spoerrireto
/
UP-Viagg-io
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: spoerrireto:master
Branches Tags
spoerrireto:leveldesign
spoerrireto:programming
spoerrireto:master
spoerrireto:dev
spoerrireto:story_process
spoerrireto:interaction
spoerrireto:scene_loading
spoerrireto:depthkit_samples
spoerrireto:btc_debug
..
compare: spoerrireto:programming
Branches Tags
spoerrireto:leveldesign
spoerrireto:programming
spoerrireto:master
spoerrireto:dev
spoerrireto:story_process
spoerrireto:interaction
spoerrireto:scene_loading
spoerrireto:depthkit_samples
spoerrireto:btc_debug

2 Commits
master ... programmin

Author SHA1 Message Date
Nadine Ganz e61d80f2aa Handling speech errors 2025-01-22 15:45:15 +01:00
Nadine Ganz 2bb1202530 Update SpeechPackage 2025-01-22 13:33:28 +01:00
8 changed files with 228 additions and 111 deletions
Show Stats Expand all files Collapse all files

29
Viagg-io/Assets/BehaviourTrees/1.0-SBB.bt.txt
View File

@ -1,9 +1,32 @@
Tree("Root") { Tree("Root") {
Composite(Sequence) { Composite(Sequence) {
BTC.InitializeSpeechManager() BTC.InitializeSpeechManager()
BTC.SetVoiceName("de-DE-SeraphinaMultilingualNeural")
RunTree("10_SBB_Zugabteil_Intro") Composite(Race) {
RunTree("10_SBB_Zugabteil_Szenenwahl_Selection") Composite(Sequence) {
// Error in Speech Service: Cancel BTC Trees
BTC.SpeechErrorOccured()
BTC.SetBool("error")
}
Composite(Sequence) {
BTC.SetVoiceName("de-DE-SeraphinaMultilingualNeural")
RunTree("10_SBB_Zugabteil_Intro")
}
}
Composite(Sequence) {
BTC.CompareBool("error")
// Error Handling
BTC.AbortSpeechEventListener()
BTC.StopSpeechIntentRecognition()
BTC.ClearPossbileSpeechIntents()
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Restart")
BTC.Show("GO.HANDMENU.Option1Button")
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button")
BTC.Hide("GO.HANDMENU.Option1Button")
}
} }
} }

162
Viagg-io/Assets/BehaviourTrees/3.2-Grotto.bt.txt
View File

@ -1,17 +1,39 @@
Tree("Root") { Tree("Root") {
Composite(Sequence) { Composite(Sequence) {
BTC.InitializeSpeechManager() BTC.InitializeSpeechManager() // delete in finale productive version
//BTC.SetSpeechRecognitionLanguage("it-IT") //BTC.SetSpeechRecognitionLanguage("it-IT")
//RunTree("32_Grotto_Story_B_Grotto_Intro") Composite(Race) {
Composite(Sequence) {
// Error in Speech Service: Cancel BTC Trees
BTC.SpeechErrorOccured()
BTC.SetBool("error")
}
RunTree("32_Grotto_Kueche_Intro") Composite(Sequence) {
RunTree("32_Grotto_Kueche_Zwiebeln_schneiden") //RunTree("32_Grotto_Story_B_Grotto_Intro")
RunTree("32_Grotto_Kueche_alles_gefunden")
//RunTree("32_Grotto_Kueche_Kochen_Anzahl")
//RunTree("32_Grotto_Kueche_Zusammen_Kochen")
//RunTree("32_Grotto_Im_Grotto_Essen_Intro")
RunTree("32_Grotto_Kueche_Intro")
RunTree("32_Grotto_Kueche_Zwiebeln_schneiden")
RunTree("32_Grotto_Kueche_alles_gefunden")
//RunTree("32_Grotto_Kueche_Kochen_Anzahl")
//RunTree("32_Grotto_Kueche_Zusammen_Kochen")
//RunTree("32_Grotto_Im_Grotto_Essen_Intro")
}
}
Composite(Sequence) {
BTC.CompareBool("error")
// Error Handling
BTC.AbortSpeechEventListener()
BTC.StopSpeechIntentRecognition()
BTC.ClearPossbileSpeechIntents()
BTC.Set("TextMeshPro.GO.Option1Button", "text", "Restart")
BTC.Show("GO.GO.Option1Button")
BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.Hide("GO.GO.Option1Button")
}
} }
} }
@ -234,27 +256,27 @@ Tree("32_Grotto_Story_B_Grotto_Intro") {
// Selector 3: Fallback Button // Selector 3: Fallback Button
Composite(Sequence) { Composite(Sequence) {
Composite(Marathon) { Composite(Marathon) {
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Küche") BTC.Set("TextMeshPro.GO.Option1Button", "text", "Küche")
BTC.Show("GO.HANDMENU.Option1Button") BTC.Show("GO.GO.Option1Button")
BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "Garten") BTC.Set("TextMeshPro.GO.Option2Button", "text", "Garten")
BTC.Show("GO.HANDMENU.Option2Button") BTC.Show("GO.GO.Option2Button")
} }
Composite(Race) { Composite(Race) {
Composite(Sequence) { Composite(Sequence) {
// Kueche // Kueche
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Story_B_Grotto_Antwort_Nonna_Kueche") RunTree("32_Grotto_Story_B_Grotto_Antwort_Nonna_Kueche")
} }
Composite(Sequence) { Composite(Sequence) {
// Garten // Garten
BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") BTC.Run("NamedEventTrigger.GO.Option2Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Story_B_Grotto_Antwort_Nonna_Grotto") RunTree("32_Grotto_Story_B_Grotto_Antwort_Nonna_Grotto")
} }
} }
@ -843,27 +865,27 @@ Tree("32_Grotto_Kueche_alles_gefunden") {
// Fallback Button // Fallback Button
Composite(Marathon) { Composite(Marathon) {
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Ja") BTC.Set("TextMeshPro.GO.Option1Button", "text", "Ja")
BTC.Show("GO.HANDMENU.Option1Button") BTC.Show("GO.GO.Option1Button")
BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "Nein") BTC.Set("TextMeshPro.GO.Option2Button", "text", "Nein")
BTC.Show("GO.HANDMENU.Option2Button") BTC.Show("GO.GO.Option2Button")
} }
Composite(Race) { Composite(Race) {
Composite(Sequence) { Composite(Sequence) {
// Ja // Ja
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Kueche_Interesse_Kochen") RunTree("32_Grotto_Kueche_Interesse_Kochen")
} }
Composite(Sequence) { Composite(Sequence) {
// Nein // Nein
BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") BTC.Run("NamedEventTrigger.GO.Option2Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Kueche_Zusammen_Kochen") RunTree("32_Grotto_Kueche_Zusammen_Kochen")
} }
} }
@ -881,27 +903,27 @@ Tree("32_Grotto_Kueche_alles_gefunden") {
// Fallback Button // Fallback Button
Composite(Marathon) { Composite(Marathon) {
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Ja") BTC.Set("TextMeshPro.GO.Option1Button", "text", "Ja")
BTC.Show("GO.HANDMENU.Option1Button") BTC.Show("GO.GO.Option1Button")
BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "Nein") BTC.Set("TextMeshPro.GO.Option2Button", "text", "Nein")
BTC.Show("GO.HANDMENU.Option2Button") BTC.Show("GO.GO.Option2Button")
} }
Composite(Race) { Composite(Race) {
Composite(Sequence) { Composite(Sequence) {
// Ja // Ja
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Kueche_Interesse_Kochen") RunTree("32_Grotto_Kueche_Interesse_Kochen")
} }
Composite(Sequence) { Composite(Sequence) {
// Nein // Nein
BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") BTC.Run("NamedEventTrigger.GO.Option2Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Kueche_Zusammen_Kochen") RunTree("32_Grotto_Kueche_Zusammen_Kochen")
} }
} }
@ -915,24 +937,24 @@ Tree("32_Grotto_Kueche_Interesse_Kochen") {
Composite(Sequence) { Composite(Sequence) {
BTC.Run("AudioSource.AUDIO.Kueche1FInteresseK") BTC.Run("AudioSource.AUDIO.Kueche1FInteresseK")
// ToDo: Freie Antwort als Speech Input // ToDo: Freie Antwort als Speech Input
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Lasagne") BTC.Set("TextMeshPro.GO.Option1Button", "text", "Lasagne")
BTC.Show("GO.HANDMENU.Option1Button") BTC.Show("GO.GO.Option1Button")
BTC.SetBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked", false) BTC.SetBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked", false)
Composite(Race) { Composite(Race) {
Composite(Sequence) { Composite(Sequence) {
// Lieblingsessen nennen // Lieblingsessen nennen
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.SetBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked") BTC.SetBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
RunTree("32_Grotto_Kueche_Kochen_Anzahl") RunTree("32_Grotto_Kueche_Kochen_Anzahl")
} }
Composite(Sequence) { Composite(Sequence) {
// keine Antwort // keine Antwort
BTC.Wait(5) BTC.Wait(5)
BTC.CompareBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked", false) BTC.CompareBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked", false)
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
RunTree("32_Grotto_Kueche_Zusammen_Kochen") RunTree("32_Grotto_Kueche_Zusammen_Kochen")
} }
} }
@ -980,27 +1002,27 @@ Tree("32_Grotto_Kueche_Kochen_Anzahl") {
// Fallback Button // Fallback Button
Composite(Marathon) { Composite(Marathon) {
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "sehr oft") BTC.Set("TextMeshPro.GO.Option1Button", "text", "sehr oft")
BTC.Show("GO.HANDMENU.Option1Button") BTC.Show("GO.GO.Option1Button")
BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "selten") BTC.Set("TextMeshPro.GO.Option2Button", "text", "selten")
BTC.Show("GO.HANDMENU.Option2Button") BTC.Show("GO.GO.Option2Button")
} }
Composite(Race) { Composite(Race) {
Composite(Sequence) { Composite(Sequence) {
// sehr oft // sehr oft
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Kueche_Selbststaendig_Kochen") RunTree("32_Grotto_Kueche_Selbststaendig_Kochen")
} }
Composite(Sequence) { Composite(Sequence) {
// selten // selten
BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") BTC.Run("NamedEventTrigger.GO.Option2Button")
BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option2Button") BTC.Hide("GO.GO.Option2Button")
RunTree("32_Grotto_Kueche_Zusammen_Kochen") RunTree("32_Grotto_Kueche_Zusammen_Kochen")
} }
} }
@ -1109,13 +1131,13 @@ Tree("32_Grotto_Kueche_Parmigiano") {
// Antwort User // Antwort User
// ToDo: Sprache fehlt // ToDo: Sprache fehlt
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Ja") BTC.Set("TextMeshPro.GO.Option1Button", "text", "Ja")
BTC.Show("GO.HANDMENU.Option1Button") BTC.Show("GO.GO.Option1Button")
Composite(Race) { Composite(Race) {
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.Wait(5) BTC.Wait(5)
} }
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
BTC.SynthesizeText("Bitte stelle den Risotto hier hin.") BTC.SynthesizeText("Bitte stelle den Risotto hier hin.")
BTC.SpeechOutputEnded() BTC.SpeechOutputEnded()
@ -1159,10 +1181,10 @@ Tree("32_Grotto_Kueche_Parmigiano") {
BTC.ClearPossbileSpeechIntents() BTC.ClearPossbileSpeechIntents()
// Fallback Button // Fallback Button
BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Mein Name ist User.") BTC.Set("TextMeshPro.GO.Option1Button", "text", "Mein Name ist User.")
BTC.Show("GO.HANDMENU.Option1Button") BTC.Show("GO.GO.Option1Button")
BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") BTC.Run("NamedEventTrigger.GO.Option1Button")
BTC.Hide("GO.HANDMENU.Option1Button") BTC.Hide("GO.GO.Option1Button")
} }
} }
} }

24
Viagg-io/Assets/Packages/MyBT/BTC/BTC.cs
View File

@ -661,12 +661,14 @@ public class BTC : MonoBehaviour {
private ViaggioAIManager _speechMng { get { return ViaggioAIManager.Instance; } } private ViaggioAIManager _speechMng { get { return ViaggioAIManager.Instance; } }
private RequestDataModel _requestDataModel = new RequestDataModel(); private RequestDataModel _requestDataModel = new RequestDataModel();
private string _recognizedIntentID = ""; private string _recognizedIntentID = "";
private string _speechErrorText = "";
private bool _onIntentRecognitionSucceededEventTriggered = false; private bool _onIntentRecognitionSucceededEventTriggered = false;
private bool _onUserSpeechInputStartedEventTriggered = false; private bool _onUserSpeechInputStartedEventTriggered = false;
private bool _onIntentRecognitionFailedEventTriggered = false; private bool _onIntentRecognitionFailedEventTriggered = false;
private bool _onSpeechOutputStartedEventTriggered = false; private bool _onSpeechOutputStartedEventTriggered = false;
private bool _onSpeechOutputEndedEventTriggered = false; private bool _onSpeechOutputEndedEventTriggered = false;
private bool _abortSpeechEventListener = false; private bool _abortSpeechEventListener = false;
private bool _onErrorEventTriggered = false;
void OnDisable() void OnDisable()
{ {
@ -686,6 +688,9 @@ public class BTC : MonoBehaviour {
_speechMng.OnSpeechOutputStartedEvent += SpeechOutputStartedEventHandler; _speechMng.OnSpeechOutputStartedEvent += SpeechOutputStartedEventHandler;
_speechMng.OnSpeechOutputEndedEvent += SpeechOutputEndedEventHandler; _speechMng.OnSpeechOutputEndedEvent += SpeechOutputEndedEventHandler;
// Error Event
_speechMng.OnViaggioAIErrorEvent += SpeechErrorEventHandler;
Debug.Log("SubscribeToEvents successfull."); Debug.Log("SubscribeToEvents successfull.");
} }
} }
@ -699,6 +704,7 @@ public class BTC : MonoBehaviour {
_speechMng.OnIntentRecognitionFailedEvent -= IntentRecognitionFailedEventHandler; _speechMng.OnIntentRecognitionFailedEvent -= IntentRecognitionFailedEventHandler;
_speechMng.OnSpeechOutputStartedEvent -= SpeechOutputStartedEventHandler; _speechMng.OnSpeechOutputStartedEvent -= SpeechOutputStartedEventHandler;
_speechMng.OnSpeechOutputEndedEvent -= SpeechOutputEndedEventHandler; _speechMng.OnSpeechOutputEndedEvent -= SpeechOutputEndedEventHandler;
_speechMng.OnViaggioAIErrorEvent -= SpeechErrorEventHandler;
} }
} }
@ -728,6 +734,12 @@ public class BTC : MonoBehaviour {
_onSpeechOutputEndedEventTriggered = true; _onSpeechOutputEndedEventTriggered = true;
} }
private void SpeechErrorEventHandler(object sender, string errorText)
{
_onErrorEventTriggered = true;
_speechErrorText = errorText;
}
[Task] [Task]
public async void InitializeSpeechManager() public async void InitializeSpeechManager()
{ {
@ -943,6 +955,18 @@ public class BTC : MonoBehaviour {
return; return;
} }
} }
[Task]
public void SpeechErrorOccured()
{
if (_onErrorEventTriggered)
{
Task.SetSucceeded();
Debug.Log($"ERROR: Speech Service not working. Reason: {_speechErrorText}");
_onErrorEventTriggered = false;
return;
}
}
#endregion #endregion
#region Visited Stories Manager #region Visited Stories Manager

8
Viagg-io/Assets/afca/ViaggioAI/Prefabs/ViaggioAI.prefab
View File

@ -2118,7 +2118,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 2c0bfe10253cbe44aa6bdc7526969660, type: 3} m_Script: {fileID: 11500000, guid: 2c0bfe10253cbe44aa6bdc7526969660, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
debugModeIsActive: 1 debugModeIsActive: 0
speechKey: 4e6ecd1ff0b246e5b782b60d9af0514a speechKey: 4e6ecd1ff0b246e5b782b60d9af0514a
speechRegion: westeurope speechRegion: westeurope
initialRecognitionLanguageCode: de-DE initialRecognitionLanguageCode: de-DE
@ -2134,7 +2134,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 4d4b95d5c09d8b94f97693df0bf9a567, type: 3} m_Script: {fileID: 11500000, guid: 4d4b95d5c09d8b94f97693df0bf9a567, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
debugModeIsActive: 1 debugModeIsActive: 0
speechOutputType: 0 speechOutputType: 0
speechKey: 01e33976fd254c608f21da343dc5cc01 speechKey: 01e33976fd254c608f21da343dc5cc01
speechRegion: switzerlandnorth speechRegion: switzerlandnorth
@ -2159,7 +2159,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 701f2da9dc1b54cb4b6e765775a54c4c, type: 3} m_Script: {fileID: 11500000, guid: 701f2da9dc1b54cb4b6e765775a54c4c, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
debugModeIsActive: 1 debugModeIsActive: 0
ignoreReplyToStartInstructions: 1 ignoreReplyToStartInstructions: 1
clientInitDelay: 0.5 clientInitDelay: 0.5
responsePollingInterval: 0.5 responsePollingInterval: 0.5
@ -3074,7 +3074,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 9c17ebccf009d574598fcda2ad1850db, type: 3} m_Script: {fileID: 11500000, guid: 9c17ebccf009d574598fcda2ad1850db, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
debugModeIsActive: 1 debugModeIsActive: 0
soundEffectsEnabled: 1 soundEffectsEnabled: 1
audioSource: {fileID: 645973765163538692} audioSource: {fileID: 645973765163538692}
servicesContainer: {fileID: 4548206053279514970} servicesContainer: {fileID: 4548206053279514970}

57
Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/OpenAIServices.cs
View File

@ -72,7 +72,7 @@ public class OpenAIServices : MonoBehaviour
{ {
if (value != this._openAIServiceState) if (value != this._openAIServiceState)
{ {
Debug.Log($"OpenAIServiceState changed, new value= {value}"); this.logIfInDebugMode($"OpenAIServiceState changed, new value= {value}");
this._openAIServiceState = value; this._openAIServiceState = value;
@ -204,41 +204,48 @@ public class OpenAIServices : MonoBehaviour
private async void init() private async void init()
{ {
this.logIfInDebugMode("Init started"); this.logIfInDebugMode("OpenAIServices Init started");
this.OpenAIServiceState = EOpenAIServiceState.StartingUp; try
{
this.OpenAIServiceState = EOpenAIServiceState.StartingUp;
this.client = new AssistantsClient(new Uri(this.azureResourceUrl), new AzureKeyCredential(this.azureApiKey)); this.client = new AssistantsClient(new Uri(this.azureResourceUrl), new AzureKeyCredential(this.azureApiKey));
await Task.Delay(TimeSpan.FromSeconds(this.clientInitDelay)); await Task.Delay(TimeSpan.FromSeconds(this.clientInitDelay));
Response<Assistant> assistantResponse = await this.client.CreateAssistantAsync( Response<Assistant> assistantResponse = await this.client.CreateAssistantAsync(
new AssistantCreationOptions(assistantModel) new AssistantCreationOptions(assistantModel)
{ {
Name = assistantName, Name = assistantName,
Instructions = assistantInstructions, Instructions = assistantInstructions,
}); });
this.assistant = assistantResponse.Value; this.assistant = assistantResponse.Value;
Response<AssistantThread> threadResponse = await this.client.CreateThreadAsync(); Response<AssistantThread> threadResponse = await this.client.CreateThreadAsync();
this.thread = threadResponse.Value; this.thread = threadResponse.Value;
this.runResponse = await client.CreateRunAsync( this.runResponse = await client.CreateRunAsync(
this.thread.Id, this.thread.Id,
new CreateRunOptions(assistant.Id) new CreateRunOptions(assistant.Id)
{ {
AdditionalInstructions = assistantStartInstructions, AdditionalInstructions = assistantStartInstructions,
}); });
ThreadRun run = runResponse.Value; ThreadRun run = runResponse.Value;
this.logIfInDebugMode($"Init completed, ignoreReplyToStartInstructions={this.ignoreReplyToStartInstructions}"); this.logIfInDebugMode($"Init completed, ignoreReplyToStartInstructions={this.ignoreReplyToStartInstructions}");
this.ignoreIncomingReplies = this.ignoreReplyToStartInstructions; this.ignoreIncomingReplies = this.ignoreReplyToStartInstructions;
this.OpenAIServiceState = EOpenAIServiceState.WaitingForInstructionsReply; this.OpenAIServiceState = EOpenAIServiceState.WaitingForInstructionsReply;
await this.listen(); await this.listen();
}
catch (Exception ex)
{
this.lastError = ex.ToString();
}
} }
private async Task send(string text) private async Task send(string text)

11
Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechRecognitionService.cs
View File

@ -76,6 +76,7 @@ public class SpeechRecognitionService : MonoBehaviour
private string pendingPartialTranscription; private string pendingPartialTranscription;
private string pendingFullTranscription; private string pendingFullTranscription;
private string invokeUserSpeechInputEndedEventIsPending; private string invokeUserSpeechInputEndedEventIsPending;
private string pendingRecoError;
#endregion #endregion
@ -136,6 +137,8 @@ public class SpeechRecognitionService : MonoBehaviour
public event EventHandler<string> OnUserSpeechInputEndedEvent; public event EventHandler<string> OnUserSpeechInputEndedEvent;
public event EventHandler<string> OnLastRecoErrorChangedEvent;
#endregion #endregion
#region Public Functions #region Public Functions
@ -187,7 +190,7 @@ public class SpeechRecognitionService : MonoBehaviour
else if (result.Reason == ResultReason.Canceled) else if (result.Reason == ResultReason.Canceled)
{ {
var cancellation = CancellationDetails.FromResult(result); var cancellation = CancellationDetails.FromResult(result);
this.logIfInDebugMode($"SpeechIntentService Canceled: Reason={cancellation.Reason} ErrorDetails={cancellation.ErrorDetails}"); this.pendingRecoError = $"SpeechIntentService Canceled: Reason={cancellation.Reason} ErrorDetails={cancellation.ErrorDetails}";
} }
lock (threadLocker) lock (threadLocker)
@ -262,6 +265,12 @@ public class SpeechRecognitionService : MonoBehaviour
this.OnUserSpeechInputEndedEvent?.Invoke(this, invokeUserSpeechInputEndedEventIsPending); this.OnUserSpeechInputEndedEvent?.Invoke(this, invokeUserSpeechInputEndedEventIsPending);
this.invokeUserSpeechInputEndedEventIsPending = null; this.invokeUserSpeechInputEndedEventIsPending = null;
} }
if (this.pendingRecoError != null)
{
this.OnLastRecoErrorChangedEvent?.Invoke(this, this.pendingRecoError);
this.pendingRecoError = null;
}
} }
private void logIfInDebugMode(string message) private void logIfInDebugMode(string message)

11
Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechSynthesizerService.cs
View File

@ -122,7 +122,7 @@ public class SpeechSynthesizerService : MonoBehaviour
async void OnDisable() async void OnDisable()
{ {
Debug.Log("SpeechSynthesizerService disposing speechSynthesizer from OnDisable"); this.logIfInDebugMode("SpeechSynthesizerService disposing speechSynthesizer from OnDisable");
await this.disposeSynthesizer(); await this.disposeSynthesizer();
} }
@ -138,6 +138,8 @@ public class SpeechSynthesizerService : MonoBehaviour
public event EventHandler<bool> OnSpeechOutputEndedEvent; public event EventHandler<bool> OnSpeechOutputEndedEvent;
public event EventHandler<string> OnSynthErrorChangedEvent;
#endregion #endregion
#region Public Functions #region Public Functions
@ -164,6 +166,11 @@ public class SpeechSynthesizerService : MonoBehaviour
SpeechSynthesisResult synthesisResult = await this.speechSynthesizer.SpeakTextAsync(text); SpeechSynthesisResult synthesisResult = await this.speechSynthesizer.SpeakTextAsync(text);
if (synthesisResult.Reason == ResultReason.Canceled)
{
this.OnSynthErrorChangedEvent?.Invoke(this, "Speech synthesis failed! Check internet connection");
}
if (this.speechOutputType == ESpeechOutputType.GenerateAudioClip) if (this.speechOutputType == ESpeechOutputType.GenerateAudioClip)
{ {
// Todo add WavUtility // Todo add WavUtility
@ -189,7 +196,7 @@ public class SpeechSynthesizerService : MonoBehaviour
if (this.speechSynthesizer != null) if (this.speechSynthesizer != null)
{ {
Debug.Log("SpeechSynthesizerService disposing speechSynthesizer from initialize"); this.logIfInDebugMode("SpeechSynthesizerService disposing speechSynthesizer from initialize");
await this.disposeSynthesizer(); await this.disposeSynthesizer();
} }

31
Viagg-io/Assets/afca/ViaggioAI/Scripts/ViaggioAI/ViaggioAIManager.cs
View File

@ -107,7 +107,7 @@ public class ViaggioAIManager : MonoBehaviour
{ {
if (value != this._partialTranscription) if (value != this._partialTranscription)
{ {
//Debug.Log("PartialTranscription changed, new value= " + value); this.LogIfInDebugMode("PartialTranscription changed, new value= " + value);
this._partialTranscription = value; this._partialTranscription = value;
@ -134,7 +134,7 @@ public class ViaggioAIManager : MonoBehaviour
{ {
if (value != this._fullTranscription) if (value != this._fullTranscription)
{ {
//Debug.Log("FullTranscription changed, new value= " + value); this.LogIfInDebugMode("FullTranscription changed, new value= " + value);
this._fullTranscription = value; this._fullTranscription = value;
@ -196,12 +196,14 @@ public class ViaggioAIManager : MonoBehaviour
this.speechRecognitionService.OnPartialTranscriptionChangedEvent += this.onPartialTranscriptionChanged; this.speechRecognitionService.OnPartialTranscriptionChangedEvent += this.onPartialTranscriptionChanged;
this.speechRecognitionService.OnFullTranscriptionChangedEvent += this.onFullTranscriptionChanged; this.speechRecognitionService.OnFullTranscriptionChangedEvent += this.onFullTranscriptionChanged;
this.speechRecognitionService.OnUserSpeechInputEndedEvent += this.onUserSpeechInputEnded; this.speechRecognitionService.OnUserSpeechInputEndedEvent += this.onUserSpeechInputEnded;
this.speechRecognitionService.OnLastRecoErrorChangedEvent += this.onLastRecoErrorChanged;
} }
if (this.speechSynthesizerService != null) if (this.speechSynthesizerService != null)
{ {
this.speechSynthesizerService.OnSpeechOutputStartedEvent += this.onSpeechOutputStarted; this.speechSynthesizerService.OnSpeechOutputStartedEvent += this.onSpeechOutputStarted;
this.speechSynthesizerService.OnSpeechOutputEndedEvent += this.onSpeechOutputEnded; this.speechSynthesizerService.OnSpeechOutputEndedEvent += this.onSpeechOutputEnded;
this.speechSynthesizerService.OnSynthErrorChangedEvent += this.onSynthErrorChanged;
} }
if (this.openAIServices) if (this.openAIServices)
@ -219,12 +221,14 @@ public class ViaggioAIManager : MonoBehaviour
this.speechRecognitionService.OnPartialTranscriptionChangedEvent -= this.onPartialTranscriptionChanged; this.speechRecognitionService.OnPartialTranscriptionChangedEvent -= this.onPartialTranscriptionChanged;
this.speechRecognitionService.OnFullTranscriptionChangedEvent -= this.onFullTranscriptionChanged; this.speechRecognitionService.OnFullTranscriptionChangedEvent -= this.onFullTranscriptionChanged;
this.speechRecognitionService.OnUserSpeechInputEndedEvent -= this.onUserSpeechInputEnded; this.speechRecognitionService.OnUserSpeechInputEndedEvent -= this.onUserSpeechInputEnded;
this.speechRecognitionService.OnLastRecoErrorChangedEvent -= this.onLastRecoErrorChanged;
} }
if (this?.speechSynthesizerService != null) if (this?.speechSynthesizerService != null)
{ {
this.speechSynthesizerService.OnSpeechOutputStartedEvent -= this.onSpeechOutputStarted; this.speechSynthesizerService.OnSpeechOutputStartedEvent -= this.onSpeechOutputStarted;
this.speechSynthesizerService.OnSpeechOutputEndedEvent -= this.onSpeechOutputEnded; this.speechSynthesizerService.OnSpeechOutputEndedEvent -= this.onSpeechOutputEnded;
this.speechSynthesizerService.OnSynthErrorChangedEvent -= this.onSynthErrorChanged;
} }
if (this?.openAIServices) if (this?.openAIServices)
@ -264,6 +268,11 @@ public class ViaggioAIManager : MonoBehaviour
this.sendIntentRequest(fullTranscription); this.sendIntentRequest(fullTranscription);
} }
private void onLastRecoErrorChanged(object sender, string errorText)
{
this.handleError(sender, errorText);
}
private void onSpeechOutputStarted(object sender, bool dummy) private void onSpeechOutputStarted(object sender, bool dummy)
{ {
this.OnSpeechOutputStartedEvent?.Invoke(this, dummy); this.OnSpeechOutputStartedEvent?.Invoke(this, dummy);
@ -274,6 +283,11 @@ public class ViaggioAIManager : MonoBehaviour
this.OnSpeechOutputEndedEvent?.Invoke(this, dummy); this.OnSpeechOutputEndedEvent?.Invoke(this, dummy);
} }
private void onSynthErrorChanged(object sender, string errorText)
{
this.handleError(sender, errorText);
}
private void onLastBotReplyChanged(object sender, string replyText) private void onLastBotReplyChanged(object sender, string replyText)
{ {
this.processBotReply(replyText); this.processBotReply(replyText);
@ -281,7 +295,7 @@ public class ViaggioAIManager : MonoBehaviour
private void onLastBotErrorChanged(object sender, string errorText) private void onLastBotErrorChanged(object sender, string errorText)
{ {
// todo handle error this.handleError(sender, errorText);
} }
#endregion #endregion
@ -301,6 +315,9 @@ public class ViaggioAIManager : MonoBehaviour
public event EventHandler<string> OnIntentRecognitionSucceededEvent; public event EventHandler<string> OnIntentRecognitionSucceededEvent;
public event EventHandler<bool> OnIntentRecognitionFailedEvent; public event EventHandler<bool> OnIntentRecognitionFailedEvent;
// Error Handling
public event EventHandler<string> OnViaggioAIErrorEvent;
#endregion #endregion
#region Public Functions #region Public Functions
@ -417,6 +434,14 @@ public class ViaggioAIManager : MonoBehaviour
this.audioSource.PlayOneShot(clip); this.audioSource.PlayOneShot(clip);
} }
private void handleError(object sender, string errorText)
{
string logText = $"{sender} Error: {errorText}";
Debug.LogError(logText);
this.OnViaggioAIErrorEvent?.Invoke(this, logText);
}
#endregion #endregion
} }