From e61d80f2aaf5e5cc6031ec8e9d561040afe90224 Mon Sep 17 00:00:00 2001 From: Nadine Ganz Date: Wed, 22 Jan 2025 15:45:15 +0100 Subject: [PATCH] Handling speech errors --- Viagg-io/Assets/BehaviourTrees/1.0-SBB.bt.txt | 29 +++- .../Assets/BehaviourTrees/3.2-Grotto.bt.txt | 164 ++++++++++-------- Viagg-io/Assets/Packages/MyBT/BTC/BTC.cs | 24 +++ 3 files changed, 143 insertions(+), 74 deletions(-) diff --git a/Viagg-io/Assets/BehaviourTrees/1.0-SBB.bt.txt b/Viagg-io/Assets/BehaviourTrees/1.0-SBB.bt.txt index eb9afcd..e3bce34 100644 --- a/Viagg-io/Assets/BehaviourTrees/1.0-SBB.bt.txt +++ b/Viagg-io/Assets/BehaviourTrees/1.0-SBB.bt.txt @@ -1,9 +1,32 @@ Tree("Root") { Composite(Sequence) { BTC.InitializeSpeechManager() - BTC.SetVoiceName("de-DE-SeraphinaMultilingualNeural") - RunTree("10_SBB_Zugabteil_Intro") - RunTree("10_SBB_Zugabteil_Szenenwahl_Selection") + + Composite(Race) { + Composite(Sequence) { + // Error in Speech Service: Cancel BTC Trees + BTC.SpeechErrorOccured() + BTC.SetBool("error") + } + + Composite(Sequence) { + BTC.SetVoiceName("de-DE-SeraphinaMultilingualNeural") + RunTree("10_SBB_Zugabteil_Intro") + } + } + + Composite(Sequence) { + BTC.CompareBool("error") + // Error Handling + BTC.AbortSpeechEventListener() + BTC.StopSpeechIntentRecognition() + BTC.ClearPossbileSpeechIntents() + + BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Restart") + BTC.Show("GO.HANDMENU.Option1Button") + BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") + BTC.Hide("GO.HANDMENU.Option1Button") + } } } diff --git a/Viagg-io/Assets/BehaviourTrees/3.2-Grotto.bt.txt b/Viagg-io/Assets/BehaviourTrees/3.2-Grotto.bt.txt index cd580f8..7b10356 100644 --- a/Viagg-io/Assets/BehaviourTrees/3.2-Grotto.bt.txt +++ b/Viagg-io/Assets/BehaviourTrees/3.2-Grotto.bt.txt @@ -1,17 +1,39 @@ Tree("Root") { Composite(Sequence) { - BTC.InitializeSpeechManager() + BTC.InitializeSpeechManager() // delete in finale productive version //BTC.SetSpeechRecognitionLanguage("it-IT") - //RunTree("32_Grotto_Story_B_Grotto_Intro") + Composite(Race) { + Composite(Sequence) { + // Error in Speech Service: Cancel BTC Trees + BTC.SpeechErrorOccured() + BTC.SetBool("error") + } - RunTree("32_Grotto_Kueche_Intro") - RunTree("32_Grotto_Kueche_Zwiebeln_schneiden") - RunTree("32_Grotto_Kueche_alles_gefunden") - //RunTree("32_Grotto_Kueche_Kochen_Anzahl") - //RunTree("32_Grotto_Kueche_Zusammen_Kochen") - //RunTree("32_Grotto_Im_Grotto_Essen_Intro") - + Composite(Sequence) { + //RunTree("32_Grotto_Story_B_Grotto_Intro") + + RunTree("32_Grotto_Kueche_Intro") + RunTree("32_Grotto_Kueche_Zwiebeln_schneiden") + RunTree("32_Grotto_Kueche_alles_gefunden") + //RunTree("32_Grotto_Kueche_Kochen_Anzahl") + //RunTree("32_Grotto_Kueche_Zusammen_Kochen") + //RunTree("32_Grotto_Im_Grotto_Essen_Intro") + } + } + + Composite(Sequence) { + BTC.CompareBool("error") + // Error Handling + BTC.AbortSpeechEventListener() + BTC.StopSpeechIntentRecognition() + BTC.ClearPossbileSpeechIntents() + + BTC.Set("TextMeshPro.GO.Option1Button", "text", "Restart") + BTC.Show("GO.GO.Option1Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") + BTC.Hide("GO.GO.Option1Button") + } } } @@ -234,27 +256,27 @@ Tree("32_Grotto_Story_B_Grotto_Intro") { // Selector 3: Fallback Button Composite(Sequence) { Composite(Marathon) { - BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Küche") - BTC.Show("GO.HANDMENU.Option1Button") - BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "Garten") - BTC.Show("GO.HANDMENU.Option2Button") + BTC.Set("TextMeshPro.GO.Option1Button", "text", "Küche") + BTC.Show("GO.GO.Option1Button") + BTC.Set("TextMeshPro.GO.Option2Button", "text", "Garten") + BTC.Show("GO.GO.Option2Button") } Composite(Race) { Composite(Sequence) { // Kueche - BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Story_B_Grotto_Antwort_Nonna_Kueche") } Composite(Sequence) { // Garten - BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option2Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Story_B_Grotto_Antwort_Nonna_Grotto") } } @@ -843,27 +865,27 @@ Tree("32_Grotto_Kueche_alles_gefunden") { // Fallback Button Composite(Marathon) { - BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Ja") - BTC.Show("GO.HANDMENU.Option1Button") - BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "Nein") - BTC.Show("GO.HANDMENU.Option2Button") + BTC.Set("TextMeshPro.GO.Option1Button", "text", "Ja") + BTC.Show("GO.GO.Option1Button") + BTC.Set("TextMeshPro.GO.Option2Button", "text", "Nein") + BTC.Show("GO.GO.Option2Button") } Composite(Race) { Composite(Sequence) { // Ja - BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Kueche_Interesse_Kochen") } Composite(Sequence) { // Nein - BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option2Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Kueche_Zusammen_Kochen") } } @@ -881,27 +903,27 @@ Tree("32_Grotto_Kueche_alles_gefunden") { // Fallback Button Composite(Marathon) { - BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Ja") - BTC.Show("GO.HANDMENU.Option1Button") - BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "Nein") - BTC.Show("GO.HANDMENU.Option2Button") + BTC.Set("TextMeshPro.GO.Option1Button", "text", "Ja") + BTC.Show("GO.GO.Option1Button") + BTC.Set("TextMeshPro.GO.Option2Button", "text", "Nein") + BTC.Show("GO.GO.Option2Button") } Composite(Race) { Composite(Sequence) { // Ja - BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Kueche_Interesse_Kochen") } Composite(Sequence) { // Nein - BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option2Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Kueche_Zusammen_Kochen") } } @@ -915,24 +937,24 @@ Tree("32_Grotto_Kueche_Interesse_Kochen") { Composite(Sequence) { BTC.Run("AudioSource.AUDIO.Kueche1FInteresseK") // ToDo: Freie Antwort als Speech Input - BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Lasagne") - BTC.Show("GO.HANDMENU.Option1Button") + BTC.Set("TextMeshPro.GO.Option1Button", "text", "Lasagne") + BTC.Show("GO.GO.Option1Button") BTC.SetBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked", false) Composite(Race) { Composite(Sequence) { // Lieblingsessen nennen - BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") BTC.SetBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked") - BTC.Hide("GO.HANDMENU.Option1Button") + BTC.Hide("GO.GO.Option1Button") RunTree("32_Grotto_Kueche_Kochen_Anzahl") } Composite(Sequence) { // keine Antwort BTC.Wait(5) BTC.CompareBool("32_Grotto_Kueche_Interesse_Kochen_Button_Clicked", false) - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option1Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button") + BTC.Hide("GO.GO.Option1Button") RunTree("32_Grotto_Kueche_Zusammen_Kochen") } } @@ -980,27 +1002,27 @@ Tree("32_Grotto_Kueche_Kochen_Anzahl") { // Fallback Button Composite(Marathon) { - BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "sehr oft") - BTC.Show("GO.HANDMENU.Option1Button") - BTC.Set("TextMeshPro.HANDMENU.Option2Button", "text", "selten") - BTC.Show("GO.HANDMENU.Option2Button") + BTC.Set("TextMeshPro.GO.Option1Button", "text", "sehr oft") + BTC.Show("GO.GO.Option1Button") + BTC.Set("TextMeshPro.GO.Option2Button", "text", "selten") + BTC.Show("GO.GO.Option2Button") } Composite(Race) { Composite(Sequence) { // sehr oft - BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option2Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option2Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Kueche_Selbststaendig_Kochen") } Composite(Sequence) { // selten - BTC.Run("NamedEventTrigger.HANDMENU.Option2Button") - BTC.AbortEventListener("NamedEventTrigger.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option2Button") + BTC.Run("NamedEventTrigger.GO.Option2Button") + BTC.AbortEventListener("NamedEventTrigger.GO.Option1Button") + BTC.Hide("GO.GO.Option1Button") + BTC.Hide("GO.GO.Option2Button") RunTree("32_Grotto_Kueche_Zusammen_Kochen") } } @@ -1109,13 +1131,13 @@ Tree("32_Grotto_Kueche_Parmigiano") { // Antwort User // ToDo: Sprache fehlt - BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Ja") - BTC.Show("GO.HANDMENU.Option1Button") + BTC.Set("TextMeshPro.GO.Option1Button", "text", "Ja") + BTC.Show("GO.GO.Option1Button") Composite(Race) { - BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") BTC.Wait(5) } - BTC.Hide("GO.HANDMENU.Option1Button") + BTC.Hide("GO.GO.Option1Button") BTC.SynthesizeText("Bitte stelle den Risotto hier hin.") BTC.SpeechOutputEnded() @@ -1159,10 +1181,10 @@ Tree("32_Grotto_Kueche_Parmigiano") { BTC.ClearPossbileSpeechIntents() // Fallback Button - BTC.Set("TextMeshPro.HANDMENU.Option1Button", "text", "Mein Name ist User.") - BTC.Show("GO.HANDMENU.Option1Button") - BTC.Run("NamedEventTrigger.HANDMENU.Option1Button") - BTC.Hide("GO.HANDMENU.Option1Button") + BTC.Set("TextMeshPro.GO.Option1Button", "text", "Mein Name ist User.") + BTC.Show("GO.GO.Option1Button") + BTC.Run("NamedEventTrigger.GO.Option1Button") + BTC.Hide("GO.GO.Option1Button") } } } diff --git a/Viagg-io/Assets/Packages/MyBT/BTC/BTC.cs b/Viagg-io/Assets/Packages/MyBT/BTC/BTC.cs index b3a326d..18804af 100644 --- a/Viagg-io/Assets/Packages/MyBT/BTC/BTC.cs +++ b/Viagg-io/Assets/Packages/MyBT/BTC/BTC.cs @@ -661,12 +661,14 @@ public class BTC : MonoBehaviour { private ViaggioAIManager _speechMng { get { return ViaggioAIManager.Instance; } } private RequestDataModel _requestDataModel = new RequestDataModel(); private string _recognizedIntentID = ""; + private string _speechErrorText = ""; private bool _onIntentRecognitionSucceededEventTriggered = false; private bool _onUserSpeechInputStartedEventTriggered = false; private bool _onIntentRecognitionFailedEventTriggered = false; private bool _onSpeechOutputStartedEventTriggered = false; private bool _onSpeechOutputEndedEventTriggered = false; private bool _abortSpeechEventListener = false; + private bool _onErrorEventTriggered = false; void OnDisable() { @@ -686,6 +688,9 @@ public class BTC : MonoBehaviour { _speechMng.OnSpeechOutputStartedEvent += SpeechOutputStartedEventHandler; _speechMng.OnSpeechOutputEndedEvent += SpeechOutputEndedEventHandler; + // Error Event + _speechMng.OnViaggioAIErrorEvent += SpeechErrorEventHandler; + Debug.Log("SubscribeToEvents successfull."); } } @@ -699,6 +704,7 @@ public class BTC : MonoBehaviour { _speechMng.OnIntentRecognitionFailedEvent -= IntentRecognitionFailedEventHandler; _speechMng.OnSpeechOutputStartedEvent -= SpeechOutputStartedEventHandler; _speechMng.OnSpeechOutputEndedEvent -= SpeechOutputEndedEventHandler; + _speechMng.OnViaggioAIErrorEvent -= SpeechErrorEventHandler; } } @@ -728,6 +734,12 @@ public class BTC : MonoBehaviour { _onSpeechOutputEndedEventTriggered = true; } + private void SpeechErrorEventHandler(object sender, string errorText) + { + _onErrorEventTriggered = true; + _speechErrorText = errorText; + } + [Task] public async void InitializeSpeechManager() { @@ -943,6 +955,18 @@ public class BTC : MonoBehaviour { return; } } + + [Task] + public void SpeechErrorOccured() + { + if (_onErrorEventTriggered) + { + Task.SetSucceeded(); + Debug.Log($"ERROR: Speech Service not working. Reason: {_speechErrorText}"); + _onErrorEventTriggered = false; + return; + } + } #endregion #region Visited Stories Manager