diff --git a/Viagg-io/Assets/afca/ViaggioAI/Prefabs/ViaggioAI.prefab b/Viagg-io/Assets/afca/ViaggioAI/Prefabs/ViaggioAI.prefab index a2cecc3..37450e4 100755 --- a/Viagg-io/Assets/afca/ViaggioAI/Prefabs/ViaggioAI.prefab +++ b/Viagg-io/Assets/afca/ViaggioAI/Prefabs/ViaggioAI.prefab @@ -2118,7 +2118,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 2c0bfe10253cbe44aa6bdc7526969660, type: 3} m_Name: m_EditorClassIdentifier: - debugModeIsActive: 1 + debugModeIsActive: 0 speechKey: 4e6ecd1ff0b246e5b782b60d9af0514a speechRegion: westeurope initialRecognitionLanguageCode: de-DE @@ -2134,7 +2134,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 4d4b95d5c09d8b94f97693df0bf9a567, type: 3} m_Name: m_EditorClassIdentifier: - debugModeIsActive: 1 + debugModeIsActive: 0 speechOutputType: 0 speechKey: 01e33976fd254c608f21da343dc5cc01 speechRegion: switzerlandnorth @@ -2159,7 +2159,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 701f2da9dc1b54cb4b6e765775a54c4c, type: 3} m_Name: m_EditorClassIdentifier: - debugModeIsActive: 1 + debugModeIsActive: 0 ignoreReplyToStartInstructions: 1 clientInitDelay: 0.5 responsePollingInterval: 0.5 @@ -3074,7 +3074,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 9c17ebccf009d574598fcda2ad1850db, type: 3} m_Name: m_EditorClassIdentifier: - debugModeIsActive: 1 + debugModeIsActive: 0 soundEffectsEnabled: 1 audioSource: {fileID: 645973765163538692} servicesContainer: {fileID: 4548206053279514970} diff --git a/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/OpenAIServices.cs b/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/OpenAIServices.cs index 014c7b8..786d3b1 100755 --- a/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/OpenAIServices.cs +++ b/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/OpenAIServices.cs @@ -72,7 +72,7 @@ public class OpenAIServices : MonoBehaviour { if (value != this._openAIServiceState) { - Debug.Log($"OpenAIServiceState changed, new value= {value}"); + this.logIfInDebugMode($"OpenAIServiceState changed, new value= {value}"); this._openAIServiceState = value; @@ -204,41 +204,48 @@ public class OpenAIServices : MonoBehaviour private async void init() { - this.logIfInDebugMode("Init started"); + this.logIfInDebugMode("OpenAIServices Init started"); - this.OpenAIServiceState = EOpenAIServiceState.StartingUp; + try + { + this.OpenAIServiceState = EOpenAIServiceState.StartingUp; - this.client = new AssistantsClient(new Uri(this.azureResourceUrl), new AzureKeyCredential(this.azureApiKey)); + this.client = new AssistantsClient(new Uri(this.azureResourceUrl), new AzureKeyCredential(this.azureApiKey)); - await Task.Delay(TimeSpan.FromSeconds(this.clientInitDelay)); + await Task.Delay(TimeSpan.FromSeconds(this.clientInitDelay)); - Response assistantResponse = await this.client.CreateAssistantAsync( - new AssistantCreationOptions(assistantModel) - { - Name = assistantName, - Instructions = assistantInstructions, + Response assistantResponse = await this.client.CreateAssistantAsync( + new AssistantCreationOptions(assistantModel) + { + Name = assistantName, + Instructions = assistantInstructions, - }); - this.assistant = assistantResponse.Value; + }); + this.assistant = assistantResponse.Value; - Response threadResponse = await this.client.CreateThreadAsync(); - this.thread = threadResponse.Value; + Response threadResponse = await this.client.CreateThreadAsync(); + this.thread = threadResponse.Value; - this.runResponse = await client.CreateRunAsync( - this.thread.Id, - new CreateRunOptions(assistant.Id) - { - AdditionalInstructions = assistantStartInstructions, - }); - ThreadRun run = runResponse.Value; + this.runResponse = await client.CreateRunAsync( + this.thread.Id, + new CreateRunOptions(assistant.Id) + { + AdditionalInstructions = assistantStartInstructions, + }); + ThreadRun run = runResponse.Value; - this.logIfInDebugMode($"Init completed, ignoreReplyToStartInstructions={this.ignoreReplyToStartInstructions}"); + this.logIfInDebugMode($"Init completed, ignoreReplyToStartInstructions={this.ignoreReplyToStartInstructions}"); - this.ignoreIncomingReplies = this.ignoreReplyToStartInstructions; + this.ignoreIncomingReplies = this.ignoreReplyToStartInstructions; - this.OpenAIServiceState = EOpenAIServiceState.WaitingForInstructionsReply; + this.OpenAIServiceState = EOpenAIServiceState.WaitingForInstructionsReply; - await this.listen(); + await this.listen(); + } + catch (Exception ex) + { + this.lastError = ex.ToString(); + } } private async Task send(string text) diff --git a/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechRecognitionService.cs b/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechRecognitionService.cs index 577b4ec..f3f4e52 100755 --- a/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechRecognitionService.cs +++ b/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechRecognitionService.cs @@ -76,11 +76,12 @@ public class SpeechRecognitionService : MonoBehaviour private string pendingPartialTranscription; private string pendingFullTranscription; private string invokeUserSpeechInputEndedEventIsPending; + private string pendingRecoError; #endregion #region Framework Functions - + void OnEnable() { this.initialize(this.initialRecognitionLanguageCode); @@ -136,6 +137,8 @@ public class SpeechRecognitionService : MonoBehaviour public event EventHandler OnUserSpeechInputEndedEvent; + public event EventHandler OnLastRecoErrorChangedEvent; + #endregion #region Public Functions @@ -187,7 +190,7 @@ public class SpeechRecognitionService : MonoBehaviour else if (result.Reason == ResultReason.Canceled) { var cancellation = CancellationDetails.FromResult(result); - this.logIfInDebugMode($"SpeechIntentService Canceled: Reason={cancellation.Reason} ErrorDetails={cancellation.ErrorDetails}"); + this.pendingRecoError = $"SpeechIntentService Canceled: Reason={cancellation.Reason} ErrorDetails={cancellation.ErrorDetails}"; } lock (threadLocker) @@ -262,6 +265,12 @@ public class SpeechRecognitionService : MonoBehaviour this.OnUserSpeechInputEndedEvent?.Invoke(this, invokeUserSpeechInputEndedEventIsPending); this.invokeUserSpeechInputEndedEventIsPending = null; } + + if (this.pendingRecoError != null) + { + this.OnLastRecoErrorChangedEvent?.Invoke(this, this.pendingRecoError); + this.pendingRecoError = null; + } } private void logIfInDebugMode(string message) diff --git a/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechSynthesizerService.cs b/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechSynthesizerService.cs index 24c6611..16e4248 100755 --- a/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechSynthesizerService.cs +++ b/Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechSynthesizerService.cs @@ -122,7 +122,7 @@ public class SpeechSynthesizerService : MonoBehaviour async void OnDisable() { - Debug.Log("SpeechSynthesizerService disposing speechSynthesizer from OnDisable"); + this.logIfInDebugMode("SpeechSynthesizerService disposing speechSynthesizer from OnDisable"); await this.disposeSynthesizer(); } @@ -138,6 +138,8 @@ public class SpeechSynthesizerService : MonoBehaviour public event EventHandler OnSpeechOutputEndedEvent; + public event EventHandler OnSynthErrorChangedEvent; + #endregion #region Public Functions @@ -164,6 +166,11 @@ public class SpeechSynthesizerService : MonoBehaviour SpeechSynthesisResult synthesisResult = await this.speechSynthesizer.SpeakTextAsync(text); + if (synthesisResult.Reason == ResultReason.Canceled) + { + this.OnSynthErrorChangedEvent?.Invoke(this, "Speech synthesis failed! Check internet connection"); + } + if (this.speechOutputType == ESpeechOutputType.GenerateAudioClip) { // Todo add WavUtility @@ -189,7 +196,7 @@ public class SpeechSynthesizerService : MonoBehaviour if (this.speechSynthesizer != null) { - Debug.Log("SpeechSynthesizerService disposing speechSynthesizer from initialize"); + this.logIfInDebugMode("SpeechSynthesizerService disposing speechSynthesizer from initialize"); await this.disposeSynthesizer(); } diff --git a/Viagg-io/Assets/afca/ViaggioAI/Scripts/ViaggioAI/ViaggioAIManager.cs b/Viagg-io/Assets/afca/ViaggioAI/Scripts/ViaggioAI/ViaggioAIManager.cs index 7f15d77..eeaaaa5 100755 --- a/Viagg-io/Assets/afca/ViaggioAI/Scripts/ViaggioAI/ViaggioAIManager.cs +++ b/Viagg-io/Assets/afca/ViaggioAI/Scripts/ViaggioAI/ViaggioAIManager.cs @@ -107,7 +107,7 @@ public class ViaggioAIManager : MonoBehaviour { if (value != this._partialTranscription) { - //Debug.Log("PartialTranscription changed, new value= " + value); + this.LogIfInDebugMode("PartialTranscription changed, new value= " + value); this._partialTranscription = value; @@ -134,7 +134,7 @@ public class ViaggioAIManager : MonoBehaviour { if (value != this._fullTranscription) { - //Debug.Log("FullTranscription changed, new value= " + value); + this.LogIfInDebugMode("FullTranscription changed, new value= " + value); this._fullTranscription = value; @@ -196,12 +196,14 @@ public class ViaggioAIManager : MonoBehaviour this.speechRecognitionService.OnPartialTranscriptionChangedEvent += this.onPartialTranscriptionChanged; this.speechRecognitionService.OnFullTranscriptionChangedEvent += this.onFullTranscriptionChanged; this.speechRecognitionService.OnUserSpeechInputEndedEvent += this.onUserSpeechInputEnded; + this.speechRecognitionService.OnLastRecoErrorChangedEvent += this.onLastRecoErrorChanged; } if (this.speechSynthesizerService != null) { this.speechSynthesizerService.OnSpeechOutputStartedEvent += this.onSpeechOutputStarted; this.speechSynthesizerService.OnSpeechOutputEndedEvent += this.onSpeechOutputEnded; + this.speechSynthesizerService.OnSynthErrorChangedEvent += this.onSynthErrorChanged; } if (this.openAIServices) @@ -219,12 +221,14 @@ public class ViaggioAIManager : MonoBehaviour this.speechRecognitionService.OnPartialTranscriptionChangedEvent -= this.onPartialTranscriptionChanged; this.speechRecognitionService.OnFullTranscriptionChangedEvent -= this.onFullTranscriptionChanged; this.speechRecognitionService.OnUserSpeechInputEndedEvent -= this.onUserSpeechInputEnded; + this.speechRecognitionService.OnLastRecoErrorChangedEvent -= this.onLastRecoErrorChanged; } if (this?.speechSynthesizerService != null) { this.speechSynthesizerService.OnSpeechOutputStartedEvent -= this.onSpeechOutputStarted; this.speechSynthesizerService.OnSpeechOutputEndedEvent -= this.onSpeechOutputEnded; + this.speechSynthesizerService.OnSynthErrorChangedEvent -= this.onSynthErrorChanged; } if (this?.openAIServices) @@ -264,6 +268,11 @@ public class ViaggioAIManager : MonoBehaviour this.sendIntentRequest(fullTranscription); } + private void onLastRecoErrorChanged(object sender, string errorText) + { + this.handleError(sender, errorText); + } + private void onSpeechOutputStarted(object sender, bool dummy) { this.OnSpeechOutputStartedEvent?.Invoke(this, dummy); @@ -274,6 +283,11 @@ public class ViaggioAIManager : MonoBehaviour this.OnSpeechOutputEndedEvent?.Invoke(this, dummy); } + private void onSynthErrorChanged(object sender, string errorText) + { + this.handleError(sender, errorText); + } + private void onLastBotReplyChanged(object sender, string replyText) { this.processBotReply(replyText); @@ -281,7 +295,7 @@ public class ViaggioAIManager : MonoBehaviour private void onLastBotErrorChanged(object sender, string errorText) { - // todo handle error + this.handleError(sender, errorText); } #endregion @@ -301,6 +315,9 @@ public class ViaggioAIManager : MonoBehaviour public event EventHandler OnIntentRecognitionSucceededEvent; public event EventHandler OnIntentRecognitionFailedEvent; + // Error Handling + public event EventHandler OnViaggioAIErrorEvent; + #endregion #region Public Functions @@ -317,7 +334,7 @@ public class ViaggioAIManager : MonoBehaviour this.ViaggioAIState = EViaggioAIState.Listening; } - + public void SetSpeechRecognitionLanguage(string languageCode) { this.speechRecognitionService.SetRecognitionLanguage(languageCode); @@ -417,6 +434,14 @@ public class ViaggioAIManager : MonoBehaviour this.audioSource.PlayOneShot(clip); } + private void handleError(object sender, string errorText) + { + string logText = $"{sender} Error: {errorText}"; + + Debug.LogError(logText); + this.OnViaggioAIErrorEvent?.Invoke(this, logText); + } + #endregion }