spoerrireto/UP-Viagg-io
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update SpeechPackage

Browse Source
This commit is contained in:
Nadine Ganz 2025-01-22 13:33:28 +01:00
parent ed7c9b24e8
commit 2bb1202530
5 changed files with 85 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Viagg-io/Assets/afca/ViaggioAI/Prefabs/ViaggioAI.prefab
View File

@ -2118,7 +2118,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 2c0bfe10253cbe44aa6bdc7526969660, type: 3}
m_Name:
m_EditorClassIdentifier:
debugModeIsActive: 1
debugModeIsActive: 0
speechKey: 4e6ecd1ff0b246e5b782b60d9af0514a
speechRegion: westeurope
initialRecognitionLanguageCode: de-DE
@ -2134,7 +2134,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 4d4b95d5c09d8b94f97693df0bf9a567, type: 3}
m_Name:
m_EditorClassIdentifier:
debugModeIsActive: 1
debugModeIsActive: 0
speechOutputType: 0
speechKey: 01e33976fd254c608f21da343dc5cc01
speechRegion: switzerlandnorth
@ -2159,7 +2159,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 701f2da9dc1b54cb4b6e765775a54c4c, type: 3}
m_Name:
m_EditorClassIdentifier:
debugModeIsActive: 1
debugModeIsActive: 0
ignoreReplyToStartInstructions: 1
clientInitDelay: 0.5
responsePollingInterval: 0.5
@ -3074,7 +3074,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 9c17ebccf009d574598fcda2ad1850db, type: 3}
m_Name:
m_EditorClassIdentifier:
debugModeIsActive: 1
debugModeIsActive: 0
soundEffectsEnabled: 1
audioSource: {fileID: 645973765163538692}
servicesContainer: {fileID: 4548206053279514970}

57
Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/OpenAIServices.cs
View File

@ -72,7 +72,7 @@ public class OpenAIServices : MonoBehaviour
{
if (value != this._openAIServiceState)
{
Debug.Log($"OpenAIServiceState changed, new value= {value}");
this.logIfInDebugMode($"OpenAIServiceState changed, new value= {value}");
this._openAIServiceState = value;
@ -204,41 +204,48 @@ public class OpenAIServices : MonoBehaviour
private async void init()
{
this.logIfInDebugMode("Init started");
this.logIfInDebugMode("OpenAIServices Init started");
this.OpenAIServiceState = EOpenAIServiceState.StartingUp;
try
{
this.OpenAIServiceState = EOpenAIServiceState.StartingUp;
this.client = new AssistantsClient(new Uri(this.azureResourceUrl), new AzureKeyCredential(this.azureApiKey));
this.client = new AssistantsClient(new Uri(this.azureResourceUrl), new AzureKeyCredential(this.azureApiKey));
await Task.Delay(TimeSpan.FromSeconds(this.clientInitDelay));
await Task.Delay(TimeSpan.FromSeconds(this.clientInitDelay));
Response<Assistant> assistantResponse = await this.client.CreateAssistantAsync(
new AssistantCreationOptions(assistantModel)
{
Name = assistantName,
Instructions = assistantInstructions,
Response<Assistant> assistantResponse = await this.client.CreateAssistantAsync(
new AssistantCreationOptions(assistantModel)
{
Name = assistantName,
Instructions = assistantInstructions,
});
this.assistant = assistantResponse.Value;
});
this.assistant = assistantResponse.Value;
Response<AssistantThread> threadResponse = await this.client.CreateThreadAsync();
this.thread = threadResponse.Value;
Response<AssistantThread> threadResponse = await this.client.CreateThreadAsync();
this.thread = threadResponse.Value;
this.runResponse = await client.CreateRunAsync(
this.thread.Id,
new CreateRunOptions(assistant.Id)
{
AdditionalInstructions = assistantStartInstructions,
});
ThreadRun run = runResponse.Value;
this.runResponse = await client.CreateRunAsync(
this.thread.Id,
new CreateRunOptions(assistant.Id)
{
AdditionalInstructions = assistantStartInstructions,
});
ThreadRun run = runResponse.Value;
this.logIfInDebugMode($"Init completed, ignoreReplyToStartInstructions={this.ignoreReplyToStartInstructions}");
this.logIfInDebugMode($"Init completed, ignoreReplyToStartInstructions={this.ignoreReplyToStartInstructions}");
this.ignoreIncomingReplies = this.ignoreReplyToStartInstructions;
this.ignoreIncomingReplies = this.ignoreReplyToStartInstructions;
this.OpenAIServiceState = EOpenAIServiceState.WaitingForInstructionsReply;
this.OpenAIServiceState = EOpenAIServiceState.WaitingForInstructionsReply;
await this.listen();
await this.listen();
}
catch (Exception ex)
{
this.lastError = ex.ToString();
}
}
private async Task send(string text)

13
Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechRecognitionService.cs
View File

@ -76,11 +76,12 @@ public class SpeechRecognitionService : MonoBehaviour
private string pendingPartialTranscription;
private string pendingFullTranscription;
private string invokeUserSpeechInputEndedEventIsPending;
private string pendingRecoError;
#endregion
#region Framework Functions
void OnEnable()
{
this.initialize(this.initialRecognitionLanguageCode);
@ -136,6 +137,8 @@ public class SpeechRecognitionService : MonoBehaviour
public event EventHandler<string> OnUserSpeechInputEndedEvent;
public event EventHandler<string> OnLastRecoErrorChangedEvent;
#endregion
#region Public Functions
@ -187,7 +190,7 @@ public class SpeechRecognitionService : MonoBehaviour
else if (result.Reason == ResultReason.Canceled)
{
var cancellation = CancellationDetails.FromResult(result);
this.logIfInDebugMode($"SpeechIntentService Canceled: Reason={cancellation.Reason} ErrorDetails={cancellation.ErrorDetails}");
this.pendingRecoError = $"SpeechIntentService Canceled: Reason={cancellation.Reason} ErrorDetails={cancellation.ErrorDetails}";
}
lock (threadLocker)
@ -262,6 +265,12 @@ public class SpeechRecognitionService : MonoBehaviour
this.OnUserSpeechInputEndedEvent?.Invoke(this, invokeUserSpeechInputEndedEventIsPending);
this.invokeUserSpeechInputEndedEventIsPending = null;
}
if (this.pendingRecoError != null)
{
this.OnLastRecoErrorChangedEvent?.Invoke(this, this.pendingRecoError);
this.pendingRecoError = null;
}
}
private void logIfInDebugMode(string message)

11
Viagg-io/Assets/afca/ViaggioAI/Scripts/AIServices/SpeechSynthesizerService.cs
View File

@ -122,7 +122,7 @@ public class SpeechSynthesizerService : MonoBehaviour
async void OnDisable()
{
Debug.Log("SpeechSynthesizerService disposing speechSynthesizer from OnDisable");
this.logIfInDebugMode("SpeechSynthesizerService disposing speechSynthesizer from OnDisable");
await this.disposeSynthesizer();
}
@ -138,6 +138,8 @@ public class SpeechSynthesizerService : MonoBehaviour
public event EventHandler<bool> OnSpeechOutputEndedEvent;
public event EventHandler<string> OnSynthErrorChangedEvent;
#endregion
#region Public Functions
@ -164,6 +166,11 @@ public class SpeechSynthesizerService : MonoBehaviour
SpeechSynthesisResult synthesisResult = await this.speechSynthesizer.SpeakTextAsync(text);
if (synthesisResult.Reason == ResultReason.Canceled)
{
this.OnSynthErrorChangedEvent?.Invoke(this, "Speech synthesis failed! Check internet connection");
}
if (this.speechOutputType == ESpeechOutputType.GenerateAudioClip)
{
// Todo add WavUtility
@ -189,7 +196,7 @@ public class SpeechSynthesizerService : MonoBehaviour
if (this.speechSynthesizer != null)
{
Debug.Log("SpeechSynthesizerService disposing speechSynthesizer from initialize");
this.logIfInDebugMode("SpeechSynthesizerService disposing speechSynthesizer from initialize");
await this.disposeSynthesizer();
}

33
Viagg-io/Assets/afca/ViaggioAI/Scripts/ViaggioAI/ViaggioAIManager.cs
View File

@ -107,7 +107,7 @@ public class ViaggioAIManager : MonoBehaviour
{
if (value != this._partialTranscription)
{
//Debug.Log("PartialTranscription changed, new value= " + value);
this.LogIfInDebugMode("PartialTranscription changed, new value= " + value);
this._partialTranscription = value;
@ -134,7 +134,7 @@ public class ViaggioAIManager : MonoBehaviour
{
if (value != this._fullTranscription)
{
//Debug.Log("FullTranscription changed, new value= " + value);
this.LogIfInDebugMode("FullTranscription changed, new value= " + value);
this._fullTranscription = value;
@ -196,12 +196,14 @@ public class ViaggioAIManager : MonoBehaviour
this.speechRecognitionService.OnPartialTranscriptionChangedEvent += this.onPartialTranscriptionChanged;
this.speechRecognitionService.OnFullTranscriptionChangedEvent += this.onFullTranscriptionChanged;
this.speechRecognitionService.OnUserSpeechInputEndedEvent += this.onUserSpeechInputEnded;
this.speechRecognitionService.OnLastRecoErrorChangedEvent += this.onLastRecoErrorChanged;
}
if (this.speechSynthesizerService != null)
{
this.speechSynthesizerService.OnSpeechOutputStartedEvent += this.onSpeechOutputStarted;
this.speechSynthesizerService.OnSpeechOutputEndedEvent += this.onSpeechOutputEnded;
this.speechSynthesizerService.OnSynthErrorChangedEvent += this.onSynthErrorChanged;
}
if (this.openAIServices)
@ -219,12 +221,14 @@ public class ViaggioAIManager : MonoBehaviour
this.speechRecognitionService.OnPartialTranscriptionChangedEvent -= this.onPartialTranscriptionChanged;
this.speechRecognitionService.OnFullTranscriptionChangedEvent -= this.onFullTranscriptionChanged;
this.speechRecognitionService.OnUserSpeechInputEndedEvent -= this.onUserSpeechInputEnded;
this.speechRecognitionService.OnLastRecoErrorChangedEvent -= this.onLastRecoErrorChanged;
}
if (this?.speechSynthesizerService != null)
{
this.speechSynthesizerService.OnSpeechOutputStartedEvent -= this.onSpeechOutputStarted;
this.speechSynthesizerService.OnSpeechOutputEndedEvent -= this.onSpeechOutputEnded;
this.speechSynthesizerService.OnSynthErrorChangedEvent -= this.onSynthErrorChanged;
}
if (this?.openAIServices)
@ -264,6 +268,11 @@ public class ViaggioAIManager : MonoBehaviour
this.sendIntentRequest(fullTranscription);
}
private void onLastRecoErrorChanged(object sender, string errorText)
{
this.handleError(sender, errorText);
}
private void onSpeechOutputStarted(object sender, bool dummy)
{
this.OnSpeechOutputStartedEvent?.Invoke(this, dummy);
@ -274,6 +283,11 @@ public class ViaggioAIManager : MonoBehaviour
this.OnSpeechOutputEndedEvent?.Invoke(this, dummy);
}
private void onSynthErrorChanged(object sender, string errorText)
{
this.handleError(sender, errorText);
}
private void onLastBotReplyChanged(object sender, string replyText)
{
this.processBotReply(replyText);
@ -281,7 +295,7 @@ public class ViaggioAIManager : MonoBehaviour
private void onLastBotErrorChanged(object sender, string errorText)
{
// todo handle error
this.handleError(sender, errorText);
}
#endregion
@ -301,6 +315,9 @@ public class ViaggioAIManager : MonoBehaviour
public event EventHandler<string> OnIntentRecognitionSucceededEvent;
public event EventHandler<bool> OnIntentRecognitionFailedEvent;
// Error Handling
public event EventHandler<string> OnViaggioAIErrorEvent;
#endregion
#region Public Functions
@ -317,7 +334,7 @@ public class ViaggioAIManager : MonoBehaviour
this.ViaggioAIState = EViaggioAIState.Listening;
}
public void SetSpeechRecognitionLanguage(string languageCode)
{
this.speechRecognitionService.SetRecognitionLanguage(languageCode);
@ -417,6 +434,14 @@ public class ViaggioAIManager : MonoBehaviour
this.audioSource.PlayOneShot(clip);
}
private void handleError(object sender, string errorText)
{
string logText = $"{sender} Error: {errorText}";
Debug.LogError(logText);
this.OnViaggioAIErrorEvent?.Invoke(this, logText);
}
#endregion
}