/* * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under the license found in the * LICENSE file in the root directory of this source tree. */ using System; using System.Collections.Generic; using System.Linq; using Meta.Conduit; using Meta.Voice; using Meta.WitAi.Configuration; using Meta.WitAi.Data; using Meta.WitAi.Data.Configuration; using Meta.WitAi.Data.Intents; using Meta.WitAi.Events; using Meta.WitAi.Events.UnityEventListeners; using Meta.WitAi.Interfaces; using Meta.WitAi.Json; using Meta.WitAi.Requests; using UnityEngine; namespace Meta.WitAi { public abstract class VoiceService : MonoBehaviour, IVoiceService, IInstanceResolver, IAudioEventProvider { /// /// When set to true, Conduit will be used. Otherwise, the legacy dispatching will be used. /// private bool UseConduit => WitConfiguration && WitConfiguration.useConduit; /// /// When set to true, the service will use platform integration. /// public virtual bool UsePlatformIntegrations { get => false; set => throw new NotImplementedException(); } /// /// Wit configuration accessor via IWitConfigurationProvider /// public WitConfiguration WitConfiguration { get { if (_witConfiguration == null) { _witConfiguration = GetComponent()?.Configuration; } return _witConfiguration; } } private WitConfiguration _witConfiguration; /// /// The Conduit parameter provider. /// private readonly IParameterProvider _conduitParameterProvider = new ParameterProvider(); /// /// This field should not be accessed outside the Wit-Unity library. If you need access /// to events you should be using the VoiceService.VoiceEvents property instead. /// [Tooltip("Events that will fire before, during and after an activation")] [SerializeField] protected VoiceEvents events = new VoiceEvents(); /// /// Internal events used to report telemetry. These events are reserved for internal /// use only and should not be used for any other purpose. /// protected TelemetryEvents telemetryEvents = new TelemetryEvents(); /// /// Returns true if this voice service is currently active and listening with the mic /// public virtual bool Active => Requests != null && Requests.Count > 0; /// /// The Conduit-based dispatcher that dispatches incoming invocations based on a manifest. /// internal IConduitDispatcher ConduitDispatcher { get; set; } /// /// Returns true if the service is actively communicating with Wit.ai during an Activation. The mic may or may not still be active while this is true. /// public virtual bool IsRequestActive => Requests.Count > 0; /// /// Gets/Sets a custom transcription provider. This can be used to replace any built in asr /// with an on device model or other provided source /// public abstract ITranscriptionProvider TranscriptionProvider { get; set; } /// /// Returns true if this voice service is currently reading data from the microphone /// public abstract bool MicActive { get; } public virtual VoiceEvents VoiceEvents { get => events; set => events = value; } public virtual TelemetryEvents TelemetryEvents { get => telemetryEvents; set => telemetryEvents = value; } /// /// A subset of events around collection of audio data /// public IAudioInputEvents AudioEvents => VoiceEvents; /// /// A subset of events around receiving transcriptions /// public ITranscriptionEvent TranscriptionEvents => VoiceEvents; /// /// Returns true if the audio input should be read in an activation /// protected abstract bool ShouldSendMicData { get; } /// /// All currently running requests /// public HashSet Requests { get; } = new HashSet(); /// /// Constructs a /// protected VoiceService() { _conduitParameterProvider.SetSpecializedParameter(ParameterProvider.WitResponseNodeReservedName, typeof(WitResponseNode)); _conduitParameterProvider.SetSpecializedParameter(ParameterProvider.VoiceSessionReservedName, typeof(VoiceSession)); var conduitDispatcherFactory = new ConduitDispatcherFactory(this); ConduitDispatcher = conduitDispatcherFactory.GetDispatcher(); } #region TEXT REQUESTS /// /// Send text data for NLU processing. Results will return the same way a voice based activation would. /// /// Text to be used for NLU processing public void Activate(string text) => Activate(text, new WitRequestOptions()); /// /// Send text data for NLU processing. Results will return the same way a voice based activation would. /// /// Text to be used for NLU processing /// Additional options such as dynamic entities public void Activate(string text, WitRequestOptions requestOptions) => Activate(text, requestOptions, new VoiceServiceRequestEvents()); /// /// Send text data for NLU processing. Results will return the same way a voice based activation would. /// /// Text to be used for NLU processing /// Events specific to the request's lifecycle public VoiceServiceRequest Activate(string text, VoiceServiceRequestEvents requestEvents) => Activate(text, new WitRequestOptions(), requestEvents); /// /// Send text data for NLU processing with custom request options. /// /// Text to be used for NLU processing /// Additional options such as dynamic entities /// Events specific to the request's lifecycle public abstract VoiceServiceRequest Activate(string text, WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents); /// /// Called on text request creation /// /// protected virtual void OnTextRequestCreated(VoiceServiceRequest textRequest) { if (textRequest == null) { return; } if (!textRequest.IsActive) { HandleRequestResults(textRequest); return; } textRequest.Events.OnCancel.AddListener(HandleRequestResults); textRequest.Events.OnFailed.AddListener(HandleRequestResults); textRequest.Events.OnSuccess.AddListener(HandleRequestResults); Requests.Add(textRequest); } #endregion TEXT REQUESTS #region SHARED /// /// Whether voice requests can be sent or not /// /// public virtual bool CanSend() => string.IsNullOrEmpty(GetSendError()); /// /// Check for error that will occur if attempting to send data /// /// Returns an error if send will not be allowed. protected virtual string GetSendError() { // Cannot send if internet is not reachable (Only works on Mobile) if (Application.internetReachability == NetworkReachability.NotReachable) { return "Unable to reach the internet. Check your connection."; } // No error return string.Empty; } /// /// Called after request cancellation, failure or success /// protected virtual void HandleRequestResults(VoiceServiceRequest request) { // Remove request from requests list if (Requests.Contains(request)) { Requests.Remove(request); } } #endregion SHARED #region AUDIO REQUESTS /// /// Whether audio can be activated or not /// /// public virtual bool CanActivateAudio() => string.IsNullOrEmpty(GetActivateAudioError()); /// /// Check for error that will occur if attempting to read an audio source /// /// Returns an error if audio cannot be read. protected abstract string GetActivateAudioError(); /// /// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected. /// public void Activate() => Activate(new WitRequestOptions()); /// /// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected. /// /// Additional options such as dynamic entities public void Activate(WitRequestOptions requestOptions) => Activate(requestOptions, new VoiceServiceRequestEvents()); /// /// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected. /// /// Events specific to the request's lifecycle public VoiceServiceRequest Activate(VoiceServiceRequestEvents requestEvents) => Activate(new WitRequestOptions(), requestEvents); /// /// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected. /// /// Additional options such as dynamic entities /// Events specific to the request's lifecycle public abstract VoiceServiceRequest Activate(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents); /// /// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin. /// public void ActivateImmediately() => ActivateImmediately(new WitRequestOptions()); /// /// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin. /// /// Additional options such as dynamic entities public void ActivateImmediately(WitRequestOptions requestOptions) => ActivateImmediately(requestOptions, new VoiceServiceRequestEvents()); /// /// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin. /// /// Events specific to the request's lifecycle public VoiceServiceRequest ActivateImmediately(VoiceServiceRequestEvents requestEvents) => ActivateImmediately(new WitRequestOptions(), requestEvents); /// /// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin. /// /// Additional options such as dynamic entities /// Events specific to the request's lifecycle public abstract VoiceServiceRequest ActivateImmediately(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents); /// /// Called on creation /// /// protected virtual void OnAudioRequestCreated(VoiceServiceRequest audioRequest) { if (audioRequest == null) { return; } if (!audioRequest.IsActive) { HandleRequestResults(audioRequest); return; } audioRequest.Events.OnPartialResponse.AddListener((response) => OnAudioPartialResponse(audioRequest)); audioRequest.Events.OnCancel.AddListener(HandleRequestResults); audioRequest.Events.OnFailed.AddListener(HandleRequestResults); audioRequest.Events.OnSuccess.AddListener(HandleRequestResults); Requests.Add(audioRequest); } // Callback for early validation protected virtual void OnAudioPartialResponse(VoiceServiceRequest audioRequest) { // Ignore unless can be validated if (VoiceEvents.OnValidatePartialResponse == null || audioRequest == null || audioRequest.State != VoiceRequestState.Transmitting) { return; } // Create short response data WitResponseNode response = audioRequest?.Results?.ResponseData; VoiceSession validationData = GetVoiceSession(response); // Call short response VoiceEvents.OnValidatePartialResponse.Invoke(validationData); // Invoke if (UseConduit) { // Ignore without an intent WitIntentData intent = response.GetFirstIntentData(); if (intent != null) { _conduitParameterProvider.PopulateParametersFromNode(response); _conduitParameterProvider.AddParameter(ParameterProvider.VoiceSessionReservedName, validationData); _conduitParameterProvider.AddParameter(ParameterProvider.WitResponseNodeReservedName, response); ConduitDispatcher.InvokeAction(_conduitParameterProvider, intent.name, _witConfiguration.relaxedResolution, intent.confidence, true); } } // Deactivate & abort immediately but use the response data as results if (validationData.validResponse) { VLog.I("Validated Early"); audioRequest.CompleteEarly(); } } #endregion AUDIO REQUESTS /// /// Stop listening and submit any remaining buffered microphone data for processing. /// public abstract void Deactivate(); /// /// Stop listening and abort any requests that may be active without waiting for a response. /// public abstract void DeactivateAndAbortRequest(); /// /// Abort a specific request /// public virtual void DeactivateAndAbortRequest(VoiceServiceRequest request) => request.Cancel(); /// /// Returns objects of the specified type. /// /// The type. /// Objects of the specified type. public IEnumerable GetObjectsOfType(Type type) { return FindObjectsOfType(type); } protected virtual void Awake() { InitializeEventListeners(); if (!UseConduit) { MatchIntentRegistry.Initialize(); } } private void InitializeEventListeners() { var audioEventListener = GetComponent(); if (!audioEventListener) { gameObject.AddComponent(); } var transcriptionEventListener = GetComponent(); if (!transcriptionEventListener) { gameObject.AddComponent(); } } protected virtual void OnEnable() { if (UseConduit) { ConduitDispatcher.Initialize(_witConfiguration.ManifestLocalPath); if (_witConfiguration.relaxedResolution) { if (!ConduitDispatcher.Manifest.ResolveEntities()) { VLog.E("Failed to resolve Conduit entities"); } foreach (var entity in ConduitDispatcher.Manifest.CustomEntityTypes) { _conduitParameterProvider.AddCustomType(entity.Key, entity.Value); } } } TranscriptionProvider?.OnFullTranscription.AddListener(OnFinalTranscription); VoiceEvents.OnResponse.AddListener(HandleResponse); } protected virtual void OnDisable() { TranscriptionProvider?.OnFullTranscription.RemoveListener(OnFinalTranscription); VoiceEvents.OnResponse.RemoveListener(HandleResponse); } /// /// Activate message if transcription provider returns a final transcription /// protected virtual void OnFinalTranscription(string transcription) { if (TranscriptionProvider != null) { Activate(transcription); } } private VoiceSession GetVoiceSession(WitResponseNode response) { return new VoiceSession { service = this, response = response, validResponse = false }; } protected virtual void HandleResponse(WitResponseNode response) { HandleIntents(response); } private void HandleIntents(WitResponseNode response) { var intents = response.GetIntents(); foreach (var intent in intents) { HandleIntent(intent, response); } } private void HandleIntent(WitIntentData intent, WitResponseNode response) { if (UseConduit) { _conduitParameterProvider.PopulateParametersFromNode(response); _conduitParameterProvider.AddParameter(ParameterProvider.WitResponseNodeReservedName, response); ConduitDispatcher.InvokeAction(_conduitParameterProvider, intent.name, _witConfiguration.relaxedResolution, intent.confidence, false); } else { var methods = MatchIntentRegistry.RegisteredMethods[intent.name]; foreach (var method in methods) { ExecuteRegisteredMatch(method, intent, response); } } } private void ExecuteRegisteredMatch(RegisteredMatchIntent registeredMethod, WitIntentData intent, WitResponseNode response) { if (intent.confidence >= registeredMethod.matchIntent.MinConfidence && intent.confidence <= registeredMethod.matchIntent.MaxConfidence) { foreach (var obj in GetObjectsOfType(registeredMethod.type)) { var parameters = registeredMethod.method.GetParameters(); if (parameters.Length == 0) { registeredMethod.method.Invoke(obj, Array.Empty()); continue; } if (parameters[0].ParameterType != typeof(WitResponseNode) || parameters.Length > 2) { VLog.E("Match intent only supports methods with no parameters or with a WitResponseNode parameter. Enable Conduit or adjust the parameters"); continue; } if (parameters.Length == 1) { registeredMethod.method.Invoke(obj, new object[] {response}); } } } } } public interface IVoiceService : IVoiceEventProvider, ITelemetryEventsProvider, IVoiceActivationHandler { /// /// Returns true if voice service is currently active or request is transmitting /// bool IsRequestActive { get; } /// /// When set to true, the service will use platform integration. /// bool UsePlatformIntegrations { get; set; } /// /// The current running voice requests /// HashSet Requests { get; } /// /// Returns true Mic is still enabled /// bool MicActive { get; } /// /// All events used for a voice service /// new VoiceEvents VoiceEvents { get; set; } /// /// All events used for a voice service telemetry /// new TelemetryEvents TelemetryEvents { get; set; } /// /// Easy acccess for transcription /// ITranscriptionProvider TranscriptionProvider { get; set; } /// /// Whether or not this service can listen to audio /// /// True if audio can be listened to bool CanActivateAudio(); /// /// Whether or not this service can perform requests /// /// True if a request can be sent bool CanSend(); } public interface IVoiceActivationHandler { /// /// Returns true if this voice service is currently active and listening with the mic /// bool Active { get; } /// /// Send text data for NLU processing with custom request options & events. /// /// Text to be used for NLU processing /// Additional options such as dynamic entities /// Events specific to the request's lifecycle VoiceServiceRequest Activate(string text, WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents); /// /// Activate the microphone and wait for threshold and then send data /// /// Additional options such as dynamic entities /// Events specific to the request's lifecycle VoiceServiceRequest Activate(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents); /// /// Activate the microphone and send data for NLU processing with custom request options. /// /// Additional options such as dynamic entities /// Events specific to the request's lifecycle VoiceServiceRequest ActivateImmediately(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents); /// /// Stop listening and submit the collected microphone data for processing. /// void Deactivate(); /// /// Stop listening and abort any requests that may be active without waiting for a response. /// void DeactivateAndAbortRequest(); /// /// Deactivate mic & abort a specific request /// void DeactivateAndAbortRequest(VoiceServiceRequest request); } }