/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the license found in the
* LICENSE file in the root directory of this source tree.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using Meta.Conduit;
using Meta.Voice;
using Meta.WitAi.Configuration;
using Meta.WitAi.Data;
using Meta.WitAi.Data.Configuration;
using Meta.WitAi.Data.Intents;
using Meta.WitAi.Events;
using Meta.WitAi.Events.UnityEventListeners;
using Meta.WitAi.Interfaces;
using Meta.WitAi.Json;
using Meta.WitAi.Requests;
using UnityEngine;
namespace Meta.WitAi
{
public abstract class VoiceService : MonoBehaviour, IVoiceService, IInstanceResolver, IAudioEventProvider
{
///
/// When set to true, Conduit will be used. Otherwise, the legacy dispatching will be used.
///
private bool UseConduit => WitConfiguration && WitConfiguration.useConduit;
///
/// When set to true, the service will use platform integration.
///
public virtual bool UsePlatformIntegrations
{
get => false;
set => throw new NotImplementedException();
}
///
/// Wit configuration accessor via IWitConfigurationProvider
///
public WitConfiguration WitConfiguration
{
get
{
if (_witConfiguration == null)
{
_witConfiguration = GetComponent()?.Configuration;
}
return _witConfiguration;
}
}
private WitConfiguration _witConfiguration;
///
/// The Conduit parameter provider.
///
private readonly IParameterProvider _conduitParameterProvider = new ParameterProvider();
///
/// This field should not be accessed outside the Wit-Unity library. If you need access
/// to events you should be using the VoiceService.VoiceEvents property instead.
///
[Tooltip("Events that will fire before, during and after an activation")] [SerializeField]
protected VoiceEvents events = new VoiceEvents();
///
/// Internal events used to report telemetry. These events are reserved for internal
/// use only and should not be used for any other purpose.
///
protected TelemetryEvents telemetryEvents = new TelemetryEvents();
///
/// Returns true if this voice service is currently active and listening with the mic
///
public virtual bool Active => Requests != null && Requests.Count > 0;
///
/// The Conduit-based dispatcher that dispatches incoming invocations based on a manifest.
///
internal IConduitDispatcher ConduitDispatcher { get; set; }
///
/// Returns true if the service is actively communicating with Wit.ai during an Activation. The mic may or may not still be active while this is true.
///
public virtual bool IsRequestActive => Requests.Count > 0;
///
/// Gets/Sets a custom transcription provider. This can be used to replace any built in asr
/// with an on device model or other provided source
///
public abstract ITranscriptionProvider TranscriptionProvider { get; set; }
///
/// Returns true if this voice service is currently reading data from the microphone
///
public abstract bool MicActive { get; }
public virtual VoiceEvents VoiceEvents
{
get => events;
set => events = value;
}
public virtual TelemetryEvents TelemetryEvents
{
get => telemetryEvents;
set => telemetryEvents = value;
}
///
/// A subset of events around collection of audio data
///
public IAudioInputEvents AudioEvents => VoiceEvents;
///
/// A subset of events around receiving transcriptions
///
public ITranscriptionEvent TranscriptionEvents => VoiceEvents;
///
/// Returns true if the audio input should be read in an activation
///
protected abstract bool ShouldSendMicData { get; }
///
/// All currently running requests
///
public HashSet Requests { get; } = new HashSet();
///
/// Constructs a
///
protected VoiceService()
{
_conduitParameterProvider.SetSpecializedParameter(ParameterProvider.WitResponseNodeReservedName, typeof(WitResponseNode));
_conduitParameterProvider.SetSpecializedParameter(ParameterProvider.VoiceSessionReservedName, typeof(VoiceSession));
var conduitDispatcherFactory = new ConduitDispatcherFactory(this);
ConduitDispatcher = conduitDispatcherFactory.GetDispatcher();
}
#region TEXT REQUESTS
///
/// Send text data for NLU processing. Results will return the same way a voice based activation would.
///
/// Text to be used for NLU processing
public void Activate(string text) => Activate(text, new WitRequestOptions());
///
/// Send text data for NLU processing. Results will return the same way a voice based activation would.
///
/// Text to be used for NLU processing
/// Additional options such as dynamic entities
public void Activate(string text, WitRequestOptions requestOptions) => Activate(text, requestOptions, new VoiceServiceRequestEvents());
///
/// Send text data for NLU processing. Results will return the same way a voice based activation would.
///
/// Text to be used for NLU processing
/// Events specific to the request's lifecycle
public VoiceServiceRequest Activate(string text, VoiceServiceRequestEvents requestEvents) => Activate(text, new WitRequestOptions(), requestEvents);
///
/// Send text data for NLU processing with custom request options.
///
/// Text to be used for NLU processing
/// Additional options such as dynamic entities
/// Events specific to the request's lifecycle
public abstract VoiceServiceRequest Activate(string text, WitRequestOptions requestOptions,
VoiceServiceRequestEvents requestEvents);
///
/// Called on text request creation
///
///
protected virtual void OnTextRequestCreated(VoiceServiceRequest textRequest)
{
if (textRequest == null)
{
return;
}
if (!textRequest.IsActive)
{
HandleRequestResults(textRequest);
return;
}
textRequest.Events.OnCancel.AddListener(HandleRequestResults);
textRequest.Events.OnFailed.AddListener(HandleRequestResults);
textRequest.Events.OnSuccess.AddListener(HandleRequestResults);
Requests.Add(textRequest);
}
#endregion TEXT REQUESTS
#region SHARED
///
/// Whether voice requests can be sent or not
///
///
public virtual bool CanSend() => string.IsNullOrEmpty(GetSendError());
///
/// Check for error that will occur if attempting to send data
///
/// Returns an error if send will not be allowed.
protected virtual string GetSendError()
{
// Cannot send if internet is not reachable (Only works on Mobile)
if (Application.internetReachability == NetworkReachability.NotReachable)
{
return "Unable to reach the internet. Check your connection.";
}
// No error
return string.Empty;
}
///
/// Called after request cancellation, failure or success
///
protected virtual void HandleRequestResults(VoiceServiceRequest request)
{
// Remove request from requests list
if (Requests.Contains(request))
{
Requests.Remove(request);
}
}
#endregion SHARED
#region AUDIO REQUESTS
///
/// Whether audio can be activated or not
///
///
public virtual bool CanActivateAudio() => string.IsNullOrEmpty(GetActivateAudioError());
///
/// Check for error that will occur if attempting to read an audio source
///
/// Returns an error if audio cannot be read.
protected abstract string GetActivateAudioError();
///
/// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected.
///
public void Activate() => Activate(new WitRequestOptions());
///
/// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected.
///
/// Additional options such as dynamic entities
public void Activate(WitRequestOptions requestOptions) =>
Activate(requestOptions, new VoiceServiceRequestEvents());
///
/// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected.
///
/// Events specific to the request's lifecycle
public VoiceServiceRequest Activate(VoiceServiceRequestEvents requestEvents) =>
Activate(new WitRequestOptions(), requestEvents);
///
/// Start listening for sound or speech from the user and start sending data to Wit.ai once sound or speech has been detected.
///
/// Additional options such as dynamic entities
/// Events specific to the request's lifecycle
public abstract VoiceServiceRequest Activate(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents);
///
/// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin.
///
public void ActivateImmediately() => ActivateImmediately(new WitRequestOptions());
///
/// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin.
///
/// Additional options such as dynamic entities
public void ActivateImmediately(WitRequestOptions requestOptions) =>
ActivateImmediately(requestOptions, new VoiceServiceRequestEvents());
///
/// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin.
///
/// Events specific to the request's lifecycle
public VoiceServiceRequest ActivateImmediately(VoiceServiceRequestEvents requestEvents) =>
ActivateImmediately(new WitRequestOptions(), requestEvents);
///
/// Activate the microphone and send data for NLU processing immediately without waiting for sound/speech from the user to begin.
///
/// Additional options such as dynamic entities
/// Events specific to the request's lifecycle
public abstract VoiceServiceRequest ActivateImmediately(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents);
///
/// Called on creation
///
///
protected virtual void OnAudioRequestCreated(VoiceServiceRequest audioRequest)
{
if (audioRequest == null)
{
return;
}
if (!audioRequest.IsActive)
{
HandleRequestResults(audioRequest);
return;
}
audioRequest.Events.OnPartialResponse.AddListener((response) => OnAudioPartialResponse(audioRequest));
audioRequest.Events.OnCancel.AddListener(HandleRequestResults);
audioRequest.Events.OnFailed.AddListener(HandleRequestResults);
audioRequest.Events.OnSuccess.AddListener(HandleRequestResults);
Requests.Add(audioRequest);
}
// Callback for early validation
protected virtual void OnAudioPartialResponse(VoiceServiceRequest audioRequest)
{
// Ignore unless can be validated
if (VoiceEvents.OnValidatePartialResponse == null || audioRequest == null || audioRequest.State != VoiceRequestState.Transmitting)
{
return;
}
// Create short response data
WitResponseNode response = audioRequest?.Results?.ResponseData;
VoiceSession validationData = GetVoiceSession(response);
// Call short response
VoiceEvents.OnValidatePartialResponse.Invoke(validationData);
// Invoke
if (UseConduit)
{
// Ignore without an intent
WitIntentData intent = response.GetFirstIntentData();
if (intent != null)
{
_conduitParameterProvider.PopulateParametersFromNode(response);
_conduitParameterProvider.AddParameter(ParameterProvider.VoiceSessionReservedName,
validationData);
_conduitParameterProvider.AddParameter(ParameterProvider.WitResponseNodeReservedName, response);
ConduitDispatcher.InvokeAction(_conduitParameterProvider, intent.name, _witConfiguration.relaxedResolution, intent.confidence, true);
}
}
// Deactivate & abort immediately but use the response data as results
if (validationData.validResponse)
{
VLog.I("Validated Early");
audioRequest.CompleteEarly();
}
}
#endregion AUDIO REQUESTS
///
/// Stop listening and submit any remaining buffered microphone data for processing.
///
public abstract void Deactivate();
///
/// Stop listening and abort any requests that may be active without waiting for a response.
///
public abstract void DeactivateAndAbortRequest();
///
/// Abort a specific request
///
public virtual void DeactivateAndAbortRequest(VoiceServiceRequest request) => request.Cancel();
///
/// Returns objects of the specified type.
///
/// The type.
/// Objects of the specified type.
public IEnumerable