Initial Commit

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Configuration.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: ea91106e590de2945938dd234262b37e
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Configuration/DictationConfiguration.cs

@@ -0,0 +1,16 @@
+using System;
+using UnityEngine;
+
+namespace Oculus.Voice.Dictation.Configuration
+{
+    [Serializable]
+    public class DictationConfiguration
+    {
+        [Tooltip("Re-open the mic after the final transcription. Useful for long form content/messaging.")]
+        public bool multiPhrase;
+        [Tooltip("Hint about the scenario that the user is dictating. Default to package name. In the future we might have messaging, search, general, etc")]
+        public string scenario = "default";
+        [Tooltip("Input types: text_default: Normal text, numeric: Numbers, email: Email addresses")]
+        public string inputType = "text_default";
+    }
+}

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Configuration/DictationConfiguration.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 07b07eeddd4fd7847a488c8fb590b5e9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Listeners.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 6b7302946fb0f1e4299eb699e30c7449
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Listeners/DictationListener.cs

@@ -0,0 +1,40 @@
+namespace Oculus.Voice.Dictation.Listeners
+{
+    public interface DictationListener
+    {
+        /// <summary>
+        /// Called when dictation has started
+        /// </summary>
+        void OnStart(DictationListener listener);
+
+        /// <summary>
+        /// Called when mic level changes. Used for building UI.
+        /// </summary>
+        /// <param name="micLevel"></param>
+        void OnMicAudioLevel(float micLevel);
+
+        /// <summary>
+        /// Called with current predicted transcription. Could change as user speaks.
+        /// </summary>
+        /// <param name="transcription"></param>
+        void OnPartialTranscription(string transcription);
+
+        /// <summary>
+        /// Final transcription of what the user has said
+        /// </summary>
+        /// <param name="transcription"></param>
+        void OnFinalTranscription(string transcription);
+
+        /// <summary>
+        /// Called when there was an error with the dictation service
+        /// </summary>
+        /// <param name="errorType">The type of error encountered</param>
+        /// <param name="errorMessage">Human readable message describing the error</param>
+        void OnError(string errorType, string errorMessage);
+
+        /// <summary>
+        /// Called when the dictation session is done
+        /// </summary>
+        void OnStopped(DictationListener listener);
+    }
+}

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Listeners/DictationListener.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 809d25c26f051e44991a37caa328aa7a
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Service.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 0768efab3897ba744a6ad727b7bc28ee
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Service/AppDictationExperience.cs

@@ -0,0 +1,458 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * Licensed under the Oculus SDK License Agreement (the "License");
+ * you may not use the Oculus SDK except in compliance with the License,
+ * which is provided at the time of installation or download, or which
+ * otherwise accompanies this software in either electronic or hard copy form.
+ *
+ * You may obtain a copy of the License at
+ *
+ * https://developer.oculus.com/licenses/oculussdk/
+ *
+ * Unless required by applicable law or agreed to in writing, the Oculus SDK
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Globalization;
+using Meta.Voice;
+using Meta.WitAi;
+using Meta.WitAi.Configuration;
+using Meta.WitAi.Data.Configuration;
+using Meta.WitAi.Dictation;
+using Meta.WitAi.Dictation.Data;
+using Meta.WitAi.Interfaces;
+using Meta.WitAi.Requests;
+using Oculus.Voice.Dictation.Bindings.Android;
+using Oculus.VoiceSDK.Utilities;
+using Oculus.Voice.Core.Bindings.Android.PlatformLogger;
+using Oculus.Voice.Core.Bindings.Interfaces;
+using UnityEngine;
+
+namespace Oculus.Voice.Dictation
+{
+    public class AppDictationExperience : DictationService, IWitRuntimeConfigProvider, IWitConfigurationProvider
+    {
+        [SerializeField] private WitDictationRuntimeConfiguration runtimeConfiguration;
+        [Tooltip("Uses platform dictation service instead of accessing wit directly from within the application.")]
+        [SerializeField] private bool usePlatformServices;
+
+        [Tooltip("Dictation will not fallback to Wit if platform dictation is not available. Not applicable in Unity Editor")]
+        [SerializeField] private bool doNotFallbackToWit;
+        [Tooltip("Enables logs related to the interaction to be displayed on console")]
+        [SerializeField] private bool enableConsoleLogging;
+
+        public WitRuntimeConfiguration RuntimeConfiguration => runtimeConfiguration;
+        public WitDictationRuntimeConfiguration RuntimeDictationConfiguration
+        {
+            get => runtimeConfiguration;
+            set => runtimeConfiguration = value;
+        }
+        public WitConfiguration Configuration => RuntimeConfiguration?.witConfiguration;
+
+        private IDictationService _dictationServiceImpl;
+        private IVoiceSDKLogger _voiceSDKLogger;
+        /// <summary>
+        /// True if the user currently has requested dictation to be active. This will remain true until a Deactivate
+        /// method is called and we will reactivate when the mic stops as a result.
+        /// </summary>
+        private bool _isActive;
+
+        private DictationSession _activeSession;
+        private WitRequestOptions _activeRequestOptions;
+
+        public DictationSession ActiveSession => _activeSession;
+        public WitRequestOptions ActiveRequestOptions => _activeRequestOptions;
+
+        public event Action OnInitialized;
+
+        private static string PACKAGE_VERSION => VoiceSDKConstants.SdkVersion;
+
+#if UNITY_ANDROID && !UNITY_EDITOR
+        public bool HasPlatformIntegrations => usePlatformServices && _dictationServiceImpl is PlatformDictationImpl;
+#else
+        public bool HasPlatformIntegrations => false;
+#endif
+
+        public bool UsePlatformIntegrations
+        {
+            get => usePlatformServices;
+            set
+            {
+                // If we're trying to turn on platform services and they're not currently active we
+                // will forcably reinit and try to set the state.
+                if (usePlatformServices != value || HasPlatformIntegrations != value)
+                {
+                    usePlatformServices = value;
+#if UNITY_ANDROID && !UNITY_EDITOR
+                    VLog.D($"{(usePlatformServices ? "Enabling" : "Disabling")} platform integration.");
+                    InitDictation();
+#endif
+                }
+            }
+        }
+
+        public bool DoNotFallbackToWit
+        {
+            get => doNotFallbackToWit;
+            set => doNotFallbackToWit = value;
+        }
+
+        private void InitDictation()
+        {
+            // Check voice sdk version
+            if (string.IsNullOrEmpty(PACKAGE_VERSION))
+            {
+                VLog.E("No SDK Version Set");
+            }
+            // Clean up if we're switching to native C# wit impl
+            if (!UsePlatformIntegrations)
+            {
+                if (_dictationServiceImpl is PlatformDictationImpl)
+                {
+                    ((PlatformDictationImpl) _dictationServiceImpl).Disconnect();
+                }
+
+                if (_voiceSDKLogger is VoiceSDKPlatformLoggerImpl)
+                {
+                    try
+                    {
+                        ((VoiceSDKPlatformLoggerImpl)_voiceSDKLogger).Disconnect();
+                    }
+                    catch (Exception e)
+                    {
+                        VLog.E($"Disconnection error: {e.Message}");
+                    }
+                }
+            }
+#if UNITY_ANDROID && !UNITY_EDITOR
+            var loggerImpl = new VoiceSDKPlatformLoggerImpl();
+            loggerImpl.Connect(PACKAGE_VERSION);
+            _voiceSDKLogger = loggerImpl;
+
+            if (UsePlatformIntegrations)
+            {
+                VLog.D("Checking platform dictation capabilities...");
+                var platformDictationImpl = new PlatformDictationImpl(this);
+                platformDictationImpl.OnServiceNotAvailableEvent += OnPlatformServiceNotAvailable;
+                platformDictationImpl.Connect(PACKAGE_VERSION);
+                if (platformDictationImpl.PlatformSupportsDictation)
+                {
+                    _dictationServiceImpl = platformDictationImpl;
+                    _dictationServiceImpl.DictationEvents = DictationEvents;
+                    _dictationServiceImpl.TelemetryEvents = TelemetryEvents;
+                    platformDictationImpl.SetDictationRuntimeConfiguration(RuntimeDictationConfiguration);
+                    VLog.D("Dictation platform init complete");
+                    _voiceSDKLogger.IsUsingPlatformIntegration = true;
+                }
+                else
+                {
+                    OnPlatformServiceNotAvailable();
+                }
+            }
+            else
+            {
+                RevertToWitDictation();
+            }
+#else
+            _voiceSDKLogger = new VoiceSDKConsoleLoggerImpl();
+            RevertToWitDictation();
+#endif
+            _voiceSDKLogger.WitApplication = RuntimeDictationConfiguration?.witConfiguration?.GetLoggerAppId();
+            _voiceSDKLogger.ShouldLogToConsole = enableConsoleLogging;
+
+            OnInitialized?.Invoke();
+        }
+
+        private void OnPlatformServiceNotAvailable()
+        {
+#if !UNITY_EDITOR
+            if (DoNotFallbackToWit)
+            {
+                VLog.D("Platform dictation service unavailable. Falling back to WitDictation is disabled");
+                DictationEvents.OnError?.Invoke("Platform dictation unavailable", "Platform dictation service is not available");
+                return;
+            }
+#endif
+
+            VLog.D("Platform dictation service unavailable. Falling back to WitDictation");
+            RevertToWitDictation();
+        }
+
+        private void OnDictationServiceNotAvailable()
+        {
+            VLog.D("Dictation service unavailable");
+            DictationEvents.OnError?.Invoke("Dictation unavailable", "Dictation service is not available");
+        }
+
+        private void RevertToWitDictation()
+        {
+            WitDictation witDictation = GetComponent<WitDictation>();
+            if (null == witDictation)
+            {
+                witDictation = gameObject.AddComponent<WitDictation>();
+                witDictation.hideFlags = HideFlags.HideInInspector;
+            }
+
+            witDictation.RuntimeConfiguration = RuntimeDictationConfiguration;
+            witDictation.DictationEvents = DictationEvents;
+            witDictation.TelemetryEvents = TelemetryEvents;
+            _dictationServiceImpl = witDictation;
+            VLog.D("WitDictation init complete");
+            _voiceSDKLogger.IsUsingPlatformIntegration = false;
+        }
+
+        protected override void OnEnable()
+        {
+            base.OnEnable();
+            if (MicPermissionsManager.HasMicPermission())
+            {
+                InitDictation();
+            }
+            else
+            {
+                MicPermissionsManager.RequestMicPermission((e) => InitDictation());
+            }
+
+            DictationEvents.OnRequestInitialized.AddListener(OnRequestInit);
+            DictationEvents.OnStartListening.AddListener(OnStarted);
+            DictationEvents.OnStoppedListening.AddListener(OnStopped);
+            DictationEvents.OnComplete.AddListener(OnComplete);
+            DictationEvents.OnDictationSessionStarted.AddListener(OnDictationSessionStarted);
+            DictationEvents.OnPartialTranscription.AddListener(OnPartialTranscription);
+            DictationEvents.OnFullTranscription.AddListener(OnFullTranscription);
+            TelemetryEvents.OnAudioTrackerFinished.AddListener(OnAudioDurationTrackerFinished);
+        }
+
+        protected override void OnDisable()
+        {
+#if UNITY_ANDROID
+            if (_dictationServiceImpl is PlatformDictationImpl platformDictationImpl)
+            {
+                platformDictationImpl.Disconnect();
+            }
+
+            if (_voiceSDKLogger is VoiceSDKPlatformLoggerImpl loggerImpl)
+            {
+                loggerImpl.Disconnect();
+            }
+#endif
+            _dictationServiceImpl = null;
+            _voiceSDKLogger = null;
+            DictationEvents.OnRequestInitialized.RemoveListener(OnRequestInit);
+            DictationEvents.OnStartListening.RemoveListener(OnStarted);
+            DictationEvents.OnStoppedListening.RemoveListener(OnStopped);
+            DictationEvents.OnComplete.RemoveListener(OnComplete);
+            DictationEvents.OnDictationSessionStarted.RemoveListener(OnDictationSessionStarted);
+            DictationEvents.OnPartialTranscription.RemoveListener(OnPartialTranscription);
+            DictationEvents.OnFullTranscription.RemoveListener(OnFullTranscription);
+            TelemetryEvents.OnAudioTrackerFinished.RemoveListener(OnAudioDurationTrackerFinished);
+            base.OnDisable();
+        }
+
+        #region DictationService properties
+
+        public override bool Active => _dictationServiceImpl != null && _dictationServiceImpl.Active;
+        public override bool IsRequestActive => _dictationServiceImpl != null && _dictationServiceImpl.IsRequestActive;
+
+        public override ITranscriptionProvider TranscriptionProvider
+        {
+            get => _dictationServiceImpl.TranscriptionProvider;
+            set => _dictationServiceImpl.TranscriptionProvider = value;
+
+        }
+        public override bool MicActive => null != _dictationServiceImpl && _dictationServiceImpl.MicActive;
+        protected override bool ShouldSendMicData => RuntimeConfiguration.sendAudioToWit ||
+                                                     null == TranscriptionProvider;
+        #endregion
+
+        #region DictationService APIs
+        /// <summary>
+        /// Toggle dictation activation from on->off or off->on depending on the current active state.
+        /// </summary>
+        public void Toggle()
+        {
+            if(Active) Deactivate();
+            else Activate();
+        }
+
+        /// <summary>
+        /// Activate the microphone and send data to Wit for NLU processing.
+        /// </summary>
+        public override VoiceServiceRequest Activate(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents)
+        {
+            if (_dictationServiceImpl == null)
+            {
+                OnDictationServiceNotAvailable();
+                return null;
+            }
+            if (!_isActive)
+            {
+                _activeSession = new DictationSession();
+                DictationEvents.OnDictationSessionStarted.Invoke(_activeSession);
+            }
+            _isActive = true;
+            return _dictationServiceImpl.Activate(requestOptions, requestEvents);
+        }
+
+        /// <summary>
+        /// Activates immediately and starts sending data to the server. This will not wait for min wake threshold
+        /// </summary>
+        /// <param name="options"></param>
+        public override VoiceServiceRequest ActivateImmediately(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents)
+        {
+            if (_dictationServiceImpl == null)
+            {
+                OnDictationServiceNotAvailable();
+                return null;
+            }
+            if (!_isActive)
+            {
+                _activeSession = new DictationSession();
+                DictationEvents.OnDictationSessionStarted.Invoke(_activeSession);
+            }
+            _isActive = true;
+            return _dictationServiceImpl.ActivateImmediately(requestOptions, requestEvents);
+        }
+
+        /// <summary>
+        /// Deactivates. If a transcription is in progress the network request will complete and any additional
+        /// transcription values will be returned.
+        /// </summary>
+        public override void Deactivate()
+        {
+            if (_dictationServiceImpl == null)
+            {
+                OnDictationServiceNotAvailable();
+                return;
+            }
+
+            _isActive = false;
+            _dictationServiceImpl.Deactivate();
+        }
+
+        /// <summary>
+        /// Deactivates and ignores any pending transcription content.
+        /// </summary>
+        public override void Cancel()
+        {
+            if (_dictationServiceImpl == null)
+            {
+                OnDictationServiceNotAvailable();
+                return;
+            }
+
+            _dictationServiceImpl.Cancel();
+            CleanupSession();
+        }
+        #endregion
+
+        #region Listeners for logging
+        void OnRequestInit(VoiceServiceRequest request)
+        {
+            _activeRequestOptions = request?.Options;
+
+            _voiceSDKLogger.LogInteractionStart(request?.Options?.RequestId, "dictation");
+
+#if UNITY_ANDROID && !UNITY_EDITOR
+            _voiceSDKLogger.LogAnnotation("clientSDKVersion", PACKAGE_VERSION);
+#endif
+            _voiceSDKLogger.LogAnnotation("minWakeThreshold",
+                RuntimeConfiguration?.soundWakeThreshold.ToString(CultureInfo.InvariantCulture));
+            _voiceSDKLogger.LogAnnotation("minKeepAliveTimeSec",
+                RuntimeConfiguration?.minKeepAliveTimeInSeconds.ToString(CultureInfo.InvariantCulture));
+            _voiceSDKLogger.LogAnnotation("minTranscriptionKeepAliveTimeSec",
+                RuntimeConfiguration?.minTranscriptionKeepAliveTimeInSeconds.ToString(CultureInfo.InvariantCulture));
+            _voiceSDKLogger.LogAnnotation("maxRecordingTime",
+                RuntimeConfiguration?.maxRecordingTime.ToString(CultureInfo.InvariantCulture));
+        }
+
+        void OnStarted()
+        {
+            _voiceSDKLogger.LogInteractionPoint("startedListening");
+        }
+
+        void OnStopped()
+        {
+            _voiceSDKLogger.LogInteractionPoint("stoppedListening");
+
+            if (RuntimeDictationConfiguration.dictationConfiguration.multiPhrase && _isActive)
+            {
+                Activate(_activeRequestOptions);
+            }
+        }
+
+        void OnDictationSessionStarted(DictationSession session)
+        {
+            if (session is PlatformDictationSession platformDictationSession)
+            {
+                _activeSession = session;
+                _voiceSDKLogger.LogAnnotation("platformInteractionId", platformDictationSession.platformSessionId);
+            }
+        }
+
+        void OnAudioDurationTrackerFinished(long timestamp, double audioDuration)
+        {
+            _voiceSDKLogger.LogAnnotation("adt_duration", audioDuration.ToString(CultureInfo.InvariantCulture));
+            _voiceSDKLogger.LogAnnotation("adt_finished", timestamp.ToString());
+        }
+
+        void OnPartialTranscription(string text)
+        {
+            _voiceSDKLogger.LogFirstTranscriptionTime();
+        }
+
+        void OnFullTranscription(string text)
+        {
+            _voiceSDKLogger.LogInteractionPoint("fullTranscriptionTime");
+        }
+
+        void OnComplete(VoiceServiceRequest request)
+        {
+            if (request.State == VoiceRequestState.Failed)
+            {
+                VLog.E($"Dictation Request Failed\nError: {request.Results.Message}");
+                _voiceSDKLogger.LogInteractionEndFailure(request.Results.Message);
+            }
+            else if (request.State == VoiceRequestState.Canceled)
+            {
+                VLog.W($"Dictation Request Canceled\nMessage: {request.Results.Message}");
+                _voiceSDKLogger.LogInteractionEndFailure("aborted");
+            }
+            else
+            {
+                VLog.D($"Dictation Request Success");
+                var tokens = request.ResponseData?["speech"]?["tokens"];
+                if (tokens != null)
+                {
+                    int speechTokensLength = tokens.Count;
+                    string speechLength = request.ResponseData["speech"]["tokens"][speechTokensLength - 1]?["end"]?.Value;
+                    _voiceSDKLogger.LogAnnotation("audioLength", speechLength);
+                }
+                _voiceSDKLogger.LogInteractionEndSuccess();
+            }
+            if (!_isActive)
+            {
+                DictationEvents.OnDictationSessionStopped?.Invoke(_activeSession);
+                CleanupSession();
+            }
+        }
+        #endregion
+
+        #region Cleanup
+
+        private void CleanupSession()
+        {
+            _activeSession = null;
+            _activeRequestOptions = null;
+            _isActive = false;
+        }
+
+        #endregion
+    }
+}

Assets/Oculus/Voice/Features/Dictation/Scripts/Runtime/API/Service/AppDictationExperience.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: f6da0ed72f4541d7b52607891bafa6f3
+timeCreated: 1652467620