Merge pull request #4 from neocortex-link/feature/webgl-audio

WebGL Audio Support

Editor/NeocortexAudioReceiverEditor.cs → Editor/AudioReceiverEditor.cs

@@ -3,8 +3,8 @@
 
 namespace Neocortex.Editor
 {
-    [CustomEditor(typeof(NeocortexAudioReceiver))]
-    public class NeocortexAudioReceiverEditor : UnityEditor.Editor
+    [CustomEditor(typeof(AudioReceiver), true)]
+    public class AudioReceiverEditor : UnityEditor.Editor
     {
         private const string MIC_INDEX_KEY = "neocortex-mic-index";
 
@@ -18,9 +18,11 @@ private void OnEnable()
         {
             microphoneOptions = Microphone.devices;
             selectedMicrophoneIndex = PlayerPrefs.GetInt(MIC_INDEX_KEY, 0);
+
+            AudioReceiver audioReceiver = (AudioReceiver)target;
 
-            NeocortexAudioReceiver audioReceiver = (NeocortexAudioReceiver)target;
-            audioReceiver.SelectedMicrophone = microphoneOptions[selectedMicrophoneIndex];
+            if(audioReceiver is NeocortexAudioReceiver neocortexAudioReceiver)
+                neocortexAudioReceiver.SelectedMicrophone = microphoneOptions[selectedMicrophoneIndex];
 
             onAudioRecorded = serializedObject.FindProperty("OnAudioRecorded");
             onRecordingFailed = serializedObject.FindProperty("OnRecordingFailed");
@@ -30,13 +32,13 @@ public override void OnInspectorGUI()
         {
             DrawDefaultInspector();
 
-            NeocortexAudioReceiver audioReceiver = (NeocortexAudioReceiver)target;
-
-            if (microphoneOptions is { Length: > 0 })
+            AudioReceiver audioReceiver = (AudioReceiver)target;
+            
+            if (microphoneOptions is { Length: > 0 } && audioReceiver is NeocortexAudioReceiver neocortexAudioReceiver)
             {
                 selectedMicrophoneIndex = EditorGUILayout.Popup("Select Microphone", selectedMicrophoneIndex, microphoneOptions);
                 PlayerPrefs.SetInt(MIC_INDEX_KEY, selectedMicrophoneIndex);
-                audioReceiver.SelectedMicrophone = microphoneOptions[selectedMicrophoneIndex];
+                neocortexAudioReceiver.SelectedMicrophone = microphoneOptions[selectedMicrophoneIndex];
             }
             else
             {

Editor/NeocortexSettingsWindow.cs

@@ -8,7 +8,7 @@ public class NeocortexSettingsWindow : EditorWindow
     {
         private static NeocortexSettings settings;
 
-        [MenuItem("Tools/Neocortex Settings", false, 0)]
+        [MenuItem("Tools/Neocortex/API Key Setup", false, 0)]
         public static void ShowWindow()
         {
             NeocortexSettingsWindow window = GetWindow<NeocortexSettingsWindow>("Neocortex Settings");

Editor/UI/NeocortexAudioChatInput.cs

@@ -1,4 +1,3 @@
-using System;
 using UnityEditor;
 using UnityEngine;
 

Editor/WebGLTemplateImporter.cs

@@ -0,0 +1,75 @@
+using System.IO;
+using UnityEditor;
+using UnityEngine;
+
+namespace Neocortex.Editor
+{
+    [InitializeOnLoad]
+    public class WebGLTemplateImporter
+    {
+        private const string SourceFolder = "Packages/link.neocortex.sdk/WebGLTemplates/Neocortex";
+        private const string DestinationFolder = "Assets/WebGLTemplates/Neocortex";
+        private const string ImportCompletedKey = "Neocortex.WebGLTemplateImported";
+
+        static WebGLTemplateImporter()
+        {
+            EditorApplication.delayCall += OnEditorLoaded;
+        }
+
+        [MenuItem("Tools/Neocortex/Import WebGL Template", false, 0)]
+        public static void ImportWebGLTemplate()
+        {
+            if (EditorUtility.DisplayDialog("Import WebGL Template", "This will overwrite any changes you have made in the WebGL template. Are you sure you want to continue?", "Yes", "No"))
+            {
+                if (Directory.Exists(DestinationFolder))
+                {
+                    Directory.Delete(DestinationFolder, recursive: true);
+                }
+
+                OnEditorLoaded();
+            }
+        }
+
+        private static void OnEditorLoaded()
+        {
+            if (EditorPrefs.HasKey(ImportCompletedKey) && Directory.Exists(DestinationFolder))
+            {
+                return;
+            }
+
+            EditorPrefs.DeleteKey(ImportCompletedKey);
+
+            try{
+                CopyDirectory(SourceFolder, DestinationFolder);
+                AssetDatabase.Refresh();
+                Debug.Log($"WebGL Template copied to {DestinationFolder} successfully.");
+            }
+            catch (System.Exception ex)
+            {
+                Debug.LogError($"Failed to copy WebGL Template: {ex.Message}");
+            }
+        }
+
+        private static void CopyDirectory(string sourceDir, string destinationDir)
+        {
+            if (!Directory.Exists(destinationDir))
+            {
+                Directory.CreateDirectory(destinationDir);
+            }
+
+            foreach (var file in Directory.GetFiles(sourceDir))
+            {
+                string destFile = Path.Combine(destinationDir, Path.GetFileName(file));
+                File.Copy(file, destFile, overwrite: true);
+            }
+
+            foreach (var directory in Directory.GetDirectories(sourceDir))
+            {
+                string destDir = Path.Combine(destinationDir, Path.GetFileName(directory));
+                CopyDirectory(directory, destDir);
+            }
+
+            EditorPrefs.SetBool(ImportCompletedKey, true);
+        }
+    }
+}

Runtime/AudioReceiver.cs

@@ -0,0 +1,20 @@
+using System;
+using UnityEngine;
+using UnityEngine.Events;
+
+namespace Neocortex
+{
+    public abstract class AudioReceiver: MonoBehaviour
+    {
+        [SerializeField] private bool usePushToTalk;
+        public bool UsePushToTalk { get => usePushToTalk; protected set => usePushToTalk = value; }
+        public float Amplitude { get; protected set; }
+        public float ElapsedWaitTime { get; protected set; }
+
+        public abstract void StartMicrophone();
+        public abstract void StopMicrophone();
+
+        [HideInInspector] public UnityEvent<AudioClip> OnAudioRecorded;
+        [HideInInspector] public UnityEvent<string> OnRecordingFailed;
+    }
+}

Runtime/Data/Enums/MicrophoneState.cs

@@ -0,0 +1,9 @@
+namespace Neocortex.Data
+{
+    public enum MicrophoneState
+    {
+        NotActive = 0,
+        Booting = 1,
+        Recording = 2,
+    }
+}

Runtime/Data/FloatArray.cs

@@ -0,0 +1,8 @@
+namespace Neocortex.Data
+{
+    public struct FloatArray
+    { 
+        public float [] Buffer;
+        public int Written;
+    }
+}

Runtime/Extension Methods/AudioClipExtensions.cs

@@ -17,6 +17,13 @@ public static AudioClip Trim(this AudioClip audioClip, float treshold = 0.002f)
 
             sampleList.RemoveAll(sample => Mathf.Abs(sample) < treshold);
 
+            // if audio is shorter than 0.2 seconds, return null
+            // this is to prevent sound such as mouse clicks from being sent
+            if (sampleList.Count < audioClip.frequency / 4)
+            {
+                return null;
+            }
+
             if (sampleList.Count > 0)
             {
                 var lengthSamples = Mathf.Max(sampleList.Count, audioClip.frequency);

Runtime/NeocortexAudioReceiver.cs

@@ -1,10 +1,9 @@
 using System;
 using UnityEngine;
-using UnityEngine.Events;
 
 namespace Neocortex
 {
-    public class NeocortexAudioReceiver : MonoBehaviour
+    public class NeocortexAudioReceiver : AudioReceiver
     {
         private const int FREQUENCY = 22050;
         private const int AUDIO_SAMPLE_WINDOW = 64;
@@ -14,24 +13,16 @@ public class NeocortexAudioReceiver : MonoBehaviour
         private bool initialized;
 
         public string SelectedMicrophone { get; set; }
-        public float Amplitude { get; private set; }
         public bool IsUserSpeaking { get; private set; }
-        public float ElapsedWaitTime { get; private set; }
-
-        public bool UsePushToTalk => usePushToTalk;
-
-        [HideInInspector] public UnityEvent<AudioClip> OnAudioRecorded;
-        [HideInInspector] public UnityEvent<string> OnRecordingFailed;
 
-        [SerializeField, Range(0, 1)] private float amplitudeTreshold = 0.1f;
+        [SerializeField, Range(0, 1)] private float amplitudeThreshold = 0.1f;
         [SerializeField] private float maxWaitTime = 1f;
-        [SerializeField] private bool usePushToTalk;
 
-        public void StartMicrophone()
+        public override void StartMicrophone()
         {
             try
             {
-                audioClip = Microphone.Start(SelectedMicrophone, true, 999, FREQUENCY);
+                audioClip = NeocortexMicrophone.Start(SelectedMicrophone, true, 999, FREQUENCY);
                 initialized = true;
             }
             catch (Exception e)
@@ -40,9 +31,9 @@ public void StartMicrophone()
             }
         }
 
-        public void StopMicrophone()
+        public override void StopMicrophone()
         {
-            Microphone.End(SelectedMicrophone);
+            NeocortexMicrophone.End(SelectedMicrophone);
             initialized = false;
             IsUserSpeaking = false;
             AudioRecorded();
@@ -54,16 +45,16 @@ private void Update()
 
             UpdateAmplitude();
 
-            if (usePushToTalk) return;
+            if (UsePushToTalk) return;
 
-            if(!IsUserSpeaking && Amplitude > amplitudeTreshold)
+            if(!IsUserSpeaking && Amplitude > amplitudeThreshold)
             {
                 IsUserSpeaking = true;
             }
 
             if (IsUserSpeaking)
             {
-                if (Amplitude < amplitudeTreshold)
+                if (Amplitude < amplitudeThreshold)
                 {
                     ElapsedWaitTime += Time.deltaTime;
 
@@ -83,12 +74,19 @@ private void Update()
         private void AudioRecorded()
         {
             AudioClip trimmed = audioClip.Trim();
-            OnAudioRecorded?.Invoke(trimmed);
+            if (!trimmed)
+            {
+                StartMicrophone();
+            }
+            else
+            {
+                OnAudioRecorded?.Invoke(trimmed);
+            }
         }
 
         private void UpdateAmplitude()
         {
-            int clipPosition = Microphone.GetPosition(SelectedMicrophone);
+            int clipPosition = NeocortexMicrophone.GetPosition(SelectedMicrophone);
             int startPosition = Mathf.Max(0, clipPosition - AUDIO_SAMPLE_WINDOW);
             float[] audioSamples = new float[AUDIO_SAMPLE_WINDOW];
             audioClip.GetData(audioSamples, startPosition);
@@ -106,7 +104,7 @@ private void OnDestroy()
         {
             if (initialized)
             {
-                Microphone.End(SelectedMicrophone);
+                NeocortexMicrophone.End(SelectedMicrophone);
             }
         }
     }

Runtime/NeocortexMicrophone.cs

@@ -0,0 +1,34 @@
+using UnityEngine;
+
+namespace Neocortex
+{
+    public sealed class NeocortexMicrophone
+    {
+        public static AudioClip Start(string deviceName, bool loop, int lengthSec, int frequency)
+        {
+            #if !UNITY_WEBGL || UNITY_EDITOR
+            return Microphone.Start(deviceName, loop, lengthSec, frequency);
+            #endif
+
+            return null;
+        }
+
+        public static void End(string deviceName)
+        {
+            #if !UNITY_WEBGL || UNITY_EDITOR
+            Microphone.End(deviceName);
+            #endif
+
+            return;
+        }
+
+        public static int GetPosition(string deviceName)
+        {
+            #if !UNITY_WEBGL || UNITY_EDITOR
+            return Microphone.GetPosition(deviceName);
+            #endif
+
+            return 0;
+        }
+    }
+}