From c0791aebf06f9c4e255cdcd1f0bcfa9297997d96 Mon Sep 17 00:00:00 2001 From: UUoocl <99063397+UUoocl@users.noreply.github.com> Date: Sun, 4 Aug 2024 14:52:47 -0400 Subject: [PATCH 1/5] added shortcuts Input --- .gitignore | 1 + index.html | 16 ++++++++++++++++ lib/shortcutsScripts.js | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 .gitignore create mode 100644 lib/shortcutsScripts.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3e0efda --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +obs_browser_examples/.DS_Store diff --git a/index.html b/index.html index 229a326..d44d94a 100644 --- a/index.html +++ b/index.html @@ -251,6 +251,16 @@

Text-to-Text large language model

+ +
+
+

Apple Shortcuts Input

+
+

Results from Apple Shortcuts

+ +
+
+ @@ -263,6 +273,12 @@

Text-to-Text large language model

+ + \ No newline at end of file diff --git a/lib/shortcutsScripts.js b/lib/shortcutsScripts.js new file mode 100644 index 0000000..0954e61 --- /dev/null +++ b/lib/shortcutsScripts.js @@ -0,0 +1,16 @@ +const shortcutsInput = document.getElementById("shortcutsInput"); +shortcutsInput.addEventListener("click", shortcutsSend); + +function shortcutsSend(){ + console.log("input clicked") + console.log(shortcutsInput.value) + //send obs_textSourceName to OBS + obs.call("CallVendorRequest", { + vendorName: "obs-browser", + requestType: "emit_event", + requestData: { + event_name: "shortcuts-result", + event_data: { shortcutsResult }, + }, + }); +} \ No newline at end of file From 43d76bbc202e806b216334347e0d61491fbfb5fd Mon Sep 17 00:00:00 2001 From: UUoocl <99063397+UUoocl@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:24:28 -0400 Subject: [PATCH 2/5] mouse Input for mac --- index.html | 6 ++++++ lib/shortcutsScripts.js | 4 ++-- lib/webmidi-Module.js | 11 ++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/index.html b/index.html index d44d94a..a2cf200 100644 --- a/index.html +++ b/index.html @@ -46,6 +46,12 @@
  • Gemma
    LLM Inference
    • +
  • + Apple Shortcuts + +
    • diff --git a/lib/shortcutsScripts.js b/lib/shortcutsScripts.js index 0954e61..51ba477 100644 --- a/lib/shortcutsScripts.js +++ b/lib/shortcutsScripts.js @@ -4,12 +4,12 @@ shortcutsInput.addEventListener("click", shortcutsSend); function shortcutsSend(){ console.log("input clicked") console.log(shortcutsInput.value) - //send obs_textSourceName to OBS + let shortcutsResult = shortcutsInput.value; obs.call("CallVendorRequest", { vendorName: "obs-browser", requestType: "emit_event", requestData: { - event_name: "shortcuts-result", + event_name: "shortcuts-message", event_data: { shortcutsResult }, }, }); diff --git a/lib/webmidi-Module.js b/lib/webmidi-Module.js index 72901f2..bfd3030 100644 --- a/lib/webmidi-Module.js +++ b/lib/webmidi-Module.js @@ -10,8 +10,7 @@ function onEnabled() { document.body.innerHTML += "No device detected."; } else { WebMidi.inputs.forEach((device, index) => { - addDeviceListener(index); - + addDeviceListener(index); const midiElem = document.createElement('div'); midiElem.id = `midi-${index}`; midiElem.innerHTML = `${index}: "midi-${device.name}"
    `; @@ -20,6 +19,12 @@ function onEnabled() { midiDataElem.id = `midi-${index}-data`; document.getElementById(`midi-${index}`).appendChild(midiDataElem); }); + WebMidi.outputs.forEach(output => console.log(output.manufacturer, output.name)); + let output = WebMidi.outputs[0]; + let channel7 = output.channels[7]; + let channel1 = output.channels[1]; + channel7.playNote(32, {rawAttack: 5}); + channel1.playNote(100, {rawAttack: 1}); } // Listen to 'midi message' events on channels 1, 2 and 3 of the selected input MIDI device @@ -55,4 +60,4 @@ function onEnabled() { }); } } -} +} \ No newline at end of file From d757db133cc252225333bcbb45e709d836f8c986 Mon Sep 17 00:00:00 2001 From: UUoocl <99063397+UUoocl@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:28:41 -0400 Subject: [PATCH 3/5] commit --- index.html | 21 +++++++++++---------- lib/shortcutsScripts.js | 31 +++++++++++++++++++++++-------- lib/webmidi-Module.js | 14 +++++++------- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/index.html b/index.html index a2cf200..36c0e4c 100644 --- a/index.html +++ b/index.html @@ -49,7 +49,8 @@
  • Apple Shortcuts
    • @@ -261,9 +262,13 @@

    Text-to-Text large language model

    Apple Shortcuts Input

    -
    -

    Results from Apple Shortcuts

    - +
    +

    Mouse coordinates

    + +
    +
    +

    PTZ coordinates

    +
    @@ -280,11 +285,7 @@

    Apple Shortcuts Input

    - - \ No newline at end of file + + diff --git a/lib/shortcutsScripts.js b/lib/shortcutsScripts.js index 51ba477..1315eb8 100644 --- a/lib/shortcutsScripts.js +++ b/lib/shortcutsScripts.js @@ -1,16 +1,31 @@ -const shortcutsInput = document.getElementById("shortcutsInput"); -shortcutsInput.addEventListener("click", shortcutsSend); +const ptzInput = document.getElementById("ptzValues"); +const mouseInput = document.getElementById("mouseCoordinates"); +ptzInput.addEventListener("click", ptzSend); +mouseInput.addEventListener("click", mouseSend); -function shortcutsSend(){ - console.log("input clicked") - console.log(shortcutsInput.value) - let shortcutsResult = shortcutsInput.value; +function ptzSend() { + console.log("ptz input clicked"); + console.log(ptzInput.value); + let shortcutsResult = ptzInput.value; obs.call("CallVendorRequest", { vendorName: "obs-browser", requestType: "emit_event", requestData: { - event_name: "shortcuts-message", + event_name: "ptz-message", event_data: { shortcutsResult }, }, }); -} \ No newline at end of file +} +function mouseSend() { + console.log("mouse input clicked"); + console.log(mouseInput.value); + let shortcutsResult = mouseInput.value; + obs.call("CallVendorRequest", { + vendorName: "obs-browser", + requestType: "emit_event", + requestData: { + event_name: "mouse-message", + event_data: { shortcutsResult }, + }, + }); +} diff --git a/lib/webmidi-Module.js b/lib/webmidi-Module.js index bfd3030..0a87d1b 100644 --- a/lib/webmidi-Module.js +++ b/lib/webmidi-Module.js @@ -7,7 +7,7 @@ WebMidi.enable() function onEnabled() { // Display available MIDI input devices if (WebMidi.inputs.length < 1) { - document.body.innerHTML += "No device detected."; + //document.body.innerHTML += "No device detected."; } else { WebMidi.inputs.forEach((device, index) => { addDeviceListener(index); @@ -19,12 +19,12 @@ function onEnabled() { midiDataElem.id = `midi-${index}-data`; document.getElementById(`midi-${index}`).appendChild(midiDataElem); }); - WebMidi.outputs.forEach(output => console.log(output.manufacturer, output.name)); - let output = WebMidi.outputs[0]; - let channel7 = output.channels[7]; - let channel1 = output.channels[1]; - channel7.playNote(32, {rawAttack: 5}); - channel1.playNote(100, {rawAttack: 1}); + // WebMidi.outputs.forEach(output => console.log(output.manufacturer, output.name)); + // let output = WebMidi.outputs[0]; + // let channel7 = output.channels[7]; + // let channel1 = output.channels[1]; + // channel7.playNote(32, {rawAttack: 5}); + // channel1.playNote(100, {rawAttack: 1}); } // Listen to 'midi message' events on channels 1, 2 and 3 of the selected input MIDI device From 7c14a71b35ea940a3820b3fac3bcbf3d83cb90d6 Mon Sep 17 00:00:00 2001 From: UUoocl <99063397+UUoocl@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:01:48 -0400 Subject: [PATCH 4/5] typo --- lib/loadScripts.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/loadScripts.js b/lib/loadScripts.js index aaf173b..3cdc583 100644 --- a/lib/loadScripts.js +++ b/lib/loadScripts.js @@ -1,13 +1,13 @@ //Load MediaPipe libraries when needed -//Pose Dectection +//Pose Detection const mpPoseButton = document.getElementById("startPoseDetectButton"); mpPoseButton.addEventListener("click", () => loadScript('mpPose.js',true)); -//Hand landmark dectection +//Hand landmark detection const mpHandButton = document.getElementById("startHandLandmarks"); mpHandButton.addEventListener("click", () => loadScript('mpHand.js',true)); -//Face landmark dectection +//Face landmark detection const mpFaceButton = document.getElementById("startFaceLandmarks"); mpFaceButton.addEventListener("click", () => loadScript('mpFace.js',true)); From 7a29aacfa5585aaf59af1a99381a6b7e58e2cf6a Mon Sep 17 00:00:00 2001 From: UUoocl <99063397+UUoocl@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:31:05 -0400 Subject: [PATCH 5/5] readme sync --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index e0ce612..2b91e38 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,33 @@ - [Get User Media(GUM) for OBS](#get-user-mediagum-for-obs) - [Overview](#overview) - [Getting Started](#getting-started) + - [Enable the OBS WebSocket Server](#enable-the-obs-websocket-server) + - [Connect to the OBS WebSocket Server](#connect-to-the-obs-websocket-server) + - [Capture Audio or Desktop](#capture-audio-or-desktop) - [Audio Frequency Analysis](#audio-frequency-analysis) + - [Audio Frequency Message](#audio-frequency-message) - [MIDI Input](#midi-input) + - [MIDI Output Message](#midi-output-message) - [Gamepad Input](#gamepad-input) + - [Gamepad Output Message](#gamepad-output-message) - [MediaPipe Landmarks](#mediapipe-landmarks) - [MediaPipe Pose Landmarks](#mediapipe-pose-landmarks) + - [Output Message](#output-message) - [MediaPipe Hand Landmarks](#mediapipe-hand-landmarks) + - [Output Message](#output-message-1) - [MediaPipe Face Landmarks](#mediapipe-face-landmarks) + - [Output Message](#output-message-2) - [MediaPipe Image Segmentation](#mediapipe-image-segmentation) + - [Output Message](#output-message-3) - [MediaPipe Text Sentiment](#mediapipe-text-sentiment) + - [Text Sentiment Result Message](#text-sentiment-result-message) - [MediaPipe Gemma LLM Inference](#mediapipe-gemma-llm-inference) + - [Output Message](#output-message-4) - [WebRTC Audio Video](#webrtc-audio-video) + - [Output Message](#output-message-5) - [Speech Recognition](#speech-recognition) + - [Speech Recognition Started Event Message](#speech-recognition-started-event-message) + - [Speech Recognition Event Message](#speech-recognition-event-message) @@ -177,6 +192,32 @@ sequenceDiagram ``` ## MediaPipe Pose Landmarks + + +|0 - nose |10 - mouth (right)|20 - right index|30 - right heel| +|---|---|---|---| +|1 - left eye (inner) |11 - left shoulder|21 - left thumb|31 - left foot index| +|2 - left eye |12 - right shoulder|22 - right thumb|32 - right foot index| +|3 - left eye (outer)|13 - left elbow|23 - left hip| +|4 - right eye (inner) |14 - right elbow|24 - right hip| +|5 - right eye |15 - left wrist|25 - left knee| +|6 - right eye (outer) |16 - right wrist|26 - right knee| +|7 - left ear |17 - left pinky|27 - left ankle| +|8 - right ear |18 - right pinky|28 - right ankle| +|9 - mouth (left) |19 - left index|29 - left heel| + + + + + + + + + + + + + ### Output Message **Message name:**