feat: implement API changes and enhance model interaction

- Updated all API endpoints to have '/api/' prefix for better organization. - Added new libraries to the project configuration for additional features. - Changed main.py functions to align with API updates and remove redundant code. - Updated html language for better SEO and removed unnecessary script tags in index.html. - Enhanced the model interaction in index.js by implementing a blinking function and integrating it with the model loading. - Updated the Chatbot model used in Config.py. - Improved the text descriptions in the user_input part of the Chatbot.py.
IRedDragonICY · May 15, 2024 · 0d9507a · 0d9507a
1 parent 2c61c26
commit 0d9507a
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 24 deletions.
diff --git a/.idea/jsLibraryMappings.xml b/.idea/jsLibraryMappings.xml
diff --git a/.idea/vixevia.iml b/.idea/vixevia.iml
diff --git a/Chatbot.py b/Chatbot.py
@@ -106,14 +106,14 @@ def process_audio(self, audio_bytes):
         user_input = {
             "role": "user",
             "parts": [
-                {"text": "Penglihatan:"},
+                {"text": "Penglihatan Vixevia:"},
                 *[
                     {"mime_type": "image/jpeg", "data": cv2.imencode('.jpg', frame)[1].tobytes()}
                     for frame in self.frames
                 ],
-                {"text": "Pendengaran:"},
+                {"text": "Pendengaran Vixevia:"},
                 {"mime_type": "audio/wav", "data": audio_bytes},
             ],
         }
         self._handle_response(user_input)
-        self.frames = []
+        self.frames = []
diff --git a/Config.py b/Config.py
@@ -1,5 +1,5 @@
 class Config:
-    MODEL_NAME = "gemini-1.5-pro-latest"
+    MODEL_NAME = "gemini-1.5-flash-latest"
     HARM_CATEGORIES = ["HARM_CATEGORY_HARASSMENT", "HARM_CATEGORY_HATE_SPEECH", "HARM_CATEGORY_SEXUALLY_EXPLICIT",
                        "HARM_CATEGORY_DANGEROUS_CONTENT"]
     BLOCK_NONE = "BLOCK_NONE"

diff --git a/main.py b/main.py
@@ -25,24 +25,24 @@
 
 
 @app.get("/")
-async def read_items():
+async def index():
     with open('static/index.html', 'r') as f:
         html_content = f.read()
     return HTMLResponse(content=html_content, status_code=200)
 
 
-@app.get("/audio_status")
+@app.get("/api/audio_status")
 async def get_audio_status():
     return {"audio_ready": chatbot.audio_ready}
 
 
-@app.get("/reset_audio_status")
+@app.post("/api/reset_audio_status")
 async def reset_audio_status():
     chatbot.audio_ready = False
     return {"audio_ready": chatbot.audio_ready}
 
 
-@app.post("/upload_frame")
+@app.post("/api/upload_frame")
 async def upload_frame(image: UploadFile = File(...)):
     try:
         image_bytes = await image.read()
@@ -52,7 +52,7 @@ async def upload_frame(image: UploadFile = File(...)):
         print(f"Error processing frame: {e}")
 
 
-@app.post("/upload_audio")
+@app.post("/api/upload_audio")
 async def upload_audio(audio: UploadFile = File(...)):
     try:
         audio_bytes = await audio.read()
@@ -66,10 +66,6 @@ def run_server():
     uvicorn.run(app, host="localhost", port=8000)
 
 
-def run_server():
-    import uvicorn
-    uvicorn.run(app, host="localhost", port=8000)
-
 if __name__ == "__main__":
     threading.Thread(target=run_server).start()
     public_url = ngrok.connect(8000)
@@ -80,4 +76,4 @@ def run_server():
         ngrok_process.proc.wait()
     except KeyboardInterrupt:
         print(" Shutting down server.")
-        ngrok.kill()
+        ngrok.kill()
diff --git a/static/index.html b/static/index.html
@@ -1,14 +1,12 @@
 <!DOCTYPE html>
-<html>
+<html lang="id">
 <head>
     <title>Vixevia</title>
 
     <!-- Load Cubism and PixiJS -->
     <script src="https://cubism.live2d.com/sdk-web/cubismcore/live2dcubismcore.min.js"></script>
     <script src="https://cdn.jsdelivr.net/gh/dylanNew/live2d/webgl/Live2D/lib/live2d.min.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/pixi.js@6.5.2/dist/browser/pixi.min.js"></script>
-
-    <!-- if only Cubism 4 -->
     <script src="https://cdn.jsdelivr.net/gh/RaSan147/pixi-live2d-display@v0.4.0-ls-2/dist/cubism4.min.js"></script>
 </head>
 <body>

diff --git a/static/js/index.js b/static/js/index.js
@@ -2,9 +2,12 @@ const app = new PIXI.Application({ view: document.getElementById("canvas"), auto
 const audio_link = "/temp/response.wav";
 const statusDiv = document.getElementById('status');
 
-let model, video, audioPlaying = false, volume = 1;
+let model, video, audioPlaying = false, volume = 1, blinkInterval;
 
-loadModel().then(result => model = result);
+loadModel().then(result => {
+    model = result;
+    startBlinking();
+});
 setupVideo();
 
 async function loadModel() {
@@ -35,15 +38,15 @@ function captureFrame() {
 function sendFrameToServer(blob) {
     let formData = new FormData();
     formData.append('image', blob, 'frame.jpg');
-    fetch('/upload_frame', { method: 'POST', body: formData }).then(console.log);
+    fetch('/api/upload_frame', { method: 'POST', body: formData }).then(console.log);
 }
 
 async function initiateAudioPlay() {
     if (!audioPlaying) await playAudioWhenReady();
 }
 
 async function checkAudioStatus() {
-    return (await fetch('/audio_status').then(res => res.json())).audio_ready;
+    return (await fetch('/api/audio_status').then(res => res.json())).audio_ready;
 }
 
 async function playAudioWhenReady() {
@@ -69,7 +72,7 @@ function setupAnalyser(audio) {
     analyser.connect(audioCtx.destination);
     analyseVolume(analyser, dataArray, audio);
     audio.onended = async function() {
-        await fetch('/reset_audio_status');
+        await fetch('/api/reset_audio_status', { method: 'POST' });
         audioPlaying = false;
         initiateAudioPlay();
         recognition.start();
@@ -110,9 +113,38 @@ navigator.mediaDevices.getUserMedia({ audio: true })
 function sendAudioToServer(blob) {
     let formData = new FormData();
     formData.append('audio', blob, 'audio.wav');
-    fetch('/upload_audio', { method: 'POST', body: formData })
+    fetch('/api/upload_audio', { method: 'POST', body: formData })
         .then(response => { console.log(response); statusDiv.innerHTML = ""; })
         .catch(error => console.error('Error uploading audio:', error));
 }
 
 initiateAudioPlay();
+
+function blink() {
+    let blinkValue = 1;
+    let blinkSpeed = 0.1;
+    const blinkInterval = setInterval(() => {
+        blinkValue -= blinkSpeed;
+        if (blinkValue <= 0) {
+            blinkValue = 0;
+            clearInterval(blinkInterval);
+            setTimeout(() => {
+                const closeEyeInterval = setInterval(() => {
+                    blinkValue += blinkSpeed;
+                    if (blinkValue >= 1) {
+                        blinkValue = 1;
+                        clearInterval(closeEyeInterval);
+                    }
+                    model.internalModel.coreModel.setParameterValueById('ParamEyeLOpen', blinkValue);
+                    model.internalModel.coreModel.setParameterValueById('ParamEyeROpen', blinkValue);
+                }, 10);
+            }, 100);
+        }
+        model.internalModel.coreModel.setParameterValueById('ParamEyeLOpen', blinkValue);
+        model.internalModel.coreModel.setParameterValueById('ParamEyeROpen', blinkValue);
+    }, 10);
+}
+
+function startBlinking() {
+    setInterval(blink, Math.random() * 4000 + 2000);
+}