part_2_dev_api/chapter_08/index.html

<!DOCTYPE html>
<!--
 Copyright 2024 Google LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-->

<html>
<head>
  <title>Gemini Multimodal Live WebSocket Demo - Developer API</title>
  <link rel="stylesheet" href="style.css">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, viewport-fit=cover">
  <meta name="theme-color" content="#000000">
  <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" />
</head>
<body>
  <div class="controls">
    <button id="micButton" onclick="toggleMicrophone()" disabled>
      <span class="material-symbols-outlined">play_arrow</span>
    </button>
    <button id="muteButton" onclick="toggleMute()" class="action-button" disabled>
      <span class="material-symbols-outlined">mic</span>
    </button>
    <button id="webcamButton" onclick="toggleWebcam()" class="action-button">
      <span class="material-symbols-outlined">videocam</span>
    </button>
    <button id="switchCameraButton" onclick="switchCamera()" class="action-button hidden">
      <span class="material-symbols-outlined">flip_camera_ios</span>
    </button>
    <button id="screenButton" onclick="toggleScreen()" class="action-button hidden">
      <span class="material-symbols-outlined">present_to_all</span>
    </button>
  </div>
  <div id="functionInfo" class="function-info">
    Waiting for function calls (available functions: get_weather, get_stock_price)
  </div>
  <div class="video-container">
    <video id="videoPreview" autoplay playsinline class="hidden"></video>
  </div>

  <!-- Load EventEmitter3 first -->
  <script src="https://cdn.jsdelivr.net/npm/eventemitter3@5.0.1/dist/eventemitter3.umd.min.js"></script>

  <!-- Load our API class -->
  <script src="shared/gemini-live-api.js"></script>

  <!-- Then load our module code -->
  <script type="module">
    import { AudioRecorder } from './shared/audio-recorder.js';
    import { AudioStreamer } from './shared/audio-streamer.js';
    import { MediaHandler } from './shared/media-handler.js';
    import { getWeather } from './shared/tools/weather-api.js';
    import { getStockPrice } from './shared/tools/stock-api.js';
    import { statusHandler } from './status-handler.js';

    console.log('Script started, setting up WebSocket...');
    const apiKey = '<YOUR_API_KEY>';
    const host = 'generativelanguage.googleapis.com';
    const endpoint = `wss://${host}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key=${apiKey}`;

    let audioContext;
    let audioStreamer;
    let audioRecorder;
    let isRecording = false;
    let initialized = false;
    let isInterrupted = false;
    let isMuted = false;
    let mediaHandler;
    let systemInstructions = '';  // Store system instructions for reconnection
    let geminiAPI;  // Will be initialized after loading system instructions

    document.addEventListener('DOMContentLoaded', () => {
      console.log('DOM loaded, initializing media handler');
      mediaHandler = new MediaHandler();
      const videoElement = document.getElementById('videoPreview');
      if (!videoElement) {
        console.error('Video element not found!');
        return;
      }
      mediaHandler.initialize(videoElement);

      // Load system instructions and send custom setup message
      fetch('system-instructions.txt')
        .then(response => response.text())
        .then(instructions => {
          systemInstructions = instructions;  // Store for later use
          // Create initial API instance
          geminiAPI = new GeminiLiveAPI(endpoint, false);
          setupGeminiHandlers();
          sendCustomSetup(instructions);
        });
    });

    function sendCustomSetup(systemInstructions) {
      const setupMessage = {
        setup: {
          model: "models/gemini-2.0-flash-exp",
          system_instruction: {
            role: "user",
            parts: [{
              text: systemInstructions
            }]
          },
          tools: [{
            functionDeclarations: [{
              name: "get_weather",
              description: "Get current weather information for a city",
              parameters: {
                type: "OBJECT",
                properties: {
                  city: {
                    type: "STRING",
                    description: "The name of the city to get weather for"
                  }
                },
                required: ["city"]
              }
            },
            {
              name: "get_stock_price",
              description: "Get current stock price and related information for a given stock symbol",
              parameters: {
                type: "OBJECT",
                properties: {
                  symbol: {
                    type: "STRING",
                    description: "The stock symbol to get price information for (e.g., AAPL, GOOGL)"
                  }
                },
                required: ["symbol"]
              }
            }]
          },
          {
            codeExecution: {}
          },
          {
            googleSearch: {}
          }],
          generation_config: {
            response_modalities: ["audio"],
            speech_config: {
              voice_config: {
                prebuilt_voice_config: {
                  voice_name: "Puck"
                }
              }
            }
          }
        }
      };
      console.log('Sending setup message:', setupMessage);
      geminiAPI.sendSetupMessage(setupMessage);
    }

    async function ensureAudioInitialized() {
      if (!initialized) {
        try {
          console.log('Initializing audio context...');
          audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 24000 });
          console.log('Audio context state:', audioContext.state);
          
          if (audioContext.state === 'suspended') {
            console.log('Resuming suspended audio context...');
            await audioContext.resume();
            console.log('Audio context resumed, new state:', audioContext.state);
          }
          
          console.log('Creating audio streamer...');
          audioStreamer = new AudioStreamer(audioContext);
          initialized = true;
          console.log('Audio initialization complete');
          return true;
        } catch (error) {
          console.error('Audio initialization failed:', error);
          return false;
        }
      }
      return true;
    }

    async function startRecording() {
      try {
        console.log('Starting recording...');
        if (!await ensureAudioInitialized()) return;
        
        // Reset state when starting new recording
        isInterrupted = false;
        if (audioStreamer) {
          console.log('Stopping previous audio streamer');
          audioStreamer.stop();
        }

        // Reinitialize Gemini API if WebSocket is closed
        if (!geminiAPI || geminiAPI.ws.readyState !== WebSocket.OPEN) {
          console.log('WebSocket not open, reinitializing...');
          geminiAPI = new GeminiLiveAPI(endpoint, false);
          // Reattach event handlers
          setupGeminiHandlers();
          // Send custom setup and wait for completion
          await new Promise((resolve) => {
            geminiAPI.onSetupComplete = () => {
              resolve();
              document.getElementById('micButton').disabled = false;
              document.getElementById('muteButton').disabled = false;
              // Only show screen share button on non-mobile devices
              if (!(/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent))) {
                document.getElementById('screenButton').classList.remove('hidden');
              }
            };
            sendCustomSetup(systemInstructions);
          });
        }

        console.log('Creating new audio recorder');
        audioRecorder = new AudioRecorder();
        await audioRecorder.start();

        audioRecorder.on('data', (base64Data) => {
          console.log('Got audio data, length:', base64Data.length);
          geminiAPI.sendAudioChunk(base64Data);
        });

        isRecording = true;
        document.getElementById('micButton').innerHTML = '<span class="material-symbols-outlined">stop</span>';
        console.log('Recording started successfully');
      } catch (error) {
        console.error('Error starting recording:', error);
      }
    }

    function stopRecording() {
      if (audioRecorder) {
        console.log('Stopping recording...');
        audioRecorder.stop();
        audioRecorder.off('data');
        isRecording = false;
        isMuted = false;
        document.getElementById('micButton').innerHTML = '<span class="material-symbols-outlined">play_arrow</span>';
        document.getElementById('muteButton').disabled = true;
        document.getElementById('muteButton').innerHTML = '<span class="material-symbols-outlined">mic</span>';
        
        // Stop video streams when stopping recording
        mediaHandler.stopAll();
        document.getElementById('webcamButton').innerHTML = '<span class="material-symbols-outlined">videocam</span>';
        document.getElementById('switchCameraButton').classList.add('hidden');
        document.getElementById('screenButton').innerHTML = '<span class="material-symbols-outlined">present_to_all</span>';
        
        // Send end message before closing
        geminiAPI.sendEndMessage();
        console.log('Recording stopped');
      }
    }

    window.toggleMicrophone = function() {
      console.log('Toggle microphone, current state:', isRecording);
      if (isRecording) {
        stopRecording();
      } else {
        startRecording();
      }
    };

    window.toggleWebcam = async function() {
      console.log('Toggle webcam, current state:', mediaHandler.isWebcamActive);
      if (mediaHandler.isWebcamActive) {
        mediaHandler.stopAll();
        document.getElementById('webcamButton').innerHTML = '<span class="material-symbols-outlined">videocam</span>';
        document.getElementById('switchCameraButton').classList.add('hidden');
      } else {
        const success = await mediaHandler.startWebcam();
        if (success) {
          document.getElementById('webcamButton').innerHTML = '<span class="material-symbols-outlined">videocam_off</span>';
          // Only show camera switch button on mobile devices
          if (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
            document.getElementById('switchCameraButton').classList.remove('hidden');
          }
          mediaHandler.startFrameCapture((base64Image) => {
            if (geminiAPI.ws.readyState === WebSocket.OPEN) {
              const message = {
                realtimeInput: {
                  mediaChunks: [{
                    mime_type: "image/jpeg",
                    data: base64Image
                  }]
                }
              };
              geminiAPI.ws.send(JSON.stringify(message));
            }
          });
        }
      }
    };

    window.toggleScreen = async function() {
      console.log('Toggle screen, current state:', mediaHandler.isScreenActive);
      if (mediaHandler.isScreenActive) {
        mediaHandler.stopAll();
        document.getElementById('screenButton').innerHTML = '<span class="material-symbols-outlined">present_to_all</span>';
      } else {
        const success = await mediaHandler.startScreenShare();
        if (success) {
          document.getElementById('screenButton').innerHTML = '<span class="material-symbols-outlined">cancel_presentation</span>';
          mediaHandler.startFrameCapture((base64Image) => {
            if (geminiAPI.ws.readyState === WebSocket.OPEN) {
              const message = {
                realtimeInput: {
                  mediaChunks: [{
                    mime_type: "image/jpeg",
                    data: base64Image
                  }]
                }
              };
              geminiAPI.ws.send(JSON.stringify(message));
            }
          });
        }
      }
    };

    window.switchCamera = async function() {
      await mediaHandler.switchCamera();
    };

    window.toggleMute = function() {
      if (!audioRecorder) return;
      
      isMuted = !isMuted;
      const muteButton = document.getElementById('muteButton');
      
      if (isMuted) {
        muteButton.innerHTML = '<span class="material-symbols-outlined">mic_off</span>';
        audioRecorder.mute();
      } else {
        muteButton.innerHTML = '<span class="material-symbols-outlined">mic</span>';
        audioRecorder.unmute();
      }
    };

    // Function to set up Gemini API event handlers
    function setupGeminiHandlers() {
      console.log('Setting up Gemini API handlers');
      
      geminiAPI.onSetupComplete = () => {
        console.log('Setup complete, enabling mic button');
        document.getElementById('micButton').disabled = false;
        document.getElementById('muteButton').disabled = false;
        // Only show screen share button on non-mobile devices
        if (!(/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent))) {
          document.getElementById('screenButton').classList.remove('hidden');
        }
      };

      geminiAPI.onAudioData = async (audioData) => {
        console.log('Received audio data');
        if (!audioStreamer?.isPlaying) {
          console.log('Gemini speaking...');
        }
        await playAudioChunk(audioData);
      };

      geminiAPI.onToolCall = async (toolCall) => {
        console.log('Received tool call:', toolCall);
        const functionCalls = toolCall.functionCalls;
        const functionResponses = [];

        for (const call of functionCalls) {
          if (call.name === 'get_weather') {
            console.log('Executing weather function call for:', call.args.city);
            // Update status before making the API call
            statusHandler.update('get_weather', { 
              city: call.args.city, 
              status: 'requesting' 
            });

            const weather = await getWeather(call.args.city);
            console.log('Weather response:', weather);
            
            // Update status with the weather results
            statusHandler.update('get_weather', { 
              city: call.args.city, 
              status: 'received',
              weather: weather 
            });

            functionResponses.push({
              id: call.id,
              name: call.name,
              response: {
                result: {
                  object_value: weather
                }
              }
            });
          } else if (call.name === 'get_stock_price') {
            console.log('Executing stock price function call for:', call.args.symbol);
            // Update status before making the API call
            statusHandler.update('get_stock_price', { 
              symbol: call.args.symbol, 
              status: 'requesting' 
            });

            const stockData = await getStockPrice(call.args.symbol);
            console.log('Stock response:', stockData);
            
            // Update status with the stock results
            statusHandler.update('get_stock_price', { 
              symbol: call.args.symbol, 
              status: 'received',
              stock: stockData 
            });

            functionResponses.push({
              id: call.id,
              name: call.name,
              response: {
                result: {
                  object_value: stockData
                }
              }
            });
          }
        }

        if (functionResponses.length > 0) {
          geminiAPI.sendToolResponse(functionResponses);
        }
      };

      geminiAPI.onInterrupted = () => {
        console.log('Gemini interrupted');
        isInterrupted = true;
        audioStreamer?.stop();
      };

      geminiAPI.onTurnComplete = () => {
        console.log('Gemini finished speaking');
        isInterrupted = false;
        audioStreamer?.complete();
      };

      geminiAPI.onError = (error) => {
        console.error('Error:', error);
      };

      geminiAPI.onClose = (event) => {
        console.log('Connection closed:', event.code, event.reason);
      };
    }

    async function playAudioChunk(base64AudioChunk) {
      try {
        await ensureAudioInitialized();
        const arrayBuffer = base64ToArrayBuffer(base64AudioChunk);
        const uint8Array = new Uint8Array(arrayBuffer);
        audioStreamer.addPCM16(uint8Array);
        audioStreamer.resume();
      } catch (error) {
        console.error('Error queuing audio chunk:', error);
      }
    }

    function base64ToArrayBuffer(base64) {
      const binaryString = atob(base64);
      const bytes = new Uint8Array(binaryString.length);
      for (let i = 0; i < binaryString.length; i++) {
        bytes[i] = binaryString.charCodeAt(i);
      }
      return bytes.buffer;
    }
  </script>
</body>
</html>