telegram bot can now undertand voice messages

n4ze3m · Dec 2, 2023 · 6f2de62 · 6f2de62
1 parent 8192d73
commit 6f2de62
Show file tree

Hide file tree

Showing 10 changed files with 115 additions and 37 deletions.
diff --git a/app/ui/package.json b/app/ui/package.json
@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "1.4.4",
+  "version": "1.4.5",
   "type": "module",
   "scripts": {
     "dev": "vite",

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

diff --git a/server/package.json b/server/package.json
@@ -20,7 +20,7 @@
     "dev:start": "fastify start --ignore-watch=uploads,.ts$ -w  -P dist/app.js"
   },
   "keywords": [],
-  "author": "", 
+  "author": "",
   "license": "MIT",
   "dependencies": {
     "@fastify/autoload": "^5.0.0",
@@ -34,6 +34,7 @@
     "@ffmpeg.wasm/core-mt": "^0.12.0",
     "@ffmpeg.wasm/main": "^0.12.0",
     "@google-ai/generativelanguage": "^0.2.0",
+    "@grammyjs/files": "^1.0.4",
     "@huggingface/inference": "1",
     "@prisma/client": "5",
     "@slack/bolt": "^3.13.2",
@@ -70,9 +71,9 @@
     "pdfjs-dist": "^3.7.107",
     "pubsub-js": "^1.9.4",
     "sitemapper": "^3.2.6",
+    "ts-node": "^10.9.1",
     "unique-names-generator": "^4.7.1",
     "wavefile": "^11.0.0",
-    "ts-node": "^10.9.1",
     "ytdl-core": "^4.11.5"
   },
   "devDependencies": {
@@ -88,4 +89,4 @@
   "prisma": {
     "seed": "ts-node prisma/seed.ts"
   }
-}
+}
diff --git a/server/src/integration/handlers/telegram.handler.ts b/server/src/integration/handlers/telegram.handler.ts
@@ -38,7 +38,7 @@ export const telegramBotHandler = async (
       human: message.human,
       ai: message.bot,
     }));
-    
+
     if (history.length > 20) {
       history.splice(0, history.length - 20);
     }

diff --git a/server/src/integration/handlers/utils/audio-to-text.ts b/server/src/integration/handlers/utils/audio-to-text.ts
@@ -0,0 +1,36 @@
+import {
+  piplelineTransformer,
+  //@ts-ignore
+} from "../../../utils/pipleline.js";
+import { WaveFile } from "wavefile";
+
+function _convert(audio: Buffer) {
+  const wav = new WaveFile(audio);
+  wav.toBitDepth("32f");
+  wav.toSampleRate(16000);
+  let audioData = wav.getSamples();
+  if (Array.isArray(audioData)) {
+    console.log(
+      "Multiple channels detected. Selecting the first channel only."
+    );
+    audioData = audioData[0];
+  }
+  return audioData;
+}
+
+export const convertTextToAudio = async (audioData: Buffer) => {
+  const pipeline = await piplelineTransformer();
+  const transcriber = await pipeline(
+    "automatic-speech-recognition",
+    process.env.WHISPER_MODEL || "distil-whisper/distil-medium.en"
+  );
+
+  const audio = _convert(Buffer.from(audioData));
+  let output = (await transcriber(audio, {
+    chunk_length_s: 30,
+  })) as {
+    text: string;
+  };
+
+  return output;
+};
diff --git a/server/src/integration/telegram.ts b/server/src/integration/telegram.ts
@@ -1,17 +1,21 @@
-import { Bot } from "grammy";
+import { Bot, Context } from "grammy";
 import axios from "axios";
 import {
   deleteTelegramChatHistory,
   telegramBotHandler,
   welcomeMessage,
 } from "./handlers/telegram.handler";
-
+import { convertTextToAudio } from "./handlers/utils/audio-to-text";
+import { FileFlavor, hydrateFiles } from "@grammyjs/files";
+import * as fs from "fs/promises";
+import { convertOggToWave } from "../utils/ffmpeg";
+type DialoqBaseContext = FileFlavor<Context>;
 export default class TelegramBot {
   static get clients() {
     return this._clients.values();
   }
 
-  private static _clients: Map<string, Bot> = new Map();
+  private static _clients: Map<string, any> = new Map();
 
   static totolClients() {
     return this._clients.size;
@@ -23,7 +27,9 @@ export default class TelegramBot {
         await this.disconnect(identifier);
       }
 
-      const bot = new Bot(token);
+      const bot = new Bot<DialoqBaseContext>(token);
+      bot.api.config.use(hydrateFiles(bot.token));
+
       await bot.api.setMyCommands([
         { command: "start", description: "Start the bot" },
         { command: "ping", description: "Ping the bot" },
@@ -70,6 +76,37 @@ export default class TelegramBot {
         return await ctx.reply(message);
       });
 
+      bot.on("message:voice", async (ctx) => {
+        try {
+          if (ctx.chat.type !== "private") {
+            return ctx.reply("I can only work in private chats.");
+          }
+
+          await ctx.replyWithChatAction("typing");
+
+          const file = await ctx.getFile();
+          const path = await file.download();
+
+          const audioWav = await convertOggToWave(path);
+          const audio = await fs.readFile(audioWav);
+
+          const response = await convertTextToAudio(audio);
+
+          const user_id = ctx.from.id;
+
+          const message = await telegramBotHandler(
+            identifier,
+            response.text,
+            user_id
+          );
+
+          return await ctx.reply(message);
+        } catch (error) {
+          console.log(error);
+          return await ctx.reply("Opps! Something went wrong");
+        }
+      });
+
       bot.start();
       bot.catch((err) => console.log(`${identifier} error: ${err}`));
       this._clients.set(identifier, bot);

diff --git a/server/src/utils/ffmpeg.ts b/server/src/utils/ffmpeg.ts
@@ -22,7 +22,7 @@ export const convertMp4ToWave = async (file_path: string) => {
     "1",
     "-ar",
     "16000",
-    "./audio.wav",
+    "./audio.wav"
   );
   const data = ffmpeg.FS("readFile", "./audio.wav");
   const new_file_path = file_path.replace(".mp4", ".wav");
@@ -44,7 +44,7 @@ export const convertMp3ToWave = async (file_path: string) => {
     "1",
     "-ar",
     "16000",
-    "./audio.wav",
+    "./audio.wav"
   );
   const data = ffmpeg.FS("readFile", "./audio.wav");
   const new_file_path = file_path.replace(".mp3", ".wav");
@@ -53,6 +53,28 @@ export const convertMp3ToWave = async (file_path: string) => {
   return new_file_path;
 };
 
+export const convertOggToWave = async (file_path: string) => {
+  await _init();
+  const audio = await fs.readFile(file_path);
+  ffmpeg.FS("writeFile", "./audio.ogg", audio);
+  await ffmpeg.run(
+    "-i",
+    "./audio.ogg",
+    "-acodec",
+    "pcm_s16le",
+    "-ac",
+    "1",
+    "-ar",
+    "16000",
+    "./audio.wav"
+  );
+
+  const data = ffmpeg.FS("readFile", "./audio.wav");
+  const new_file_path = file_path.replace(".ogg", ".wav");
+  await fs.writeFile(new_file_path, data);
+  ffmpeg.exit();
+  return new_file_path;
+};
 
 export const convertMp3ToWaveFromBuffer = async (audio: Buffer) => {
   await _init();
@@ -66,9 +88,9 @@ export const convertMp3ToWaveFromBuffer = async (audio: Buffer) => {
     "1",
     "-ar",
     "16000",
-    "./audio.wav",
+    "./audio.wav"
   );
   const data = ffmpeg.FS("readFile", "./audio.wav");
   ffmpeg.exit();
   return data;
-};
+};
diff --git a/server/test/plugins/support.test.ts b/server/test/plugins/support.test.ts
diff --git a/server/test/routes/example.test.ts b/server/test/routes/example.test.ts
diff --git a/server/yarn.lock b/server/yarn.lock
@@ -497,6 +497,11 @@
   dependencies:
     google-gax "^3.5.8"
 
+"@grammyjs/files@^1.0.4":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@grammyjs/files/-/files-1.0.4.tgz#255d8e92bde0f43768840b80a230394f0a97dba6"
+  integrity sha512-GT7JWRH9cs5kIrBtUGwMwVEhb1w7Vtnc93UT3zM/HYigmeVd93wWaIqyuSYsZq1VIB+9CSk0oz+ZsnxK83z7tg==
+
 "@grammyjs/types@3.1.1":
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/@grammyjs/types/-/types-3.1.1.tgz#dd80c018ffaf44bd89c56160edb3d9e86b84251f"