From e134f30a56f3a97e9b309e5b9a60a0e6d76c5125 Mon Sep 17 00:00:00 2001 From: wraps Date: Mon, 1 Jan 2024 03:20:54 +0100 Subject: [PATCH 1/8] feat(embeddings): add Ollama Embeddings --- app/ui/src/utils/embeddings.ts | 4 ++++ server/.env.example | 4 +++- server/src/routes/api/v1/bot/handlers/get.handler.ts | 8 ++++++++ server/src/utils/embeddings.ts | 5 +++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/ui/src/utils/embeddings.ts b/app/ui/src/utils/embeddings.ts index b09b2f5a..faf6e97a 100644 --- a/app/ui/src/utils/embeddings.ts +++ b/app/ui/src/utils/embeddings.ts @@ -6,6 +6,10 @@ export const availableEmbeddingTypes = [ value: "transformer", label: "Xenova/all-MiniLM-L6-v2 (cpu)", }, + { + value: "ollama", + label: "Ollama Embeddings", + }, { value: "supabase", label: "Supabase/gte-small (cpu)", diff --git a/server/.env.example b/server/.env.example index dd7c7571..06a594e6 100644 --- a/server/.env.example +++ b/server/.env.example @@ -14,4 +14,6 @@ ELEVENLABS_API_KEY="" DB_QUEUE_CONCURRENCY=2 DB_SESSION_SECRET="a8F2h6T9j4Kl0Pz8W7eX3rB5y1VcQ6mN" DB_SESSION_SECURE="false" -JINA_API_KEY="" \ No newline at end of file +JINA_API_KEY="" +OLLAMA_EMBEDDING_API_URL="" +OLLAMA_EMBEDDING_API_MODEL="" \ No newline at end of file diff --git a/server/src/routes/api/v1/bot/handlers/get.handler.ts b/server/src/routes/api/v1/bot/handlers/get.handler.ts index 5731a1f0..aa0a62dd 100644 --- a/server/src/routes/api/v1/bot/handlers/get.handler.ts +++ b/server/src/routes/api/v1/bot/handlers/get.handler.ts @@ -146,6 +146,10 @@ export const getCreateBotConfigHandler = async ( value: "transformer", label: "all-MiniLM-L6-v2 (cpu)", }, + { + value: "ollama", + label: "Ollama Embeddings", + }, { value: "google-gecko", label: "Google text-gecko-001", @@ -206,6 +210,10 @@ export const getBotByIdSettingsHandler = async ( value: "transformer", label: "all-MiniLM-L6-v2 (cpu)", }, + { + value: "ollama", + label: "Ollama Embeddings", + }, { value: "google-gecko", label: "Google text-gecko-001", diff --git a/server/src/utils/embeddings.ts b/server/src/utils/embeddings.ts index e2094669..29a67b1e 100644 --- a/server/src/utils/embeddings.ts +++ b/server/src/utils/embeddings.ts @@ -4,6 +4,8 @@ import { HuggingFaceInferenceEmbeddings } from "langchain/embeddings/hf"; import { TransformersEmbeddings } from "../embeddings/transformer-embedding"; import { GooglePaLMEmbeddings } from "langchain/embeddings/googlepalm"; import { GoogleGenerativeAIEmbeddings } from "@langchain/google-genai"; +import { OllamaEmbeddings } from "langchain/embeddings/ollama"; + export const embeddings = (embeddingsType: string) => { switch (embeddingsType) { @@ -17,6 +19,8 @@ export const embeddings = (embeddingsType: string) => { return new TransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }); + case "ollama": + return new OllamaEmbeddings({ baseUrl: process.env.OLLAMA_EMBEDDING_API_URL, model: process.env.OLLAMA_EMBEDDING_API_MODEL }); case "jina": return new TransformersEmbeddings({ model: "Xenova/jina-embeddings-v2-small-en", @@ -52,6 +56,7 @@ export const supportedEmbeddings = [ "cohere", "huggingface-api", "transformer", + "ollama", "google-gecko", "supabase", "jina", From 28e8d1cb307ba3a851c2bbd6fac534cda2dc40f7 Mon Sep 17 00:00:00 2001 From: wraps Date: Mon, 1 Jan 2024 12:08:16 +0100 Subject: [PATCH 2/8] Rename env variable --- server/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/.env.example b/server/.env.example index 06a594e6..f03f51fc 100644 --- a/server/.env.example +++ b/server/.env.example @@ -16,4 +16,4 @@ DB_SESSION_SECRET="a8F2h6T9j4Kl0Pz8W7eX3rB5y1VcQ6mN" DB_SESSION_SECURE="false" JINA_API_KEY="" OLLAMA_EMBEDDING_API_URL="" -OLLAMA_EMBEDDING_API_MODEL="" \ No newline at end of file +OLLAMA_EMBEDDING_MODEL="" \ No newline at end of file From be57adf733a18d05da7a37dcdd5ee4df2d96060b Mon Sep 17 00:00:00 2001 From: wraps Date: Mon, 1 Jan 2024 12:13:38 +0100 Subject: [PATCH 3/8] Update readme and add check if env variable is empty --- README.md | 77 +++++++++++++++++------------------- server/src/utils/validate.ts | 10 ++++- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 2625473b..854eb1c6 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ Create chatbots with ease

-
- +
+ [![Join dialoqbase #welcome](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/SPE3npH7Wu) [![Build Status](https://github.com/n4ze3m/dialoqbase/actions/workflows/build.yml/badge.svg)](https://github.com/n4ze3m/dialoqbase/actions/workflows/build.yml) [![License: MIT](https://img.shields.io/github/license/n4ze3m/dialoqbase)](https://github.com/n4ze3m/dialoqbase/blob/master/LICENSE) +
Dialoqbase is an open-source application designed to facilitate the creation of custom chatbots using a personalized knowledge base. The application leverages advanced language models to generate accurate and context-aware responses. Additionally, it utilizes PostgreSQL, a robust relational database management system, for efficient vector search operations and for storing the knowledge base. @@ -24,7 +25,7 @@ Want to check more demo videos? Follow me on [Twitter](https://twitter.com/n4ze3 ## Quick Deployments πŸš€ -### Railway with Supabase +### Railway with Supabase [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/TXdjD7?referralCode=olbszX) @@ -75,7 +76,6 @@ docker compose up -d ``` 4. Open your browser and go to `http://localhost:3000`. - 5. Log in using the default credentials: ```bash @@ -88,11 +88,8 @@ _Important: After the first login, remember to change the default credentials._ ## Features πŸš€ - Create custom chatbots with your own knowledge base - - Utilize powerful language models to generate responses - - Utilize PostgreSQL for vector search and storing the knowledge base. - - Use any language models or embedding models you want ## Stack πŸ“š @@ -113,61 +110,61 @@ Dialoqbase is a side project and is not ready for production. It is still in the ## Roadmap πŸ—ΊοΈ - ### Data loaders -- [x] Website (_one page only_) -- [x] Plane text -- [x] PDF (beta) -- [x] Web crawler (beta) -- [x] Microsoft Word documents (beta) -- [x] Github repository -- [x] mp3 -- [x] mp4 +- [X] Website (_one page only_) +- [X] Plane text +- [X] PDF (beta) +- [X] Web crawler (beta) +- [X] Microsoft Word documents (beta) +- [X] Github repository +- [X] mp3 +- [X] mp4 - [X] Sitemap -- [x] Youtube +- [X] Youtube - [ ] Notion and more... ### Language models -- [x] OpenAI -- [x] Anthropic -- [x] Google chat-bison-001 -- [x] fireworks.ai's llama 2 models -- [x] fireworks.ai's mistral +- [X] OpenAI +- [X] Anthropic +- [X] Google chat-bison-001 +- [X] fireworks.ai's llama 2 models +- [X] fireworks.ai's mistral - [X] Local AI models - [X] Ollama ### Embedding models -- [x] OpenAI -- [x] TensorFlow (removed) -- [x] Hugging Face -- [x] Cohere -- [x] all-MiniLM-L6-v2 using [xenova/transformers.js](https://github.com/xenova/transformers.js/) -- [x] Google text-gecko-001 +- [X] OpenAI +- [X] TensorFlow (removed) +- [X] Hugging Face +- [X] Cohere +- [X] all-MiniLM-L6-v2 using [xenova/transformers.js](https://github.com/xenova/transformers.js/) +- [X] Ollama embedding +- [X] Google text-gecko-001 need more ? create an issue... ### Application -- [x] Create chatbots -- [x] Create knowledge base -- [x] Generate responses -- [x] Vector search -- [x] Bot embed script for websites -- [x] Prompt Editor +- [X] Create chatbots +- [X] Create knowledge base +- [X] Generate responses +- [X] Vector search +- [X] Bot embed script for websites +- [X] Prompt Editor - [X] Chatbot API -- [x] Chatbot theme editor -- [x] Streaming responses +- [X] Chatbot theme editor +- [X] Streaming responses -### Integrations +### Integrations -- [x] Web embed script -- [x] Telegram (beta) -- [x] Discord (beta) +- [X] Web embed script +- [X] Telegram (beta) +- [X] Discord (beta) - [ ] Slack - [X] Whatsapp (experimental) diff --git a/server/src/utils/validate.ts b/server/src/utils/validate.ts index 1a6e3dc4..2c925795 100644 --- a/server/src/utils/validate.ts +++ b/server/src/utils/validate.ts @@ -40,9 +40,13 @@ export const apiKeyValidaton = (embeddingsType: string) => { return process.env.FIREWORKS_API_KEY ? process.env.FIREWORKS_API_KEY.length > 0 : false; - case "local": case "ollama": - return true; + return process.env.OLLAMA_EMBEDDING_API_URL && process.env.OLLAMA_EMBEDDING_MODEL + ? process.env.OLLAMA_EMBEDDING_API_URL.length > 0 && process.env.OLLAMA_EMBEDDING_MODEL.length > 0 + : false; + + case "local": + return true; default: return false; } @@ -57,6 +61,8 @@ export const apiKeyValidatonMessage = (embeddingsType: string) => { return "Please add COHERE_API_KEY to your .env file"; case "huggingface-api": return "Please add HUGGINGFACEHUB_API_KEY to your .env file"; + case "ollama": + return "Please add OLLAMA_EMBEDDING_API_URL and OLLAMA_EMBEDDING_MODEL to your .env file"; case "google-bison": case "google-gecko": case "google": From 9d8186d21b25494f380afc3c7beb26c80d30aa5b Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Wed, 3 Jan 2024 00:05:26 +0530 Subject: [PATCH 4/8] Remove sponsors section from README.md --- README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/README.md b/README.md index 854eb1c6..9c425541 100644 --- a/README.md +++ b/README.md @@ -178,16 +178,6 @@ need more ? create an issue... Interested in contributing? Check out the [contributing guide](CONTRIBUTION.md). -## Sponsors πŸ’– - -

- - - -

- ## License πŸ“ [MIT](LICENSE) From 09f3ab9c2716ed50583334c274d720fcb6f7bd46 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Wed, 3 Jan 2024 18:26:41 +0530 Subject: [PATCH 5/8] Add sponsors section to README.md --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 9c425541..842a3d6a 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,17 @@ need more ? create an issue... Interested in contributing? Check out the [contributing guide](CONTRIBUTION.md). + +## Sponsors πŸ’– + +

+ + + +

+ ## License πŸ“ [MIT](LICENSE) From 25e3feab49814934a6e322dc487b7136c390ce6d Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sat, 6 Jan 2024 21:15:09 +0530 Subject: [PATCH 6/8] Update package versions --- app/ui/package.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/ui/package.json b/app/ui/package.json index a2440829..ff5702eb 100644 --- a/app/ui/package.json +++ b/app/ui/package.json @@ -1,7 +1,7 @@ { "name": "app", "private": true, - "version": "1.6.0", + "version": "1.6.1", "type": "module", "scripts": { "dev": "vite", diff --git a/package.json b/package.json index cf27b51b..e4ea90d8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dialoqbase", - "version": "1.6.0", + "version": "1.6.1", "description": "Create chatbots with ease", "scripts": { "ui:dev": "pnpm run --filter ui dev", From 058d398f332a978497388d934112e91a2f4ec7ad Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 7 Jan 2024 16:25:20 +0530 Subject: [PATCH 7/8] Update README.md with loader and integration status --- README.md | 86 ++++++++++++++++++++++++++----------------------------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 842a3d6a..02d43b34 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ git clone https://github.com/n4ze3m/dialoqbase.git cd dialoqbase/docker ``` -2. Edit the `.env` file and set the following environment variables: +2. Edit the `.env` : - On Linux: @@ -60,7 +60,6 @@ vim .env notepad .env ``` -Set the `OPENAI_API_KEY` variable to your OpenAI API key. You can obtain an API key [here](https://platform.openai.com/account/api-keys). Set the `DB_SECRET_KEY` varible. 3. Run the docker-compose file: @@ -112,64 +111,61 @@ Dialoqbase is a side project and is not ready for production. It is still in the ### Data loaders -- [X] Website (_one page only_) -- [X] Plane text -- [X] PDF (beta) -- [X] Web crawler (beta) -- [X] Microsoft Word documents (beta) -- [X] Github repository -- [X] mp3 -- [X] mp4 -- [X] Sitemap -- [X] Youtube -- [ ] Notion +| Loader | Description | Status | +| ----------- | ----------- | ------ | +| Website | Load data from a website | βœ… | +| Plane text | Load data from a plane text file | βœ… | +| PDF | Load data from a PDF file | βœ… | +| Web crawler | Load data from a website using a web crawler | βœ… | +| Microsoft Word documents | Load data from a Microsoft Word document | βœ… | +| Github repository | Load data from a Github repository | βœ… | +| mp3 | Load data from a mp3 file | βœ… | +| mp4 | Load data from a mp4 file | βœ… | +| Sitemap | Load data from a sitemap | βœ… | +| Youtube | Load data from a Youtube video | βœ… | +| Notion | Load data from a Notion page | ❌ | -and more... ### Language models -- [X] OpenAI -- [X] Anthropic -- [X] Google chat-bison-001 -- [X] fireworks.ai's llama 2 models -- [X] fireworks.ai's mistral -- [X] Local AI models -- [X] Ollama + +| Provider | Models | +| ----------- | ----------- | +| OpenAI | gpt-3.5-turbo, gpt-3.5-turbo-16k, gpt-4, gpt-4-0613, gpt-3.5-instruct, gpt-3.5-turbo-1106, gpt-4-1106-preview | +| Anthropic | Claude 1, Claude 2 | +| Google | chat-bison-001, Gemini | +| Fireworks | Llama 2, Mistral, Mixtral and more | +| Local AI | Local AI models | +| Ollama | All Ollama supported models | ### Embedding models -- [X] OpenAI -- [X] TensorFlow (removed) -- [X] Hugging Face -- [X] Cohere -- [X] all-MiniLM-L6-v2 using [xenova/transformers.js](https://github.com/xenova/transformers.js/) -- [X] Ollama embedding -- [X] Google text-gecko-001 +| Provider | Models | Status | +| ----------- | ----------- | ------ | +| OpenAI | text-embedding-ada-002 | βœ… | +| TensorFlow | universal-sentence-encoder | ❌ | +| Hugging Face | distilbert-base-uncased | βœ… | +| Cohere | Embed | βœ… | +| Huggingface Transformers.js | all-MiniLM-L6-v2 | βœ… | +| Ollama | Ollama embedding | βœ… | +| Google | text-gecko-001, embedding-001 | βœ… | +| Jina | Jina embedding | βœ… | need more ? create an issue... -### Application - -- [X] Create chatbots -- [X] Create knowledge base -- [X] Generate responses -- [X] Vector search -- [X] Bot embed script for websites -- [X] Prompt Editor -- [X] Chatbot API -- [X] Chatbot theme editor -- [X] Streaming responses - ### Integrations -- [X] Web embed script -- [X] Telegram (beta) -- [X] Discord (beta) -- [ ] Slack -- [X] Whatsapp (experimental) +| Type | Stage | Status | +| ----------- | ----------- | ------ | +| Web embed script | stable | βœ… | +| Telegram | beta | βœ… | +| Discord | beta | βœ… | +| Slack | beta | ❌ | +| Whatsapp | experimental | βœ… | need more ? create an issue... + ## Contributors ✨ From fb66a594ad7f725f846744ff95d093b5fa4f5e15 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 7 Jan 2024 16:29:36 +0530 Subject: [PATCH 8/8] Update data loaders and language models --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 02d43b34..ccc7eeaf 100644 --- a/README.md +++ b/README.md @@ -111,19 +111,19 @@ Dialoqbase is a side project and is not ready for production. It is still in the ### Data loaders -| Loader | Description | Status | -| ----------- | ----------- | ------ | -| Website | Load data from a website | βœ… | -| Plane text | Load data from a plane text file | βœ… | -| PDF | Load data from a PDF file | βœ… | -| Web crawler | Load data from a website using a web crawler | βœ… | -| Microsoft Word documents | Load data from a Microsoft Word document | βœ… | -| Github repository | Load data from a Github repository | βœ… | -| mp3 | Load data from a mp3 file | βœ… | -| mp4 | Load data from a mp4 file | βœ… | -| Sitemap | Load data from a sitemap | βœ… | -| Youtube | Load data from a Youtube video | βœ… | -| Notion | Load data from a Notion page | ❌ | +| Loader |Status | +| ----------- | ------ | +| Website | βœ… | +| Plane text | βœ… | +| PDF | βœ… | +| Web crawler | βœ… | +| Microsoft Word documents | βœ… | +| Github repository | βœ… | +| mp3 | βœ… | +| mp4 | βœ… | +| Sitemap | βœ… | +| Youtube | βœ… | +| Notion | ❌ | ### Language models @@ -160,7 +160,7 @@ need more ? create an issue... | Web embed script | stable | βœ… | | Telegram | beta | βœ… | | Discord | beta | βœ… | -| Slack | beta | ❌ | +| Slack | development | ❌ | | Whatsapp | experimental | βœ… | need more ? create an issue...