v1.6.1

README.md

@@ -3,11 +3,12 @@
  Create chatbots with ease
 </p>
 
-<div align="center"> 
-  
+<div align="center">
+
   [![Join dialoqbase #welcome](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/SPE3npH7Wu)
   [![Build Status](https://github.com/n4ze3m/dialoqbase/actions/workflows/build.yml/badge.svg)](https://github.com/n4ze3m/dialoqbase/actions/workflows/build.yml)
   [![License: MIT](https://img.shields.io/github/license/n4ze3m/dialoqbase)](https://github.com/n4ze3m/dialoqbase/blob/master/LICENSE)
+
 </div>
 
 Dialoqbase is an open-source application designed to facilitate the creation of custom chatbots using a personalized knowledge base. The application leverages advanced language models to generate accurate and context-aware responses. Additionally, it utilizes PostgreSQL, a robust relational database management system, for efficient vector search operations and for storing the knowledge base.
@@ -24,7 +25,7 @@ Want to check more demo videos? Follow me on [Twitter](https://twitter.com/n4ze3
 
 ## Quick Deployments 🚀
 
-### Railway with Supabase 
+### Railway with Supabase
 
 [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/TXdjD7?referralCode=olbszX)
 
@@ -39,7 +40,7 @@ git clone https://github.com/n4ze3m/dialoqbase.git
 cd dialoqbase/docker
 ```
 
-2. Edit the `.env` file and set the following environment variables:
+2. Edit the `.env` <!--(see the [Environment Variables](#environment-variables) section for more details)-->:
 
 - On Linux:
 
@@ -59,7 +60,6 @@ vim .env
 notepad .env
 ```
 
-Set the `OPENAI_API_KEY` variable to your OpenAI API key. You can obtain an API key [here](https://platform.openai.com/account/api-keys).
 Set the `DB_SECRET_KEY` varible.
 
 3. Run the docker-compose file:
@@ -75,7 +75,6 @@ docker compose up -d
 ```
 
 4. Open your browser and go to `http://localhost:3000`.
-
 5. Log in using the default credentials:
 
 ```bash
@@ -88,11 +87,8 @@ _Important: After the first login, remember to change the default credentials._
 ## Features 🚀
 
 - Create custom chatbots with your own knowledge base
-
 - Utilize powerful language models to generate responses
-
 - Utilize PostgreSQL for vector search and storing the knowledge base.
-
 - Use any language models or embedding models you want
 
 ## Stack 📚
@@ -113,66 +109,63 @@ Dialoqbase is a side project and is not ready for production. It is still in the
 
 ## Roadmap 🗺️
 
-
 ### Data loaders
 
-- [x] Website (_one page only_)
-- [x] Plane text
-- [x] PDF (beta)
-- [x] Web crawler (beta)
-- [x] Microsoft Word documents (beta)
-- [x] Github repository
-- [x] mp3
-- [x] mp4
-- [X] Sitemap
-- [x] Youtube
-- [ ] Notion
+| Loader |Status |
+| ----------- |  ------ |
+| Website | ✅ |
+| Plane text |  ✅ |
+| PDF |  ✅ |
+| Web crawler |  ✅ |
+| Microsoft Word documents |  ✅ |
+| Github repository | ✅ |
+| mp3 | ✅ |
+| mp4 |  ✅ |
+| Sitemap |  ✅ |
+| Youtube | ✅ |
+| Notion |  ❌ |
 
-and more...
 
 ### Language models
 
-- [x] OpenAI
-- [x] Anthropic
-- [x] Google chat-bison-001
-- [x] fireworks.ai's llama 2 models
-- [x] fireworks.ai's mistral
-- [X] Local AI models
-- [X] Ollama
+
+| Provider | Models |
+| ----------- | ----------- |
+| OpenAI | gpt-3.5-turbo, gpt-3.5-turbo-16k, gpt-4, gpt-4-0613, gpt-3.5-instruct, gpt-3.5-turbo-1106, gpt-4-1106-preview	| 
+| Anthropic | Claude 1, Claude 2 |
+| Google | chat-bison-001, Gemini |
+| Fireworks | Llama 2, Mistral, Mixtral and more |
+| Local AI | Local AI models |
+| Ollama | All Ollama supported models |
 
 ### Embedding models
 
-- [x] OpenAI
-- [x] TensorFlow (removed)
-- [x] Hugging Face
-- [x] Cohere
-- [x] all-MiniLM-L6-v2 using [xenova/transformers.js](https://github.com/xenova/transformers.js/)
-- [x] Google text-gecko-001
+| Provider | Models | Status |
+| ----------- | ----------- | ------ |
+| OpenAI | text-embedding-ada-002 | ✅ |
+| TensorFlow | universal-sentence-encoder | ❌ |
+| Hugging Face | distilbert-base-uncased | ✅ |
+| Cohere | Embed | ✅ |
+| Huggingface Transformers.js | all-MiniLM-L6-v2 | ✅ |
+| Ollama | Ollama embedding | ✅ |
+| Google | text-gecko-001, embedding-001 | ✅ |
+| Jina | Jina embedding | ✅ |
 
 need more ? create an issue...
 
-### Application
-
-- [x] Create chatbots
-- [x] Create knowledge base
-- [x] Generate responses
-- [x] Vector search
-- [x] Bot embed script for websites
-- [x] Prompt Editor
-- [X] Chatbot API
-- [x] Chatbot theme editor
-- [x] Streaming responses
+### Integrations
 
-### Integrations 
-
-- [x] Web embed script
-- [x] Telegram (beta)
-- [x] Discord (beta)
-- [ ] Slack
-- [X] Whatsapp (experimental)
+| Type | Stage | Status |
+| ----------- | ----------- | ------ |
+| Web embed script |  stable | ✅ |
+| Telegram | beta | ✅ |
+| Discord | beta | ✅ |
+| Slack | development | ❌ |
+| Whatsapp | experimental | ✅ |
 
 need more ? create an issue...
 
+
 ## Contributors ✨
 
 <a href="https://github.com/n4ze3m/dialoqbase/graphs/contributors">
@@ -181,6 +174,7 @@ need more ? create an issue...
 
 Interested in contributing? Check out the [contributing guide](CONTRIBUTION.md).
 
+
 ## Sponsors 💖
 
 <p>

app/ui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "1.6.0",
+  "version": "1.6.1",
   "type": "module",
   "scripts": {
     "dev": "vite",

app/ui/src/utils/embeddings.ts

@@ -6,6 +6,10 @@ export const availableEmbeddingTypes = [
     value: "transformer",
     label: "Xenova/all-MiniLM-L6-v2 (cpu)",
   },
+  {
+    value: "ollama",
+    label: "Ollama Embeddings",
+  },
   {
     value: "supabase",
     label: "Supabase/gte-small (cpu)",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "1.6.0",
+  "version": "1.6.1",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

server/.env.example

@@ -14,4 +14,6 @@ ELEVENLABS_API_KEY=""
 DB_QUEUE_CONCURRENCY=2
 DB_SESSION_SECRET="a8F2h6T9j4Kl0Pz8W7eX3rB5y1VcQ6mN"
 DB_SESSION_SECURE="false"
-JINA_API_KEY=""
+JINA_API_KEY=""
+OLLAMA_EMBEDDING_API_URL=""
+OLLAMA_EMBEDDING_MODEL=""

server/src/routes/api/v1/bot/handlers/get.handler.ts

@@ -146,6 +146,10 @@ export const getCreateBotConfigHandler = async (
       value: "transformer",
       label: "all-MiniLM-L6-v2 (cpu)",
     },
+    {
+      value: "ollama",
+      label: "Ollama Embeddings",
+    },
     {
       value: "google-gecko",
       label: "Google text-gecko-001",
@@ -206,6 +210,10 @@ export const getBotByIdSettingsHandler = async (
       value: "transformer",
       label: "all-MiniLM-L6-v2 (cpu)",
     },
+    {
+      value: "ollama",
+      label: "Ollama Embeddings",
+    },
     {
       value: "google-gecko",
       label: "Google text-gecko-001",

server/src/utils/embeddings.ts

@@ -4,6 +4,8 @@ import { HuggingFaceInferenceEmbeddings } from "langchain/embeddings/hf";
 import { TransformersEmbeddings } from "../embeddings/transformer-embedding";
 import { GooglePaLMEmbeddings } from "langchain/embeddings/googlepalm";
 import { GoogleGenerativeAIEmbeddings } from "@langchain/google-genai";
+import { OllamaEmbeddings } from "langchain/embeddings/ollama";
+
 
 export const embeddings = (embeddingsType: string) => {
   switch (embeddingsType) {
@@ -17,6 +19,8 @@ export const embeddings = (embeddingsType: string) => {
       return new TransformersEmbeddings({
         model: "Xenova/all-MiniLM-L6-v2",
       });
+    case "ollama":
+      return new OllamaEmbeddings({ baseUrl: process.env.OLLAMA_EMBEDDING_API_URL, model: process.env.OLLAMA_EMBEDDING_API_MODEL });
     case "jina":
       return new TransformersEmbeddings({
         model: "Xenova/jina-embeddings-v2-small-en",
@@ -52,6 +56,7 @@ export const supportedEmbeddings = [
   "cohere",
   "huggingface-api",
   "transformer",
+  "ollama",
   "google-gecko",
   "supabase",
   "jina",

server/src/utils/validate.ts

@@ -40,9 +40,13 @@ export const apiKeyValidaton = (embeddingsType: string) => {
       return process.env.FIREWORKS_API_KEY
         ? process.env.FIREWORKS_API_KEY.length > 0
         : false;
-    case "local":
     case "ollama":
-        return true;
+      return process.env.OLLAMA_EMBEDDING_API_URL && process.env.OLLAMA_EMBEDDING_MODEL
+        ? process.env.OLLAMA_EMBEDDING_API_URL.length > 0 && process.env.OLLAMA_EMBEDDING_MODEL.length > 0
+        : false;
+
+    case "local":
+      return true;
     default:
       return false;
   }
@@ -57,6 +61,8 @@ export const apiKeyValidatonMessage = (embeddingsType: string) => {
       return "Please add COHERE_API_KEY to your .env file";
     case "huggingface-api":
       return "Please add HUGGINGFACEHUB_API_KEY to your .env file";
+    case "ollama":
+      return "Please add OLLAMA_EMBEDDING_API_URL and OLLAMA_EMBEDDING_MODEL to your .env file";
     case "google-bison":
     case "google-gecko":
     case "google":