feat: kobold service, better doctor

av · Jan 11, 2025 · 0c6282a · 0c6282a
1 parent baa01e0
commit 0c6282a
Show file tree

Hide file tree

Showing 13 changed files with 374 additions and 193 deletions.
diff --git a/README.md b/README.md
@@ -44,7 +44,8 @@ Harbor is a containerized LLM toolkit that allows you to run LLMs and additional
 [AirLLM](https://github.com/av/harbor/wiki/2.2.11-Backend:-AirLLM) ⦁︎
 [SGLang](https://github.com/av/harbor/wiki/2.2.12-Backend:-SGLang) ⦁︎
 [KTransformers](https://github.com/av/harbor/wiki/2.2.13-Backend:-KTransformers) ⦁︎
-[Nexa SDK](https://github.com/av/harbor/wiki/2.2.15-Backend:-Nexa-SDK)
+[Nexa SDK](https://github.com/av/harbor/wiki/2.2.15-Backend:-Nexa-SDK) ⦁︎
+[KoboldCpp](https://github.com/av/harbor/wiki/2.2.16-Backend:-KoboldCpp)
 
 ##### Satellites
 [Harbor Bench](https://github.com/av/harbor/wiki/5.1.-Harbor-Bench) ⦁︎

diff --git a/app/src/serviceMetadata.tsx b/app/src/serviceMetadata.tsx
@@ -270,4 +270,8 @@ export const serviceMetadata: Record<string, Partial<HarborService>> = {
         tags: [HST.satellite, HST.api],
         wikiUrl: 'https://github.com/av/harbor/wiki/2.3.33-Satellite:-OptiLLM',
     },
+    kobold: {
+        tags: [HST.satellite, HST.frontend, HST.backend],
+        wikiUrl: 'https://github.com/av/harbor/wiki/2.2.16-Backend:-KoboldCpp',
+    }
 };
diff --git a/compose.kobold.yml b/compose.kobold.yml
@@ -0,0 +1,20 @@
+services:
+  kobold:
+    container_name: ${HARBOR_CONTAINER_PREFIX}.kobold
+    image: ${HARBOR_KOBOLD_IMAGE}:${HARBOR_KOBOLD_VERSION}
+    env_file:
+      - ./.env
+      - kobold/override.env
+    volumes:
+      - ${HARBOR_KOBOLD_WORKSPACE}:/workspace/:rw
+      - ${HARBOR_OLLAMA_CACHE}:/ollama
+      - ${HARBOR_HF_CACHE}:/hf
+      - ${HARBOR_LLAMACPP_CACHE}:/llamacpp
+      - ${HARBOR_VLLM_CACHE}:/vllm
+    ports:
+      - ${HARBOR_KOBOLD_HOST_PORT}:5001
+    environment:
+      - KCPP_MODEL=${HARBOR_KOBOLD_MODEL}
+      - KCPP_ARGS=${HARBOR_KOBOLD_ARGS}
+    networks:
+      - harbor-network
diff --git a/compose.perplexideez.yml b/compose.perplexideez.yml
@@ -69,69 +69,3 @@ services:
         condition: service_healthy
     networks:
       - harbor-network
-
-configs:
-  perplexideez_proxy_config:
-    content: |
-      server {
-          listen 80;
-          gzip_static off;
-          server_name perplexideez.com;
-
-          location / {
-              proxy_pass http://perplexideez-service:3000;
-              proxy_set_header Host perplexideez.com;
-              proxy_set_header X-Real-IP $$remote_addr;
-              proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for;
-              proxy_set_header X-Forwarded-Proto https;
-
-              # Disable CSRF protection in Nginx
-              proxy_set_header X-Forwarded-Host $$host;
-              proxy_set_header Origin https://perplexideez.com;
-              proxy_set_header Referer https://perplexideez.com;
-
-              # Rewrite Location header to use localhost
-              proxy_redirect https://perplexideez.com/ http://localhost:34261/;
-              proxy_redirect http://perplexideez.com/ http://localhost:34261/;
-
-              # Cookie rewriting
-              proxy_cookie_domain perplexideez.com localhost;
-              proxy_cookie_path / /;
-              proxy_cookie_flags ~ -secure;
-
-              # Force uncompressed response to allow substitutions
-              proxy_set_header Accept-Encoding "";
-
-              # Rewrite Secure-prefixed cookies
-              proxy_hide_header set-cookie;
-              proxy_set_header Accept-Encoding "";
-              sub_filter_types *;
-              sub_filter_once off;
-              sub_filter '__Secure-' '';
-              sub_filter '; Secure' '';
-              sub_filter 'perplexideez.com' 'localhost:34261';
-              sub_filter "https://localhost:34261" "http://localhost:34261";
-              sub_filter "https%3A%2F%2Flocalhost%3A34261" "http%3A%2F%2Flocalhost%3A34261";
-              sub_filter 'href="https://localhost:34261' 'href="http://localhost:34261';
-              sub_filter 'content="https://localhost:34261' 'content="http://localhost:34261';
-
-              # CORS headers
-              add_header 'Access-Control-Allow-Origin' '*';
-              add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-              add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization';
-
-              # Handle OPTIONS method
-              if ($$request_method = 'OPTIONS') {
-                  add_header 'Access-Control-Allow-Origin' '*';
-                  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-                  add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization';
-                  add_header 'Access-Control-Max-Age' 1728000;
-                  add_header 'Content-Type' 'text/plain; charset=utf-8';
-                  add_header 'Content-Length' 0;
-                  return 204;
-              }
-
-              # Force processing of response
-              proxy_force_ranges on;
-          }
-      }
diff --git a/compose.x.kobold.nvidia.yml b/compose.x.kobold.nvidia.yml
@@ -0,0 +1,9 @@
+services:
+  kobold:
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
diff --git a/compose.x.perplexideez.mdc.yml b/compose.x.perplexideez.mdc.yml
@@ -0,0 +1,65 @@
+configs:
+  perplexideez_proxy_config:
+    content: |
+      server {
+          listen 80;
+          gzip_static off;
+          server_name perplexideez.com;
+
+          location / {
+              proxy_pass http://perplexideez-service:3000;
+              proxy_set_header Host perplexideez.com;
+              proxy_set_header X-Real-IP $$remote_addr;
+              proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for;
+              proxy_set_header X-Forwarded-Proto https;
+
+              # Disable CSRF protection in Nginx
+              proxy_set_header X-Forwarded-Host $$host;
+              proxy_set_header Origin https://perplexideez.com;
+              proxy_set_header Referer https://perplexideez.com;
+
+              # Rewrite Location header to use localhost
+              proxy_redirect https://perplexideez.com/ http://localhost:34261/;
+              proxy_redirect http://perplexideez.com/ http://localhost:34261/;
+
+              # Cookie rewriting
+              proxy_cookie_domain perplexideez.com localhost;
+              proxy_cookie_path / /;
+              proxy_cookie_flags ~ -secure;
+
+              # Force uncompressed response to allow substitutions
+              proxy_set_header Accept-Encoding "";
+
+              # Rewrite Secure-prefixed cookies
+              proxy_hide_header set-cookie;
+              proxy_set_header Accept-Encoding "";
+              sub_filter_types *;
+              sub_filter_once off;
+              sub_filter '__Secure-' '';
+              sub_filter '; Secure' '';
+              sub_filter 'perplexideez.com' 'localhost:34261';
+              sub_filter "https://localhost:34261" "http://localhost:34261";
+              sub_filter "https%3A%2F%2Flocalhost%3A34261" "http%3A%2F%2Flocalhost%3A34261";
+              sub_filter 'href="https://localhost:34261' 'href="http://localhost:34261';
+              sub_filter 'content="https://localhost:34261' 'content="http://localhost:34261';
+
+              # CORS headers
+              add_header 'Access-Control-Allow-Origin' '*';
+              add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+              add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization';
+
+              # Handle OPTIONS method
+              if ($$request_method = 'OPTIONS') {
+                  add_header 'Access-Control-Allow-Origin' '*';
+                  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+                  add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization';
+                  add_header 'Access-Control-Max-Age' 1728000;
+                  add_header 'Content-Type' 'text/plain; charset=utf-8';
+                  add_header 'Content-Length' 0;
+                  return 204;
+              }
+
+              # Force processing of response
+              proxy_force_ranges on;
+          }
+      }
diff --git a/compose.x.webui.kobold.yml b/compose.x.webui.kobold.yml
@@ -0,0 +1,4 @@
+services:
+  webui:
+    volumes:
+      - ./open-webui/configs/config.kobold.json:/app/configs/config.kobold.json