From 27b7425659b4c29805a7eb04a28bde097648509e Mon Sep 17 00:00:00 2001 From: David Cannan <104325852+Cdaprod@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:58:42 -0400 Subject: [PATCH] Add docker-compose file for deploying MinIO and Weaviate servers with Tailscale container. --- docker-compose.tailscale.yaml | 167 ++++++++++++++++++++++++++++++++++ docker-compose.yaml | 2 +- 2 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 docker-compose.tailscale.yaml diff --git a/docker-compose.tailscale.yaml b/docker-compose.tailscale.yaml new file mode 100644 index 0000000..b5b0e5b --- /dev/null +++ b/docker-compose.tailscale.yaml @@ -0,0 +1,167 @@ +Here's the updated `docker-compose.yml` file for deploying MinIO and Weaviate servers, each with a dedicated Tailscale container. This setup ensures that both services are accessible over your Tailscale tailnet with the appropriate environment variables and configurations. + +### docker-compose.yml + +```yaml +version: '3.8' + +# Set Variables: TS_AUTHKEY, TS_CERT_DOMAIN, MINIO_DOMAIN, MINIO_BROWSER_REDIRECT_URL, MINIO_ROOT_USER, MINIO_ROOT_PASSWORD, WEAVIATE_ORIGIN +# Configuration Files: TS_SERVE_CONFIG + +services: + tailscale-minio: + image: tailscale/tailscale:latest + container_name: tailscale-minio + hostname: tailscale-minio # The Tailnet node will be named this for use with MagicDNS + volumes: + - /dev/net/tun:/dev/net/tun + - ${PWD}/tailscale-minio/state:/var/lib/tailscale + environment: + - TS_AUTHKEY=${TS_AUTHKEY} + - TS_STATE_DIR=/var/lib/tailscale + - TS_SERVE_CONFIG=/tailscale/minio.json + - TS_EXTRA_ARGS=--advertise-routes=10.0.0.0/24 --advertise-tags=tag:container --accept-routes --advertise-exit-node + cap_add: + - NET_ADMIN + - SYS_MODULE + network_mode: host + restart: unless-stopped + + minio: + image: minio/minio:latest + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_DOMAIN: minio.${TS_CERT_DOMAIN} + MINIO_BROWSER_REDIRECT_URL: https://${MINIO_DOMAIN} + command: server /data --address ":9000" --console-address ":9001" + volumes: + - minio_data:/data + depends_on: + - tailscale-minio + network_mode: service:tailscale-minio + + tailscale-weaviate: + image: tailscale/tailscale:latest + container_name: tailscale-weaviate + hostname: tailscale-weaviate # The Tailnet node will be named this for use with MagicDNS + volumes: + - /dev/net/tun:/dev/net/tun + - ${PWD}/tailscale-weaviate/state:/var/lib/tailscale + environment: + - TS_AUTHKEY=${TS_AUTHKEY} + - TS_STATE_DIR=/var/lib/tailscale + - TS_SERVE_CONFIG=/tailscale/weaviate.json + - TS_EXTRA_ARGS=--advertise-routes=10.0.0.0/24 --advertise-tags=tag:container --accept-routes --advertise-exit-node + cap_add: + - NET_ADMIN + - SYS_MODULE + network_mode: host + restart: unless-stopped + + weaviate: + image: semitechnologies/weaviate:latest + environment: + QUERY_DEFAULTS_LIMIT: 20 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + WEAVIATE_ORIGIN: ${WEAVIATE_ORIGIN} + ports: + - "8080:8080" + depends_on: + - tailscale-weaviate + network_mode: service:tailscale-weaviate + +volumes: + minio_data: + driver: local + tailscale-minio: + driver: local + tailscale-weaviate: + driver: local + +networks: + app_network: + driver: bridge +``` + +### Bash Functions and Aliases + +Add the following bash functions and aliases to your `~/.bashrc` or `~/.bash_profile` to interact with your MinIO and Weaviate services via the CLI. + +#### Add to `~/.bashrc` or `~/.bash_profile` + +```bash +# Function to upload a file to MinIO +upload_to_minio() { + python3 minio_weaviate.py upload "$1" "$2" +} + +# Function to index a file in Weaviate +index_in_weaviate() { + python3 minio_weaviate.py index "$1" "$2" "$3" "$4" +} + +# Function to upload a file to MinIO and index it in Weaviate +upload_and_index() { + python3 minio_weaviate.py upload_and_index "$1" "$2" "$3" "$4" +} + +# Aliases for convenience +alias minio_upload=upload_to_minio +alias weaviate_index=index_in_weaviate +alias upload_index=upload_and_index +``` + +After adding these lines, reload your bash profile: + +```sh +source ~/.bashrc +# or +source ~/.bash_profile +``` + +### Python Script for MinIO and Weaviate + +Create a Python script called `minio_weaviate.py` to handle the upload and indexing operations. + +#### minio_weaviate.py + +```python +from minio import Minio +from weaviate import Client +import sys +import json + +# MinIO Configuration +minio_client = Minio( + "minio.${TS_CERT_DOMAIN}:9000", # MinIO server address + access_key="minioadmin", # MinIO access key + secret_key="minioadmin", # MinIO secret key + secure=True # Using HTTPS +) + +# Weaviate Configuration +weaviate_client = Client( + "https://weaviate.${TS_CERT_DOMAIN}:8080", # Weaviate server address +) + +def upload_to_minio(file_path, bucket_name): + # Upload file to MinIO + minio_client.fput_object(bucket_name, file_path, file_path) + print(f"Uploaded {file_path} to bucket {bucket_name}.") + +def index_in_weaviate(file_path, bucket_name, class_name, schema): + # Create an object in Weaviate + weaviate_client.data_object.create({ + "file_name": file_path, + "bucket": bucket_name + }, class_name, schema) + print(f"Indexed {file_path} in Weaviate class {class_name}.") + +def upload_and_index(file_path, bucket_name, class_name, schema): + upload_to_minio(file_path, bucket_name) + index_in_weaviate(file_path, bucket_name, class_name, schema) + +if __name__ == "__main__": + action = sys.argv[1] + file_path = sys.argv[2 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 5120df2..8a3d473 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -35,7 +35,7 @@ services: DEFAULT_VECTORIZER_MODULE: 'none' AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' PERSISTENCE_DATA_PATH: '/var/lib/weaviate' - ENABLE_MODULES: 'backup-s3' + ENABLE_MODULES: 'backup-s3, text2vec-cohere,text2vec-huggingface,text2vec-palm,text2vec-openai,generative-openai,generative-cohere,generative-palm,ref2vec-centroid,reranker-cohere,qna-openai' BACKUP_S3_BUCKET: 'weaviate-backups' BACKUP_S3_ENDPOINT: 'minio:9000' BACKUP_S3_ACCESS_KEY_ID: 'minio'