Merge pull request #30 from vnbl/docker-compose-setup

working docker-compose: solved .mage_data persistance issue
ProyectoRespira · Oct 21, 2024 · 8556737 · 8556737
2 parents d36510e + 1dd7bd3
commit 8556737
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 6 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,2 +1,4 @@
+# ignore src/
 src/
+# ignore venv/
 venv/
diff --git a/.env.example b/.env.example
@@ -10,5 +10,6 @@ MYSQL_HOST='mysql_host'
 MYSQL_DB='mysql_db'
 MYSQL_PORT='3306'
 
+MAGE_DATA_DIR= 'path/to/mage/data' 
 PIPELINE_HOST='localhost'
 PIPELINE_PORT='6789'
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -25,7 +25,7 @@ services:
       dockerfile: Dockerfile
     restart: always
     environment:
-      MAGE_DATA_DIR: ../.mage_data
+      MAGE_DATA_DIR: ${MAGE_DATA_DIR}
       PIPELINE_POSTGRES_USER: ${POSTGRES_USER}
       PIPELINE_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       PIPELINE_POSTGRES_DB: ${POSTGRES_DB}
@@ -48,7 +48,9 @@ services:
       retries: 5
       start_period: 10s
     volumes:
-      - /home/vnbl/data_retriever:/app
-
+      - ./:/app
+      # manually transfer .mage_data with already created secrets to server or store secrets in server's mage UI during deployment
+      - ../.mage_data:/root/.mage_data 
+
 volumes:
   postgres_data:
diff --git a/etl-pipeline/metadata.yaml b/etl-pipeline/metadata.yaml
@@ -1,6 +1,6 @@
 project_type: standalone
 
-variables_dir: /app/.mage_data/
+variables_dir: "{{ env_var('MAGE_DATA_DIR') }}"
 # remote_variables_dir: s3://bucket/path_prefix
 
 variables_retention_period: '90d'

diff --git a/requirements.txt b/requirements.txt
@@ -286,7 +286,7 @@ typer==0.9.0
 typing_extensions==4.10.0
 tzdata==2024.2
 tzlocal==5.2
-urllib3==2.2.3
+urllib3==1.26.20
 utilsforecast==0.2.5
 watchdog==4.0.0
 wcwidth==0.2.13
@@ -298,4 +298,4 @@ xarray==2024.9.0
 xgboost==2.1.1
 yarl==1.13.1
 zipp==3.20.2
-zstandard==0.23.0
+zstandard==0.23.0