From 1dd7bd34d0e5d1cf2cf07637f83c895feeec20c0 Mon Sep 17 00:00:00 2001
From: vnbl <fernanda.carles@gmail.com>
Date: Mon, 21 Oct 2024 17:39:46 -0300
Subject: [PATCH] working docker-compose: solved .mage_data persistance issue

---
 .dockerignore              | 2 ++
 .env.example               | 1 +
 docker-compose.yml         | 8 +++++---
 etl-pipeline/metadata.yaml | 2 +-
 requirements.txt           | 4 ++--
 5 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index e5ed15c..7f25d47 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,2 +1,4 @@
+# ignore src/
 src/
+# ignore venv/
 venv/
diff --git a/.env.example b/.env.example
index 834ad5f..42a53a8 100644
--- a/.env.example
+++ b/.env.example
@@ -10,5 +10,6 @@ MYSQL_HOST='mysql_host'
 MYSQL_DB='mysql_db'
 MYSQL_PORT='3306'
 
+MAGE_DATA_DIR= 'path/to/mage/data' 
 PIPELINE_HOST='localhost'
 PIPELINE_PORT='6789'
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 356574c..5891343 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -25,7 +25,7 @@ services:
       dockerfile: Dockerfile
     restart: always
     environment:
-      MAGE_DATA_DIR: ../.mage_data
+      MAGE_DATA_DIR: ${MAGE_DATA_DIR}
       PIPELINE_POSTGRES_USER: ${POSTGRES_USER}
       PIPELINE_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       PIPELINE_POSTGRES_DB: ${POSTGRES_DB}
@@ -48,7 +48,9 @@ services:
       retries: 5
       start_period: 10s
     volumes:
-      - /home/vnbl/data_retriever:/app
-
+      - ./:/app
+      # manually transfer .mage_data with already created secrets to server or store secrets in server's mage UI during deployment
+      - ../.mage_data:/root/.mage_data 
+      
 volumes:
   postgres_data:
diff --git a/etl-pipeline/metadata.yaml b/etl-pipeline/metadata.yaml
index 84dd7bd..c0e7b78 100755
--- a/etl-pipeline/metadata.yaml
+++ b/etl-pipeline/metadata.yaml
@@ -1,6 +1,6 @@
 project_type: standalone
 
-variables_dir: /app/.mage_data/
+variables_dir: "{{ env_var('MAGE_DATA_DIR') }}"
 # remote_variables_dir: s3://bucket/path_prefix
 
 variables_retention_period: '90d'
diff --git a/requirements.txt b/requirements.txt
index 2e005ec..3b7aca3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -286,7 +286,7 @@ typer==0.9.0
 typing_extensions==4.10.0
 tzdata==2024.2
 tzlocal==5.2
-urllib3==2.2.3
+urllib3==1.26.20
 utilsforecast==0.2.5
 watchdog==4.0.0
 wcwidth==0.2.13
@@ -298,4 +298,4 @@ xarray==2024.9.0
 xgboost==2.1.1
 yarl==1.13.1
 zipp==3.20.2
-zstandard==0.23.0
\ No newline at end of file
+zstandard==0.23.0