etl

nogibjj · Dec 13, 2024 · 068b44f · 068b44f
1 parent 765e5a5
commit 068b44f
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 198 deletions.
diff --git a/country_capital.json b/country_capital.json
diff --git a/databricks_scripts/extract.py b/databricks_scripts/extract.py
@@ -25,6 +25,7 @@ def do_extract():
     extract(
         "https://data.cityofnewyork.us/resource/c3uy-2p5r.csv?$limit=200000",
         "air_quality.csv",
+        on_databricks=True,
     )
 
     log_tests("Testing if CSV file exists...")

diff --git a/databricks_scripts/query.py b/databricks_scripts/query.py
@@ -24,11 +24,19 @@
 """
 
 
-def query():
+def query(on_databricks=False):
     result_df = execute_read_query(script_to_execute)
     result_df.show()
     save_output(result_df.toPandas().to_markdown())
-    result_df.write.csv("/tmp/Aggregation_Query_Result", header=True, mode="overwrite")
+    result_df.write.csv(
+        (
+            "/Workspace/Workspace/Shared/Leonard_Eshun_Mini_Project_Eleven/data/"
+            if on_databricks
+            else "./Aggregation_Query_Result"
+        ),
+        header=True,
+        mode="overwrite",
+    )
 
 
 if __name__ == "__main__":

diff --git a/databricks_scripts/transform_n_load.py b/databricks_scripts/transform_n_load.py
@@ -48,6 +48,7 @@ def transform_and_load():
             "fn_geo_id": 6,
             "fn_indicator_id": 1,
         },
+        on_databricks=True,
     )
     log_tests("Transform and Load Test Successful", last_in_group=True)
     print("Transform and Load Test Successful")