This code imports data from NBP website and creates a report using Airflow, GCP and DuckDB
git clone https://github.com/MariuszKu/analiza-duckdb-mieszkania.git
cd analiza-duckdb-mieszkania
chmod +x deploy.sh
./deploy.sh
If you want to work with GCP you need to upload your secrete key and change the working path in env.py file
Location for key dags/key.json
LINK = "gs://mk-dev-gcs/data/" # for GCP
LINK = 'data/' #for local
LINK = "/opt/airflow/dags/data/" # for airflow local
create or replace EXTERNAL table mk.flats_report
OPTIONS(
format = "PARQUET",
uris = ['gs://mk-dev-gcs/data/data/*.parquet']
);
