From c2232218ea1ed38911adf1aeb844313c6725320c Mon Sep 17 00:00:00 2001 From: David Gasquez Date: Mon, 8 Jan 2024 14:50:01 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E2=9A=A1=EF=B8=8F=20render=20Quart?= =?UTF-8?q?o=20on=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- .github/workflows/run.yml | 15 +++++++++++---- Makefile | 9 ++++++++- portal/_quarto.yml | 2 ++ portal/catalog.ipynb | 4 ++-- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9f9c38..aaa4fa5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - name: Export Database and Log File Sizes run: | - python -c 'from ggdp import db; db.export_database_to_parquet("data/local.duckdb", "tables");' + make tables echo "Files exported:" >> $GITHUB_STEP_SUMMARY { echo "| Filename | File Size |" diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 62b58c7..76ecf97 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -32,7 +32,7 @@ jobs: - name: Export Database and Log File Sizes run: | - python -c 'from ggdp import db; db.export_database_to_parquet("data/local.duckdb", "tables");' + make tables echo "Files exported:" >> $GITHUB_STEP_SUMMARY { echo "| Filename | File Size |" @@ -40,11 +40,18 @@ jobs: ls -lh tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}' } >> $GITHUB_STEP_SUMMARY ls -lh tables - + - name: Install Quarto + uses: quarto-dev/quarto-actions/setup@v2 + - name: Render + run: | + make render + - name: Copy Portal + run: | + cp -r portal/_site data/tables/portal - uses: filebase/ipfs-action@master id: ipfs with: - path: ./tables + path: ./data/tables service: filebase pinName: gitcoin-grants-data-portal-tables filebaseBucket: gitcoin-grants-data-portal @@ -52,7 +59,7 @@ jobs: filebaseSecret: ${{ secrets.FILEBASE_SECRET }} key: gitcoin-grants-data-portal-db - - name: Create CID file + - name: Update CID file run: echo ${{ steps.ipfs.outputs.cid }} > data/IPFS_CID - name: Commit and Push diff --git a/Makefile b/Makefile index f898f1f..317b2e3 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ run: @dagster asset materialize --select \* -m ggdp +tables: + @python -c 'from ggdp import db; db.export_database_to_parquet("data/local.duckdb", "data/tables");' + dev: @dagster dev -m ggdp @@ -13,11 +16,15 @@ preview: @quarto preview portal render: + @cd dbt && dbt docs generate + @mkdir -p dbt/target/docs + @cp dbt/target/*.json dbt/target/index.html dbt/target/graph.gpickle dbt/target/docs/ @quarto render portal + @cp -r dbt/target/docs/ portal/_site/dbt publish: @quarto publish gh-pages portal --no-prompt clean: - @rm -rf portal/.quarto data/*.parquet data/*.duckdb + @rm -rf portal/.quarto data/*.parquet data/*.duckdb portal/_site @rm -rf dbt/target dbt/logs dbt/dbt_packages diff --git a/portal/_quarto.yml b/portal/_quarto.yml index f328126..f75fc36 100644 --- a/portal/_quarto.yml +++ b/portal/_quarto.yml @@ -13,6 +13,8 @@ website: text: Notebook - href: catalog.ipynb text: Catalog + - href: dbt + text: dbt Docs format: html: diff --git a/portal/catalog.ipynb b/portal/catalog.ipynb index 1c86106..91d5f54 100644 --- a/portal/catalog.ipynb +++ b/portal/catalog.ipynb @@ -286,8 +286,8 @@ "\n", "import os\n", "\n", - "# Make sure to `make run` first so `tables` folder is actually populated.\n", - "EXPORT_PATH = os.path.join(\"..\", \"tables\")\n", + "# Make sure to `make export-db`\n", + "EXPORT_PATH = os.path.join(\"..\", \"data/tables\")\n", "EXTENSIONS = \".parquet\"\n", "\n", "raw_files = []\n", From f6928a1bb36d7c0b0efcd3e92514107b5fea820b Mon Sep 17 00:00:00 2001 From: David Gasquez Date: Mon, 8 Jan 2024 14:53:32 +0000 Subject: [PATCH 2/3] =?UTF-8?q?ci:=20=E2=99=BB=EF=B8=8F=20render=20on=20ci?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aaa4fa5..836dfea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,3 +38,8 @@ jobs: ls -lh tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}' } >> $GITHUB_STEP_SUMMARY ls -lh tables + - name: Install Quarto + uses: quarto-dev/quarto-actions/setup@v2 + - name: Render + run: | + make render From 6faad4fe366738260fb31c1d2a64df87c3562c43 Mon Sep 17 00:00:00 2001 From: David Gasquez Date: Mon, 8 Jan 2024 14:58:47 +0000 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=F0=9F=9A=9A=20update=20folder=20nam?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- .github/workflows/run.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 836dfea..75d738c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,9 +35,9 @@ jobs: { echo "| Filename | File Size |" echo "| --- | --- |" - ls -lh tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}' + ls -lh data/tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}' } >> $GITHUB_STEP_SUMMARY - ls -lh tables + ls -lh data/tables - name: Install Quarto uses: quarto-dev/quarto-actions/setup@v2 - name: Render diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 76ecf97..92668bc 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -37,9 +37,9 @@ jobs: { echo "| Filename | File Size |" echo "| --- | --- |" - ls -lh tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}' + ls -lh data/tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}' } >> $GITHUB_STEP_SUMMARY - ls -lh tables + ls -lh data/tables - name: Install Quarto uses: quarto-dev/quarto-actions/setup@v2 - name: Render