feat: ⚡️ render Quarto on CI

.github/workflows/ci.yml

@@ -30,11 +30,16 @@ jobs:
 
     - name: Export Database and Log File Sizes
       run: |
-        python -c 'from ggdp import db; db.export_database_to_parquet("data/local.duckdb", "tables");'
+        make tables
         echo "Files exported:" >> $GITHUB_STEP_SUMMARY
         {
           echo "| Filename | File Size |"
           echo "| --- | --- |"
-          ls -lh tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}'
+          ls -lh data/tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}'
         } >> $GITHUB_STEP_SUMMARY
-        ls -lh tables
+        ls -lh data/tables
+    - name: Install Quarto
+      uses: quarto-dev/quarto-actions/setup@v2
+    - name: Render
+      run: |
+        make render

.github/workflows/run.yml

@@ -32,27 +32,34 @@ jobs:
 
     - name: Export Database and Log File Sizes
       run: |
-        python -c 'from ggdp import db; db.export_database_to_parquet("data/local.duckdb", "tables");'
+        make tables
         echo "Files exported:" >> $GITHUB_STEP_SUMMARY
         {
           echo "| Filename | File Size |"
           echo "| --- | --- |"
-          ls -lh tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}'
+          ls -lh data/tables | tail -n +2 | awk '{printf "| %s | %s |\n", $9, $5}'
         } >> $GITHUB_STEP_SUMMARY
-        ls -lh tables
-
+        ls -lh data/tables
+    - name: Install Quarto
+      uses: quarto-dev/quarto-actions/setup@v2
+    - name: Render
+      run: |
+        make render
+    - name: Copy Portal
+      run: |
+        cp -r portal/_site data/tables/portal
     - uses: filebase/ipfs-action@master
       id: ipfs
       with:
-        path: ./tables
+        path: ./data/tables
         service: filebase
         pinName: gitcoin-grants-data-portal-tables
         filebaseBucket: gitcoin-grants-data-portal
         filebaseKey: ${{ secrets.FILEBASE_KEY }}
         filebaseSecret: ${{ secrets.FILEBASE_SECRET }}
         key: gitcoin-grants-data-portal-db
 
-    - name: Create CID file
+    - name: Update CID file
       run: echo ${{ steps.ipfs.outputs.cid }} > data/IPFS_CID
 
     - name: Commit and Push

Makefile

@@ -3,6 +3,9 @@
 run:
 	@dagster asset materialize --select \* -m ggdp
 
+tables:
+	@python -c 'from ggdp import db; db.export_database_to_parquet("data/local.duckdb", "data/tables");'
+
 dev:
 	@dagster dev -m ggdp
 
@@ -13,11 +16,15 @@ preview:
 	@quarto preview portal
 
 render:
+	@cd dbt && dbt docs generate
+	@mkdir -p dbt/target/docs
+	@cp dbt/target/*.json dbt/target/index.html dbt/target/graph.gpickle dbt/target/docs/
 	@quarto render portal
+	@cp -r dbt/target/docs/ portal/_site/dbt
 
 publish:
 	@quarto publish gh-pages portal --no-prompt
 
 clean:
-	@rm -rf portal/.quarto data/*.parquet data/*.duckdb
+	@rm -rf portal/.quarto data/*.parquet data/*.duckdb portal/_site
 	@rm -rf dbt/target dbt/logs dbt/dbt_packages

portal/_quarto.yml

@@ -13,6 +13,8 @@ website:
         text: Notebook
       - href: catalog.ipynb
         text: Catalog
+      - href: dbt
+        text: dbt Docs
 
 format:
   html:

portal/catalog.ipynb

@@ -286,8 +286,8 @@
     "\n",
     "import os\n",
     "\n",
-    "# Make sure to `make run` first so `tables` folder is actually populated.\n",
-    "EXPORT_PATH = os.path.join(\"..\", \"tables\")\n",
+    "# Make sure to `make export-db`\n",
+    "EXPORT_PATH = os.path.join(\"..\", \"data/tables\")\n",
     "EXTENSIONS = \".parquet\"\n",
     "\n",
     "raw_files = []\n",