chore(database): move from planetsacle to rds sniff (#105)

* chore(database): move from planetsacle to rds sniff * fix: doppler install * fix: migration diff step * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests * fix: migration diff step n tests
tericcabrel · Sep 1, 2024 · be32440 · be32440
1 parent 2d2d8c3
commit be32440
Show file tree

Hide file tree

Showing 17 changed files with 168 additions and 260 deletions.
diff --git a/.github/workflows/migration-db-check.yml b/.github/workflows/migration-db-check.yml
@@ -6,93 +6,75 @@ on:
     paths:
       - 'packages/domain/prisma/migrations/**'
 
-env:
-  PLANETSCALE_SERVICE_TOKEN_ID: ${{ secrets.PLANETSCALE_SERVICE_TOKEN_ID }}
-  PLANETSCALE_SERVICE_TOKEN: ${{ secrets.PLANETSCALE_SERVICE_TOKEN }}
-
 jobs:
   migration-detail:
     runs-on: ubuntu-latest
+    env:
+      MYSQL_ROOT_PASSWORD: root
+      MYSQL_DATABASE: db
+      MYSQL_PORT: 3306
+      DATABASE_URL: mysql://root:root@127.0.0.1:3306/db
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup pscale
-        uses: planetscale/setup-pscale-action@v1
-
-      - name: Set database branch name
-        run: echo "PSCALE_BRANCH_NAME=$(echo ${{ github.head_ref }} | tr -cd '[:alnum:]-'| tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
-
-      - name: Create deploy request on development branch
-        run: |
-          DEPLOY_REQUEST_STATE=$(pscale deploy-request show ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.PSCALE_BRANCH_NAME }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json | jq -r '.state')
-
-          echo "Deploy request State: $DEPLOY_REQUEST_STATE"
-
-          if [ "$DEPLOY_REQUEST_STATE" = "open" ]; then
-            echo "Deploy request exists: skipping creation"
-          else
-            echo "Deploy request does not exist : creating..."
-            pscale deploy-request create ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.PSCALE_BRANCH_NAME }} \
-            --org ${{ secrets.PLANETSCALE_ORG_NAME }} \
-            --into dev \
-            --notes "Apply changes to the dev branch"
-          fi
+      - name: Start MySQL server
+        run: sudo systemctl start mysql.service
 
-      - name: Get deploy request number
-        run: |
-          DEPLOY_REQUEST_NUMBER=$(pscale deploy-request show ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.PSCALE_BRANCH_NAME }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json | jq -r '.number')
-          echo "DEPLOY_REQUEST_NUMBER=$DEPLOY_REQUEST_NUMBER" >> $GITHUB_ENV
+      - name: Install dependencies
+        uses: ./.github/actions/install-dependencies
 
-      - name: Check deployment state
-        continue-on-error: false
-        run: |
-          for i in {1..10}; do
-            DEPLOYMENT_STATE=$(pscale deploy-request show ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.DEPLOY_REQUEST_NUMBER }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} --format json | jq -r '.deployment_state')
-            echo "Deployment State: $DEPLOYMENT_STATE"
+      - name: Run migrations
+        run: yarn workspace @snipcode/domain db:migrate
 
-            if [ "$DEPLOYMENT_STATE" = "ready" ]; then
-              echo "Deployment state is ready. Continuing."
-              echo "DEPLOY_REQUEST_OPENED=true" >> $GITHUB_ENV
-              break
-            fi
+      - name: Install Doppler CLI
+        uses: dopplerhq/cli-action@v3
 
-            echo "Deployment state is not ready. Waiting 2 seconds before checking again."
-            sleep 2
-          done
+      - name: Retrieve and export the production database URL
+        run: echo "PROD_DATABASE_URL=$(doppler secrets get CONNECTION_STRING --plain)" >> $GITHUB_ENV
+        env:
+          DOPPLER_TOKEN: ${{ secrets.DOPPLER_DATABASE_TOKEN }}
 
       - name: Collect the migration diff
         continue-on-error: false
-        if: ${{ env.DEPLOY_REQUEST_OPENED }}
+        working-directory: packages/domain
         run: |
-          DEPLOY_DATA=$(pscale api organizations/${{ secrets.PLANETSCALE_ORG_NAME }}/databases/${{ secrets.PLANETSCALE_DATABASE_NAME }}/deploy-requests/${{ env.DEPLOY_REQUEST_NUMBER }}/deployment --org planetscale)
-          CAN_DROP_DATA=$(echo "$DEPLOY_DATA" | jq -r '.deploy_operations[] | select(.can_drop_data == true) | .can_drop_data')
-
-          echo "Deploy request opened: https://app.planetscale.com/${{ secrets.PLANETSCALE_ORG_NAME }}/${{ secrets.PLANETSCALE_DATABASE_NAME }}/deploy-requests/${{ env.DEPLOY_REQUEST_NUMBER }}" >> migration-message.txt
+          set -e
+          echo "### Database Migration Detected" > migration-message.txt
+          echo "The following migration will be applied to the production database:" >> migration-message.txt
           echo "" >> migration-message.txt
-
-          if [ "$CAN_DROP_DATA" = "true" ]; then
-            echo ":rotating_light: You are dropping a column. Before running the migration make sure to do the following:" >> migration-message.txt
-            echo "" >> migration-message.txt
-
-            echo "1. [ ] Deploy app changes to ensure the column is no longer being used." >> migration-message.txt
-            echo "2. [ ] Once you've verified it's no used, run the deploy request." >> migration-message.txt
-            echo "" >> migration-message.txt
-          else
-            echo "When adding to the schema, the Deploy Request must be run **before** the code is deployed." >> migration-message.txt
-            echo "Please ensure your schema changes are compatible with the application code currently running in production." >> migration-message.txt
-            echo "" >> migration-message.txt
-
-            echo "1. [ ] Successfully run the Deploy Request" >> migration-message.txt
-            echo "2. [ ] Deploy this PR" >> migration-message.txt
-            echo "" >> migration-message.txt
-          fi
-
-          echo "\`\`\`diff" >> migration-message.txt
-          pscale deploy-request diff ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.DEPLOY_REQUEST_NUMBER }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json | jq -r '.[].raw' >> migration-message.txt
+          echo "\`\`\`sql" >> migration-message.txt
+          yarn prisma migrate diff --from-url "$PROD_DATABASE_URL" --to-url "$DATABASE_URL" --script >> migration-message.txt
           echo "\`\`\`" >> migration-message.txt
+          echo "" >> migration-message.txt
+          echo "" >> migration-message.txt
+          echo "Please ensure your schema changes are compatible with the application code currently running in production." >> migration-message.txt
 
-      - name: Comment pull request with the migration diff
-        uses: thollander/actions-comment-pull-request@v2
-        if: ${{ env.DEPLOY_REQUEST_OPENED }}
+      - name: Create or update the pull request with the migration diff
+        uses: actions/github-script@v6
+        id: plan-comment
         with:
-          filePath: migration-message.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const botComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes('Database Migration Detected')
+            });
+            const output = fs.readFileSync('packages/domain/migration-message.txt', 'utf8');
+            if (botComment) {
+              github.rest.issues.deleteComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+              });
+            }
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: output
+            });
diff --git a/.github/workflows/migration-db-deploy.yml b/.github/workflows/migration-db-deploy.yml
@@ -6,85 +6,22 @@ on:
     paths:
       - 'packages/domain/prisma/migrations/**'
 
-env:
-  PLANETSCALE_SERVICE_TOKEN_ID: ${{ secrets.PLANETSCALE_SERVICE_TOKEN_ID }}
-  PLANETSCALE_SERVICE_TOKEN: ${{ secrets.PLANETSCALE_SERVICE_TOKEN }}
-
 jobs:
   apply-migration:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup pscale
-        uses: planetscale/setup-pscale-action@v1
-
-      - name: Get the deploy request number and branch name in development branch
-        run: |
-          DEPLOYER_REQUEST_LIST=$(pscale deploy-request list ${{ secrets.PLANETSCALE_DATABASE_NAME }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json)
-          DEV_DEPLOY_REQUEST_NUMBER=$(echo "$DEPLOYER_REQUEST_LIST" | jq -r '.[] | select(.into_branch == "dev" and .state == "open") | .number')
-          PSCALE_BRANCH_NAME=$(echo "$DEPLOYER_REQUEST_LIST" | jq -r '.[] | select(.into_branch == "dev" and .state == "open") | .branch')
-
-          if [ -z "$DEV_DEPLOY_REQUEST_NUMBER" ]; then
-            echo "No open deploy request found in the development branch."
-            exit 1
-          fi
-          echo "DEV_DEPLOY_REQUEST_NUMBER=$DEV_DEPLOY_REQUEST_NUMBER" >> $GITHUB_ENV
-          echo "PSCALE_BRANCH_NAME=$PSCALE_BRANCH_NAME" >> $GITHUB_ENV
-
-      - name: Check deployment state
-        continue-on-error: false
-        run: |
-          for i in {1..10}; do
-            DEPLOYMENT_STATE=$(pscale deploy-request show ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.DEV_DEPLOY_REQUEST_NUMBER }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json | jq -r '.deployment_state')
-            echo "Deployment State: $DEPLOYMENT_STATE"
-
-            if [ "$DEPLOYMENT_STATE" = "ready" ]; then
-              echo "Deployment state is ready. Continuing."
-              echo "DEPLOY_REQUEST_OPENED=true" >> $GITHUB_ENV
-              break
-            fi
-
-            echo "Deployment state is not ready. Waiting 2 seconds before checking again."
-            sleep 2
-          done
-
-      - name: Deploy schema migration in the development branch
-        continue-on-error: false
-        run: |
-          pscale deploy-request deploy ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.DEV_DEPLOY_REQUEST_NUMBER }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} --wait
-          pscale deploy-request skip-revert ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.DEV_DEPLOY_REQUEST_NUMBER }} --org ${{ secrets.PLANETSCALE_ORG_NAME }}
-
-      - name: Deploy schema migration in the production branch
-        if: ${{ success() }}
-        continue-on-error: false
-        run: |
-          pscale deploy-request create ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.PSCALE_BRANCH_NAME }} \
-          --org ${{ secrets.PLANETSCALE_ORG_NAME }} \
-          --into main \
-          --notes "Apply changes to the production branch"
-
-          PROD_DEPLOY_REQUEST_NUMBER=$(pscale deploy-request show ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.PSCALE_BRANCH_NAME }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json | jq -r '.number')
-
-          for i in {1..10}; do
-            DEPLOYMENT_STATE=$(pscale deploy-request show ${{ secrets.PLANETSCALE_DATABASE_NAME }} $PROD_DEPLOY_REQUEST_NUMBER --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json | jq -r '.deployment_state')
-            echo "Deployment State: $DEPLOYMENT_STATE"
-
-            if [ "$DEPLOYMENT_STATE" = "ready" ]; then
-              echo "Deployment state is ready. Continuing."
-              break
-            fi
+      - name: Install dependencies
+        uses: ./.github/actions/install-dependencies
 
-            echo "Deployment state is not ready. Waiting 2 seconds before checking again."
-            sleep 2
-          done
+      - name: Install Doppler CLI
+        run: curl -Ls https://cli.doppler.com/install.sh | sh
 
-          pscale deploy-request deploy ${{ secrets.PLANETSCALE_DATABASE_NAME }} $PROD_DEPLOY_REQUEST_NUMBER --org ${{ secrets.PLANETSCALE_ORG_NAME }} --wait
+      - name: Retrieve and export the database URL
+        run: echo "DATABASE_URL=$(doppler secrets get CONNECTION_STRING --plain)" >> $GITHUB_ENV
+        env:
+          DOPPLER_TOKEN: ${{ secrets.DOPPLER_DATABASE_TOKEN }}
 
-      - name: Delete the database branch
-        if: ${{ success() }}
-        run: |
-          PROD_DEPLOY_REQUEST_NUMBER=$(pscale deploy-request show ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.PSCALE_BRANCH_NAME }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} -f json | jq -r '.number')
-          pscale deploy-request skip-revert ${{ secrets.PLANETSCALE_DATABASE_NAME }} $PROD_DEPLOY_REQUEST_NUMBER --org ${{ secrets.PLANETSCALE_ORG_NAME }}
-          pscale branch delete ${{ secrets.PLANETSCALE_DATABASE_NAME }} ${{ env.PSCALE_BRANCH_NAME }} --org ${{ secrets.PLANETSCALE_ORG_NAME }} --force
-          echo "The branch \"${{ env.PSCALE_BRANCH_NAME }}\" has been successfully."
+      - name: Deploy schema migration to the production database
+        run: yarn workspace @snipcode/domain db:deploy
diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@ Snipcode is an open-source code-sharing platform that makes it easy to create co
 * [Node.js](https://nodejs.org/en/)
 * [TypeScript](https://www.typescriptlang.org/)
 * [GraphQL](https://graphql.org/)
-* [MySQL 8 with PlanetScale](https://planetscale.com/)
+* [MySQL 8.0](https://dev.mysql.com/doc/refman/8.0/en/)
 * [Prisma](https://www.prisma.io/)
 * [Tailwind CSS](https://tailwindcss.com/)
 * [AWS](https://aws.amazon.com)
@@ -29,11 +29,9 @@ snipcode
 │  ├─ backend
 │  ├─ web
 ├─ packages
-│  ├─ database
 │  ├─ embed
 │  ├─ domain
 │  ├─ front
-│  ├─ logger
 │  ├─ utils
 package.json
 tsconfig.base.json

diff --git a/_infra/credentials/database_prod.tf b/_infra/credentials/database_prod.tf
@@ -24,6 +24,7 @@ variable "database_prod_secrets_map" {
     "ADMIN_PASSWORD" = "",
     "DATABASE_NAME" = "",
     "PORT" = "3306"
+    "CONNECTION_STRING" = ""
   }
 }
 

diff --git a/apps/backend/.env.example b/apps/backend/.env.example
@@ -3,7 +3,7 @@ APP_VERSION=1.0.0
 HOST=http://localhost
 PORT=7501
 INTROSPECTION_ENABLED=true
-DATABASE_URL="mysql://root:@127.0.0.1:3311/snipcode"
+DATABASE_URL=mysql://root:secret@127.0.0.1:3311/snipcode
 ADMIN_PASSWORD=nwHSvXuQxjey43Bp
 CONVERTKIT_API_KEY=convertKitApiKey
 CONVERTKIT_FORM_ID=formId

diff --git a/apps/backend/README.md b/apps/backend/README.md
@@ -1,4 +1,4 @@
-# Snipcode Core
+# Snipcode Backend
 
 This is the backend of Snipcode, containing the business logics related to  . 
 
@@ -15,7 +15,7 @@ Make sure you have these tools installed before running the project
 * NPM or Yarn
 * Docker
 * AWS CLI v2
-* MySQL 8 on PlanetScale
+* MySQL 8.0
 
 ## Packages dependencies
 We use Yarn workspace to create packages we can share with other applications.

diff --git a/apps/backend/package.json b/apps/backend/package.json
@@ -28,7 +28,7 @@
     "@nestjs/graphql": "12.1.1",
     "@nestjs/platform-express": "10.3.9",
     "@nestjs/serve-static": "4.0.2",
-    "@prisma/client": "5.14.0",
+    "@prisma/client": "5.19.0",
     "@sentry/node": "8.11.0",
     "@snipcode/domain": "workspace:*",
     "@snipcode/embed": "workspace:*",

diff --git a/apps/backend/src/utils/tests/helpers.ts b/apps/backend/src/utils/tests/helpers.ts
@@ -47,7 +47,9 @@ export class TestHelper {
     await prismaService.snippet.deleteMany();
 
     // Recursive relationship between folders makes it hard to delete all folders using folder.deleteMany()
+    await prismaService.$executeRaw`SET FOREIGN_KEY_CHECKS=0;`;
     await prismaService.$executeRaw`TRUNCATE TABLE folders;`;
+    await prismaService.$executeRaw`SET FOREIGN_KEY_CHECKS=1;`;
 
     await prismaService.session.deleteMany();
 

diff --git a/packages/domain/.env.template b/packages/domain/.env.template
@@ -1,2 +1,2 @@
-DATABASE_URL=mysql://root:@127.0.0.1:3311/core-db
+DATABASE_URL=mysql://root:secret@127.0.0.1:3311/snipcode
 SHADOW_DATABASE_URL=mysql://root:secret@127.0.0.1:3312/snipcode
diff --git a/packages/domain/README.md b/packages/domain/README.md
@@ -11,7 +11,6 @@ Make sure you have this tools installed before running the project
 * Node.js 20+
 * Yarn 4
 * Docker
-* [The PlanetScale CLI](https://planetscale.com/cli)
 
 ## Set up the project
 Delete the existing folders output from build commands
@@ -29,42 +28,23 @@ Create the .env file from the template. This file is useful for test executions,
 cp .env.template .env
 ```
 
-Create git branch for your feature; there is no convention for the branch name.
+### Start the local database and run the migrations
 ```shell
-git checkout -b your-feature-name
-```
-
-### Connect to the local database locally
-This project uses PlanetScale as the database.<br>
-To connect to the database locally, you must authenticate first (Ask the credentials to [@tericcabrel](https://github.com/tericcabrel)).<br>
-Once authenticated from the terminal, execute the commands below to create a database branch for your feature and create local tunnel to it.
-```shell
-yarn db:branch:dev
-yarn db:branch:connect
-```
-Open a second terminal and run the command below to start the shadow database; 
-this is only necessary to run generate or execute database migrations with prisma schema
-```shell
-yarn db:shadow
+yarn db:local
+yarn db:migrate
 ```
 
-The shadow database runs on Docker, you can stop it when you don't need it
+### Generate Prisma types and seed the database with default data
 ```shell
-yarn db:shadow:stop
+yarn db:generate
+yarn db:seed
 ```
 
-### Connect to the development database
-The development database is used to beta test features. Execute the command below to connect to it
+To stop the local database run the command below:
 ```shell
-yarn db:dev:connect
+yarn db:local:stop
 ```
 
-### Run the database migration, generate Prisma types and seed the database with default data
-```shell
-yarn db:generate
-yarn db:migrate
-yarn db:seed
-```
 
 ### Generate a database migration
 To create a database migration that generate the SQL file, run the command below:
@@ -78,13 +58,7 @@ yarn db:migrate --name <migration-name>
 - Open Prisma Studio to browse your database: `db:view`
 - Lint the Prisma schema file: `db:format`
 
-### Create a deployment request
-To publish the database schema changes in production, you must create a deployment request. Run the command below to do that
-```shell
-yarn db:deploy:create
-```
-
-Build the package to generate types declaration required to provide autocompletion while using the functions in the core or Lambda functions
+Build the package to generate types declaration required to provide autocompletion while using the functions in the backend application
 ```bash
 yarn build
 ```