Merge pull request #496 from bcnmy/INF-259/create-cicd-pipeline-with-…

…github-actions-to-deploy-to-new-bunder

Add github Actions CI/CD pipelines

.github/workflows/merge_on_master_tw.yaml

@@ -0,0 +1,66 @@
+# This workflow will do a clean installation of node dependencies, build the 
+# source code and run tests across different versions of node
+# For more information see: 
+# https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Main_or_Master
+on:
+# The trigger event is for every pull request. If we would trigger when we push
+# to any branch and on pull request we would have double workflow runs and will
+# consume more minutes. I chose to trigger the workflow run on pull requests only.
+  push:
+    branches:
+      - 'dedicated-bundler-setup_new'
+
+jobs:
+  js_build_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+        # node-version: [14.x, 16.x, 18.x, 20.x]
+        # See supported Node.js release schedule at
+        # https://nodejs.org/en/about/releases/
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Yarn install, build and test
+        run: echo "In the steps below we will build and run first set of tests for all components"
+      - run: echo yarn install
+      - run: echo yarn lint
+      - run: echo yarn build
+      - run: echo yarn test
+
+  container_img_build_push_gar:
+    needs: [js_build_test]
+    # Allow the job to fetch a GitHub ID token
+    permissions:
+      id-token: write
+      contents: read
+    # The plan is to build and push each docker image in parallel.
+    strategy:
+      matrix:
+        image:
+          - us-docker.pkg.dev/biconomy-prod/bundler/trustwallet
+          - us-docker.pkg.dev/prj-biconomy-prod-001/bundler/bundler
+          # LOCATION-docker.pkg.dev/PROJECT-ID/REPOSITORY/IMAGE
+          # {owner}/{repo}/.github/workflows/{filename}@{ref}
+    uses: bcnmy/devops/.github/workflows/container_img_build_push_gar.yaml@master
+    with:
+      image: ${{ matrix.image }}
+      dockerfile: Dockerfile.helm
+      # GCP project where the identity provider is
+      # gcloud projects describe prj-workload-identity-001
+      gcp_project_number: '766873424314'
+      gcp_pool_id: 'pool-id-github-actions'
+      # gcp_provider_id: 'ga-GITHUB_REPO_NAME'
+      gcp_provider_id: 'ga-bundler'
+      # LOCATION-docker.pkg.dev/PROJECT-ID/REPOSITORY/IMAGE
+      gcp_registry: 'us-docker.pkg.dev/biconomy-prod/bundler/trustwallet'
+      gcp_service_account: 'sa-bundler@prj-workload-identity-001.iam.gserviceaccount.com'
+
+  # TODO: Add integrations tests here

.github/workflows/pr.yaml

@@ -0,0 +1,64 @@
+---
+# This workflow will do a clean installation of node dependencies, build the
+# source code and run tests across different versions of node
+# For more information see:
+# https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+# yamllint disable rule:line-length
+name: PR
+on:
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  js_build_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+        # node-version: [14.x, 16.x, 18.x, 20.x]
+        # See supported Node.js release schedule at
+        # https://nodejs.org/en/about/releases/
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Yarn install, build and test
+        run: echo "In the steps below we will build and run first set of tests for all components"
+      - run: echo yarn install
+      - run: echo yarn lint
+      - run: echo yarn build
+      - run: echo yarn test
+
+  container_img_build_push_gar:
+    needs: [js_build_test]
+    # Allow the job to fetch a GitHub ID token
+    permissions:
+      id-token: write
+      contents: read
+    # The plan is to build and push each docker image in parallel.
+    strategy:
+      matrix:
+        image:
+          - us-docker.pkg.dev/biconomy-prod/bundler/trustwallet
+          - us-docker.pkg.dev/prj-biconomy-prod-001/bundler/bundler
+          # LOCATION-docker.pkg.dev/PROJECT-ID/REPOSITORY/IMAGE
+          # {owner}/{repo}/.github/workflows/{filename}@{ref}
+    uses: bcnmy/devops/.github/workflows/container_img_build_push_gar.yaml@master
+    with:
+      image: ${{ matrix.image }}
+      dockerfile: Dockerfile.helm
+      # GCP project where the identity provider is
+      # gcloud projects describe prj-workload-identity-001
+      gcp_project_number: '766873424314'
+      gcp_pool_id: 'pool-id-github-actions'
+      # gcp_provider_id: 'ga-GITHUB_REPO_NAME'
+      gcp_provider_id: 'ga-bundler'
+      # LOCATION-docker.pkg.dev/PROJECT-ID/REPOSITORY/IMAGE
+      gcp_registry: 'us-docker.pkg.dev/biconomy-prod/bundler/trustwallet'
+      gcp_service_account: 'sa-bundler@prj-workload-identity-001.iam.gserviceaccount.com'
+      #

.github/workflows/release.yaml

@@ -0,0 +1,125 @@
+---
+# This workflow will build a docker image and deploy it to trustwallet stating \
+# and production environments
+
+# yamllint disable rule:line-length
+name: Version_Release
+on:
+  push:
+    tags:
+      - v0.**
+jobs:
+  js_build_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+        # node-version: [14.x, 16.x, 18.x, 20.x]
+        # See supported Node.js release schedule at
+        # https://nodejs.org/en/about/releases/
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Yarn install, build and test
+        run: echo "In the steps below we will build and run first set of tests for all components"
+      - run: echo yarn install
+      - run: echo yarn lint
+      - run: echo yarn build
+      - run: echo yarn test
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Dump GITHUB_REF
+        run: echo "${GITHUB_REF}"
+
+  container_img_build_push_gar:
+    needs: [js_build_test]
+    # Allow the job to fetch a GitHub ID token
+    permissions:
+      id-token: write
+      contents: read
+    # The plan is to build and push each docker image in parallel.
+    strategy:
+      matrix:
+        image:
+          - us-docker.pkg.dev/biconomy-prod/bundler/trustwallet
+          - us-docker.pkg.dev/prj-biconomy-prod-001/bundler/bundler
+          # LOCATION-docker.pkg.dev/PROJECT-ID/REPOSITORY/IMAGE
+          # {owner}/{repo}/.github/workflows/{filename}@{ref}
+    uses: bcnmy/devops/.github/workflows/container_img_build_push_gar.yaml@master
+    with:
+      image: ${{ matrix.image }}
+      dockerfile: Dockerfile.helm
+      # GCP project where the identity provider is
+      # gcloud projects describe prj-workload-identity-001
+      gcp_project_number: '766873424314'
+      gcp_pool_id: 'pool-id-github-actions'
+      # gcp_provider_id: 'ga-GITHUB_REPO_NAME'
+      gcp_provider_id: 'ga-bundler'
+      # LOCATION-docker.pkg.dev/PROJECT-ID/REPOSITORY/IMAGE
+      gcp_registry: 'us-docker.pkg.dev/biconomy-prod/bundler/trustwallet'
+      gcp_service_account: 'sa-bundler@prj-workload-identity-001.iam.gserviceaccount.com'
+
+  deploy_tw_staging:
+    needs: [container_img_build_push_gar]
+    # Allow the job to fetch a GitHub ID token
+    # runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    uses: bcnmy/devops/.github/workflows/deploy_to_gke.yaml@master
+    with:
+      # GCP project where the identity provider is
+      # gcloud projects describe prj-workload-identity-001
+      gcp_project_number: '766873424314'
+      gcp_project_id: 'biconomy-prod'
+      gcp_bastion: 'bastion02'
+      gcp_bastion_zone: 'us-east1-b'
+      gcp_pool_id: 'pool-id-github-actions'
+      # created by devops/gcp/github-actions/configure_workload_identity_federation_with_github_actions_pipelines.sh
+      # gcp_provider_id: 'ga-GITHUB_REPO_NAME'
+      gcp_provider_id: 'ga-bundler'
+      # SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT}@${PROJECT}.iam.gserviceaccount.com"
+      gcp_service_account: 'sa-bundler@prj-workload-identity-001.iam.gserviceaccount.com'
+      gcp_cluster_name: 'dedicated-bundler'
+      gcp_cluster_location: 'us-east1'
+      use_internal_ip: true
+      # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+      # deploy_command: 'helm ls --all-namespaces'
+      # deploy_command: 'echo IMG_VERSION is ${GITHUB_REF:10}' # for example extracts v0.0.5 from "refs/tags/v0.0.5"  
+      deploy_command: './install-bundler/bundler-update-release.sh bundler-tw-staging.cfg ${GITHUB_REF:10}'
+
+  deploy_tw_prod:
+    needs: [deploy_tw_staging]
+    # environment: tw-prod
+    # Allow the job to fetch a GitHub ID token
+    # runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    uses: bcnmy/devops/.github/workflows/deploy_to_gke.yaml@master
+    with:
+      environment: 'tw-prod'
+      # GCP project where the identity provider is
+      # gcloud projects describe prj-workload-identity-001
+      gcp_project_number: '766873424314'
+      # GCP project ID where the workload will be deployed
+      gcp_project_id: 'prj-biconomy-prod-001'
+      gcp_bastion: 'bastion02'
+      gcp_bastion_zone: 'us-central1-a'
+      gcp_pool_id: 'pool-id-github-actions'
+      # created by devops/gcp/github-actions/configure_workload_identity_federation_with_github_actions_pipelines.sh
+      # gcp_provider_id: 'ga-GITHUB_REPO_NAME'
+      gcp_provider_id: 'ga-bundler'
+      # SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT}@${PROJECT}.iam.gserviceaccount.com"
+      gcp_service_account: 'sa-bundler@prj-workload-identity-001.iam.gserviceaccount.com'
+      gcp_cluster_name: 'trustwallet'
+      gcp_cluster_location: 'us-central1'
+      use_internal_ip: true
+      deploy_command: './install-bundler/bundler-update-release.sh bundler-tw-production.cfg ${GITHUB_REF:10}'
+      # deploy_command: 'helm ls --all-namespaces'

.gitignore

@@ -20,6 +20,5 @@ refundAddresses.ts
 # However, if it's actually index.json, include it like so:
 #!index.json
 
-*.cfg
 !example.cfg
 

README.md

@@ -123,4 +123,127 @@ To update the config.json.enc file run ts-node encrypt-config.ts
 ```jsx
 yarn run dev
 ```
+## Install bundler for the first time in a new k8s cluster
+Long story short is that you need to run 
+`bundler/install-bundler/bundler-initial-setup.sh`
+
+### 1. Manual step - create `config/config.json` ;
+
+content should look like
+```
+{
+  "relayerManagers": [
+    {
+      "relayerSeed": "******",
+      "ownerPublicKey": "0x******",
+      "ownerPrivateKey": "0x0e1*****",
+      "ownerAccountDetails": {
+        "137": {
+          "publicKey": "0x*****",
+          "privateKey": "0x*******"
+        }
+      }
+    }
+  ]
+}
+```
+
+### 2. Manual step - create `config.json.enc` at the root of the repo;<br>
+this is done by running `encrypt-config.ts`; <br>
+example:<br>
+install `ts-node`
+```bash
+npm install --location=global ts-node typescript
+```
+create a test file in `config` dir
+
+```bash
+cat config/radu-test-config.json
+'{"a": "b"}'
+```
+set the passphares
+```bash
+export CONFIG_PASSPHRASE=secret 
+```
+finally run `encrypt-config.ts`
+```
+ts-node encrypt-config.ts radu-test-config.json
+```
+
+to decrypt:
+```bash
+cat radu-test-config.json.enc
+xPJ9gyGx7U5q+NAoKswoGM28dDYEICBRUWcczvgI7aE=422142eae514c31c05e345669e56b4e9141649c92d60f3b83ab02c22b072cf76dSsomU6+8Fb6EgIFBoSAIQ==
+
+ts-node decrypt-config.ts radu-test-config.json.enc
+'{"a": "b"}'
+```
+
+- save the password that was used `config.json.enc` ; the script will ask 
+for it ;
+
+### 3. create a config file that can be saved in this repo `install-bundler/configs`
+e.g. `install-bundler/configs/bundler-tw-staging.cfg`
+```
+NAMESPACE="bundler-tw-staging"
+PROJECT_ID="biconomy-prod"
+IMAGE="us-docker.pkg.dev/biconomy-prod/bundler/trustwallet"
+IMAGE_TAG="7064007"
+DNS_NAME="bundler-tw-staging.biconomy.io"
+IP_NAME="bundler-tw-staging"
+CHAINS_CFG_FILENAME="staging-trust-wallet-chains.sh"
+CONTEXT="gke_biconomy-prod_us-east1_dedicated-bundler"
+```
+
+### 4. Run `bundler-initial-setup.sh`
+This will install all dependency apps required to run the bundler.
+It will also create secrets in GCP Secret Manager.
+```bash
+./bundler-initial-setup.sh configs/bundler-tw-staging.cfg
+```
+Output saved in `doc/bundler-tw-staging.md`
+
+### 5. Update GCP secret containing required configuration secret values.
+By running the script in previous step you have created a secret that contains
+configuration information based on this pattern. <br>
+```bash
+TRIMMED_CLUSTER_NAME=$(echo "$CLUSTER_NAME" | cut -c 1-8)
+TRIMMED_NAMESPACE=$(echo "$NAMESPACE" | cut -c 1-10)
+GCP_PLAINTEXT_CONFIG_SECRET="$TRIMMED_CLUSTER_NAME-$TRIMMED_NAMESPACE-cfg-plain"
+```
+
+Use console.cloud.google.com UI to update the values of the secret.
+Content of the secret should look like:
+```bash
+SIMULATION_DATA_JSON=<value>
+TOKEN_PRICE_JSON=<value>
+SLACK_JSON=<value>
+PROVIDER_JSON=<value>
+DATASOURCES_JSON=<value>
+SOCKET_SERVICE_JSON=<value>
+QUEUE_URL=<value>
+```
+
+### 6. Add DNS record in cloudflare
+The script will tell you what DNS record you need to add in cloudflare and
+for which IP.
+
+### 7. Deploy the bundler.
+
+Dependency: `configs/bundler-tw-staging.cfg` based on `configs/example.cfg`
+
+```bash
+./bundler-update-release.sh configs/bundler-tw-staging.cfg
+```
+
+## CI/CD with Github Actions
+
+To create the GCP workload identity service account 
+  - go to https://github.com/bcnmy/devops/tree/master/gcp/github-actions
+  - create new sh file `configure_bcnmy_<github_repo_name>`;
+    e.g. `configure_bcnmy_bundler.sh`
+  - run the new script
+  - get value of service account
+    e.g. `sa-bundler@prj-workload-identity-001.iam.gserviceaccount.com`
+