continuous.yaml

name: Continuous
on:
  push:
  pull_request:

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

jobs:
  duplication-check:
    runs-on: ubuntu-latest
    outputs:
      inPR: ${{ steps.check.outputs.number != '' }}
    steps:
      - name: Check if push is in PR
        id: check
        uses: 8BitJonny/gh-get-current-pr@2.2.0
        with:
          filterOutClosed: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
  build-generic:
    permissions:
      contents: 'read'
      id-token: 'write'
    runs-on: ubuntu-latest
    needs: duplication-check
    if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && needs.duplication-check.outputs.inPR == 'false' ) }}
    strategy:
      matrix:
        app: [ web, node ]
    steps:
      - uses: actions/checkout@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v1
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Login to GCR
        uses: docker/login-action@v1
        with:
          registry: gcr.io
          username: oauth2accesstoken
          password: '${{ steps.auth.outputs.access_token }}'
      - name: Login to GAR
        uses: docker/login-action@v1
        with:
          registry: us-east1-docker.pkg.dev
          username: oauth2accesstoken
          password: '${{ steps.auth.outputs.access_token }}'
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
      - name: Generate image metadata
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: |
            gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
            type=sha
          flavor: |
            latest=true
      - name: build and push
        uses: docker/build-push-action@v3
        with:
#         cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache
#         cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache, mode=max
          context: .
          push: true
          build-args: |
            TYPE=build
          file: ./build/${{ matrix.app }}/Dockerfile
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
  build-derived:
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
      id-token: 'write'
    needs:
      - duplication-check
      - build-generic
    strategy:
      matrix:
        app: [ asset, linker ]
    if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && needs.duplication-check.outputs.inPR == 'false' ) }}
    steps:
      - uses: actions/checkout@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v1
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Login to GCR
        uses: docker/login-action@v1
        with:
          registry: gcr.io
          username: oauth2accesstoken
          password: '${{ steps.auth.outputs.access_token }}'
      - name: Login to GAR
        uses: docker/login-action@v1
        with:
          registry: us-east1-docker.pkg.dev
          username: oauth2accesstoken
          password: '${{ steps.auth.outputs.access_token }}'
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
      - name: Generate image metadata
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: |
            gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
            type=sha
          flavor: |
            latest=true
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: build and push
        uses: docker/build-push-action@v3
        with:
#         cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache
#         cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache,mode=max
          context: .
          push: true
          build-args: |
            SRC_IMG=us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-web-${{ steps.branch-name.outputs.current_branch }}:sha-${{ steps.get-sha.outputs.sha_short }}
          file: ./build/${{ matrix.app }}/Dockerfile
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
#######
# Below Tests only run on PR
#######
  jest-tests:
    name: "Jest"   # This name is referenced when slacking status
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'pull_request' }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@v2
      - name: Install dependencies
        run: |
          npm install
          npm run build-prod
      - run: ls
      - run: pwd
      - name: Run Jest Tests
        run: npm run jest-gha
      - name: Handle Jest Test Results
        run: cat /home/runner/jestResults.json; STATUS=`jq ".numFailedTestSuites" /home/runner/jestResults.json`; exit $STATUS
        if: ${{ always() }}
  sandbox-deploy:
    concurrency:
      group: dev-mongo
      cancel-in-progress: false
    if: ${{ github.event_name == 'pull_request' }}
    runs-on: ubuntu-latest
    needs: build-derived
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - name: Checkout Code
        uses: actions/checkout@v2
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v1
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Setup GCloud
        uses: google-github-actions/setup-gcloud@v0
        with:
          project_id: ${{ secrets.DEV_PROJECT }}
          install_components: 'gke-gcloud-auth-plugin'
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: Set up yq
        uses: frenck/action-setup-yq@v1
          #      - name: setup helm
          #        uses: azure/setup-helm@v3
      - name: Authenticate GHA Runner To Target Cluster
        uses: google-github-actions/get-gke-credentials@v0
        with:
          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
          location: ${{secrets.DEV_GKE_REGION}}
          project_id: ${{secrets.DEV_GCP_PROJECT}}
      - name: Deploy Sandbox
        run: ./build/ci/sandbox-helm-deploy.sh build/ci/sandbox-values.yaml
        env:
          GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}"
          BRANCH: "${{ steps.branch-name.outputs.current_branch }}"
          PROJECT_ID: "${{ secrets.DEV_PROJECT }}"
          NAMESPACE: "${{secrets.DEV_SANDBOX_NAMESPACE}}"
  sandbox-ready:
    if: ${{ github.event_name == 'pull_request' }}
    needs: sandbox-deploy
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: Wait for test sandbox to become available
        # https://gist.github.com/rgl/f90ff293d56dbb0a1e0f7e7e89a81f42
        run: ./build/ci/waitForSandbox.bash
        env:
          WAIT_DURATION: "3000"
          GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}"
  pytest-job:
    concurrency:
      group: dev-mongo
      cancel-in-progress: false
    if: ${{ github.event_name == 'pull_request' }}
    name: "PyTest"  # This name is referenced when slacking status
    needs:
      - sandbox-ready
    permissions:
      contents: 'read'
      id-token: 'write'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v1
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Setup GCloud
        uses: google-github-actions/setup-gcloud@v0
        with:
          project_id: ${{ secrets.DEV_PROJECT }}
          install_components: 'gke-gcloud-auth-plugin'
      - name: Authenticate GHA Runner To Target Cluster
        uses: google-github-actions/get-gke-credentials@v0
        with:
          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
          location: ${{secrets.DEV_GKE_REGION}}
          project_id: ${{secrets.DEV_GCP_PROJECT}}
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Start Job
        run: ./build/ci/createJobFromRollout.sh $GITHUB_RUN_ID $DEPLOY_ENV
        env:
          # dependent on GITHUB_RUN_ID, which is implicitly passed in
          DEPLOY_ENV: sandbox-${{ steps.get-sha.outputs.sha_short }}
      - name: Wait For Job To Finish
        run: ./build/ci/waitForCIJob.bash
        timeout-minutes: 60
        env:
          # dependent on GITHUB_RUN_ID, which is implicitly passed in
          TEST_NAME: pytest
      - name: Get Logs From Cluster and propogate test result
        run: "kubectl logs --tail=-1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest; LASTLINE=`kubectl logs --tail=1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest`; STAT=${LASTLINE: -1}; exit $STAT"
      - name: Cleanup pyTest Pod
        run: kubectl delete jobs -l ci-run=$GITHUB_RUN_ID,test-name=pytest
        if: always()
  ending-notification:
    runs-on: ubuntu-latest
    if: ${{ always() && github.event_name == 'pull_request' }}
    needs:
      - pytest-job
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-node@v2
        with:
          node-version: '14'
      - run: npm ci
        working-directory: ./build/notify
      - run: node notifyEnd.js
        working-directory: ./build/notify
        env:
          SLACK_TEST_SUCCESS_WEBHOOK_URL: ${{secrets.SLACK_TEST_SUCCESS_WEBHOOK_URL}}
          SLACK_TEST_FAILURE_WEBHOOK_URL: ${{secrets.SLACK_TEST_FAILURE_WEBHOOK_URL}}
          GITUSER_SLACK_MAP: ${{secrets.GITUSER_SLACK_MAP}}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  sandbox-cleanup:
    name: "Clean up CI Sandbox"
    if: ${{ always() && github.event_name == 'pull_request' }}
    needs:
      - pytest-job
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v1
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Setup GCloud
        uses: google-github-actions/setup-gcloud@v0
        with:
          project_id: ${{ secrets.DEV_PROJECT }}
          install_components: 'gke-gcloud-auth-plugin'
      - name: Authenticate GHA Runner To Target Cluster
        uses: google-github-actions/get-gke-credentials@v0
        with:
          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
          location: ${{secrets.DEV_GKE_REGION}}
          project_id: ${{secrets.DEV_GCP_PROJECT}}
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: check helm
        id: get-helm
        run: echo "count=$(helm list -n $NAMESPACE | grep $NAME | wc -l)" >> $GITHUB_OUTPUT
        env:
          NAMESPACE: ${{ secrets.DEV_SANDBOX_NAMESPACE }}
          NAME: sandbox-${{ steps.get-sha.outputs.sha_short }}
      - name: Uninstall
        run: helm delete sandbox-${{ steps.get-sha.outputs.sha_short }} -n ${{ secrets.DEV_SANDBOX_NAMESPACE }} --debug --timeout 10m0s
        if: steps.get-helm.outputs.count > 0