diff --git a/.github/workflows/build_push.yaml b/.github/workflows/build_push.yaml deleted file mode 100644 index 92b0775ca..000000000 --- a/.github/workflows/build_push.yaml +++ /dev/null @@ -1,247 +0,0 @@ -# This workflow: -# * Builds, tests, and scans all images -# * (optionally) pushes the images to ACR -# -# This workflow triggers on: -# * a push to master -# * any create/synchronize to a PR (eg: any time you push an update to a PR). -# -# Image build/test/scan will run on any of the above events. -# Image push will run only if: -# * this is a push to master -# * if the PR triggering this event has the label 'auto-deploy' -# -# To configure this workflow: -# -# 1. Set up the following secrets in your workspace: -# a. REGISTRY_USERNAME with ACR username -# b. REGISTRY_PASSWORD with ACR Password -# c. AZURE_CREDENTIALS with the output of `az ad sp create-for-rbac --sdk-auth` -# d. DEV_REGISTRY_USERNAME with the DEV ACR username -# e. DEV_REGISTRY_PASSWORD with the DEV ACR Password -# -# 2. Change the values for the REGISTRY_NAME, CLUSTER_NAME, CLUSTER_RESOURCE_GROUP and NAMESPACE environment variables (below in build-push). -name: build_and_push -on: - schedule: - # Execute at 2am EST every day - - cron: '0 21 * * *' - push: - branches: - - 'master' - pull_request: - types: - - 'opened' - - 'synchronize' - - 'reopened' -env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - -jobs: - # Any checks that run pre-build - pre-build-checks: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - - name: Assert committed ./output folder matches `make generate-dockerfiles` output - run: | - sudo apt-get install --yes make - make clean - make generate-dockerfiles - if ! git diff --quiet output/; then - echo 'output folder and docker-bits/resources out of sync!' - exit 1 - fi - - build-push: - env: - REGISTRY_NAME: k8scc01covidacr - DEV_REGISTRY_NAME: k8scc01covidacrdev - CLUSTER_NAME: k8s-cancentral-01-covid-aks - CLUSTER_RESOURCE_GROUP: k8s-cancentral-01-covid-aks - LOCAL_REPO: localhost:5000 - TRIVY_VERSION: "v0.57.0" - TRIVY_DATABASES: '"ghcr.io/aquasecurity/trivy-db:2","public.ecr.aws/aquasecurity/trivy-db"' - TRIVY_JAVA_DATABASES: '"ghcr.io/aquasecurity/trivy-java-db:1","public.ecr.aws/aquasecurity/trivy-java-db"' - TRIVY_MAX_RETRIES: 5 - TRIVY_RETRY_DELAY: 20 - HADOLINT_VERSION: "2.12.0" - ACTIONS_RUNNER_DEBUG: true - - strategy: - fail-fast: false - matrix: - notebook: - # TODO: Pull this from a settings file or Makefile, that way Make can have the same list - # - docker-stacks-datascience-notebook # Debugging - - rstudio - - sas - - jupyterlab-cpu - - jupyterlab-pytorch - # - jupyterlab-tensorflow removed from build. https://jirab.statcan.ca/browse/BTIS-421 - - remote-desktop - needs: pre-build-checks - runs-on: ubuntu-latest - services: - registry: - image: registry:2 - ports: - - 5000:5000 - steps: - - name: Set ENV variables for a PR containing the auto-deploy tag - if: github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'auto-deploy') - run: | - echo "REGISTRY=k8scc01covidacrdev.azurecr.io" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=dev" >> "$GITHUB_ENV" - - - name: Set ENV variables for pushes to master - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - echo "REGISTRY=k8scc01covidacr.azurecr.io" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=v1" >> "$GITHUB_ENV" - echo "IS_LATEST=true" >> "$GITHUB_ENV" - - - uses: actions/checkout@master - - - name: Echo disk usage before clean up - run: ./.github/scripts/echo_usage.sh - - - name: Free up all available disk space before building - run: ./.github/scripts/cleanup_runner.sh - - - name: Echo disk usage before build start - run: ./.github/scripts/echo_usage.sh - - - name: Get current notebook name - id: notebook-name - shell: bash - run: | - echo NOTEBOOK_NAME=${{ matrix.notebook }} >> $GITHUB_OUTPUT - - # Connect to Azure Container registry (ACR) - - uses: azure/docker-login@v1 - with: - login-server: ${{ env.REGISTRY_NAME }}.azurecr.io - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_PASSWORD }} - - # Connect to Azure DEV Container registry (ACR) - - uses: azure/docker-login@v1 - with: - login-server: ${{ env.DEV_REGISTRY_NAME }}.azurecr.io - username: ${{ secrets.DEV_REGISTRY_USERNAME }} - password: ${{ secrets.DEV_REGISTRY_PASSWORD }} - - # Image building/storing locally - - name: Make Dockerfiles - run: make generate-dockerfiles - - - name: Run Hadolint - run: | - sudo curl -L https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 --output hadolint - sudo chmod +x hadolint - ./hadolint output/${{ matrix.notebook }}/Dockerfile --no-fail - - # make build emits full_image_name, image_tag, and image_repo outputs - - name: Build image - id: build-image - run: make build/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} - - - name: Echo disk usage after build completion - run: ./.github/scripts/echo_usage.sh - - - name: Add standard tag names (short sha, sha, and branch) and any other post-build activity - run: make post-build/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} - - - name: Push image to local registry (default pushes all tags) - run: make push/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} - - # Image testing - - name: Set Up Python for Test Suite - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Set up venv for Test Suite - run: | - python -m pip install --upgrade pip - make install-python-dev-venv - - - name: Test image - run: make test/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} - - # Free up space from build process (containerscan action will run out of space if we don't) - - run: ./.github/scripts/cleanup_runner.sh - - # Scan image for vulnerabilities - - name: Aqua Security Trivy image scan - # see https://github.com/StatCan/aaw-private/issues/11 -- should be re-enabled - if: steps.notebook-name.outputs.NOTEBOOK_NAME != 'sas' - run: | - printf ${{ secrets.CVE_ALLOWLIST }} > .trivyignore - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }} - - set +e - - for ((i=0; i<${{ env.TRIVY_MAX_RETRIES }}; i++)); do - echo "Attempt $((i + 1)) of ${{ env.TRIVY_MAX_RETRIES }}..." - - trivy image \ - --db-repository ${{ env.TRIVY_DATABASES }} \ - --java-db-repository ${{ env.TRIVY_JAVA_DATABASES }} \ - ${{ steps.build-image.outputs.full_image_name }} \ - --exit-code 10 --timeout=20m --scanners vuln --severity CRITICAL - EXIT_CODE=$? - - if [[ $EXIT_CODE -eq 0 ]]; then - echo "Trivy scan completed successfully." - exit 0 - elif [[ $EXIT_CODE -eq 10 ]]; then - echo "Trivy scan completed successfully. Some vulnerabilities were found." - exit 10 - elif [[ $i -lt $(( ${{ env.TRIVY_MAX_RETRIES }} - 1)) ]]; then - echo "Encountered unexpected error. Retrying in ${{ env.TRIVY_RETRY_DELAY }} seconds..." - sleep ${{ env.TRIVY_RETRY_DELAY }} - else - echo "Unexpected error persists after ${{ env.TRIVY_MAX_RETRIES }} attempts. Exiting." - exit 1 - fi - done - - # Push image to ACR - # Pushes if this is a push to master or an update to a PR that has auto-deploy label - - name: Test if we should push to ACR - id: should-i-push - if: | - github.event_name == 'push' || - ( - github.event_name == 'pull_request' && - contains( github.event.pull_request.labels.*.name, 'auto-deploy') - ) - run: echo 'boolean=true' >> $GITHUB_OUTPUT - - # Pull the local image back, then "build" it (will just tag the pulled image) - - name: Pull image back from local repo - if: steps.should-i-push.outputs.boolean == 'true' - run: docker pull ${{ steps.build-image.outputs.full_image_name }} - - # Rename the localhost:5000/imagename:tag built above to use the real repo - # (get above's name from build-image's output) - - name: Tag images with real repository - if: steps.should-i-push.outputs.boolean == 'true' - run: > - make post-build/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY IS_LATEST=$IS_LATEST - IMAGE_VERSION=$IMAGE_VERSION SOURCE_FULL_IMAGE_NAME=${{ steps.build-image.outputs.full_image_name }} - - - name: Push image to registry - if: steps.should-i-push.outputs.boolean == 'true' - run: | - make push/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY - - - name: Slack Notification - if: failure() && github.event_name=='schedule' - uses: act10ns/slack@v1 - with: - status: failure - message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}} diff --git a/.github/workflows/check-diff.yaml b/.github/workflows/check-diff.yaml new file mode 100644 index 000000000..b9a44859f --- /dev/null +++ b/.github/workflows/check-diff.yaml @@ -0,0 +1,62 @@ +name: Check for changes in subdirectory + +on: + workflow_call: + inputs: + directory: + description: The directory of the image files + required: true + type: string + parent-image-is-diff: + description: Parent image has been changed? + required: true + type: string + branch-name: + description: The name of the current branch + required: true + type: string + outputs: + is-diff: + description: Is there a difference between the master branch and the current branch + value: ${{ jobs.check-diff.outputs.is-diff }} + +jobs: + check-diff: + runs-on: ubuntu-latest + outputs: + is-diff: ${{ steps.check-changes.outputs.is-diff }} + + steps: + - uses: actions/checkout@v4 + + - name: Fetch master branch + run: | + git fetch origin master + + - name: Check for changes + id: check-changes + run: | # Check for changes excluding README.md + if [ "${{ inputs.branch-name }}" == "master" ]; then + echo "Always build master" + echo "is-diff=true" >> $GITHUB_OUTPUT + elif [ "${{ inputs.parent-image-is-diff }}" == "true" ]; then + echo "Parent is diff" + echo "is-diff=true" >> $GITHUB_OUTPUT + # Check if the subdirectory exists in the base branch + elif ! git ls-tree -d origin/master -- "images/${{ inputs.directory }}" >/dev/null 2>&1; then + echo "Subdirectory does not exist in the base branch" + echo "is-diff='true'" >> $GITHUB_OUTPUT + else + CHANGES=$(git diff --name-only origin/master HEAD -- "images/${{ inputs.directory }}" | grep -v "README.md" || true) + NEW_FILES=$(git diff --name-only --diff-filter=A origin/master HEAD -- "images/${{ inputs.directory }}" | grep -v "README.md" || true) + + CHANGES="${CHANGES}${NEW_FILES}" + + if [ -n "$CHANGES" ]; then + echo "Changes detected (excluding README.md)" + echo "is-diff=true" >> $GITHUB_OUTPUT + else + echo "No changes detected" + echo "is-diff=false" >> $GITHUB_OUTPUT + fi + fi diff --git a/.github/workflows/docker-build-upload.yaml b/.github/workflows/docker-build-upload.yaml new file mode 100644 index 000000000..592835d41 --- /dev/null +++ b/.github/workflows/docker-build-upload.yaml @@ -0,0 +1,116 @@ +name: Download a parent image, build a new one, tag it, then upload the image + +env: + HADOLINT_VERSION: "2.12.0" + +on: + workflow_call: + inputs: + parent-image: + description: Parent image name + required: true + type: string + directory: + description: The directory of the image files + required: true + type: string + image: + description: Image name + required: true + type: string + base-image: + description: The base image to build from if not located on our own repo + required: false + type: string + registry-name: + description: url of the registry + required: true + type: string + buildkit: + description: buildkit version, legacy is 0 and deprecated + required: true + type: number + branch-name: + description: The name of the current branch + required: true + type: string + secrets: + REGISTRY_USERNAME: + description: The username for the container registry + required: true + REGISTRY_PASSWORD: + description: The password for the container registry + required: true + +jobs: + build-upload: + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + + steps: + - uses: actions/checkout@v4 + + - name: Run Hadolint + run: | + sudo curl -L https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 --output hadolint + sudo chmod +x hadolint + ./hadolint images/${{ inputs.directory }}/Dockerfile --no-fail + + - name: Echo disk usage before clean up + run: ./.github/scripts/echo_usage.sh + + - name: Free up all available disk space before building + run: ./.github/scripts/cleanup_runner.sh + + - name: Echo disk usage before build start + run: ./.github/scripts/echo_usage.sh + + # Connect to Azure Container registry (ACR) + - uses: azure/docker-login@v1 + with: + login-server: ${{ inputs.registry-name }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Pull parent image + id: pull-parent + if: inputs.parent-image != '' + run: make pull/${{ inputs.parent-image }} REPO=${{ inputs.registry-name }} + + - name: Set BASE_IMAGE variable + run: | + if [ "${{ inputs.base-image }}" == "" ]; then + echo "BASE_IMAGE=${{ steps.pull-parent.outputs.image_name }}" >> $GITHUB_ENV + else + echo "BASE_IMAGE=${{ inputs.base-image }}" >> $GITHUB_ENV + fi + + - name: Set FROM and as in Docerfile + run: | + sed -i '1i FROM ${{ env.BASE_IMAGE}} as ${{ inputs.image }}' ./images/${{ inputs.directory }}/Dockerfile + + # make build emits full_image_name, image_tag, and image_repo outputs + - name: Build image + id: build-image + run: make build/${{ inputs.image }} REPO=${{ inputs.registry-name }} DIRECTORY=${{ inputs.directory }} BUILDKIT=${{ inputs.buildkit }} + + - name: Echo disk usage after build completion + run: ./.github/scripts/echo_usage.sh + + - name: Add standard tags (short sha, sha, and branch) and any other post-build activity + if: ${{ inputs.branch-name != 'master' }} + run: make post-build/${{ inputs.image }} REPO=${{ inputs.registry-name }} + + - name: Add master and standard tags (v1, latest; short sha, sha, and branch) and any other post-build activity + if: ${{ inputs.branch-name == 'master' }} + run: make post-build/${{ inputs.image }} REPO=${{ inputs.registry-name }} IMAGE_VERSION=v1 IS_LATEST=true + + - name: Push image to registry (default pushes all tags) + run: make push/${{ inputs.image }} REPO=${{ inputs.registry-name }} + + # Free up space from build process (containerscan action will run out of space if we don't) + - run: ./.github/scripts/cleanup_runner.sh diff --git a/.github/workflows/docker-nightly.yaml b/.github/workflows/docker-nightly.yaml new file mode 100644 index 000000000..4b89d052c --- /dev/null +++ b/.github/workflows/docker-nightly.yaml @@ -0,0 +1,85 @@ +name: Nightly test +on: + schedule: + # Execute at 2am EST every day + - cron: '0 7 * * *' + +jobs: + vars: + runs-on: ubuntu-latest + outputs: + REGISTRY_NAME: "k8scc01covidacr" + DEV_REGISTRY_NAME: "k8scc01covidacrdev" + branch-name: "master" + steps: + - uses: actions/checkout@v4 + + - name: Get branch name + id: getBranch + run: | + chmod +x ./make_helpers/get_branch_name.sh + BRANCH_NAME=$(./make_helpers/get_branch_name.sh) + echo "branch-name=$BRANCH_NAME" + echo "branch-name=$BRANCH_NAME" >> $GITHUB_OUTPUT + - name: Set up environment + run: echo "Environment has been set up." + + jupyterlab-cpu-test: + needs: [vars] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "jupyterlab-cpu" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + jupyterlab-tensorflow-test: + needs: [vars] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "jupyterlab-tensorflow" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + rstudio-test: + needs: [vars] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "rstudio" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + sas-test: + needs: [vars] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "sas" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + remote-desktop-test: + needs: [vars] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "remote-desktop" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} diff --git a/.github/workflows/docker-pull-test.yaml b/.github/workflows/docker-pull-test.yaml new file mode 100644 index 000000000..5cd21e42c --- /dev/null +++ b/.github/workflows/docker-pull-test.yaml @@ -0,0 +1,111 @@ +name: Tests the image built or copied from the previous step + +env: + HADOLINT_VERSION: "2.12.0" + +on: + workflow_call: + inputs: + image: + description: Image name + required: true + type: string + registry-name: + description: url of the registry + required: true + type: string + branch-name: + description: The name of the current branch + required: true + type: string + secrets: + REGISTRY_USERNAME: + description: The username for the container registry + required: true + REGISTRY_PASSWORD: + description: The password for the container registry + required: true + CVE_ALLOWLIST: + description: The list of Trivy exemptions + required: true + +jobs: + pull-test: + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + env: + TRIVY_VERSION: "v0.57.0" + TRIVY_DATABASES: '"ghcr.io/aquasecurity/trivy-db:2","public.ecr.aws/aquasecurity/trivy-db"' + TRIVY_JAVA_DATABASES: '"ghcr.io/aquasecurity/trivy-java-db:1","public.ecr.aws/aquasecurity/trivy-java-db"' + TRIVY_MAX_RETRIES: 5 + TRIVY_RETRY_DELAY: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Free up all available disk space before building + run: ./.github/scripts/cleanup_runner.sh + + # Connect to Azure Container registry (ACR) + - uses: azure/docker-login@v1 + with: + login-server: ${{ inputs.registry-name }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Pull existing image + id: pull-existing + run: make pull/${{ inputs.image }} REPO=${{ inputs.registry-name }} TAG=${{ inputs.branch-name }} + + - name: Set Up Python for Test Suite + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Set up venv for Test Suite + run: | + python -m pip install --upgrade pip + make install-python-dev-venv + + - name: Test image + run: make test/${{ inputs.image }} + + # Free up space from build process (containerscan action will run out of space if we don't) + - name: cleanup runner + run: ./.github/scripts/cleanup_runner.sh + + # Scan image for vulnerabilities + - name: Aqua Security Trivy image scan + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }} + + set +e + + for ((i=0; i<${{ env.TRIVY_MAX_RETRIES }}; i++)); do + echo "Attempt $((i + 1)) of ${{ env.TRIVY_MAX_RETRIES }}..." + + trivy image \ + --db-repository ${{ env.TRIVY_DATABASES }} \ + --java-db-repository ${{ env.TRIVY_JAVA_DATABASES }} \ + ${{ inputs.registry-name }}/${{ inputs.image }}:${{ inputs.branch-name }} \ + --exit-code 10 --timeout=20m --scanners vuln --severity CRITICAL --quiet + EXIT_CODE=$? + + if [[ $EXIT_CODE -eq 0 ]]; then + echo "Trivy scan completed successfully." + exit 0 + elif [[ $EXIT_CODE -eq 10 ]]; then + echo "Trivy scan completed successfully. Some vulnerabilities were found." + exit 0 + elif [[ $i -lt $(( ${{ env.TRIVY_MAX_RETRIES }} - 1)) ]]; then + echo "Encountered unexpected error. Retrying in ${{ env.TRIVY_RETRY_DELAY }} seconds..." + sleep ${{ env.TRIVY_RETRY_DELAY }} + else + echo "Unexpected error persists after ${{ env.TRIVY_MAX_RETRIES }} attempts. Exiting." + exit 1 + fi + done diff --git a/.github/workflows/docker-pull-upload.yaml b/.github/workflows/docker-pull-upload.yaml new file mode 100644 index 000000000..4345e00e4 --- /dev/null +++ b/.github/workflows/docker-pull-upload.yaml @@ -0,0 +1,58 @@ +name: Download a copy of the image of of main, retag it, and upload + +env: + HADOLINT_VERSION: "2.12.0" + +on: + workflow_call: + inputs: + image: + description: Image name + required: true + type: string + registry-name: + description: url of the registry + required: true + type: string + secrets: + REGISTRY_USERNAME: + description: The username for the container registry + required: true + REGISTRY_PASSWORD: + description: The password for the container registry + required: true + +jobs: + pull-upload: + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + + steps: + - uses: actions/checkout@v4 + + - name: Free up all available disk space before building + run: ./.github/scripts/cleanup_runner.sh + + # Connect to Azure Container registry (ACR) + - uses: azure/docker-login@v1 + with: + login-server: ${{ inputs.registry-name }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Pull existing image + id: pull-existing + run: make pull/${{ inputs.image }} REPO=${{ inputs.registry-name }} TAG=master + + - name: Retag existing image + run: make post-build/${{ inputs.image }} REPO=${{ inputs.registry-name }} SOURCE_FULL_IMAGE_NAME=${{ steps.pull-existing.outputs.image_name }} + + - name: Push image to registry (default pushes all tags) + run: make push/${{ inputs.image }} REPO=${{ inputs.registry-name }} + + # Free up space from build process (containerscan action will run out of space if we don't) + - run: ./.github/scripts/cleanup_runner.sh diff --git a/.github/workflows/docker-steps.yaml b/.github/workflows/docker-steps.yaml new file mode 100644 index 000000000..7f93bca73 --- /dev/null +++ b/.github/workflows/docker-steps.yaml @@ -0,0 +1,94 @@ +name: organizes the steps taken to build, upload, and test the images + +on: + workflow_call: + inputs: + image: + description: Image name + required: true + type: string + directory: + description: The directory of the image files + required: true + type: string + parent-image: + description: Parent image name + required: false + type: string + parent-image-is-diff: + description: Parent image has been changed? + required: false + type: string + default: "false" + base-image: + description: The base image to build from if not located on our own repo + required: false + type: string + registry-name: + description: url of the registry + required: true + type: string + buildkit: + description: buildkit version, legacy is 0 and deprecated + required: false + type: number + default: 1 + branch-name: + description: The name of the current branch + required: true + type: string + secrets: + REGISTRY_USERNAME: + description: The username for the container registry + required: true + REGISTRY_PASSWORD: + description: The password for the container registry + required: true + outputs: + is-diff: + description: Is there a difference between the master branch and the current branch + value: ${{ jobs.check-diff.outputs.is-diff }} + +jobs: + check-diff: + uses: ./.github/workflows/check-diff.yaml + with: + directory: ${{ inputs.directory }} + parent-image-is-diff: ${{ inputs.parent-image-is-diff }} + branch-name: ${{ inputs.branch-name }} + + print-variables: + needs: [check-diff] + runs-on: ubuntu-latest + steps: + - name: print inputs + run: echo "${{ toJSON(inputs) }}" + - name: print check-diff + run: echo "${{ toJSON(needs.check-diff) }}" + + build-upload: + needs: [check-diff] + if: ${{ needs.check-diff.outputs.is-diff == 'true' }} + uses: ./.github/workflows/docker-build-upload.yaml + with: + parent-image: ${{ inputs.parent-image }} + directory: ${{ inputs.directory }} + image: ${{ inputs.image }} + base-image: ${{ inputs.base-image }} + registry-name: ${{ inputs.registry-name }} + buildkit: ${{ inputs.buildkit }} + branch-name: ${{ inputs.branch-name }} + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + pull-upload: + needs: [check-diff] + if: ${{ needs.check-diff.outputs.is-diff == 'false' }} + uses: ./.github/workflows/docker-pull-upload.yaml + with: + image: ${{ inputs.image }} + registry-name: ${{ inputs.registry-name }} + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 000000000..ae6438805 --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,259 @@ +name: Build, test, and push Docker Images +on: + push: + branches: + - "master" + paths: + - ".github/workflows/*" + + - "images/**" + - "!images/*/README.md" + - "tests/**" + - "!tests/README.md" + - "requirements-dev.txt" + pull_request: + paths: + - ".github/workflows/*" + + - "images/**" + - "!images/*/README.md" + - "tests/**" + - "!tests/README.md" + - "requirements-dev.txt" + +jobs: + vars: + runs-on: ubuntu-latest + outputs: + REGISTRY_NAME: "k8scc01covidacr.azurecr.io" + DEV_REGISTRY_NAME: "k8scc01covidacrdev.azurecr.io" + branch-name: ${{ steps.getBranch.outputs.branch-name }} + steps: + - uses: actions/checkout@v4 + + - name: Get branch name + id: getBranch + run: | + chmod +x ./make_helpers/get_branch_name.sh + BRANCH_NAME=$(./make_helpers/get_branch_name.sh) + echo "branch-name=$BRANCH_NAME" + echo "branch-name=$BRANCH_NAME" >> $GITHUB_OUTPUT + - name: Set up environment + run: echo "Environment has been set up." + + base: + needs: [vars] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "base" + directory: "base" + base-image: "quay.io/jupyter/datascience-notebook:2024-06-17" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + platform-jupyterlab: + needs: [vars, base] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "platform-jupyterlab" + directory: "platform" + parent-image: "base" + parent-image-is-diff: "${{ needs.base.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + jupyterlab-cpu: + needs: [vars, platform-jupyterlab] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "jupyterlab-cpu" + directory: "cmd" + parent-image: "platform-jupyterlab" + parent-image-is-diff: "${{ needs.platform-jupyterlab.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + mid-tensorflow: + needs: [vars, platform-jupyterlab] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "mid-tensorflow" + directory: "tensorflow" + parent-image: "platform-jupyterlab" + parent-image-is-diff: "${{ needs.platform-jupyterlab.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + jupyterlab-tensorflow: + needs: [vars, mid-tensorflow] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "jupyterlab-tensorflow" + directory: "cmd" + parent-image: "mid-tensorflow" + parent-image-is-diff: "${{ needs.mid-tensorflow.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + mid-rstudio: + needs: [vars, base] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "mid-rstudio" + directory: "rstudio" + parent-image: "base" + parent-image-is-diff: "${{ needs.base.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + platform-rstudio: + needs: [vars, mid-rstudio] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "platform-rstudio" + directory: "platform" + parent-image: "mid-rstudio" + parent-image-is-diff: "${{ needs.mid-rstudio.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + rstudio: + needs: [vars, platform-rstudio] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "rstudio" + directory: "cmd" + parent-image: "platform-rstudio" + parent-image-is-diff: "${{ needs.platform-rstudio.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + mid-sas: + needs: [vars, mid-rstudio] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "mid-sas" + directory: "sas" + parent-image: "mid-rstudio" + parent-image-is-diff: "${{ needs.mid-rstudio.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + sas: + needs: [vars, mid-sas] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "sas" + directory: "cmd" + parent-image: "mid-sas" + parent-image-is-diff: "${{ needs.mid-sas.outputs.is-diff }}" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + remote-desktop: + needs: [vars] + uses: ./.github/workflows/docker-steps.yaml + with: + image: "remote-desktop" + directory: "remote-desktop" + # Rocker/geospatial is tagged by R version number. They are not clear on whether they'll change those tagged + # images for hotfixes, so always pin tag and digest to prevent unexpected upstream changes + base-image: "rocker/geospatial:4.2.1@sha256:5caca36b8962233f8636540b7c349d3f493f09e864b6e278cb46946ccf60d4d2" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + buildkit: 0 + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + + # Test the Images + + jupyterlab-cpu-test: + needs: [vars, jupyterlab-cpu] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "jupyterlab-cpu" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + jupyterlab-tensorflow-test: + needs: [vars, jupyterlab-tensorflow] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "jupyterlab-tensorflow" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + rstudio-test: + needs: [vars, rstudio] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "rstudio" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + sas-test: + needs: [vars, sas] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "sas" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} + + remote-desktop-test: + needs: [vars, remote-desktop] + uses: ./.github/workflows/docker-pull-test.yaml + with: + image: "remote-desktop" + registry-name: "${{ needs.vars.outputs.REGISTRY_NAME }}" + branch-name: "${{ needs.vars.outputs.branch-name }}" + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + CVE_ALLOWLIST: ${{ secrets.CVE_ALLOWLIST}} \ No newline at end of file diff --git a/Makefile b/Makefile index aecbf1e5b..45febe9ba 100644 --- a/Makefile +++ b/Makefile @@ -184,6 +184,7 @@ docker-stacks-datascience-notebook: ###### Docker helpers ###### ################################### +pull/%: GITHUB_OUTPUT ?= .tmp/github_output.log pull/%: DARGS?= pull/%: REPO?=$(DEFAULT_REPO) pull/%: TAG?=$(DEFAULT_TAG) @@ -191,18 +192,22 @@ pull/%: # End repo with a single slash and start tag with a single colon, if they exist REPO=$$(echo "$(REPO)" | sed 's:/*$$:/:' | sed 's:^\s*/*\s*$$::') &&\ TAG=$$(echo "$(TAG)" | sed 's~^:*~:~' | sed 's~^\s*:*\s*$$~~') &&\ - echo "Pulling $${REPO}$(notdir $@)$${TAG}" &&\ - docker pull $(DARGS) "$${REPO}$(notdir $@)$${TAG}" + IMAGE_NAME="$${REPO}$(notdir $@):$(TAG)" && \ + echo "Pulling $$IMAGE_NAME" &&\ + docker pull $(DARGS) $$IMAGE_NAME &&\ + echo "image_name=$$IMAGE_NAME" >> $(GITHUB_OUTPUT) build/%: GITHUB_OUTPUT ?= .tmp/github_output.log +build/%: BUILDKIT ?= 1 +build/%: DIRECTORY?= build/%: DARGS?= build/%: REPO?=$(DEFAULT_REPO) build/%: TAG?=$(DEFAULT_TAG) build/%: ## build the latest image # End repo with exactly one trailing slash, unless it is empty - REPO=$$(echo "$(REPO)" | sed 's:/*$$:/:' | sed 's:^\s*/*\s*$$::') &&\ + REPO=$$(echo "$(REPO)" | sed 's:/*$$:/:' | sed 's:^\s*/*\s*$$::') && \ IMAGE_NAME="$${REPO}$(notdir $@):$(TAG)" && \ - DOCKER_BUILDKIT=0 docker build $(DARGS) --rm --force-rm -t $$IMAGE_NAME ./output/$(notdir $@) && \ + DOCKER_BUILDKIT=$(BUILDKIT) docker build $(DARGS) --rm --force-rm -t $$IMAGE_NAME ./images/$(DIRECTORY) && \ echo -n "Built image $$IMAGE_NAME of size: " && \ docker images $$IMAGE_NAME --format "{{.Size}}" && \ echo "full_image_name=$$IMAGE_NAME" >> $(GITHUB_OUTPUT) && \ diff --git a/tests/general/test_packages.py b/tests/general/test_packages.py index 574aca706..fefb54246 100644 --- a/tests/general/test_packages.py +++ b/tests/general/test_packages.py @@ -90,6 +90,7 @@ "gputil", "cudatoolkit", "cudnn", + "mamba", # Python # import PIL, not import pillow "pillow",