From 8273589f790ad08ec4a8ea5e8285f7314a7280a7 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 16 Aug 2023 14:55:37 +1000 Subject: [PATCH 1/2] Moved compiler down into base-spack, added ENVs based on ARGS for use in later Dockerfiles/scripts --- containers/Dockerfile.access-om2 | 10 ++++---- containers/Dockerfile.access-om3 | 10 ++++---- containers/Dockerfile.base-spack | 44 ++++++++++++++++++++------------ containers/setup-spack-envs.sh | 19 ++++---------- 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/containers/Dockerfile.access-om2 b/containers/Dockerfile.access-om2 index e76fa05..99ebf6a 100644 --- a/containers/Dockerfile.access-om2 +++ b/containers/Dockerfile.access-om2 @@ -2,15 +2,14 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} ARG PACKAGE_NAMES="libaccessom2 mom5 cice5 oasis3-mct" -ARG COMPILER_PACKAGE -ARG COMPILER_VERSION -ARG COMPILER_NAME WORKDIR $SPACK_ROOT -LABEL au.org.access-nri.ci.compiler ${COMPILER_PACKAGE}@${COMPILER_VERSION} LABEL au.org.access-nri.ci.packages ${PACKAGE_NAMES} LABEL au.org.access-nri.ci.base-spack-image ${BASE_IMAGE} +LABEL au.org.access-nri.ci.spack-repo-version $SPACK_REPO_VERSION +LABEL au.org.access-nri.ci.spack-packages-repo-version $SPACK_PACKAGES_REPO_VERSION +LABEL au.org.access-nri.ci.compiler $SPACK_ENV_COMPILER_PACKAGE@$SPACK_ENV_COMPILER_VERSION # Use Spack shell environment for subsequent RUN steps SHELL ["docker-shell"] @@ -18,7 +17,8 @@ SHELL ["docker-shell"] COPY setup-spack-envs.sh . RUN chmod +x setup-spack-envs.sh \ - && ./setup-spack-envs.sh ${COMPILER_PACKAGE} ${COMPILER_NAME} ${COMPILER_VERSION} "${PACKAGE_NAMES}" + && ./setup-spack-envs.sh "${PACKAGE_NAMES}" \ + && spack gc -y # Push any uncached binaries to buildcache RUN spack -d buildcache create -a -m s3_buildcache `spack find --json | jq --raw-output .[].name` diff --git a/containers/Dockerfile.access-om3 b/containers/Dockerfile.access-om3 index 709c2d4..4cb28af 100644 --- a/containers/Dockerfile.access-om3 +++ b/containers/Dockerfile.access-om3 @@ -2,15 +2,14 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} ARG PACKAGE_NAMES="access-om3" -ARG COMPILER_PACKAGE -ARG COMPILER_VERSION -ARG COMPILER_NAME WORKDIR $SPACK_ROOT -LABEL au.org.access-nri.ci.compiler ${COMPILER_PACKAGE}@${COMPILER_VERSION} LABEL au.org.access-nri.ci.packages ${PACKAGE_NAMES} LABEL au.org.access-nri.ci.base-spack-image ${BASE_IMAGE} +LABEL au.org.access-nri.ci.spack-repo-version $SPACK_REPO_VERSION +LABEL au.org.access-nri.ci.spack-packages-repo-version $SPACK_PACKAGES_REPO_VERSION +LABEL au.org.access-nri.ci.compiler $SPACK_ENV_COMPILER_PACKAGE@$SPACK_ENV_COMPILER_VERSION # Use Spack shell environment for subsequent RUN steps SHELL ["docker-shell"] @@ -18,7 +17,8 @@ SHELL ["docker-shell"] COPY setup-spack-envs.sh . RUN chmod +x setup-spack-envs.sh \ - && ./setup-spack-envs.sh ${COMPILER_PACKAGE} ${COMPILER_NAME} ${COMPILER_VERSION} "${PACKAGE_NAMES}" + && ./setup-spack-envs.sh "${PACKAGE_NAMES}" \ + && spack gc -y # Push any uncached binaries to buildcache RUN spack -d buildcache create -a -m s3_buildcache `spack find --json | jq --raw-output .[].name` diff --git a/containers/Dockerfile.base-spack b/containers/Dockerfile.base-spack index 61ffa63..46fdae4 100644 --- a/containers/Dockerfile.base-spack +++ b/containers/Dockerfile.base-spack @@ -1,16 +1,27 @@ FROM ghcr.io/access-nri/base-os:latest -SHELL ["/bin/bash", "-c"] +ARG SPACK_REPO_VERSION=v0.20.1 +ARG SPACK_PACKAGES_REPO_VERSION=main +ARG COMPILER_PACKAGE +ARG COMPILER_VERSION +ARG COMPILER_NAME ENV SPACK_ROOT=/opt/spack -ENV SPACK_PACKAGES_REPO_ROOT=/opt/spack_packages ENV GNUPGHOME=$SPACK_ROOT/opt/spack/gpg +ENV SPACK_REPO_VERSION=${SPACK_REPO_VERSION} +ENV SPACK_PACKAGES_REPO_ROOT=/opt/spack_packages +ENV SPACK_ENV_COMPILER_PACKAGE=${COMPILER_PACKAGE} +ENV SPACK_ENV_COMPILER_VERSION=${COMPILER_VERSION} +ENV SPACK_ENV_COMPILER_NAME=${COMPILER_NAME} ARG SPACK_PACKAGES_REPO_VERSION=main ARG SPACK_REPO_VERSION=v0.20.1 +LABEL au.org.access-nri.ci.spack-repo-version $SPACK_REPO_VERSION LABEL au.org.access-nri.ci.spack-packages-repo-version ${SPACK_PACKAGES_REPO_VERSION} -LABEL au.org.access-nri.ci.spack-repo-version ${SPACK_REPO_VERSION} +LABEL au.org.access-nri.ci.compiler ${SPACK_ENV_COMPILER_PACKAGE}@${SPACK_ENV_COMPILER_VERSION} + +SHELL ["/bin/bash", "-c"] # Install spack RUN git clone -c feature.manyFiles=true https://github.com/spack/spack.git $SPACK_ROOT --branch ${SPACK_REPO_VERSION} --single-branch --depth=1 @@ -38,26 +49,27 @@ RUN git clone https://github.com/ACCESS-NRI/spack_packages.git ${SPACK_PACKAGES RUN spack repo add $SPACK_PACKAGES_REPO_ROOT && \ spack bootstrap now -# TODO -# Think about setting up spack S3 build cache for CI here # Set up ACCESS Spack buildcache RUN pip3 install --no-cache-dir boto3==1.23.10 RUN spack gpg init # Add buildcache mirror -RUN --mount=type=secret,id=S3_ACCESS_KEY_ID \ - --mount=type=secret,id=S3_ACCESS_KEY_SECRET \ - spack mirror add \ - --s3-access-key-id `cat /run/secrets/S3_ACCESS_KEY_ID` \ - --s3-access-key-secret `cat /run/secrets/S3_ACCESS_KEY_SECRET` \ - s3_buildcache \ - s3://access-nri-spack-cache +# RUN --mount=type=secret,id=S3_ACCESS_KEY_ID \ +# --mount=type=secret,id=S3_ACCESS_KEY_SECRET \ +# spack mirror add \ +# --s3-access-key-id `cat /run/secrets/S3_ACCESS_KEY_ID` \ +# --s3-access-key-secret `cat /run/secrets/S3_ACCESS_KEY_SECRET` \ +# s3_buildcache \ +# s3://access-nri-spack-cache # Import buildcache keys -RUN --mount=type=secret,id=access-nri.priv \ - spack gpg trust /run/secrets/access-nri.priv -RUN --mount=type=secret,id=access-nri.pub \ - spack gpg trust /run/secrets/access-nri.pub +# RUN --mount=type=secret,id=access-nri.priv \ +# spack gpg trust /run/secrets/access-nri.priv +# RUN --mount=type=secret,id=access-nri.pub \ +# spack gpg trust /run/secrets/access-nri.pub + +# Set up compilers for this base-spack image +RUN spack install ${SPACK_ENV_COMPILER_PACKAGE}@${SPACK_ENV_COMPILER_VERSION} ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"] CMD ["interactive-shell"] diff --git a/containers/setup-spack-envs.sh b/containers/setup-spack-envs.sh index 802c5b5..23bd928 100644 --- a/containers/setup-spack-envs.sh +++ b/containers/setup-spack-envs.sh @@ -1,24 +1,15 @@ #!/bin/bash -# usage: ./setup-spack-envs -# eg: -if [ "$#" -ne 4 ]; then - echo "Invalid number of arguments: expecting 4" - exit 1 -fi +# usage: ./setup-spack-envs -COMPILER_PACKAGE="$1" -COMPILER_NAME="$2" -COMPILER_VERSION="$3" -PACKAGES="$4" +PACKAGES="$1" for PACKAGE in $PACKAGES; do spack env create $PACKAGE spack env activate $PACKAGE - spack -d install -j 4 --add --fail-fast $COMPILER_PACKAGE@$COMPILER_VERSION - spack load $COMPILER_PACKAGE@$COMPILER_VERSION + spack -d install -j 4 --add --fail-fast $SPACK_ENV_COMPILER_PACKAGE@$SPACK_ENV_COMPILER_VERSION + spack load $SPACK_ENV_COMPILER_PACKAGE@$SPACK_ENV_COMPILER_VERSION spack compiler find --scope env:$PACKAGE - spack -d install -j 4 --add --only dependencies --fail-fast $PACKAGE%$COMPILER_NAME@$COMPILER_VERSION - spack gc -y + spack -d install -j 4 --add --only dependencies --fail-fast $PACKAGE%$SPACK_ENV_COMPILER_NAME@$SPACK_ENV_COMPILER_VERSION spack env deactivate done \ No newline at end of file From 87359539cd1592948c5a16c0a9db17a211541ed4 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Mon, 21 Aug 2023 13:32:05 +1000 Subject: [PATCH 2/2] Split build-and-push-image into a compiler-based base-spack -> dependency-image pipeline, using jq --- .../workflows/build-and-push-image-build.yml | 56 +++---------- .../workflows/dependency-image-pipeline.yml | 84 +++++++++++++++++++ containers/Dockerfile.base-spack | 25 +++--- 3 files changed, 107 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/dependency-image-pipeline.yml diff --git a/.github/workflows/build-and-push-image-build.yml b/.github/workflows/build-and-push-image-build.yml index 0ce3c62..422a884 100644 --- a/.github/workflows/build-and-push-image-build.yml +++ b/.github/workflows/build-and-push-image-build.yml @@ -3,11 +3,6 @@ name: Build and push access-nri/build images on: workflow_dispatch: inputs: - spack-packages-build-required: - required: true - default: true - type: boolean - description: "A flag to note if building a new spack_packages image at the below version is required" spack-packages-version: required: true type: string @@ -23,26 +18,8 @@ on: - all jobs: - base-spack-build-and-push-image: - if: inputs.spack-packages-build-required - uses: access-nri/build-ci/.github/workflows/build-and-push-image.yml@main - with: - container-registry: ghcr.io - container-name: access-nri/base-spack-${{ github.event.inputs.spack-packages-version }} - dockerfile-directory: containers - dockerfile-name: Dockerfile.base-spack - build-args: "SPACK_PACKAGES_REPO_VERSION=${{ github.event.inputs.spack-packages-version }}" - secrets: - build-secrets: | - "S3_ACCESS_KEY_ID=${{ secrets.S3_ACCESS_KEY_ID }}" - "S3_ACCESS_KEY_SECRET=${{ secrets.S3_ACCESS_KEY_SECRET }}" - "access-nri.priv=${{ secrets.BUILDCACHE_KEY_PRIVATE }}" - "access-nri.pub=${{ secrets.BUILDCACHE_KEY_PUBLIC }}" - permissions: - contents: read - packages: write - generate-matrix: + name: Generate Matrix of compilers and models runs-on: ubuntu-latest outputs: model: ${{ steps.get-model.outputs.model }} @@ -59,30 +36,21 @@ jobs: fi - name: Determine compilers to build id: get-compiler - run: echo "compiler=$(cat containers/compilers.json | tr '\n' ' ')" >> $GITHUB_OUTPUT + run: echo "compiler=$(jq -c . containers/compilers.json)" >> $GITHUB_OUTPUT - model-build-and-push-image: + dependency-image-pipeline: + name: Generate images required for dependency image needs: - - base-spack-build-and-push-image - generate-matrix - if: always() strategy: fail-fast: false matrix: - model: ${{ fromJson(needs.generate-matrix.outputs.model) }} - compiler: ${{ fromJson(needs.generate-matrix.outputs.compiler) }} - uses: access-nri/build-ci/.github/workflows/build-and-push-image.yml@main + model: ${{ needs.generate-matrix.outputs.model }} + compiler: ${{ needs.generate-matrix.outputs.compiler }} + uses: access-nri/build-ci/.github/workflows/dependency-image-pipeline.yml@main with: - container-registry: ghcr.io - container-name: access-nri/build-${{ matrix.model}}-${{ matrix.compiler.name }}${{ matrix.compiler.version }}-${{ inputs.spack-packages-version }} - dockerfile-directory: containers - dockerfile-name: Dockerfile.${{ matrix.model }} - build-args: | - # TODO: Probably shouldn't hard code base image path - "BASE_IMAGE=ghcr.io/access-nri/base-spack-${{ github.event.inputs.spack-packages-version }}:latest" - "COMPILER_NAME=${{ matrix.compiler.name }}" - "COMPILER_PACKAGE=${{ matrix.compiler.package }}" - "COMPILER_VERSION=${{ matrix.compiler.version }}" - permissions: - contents: read - packages: write + spack-packages-version: ${{ inputs.spack-packages-version }} + compiler-name: ${{ matrix.compiler.name }} + compiler-package: ${{ matrix.compiler.package }} + compiler-version: ${{ matrix.compiler.version }} + model: ${{ matrix.model }} \ No newline at end of file diff --git a/.github/workflows/dependency-image-pipeline.yml b/.github/workflows/dependency-image-pipeline.yml new file mode 100644 index 0000000..b6766bb --- /dev/null +++ b/.github/workflows/dependency-image-pipeline.yml @@ -0,0 +1,84 @@ +name: Dependency Image Generation Pipeline +on: + workflow_call: + inputs: + spack-packages-version: + description: Tag associated with the access-nri/spack_packages repo + type: string + compiler-name: + description: Name of the compiler + type: string + compiler-package: + description: Spack package name of the compiler + type: string + compiler-version: + description: Spack package version of the compiler + type: string + model: + description: A model name (eg. access-om2) + type: string +jobs: + base-spack-image-check: + name: Check for the existence of base-spack-${{ inputs.compiler-name }}@${{ inputs.compiler-version }}-${{ inputs.spack-packages-version }} + runs-on: ubuntu-latest + outputs: + image-exists: ${{ steps.image-exists.outputs.check }} + steps: + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Check for existing package + id: image-exists + run: | + manifest=$(docker manifest inspect ghcr.io/access-nri/base-spack-${{ inputs.compiler-name }}@${{ inputs.compiler-version }}-${{ inputs.spack-packages-version }}:latest) + if [ "$manifest" != "manifest unknown" ]; then + echo "check=true" >> $GITHUB_OUTPUT + fi + + base-spack-image: + name: Create the base-spack image + needs: + - base-spack-image-check + if: needs.base-spack-image-check.outputs.image-exists + uses: access-nri/build-ci/.github/workflows/build-and-push-image.yml@main + with: + container-registry: ghcr.io + container-name: access-nri/base-spack-${{ inputs.spack-packages-version }} + dockerfile-directory: containers + dockerfile-name: Dockerfile.base-spack + build-args: | + "SPACK_PACKAGES_REPO_VERSION=${{ inputs.spack-packages-version }}" + "COMPILER_NAME=${{ inputs.compiler-name }}" + "COMPILER_PACKAGE=${{ inputs.compiler-package }}" + "COMPILER_VERSION=${{ inputs.compiler-version }}" + secrets: + build-secrets: | + "S3_ACCESS_KEY_ID=${{ secrets.S3_ACCESS_KEY_ID }}" + "S3_ACCESS_KEY_SECRET=${{ secrets.S3_ACCESS_KEY_SECRET }}" + "access-nri.priv=${{ secrets.BUILDCACHE_KEY_PRIVATE }}" + "access-nri.pub=${{ secrets.BUILDCACHE_KEY_PUBLIC }}" + permissions: + contents: read + packages: write + + dependency-image: + # running always() means it will run even if base-spack is skipped + if: always() + needs: + - base-spack-image + uses: access-nri/build-ci/.github/workflows/build-and-push-image.yml@main + with: + container-registry: ghcr.io + container-name: access-nri/build-${{ inputs.model}}-${{ inputs.compiler-name }}${{ inputs.compiler-version }}-${{ inputs.spack-packages-version }} + dockerfile-directory: containers + dockerfile-name: Dockerfile.${{ inputs.model }} + build-args: | + # TODO: Probably shouldn't hard code base image path + "BASE_IMAGE=ghcr.io/access-nri/base-spack-${{ inputs.compiler-name }}@${{ inputs.compiler-version }}-${{ inputs.spack-packages-version }}:latest" + permissions: + contents: read + packages: write + \ No newline at end of file diff --git a/containers/Dockerfile.base-spack b/containers/Dockerfile.base-spack index 46fdae4..ca98a0e 100644 --- a/containers/Dockerfile.base-spack +++ b/containers/Dockerfile.base-spack @@ -14,9 +14,6 @@ ENV SPACK_ENV_COMPILER_PACKAGE=${COMPILER_PACKAGE} ENV SPACK_ENV_COMPILER_VERSION=${COMPILER_VERSION} ENV SPACK_ENV_COMPILER_NAME=${COMPILER_NAME} -ARG SPACK_PACKAGES_REPO_VERSION=main -ARG SPACK_REPO_VERSION=v0.20.1 - LABEL au.org.access-nri.ci.spack-repo-version $SPACK_REPO_VERSION LABEL au.org.access-nri.ci.spack-packages-repo-version ${SPACK_PACKAGES_REPO_VERSION} LABEL au.org.access-nri.ci.compiler ${SPACK_ENV_COMPILER_PACKAGE}@${SPACK_ENV_COMPILER_VERSION} @@ -54,19 +51,19 @@ RUN pip3 install --no-cache-dir boto3==1.23.10 RUN spack gpg init # Add buildcache mirror -# RUN --mount=type=secret,id=S3_ACCESS_KEY_ID \ -# --mount=type=secret,id=S3_ACCESS_KEY_SECRET \ -# spack mirror add \ -# --s3-access-key-id `cat /run/secrets/S3_ACCESS_KEY_ID` \ -# --s3-access-key-secret `cat /run/secrets/S3_ACCESS_KEY_SECRET` \ -# s3_buildcache \ -# s3://access-nri-spack-cache +RUN --mount=type=secret,id=S3_ACCESS_KEY_ID \ + --mount=type=secret,id=S3_ACCESS_KEY_SECRET \ + spack mirror add \ + --s3-access-key-id `cat /run/secrets/S3_ACCESS_KEY_ID` \ + --s3-access-key-secret `cat /run/secrets/S3_ACCESS_KEY_SECRET` \ + s3_buildcache \ + s3://access-nri-spack-cache # Import buildcache keys -# RUN --mount=type=secret,id=access-nri.priv \ -# spack gpg trust /run/secrets/access-nri.priv -# RUN --mount=type=secret,id=access-nri.pub \ -# spack gpg trust /run/secrets/access-nri.pub +RUN --mount=type=secret,id=access-nri.priv \ + spack gpg trust /run/secrets/access-nri.priv +RUN --mount=type=secret,id=access-nri.pub \ + spack gpg trust /run/secrets/access-nri.pub # Set up compilers for this base-spack image RUN spack install ${SPACK_ENV_COMPILER_PACKAGE}@${SPACK_ENV_COMPILER_VERSION}