Create Multimodel Build CI Testing Image

.github/workflows/build-and-push-image-build.yml

@@ -13,6 +13,14 @@ on:
         type: string
         default: "main"
         description: "Either a git tag or branch name for the ACCESS-NRI/spack_packages repository, which defaults to the main branch"
+      model:
+        required: true
+        type: choice
+        default: "all"
+        options:
+          - access-om2
+          - access-om3
+          - all
 
 jobs:
   base-spack-build-and-push-image:
@@ -33,41 +41,48 @@ jobs:
     permissions:
       contents: read
       packages: write
-
+
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      model: ${{ steps.get-model.outputs.model }}
+      compiler: ${{ steps.get-compiler.outputs.compiler }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Determine correct model(s) to build
+        id: get-model
+        run: |
+          if [[ ${{ inputs.model }} -eq "all" ]]; then
+            echo "model=[access-om2, access-om3]" >> $GITHUB_OUTPUT
+          else
+            echo "model=[${{ inputs.model }}]" >> $GITHUB_OUTPUT
+          fi
+      - name: Determine compilers to build
+        id: get-compiler
+        run: echo "compiler=`cat containers/compilers.yaml`" >> $GITHUB_OUTPUT
+
   model-build-and-push-image:
     needs:
       - base-spack-build-and-push-image
-    if: needs.base-spack-build-and-push-image.result == 'success' || needs.base-spack-build-and-push-image.result == 'skipped'
+      - generate-matrix
+    if: always()
     strategy:
       fail-fast: false
       matrix:
-        package: [
-          "oasis3-mct",
-          "libaccessom2",
-          "access-om3",
-          "mom5",
-          "cice5",
-        ]
-        compiler: [
-          {
-            name: "intel",
-            package: "intel-oneapi-compilers",
-            version: "2021.1.2",
-          },
-        ]
+        model: ${{ needs.generate-matrix.outputs.model }}
+        compiler: ${{ needs.generate-matrix.outputs.compiler }}
     uses: access-nri/build-ci/.github/workflows/build-and-push-image.yml@main
     with:
       container-registry: ghcr.io
-      container-name: access-nri/build-${{ matrix.package }}-${{ matrix.compiler.name }}${{ matrix.compiler.version }}-${{ github.event.inputs.spack-packages-version }}
+      container-name: access-nri/build-${{ matrix.model}}-${{ matrix.compiler.name }}${{ matrix.compiler.version }}-${{ inputs.spack-packages-version }}
       dockerfile-directory: containers
-      dockerfile-name: Dockerfile.build
+      dockerfile-name: Dockerfile.${{ matrix.model }}
       build-args: |
         # TODO: Probably shouldn't hard code base image path
         "BASE_IMAGE=ghcr.io/access-nri/base-spack-${{ github.event.inputs.spack-packages-version }}:latest"
-        "PACKAGE=${{ matrix.package }}"
-        "COMPILER_NAME=${{ matrix.compiler.name}}"
-        "COMPILER_PACKAGE=${{ matrix.compiler.package}}"
-        "COMPILER_VERSION=${{ matrix.compiler.version}}"
+        "COMPILER_NAME=${{ matrix.compiler.name }}"
+        "COMPILER_PACKAGE=${{ matrix.compiler.package }}"
+        "COMPILER_VERSION=${{ matrix.compiler.version }}"
     permissions:
       contents: read
       packages: write

.github/workflows/build-package.yml

@@ -38,18 +38,33 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
+    - name: Get correct ref
+      run: echo "GH_REF=${{ github.event_name == 'pull_request' }} && ${{ github.head_ref }} || ${{ github.ref_name }}" >> $GITHUB_ENV
+
+    - name: Activate environment
+      run: spack env activate ${{ inputs.package-name }}
+
     - name: Build package via spack
+      run: spack -d install --add --only package --fail-fast --no-checksum ${{ inputs.package-name }}@=$GH_REF%${{ inputs.compiler-name }}@${{ inputs.compiler-version }}
+
+    - name: Generate lockfile
       run: |
-        spack -d install --only package --no-checksum ${{ inputs.package-name }}@=$GITHUB_REF_NAME%${{ inputs.compiler-name }}@${{ inputs.compiler-version }}
-      if: github.event_name != 'pull_request'
+        cd /opt/spack/var/spack/environments/${{ inputs.package-name }}
+        cp spack.lock ${{ inputs.package-name }}.original.spack.lock
 
-    # Workaround for this issue: https://github.com/github/docs/issues/15319
-    # In the case of a pull request, we need to use GITHUB_HEAD_REF to get the
-    # correct branch name
-    - name: Build package via spack
+    - name: Generate force-concretized lockfile
+      if: failure()
       run: |
-        spack -d install --only package --no-checksum ${{ inputs.package-name }}@=$GITHUB_HEAD_REF%${{ inputs.compiler-name }}@${{ inputs.compiler-version }}
-      if: github.event_name == 'pull_request'
+        cd /opt/spack/var/spack/environments/${{ inputs.package-name }}
+        spack -d concretize --force
+        cp spack.lock ${{ inputs.package-name }}.force.spack.lock
+
+    - name: Upload lockfiles
+      uses: actions/upload-artifact@v3
+      with: 
+        name: lockfile-output
+        path: /opt/spack/var/spack/environments/${{ inputs.package-name }}/${{ inputs.package-name }}.*.spack.lock
+
 
     # Install and launch interactive debugging on build failure
     - name: Install tmate on failure

.gitignore

@@ -1,4 +1,4 @@
 keys
 __pycache__
 .pytest_cache
-.vscode
+.vscode

containers/Dockerfile.access-om2

@@ -0,0 +1,24 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ARG PACKAGE_NAMES="libaccessom2 mom5 cice5 oasis3-mct"
+ARG COMPILER_PACKAGE
+ARG COMPILER_VERSION
+ARG COMPILER_NAME
+
+WORKDIR $SPACK_ROOT
+
+LABEL au.org.access-nri.ci.compiler ${COMPILER_PACKAGE}@${COMPILER_VERSION}
+LABEL au.org.access-nri.ci.packages ${PACKAGE_NAMES}
+LABEL au.org.access-nri.ci.base-spack-image ${BASE_IMAGE}
+
+# Use Spack shell environment for subsequent RUN steps
+SHELL ["docker-shell"]
+
+COPY setup-spack-envs.sh .
+
+RUN chmod +x setup-spack-envs.sh \ 
+ && ./setup-spack-envs.sh ${COMPILER_PACKAGE} ${COMPILER_NAME} ${COMPILER_VERSION} "${PACKAGE_NAMES}"
+
+# Push any uncached binaries to buildcache
+RUN spack -d buildcache create -a -m s3_buildcache `spack find --json | jq --raw-output .[].name`

containers/Dockerfile.access-om3

@@ -0,0 +1,24 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ARG PACKAGE_NAMES="access-om3"
+ARG COMPILER_PACKAGE
+ARG COMPILER_VERSION
+ARG COMPILER_NAME
+
+WORKDIR $SPACK_ROOT
+
+LABEL au.org.access-nri.ci.compiler ${COMPILER_PACKAGE}@${COMPILER_VERSION}
+LABEL au.org.access-nri.ci.packages ${PACKAGE_NAMES}
+LABEL au.org.access-nri.ci.base-spack-image ${BASE_IMAGE}
+
+# Use Spack shell environment for subsequent RUN steps
+SHELL ["docker-shell"]
+
+COPY setup-spack-envs.sh .
+
+RUN chmod +x setup-spack-envs.sh \ 
+ && ./setup-spack-envs.sh ${COMPILER_PACKAGE} ${COMPILER_NAME} ${COMPILER_VERSION} "${PACKAGE_NAMES}"
+
+# Push any uncached binaries to buildcache
+RUN spack -d buildcache create -a -m s3_buildcache `spack find --json | jq --raw-output .[].name`

containers/Dockerfile.base-os

@@ -0,0 +1,15 @@
+FROM rockylinux:8.8
+
+# csh currently required for some build scripts e.g. MOM5
+RUN dnf -y group install "Development Tools" \
+ && dnf -y install \
+    gcc-gfortran \
+    git \
+    jq \
+    python3 \
+    python3-pip \
+    csh \
+ && dnf clean all \
+ && rm -rf /var/cache/yum
+
+RUN pip3 install --no-cache-dir boto3==1.23.10

containers/Dockerfile.base-spack

@@ -1,9 +1,9 @@
-FROM rockylinux:8.7
+FROM ghcr.io/access-nri/base-os:latest
 
 SHELL ["/bin/bash", "-c"]
 
 ENV SPACK_ROOT=/opt/spack
-ENV SPACK_PACKAGES_ROOT=/opt/spack_packages
+ENV SPACK_PACKAGES_REPO_ROOT=/opt/spack_packages
 ENV GNUPGHOME=$SPACK_ROOT/opt/spack/gpg
 
 ARG SPACK_PACKAGES_REPO_VERSION
@@ -12,20 +12,6 @@ ARG SPACK_REPO_VERSION=v0.20.1
 LABEL au.org.access-nri.ci.spack-packages-repo-version ${SPACK_PACKAGES_REPO_VERSION}
 LABEL au.org.access-nri.ci.spack-repo-version ${SPACK_REPO_VERSION}
 
-# csh currently required for some build scripts e.g. MOM5
-RUN dnf -y group install "Development Tools" \
- && dnf -y install \
-    gcc-gfortran \
-    git \
-    jq \
-    python3 \
-    python3-pip \
-    csh \
- && dnf clean all \
- && rm -rf /var/cache/yum
-
-RUN pip3 install --no-cache-dir boto3==1.23.10
-
 # Install spack
 RUN git clone -c feature.manyFiles=true https://github.com/spack/spack.git $SPACK_ROOT --branch ${SPACK_REPO_VERSION} --single-branch --depth=1
 ENV PATH="$SPACK_ROOT/bin:$PATH"
@@ -48,7 +34,8 @@ SHELL ["docker-shell"]
 RUN spack bootstrap now
 
 # Set up ACCESS Spack package repo
-RUN git clone  https://github.com/ACCESS-NRI/spack_packages.git ${SPACK_PACKAGES_ROOT} --branch ${SPACK_PACKAGES_REPO_VERSION} --single-branch --depth=1
+RUN git clone  https://github.com/ACCESS-NRI/spack_packages.git ${SPACK_PACKAGES_REPO_ROOT} --branch ${SPACK_PACKAGES_REPO_VERSION} --single-branch --depth=1
+
 COPY repos.yaml $SPACK_ROOT/etc/spack/repos.yaml
 
 # TODO

containers/compilers.yaml

@@ -0,0 +1,7 @@
+[
+  {
+    name: "intel",
+    package: "intel-oneapi-compilers",
+    version: "2021.1.2",
+  },
+]

containers/setup-spack-envs.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# usage: ./setup-spack-envs <compiler package> <compiler name> <compiler version> <models string>
+# eg: 
+if [ "$#" -ne 4 ]; then
+  echo "Invalid number of arguments: expecting 4"
+  exit 1
+fi
+
+COMPILER_PACKAGE="$1"
+COMPILER_NAME="$2"
+COMPILER_VERSION="$3"
+PACKAGES="$4"
+
+for PACKAGE in $PACKAGES; do
+  spack env create $PACKAGE
+  spack env activate $PACKAGE
+  spack -d install -j 4 --add --fail-fast $COMPILER_PACKAGE@$COMPILER_VERSION
+  spack load $COMPILER_PACKAGE@$COMPILER_VERSION
+  spack compiler find --scope env:$PACKAGE
+  spack -d install -j 4 --add --only dependencies --fail-fast $PACKAGE%$COMPILER_NAME@$COMPILER_VERSION
+  spack gc -y
+  spack env deactivate
+done