gitlab: refactor using dependent jobs

Problem: subsequent framework projects will need to link against flux-core and its associated libraries. These projects (such as flux-pmix, flux-sched, and flux-coral2) are needed on various LC systems for testing flux with our LC flavors of MPI. Adapt the flux-core build to build in a shared workspace directory unique to each pipeline (set of GitLab CI jobs) and cluster. Break the core build and test into separate steps for speed. Build and install subsequent framework projects in the same shared directory, which will persist after the tests.
flux-framework · Jan 26, 2024 · 01cd017 · 01cd017
1 parent 4e2eb29
commit 01cd017
Show file tree

Hide file tree

Showing 2 changed files with 74 additions and 23 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -12,7 +12,8 @@ include:
   - local: '.gitlab/builds.gitlab-ci.yml'
   - local: '.gitlab/machines.gitlab-ci.yml'
 
-stages: 
+stages:
+  - build
   - test
 
 default:
@@ -25,18 +26,23 @@ default:
         CUSTOM_CI_BUILDS_DIR: "/usr/WS1/$$USER/gitlab-runner-builds-dir"
         # Note: the above will not work with /usr/workspace, you must specify WS1 or WS2
 
+## Job Specifications
 .test-core:
     extends: .lc-variables
     variables:
         PYTHON: "/usr/bin/python3"
         debug: t
         FLUX_TESTS_LOGFILE: t
     script:
+        - echo $CI_DIRECTORY
+        - cd ${CORE_BUILD_DIR}
         - lstopo --of xml >$(hostname).xml
         - export FLUX_HWLOC_XMLFILE=$(pwd)/$(hostname).xml
-        - !reference ['.build-core', 'script']
-        - cd ${CORE_BUILD_DIR}
         - make -j $(nproc) check
+        ## The following is teardown to make sure the xml file doesn't get reused
+        ## but maybe it's unnecessary. Subsequent tests will run under new shells...
+        - rm ${FLUX_HWLOC_XMLFILE}
+        - unset FLUX_HWLOC_XMLFILE
 
 .test-pmix:
     extends: .lc-variables
@@ -45,60 +51,88 @@ default:
         debug: t
         FLUX_TESTS_LOGFILE: t
     script:
-        - export FTC_DIRECTORY=$(pwd)
-        - !reference ['.build-core', 'script']
         - export PKG_CONFIG_PATH=${CORE_INSTALL_PREFIX}/lib/pkgconfig:$(pkg-config --variable pc_path pkg-config)
         - module load openmpi
         - export PKG_CONFIG_PATH=$(dirname $(which mpicc))/../lib/pkgconfig:${PKG_CONFIG_PATH} 
-        - cd ${FTC_DIRECTORY}
+        - cd ${CI_DIRECTORY}
         - git clone https://github.com/flux-framework/flux-pmix
         - cd flux-pmix
         - ./autogen.sh
-        - ./configure
+        - ./configure --prefix=${CORE_INSTALL_PREFIX}
         - make -j $(nproc) check
+        - make -j $(nproc) install
 
 .test-core-mpi:
     extends: .lc-variables
-    variables: 
+    ## this will need coral2 XOR pmix depending on system (hopefully both eventually)
+    variables:
         PYTHON: "/usr/bin/python3"
     script:
         - export MPI_TESTS_DIRECTORY=$(pwd)/mpi
         - export FTC_DIRECTORY=$(pwd)
-        - !reference ['.build-core', 'script']
         - flux run -N2 $CORE_BUILD_DIR/src/cmd/flux start $MPI_TESTS_DIRECTORY/outer_script.sh
 
-## Job Specifications
+corona-core-build:
+    extends: 
+        - .build-core
+        - .corona
+    stage: build
+
 corona-core-test:
+    needs: ["corona-core-build"]
     extends: 
         - .test-core
         - .corona
     stage: test
 
+poodle-core-build:
+    extends: 
+        - .build-core
+        - .poodle
+    stage: build
+
 poodle-core-test:
+    needs: ["poodle-core-build"]
     extends: 
         - .test-core
         - .poodle
     stage: test
 
+tioga-core-build:
+    extends: 
+        - .build-core
+        - .tioga
+    stage: build
+
 tioga-core-test:
+    needs: ["tioga-core-build"]
     extends: 
         - .test-core
         - .tioga
     stage: test
 
+quartz-core-build:
+    extends: 
+        - .build-core
+        - .quartz
+    stage: build
+
 quartz-core-test:
+    needs: ["quartz-core-build"]
     extends: 
         - .test-core
         - .quartz
     stage: test
 
 corona-pmix-test:
+    needs: ["corona-core-build"]
     extends:
         - .test-pmix
         - .corona
     stage: test
 
 corona-mpi-test:
+    needs: ["corona-core-build", "corona-pmix-test"]
     extends:
         - .test-core-mpi
         - .corona

diff --git a/.gitlab/builds.gitlab-ci.yml b/.gitlab/builds.gitlab-ci.yml
@@ -9,16 +9,33 @@
 ##############################################################
 
 .build-core:
-  script:
-    - git clone https://github.com/flux-framework/flux-core
-    - cd flux-core
-    - ./autogen.sh
-    - mkdir build
-    - cd build
-    - mkdir install
-    - ../configure --prefix=$(pwd)/install
-    - make -j $(nproc)
-    - make -j $(nproc) install
-    - export CORE_BUILD_DIR=$(pwd)
-    - export CORE_INSTALL_PREFIX=$(pwd)/install
-
+    extends: .lc-variables
+    variables:
+        PYTHON: "/usr/bin/python3"
+    script:
+        - test -n ${CI_PIPELINE_ID}
+        - export FTC_DIRECTORY=$(pwd)
+        - mkdir /usr/WS1/${USER}/cibuilds/${CI_PIPELINE_ID}_${LCSCHEDCLUSTER}
+        - export CI_DIRECTORY=/usr/WS1/${USER}/cibuilds/${CI_PIPELINE_ID}_${LCSCHEDCLUSTER}
+        - cd ${CI_DIRECTORY}
+        - git clone https://github.com/flux-framework/flux-core
+        - cd flux-core
+        - ./autogen.sh
+        - mkdir build
+        - cd build
+        - mkdir install
+        - ../configure --prefix=$(pwd)/install
+        - make -j $(nproc)
+        ## need to install to get pkgconfig files
+        - export CORE_BUILD_DIR=$(pwd)
+        - export CORE_INSTALL_PREFIX=$(pwd)/install
+        - cd $FTC_DIRECTORY
+        - echo "CI_DIRECTORY=${CI_DIRECTORY}" >> build.env
+        - echo "CORE_BUILD_DIR=${CORE_BUILD_DIR}" >> build.env
+        - echo "CORE_INSTALL_PREFIX=${CORE_INSTALL_PREFIX}" >> build.env
+    ## Save the exported directory names so later jobs can use the binaries that have already been built
+    ## send these dirnames to tests as a .env file
+    artifacts:
+        reports:
+            dotenv:
+                - build.env