From 96b2e0d359eed582269b237b85b92a3d853463df Mon Sep 17 00:00:00 2001
From: William Hobbs <hobbs17@llnl.gov>
Date: Wed, 7 Feb 2024 16:25:56 -0800
Subject: [PATCH] mpi: add testing support for openmpi5

Problem: flux-pmix is required by openmpi5, which means
we should be testing openmpi5 on LC hardware.

Utilize modulefiles on Corona to test this. Since loading new
modulefiles can cause unexpected behavior in tests for openmpi4,
only load those modulefiles when openmpi5 is the requested MPI
to test.
---
 mpi/inner_script.sh |  9 +++++++--
 mpi/outer_script.sh | 11 ++++++-----
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/mpi/inner_script.sh b/mpi/inner_script.sh
index abe5776..b4d6d59 100755
--- a/mpi/inner_script.sh
+++ b/mpi/inner_script.sh
@@ -10,17 +10,22 @@ BATCH_NNODES=$(flux resource list -n -o {nnodes})
 BATCH_NCORES=$(flux resource list -n -o {ncores})
 COMPILER=$1
 MPI=$2
-export NAME="$COMPILER"_"$MPI"
+export NAME=$(echo "$COMPILER"_"$MPI" | sed "s/\///")
 
 test -n $COMPILER || die "COMPILER (argument 1) not set"
 test -n $MPI || die "MPI (argument 2) not set"
+if [[ $MPI == "openmpi/5" ]]; then
+    ## This is the Corona OpenMPI5 side-installed modulefile. Only load when
+    ## openmpi5 is needed.
+    module use -q /usr/global/tools/mpi/toss_4_x86_64_ib/modulefiles/Core/
+fi
 module load $COMPILER || die "Compiler $COMPILER is unavailable on $LCSCHEDCLUSTER"
 module load $MPI || die "MPI implementation $MPI is unavailable on $LCSCHEDCLUSTER"
 test -n $FTC_DIRECTORY || die "FTC_DIRECTORY not set"
 mkdir $FTC_DIRECTORY/$NAME || die "Unable to create directory for $FTC_DIRECTORY/$NAME"
 cp -r $MPI_TESTS_DIRECTORY/* $FTC_DIRECTORY/$NAME
 cd $FTC_DIRECTORY/$NAME || die "Could not find $FTC_DIRECTORY/$NAME"
-echo "Running with $1 compiler and $2 MPI"
+echo "Running with $COMPILER compiler and $MPI MPI"
 flux bulksubmit -n1 --watch mpicc -o {} {}.c ::: $TESTS || die "Compilation failure in tests"
 flux bulksubmit --watch -N $BATCH_NNODES -n $BATCH_NCORES $EXTRA_FLUX_SUBMIT_OPTIONS --output=kvs ./{} ::: $TESTS
 RC=$?
diff --git a/mpi/outer_script.sh b/mpi/outer_script.sh
index fe29eca..2423a91 100755
--- a/mpi/outer_script.sh
+++ b/mpi/outer_script.sh
@@ -10,7 +10,8 @@ intel-classic
 
 corona_MPIS="
 mvapich2
-openmpi
+openmpi/4
+openmpi/5
 "
 
 export TESTS="hello
@@ -33,11 +34,11 @@ COMPILERS="${LCSCHEDCLUSTER}_COMPILERS"
 for mpi in ${!MPIS}; do
     for compiler in ${!COMPILERS}; do
         if [[ $mpi == "cray-mpich" ]]; then
-            EXTRA_FLUX_SUBMIT_OPTIONS="-o pmi=cray-pals" flux batch -N2 -n4 --flags=waitable --output=kvs $MPI_TESTS_DIRECTORY/inner_script.sh $mpi $compiler
-        elif [[ $mpi == "openmpi" ]]; then
-            EXTRA_FLUX_SUBMIT_OPTIONS="-o pmi=pmix" flux batch -N2 -n4 --flags=waitable --output=kvs $MPI_TESTS_DIRECTORY/inner_script.sh $mpi $compiler
+            EXTRA_FLUX_SUBMIT_OPTIONS="-o pmi=cray-pals" flux batch -N2 -n4 --flags=waitable --output=kvs $MPI_TESTS_DIRECTORY/inner_script.sh $compiler $mpi
+        elif [[ $mpi == "openmpi"* ]]; then
+            EXTRA_FLUX_SUBMIT_OPTIONS="-o pmi=pmix" flux batch -N2 -n4 --flags=waitable --output=kvs $MPI_TESTS_DIRECTORY/inner_script.sh $compiler $mpi
         else
-            flux batch -N2 -n4 --flags=waitable --output=kvs $MPI_TESTS_DIRECTORY/inner_script.sh $mpi $compiler
+            flux batch -N2 -n4 --flags=waitable --output=kvs $MPI_TESTS_DIRECTORY/inner_script.sh $compiler $mpi
         fi 
     done
 done