From 2ef24a949c16faea408d56922727f39b0a46bc88 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Wed, 7 Aug 2024 12:36:46 -0700
Subject: [PATCH 01/35] create CICD init script

---
 .cicd/scripts/land_init.sh | 57 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100755 .cicd/scripts/land_init.sh

diff --git a/.cicd/scripts/land_init.sh b/.cicd/scripts/land_init.sh
new file mode 100755
index 00000000..019c4878
--- /dev/null
+++ b/.cicd/scripts/land_init.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#!/usr/bin/env bash
+#
+# A unified init script for the Land-DA_workflow application. This script is expected to
+# fetch and dependent source for the LAND-DA_workflow application for all supported platforms.
+#
+pwd
+set +x
+echo "UFS_PLATFORM=${UFS_PLATFORM}"
+echo "UFS_COMPILER=${UFS_COMPILER}"
+[[ -n $1 ]] && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
+[[ -n $2 ]] && export compiler=${2} || export compiler=${UFS_COMPILER}
+[[ -n ${WORKSPACE} ]] && export workspace=${WORKSPACE} || export workspace=$(pwd)
+echo "machine=${machine}"
+echo "compiler=${compiler}"
+echo "workspace=${workspace}"
+
+set -e -u -x
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+
+# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
+# relative to script directory.
+declare workspace
+if [[ -d "${WORKSPACE}/${UFS_PLATFORM}" ]]; then
+    workspace="${WORKSPACE}/${UFS_PLATFORM}"
+else
+    workspace="$(cd -- "${script_dir}/../.." && pwd)"
+fi
+
+# Normalize Parallel Works cluster platform value.
+declare platform
+if [[ "${UFS_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then
+    platform='noaacloud'
+else
+    platform="${UFS_PLATFORM}"
+fi
+
+# Initialize
+cd ${workspace}
+pwd
+set +e
+/usr/bin/time -p -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' -o ${WORKSPACE}/${UFS_PLATFORM}-${UFS_COMPILER}-time-srw_init.json \
+    find . -name .git -type d
+git branch
+git log -1 --oneline
+git status
+
+init_exit=$?
+echo "STAGE_NAME=${STAGE_NAME:=manual}"
+env | grep = | sort > ${WORKSPACE}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
+set -e
+cd -
+pwd
+
+exit $init_exit
+

From 278867fa2e3442680ba390d41bffed0f198d1490 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Wed, 7 Aug 2024 16:01:25 -0700
Subject: [PATCH 02/35] add CICD build and test scripts

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/scripts/land_build.sh |  87 ++++++++++++++++++++++++++++
 .cicd/scripts/land_init.sh  |  20 ++++---
 .cicd/scripts/land_test.sh  | 109 ++++++++++++++++++++++++++++++++++++
 3 files changed, 207 insertions(+), 9 deletions(-)
 create mode 100755 .cicd/scripts/land_build.sh
 create mode 100755 .cicd/scripts/land_test.sh

diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
new file mode 100755
index 00000000..f13458d4
--- /dev/null
+++ b/.cicd/scripts/land_build.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+#
+# A unified build script for the Land-DA_workflow application. This script is expected to
+# build Land-DA_workflow model from source for all supported platforms.
+#
+pwd
+set +x
+echo "UFS_PLATFORM=${UFS_PLATFORM}"
+echo "UFS_COMPILER=${UFS_COMPILER}"
+[[ -n $1 ]] && export UFS_PLATFORM=${1} && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
+[[ -n $2 ]] && export UFS_COMPILER=${2} && export compiler=${2} || export compiler=${UFS_COMPILER}
+[[ -n ${WORKSPACE} ]] && export workspace=${WORKSPACE} || export workspace=$(pwd)
+echo "machine=${machine}"
+echo "compiler=${compiler}"
+echo "workspace=${workspace}"
+
+echo "HOME=${HOME}"
+[[ ${machine} = hera ]] && NODE_PATH="/scratch2/NAGAPE/epic/role.epic"
+[[ ${machine} = jet ]] && NODE_PATH="/mnt/lfs4/HFIP/hfv3gfs/role.epic"
+[[ ${machine} = gaea ]] && NODE_PATH=""
+[[ ${machine} = orion ]] && NODE_PATH="/work/noaa/epic/role-epic"
+[[ ${machine} = hercules ]] && NODE_PATH="/work/noaa/epic/role-epic"
+echo "NODE_PATH=${NODE_PATH}"
+
+( set -x ; ls -ld ${NODE_PATH} && ls -al ${NODE_PATH}/. )
+
+[[ ${machine} = hera ]] && ls -l /scratch2/NAGAPE/epic/UFS_Land-DA/inputs
+[[ ${machine} = jet ]] && echo "where are inputs?"
+[[ ${machine} = gaea ]] && echo "where are inputs?"
+[[ ${machine} = orion ]] && ls -l /work/noaa/epic/UFS_Land-DA/inputs
+[[ ${machine} = hercules ]] && ls -l /work/noaa/epic/UFS_Land-DA/inputs
+
+set -e -u -x
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+
+# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
+# relative to script directory.
+declare workspace
+if [[ -d "${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}" ]]; then
+    workspace="${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}"
+else
+    workspace="$(cd -- "${script_dir}/../.." && pwd)"
+fi
+
+# Normalize Parallel Works cluster platform value.
+declare platform
+if [[ "${UFS_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then
+    platform='noaacloud'
+else
+    platform="${UFS_PLATFORM}"
+fi
+
+if [[ ${platform} = derecho ]] ; then
+	export ACCNR=nral0032
+elif [[ ${platform} = jet ]] ; then
+	export ACCNR=hfv3gfs
+else
+	export ACCNR=epic
+fi
+echo "ACCNR=${ACCNR}"
+
+# Build
+cd ${workspace}
+pwd
+set +e
+echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR}."
+/usr/bin/time -p \
+	-o ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_build.json \
+	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+	sorc/app_build.sh -p=${machine} -c=${compiler} --conda=off --build 2>&1 | tee ./build-log.txt
+status=${PIPESTATUS[0]}
+cat sorc/build/log.ecbuild sorc/build/log.make >> ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}/build-log.txt
+echo "Pipeline Completed Land-DA build on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
+git status
+                                    
+ls -l sorc/build
+
+build_exit=$?
+echo "STAGE_NAME=${STAGE_NAME:=manual}"
+env | grep = | sort > ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
+set -e
+cd -
+pwd
+
+exit $build_exit
+
diff --git a/.cicd/scripts/land_init.sh b/.cicd/scripts/land_init.sh
index 019c4878..64c70d44 100755
--- a/.cicd/scripts/land_init.sh
+++ b/.cicd/scripts/land_init.sh
@@ -1,15 +1,14 @@
 #!/usr/bin/env bash
-#!/usr/bin/env bash
 #
 # A unified init script for the Land-DA_workflow application. This script is expected to
-# fetch and dependent source for the LAND-DA_workflow application for all supported platforms.
+# fetch and dependent source for the Land-DA_workflow application for all supported platforms.
 #
 pwd
 set +x
 echo "UFS_PLATFORM=${UFS_PLATFORM}"
 echo "UFS_COMPILER=${UFS_COMPILER}"
-[[ -n $1 ]] && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
-[[ -n $2 ]] && export compiler=${2} || export compiler=${UFS_COMPILER}
+[[ -n $1 ]] && export UFS_PLATFORM=${1} && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
+[[ -n $2 ]] && export UFS_COMPILER=${2} && export compiler=${2} || export compiler=${UFS_COMPILER}
 [[ -n ${WORKSPACE} ]] && export workspace=${WORKSPACE} || export workspace=$(pwd)
 echo "machine=${machine}"
 echo "compiler=${compiler}"
@@ -22,8 +21,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)
 # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
 # relative to script directory.
 declare workspace
-if [[ -d "${WORKSPACE}/${UFS_PLATFORM}" ]]; then
-    workspace="${WORKSPACE}/${UFS_PLATFORM}"
+if [[ -d "${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}" ]]; then
+    workspace="${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}"
 else
     workspace="$(cd -- "${script_dir}/../.." && pwd)"
 fi
@@ -40,15 +39,18 @@ fi
 cd ${workspace}
 pwd
 set +e
-/usr/bin/time -p -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' -o ${WORKSPACE}/${UFS_PLATFORM}-${UFS_COMPILER}-time-srw_init.json \
-    find . -name .git -type d
+rm -rf sorc/build
+/usr/bin/time -p \
+	-o ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_init.json \
+	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+	find . -name .git -type d
 git branch
 git log -1 --oneline
 git status
 
 init_exit=$?
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
-env | grep = | sort > ${WORKSPACE}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
+env | grep = | sort > ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
 set -e
 cd -
 pwd
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
new file mode 100755
index 00000000..3c164550
--- /dev/null
+++ b/.cicd/scripts/land_test.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+#
+# A unified test script for the Land-DA_workflow application. This script is expected to
+# test Land-DA_workflow model for any supported platforms.
+#
+pwd
+set +x
+echo "UFS_PLATFORM=${UFS_PLATFORM}"
+echo "UFS_COMPILER=${UFS_COMPILER}"
+echo "LAND_DA_RUN_TESTS=${LAND_DA_RUN_TESTS}"
+[[ -n $1 ]] && export UFS_PLATFORM=${1} && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
+[[ -n $2 ]] && export UFS_COMPILER=${2} && export compiler=${2} || export compiler=${UFS_COMPILER}
+[[ -n $3 ]] && export LAND_DA_RUN_TESTS=${3}
+[[ -n ${WORKSPACE} ]] && export workspace=${WORKSPACE} || export workspace=$(pwd)
+echo "machine=${machine}"
+echo "compiler=${compiler}"
+echo "workspace=${workspace}"
+
+echo "HOME=${HOME}"
+[[ ${machine} = hera ]] && NODE_PATH="/scratch2/NAGAPE/epic/role.epic"
+[[ ${machine} = jet ]] && NODE_PATH="/mnt/lfs4/HFIP/hfv3gfs/role.epic"
+[[ ${machine} = gaea ]] && NODE_PATH=""
+[[ ${machine} = orion ]] && NODE_PATH="/work/noaa/epic/role-epic"
+[[ ${machine} = hercules ]] && NODE_PATH="/work/noaa/epic/role-epic"
+echo "NODE_PATH=${NODE_PATH}"
+
+( set -x ; ls -ld ${NODE_PATH} && ls -al ${NODE_PATH}/. )
+
+[[ ${machine} = hera ]] && ls -l /scratch2/NAGAPE/epic/UFS_Land-DA/inputs
+[[ ${machine} = jet ]] && echo "where are inputs?"
+[[ ${machine} = gaea ]] && echo "where are inputs?"
+[[ ${machine} = orion ]] && ls -l /work/noaa/epic/UFS_Land-DA/inputs
+[[ ${machine} = hercules ]] && ls -l /work/noaa/epic/UFS_Land-DA/inputs
+
+set -e -u -x
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+
+# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
+# relative to script directory.
+declare workspace
+if [[ -d "${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}" ]]; then
+    workspace="${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}"
+else
+    workspace="$(cd -- "${script_dir}/../.." && pwd)"
+fi
+
+# Normalize Parallel Works cluster platform value.
+declare platform
+if [[ "${UFS_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then
+    platform='noaacloud'
+else
+    platform="${UFS_PLATFORM}"
+fi
+
+if [[ ${platform} = derecho ]] ; then
+	export ACCNR=nral0032
+elif [[ ${platform} = jet ]] ; then
+	export ACCNR=hfv3gfs
+else
+	export ACCNR=epic
+fi
+echo "ACCNR=${ACCNR}"
+
+# Test
+cd ${workspace}
+pwd
+set +e
+ls -l sorc/build
+
+if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
+
+	cp sorc/test/${machine}_ctest.sh sorc/build/
+	cp sorc/test/check_ctest.sh sorc/build/
+	cp sorc/test/run_${machine}_ctest.sh sorc/build/
+                                        
+	cd sorc/build/
+	set +x
+ 	echo $(pwd)
+                                            
+	echo "Pipeline Running Land-DA Tests on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR}."
+	/usr/bin/time -p \
+		-o ${WORKSPACE}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_test.json \
+		-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+		./run_${machine}_ctest.sh | awk 'f;/^+ ctest$/{f=1}' | tee ${WORKSPACE}/${UFS_PLATFORM}/test-log.txt
+	status=${PIPESTATUS[0]}
+	echo "Pipeline Completed Land-DA Tests on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
+	rc=$(( status+=$(egrep " ***Failed " ${WORKSPACE}/${UFS_PLATFORM}/test-log.txt 2>/dev/null | wc -l) ))
+	echo "rc=$rc status=$status"
+	set -x
+	cd -
+else
+	echo "Pipeline skipping Tests on ${UFS_PLATFORM} (${machine})"
+	rc=0
+	status=0
+fi
+
+echo "Pipeline Completed Land-DA test on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
+git status
+
+test_exit=$?
+echo "STAGE_NAME=${STAGE_NAME:=manual}"
+env | grep = | sort > ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
+set -e
+cd -
+pwd
+
+exit $test_exit
+

From 4dfe2d2a0cc5b9306175b2cbce2d72eb2b79ba89 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Thu, 8 Aug 2024 09:26:06 -0700
Subject: [PATCH 03/35] fix CICD script location of test-log

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/scripts/land_build.sh | 4 ++--
 .cicd/scripts/land_test.sh  | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
index f13458d4..1b3b430a 100755
--- a/.cicd/scripts/land_build.sh
+++ b/.cicd/scripts/land_build.sh
@@ -68,9 +68,9 @@ echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=
 /usr/bin/time -p \
 	-o ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_build.json \
 	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
-	sorc/app_build.sh -p=${machine} -c=${compiler} --conda=off --build 2>&1 | tee ./build-log.txt
+	sorc/app_build.sh -p=${machine} -c=${compiler} --conda=off --build 2>&1 | tee ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-build-log.txt
 status=${PIPESTATUS[0]}
-cat sorc/build/log.ecbuild sorc/build/log.make >> ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}/build-log.txt
+cat sorc/build/log.ecbuild sorc/build/log.make >> ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-build-log.txt
 echo "Pipeline Completed Land-DA build on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
 git status
                                     
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 3c164550..00495d1f 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -75,6 +75,7 @@ if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 	cp sorc/test/run_${machine}_ctest.sh sorc/build/
                                         
 	cd sorc/build/
+	rm -f out.*
 	set +x
  	echo $(pwd)
                                             
@@ -82,10 +83,10 @@ if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 	/usr/bin/time -p \
 		-o ${WORKSPACE}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_test.json \
 		-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
-		./run_${machine}_ctest.sh | awk 'f;/^+ ctest$/{f=1}' | tee ${WORKSPACE}/${UFS_PLATFORM}/test-log.txt
+		./run_${machine}_ctest.sh | awk 'f;/^+ ctest$/{f=1}' | tee ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-test-log.txt
 	status=${PIPESTATUS[0]}
 	echo "Pipeline Completed Land-DA Tests on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
-	rc=$(( status+=$(egrep " ***Failed " ${WORKSPACE}/${UFS_PLATFORM}/test-log.txt 2>/dev/null | wc -l) ))
+	rc=$(( status+=$(egrep " ***Failed " ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-test-log.txt 2>/dev/null | wc -l) ))
 	echo "rc=$rc status=$status"
 	set -x
 	cd -

From 2faa9aeddb3b25d846bdacd599156e215cf633c5 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Thu, 8 Aug 2024 12:20:59 -0700
Subject: [PATCH 04/35] fix CICD workspace variable

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/scripts/land_build.sh |  4 ++--
 .cicd/scripts/land_init.sh  | 15 ++++++++++-----
 .cicd/scripts/land_test.sh  |  4 ++--
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
index 1b3b430a..9ef74a11 100755
--- a/.cicd/scripts/land_build.sh
+++ b/.cicd/scripts/land_build.sh
@@ -66,7 +66,7 @@ pwd
 set +e
 echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR}."
 /usr/bin/time -p \
-	-o ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_build.json \
+	-o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_build.json \
 	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
 	sorc/app_build.sh -p=${machine} -c=${compiler} --conda=off --build 2>&1 | tee ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-build-log.txt
 status=${PIPESTATUS[0]}
@@ -78,7 +78,7 @@ ls -l sorc/build
 
 build_exit=$?
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
-env | grep = | sort > ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
+env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
 set -e
 cd -
 pwd
diff --git a/.cicd/scripts/land_init.sh b/.cicd/scripts/land_init.sh
index 64c70d44..edb7c205 100755
--- a/.cicd/scripts/land_init.sh
+++ b/.cicd/scripts/land_init.sh
@@ -40,17 +40,22 @@ cd ${workspace}
 pwd
 set +e
 rm -rf sorc/build
-/usr/bin/time -p \
-	-o ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_init.json \
-	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
-	find . -name .git -type d
+
+(
+set -x
 git branch
 git log -1 --oneline
 git status
+)
+
+/usr/bin/time -p \
+	-o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_init.json \
+	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+	find . -name .git -type d
 
 init_exit=$?
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
-env | grep = | sort > ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
+env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
 set -e
 cd -
 pwd
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 00495d1f..66c60010 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -81,7 +81,7 @@ if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
                                             
 	echo "Pipeline Running Land-DA Tests on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR}."
 	/usr/bin/time -p \
-		-o ${WORKSPACE}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_test.json \
+		-o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_test.json \
 		-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
 		./run_${machine}_ctest.sh | awk 'f;/^+ ctest$/{f=1}' | tee ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-test-log.txt
 	status=${PIPESTATUS[0]}
@@ -101,7 +101,7 @@ git status
 
 test_exit=$?
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
-env | grep = | sort > ${WORKSPACE:=$(pwd)}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
+env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
 set -e
 cd -
 pwd

From 7a022e2577d2154277a6f0ca4cb27ceb94fa5a20 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Fri, 9 Aug 2024 11:35:02 -0700
Subject: [PATCH 05/35] add usage help to CICD scripts

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/scripts/land_build.sh | 47 +++++++++++++++----------------------
 .cicd/scripts/land_init.sh  | 18 ++++++++++----
 .cicd/scripts/land_test.sh  | 15 ++++++++----
 3 files changed, 42 insertions(+), 38 deletions(-)

diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
index 9ef74a11..11d70f0c 100755
--- a/.cicd/scripts/land_build.sh
+++ b/.cicd/scripts/land_build.sh
@@ -3,6 +3,10 @@
 # A unified build script for the Land-DA_workflow application. This script is expected to
 # build Land-DA_workflow model from source for all supported platforms.
 #
+# Usage:
+#  UFS_PLATFORM=<platform> UFS_COMPILER=<compiler> .cicd/scripts/land_build.sh
+#  .cicd/scripts/land_build.sh <platform> <compiler>
+#
 pwd
 set +x
 echo "UFS_PLATFORM=${UFS_PLATFORM}"
@@ -14,24 +18,11 @@ echo "machine=${machine}"
 echo "compiler=${compiler}"
 echo "workspace=${workspace}"
 
-echo "HOME=${HOME}"
-[[ ${machine} = hera ]] && NODE_PATH="/scratch2/NAGAPE/epic/role.epic"
-[[ ${machine} = jet ]] && NODE_PATH="/mnt/lfs4/HFIP/hfv3gfs/role.epic"
-[[ ${machine} = gaea ]] && NODE_PATH=""
-[[ ${machine} = orion ]] && NODE_PATH="/work/noaa/epic/role-epic"
-[[ ${machine} = hercules ]] && NODE_PATH="/work/noaa/epic/role-epic"
-echo "NODE_PATH=${NODE_PATH}"
-
-( set -x ; ls -ld ${NODE_PATH} && ls -al ${NODE_PATH}/. )
-
-[[ ${machine} = hera ]] && ls -l /scratch2/NAGAPE/epic/UFS_Land-DA/inputs
-[[ ${machine} = jet ]] && echo "where are inputs?"
-[[ ${machine} = gaea ]] && echo "where are inputs?"
-[[ ${machine} = orion ]] && ls -l /work/noaa/epic/UFS_Land-DA/inputs
-[[ ${machine} = hercules ]] && ls -l /work/noaa/epic/UFS_Land-DA/inputs
-
 set -e -u -x
 
+echo "UFS_PLATFORM=${UFS_PLATFORM}"
+echo "UFS_COMPILER=${UFS_COMPILER}"
+
 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 
 # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
@@ -51,20 +42,20 @@ else
     platform="${UFS_PLATFORM}"
 fi
 
-if [[ ${platform} = derecho ]] ; then
-	export ACCNR=nral0032
-elif [[ ${platform} = jet ]] ; then
-	export ACCNR=hfv3gfs
-else
-	export ACCNR=epic
-fi
-echo "ACCNR=${ACCNR}"
+#if [[ ${platform} = derecho ]] ; then
+#	export ACCNR=nral0032
+#elif [[ ${platform} = jet ]] ; then
+#	export ACCNR=hfv3gfs
+#else
+#	export ACCNR=epic
+#fi
+echo "ACCNR=${ACCNR:=}"
 
 # Build
 cd ${workspace}
 pwd
 set +e
-echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR}."
+echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR:=}."
 /usr/bin/time -p \
 	-o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_build.json \
 	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
@@ -72,11 +63,11 @@ echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=
 status=${PIPESTATUS[0]}
 cat sorc/build/log.ecbuild sorc/build/log.make >> ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-build-log.txt
 echo "Pipeline Completed Land-DA build on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
-git status
+git status -u
                                     
-ls -l sorc/build
+ls -l sorc/build/*
 
-build_exit=$?
+build_exit=$status
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
 env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
 set -e
diff --git a/.cicd/scripts/land_init.sh b/.cicd/scripts/land_init.sh
index edb7c205..3119eaae 100755
--- a/.cicd/scripts/land_init.sh
+++ b/.cicd/scripts/land_init.sh
@@ -3,6 +3,10 @@
 # A unified init script for the Land-DA_workflow application. This script is expected to
 # fetch and dependent source for the Land-DA_workflow application for all supported platforms.
 #
+# Usage:
+#  UFS_PLATFORM=<platform> UFS_COMPILER=<compiler> [ BRANCH_NAME=<branch> ] .cicd/scripts/land_init.sh
+#  .cicd/scripts/land_init.sh <platform> <compiler> [ <branch> ]
+#
 pwd
 set +x
 echo "UFS_PLATFORM=${UFS_PLATFORM}"
@@ -16,6 +20,9 @@ echo "workspace=${workspace}"
 
 set -e -u -x
 
+echo "UFS_PLATFORM=${UFS_PLATFORM}"
+echo "UFS_COMPILER=${UFS_COMPILER}"
+
 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 
 # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
@@ -39,21 +46,22 @@ fi
 cd ${workspace}
 pwd
 set +e
+
 rm -rf sorc/build
 
-(
-set -x
+git checkout ${BRANCH_NAME:=develop}
+status=${PIPESTATUS[0]}
+
 git branch
 git log -1 --oneline
-git status
-)
+git status -u
 
 /usr/bin/time -p \
 	-o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_init.json \
 	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
 	find . -name .git -type d
 
-init_exit=$?
+init_exit=$status
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
 env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
 set -e
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 66c60010..4106959e 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -3,6 +3,10 @@
 # A unified test script for the Land-DA_workflow application. This script is expected to
 # test Land-DA_workflow model for any supported platforms.
 #
+# Usage:
+#  UFS_PLATFORM=<platform> UFS_COMPILER=<compiler> [ LAND_DA_RUN_TESTS=true ] .cicd/scripts/land_test.sh
+#  .cicd/scripts/land_test.sh <platform> <compiler> [ true|false ]
+#
 pwd
 set +x
 echo "UFS_PLATFORM=${UFS_PLATFORM}"
@@ -34,6 +38,9 @@ echo "NODE_PATH=${NODE_PATH}"
 
 set -e -u -x
 
+echo "UFS_PLATFORM=${UFS_PLATFORM}"
+echo "UFS_COMPILER=${UFS_COMPILER}"
+
 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 
 # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
@@ -66,7 +73,7 @@ echo "ACCNR=${ACCNR}"
 cd ${workspace}
 pwd
 set +e
-ls -l sorc/build
+ls -l sorc/build/*
 
 if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 
@@ -92,14 +99,12 @@ if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 	cd -
 else
 	echo "Pipeline skipping Tests on ${UFS_PLATFORM} (${machine})"
-	rc=0
 	status=0
 fi
 
-echo "Pipeline Completed Land-DA test on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
-git status
+git status -u
 
-test_exit=$?
+test_exit=$status
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
 env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
 set -e

From 074c0517bcd35634eb4d7fe56b6383236ebd5f4b Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Fri, 9 Aug 2024 12:01:15 -0700
Subject: [PATCH 06/35] fix BRANCH_NAME checkout for CICD scripts

---
 .cicd/scripts/land_init.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.cicd/scripts/land_init.sh b/.cicd/scripts/land_init.sh
index 3119eaae..be7a007b 100755
--- a/.cicd/scripts/land_init.sh
+++ b/.cicd/scripts/land_init.sh
@@ -45,11 +45,13 @@ fi
 # Initialize
 cd ${workspace}
 pwd
+rm -f ${workspace}/${UFS_PLATFORM,,}-*.txt ${workspace}/${UFS_PLATFORM,,}-*.json
+rm -f ${workspace}/${platform}-*.txt ${workspace}/${platform}-*.json
 set +e
 
 rm -rf sorc/build
 
-git checkout ${BRANCH_NAME:=develop}
+[[ -n ${BRANCH_NAME:=} ]] && git checkout ${BRANCH_NAME} || true
 status=${PIPESTATUS[0]}
 
 git branch

From 3e1ca291dcd728eb679fd9257aa31142c2c98d1c Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Fri, 9 Aug 2024 12:13:06 -0700
Subject: [PATCH 07/35] fix exit status for CICD scripts

---
 .cicd/scripts/land_build.sh | 9 +++++++--
 .cicd/scripts/land_init.sh  | 3 ++-
 .cicd/scripts/land_test.sh  | 5 ++++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
index 11d70f0c..054becd9 100755
--- a/.cicd/scripts/land_build.sh
+++ b/.cicd/scripts/land_build.sh
@@ -55,6 +55,10 @@ echo "ACCNR=${ACCNR:=}"
 cd ${workspace}
 pwd
 set +e
+
+git branch
+git log -1 --oneline
+
 echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR:=}."
 /usr/bin/time -p \
 	-o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_build.json \
@@ -63,10 +67,11 @@ echo "Pipeline Building Land-DA on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=
 status=${PIPESTATUS[0]}
 cat sorc/build/log.ecbuild sorc/build/log.make >> ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-build-log.txt
 echo "Pipeline Completed Land-DA build on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
-git status -u
-                                    
+
 ls -l sorc/build/*
 
+git status -u
+                                    
 build_exit=$status
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
 env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
diff --git a/.cicd/scripts/land_init.sh b/.cicd/scripts/land_init.sh
index be7a007b..17a4ed32 100755
--- a/.cicd/scripts/land_init.sh
+++ b/.cicd/scripts/land_init.sh
@@ -56,13 +56,14 @@ status=${PIPESTATUS[0]}
 
 git branch
 git log -1 --oneline
-git status -u
 
 /usr/bin/time -p \
 	-o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-land_init.json \
 	-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
 	find . -name .git -type d
 
+git status -u
+
 init_exit=$status
 echo "STAGE_NAME=${STAGE_NAME:=manual}"
 env | grep = | sort > ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-env.txt
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 4106959e..2acdfd13 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -74,6 +74,10 @@ cd ${workspace}
 pwd
 set +e
 ls -l sorc/build/*
+status=$?
+
+git branch
+git log -1 --oneline
 
 if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 
@@ -99,7 +103,6 @@ if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 	cd -
 else
 	echo "Pipeline skipping Tests on ${UFS_PLATFORM} (${machine})"
-	status=0
 fi
 
 git status -u

From f97bb44ccaa9ef80f8f88539c00b66277c45acfb Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Fri, 9 Aug 2024 12:50:52 -0700
Subject: [PATCH 08/35] check for bin and lib after build in CICD scripts

---
 .cicd/scripts/land_build.sh |  3 ++-
 .cicd/scripts/land_test.sh  | 13 ++++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
index 054becd9..81e80997 100755
--- a/.cicd/scripts/land_build.sh
+++ b/.cicd/scripts/land_build.sh
@@ -68,7 +68,8 @@ status=${PIPESTATUS[0]}
 cat sorc/build/log.ecbuild sorc/build/log.make >> ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-build-log.txt
 echo "Pipeline Completed Land-DA build on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
 
-ls -l sorc/build/*
+ls -l sorc/build/bin/*.exe sorc/build/lib/*.so
+status=$?
 
 git status -u
                                     
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 2acdfd13..51287f89 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -72,19 +72,23 @@ echo "ACCNR=${ACCNR}"
 # Test
 cd ${workspace}
 pwd
-set +e
 ls -l sorc/build/*
-status=$?
+set +e
+
 
 git branch
 git log -1 --oneline
 
 if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
+                                        
+    ls -l sorc/build/bin/*.exe sorc/build/lib/*.so
+    status=$?
+    if [[ ${status} = 0 ]] ; then
 
 	cp sorc/test/${machine}_ctest.sh sorc/build/
 	cp sorc/test/check_ctest.sh sorc/build/
 	cp sorc/test/run_${machine}_ctest.sh sorc/build/
-                                        
+
 	cd sorc/build/
 	rm -f out.*
 	set +x
@@ -101,6 +105,9 @@ if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 	echo "rc=$rc status=$status"
 	set -x
 	cd -
+    else
+	echo "Error: bin/* or lib/* not available."
+    fi
 else
 	echo "Pipeline skipping Tests on ${UFS_PLATFORM} (${machine})"
 fi

From d6f0c6b13d9423c498c60ee543fcc0f44d3a3200 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 11:22:02 -0700
Subject: [PATCH 09/35] Create Jenkinsfile

---
 .cicd/Jenkinsfile | 254 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 254 insertions(+)
 create mode 100644 .cicd/Jenkinsfile

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
new file mode 100644
index 00000000..650da188
--- /dev/null
+++ b/.cicd/Jenkinsfile
@@ -0,0 +1,254 @@
+pipeline {
+    agent none
+
+    options {
+        disableConcurrentBuilds()
+        overrideIndexTriggers(false)
+        skipDefaultCheckout(true)
+        timestamps()
+        timeout(time: 6, unit: 'HOURS')
+    }
+
+    parameters {
+        // Allow job runner to filter based on platform
+        choice(name: 'UFS_PLATFORM_FILTER', choices: ['all', 'hera', 'orion', 'hercules', 'jet', 'gaea', 'derecho', 'pclusternoaav2use1', 'azclusternoaav2use1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
+        // Allow job runner to filter based on compiler
+        choice(name: 'UFS_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
+    }
+
+    stages {
+        stage('Launch SonarQube') {
+            steps {
+                        echo "SonarQube: ${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                /*
+                script {
+                    build job: '/land-DA_workflow/land-da-sonarqube', parameters: [
+                        string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'),
+                        string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: '')
+                    ], wait: false
+                }
+                */
+            }
+        }
+
+        /*
+        stage('Check Nodes') {
+            steps {
+                script {
+                    build job: '/infrastructure/is_up', parameters: [string(name: 'NODE_CHOICE', value: 'pclusternoaav2use1'  ?: 'none')], wait: false
+                    build job: '/infrastructure/is_up', parameters: [string(name: 'NODE_CHOICE', value: 'azclusternoaav2use1' ?: 'none')], wait: false
+                    build job: '/infrastructure/is_up', parameters: [string(name: 'NODE_CHOICE', value: 'gclusternoaav2usc1'  ?: 'none')], wait: false
+                    sleep time: 120, unit: 'SECONDS'
+                }
+            }
+        }
+        */
+
+        // Build and test the Land-DA application on all supported platforms using the supported compilers for each platform
+        stage('Build and Test') {
+            matrix {
+                // Run on all platform/compiler combinations by default or build and test only on the platform(s) and
+                // compiler(s) specified by UFS_PLATFORM_FILTER and UFS_COMPILER_FILTER
+                when {
+                    beforeAgent true
+                    expression {
+                        return nodesByLabel(env.UFS_PLATFORM).size() > 0
+                    }
+                    
+                    //expression { env.UFS_PLATFORM != 'hera' }
+                    expression { env.UFS_PLATFORM != 'orion' }
+                    expression { env.UFS_PLATFORM != 'hercules' }
+                    expression { env.UFS_PLATFORM != 'jet' }
+                    expression { env.UFS_PLATFORM != 'gaea' }
+                    expression { env.UFS_PLATFORM != 'derecho' }
+                    expression { env.UFS_PLATFORM != 'pclusternoaav2use1'}
+                    expression { env.UFS_PLATFORM != 'azclusternoaav2use1'}
+                    expression { env.UFS_PLATFORM != 'gclusternoaav2usc1' }
+
+                    allOf {
+                        anyOf {
+                            expression { params.UFS_PLATFORM_FILTER == 'all' }
+                            expression { params.UFS_PLATFORM_FILTER == env.UFS_PLATFORM }
+                        }
+
+                        anyOf {
+                            expression { params.UFS_COMPILER_FILTER == 'all' }
+                            expression { params.UFS_COMPILER_FILTER == env.UFS_COMPILER }
+                        }
+                    }
+                }
+
+                axes {
+                    axis {
+                        name 'UFS_PLATFORM'
+                        values 'hera', 'orion', 'hercules', 'jet', 'gaea', 'derecho' , 'pclusternoaav2use1', 'azclusternoaav2use1', 'gclusternoaav2usc1'
+                    }
+
+                    axis {
+                        name 'UFS_COMPILER'
+                        values 'gnu', 'intel'
+                    }
+                }
+
+                excludes {
+                    // Exclude GNU from platforms that don't support it
+                    exclude {
+                            axis {
+                            name 'UFS_PLATFORM'
+                            values 'hera', 'orion', 'hercules', 'jet', 'gaea', 'derecho', 'pclusternoaav2use1', 'azclusternoaav2use1', 'gclusternoaav2usc1'
+                        }
+
+                        axis {
+                            name 'UFS_COMPILER'
+                            values 'gnu'
+                        }
+                    }
+                }
+
+                agent {
+                    label env.UFS_PLATFORM
+                }
+
+                environment {
+                    REPO_PROJECT="ufs-community"
+                    BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
+                    BUILD_VERSION = "${env.UFS_PLATFORM}-${env.UFS_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
+                    BUILD_NAME = "land-DA_workflow_${env.BUILD_VERSION}"
+                    INSTALL_NAME = "install_${env.UFS_COMPILER}"
+                    ACCNR = "epic"
+                    CMAKE_FLAGS = "-DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16"
+                }
+
+                stages {
+                    // Clean and create the workspace , checkout the repository into ${env.UFS_PLATFORM}
+                    stage('Checkout scm') {
+                        steps {
+                                echo "${env.STAGE_NAME} WM (${env.UFS_COMPILER}) create environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                                cleanWs()
+                                sh "env | egrep = | egrep -v ' ' | sort | tee ${env.UFS_PLATFORM}-${env.UFS_COMPILER}-build-env.txt | egrep '^HOST|^LOGNAME=|^USER=|^HOME=|^PATH=|^PWD=|^NODE|^WORKSPACE|^JOB|^BUILD|^BRANCH|^UFS'"
+                                sh '''
+                                    echo "checkout scm ${BRANCH_NAME} on ${UFS_PLATFORM}"
+                                    [[ ${BRANCH_NAME} =~ feature/cicd_ ]] && export REPO_PROJECT="BruceKropp-Raytheon"
+                                    rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
+                                    /usr/bin/time -p \
+                                        -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
+                                        -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+                                    git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}
+                                '''
+                        }
+                    }
+
+                    // In the workspace repository directory, run initialization
+                    stage('Initialize') {
+                        steps {
+                            dir ("${env.UFS_PLATFORM}") {
+                                echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                                sh '''
+                                    pwd
+                                    echo "NODE_NAME=${NODE_NAME}"
+                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
+                                    .cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}
+                                '''
+                                //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
+                            }
+                        }
+
+                        post {
+                            always {
+                                echo "#### Initialize COMPLETE."
+                            }
+                        }
+                    }
+
+                    // Run the build script; if successful create a tarball of the build and upload to S3
+                    stage('Build') {
+                        options {
+                            timeout(time: 4, unit: 'HOURS')
+                        }
+
+                        steps {
+                            dir ("${env.UFS_PLATFORM}") {
+                                echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                                sh '''
+                                    pwd
+                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
+                                    echo "UFS_COMPILER=${UFS_COMPILER}"
+                                    .cicd/scripts/land_build.sh ${UFS_PLATFORM} ${UFS_COMPILER}
+                                    ls -l sorc/build/bin/*.exe sorc/build/lib/*.so
+                                '''
+                                //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
+                            }
+                        }
+
+                        post {
+                            success {
+                                echo "#### Build SUCCESS."
+                            }
+                            always {
+                                echo "#### Build COMPLETE."
+                            }
+                        }
+                    }
+
+                    // Run the test script
+                    stage('Test') {
+                        // when { not { expression { return env.UFS_PLATFORM == jet }}}
+                        options {
+                            timeout(time: 4, unit: 'HOURS')
+                        }
+
+                        environment {
+                            EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/${env.UFS_PLATFORM}/tests"
+                        }
+
+                        steps {
+                            dir ("${env.UFS_PLATFORM}") {
+                                echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+
+                                script {
+                                    def run_tests = params.LAND_DA_RUN_TESTS
+                                sh '''
+                                    pwd
+                                    echo "UFS_PLATFORM=<${UFS_PLATFORM}>"
+                                    echo "UFS_COMPILER=<${UFS_COMPILER}>"
+                                    .cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${LAND_DA_RUN_TESTS}
+                                    status=$?
+                                    [[ $status = 0 ]]
+                                '''
+                                //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
+                                }
+                            }
+                        }
+
+                        post {
+                            success {
+                                echo "#### Test SUCCESS."
+                            }
+                            always {
+                                echo "#### Test COMPLETE."
+                            }
+                        }
+                    }
+                } // stages
+            } // matrix
+        } // stage
+    } // stages
+
+    post {
+        always {
+            script {
+                // Trigger another job to collect all build statistics
+                CI_JOB_NAME=env.JOB_NAME.replace("/${env.BRANCH_NAME}","")
+                CI_BRANCH_NAME=env.BRANCH_NAME.replace("%2F","%252F")
+                echo "post: Triggering ufs-weather-model/ufs-wm-metrics job for ${CI_JOB_NAME} on branch build ${CI_BRANCH_NAME}/${env.BUILD_NUMBER} ..."
+                /*
+                build job: '/ufs-srweather-app/ufs-srw-metrics', parameters: [
+                        string(name: 'CI_JOB_NAME', value: "${CI_JOB_NAME}"),
+                        string(name: 'CI_BUILD_NUMBER', value: "${CI_BRANCH_NAME}/${env.BUILD_NUMBER}")
+                ], wait: false
+                */
+                echo "#### post: ufs-weather-model/ufs-wm-metrics COMPLETE."
+            }
+        }
+    }
+} // pipeline

From da588e96ed130b13b9b4b613fab63a6efed67729 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 11:36:03 -0700
Subject: [PATCH 10/35] Update Jenkinsfile

---
 .cicd/Jenkinsfile | 32 +++++++-------------------------
 1 file changed, 7 insertions(+), 25 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 650da188..f7f2bf7b 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -121,34 +121,14 @@ pipeline {
 
                 stages {
                     // Clean and create the workspace , checkout the repository into ${env.UFS_PLATFORM}
-                    stage('Checkout scm') {
-                        steps {
-                                echo "${env.STAGE_NAME} WM (${env.UFS_COMPILER}) create environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                cleanWs()
-                                sh "env | egrep = | egrep -v ' ' | sort | tee ${env.UFS_PLATFORM}-${env.UFS_COMPILER}-build-env.txt | egrep '^HOST|^LOGNAME=|^USER=|^HOME=|^PATH=|^PWD=|^NODE|^WORKSPACE|^JOB|^BUILD|^BRANCH|^UFS'"
-                                sh '''
-                                    echo "checkout scm ${BRANCH_NAME} on ${UFS_PLATFORM}"
-                                    [[ ${BRANCH_NAME} =~ feature/cicd_ ]] && export REPO_PROJECT="BruceKropp-Raytheon"
-                                    rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
-                                    /usr/bin/time -p \
-                                        -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
-                                        -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
-                                    git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}
-                                '''
-                        }
-                    }
-
                     // In the workspace repository directory, run initialization
                     stage('Initialize') {
                         steps {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                sh '''
-                                    pwd
-                                    echo "NODE_NAME=${NODE_NAME}"
-                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
-                                    .cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}
-                                '''
+                                cleanWs()
+                                checkout scm
+                                sh '"${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
                         }
@@ -171,9 +151,10 @@ pipeline {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
                                 sh '''
                                     pwd
+                                    echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=${UFS_PLATFORM}"
                                     echo "UFS_COMPILER=${UFS_COMPILER}"
-                                    .cicd/scripts/land_build.sh ${UFS_PLATFORM} ${UFS_COMPILER}
+                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh ${UFS_PLATFORM} ${UFS_COMPILER}
                                     ls -l sorc/build/bin/*.exe sorc/build/lib/*.so
                                 '''
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
@@ -209,9 +190,10 @@ pipeline {
                                     def run_tests = params.LAND_DA_RUN_TESTS
                                 sh '''
                                     pwd
+                                    echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=<${UFS_PLATFORM}>"
                                     echo "UFS_COMPILER=<${UFS_COMPILER}>"
-                                    .cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${LAND_DA_RUN_TESTS}
+                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${LAND_DA_RUN_TESTS}
                                     status=$?
                                     [[ $status = 0 ]]
                                 '''

From 39c274659bbce20b33fd4ff5483e17ad94cc6110 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 12:02:52 -0700
Subject: [PATCH 11/35] Update Jenkinsfile

---
 .cicd/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index f7f2bf7b..a9ef055a 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -126,7 +126,7 @@ pipeline {
                         steps {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                cleanWs()
+                                //cleanWs()
                                 checkout scm
                                 sh '"${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'

From ac46dd3218d4896027dda4407e829cb610edcbfd Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 12:09:38 -0700
Subject: [PATCH 12/35] Update Jenkinsfile

---
 .cicd/Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index a9ef055a..ef734b94 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -111,10 +111,10 @@ pipeline {
 
                 environment {
                     REPO_PROJECT="ufs-community"
-                    BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
-                    BUILD_VERSION = "${env.UFS_PLATFORM}-${env.UFS_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
-                    BUILD_NAME = "land-DA_workflow_${env.BUILD_VERSION}"
-                    INSTALL_NAME = "install_${env.UFS_COMPILER}"
+                    // BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
+                    // BUILD_VERSION = "${env.UFS_PLATFORM}-${env.UFS_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
+                    // BUILD_NAME = "land-DA_workflow_${env.BUILD_VERSION}"
+                    // INSTALL_NAME = "install_${env.UFS_COMPILER}"
                     ACCNR = "epic"
                     CMAKE_FLAGS = "-DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16"
                 }

From eab32bceb4bdb370bcdf0d6a9de78f1bb9cecae3 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 12:23:04 -0700
Subject: [PATCH 13/35] Update Jenkinsfile to debug workspace

---
 .cicd/Jenkinsfile | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index ef734b94..2ef8d1c6 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -126,9 +126,14 @@ pipeline {
                         steps {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                //cleanWs()
+                                cleanWs()
                                 checkout scm
-                                sh '"${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}"'
+                                sh '''
+                                set -x
+                                pwd
+                                ls -al . .cicd/*
+                                ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}
+                                '''
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
                         }

From 241212dc3642c7770cf9acab7a9eed97ef342658 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 12:36:34 -0700
Subject: [PATCH 14/35] Update Jenkinsfile debug build sorc

---
 .cicd/Jenkinsfile | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 2ef8d1c6..675ed0dd 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -129,10 +129,13 @@ pipeline {
                                 cleanWs()
                                 checkout scm
                                 sh '''
-                                set -x
-                                pwd
-                                ls -al . .cicd/*
-                                ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}
+                                    echo "WORKSPACE=${WORKSPACE}"
+                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
+                                    echo "UFS_COMPILER=${UFS_COMPILER}"
+                                    set -x
+                                    pwd
+                                    ls -la . .cicd/*
+                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}
                                 '''
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
@@ -155,12 +158,14 @@ pipeline {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
                                 sh '''
-                                    pwd
                                     echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=${UFS_PLATFORM}"
                                     echo "UFS_COMPILER=${UFS_COMPILER}"
+                                    set -x
+                                    pwd
+                                    ls -la sorc/*
                                     ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh ${UFS_PLATFORM} ${UFS_COMPILER}
-                                    ls -l sorc/build/bin/*.exe sorc/build/lib/*.so
+                                    ls -la sorc/build/bin/*.exe sorc/build/lib/*.so
                                 '''
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }

From 2eef90fdef8028536686a4ed00e7ac88aa32e42d Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 13:00:57 -0700
Subject: [PATCH 15/35] Update Jenkinsfile to use git clone before init

---
 .cicd/Jenkinsfile | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 675ed0dd..dfb5204b 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -121,13 +121,30 @@ pipeline {
 
                 stages {
                     // Clean and create the workspace , checkout the repository into ${env.UFS_PLATFORM}
+                    stage('Checkout scm') {
+                        steps {
+                                echo "${env.STAGE_NAME} WM (${env.UFS_COMPILER}) create environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                                cleanWs()
+                                sh "env | egrep = | egrep -v ' ' | sort | tee ${env.UFS_PLATFORM}-${env.UFS_COMPILER}-build-env.txt | egrep '^HOST|^LOGNAME=|^USER=|^HOME=|^PATH=|^PWD=|^NODE|^WORKSPACE|^JOB|^BUILD|^BRANCH|^UFS'"
+                                sh '''
+                                    echo "checkout scm ${BRANCH_NAME} on ${UFS_PLATFORM}"
+                                    [[ ${BRANCH_NAME} =~ feature/cicd_ ]] && export REPO_PROJECT="BruceKropp-Raytheon"
+                                    rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
+                                    /usr/bin/time -p \
+                                        -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
+                                        -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+                                    git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}
+                                '''
+                        }
+                    }
+
                     // In the workspace repository directory, run initialization
                     stage('Initialize') {
                         steps {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                cleanWs()
-                                checkout scm
+                                //cleanWs()
+                                //checkout scm
                                 sh '''
                                     echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=${UFS_PLATFORM}"

From dc7c03db34314e1ef98412c4dad4b23346c0ebb8 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Mon, 12 Aug 2024 13:31:00 -0700
Subject: [PATCH 16/35] check for bin and lib files prior to CICD tests

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/scripts/land_test.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 51287f89..4553708d 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -72,13 +72,14 @@ echo "ACCNR=${ACCNR}"
 # Test
 cd ${workspace}
 pwd
-ls -l sorc/build/*
+ls -l sorc/build/bin/*.exe sorc/build/lib/*.so
 set +e
 
-
 git branch
 git log -1 --oneline
 
+status=0
+
 if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
                                         
     ls -l sorc/build/bin/*.exe sorc/build/lib/*.so

From cfb2e04c799228f89319a37462035d58e01ce0f0 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 13:40:21 -0700
Subject: [PATCH 17/35] Update Jenkinsfile boolean to run tests

---
 .cicd/Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index dfb5204b..40f572a8 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -14,6 +14,8 @@ pipeline {
         choice(name: 'UFS_PLATFORM_FILTER', choices: ['all', 'hera', 'orion', 'hercules', 'jet', 'gaea', 'derecho', 'pclusternoaav2use1', 'azclusternoaav2use1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'UFS_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
+        // Run tests ?
+        booleanParam name: 'LAND_DA_RUN_TESTS', defaultValue: true, description: 'Whether to execute the tests'
     }
 
     stages {

From 53c9b2e8945977a866d5061c02f0b95333cd441d Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 13:54:27 -0700
Subject: [PATCH 18/35] Update Jenkinsfile pass bool param to test

---
 .cicd/Jenkinsfile | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 40f572a8..ffe115ac 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -113,12 +113,13 @@ pipeline {
 
                 environment {
                     REPO_PROJECT="ufs-community"
-                    // BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
-                    // BUILD_VERSION = "${env.UFS_PLATFORM}-${env.UFS_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
-                    // BUILD_NAME = "land-DA_workflow_${env.BUILD_VERSION}"
-                    // INSTALL_NAME = "install_${env.UFS_COMPILER}"
+                    BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
+                    BUILD_VERSION = "${env.UFS_PLATFORM}-${env.UFS_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
+                    BUILD_NAME = "land-DA_workflow_${env.BUILD_VERSION}"
+                    INSTALL_NAME = "install_${env.UFS_COMPILER}"
                     ACCNR = "epic"
                     CMAKE_FLAGS = "-DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16"
+                    LAND_DA_RUN_TESTS = "${params.LAND_DA_RUN_TESTS}"
                 }
 
                 stages {
@@ -222,6 +223,7 @@ pipeline {
                                     echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=<${UFS_PLATFORM}>"
                                     echo "UFS_COMPILER=<${UFS_COMPILER}>"
+                                    echo "LAND_DA_RUN_TESTS=<${LAND_DA_RUN_TESTS}>"
                                     ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${LAND_DA_RUN_TESTS}
                                     status=$?
                                     [[ $status = 0 ]]

From 964fd61aa7e0bf4743dbbd4824ba6132e49d46f4 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 13:59:09 -0700
Subject: [PATCH 19/35] Update Jenkinsfile run tests = true

---
 .cicd/Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index ffe115ac..3d01881b 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -15,7 +15,7 @@ pipeline {
         // Allow job runner to filter based on compiler
         choice(name: 'UFS_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Run tests ?
-        booleanParam name: 'LAND_DA_RUN_TESTS', defaultValue: true, description: 'Whether to execute the tests'
+        // booleanParam name: 'LAND_DA_RUN_TESTS', defaultValue: true, description: 'Whether to execute the tests'
     }
 
     stages {
@@ -119,7 +119,7 @@ pipeline {
                     INSTALL_NAME = "install_${env.UFS_COMPILER}"
                     ACCNR = "epic"
                     CMAKE_FLAGS = "-DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16"
-                    LAND_DA_RUN_TESTS = "${params.LAND_DA_RUN_TESTS}"
+                    LAND_DA_RUN_TESTS = true
                 }
 
                 stages {

From 5fad59b9b0f81460f121267b497ef6d8644e8d30 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 14:16:57 -0700
Subject: [PATCH 20/35] Update Jenkinsfile use checkout scm

---
 .cicd/Jenkinsfile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 3d01881b..38b1c6f3 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -127,16 +127,16 @@ pipeline {
                     stage('Checkout scm') {
                         steps {
                                 echo "${env.STAGE_NAME} WM (${env.UFS_COMPILER}) create environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                cleanWs()
+                                //cleanWs()
                                 sh "env | egrep = | egrep -v ' ' | sort | tee ${env.UFS_PLATFORM}-${env.UFS_COMPILER}-build-env.txt | egrep '^HOST|^LOGNAME=|^USER=|^HOME=|^PATH=|^PWD=|^NODE|^WORKSPACE|^JOB|^BUILD|^BRANCH|^UFS'"
                                 sh '''
                                     echo "checkout scm ${BRANCH_NAME} on ${UFS_PLATFORM}"
                                     [[ ${BRANCH_NAME} =~ feature/cicd_ ]] && export REPO_PROJECT="BruceKropp-Raytheon"
-                                    rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
-                                    /usr/bin/time -p \
-                                        -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
-                                        -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
-                                    git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}
+                                    #rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
+                                    #/usr/bin/time -p \
+                                    #    -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
+                                    #    -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+                                    #git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}
                                 '''
                         }
                     }
@@ -146,8 +146,8 @@ pipeline {
                         steps {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                //cleanWs()
-                                //checkout scm
+                                cleanWs()
+                                checkout scm
                                 sh '''
                                     echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=${UFS_PLATFORM}"

From 62e7fafcac47fe202b05c4b1c31809eacc9ab443 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 14:22:03 -0700
Subject: [PATCH 21/35] Update Jenkinsfile echo git clone from init

---
 .cicd/Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 38b1c6f3..0cac6007 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -132,11 +132,11 @@ pipeline {
                                 sh '''
                                     echo "checkout scm ${BRANCH_NAME} on ${UFS_PLATFORM}"
                                     [[ ${BRANCH_NAME} =~ feature/cicd_ ]] && export REPO_PROJECT="BruceKropp-Raytheon"
-                                    #rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
-                                    #/usr/bin/time -p \
-                                    #    -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
-                                    #    -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
-                                    #git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}
+                                    rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
+                                    /usr/bin/time -p \
+                                        -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
+                                        -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
+                                    echo "git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}"
                                 '''
                         }
                     }

From cee42b394b0b06736a78790919646a65352c7545 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 14:36:57 -0700
Subject: [PATCH 22/35] Update Jenkinsfile run_tests from LAND_DA_RUN_TESTS

---
 .cicd/Jenkinsfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 0cac6007..101603be 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -15,7 +15,7 @@ pipeline {
         // Allow job runner to filter based on compiler
         choice(name: 'UFS_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Run tests ?
-        // booleanParam name: 'LAND_DA_RUN_TESTS', defaultValue: true, description: 'Whether to execute the tests'
+        booleanParam name: 'LAND_DA_RUN_TESTS', defaultValue: false, description: 'Whether to execute the tests'
     }
 
     stages {
@@ -119,7 +119,6 @@ pipeline {
                     INSTALL_NAME = "install_${env.UFS_COMPILER}"
                     ACCNR = "epic"
                     CMAKE_FLAGS = "-DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16"
-                    LAND_DA_RUN_TESTS = true
                 }
 
                 stages {
@@ -223,8 +222,8 @@ pipeline {
                                     echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=<${UFS_PLATFORM}>"
                                     echo "UFS_COMPILER=<${UFS_COMPILER}>"
-                                    echo "LAND_DA_RUN_TESTS=<${LAND_DA_RUN_TESTS}>"
-                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${LAND_DA_RUN_TESTS}
+                                    echo "LAND_DA_RUN_TESTS=<${run_tests}>"
+                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${run_tests}
                                     status=$?
                                     [[ $status = 0 ]]
                                 '''

From 173779ee9e76c69926b5bc8c8f3801c1a037d791 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 15:40:40 -0700
Subject: [PATCH 23/35] Update Jenkinsfile clear out pre-init

---
 .cicd/Jenkinsfile | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 101603be..2c1c00db 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -58,7 +58,7 @@ pipeline {
                     }
                     
                     //expression { env.UFS_PLATFORM != 'hera' }
-                    expression { env.UFS_PLATFORM != 'orion' }
+                    //expression { env.UFS_PLATFORM != 'orion' }
                     expression { env.UFS_PLATFORM != 'hercules' }
                     expression { env.UFS_PLATFORM != 'jet' }
                     expression { env.UFS_PLATFORM != 'gaea' }
@@ -123,23 +123,6 @@ pipeline {
 
                 stages {
                     // Clean and create the workspace , checkout the repository into ${env.UFS_PLATFORM}
-                    stage('Checkout scm') {
-                        steps {
-                                echo "${env.STAGE_NAME} WM (${env.UFS_COMPILER}) create environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                //cleanWs()
-                                sh "env | egrep = | egrep -v ' ' | sort | tee ${env.UFS_PLATFORM}-${env.UFS_COMPILER}-build-env.txt | egrep '^HOST|^LOGNAME=|^USER=|^HOME=|^PATH=|^PWD=|^NODE|^WORKSPACE|^JOB|^BUILD|^BRANCH|^UFS'"
-                                sh '''
-                                    echo "checkout scm ${BRANCH_NAME} on ${UFS_PLATFORM}"
-                                    [[ ${BRANCH_NAME} =~ feature/cicd_ ]] && export REPO_PROJECT="BruceKropp-Raytheon"
-                                    rm -f ${UFS_PLATFORM}-${UFS_COMPILER}-time-*
-                                    /usr/bin/time -p \
-                                        -o ${UFS_PLATFORM}-${UFS_COMPILER}-time-da_scm.json \
-                                        -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
-                                    echo "git clone --quiet --recurse-submodules https://github.com/${REPO_PROJECT}/land-DA_workflow.git -b ${BRANCH_NAME} ${UFS_PLATFORM}"
-                                '''
-                        }
-                    }
-
                     // In the workspace repository directory, run initialization
                     stage('Initialize') {
                         steps {

From f36d7f10689189e2733795850e1187c287e7d4e4 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 16:00:08 -0700
Subject: [PATCH 24/35] Update Jenkinsfile allow hercules

---
 .cicd/Jenkinsfile | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 2c1c00db..b47f6d38 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -15,7 +15,7 @@ pipeline {
         // Allow job runner to filter based on compiler
         choice(name: 'UFS_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Run tests ?
-        booleanParam name: 'LAND_DA_RUN_TESTS', defaultValue: false, description: 'Whether to execute the tests'
+        booleanParam name: 'LAND_DA_RUN_TESTS', defaultValue: true, description: 'Whether to execute the tests'
     }
 
     stages {
@@ -59,7 +59,7 @@ pipeline {
                     
                     //expression { env.UFS_PLATFORM != 'hera' }
                     //expression { env.UFS_PLATFORM != 'orion' }
-                    expression { env.UFS_PLATFORM != 'hercules' }
+                    //expression { env.UFS_PLATFORM != 'hercules' }
                     expression { env.UFS_PLATFORM != 'jet' }
                     expression { env.UFS_PLATFORM != 'gaea' }
                     expression { env.UFS_PLATFORM != 'derecho' }
@@ -131,12 +131,6 @@ pipeline {
                                 cleanWs()
                                 checkout scm
                                 sh '''
-                                    echo "WORKSPACE=${WORKSPACE}"
-                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
-                                    echo "UFS_COMPILER=${UFS_COMPILER}"
-                                    set -x
-                                    pwd
-                                    ls -la . .cicd/*
                                     ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}
                                 '''
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
@@ -191,7 +185,7 @@ pipeline {
                         }
 
                         environment {
-                            EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/${env.UFS_PLATFORM}/tests"
+                            LAND_DA_RUN_TESTS = params.LAND_DA_RUN_TESTS
                         }
 
                         steps {
@@ -205,7 +199,8 @@ pipeline {
                                     echo "WORKSPACE=${WORKSPACE}"
                                     echo "UFS_PLATFORM=<${UFS_PLATFORM}>"
                                     echo "UFS_COMPILER=<${UFS_COMPILER}>"
-                                    echo "LAND_DA_RUN_TESTS=<${run_tests}>"
+                                    echo "LAND_DA_RUN_TESTS=<${LAND_DA_RUN_TESTS}>"
+                                    echo "run_tests=<${run_tests}>"
                                     ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${run_tests}
                                     status=$?
                                     [[ $status = 0 ]]

From bb1a73cb36fc98e8b15c2027321a840f2644f4b8 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Mon, 12 Aug 2024 16:03:23 -0700
Subject: [PATCH 25/35] Update Jenkinsfile quote params

---
 .cicd/Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index b47f6d38..2d2ce46e 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -112,7 +112,7 @@ pipeline {
                 }
 
                 environment {
-                    REPO_PROJECT="ufs-community"
+                    REPO_PROJECT = "ufs-community"
                     BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
                     BUILD_VERSION = "${env.UFS_PLATFORM}-${env.UFS_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
                     BUILD_NAME = "land-DA_workflow_${env.BUILD_VERSION}"
@@ -185,7 +185,7 @@ pipeline {
                         }
 
                         environment {
-                            LAND_DA_RUN_TESTS = params.LAND_DA_RUN_TESTS
+                            LAND_DA_RUN_TESTS = "${params.LAND_DA_RUN_TESTS}"
                         }
 
                         steps {

From 78c55ddae5feaa26607a083fb60556b3c029e523 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Tue, 13 Aug 2024 16:16:26 -0700
Subject: [PATCH 26/35] dont need to set account in CICD scripts

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/Jenkinsfile          |  8 +++++++-
 .cicd/scripts/land_test.sh | 15 +++++++--------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 2d2ce46e..90cc92e7 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -131,8 +131,14 @@ pipeline {
                                 cleanWs()
                                 checkout scm
                                 sh '''
-                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}
+                                    echo "WORKSPACE=${WORKSPACE}"
+                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
+                                    echo "UFS_COMPILER=${UFS_COMPILER}"
+                                    set -x
+                                    pwd
+                                    echo "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}"
                                 '''
+                                sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
                         }
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 4553708d..eba30862 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -60,19 +60,18 @@ else
     platform="${UFS_PLATFORM}"
 fi
 
-if [[ ${platform} = derecho ]] ; then
-	export ACCNR=nral0032
-elif [[ ${platform} = jet ]] ; then
-	export ACCNR=hfv3gfs
-else
-	export ACCNR=epic
-fi
+#if [[ ${platform} = derecho ]] ; then
+#	export ACCNR=nral0032
+#elif [[ ${platform} = jet ]] ; then
+#	export ACCNR=hfv3gfs
+#else
+#	export ACCNR=epic
+#fi
 echo "ACCNR=${ACCNR}"
 
 # Test
 cd ${workspace}
 pwd
-ls -l sorc/build/bin/*.exe sorc/build/lib/*.so
 set +e
 
 git branch

From f4e6812ce6e93a5e75e5f6ad8bf4f4c1af899222 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Tue, 13 Aug 2024 16:30:59 -0700
Subject: [PATCH 27/35] Update Jenkinsfile

simplify CICD script calls
---
 .cicd/Jenkinsfile | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 90cc92e7..dcd369a9 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -130,14 +130,6 @@ pipeline {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
                                 cleanWs()
                                 checkout scm
-                                sh '''
-                                    echo "WORKSPACE=${WORKSPACE}"
-                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
-                                    echo "UFS_COMPILER=${UFS_COMPILER}"
-                                    set -x
-                                    pwd
-                                    echo "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh ${UFS_PLATFORM} ${UFS_COMPILER}"
-                                '''
                                 sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
@@ -166,9 +158,10 @@ pipeline {
                                     set -x
                                     pwd
                                     ls -la sorc/*
-                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh ${UFS_PLATFORM} ${UFS_COMPILER}
-                                    ls -la sorc/build/bin/*.exe sorc/build/lib/*.so
+                                    echo "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh ${UFS_PLATFORM} ${UFS_COMPILER}"
+                                    echo "ls -la sorc/build/bin/*.exe sorc/build/lib/*.so"
                                 '''
+                                sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
                         }
@@ -207,10 +200,11 @@ pipeline {
                                     echo "UFS_COMPILER=<${UFS_COMPILER}>"
                                     echo "LAND_DA_RUN_TESTS=<${LAND_DA_RUN_TESTS}>"
                                     echo "run_tests=<${run_tests}>"
-                                    ${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${run_tests}
+                                    echo "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${run_tests}"
                                     status=$?
                                     [[ $status = 0 ]]
                                 '''
+                                sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                                 }
                             }

From f36403fda306dcdd7244252e422235a3131270fd Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Tue, 13 Aug 2024 16:52:10 -0700
Subject: [PATCH 28/35] cleanup CICD script debugging

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/Jenkinsfile           | 27 +--------------------------
 .cicd/scripts/land_build.sh |  7 -------
 .cicd/scripts/land_test.sh  |  7 -------
 3 files changed, 1 insertion(+), 40 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index dcd369a9..0de206ad 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -151,16 +151,6 @@ pipeline {
                         steps {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                                sh '''
-                                    echo "WORKSPACE=${WORKSPACE}"
-                                    echo "UFS_PLATFORM=${UFS_PLATFORM}"
-                                    echo "UFS_COMPILER=${UFS_COMPILER}"
-                                    set -x
-                                    pwd
-                                    ls -la sorc/*
-                                    echo "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh ${UFS_PLATFORM} ${UFS_COMPILER}"
-                                    echo "ls -la sorc/build/bin/*.exe sorc/build/lib/*.so"
-                                '''
                                 sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
@@ -189,24 +179,9 @@ pipeline {
 
                         steps {
                             dir ("${env.UFS_PLATFORM}") {
-                                echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-
-                                script {
-                                    def run_tests = params.LAND_DA_RUN_TESTS
-                                sh '''
-                                    pwd
-                                    echo "WORKSPACE=${WORKSPACE}"
-                                    echo "UFS_PLATFORM=<${UFS_PLATFORM}>"
-                                    echo "UFS_COMPILER=<${UFS_COMPILER}>"
-                                    echo "LAND_DA_RUN_TESTS=<${LAND_DA_RUN_TESTS}>"
-                                    echo "run_tests=<${run_tests}>"
-                                    echo "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh ${UFS_PLATFORM} ${UFS_COMPILER} ${run_tests}"
-                                    status=$?
-                                    [[ $status = 0 ]]
-                                '''
+                                echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM}) ${env.LAND_DA_RUN_TESTS}"
                                 sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh"'
                                 //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
-                                }
                             }
                         }
 
diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
index 81e80997..06ff22b1 100755
--- a/.cicd/scripts/land_build.sh
+++ b/.cicd/scripts/land_build.sh
@@ -42,13 +42,6 @@ else
     platform="${UFS_PLATFORM}"
 fi
 
-#if [[ ${platform} = derecho ]] ; then
-#	export ACCNR=nral0032
-#elif [[ ${platform} = jet ]] ; then
-#	export ACCNR=hfv3gfs
-#else
-#	export ACCNR=epic
-#fi
 echo "ACCNR=${ACCNR:=}"
 
 # Build
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index eba30862..67fbc28b 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -60,13 +60,6 @@ else
     platform="${UFS_PLATFORM}"
 fi
 
-#if [[ ${platform} = derecho ]] ; then
-#	export ACCNR=nral0032
-#elif [[ ${platform} = jet ]] ; then
-#	export ACCNR=hfv3gfs
-#else
-#	export ACCNR=epic
-#fi
 echo "ACCNR=${ACCNR}"
 
 # Test

From 303db9c07df5f7bb2175e7b8240e577b05690688 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <Bruce.Kropp@Raytheon.com>
Date: Wed, 14 Aug 2024 09:00:30 -0700
Subject: [PATCH 29/35] add CICD script disk_usage.sh

Signed-off-by: Bruce Kropp <Bruce.Kropp@Raytheon.com>
---
 .cicd/scripts/disk_usage.sh | 48 +++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100755 .cicd/scripts/disk_usage.sh

diff --git a/.cicd/scripts/disk_usage.sh b/.cicd/scripts/disk_usage.sh
new file mode 100755
index 00000000..22ffbeaa
--- /dev/null
+++ b/.cicd/scripts/disk_usage.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Output a CSV report of disk usage on subdirs of some path
+# Usage: 
+#    [JOB_NAME=<ci_job>] [BUILD_NUMBER=<n>] [UFS_COMPILER=<intel>] [UFS_PLATFORM=<machine>] disk_usage path depth size outfile.csv
+#
+# args:
+#    directory=$1
+#    depth=$2
+#    size=$3
+#    outfile=$4
+
+[[ -n ${WORKSPACE} ]] || WORKSPACE=$(pwd)
+[[ -n ${UFS_PLATFORM} ]] || UFS_PLATFORM=$(hostname -s 2>/dev/null) || UFS_PLATFORM=$(hostname 2>/dev/null)
+[[ -n ${UFS_COMPILER} ]] || UFS_COMPILER=compiler
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+
+# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
+# relative to script directory.
+declare workspace
+if [[ -n "${WORKSPACE}/${UFS_PLATFORM}" ]]; then
+    workspace="${WORKSPACE}/${UFS_PLATFORM}"
+else
+    workspace="$(cd -- "${script_dir}/../.." && pwd)"
+fi
+
+echo "STAGE_NAME=${STAGE_NAME}" # from pipeline
+outfile="${4:-${workspace}-${UFS_COMPILER}-disk-usage${STAGE_NAME}.csv}"
+
+function disk_usage() {
+    local directory=${1:-${PWD}}
+    local depth=${2:-1}
+    local size=${3:-k}
+    echo "Disk usage: ${JOB_NAME:-ci}/${UFS_PLATFORM}/$(basename $directory)"
+    (
+    cd $directory || exit 1
+    echo "Platform,Build,Owner,Group,Inodes,${size:-k}bytes,Access Time,Filename"
+    du -Px -d ${depth:-1} --inode --exclude='./workspace' | \
+        while read line ; do
+            arr=($line); inode=${arr[0]}; filename=${arr[1]};
+            echo "${UFS_PLATFORM}-${UFS_COMPILER:-compiler},${JOB_NAME:-ci}/${BUILD_NUMBER:-0},$(stat -c '%U,%G' $filename),${inode:-0},$(du -Px -s -${size:-k} --time $filename)" | tr '\t' ',' ;
+        done | sort -t, -k5 -n #-r
+    )
+    echo ""
+}
+
+disk_usage $1 $2 $3 | tee ${outfile}

From a20df5e61d28f71fd938c17b435ee5ed9050a321 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Wed, 14 Aug 2024 09:27:52 -0700
Subject: [PATCH 30/35] Update Jenkinsfile save results and stats

---
 .cicd/Jenkinsfile | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 0de206ad..496acaf5 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -21,7 +21,7 @@ pipeline {
     stages {
         stage('Launch SonarQube') {
             steps {
-                        echo "SonarQube: ${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                echo "SonarQube: ${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
                 /*
                 script {
                     build job: '/land-DA_workflow/land-da-sonarqube', parameters: [
@@ -131,13 +131,15 @@ pipeline {
                                 cleanWs()
                                 checkout scm
                                 sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_init.sh"'
-                                //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
+                                sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
                         }
 
                         post {
                             always {
                                 echo "#### Initialize COMPLETE."
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-land_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }
@@ -152,7 +154,7 @@ pipeline {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
                                 sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_build.sh"'
-                                //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
+                                sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
                         }
 
@@ -162,6 +164,9 @@ pipeline {
                             }
                             always {
                                 echo "#### Build COMPLETE."
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-env.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-land_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }
@@ -181,16 +186,20 @@ pipeline {
                             dir ("${env.UFS_PLATFORM}") {
                                 echo "${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM}) ${env.LAND_DA_RUN_TESTS}"
                                 sh 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/land_test.sh"'
-                                //sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
+                                sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${UFS_PLATFORM}/.cicd/scripts/disk_usage.sh"'
                             }
                         }
 
                         post {
                             success {
                                 echo "#### Test SUCCESS."
+                                // Archive the test log files
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/*-test-log.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                             always {
                                 echo "#### Test COMPLETE."
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-ufs_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }

From fa3aa6c24193086543dce7637352e47c336c0623 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Wed, 14 Aug 2024 10:09:31 -0700
Subject: [PATCH 31/35] Update Jenkinsfile to use UFS_PLATFORM variable name

---
 .cicd/Jenkinsfile | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 496acaf5..d3fccbf3 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -21,15 +21,15 @@ pipeline {
     stages {
         stage('Launch SonarQube') {
             steps {
-                echo "SonarQube: ${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
-                /*
                 script {
+                    echo "SonarQube: ${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                /*
                     build job: '/land-DA_workflow/land-da-sonarqube', parameters: [
                         string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'),
                         string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: '')
                     ], wait: false
-                }
                 */
+                }
             }
         }
 
@@ -138,8 +138,8 @@ pipeline {
                         post {
                             always {
                                 echo "#### Initialize COMPLETE."
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-land_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-time-land_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }
@@ -164,9 +164,9 @@ pipeline {
                             }
                             always {
                                 echo "#### Build COMPLETE."
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-env.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-land_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-env.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-time-land_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }
@@ -194,12 +194,12 @@ pipeline {
                             success {
                                 echo "#### Test SUCCESS."
                                 // Archive the test log files
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/*-test-log.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}/*-test-log.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                             always {
                                 echo "#### Test COMPLETE."
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-ufs_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-time-ufs_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }

From f90b24070a00f1143ec1e7dacfb9732d159b089b Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Wed, 14 Aug 2024 11:10:07 -0700
Subject: [PATCH 32/35] Update Jenkinsfile to fix paths to upload files

---
 .cicd/Jenkinsfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index d3fccbf3..113844cb 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -138,7 +138,7 @@ pipeline {
                         post {
                             always {
                                 echo "#### Initialize COMPLETE."
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-time-land_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}/*-time-land_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                                 s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
@@ -164,8 +164,8 @@ pipeline {
                             }
                             always {
                                 echo "#### Build COMPLETE."
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-env.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-time-land_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}/*-env.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}/*-time-land_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                                 s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
@@ -193,12 +193,12 @@ pipeline {
                         post {
                             success {
                                 echo "#### Test SUCCESS."
-                                // Archive the test log files
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}/*-test-log.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                             always {
                                 echo "#### Test COMPLETE."
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-time-ufs_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                // Archive the test log files
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}/*-test-log.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}/*-time-land_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                                 s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.UFS_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }

From f5f4eda9ac172caa6cc9208d9efdf6f78b679540 Mon Sep 17 00:00:00 2001
From: Bruce Kropp - Raytheon
 <104453151+BruceKropp-Raytheon@users.noreply.github.com>
Date: Wed, 14 Aug 2024 11:19:25 -0700
Subject: [PATCH 33/35] Update Jenkinsfile

---
 .cicd/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 113844cb..7cdc7af4 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -22,7 +22,7 @@ pipeline {
         stage('Launch SonarQube') {
             steps {
                 script {
-                    echo "SonarQube: ${env.STAGE_NAME} Land-DA (${env.UFS_COMPILER}) build environment on ${env.UFS_PLATFORM} (using ${env.WORKSPACE}/${env.UFS_PLATFORM})"
+                    echo "SonarQube: Scan land-DA_workflow code"
                 /*
                     build job: '/land-DA_workflow/land-da-sonarqube', parameters: [
                         string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'),

From c53817c0a0a1afbd13e4125ae082891f219508e9 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <bruce.kropp@raytheon.com>
Date: Wed, 14 Aug 2024 12:22:10 -0700
Subject: [PATCH 34/35] Adjust post stage in Jenkinsfile

Signed-off-by: Bruce Kropp <bruce.kropp@raytheon.com>
---
 .cicd/Jenkinsfile           | 6 +++---
 .cicd/scripts/disk_usage.sh | 2 +-
 .cicd/scripts/land_build.sh | 4 ++--
 .cicd/scripts/land_init.sh  | 4 ++--
 .cicd/scripts/land_test.sh  | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 7cdc7af4..9aec6a20 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -214,14 +214,14 @@ pipeline {
                 // Trigger another job to collect all build statistics
                 CI_JOB_NAME=env.JOB_NAME.replace("/${env.BRANCH_NAME}","")
                 CI_BRANCH_NAME=env.BRANCH_NAME.replace("%2F","%252F")
-                echo "post: Triggering ufs-weather-model/ufs-wm-metrics job for ${CI_JOB_NAME} on branch build ${CI_BRANCH_NAME}/${env.BUILD_NUMBER} ..."
+                echo "#### post: Triggering land-DA_workflow/ufs-land-metrics job for ${CI_JOB_NAME} on branch build ${CI_BRANCH_NAME}/${env.BUILD_NUMBER} ..."
                 /*
-                build job: '/ufs-srweather-app/ufs-srw-metrics', parameters: [
+                build job: '/land-DA_workflow/ufs-land-metrics', parameters: [
                         string(name: 'CI_JOB_NAME', value: "${CI_JOB_NAME}"),
                         string(name: 'CI_BUILD_NUMBER', value: "${CI_BRANCH_NAME}/${env.BUILD_NUMBER}")
                 ], wait: false
                 */
-                echo "#### post: ufs-weather-model/ufs-wm-metrics COMPLETE."
+                echo "#### post: land-DA_workflow/ufs-land-metrics COMPLETE."
             }
         }
     }
diff --git a/.cicd/scripts/disk_usage.sh b/.cicd/scripts/disk_usage.sh
index 22ffbeaa..8c0fa08e 100755
--- a/.cicd/scripts/disk_usage.sh
+++ b/.cicd/scripts/disk_usage.sh
@@ -19,7 +19,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)
 # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
 # relative to script directory.
 declare workspace
-if [[ -n "${WORKSPACE}/${UFS_PLATFORM}" ]]; then
+if [[ -d "${WORKSPACE}/${UFS_PLATFORM}" ]]; then
     workspace="${WORKSPACE}/${UFS_PLATFORM}"
 else
     workspace="$(cd -- "${script_dir}/../.." && pwd)"
diff --git a/.cicd/scripts/land_build.sh b/.cicd/scripts/land_build.sh
index 06ff22b1..a752c4ee 100755
--- a/.cicd/scripts/land_build.sh
+++ b/.cicd/scripts/land_build.sh
@@ -9,8 +9,8 @@
 #
 pwd
 set +x
-echo "UFS_PLATFORM=${UFS_PLATFORM}"
-echo "UFS_COMPILER=${UFS_COMPILER}"
+#echo "UFS_PLATFORM=${UFS_PLATFORM}"
+#echo "UFS_COMPILER=${UFS_COMPILER}"
 [[ -n $1 ]] && export UFS_PLATFORM=${1} && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
 [[ -n $2 ]] && export UFS_COMPILER=${2} && export compiler=${2} || export compiler=${UFS_COMPILER}
 [[ -n ${WORKSPACE} ]] && export workspace=${WORKSPACE} || export workspace=$(pwd)
diff --git a/.cicd/scripts/land_init.sh b/.cicd/scripts/land_init.sh
index 17a4ed32..3c2f7630 100755
--- a/.cicd/scripts/land_init.sh
+++ b/.cicd/scripts/land_init.sh
@@ -9,8 +9,8 @@
 #
 pwd
 set +x
-echo "UFS_PLATFORM=${UFS_PLATFORM}"
-echo "UFS_COMPILER=${UFS_COMPILER}"
+#echo "UFS_PLATFORM=${UFS_PLATFORM}"
+#echo "UFS_COMPILER=${UFS_COMPILER}"
 [[ -n $1 ]] && export UFS_PLATFORM=${1} && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
 [[ -n $2 ]] && export UFS_COMPILER=${2} && export compiler=${2} || export compiler=${UFS_COMPILER}
 [[ -n ${WORKSPACE} ]] && export workspace=${WORKSPACE} || export workspace=$(pwd)
diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index 67fbc28b..e15a7b4d 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -9,8 +9,8 @@
 #
 pwd
 set +x
-echo "UFS_PLATFORM=${UFS_PLATFORM}"
-echo "UFS_COMPILER=${UFS_COMPILER}"
+#echo "UFS_PLATFORM=${UFS_PLATFORM}"
+#echo "UFS_COMPILER=${UFS_COMPILER}"
 echo "LAND_DA_RUN_TESTS=${LAND_DA_RUN_TESTS}"
 [[ -n $1 ]] && export UFS_PLATFORM=${1} && export machine=${1,,} || export machine=${UFS_PLATFORM,,}
 [[ -n $2 ]] && export UFS_COMPILER=${2} && export compiler=${2} || export compiler=${UFS_COMPILER}

From 9d55dd8f73c92efce09b5ad584f4698f6544d228 Mon Sep 17 00:00:00 2001
From: Bruce Kropp <bruce.kropp@raytheon.com>
Date: Thu, 15 Aug 2024 09:34:05 -0700
Subject: [PATCH 35/35] update CICD script land_test.sh to check if ctest
 produced output

Signed-off-by: Bruce Kropp <bruce.kropp@raytheon.com>
---
 .cicd/scripts/land_test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.cicd/scripts/land_test.sh b/.cicd/scripts/land_test.sh
index e15a7b4d..4d29e716 100755
--- a/.cicd/scripts/land_test.sh
+++ b/.cicd/scripts/land_test.sh
@@ -93,6 +93,7 @@ if [[ true = ${LAND_DA_RUN_TESTS:=false} ]] ; then
 		-f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \
 		./run_${machine}_ctest.sh | awk 'f;/^+ ctest$/{f=1}' | tee ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-test-log.txt
 	status=${PIPESTATUS[0]}
+	[[ -f out.ctest ]] || (( status+=1 ))
 	echo "Pipeline Completed Land-DA Tests on ${UFS_PLATFORM} ${UFS_COMPILER}. status=$status"
 	rc=$(( status+=$(egrep " ***Failed " ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-test-log.txt 2>/dev/null | wc -l) ))
 	echo "rc=$rc status=$status"