From 6740faae49f3227df029122926edaeb254a20743 Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 08:54:21 -0500 Subject: [PATCH 1/7] add module files for hercules --- modulefiles/build_hercules_intel.lua | 46 ++++++++++++++++++++++++++++ modulefiles/wflow_hercules.lua | 18 +++++++++++ sorc/app_build.sh | 2 +- versions/build.ver_hercules | 30 ++++++++++++++++++ versions/run.ver_hercules | 37 ++++++++++++++++++++++ 5 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 modulefiles/build_hercules_intel.lua create mode 100644 modulefiles/wflow_hercules.lua create mode 100644 versions/build.ver_hercules create mode 100644 versions/run.ver_hercules diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua new file mode 100644 index 00000000..1ab618d7 --- /dev/null +++ b/modulefiles/build_hercules_intel.lua @@ -0,0 +1,46 @@ +help([[ +loads modules necessary for building the land-DA workflow on Hercules using Intel +]]) + +whatis([===[Loads modules necessary for building the land-DA workflow on Hercules]===]) + +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("cmake", cmake_ver)) +load(pathJoin("ecbuild", ecbuild_ver)) + +load(pathJoin("jasper", jasper_ver)) +load(pathJoin("zlib", zlib_ver)) +load(pathJoin("libpng", libpng_ver)) +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("parallelio", parallelio_ver)) +load(pathJoin("esmf", esmf_ver)) +load(pathJoin("fms",fms_ver)) +load(pathJoin("bacio", bacio_ver)) +load(pathJoin("crtm", crtm_ver)) +load(pathJoin("g2", g2_ver)) +load(pathJoin("g2tmpl", g2tmpl_ver)) +load(pathJoin("ip", ip_ver)) +load(pathJoin("sp", sp_ver)) +load(pathJoin("w3emc", w3emc_ver)) +load(pathJoin("gftl-shared", gftl_shared_ver)) +load(pathJoin("mapl", mapl_ver)) +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("ufs-pyenv", ufs_pyenv_ver)) + +setenv("CFLAGS","-diag-disable=10441") +setenv("FFLAGS","-diag-disable=10441") + +setenv("CC", "mpiicc") +setenv("CXX", "mpiicpc") +setenv("FC", "mpiifort") +setenv("CMAKE_Platform", "hercules.intel") + +setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_Dev") +setenv("JEDI_INSTALL", "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules") diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua new file mode 100644 index 00000000..540a99e2 --- /dev/null +++ b/modulefiles/wflow_hercules.lua @@ -0,0 +1,18 @@ +help([[ +This module loads python environement for running the land-DA workflow on +the MSU machine Hercules +]]) + +whatis([===[Loads libraries needed for running the land-DA workflow on Hercules ]===]) + +load("contrib") +load("rocoto") + +unload("python") +load("conda") + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate land_da +]===]) +end diff --git a/sorc/app_build.sh b/sorc/app_build.sh index b147a2a3..aac77ff3 100755 --- a/sorc/app_build.sh +++ b/sorc/app_build.sh @@ -387,7 +387,7 @@ fi # Link land-DA input files to FIXlandda directory if [ "${PLATFORM}" = "hera" ]; then landda_fix_orig="/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/inputs" -elif [ "${PLATFORM}" = "orion" ]; then +elif [ "${PLATFORM}" = "orion" ] || [ "${PLATFORM}" = "hercules" ]; then landda_fix_orig="/work/noaa/epic/UFS_Land-DA_Dev/inputs" fi ln -nsf ${landda_fix_orig}/* ${HOME_DIR}/fix diff --git a/versions/build.ver_hercules b/versions/build.ver_hercules new file mode 100644 index 00000000..745034b6 --- /dev/null +++ b/versions/build.ver_hercules @@ -0,0 +1,30 @@ +#!/bin/bash + +export modulepath_spack_stack="/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core" + +export bacio_ver="2.4.1" +export cmake_ver="3.23.1" +export crtm_ver="2.4.0.1" +export ecbuild_ver="3.7.2" +export esmf_ver="8.6.0" +export fms_ver="2023.04" +export gftl_shared_ver="1.6.1" +export g2_ver="3.4.5" +export g2tmpl_ver="1.10.2" +export hdf5_ver="1.14.0" +export ip_ver="4.3.0" +export jasper_ver="2.0.32" +export libpng_ver="1.6.37" +export mapl_ver="2.40.3-esmf-8.6.0" +export netcdf_c_ver="4.9.2" +export netcdf_fortran_ver="4.6.1" +export parallelio_ver="2.5.10" +export prod_util_ver="2.1.1" +export sp_ver="2.5.0" +export stack_intel_oneapi_mpi_ver="2021.9.0" +export stack_intel_ver="2021.9.0" +export stack_python_ver="3.10.13" +export ufs_pyenv_ver="1.0.0" +export w3emc_ver="2.10.0" +export zlib_ver="1.2.13" + diff --git a/versions/run.ver_hercules b/versions/run.ver_hercules new file mode 100644 index 00000000..27463077 --- /dev/null +++ b/versions/run.ver_hercules @@ -0,0 +1,37 @@ +#!/bin/bash + +export modulepath_spack_stack="/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core" + +export bacio_ver="2.4.1" +export cmake_ver="3.23.1" +export crtm_ver="2.4.0.1" +export ecbuild_ver="3.7.2" +export esmf_ver="8.6.0" +export fms_ver="2023.04" +export gftl_shared_ver="1.6.1" +export g2_ver="3.4.5" +export g2tmpl_ver="1.10.2" +export hdf5_ver="1.14.0" +export ip_ver="4.3.0" +export jasper_ver="2.0.32" +export libpng_ver="1.6.37" +export mapl_ver="2.40.3-esmf-8.6.0" +export netcdf_c_ver="4.9.2" +export netcdf_fortran_ver="4.6.1" +export parallelio_ver="2.5.10" +export prod_util_ver="2.1.1" +export py_cartopy_ver="0.21.1" +export py_matplotlib_ver="3.7.3" +export py_netcdf4_ver="1.5.8" +export py_numpy_ver="1.22.3" +export py_pyyaml_ver="6.0" +export py_scipy_ver="1.11.3" +export py_xarray_ver="2023.7.0" +export sp_ver="2.5.0" +export stack_intel_oneapi_mpi_ver="2021.9.0" +export stack_intel_ver="2021.9.0" +export stack_python_ver="3.10.13" +export ufs_pyenv_ver="1.0.0" +export w3emc_ver="2.10.0" +export zlib_ver="1.2.13" + From 2d047c0e7e0aac0e41ae7d5aa3d7a4bb1f6ea93c Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 09:30:53 -0500 Subject: [PATCH 2/7] add ctest scripts --- sorc/test/hercules_ctest.sh | 14 ++++++++++++++ sorc/test/run_hercules_ctest.sh | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100755 sorc/test/hercules_ctest.sh create mode 100755 sorc/test/run_hercules_ctest.sh diff --git a/sorc/test/hercules_ctest.sh b/sorc/test/hercules_ctest.sh new file mode 100755 index 00000000..300b15ff --- /dev/null +++ b/sorc/test/hercules_ctest.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -o out.ctest +#SBATCH --account=epic +set -eux + +source ../../versions/build.ver_hercules +module use ../../modulefiles +module load build_hercules_intel + +ctest + +wait + +echo "ctest is done" diff --git a/sorc/test/run_hercules_ctest.sh b/sorc/test/run_hercules_ctest.sh new file mode 100755 index 00000000..01aa2142 --- /dev/null +++ b/sorc/test/run_hercules_ctest.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -eux + +JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=7 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') + +CHECK_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) + +sleep 7m + +if [ -f out.ctest ]; then + cat out.ctest +else + echo "ctest run fails to run." +fi From 75f045cda47e3d873ae193a72f2bd68dd177410b Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 10:17:25 -0500 Subject: [PATCH 3/7] add yaml file --- modulefiles/tasks/hercules/task.analysis.lua | 10 + modulefiles/tasks/hercules/task.forecast.lua | 28 ++ .../tasks/hercules/task.plot_stats.lua | 15 ++ modulefiles/tasks/hercules/task.post_anal.lua | 13 + modulefiles/tasks/hercules/task.pre_anal.lua | 10 + modulefiles/tasks/hercules/task.prep_obs.lua | 7 + parm/land_analysis_hera.yaml | 3 +- parm/land_analysis_hercules.yaml | 254 ++++++++++++++++++ parm/land_analysis_orion.yaml | 3 +- parm/launch_rocoto_wflow.sh | 3 + 10 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 modulefiles/tasks/hercules/task.analysis.lua create mode 100644 modulefiles/tasks/hercules/task.forecast.lua create mode 100644 modulefiles/tasks/hercules/task.plot_stats.lua create mode 100644 modulefiles/tasks/hercules/task.post_anal.lua create mode 100644 modulefiles/tasks/hercules/task.pre_anal.lua create mode 100644 modulefiles/tasks/hercules/task.prep_obs.lua create mode 100644 parm/land_analysis_hercules.yaml diff --git a/modulefiles/tasks/hercules/task.analysis.lua b/modulefiles/tasks/hercules/task.analysis.lua new file mode 100644 index 00000000..84d59acd --- /dev/null +++ b/modulefiles/tasks/hercules/task.analysis.lua @@ -0,0 +1,10 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) + diff --git a/modulefiles/tasks/hercules/task.forecast.lua b/modulefiles/tasks/hercules/task.forecast.lua new file mode 100644 index 00000000..cd767192 --- /dev/null +++ b/modulefiles/tasks/hercules/task.forecast.lua @@ -0,0 +1,28 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("jasper", jasper_ver)) +load(pathJoin("zlib", zlib_ver)) +load(pathJoin("libpng", libpng_ver)) +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("parallelio", parallelio_ver)) +load(pathJoin("esmf", esmf_ver)) +load(pathJoin("fms",fms_ver)) +load(pathJoin("bacio", bacio_ver)) +load(pathJoin("crtm", crtm_ver)) +load(pathJoin("g2", g2_ver)) +load(pathJoin("g2tmpl", g2tmpl_ver)) +load(pathJoin("ip", ip_ver)) +load(pathJoin("sp", sp_ver)) +load(pathJoin("w3emc", w3emc_ver)) +load(pathJoin("gftl-shared", gftl_shared_ver)) +load(pathJoin("mapl", mapl_ver)) +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) + diff --git a/modulefiles/tasks/hercules/task.plot_stats.lua b/modulefiles/tasks/hercules/task.plot_stats.lua new file mode 100644 index 00000000..92f6efa8 --- /dev/null +++ b/modulefiles/tasks/hercules/task.plot_stats.lua @@ -0,0 +1,15 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-cartopy", py_cartopy_ver)) +load(pathJoin("py-matplotlib", py_matplotlib_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) +load(pathJoin("py-pyyaml", py_pyyaml_ver)) +load(pathJoin("py-scipy", py_scipy_ver)) +load(pathJoin("py-xarray", py_xarray_ver)) + diff --git a/modulefiles/tasks/hercules/task.post_anal.lua b/modulefiles/tasks/hercules/task.post_anal.lua new file mode 100644 index 00000000..3b8b3c7b --- /dev/null +++ b/modulefiles/tasks/hercules/task.post_anal.lua @@ -0,0 +1,13 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) + diff --git a/modulefiles/tasks/hercules/task.pre_anal.lua b/modulefiles/tasks/hercules/task.pre_anal.lua new file mode 100644 index 00000000..31afb0c9 --- /dev/null +++ b/modulefiles/tasks/hercules/task.pre_anal.lua @@ -0,0 +1,10 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("prod_util", prod_util_ver)) diff --git a/modulefiles/tasks/hercules/task.prep_obs.lua b/modulefiles/tasks/hercules/task.prep_obs.lua new file mode 100644 index 00000000..4f16978f --- /dev/null +++ b/modulefiles/tasks/hercules/task.prep_obs.lua @@ -0,0 +1,7 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("prod_util", prod_util_ver)) diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml index 8eb6d42c..c58f83f9 100644 --- a/parm/land_analysis_hera.yaml +++ b/parm/land_analysis_hera.yaml @@ -7,8 +7,7 @@ workflow: cycledef: - attrs: group: cycled - spec: 200001030000 200001040000 24:00:00 # for gswp3 -# spec: 201912210000 201912220000 24:00:00 # for era5 + spec: 200001030000 200001040000 24:00:00 entities: MACHINE: "hera" SCHED: "slurm" diff --git a/parm/land_analysis_hercules.yaml b/parm/land_analysis_hercules.yaml new file mode 100644 index 00000000..e4a47e2a --- /dev/null +++ b/parm/land_analysis_hercules.yaml @@ -0,0 +1,254 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cyclethrottle: 24 + taskthrottle: 24 + cycledef: + - attrs: + group: cycled + spec: 200001030000 200001040000 24:00:00 + entities: + MACHINE: "orion" + SCHED: "slurm" + ACCOUNT: "epic" + EXP_BASEDIR: "/work2/noaa/epic/{USER}/landda_test" + JEDI_INSTALL: "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules" + WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART" + ATMOS_FORC: "gswp3" + RES: "96" + FCSTHR: "24" + NPROCS_ANALYSIS: "6" + NPROCS_FORECAST: "7" + OBSDIR: "" + OBSDIR_SUBDIR: "" + OBS_TYPES: "GHCN" + DAtype: "letkfoi_snow" + TSTUB: "oro_C96.mx100" + WE2E_VAV: "YES" + WE2E_ATOL: "1e-7" + WE2E_LOG_FN: "we2e.log" + NET: "landda" + envir: "test" + model_ver: "v2.0.0" + RUN: "landda" + HOMElandda: "&EXP_BASEDIR;/land-DA_workflow" + PTMP: "&EXP_BASEDIR;/ptmp" + COMROOT: "&PTMP;/&envir;/com" + DATAROOT: "&PTMP;/&envir;/tmp" + KEEPDATA: "YES" + LOGDIR: "&COMROOT;/output/logs" + LOGFN_SUFFIX: "_@Y@m@d@H.log" + PDY: "@Y@m@d" + cyc: "@H" + DATADEP_FILE1: "&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc" + DATADEP_FILE2: "&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.nc" + DATADEP_FILE3: "&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc" + DATADEP_FILE4: "&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.nc" + log: "&LOGDIR;/workflow.log" + tasks: + task_prep_obs: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + OBSDIR: "&OBSDIR;" + OBSDIR_SUBDIR: "&OBSDIR_SUBDIR;" + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + ATMOS_FORC: "&ATMOS_FORC;" + model_ver: "&model_ver;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "prep_obs" "&HOMElandda;" "&MACHINE;"' + jobname: prep_obs + cores: 1 + walltime: 00:02:00 + queue: batch + join: "&LOGDIR;/prep_obs&LOGFN_SUFFIX;" + task_pre_anal: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WARMSTART_DIR: "&WARMSTART_DIR;" + model_ver: "&model_ver;" + RUN: "&RUN;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "pre_anal" "&HOMElandda;" "&MACHINE;"' + jobname: pre_anal + cores: 1 + walltime: 00:05:00 + queue: batch + join: "&LOGDIR;/pre_anal&LOGFN_SUFFIX;" + dependency: + or: + datadep_file1: + attrs: + age: 5 + value: "&DATADEP_FILE1;" + datadep_file2: + attrs: + age: 5 + value: "&DATADEP_FILE2;" + datadep_file3: + attrs: + age: 5 + value: "&DATADEP_FILE3;" + datadep_file4: + attrs: + age: 5 + value: "&DATADEP_FILE4;" + task_analysis: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WE2E_VAV: "&WE2E_VAV;" + WE2E_ATOL: "&WE2E_ATOL;" + WE2E_LOG_FN: "&WE2E_LOG_FN;" + LOGDIR: "&LOGDIR;" + model_ver: "&model_ver;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + DAtype: "&DAtype;" + NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;" + JEDI_INSTALL: "&JEDI_INSTALL;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "analysis" "&HOMElandda;" "&MACHINE;"' + jobname: analysis + nodes: "1:ppn=&NPROCS_ANALYSIS;" + walltime: 00:15:00 + queue: batch + join: "&LOGDIR;/analysis&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: pre_anal + task_post_anal: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WE2E_VAV: "&WE2E_VAV;" + WE2E_ATOL: "&WE2E_ATOL;" + WE2E_LOG_FN: "&WE2E_LOG_FN;" + LOGDIR: "&LOGDIR;" + model_ver: "&model_ver;" + RUN: "&RUN;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + FCSTHR: "&FCSTHR;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "post_anal" "&HOMElandda;" "&MACHINE;"' + jobname: post_anal + cores: 1 + walltime: 00:05:00 + queue: batch + join: "&LOGDIR;/post_anal&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: analysis + task_plot_stats: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + model_ver: "&model_ver;" + RUN: "&RUN;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"' + jobname: plot_stats + cores: 1 + walltime: 00:10:00 + queue: batch + join: "&LOGDIR;/plot_stats&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: analysis + task_forecast: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + ATMOS_FORC: "&ATMOS_FORC;" + RES: "&RES;" + WARMSTART_DIR: "&WARMSTART_DIR;" + WE2E_VAV: "&WE2E_VAV;" + WE2E_ATOL: "&WE2E_ATOL;" + WE2E_LOG_FN: "&WE2E_LOG_FN;" + LOGDIR: "&LOGDIR;" + model_ver: "&model_ver;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + LOGDIR: "&LOGDIR;" + PDY: "&PDY;" + cyc: "&cyc;" + DAtype: "&DAtype;" + FCSTHR: "&FCSTHR;" + NPROCS_FORECAST: "&NPROCS_FORECAST;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"' + jobname: forecast + nodes: "1:ppn=&NPROCS_FORECAST;" + walltime: 01:00:00 + queue: batch + join: "&LOGDIR;/forecast&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: post_anal diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml index 9b913465..575f9c3e 100644 --- a/parm/land_analysis_orion.yaml +++ b/parm/land_analysis_orion.yaml @@ -7,8 +7,7 @@ workflow: cycledef: - attrs: group: cycled - spec: 200001030000 200001040000 24:00:00 # for gswp3 -# spec: 201912210000 201912220000 24:00:00 # for era5 + spec: 200001030000 200001040000 24:00:00 entities: MACHINE: "orion" SCHED: "slurm" diff --git a/parm/launch_rocoto_wflow.sh b/parm/launch_rocoto_wflow.sh index 968ee88c..a0c07ef9 100755 --- a/parm/launch_rocoto_wflow.sh +++ b/parm/launch_rocoto_wflow.sh @@ -19,6 +19,9 @@ elif [ "${MACHINE}" == "orion" ]; then module load contrib module load ruby/3.2.3 module load rocoto/1.3.7 +elif [ "${MACHINE}" == "hercules" ]; then + module load contrib + module load rocoto else echo "FATAL ERROR: modules are not loaded" fi From 88498af34ccc0f7091a5d58f332738e44006990d Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 10:26:38 -0500 Subject: [PATCH 4/7] add we2e to pr template --- .github/pull_request_template.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4ff695a8..050faa32 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -27,6 +27,8 @@ EXAMPLE: Closes ufs-community/land-DA/issues/ - [ ] Hercules - CI - [ ] Completed +- WE2E + - [ ] Completed - PW-Clouds - [ ] AWS - [ ] AZURE From 51dc0c350a8e50f538dfa81bdb62dbb99b5d89bd Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 11:06:32 -0500 Subject: [PATCH 5/7] fix typo in ctest --- .gitignore | 1 + sorc/test/run_hercules_ctest.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fc1c342b..97c34396 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ sorc/build/ sorc/conda/ sorc/lib/ sorc/lib64 +sorc/test/out.* parm/conda_loc parm/*.xml parm/*.db diff --git a/sorc/test/run_hercules_ctest.sh b/sorc/test/run_hercules_ctest.sh index 01aa2142..63121b8e 100755 --- a/sorc/test/run_hercules_ctest.sh +++ b/sorc/test/run_hercules_ctest.sh @@ -1,7 +1,7 @@ #!/bin/bash set -eux -JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=7 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') +JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=7 --nodes=1 --time=00:30:00 ./hercules_ctest.sh | awk '{print $4}') CHECK_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) From 0b2095c87397cfa56c407a33ff29e1d88210e37f Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 11:14:05 -0500 Subject: [PATCH 6/7] rm dummy line --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 97c34396..fc1c342b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ sorc/build/ sorc/conda/ sorc/lib/ sorc/lib64 -sorc/test/out.* parm/conda_loc parm/*.xml parm/*.db From 6f45444af21c5887c6ebe9d7676f7ace12ad9c22 Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 11:21:22 -0500 Subject: [PATCH 7/7] adjust time limit --- sorc/test/run_hercules_ctest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/test/run_hercules_ctest.sh b/sorc/test/run_hercules_ctest.sh index 63121b8e..7ab48df8 100755 --- a/sorc/test/run_hercules_ctest.sh +++ b/sorc/test/run_hercules_ctest.sh @@ -5,7 +5,7 @@ JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=7 CHECK_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) -sleep 7m +sleep 5m if [ -f out.ctest ]; then cat out.ctest