diff --git a/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf new file mode 100644 index 0000000000..0943aeec46 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf @@ -0,0 +1,10 @@ +[command] +default=signularity-env singularity build ${CONTAINER_DIR}/${CONTAINER_FILE} ${DOCKER_SOURCE} + +[env] +CONTAINER_DIR=${ROSE_DATAC}/container +CONTAINER_FILE=esmvaltool.sif +DOCKER_SOURCE=docker://esmvalgroup/esmvaltool:${ENV_NAME} + +[file:${CONTAINER_DIR}] +mode=mkdir diff --git a/esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf new file mode 100644 index 0000000000..f50837f11a --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf @@ -0,0 +1,10 @@ +[template variables] +DRS_CMIP5="DKRZ" +DRS_CMIP6="DKRZ" +ENV_NAME="latest" +KGO_ROOT_PATH="/work/bd0854/b382148/testing_ESMValTool_v2.11.0rc1/run1/esmvaltool_output/" +ROOTPATH_CMIP5="/work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ" +ROOTPATH_CMIP6="/work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ" +ROOTPATH_OBS="/work/bd0854/DATA/ESMValTool2/OBS" +ROOTPATH_OBS4MIPS="/work/bd0854/DATA/ESMValTool2/OBS" +SITE="dkrz" diff --git a/esmvaltool/utils/recipe_test_workflow/site/dkrz-env b/esmvaltool/utils/recipe_test_workflow/site/dkrz-env new file mode 100755 index 0000000000..bc5d414828 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/site/dkrz-env @@ -0,0 +1,59 @@ +#!/bin/bash +# +# USAGE dkrz-env COMMAND +# +# ENVIRONMENT +# ENV_NAME The name of the container being used +# QUIET_MODE Don't print confirmation messages +# PYTHONPATH_PREPEND The path to prepend to PYTHONPATH +# +# OPTIONS +# COMMAND The command to execute with options +set -eu + +# Must be run before importing numpy, see +# https://docs.dask.org/en/stable/array-best-practices.html#avoid-oversubscribing-threads +export OMP_NUM_THREADS=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export VECLIB_MAXIMUM_THREADS=1 +export NUMEXPR_NUM_THREADS=1 + +# Ensure '~/.local' isn't added to 'sys.path'. +export PYTHONNOUSERSITE=True + +WORKFLOW_RUN_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/bin" +WORKFLOW_SHARE_BIN_DIR="${CYLC_WORKFLOW_SHARE_DIR}/cycle/bin" +ROSE_APP_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/app/${ROSE_TASK_APP:-$CYLC_TASK_NAME}/bin" +CONTAINER=${ROSE_DATAC}/container/esmvaltool.sif + +# Bind paths for container. Where symbolic links are used in file paths (e.g. +# under /badc/cmip6 and /home/users) need to bind the root dirs of both the +# source and target files or directories. +export SINGULARITY_BIND="/home/b,/work" + +# Suppress an ESMValTool "file not found" warning +export SINGULARITYENV_PROJ_DATA="/opt/conda/envs/esmvaltool/share/proj" + +# Provide mkfile needed to build esmfpy package +export SINGULARITYENV_ESMFMKFILE="/opt/conda/envs/esmvaltool/lib/esmf.mk" + +# Ensure that `singularity exec` finds the right version of python +export SINGULARITYENV_PREPEND_PATH="/opt/conda/envs/esmvaltool/bin" + +# Include Rose/Cylc workflow directories in container PATH +export SINGULARITYENV_APPEND_PATH="${WORKFLOW_RUN_BIN_DIR}:${WORKFLOW_SHARE_BIN_DIR}:${ROSE_APP_BIN_DIR}" + +# If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the +# Python environment. +if [[ ! -z ${PYTHONPATH_PREPEND:-} ]]; then + echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}" + export PYTHONPATH=${PYTHONPATH_PREPEND}:${PYTHONPATH:-} +fi + + +if [[ -z ${QUIET_MODE:-} ]]; then + echo "[INFO] Using the ${ENV_NAME} container" +fi + +/usr/bin/time -v -o "${CYLC_TASK_LOG_ROOT}.time" singularity-env singularity -q exec "${CONTAINER}" "$@" diff --git a/esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env b/esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env new file mode 100644 index 0000000000..b47a7b4bd9 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env @@ -0,0 +1,27 @@ +##!/bin/bash +# +# USAGE dkrz-singularity-env COMMAND +# +# OPTIONS +# COMMAND The command to execute with options +set -eu + +module_count(){ + module list -t 2>&1 | wc -l +} + +safe_load(){ + PRE_LOAD_COUNT=$(module_count) + + module load "${1}" + # Check module count to determine whether module load was successful. + + if (( PRE_LOAD_COUNT == $(module_count) )); then + echo "[ERROR] Failed to load: ${1}" + exit 1 + fi +} +safe_load "singularity" + +command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time $@" +exec ${command} diff --git a/esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc b/esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc new file mode 100644 index 0000000000..df4039dd8f --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc @@ -0,0 +1,47 @@ +#!jinja2 +[runtime] + + # COMPUTE provides defaults for computation-heavy tasks. + # Specific tasks below override some defaults, e.g. time & memory. + [[COMPUTE]] + platform = levante + execution time limit = PT3M + [[[directives]]] + --wckey = RTW + --account = bk1088 + --partition = interactive + --mem = 2G + + [[install_env_file]] + [[[environment]]] + ROSE_APP_OPT_CONF_KEYS = {{ SITE }} + SINGULARITY_ENV_FILE=singularity-env + SINGULARITY_ENV_FILE_SITE_PATH=${CYLC_WORKFLOW_RUN_DIR}/site/${SITE}-singularity-env + + [[get_esmval]] + platform = localhost + execution time limit = PT10M # Actual: 2m17s on 2024-12-06. + [[[directives]]] + --mem = 10G + + [[configure]] + inherit = locahost + execution time limit = PT2M + [[[directives]]] + --mem = 2G + + # Resources for recipes that need more than the default. Both time and + # memory should be specified, in case the default changes. + # Variable (fast, medium) must be consistent with flow.cylc. + # Comment indicates example recorded usage on DKRZ. + [[process]] + # Actual: 1m35s, 2.1 GB on 2024-03-29. + execution time limit = PT3M + [[[directives]]] + --mem = 3G + + [[process]] + # Actual: 4m23s, 1.5 GB on 2024-03-29. + execution time limit = PT6M + [[[directives]]] + --mem = 2G