#3708: First attempt at adding DKRZ to the RTW

esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf

@@ -0,0 +1,10 @@
+[command]
+default=signularity-env singularity build ${CONTAINER_DIR}/${CONTAINER_FILE} ${DOCKER_SOURCE}
+
+[env]
+CONTAINER_DIR=${ROSE_DATAC}/container
+CONTAINER_FILE=esmvaltool.sif
+DOCKER_SOURCE=docker://esmvalgroup/esmvaltool:${ENV_NAME}
+
+[file:${CONTAINER_DIR}]
+mode=mkdir

esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf

@@ -0,0 +1,10 @@
+[template variables]
+DRS_CMIP5="DKRZ"
+DRS_CMIP6="DKRZ"
+ENV_NAME="latest"
+KGO_ROOT_PATH="/work/bd0854/b382148/testing_ESMValTool_v2.11.0rc1/run1/esmvaltool_output/"
+ROOTPATH_CMIP5="/work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ"
+ROOTPATH_CMIP6="/work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ"
+ROOTPATH_OBS="/work/bd0854/DATA/ESMValTool2/OBS"
+ROOTPATH_OBS4MIPS="/work/bd0854/DATA/ESMValTool2/OBS"
+SITE="dkrz"

esmvaltool/utils/recipe_test_workflow/site/dkrz-env

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# USAGE dkrz-env COMMAND
+#
+# ENVIRONMENT
+#   ENV_NAME            The name of the container being used
+#   QUIET_MODE          Don't print confirmation messages
+#   PYTHONPATH_PREPEND  The path to prepend to PYTHONPATH
+#
+# OPTIONS
+#   COMMAND             The command to execute with options
+set -eu
+
+# Must be run before importing numpy, see
+# https://docs.dask.org/en/stable/array-best-practices.html#avoid-oversubscribing-threads
+export OMP_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export VECLIB_MAXIMUM_THREADS=1
+export NUMEXPR_NUM_THREADS=1
+
+# Ensure '~/.local' isn't added to 'sys.path'.
+export PYTHONNOUSERSITE=True
+
+WORKFLOW_RUN_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/bin"
+WORKFLOW_SHARE_BIN_DIR="${CYLC_WORKFLOW_SHARE_DIR}/cycle/bin"
+ROSE_APP_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/app/${ROSE_TASK_APP:-$CYLC_TASK_NAME}/bin"
+CONTAINER=${ROSE_DATAC}/container/esmvaltool.sif
+
+# Bind paths for container. Where symbolic links are used in file paths (e.g.
+# under /badc/cmip6 and /home/users) need to bind the root dirs of both the
+# source and target files or directories.
+export SINGULARITY_BIND="/home/b,/work"
+
+# Suppress an ESMValTool "file not found" warning
+export SINGULARITYENV_PROJ_DATA="/opt/conda/envs/esmvaltool/share/proj"
+
+# Provide mkfile needed to build esmfpy package
+export SINGULARITYENV_ESMFMKFILE="/opt/conda/envs/esmvaltool/lib/esmf.mk"
+
+# Ensure that `singularity exec` finds the right version of python
+export SINGULARITYENV_PREPEND_PATH="/opt/conda/envs/esmvaltool/bin"
+
+# Include Rose/Cylc workflow directories in container PATH
+export SINGULARITYENV_APPEND_PATH="${WORKFLOW_RUN_BIN_DIR}:${WORKFLOW_SHARE_BIN_DIR}:${ROSE_APP_BIN_DIR}"
+
+# If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the
+# Python environment.
+if [[ ! -z ${PYTHONPATH_PREPEND:-} ]]; then
+    echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}"
+    export PYTHONPATH=${PYTHONPATH_PREPEND}:${PYTHONPATH:-}
+fi
+
+
+if [[ -z ${QUIET_MODE:-} ]]; then
+    echo "[INFO] Using the ${ENV_NAME} container"
+fi
+
+/usr/bin/time -v -o "${CYLC_TASK_LOG_ROOT}.time" singularity-env singularity -q exec "${CONTAINER}" "$@"

esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env

@@ -0,0 +1,27 @@
+##!/bin/bash
+#
+# USAGE dkrz-singularity-env COMMAND
+#
+# OPTIONS
+#   COMMAND             The command to execute with options
+set -eu
+
+module_count(){
+    module list -t 2>&1 | wc -l
+}
+
+safe_load(){
+    PRE_LOAD_COUNT=$(module_count)
+
+    module load "${1}"
+    # Check module count to determine whether module load was successful.
+
+    if (( PRE_LOAD_COUNT == $(module_count) )); then
+        echo "[ERROR] Failed to load: ${1}"
+        exit 1
+    fi
+}
+safe_load "singularity"
+
+command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time $@"
+exec ${command}

esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc

@@ -0,0 +1,47 @@
+#!jinja2
+[runtime]
+
+    # COMPUTE provides defaults for computation-heavy tasks.
+    # Specific tasks below override some defaults, e.g. time & memory.
+    [[COMPUTE]]
+        platform = levante
+        execution time limit = PT3M
+        [[[directives]]]
+            --wckey = RTW
+            --account = bk1088
+            --partition = interactive
+            --mem = 2G
+
+    [[install_env_file]]
+        [[[environment]]]
+            ROSE_APP_OPT_CONF_KEYS = {{ SITE }}
+            SINGULARITY_ENV_FILE=singularity-env
+            SINGULARITY_ENV_FILE_SITE_PATH=${CYLC_WORKFLOW_RUN_DIR}/site/${SITE}-singularity-env
+
+    [[get_esmval]]
+        platform = localhost
+        execution time limit = PT10M  # Actual: 2m17s on 2024-12-06.
+        [[[directives]]]
+            --mem = 10G
+
+    [[configure]]
+        inherit = locahost
+        execution time limit = PT2M
+        [[[directives]]]
+            --mem = 2G
+
+    # Resources for recipes that need more than the default. Both time and
+    # memory should be specified, in case the default changes.
+    # Variable (fast, medium) must be consistent with flow.cylc.
+    # Comment indicates example recorded usage on DKRZ.
+    [[process<fast=recipe_autoassess_landsurface_soilmoisture>]]
+        # Actual: 1m35s, 2.1 GB on 2024-03-29.
+        execution time limit = PT3M
+        [[[directives]]]
+            --mem = 3G
+
+    [[process<medium=recipe_ensclus>]]
+        # Actual: 4m23s, 1.5 GB on 2024-03-29.
+        execution time limit = PT6M
+        [[[directives]]]
+            --mem = 2G