diff --git a/doc/sphinx/source/utils/RTW/user_guide/quick_start.rst b/doc/sphinx/source/utils/RTW/user_guide/quick_start.rst index f15082ae84..6574aa24ee 100644 --- a/doc/sphinx/source/utils/RTW/user_guide/quick_start.rst +++ b/doc/sphinx/source/utils/RTW/user_guide/quick_start.rst @@ -38,5 +38,17 @@ Quick Start Guide export CYLC_VERSION=8 cylc vip -O jasmin -* Browse the logs using `Cylc Review`_, a web service for browsing logs via an - HTTP interface. + * on DKRZ: + + * add the following line to your ``~/.bashrc`` file to ensure the Cylc and + Rose executables can be found:: + + export PATH=/work/bd0854/metomi/bin:$PATH + + * run the RTW on DKRZ:: + + cd ESMValTool/esmvaltool/utils/recipe_test_workflow + cylc vip -O dkrz + +* Optionally browse the logs using `Cylc Review`_, + a web service for browsing logs via an HTTP interface. diff --git a/doc/sphinx/source/utils/RTW/user_guide/workflow.rst b/doc/sphinx/source/utils/RTW/user_guide/workflow.rst index 5871df7b78..f571e1e148 100644 --- a/doc/sphinx/source/utils/RTW/user_guide/workflow.rst +++ b/doc/sphinx/source/utils/RTW/user_guide/workflow.rst @@ -25,9 +25,7 @@ The |RTW| performs the following steps: GitHub, or gets the latest container image from DockerHub and converts to a singularity image, depending on ``SITE`` :Runs on: - Localhost (if cloning), or ``COMPUTE`` (if getting container), which - depends on the ``SITE``; on JASMIN, the ``get_esmval`` jobs will run on - LOTUS + Localhost, or ``COMPUTE`` on JASMIN :Executes: The ``clone_latest_esmval.sh`` script (if cloning), or a ``singularity build`` command (if getting container) from the |Rose| app @@ -48,8 +46,7 @@ The |RTW| performs the following steps: :Description: Runs the requested recipes using |ESMValTool| :Runs on: - ``COMPUTE``, which depends on the ``SITE``; at the Met Office, the - ``process`` jobs will run on SPICE + ``COMPUTE``, which depends on the ``SITE`` :Executes: The |ESMValTool| command line script from the |Rose| app :Details: @@ -60,8 +57,7 @@ The |RTW| performs the following steps: :Description: Compares the output from the ``process`` job with |KGOs| :Runs on: - ``COMPUTE``, which depends on the ``SITE``; at the Met Office, the - ``compare`` jobs will run on SPICE + ``COMPUTE``, which depends on the ``SITE`` :Executes: The :ref:`compare.py ` script from |ESMValTool| from the |Rose| app diff --git a/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf new file mode 100644 index 0000000000..7656f549bc --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-dkrz.conf @@ -0,0 +1,8 @@ +[command] +default=singularity-env singularity build ${CONTAINER_PATH} ${DOCKER_SOURCE} + +[env] +DOCKER_SOURCE=docker://esmvalgroup/esmvaltool:${ENV_NAME} + +[file:${CONTAINER_DIR}] +mode=mkdir diff --git a/esmvaltool/utils/recipe_test_workflow/app/install_env_file/opt/rose-app-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/app/install_env_file/opt/rose-app-dkrz.conf new file mode 100644 index 0000000000..8b7e9dd4f3 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/install_env_file/opt/rose-app-dkrz.conf @@ -0,0 +1,12 @@ +# Copy the site specific environment files to the 'share/bin' directory in the +# installed Cylc workflow (this directory is automatically added to the +# ${PATH} by Cylc). + +[command] +default=BASH_XTRACEFD=1 # Send the output from 'set -x' to 'stdout' rather than 'stderr'. + =set -euxo pipefail + =cp ${ENV_FILE_SITE_PATH} ${SHARE_BIN_DIR}/${ENV_FILE} + =cp ${SINGULARITY_ENV_FILE_SITE_PATH} ${SHARE_BIN_DIR}/${SINGULARITY_ENV_FILE} + +[file:${SHARE_BIN_DIR}] +mode=mkdir diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index 0dcd7e2528..0338dce5a0 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -53,7 +53,6 @@ ROSE_APP_OPT_CONF_KEYS = {{ SITE }} [[configure]] - pre-script = "mkdir -p ${USER_CONFIG_DIR}" [[[environment]]] ROSE_TASK_APP = configure DRS_CMIP3 = {{ DRS_CMIP3 }} diff --git a/esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf new file mode 100644 index 0000000000..dca068f8c2 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/opt/rose-suite-dkrz.conf @@ -0,0 +1,10 @@ +[template variables] +DRS_CMIP5="DKRZ" +DRS_CMIP6="DKRZ" +ENV_NAME="experimental" +KGO_ROOT_PATH="/work/bd0854/KGO/esmvaltool_output/" +ROOTPATH_CMIP5="/work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ" +ROOTPATH_CMIP6="/work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ" +ROOTPATH_OBS="/work/bd0854/DATA/ESMValTool2/OBS" +ROOTPATH_OBS4MIPS="/work/bd0854/DATA/ESMValTool2/OBS" +SITE="dkrz" diff --git a/esmvaltool/utils/recipe_test_workflow/site/dkrz-env b/esmvaltool/utils/recipe_test_workflow/site/dkrz-env new file mode 100755 index 0000000000..0821e59be2 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/site/dkrz-env @@ -0,0 +1,57 @@ +#!/bin/bash +# +# USAGE dkrz-env COMMAND +# +# ENVIRONMENT +# ENV_NAME The name of the container being used +# QUIET_MODE Don't print confirmation messages +# PYTHONPATH_PREPEND The path to prepend to PYTHONPATH +# +# OPTIONS +# COMMAND The command to execute with options +set -eu + +# Must be run before importing numpy, see +# https://docs.dask.org/en/stable/array-best-practices.html#avoid-oversubscribing-threads +export OMP_NUM_THREADS=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export VECLIB_MAXIMUM_THREADS=1 +export NUMEXPR_NUM_THREADS=1 + +# Ensure '~/.local' isn't added to 'sys.path'. +export PYTHONNOUSERSITE=True + +WORKFLOW_RUN_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/bin" +WORKFLOW_SHARE_BIN_DIR="${CYLC_WORKFLOW_SHARE_DIR}/cycle/bin" +ROSE_APP_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/app/${ROSE_TASK_APP:-$CYLC_TASK_NAME}/bin" + +# Bind paths for container. +export SINGULARITY_BIND="/home/b,/work,/scratch/b" + +# Suppress an ESMValTool "file not found" warning. +export SINGULARITYENV_PROJ_DATA="/opt/conda/envs/esmvaltool/share/proj" + +# Provide mkfile needed to build esmfpy package. +export SINGULARITYENV_ESMFMKFILE="/opt/conda/envs/esmvaltool/lib/esmf.mk" + +# Ensure that `singularity exec` finds the right version of python. +export SINGULARITYENV_PREPEND_PATH="/opt/conda/envs/esmvaltool/bin" + +# Include Rose/Cylc workflow directories in container PATH. +export SINGULARITYENV_APPEND_PATH="${WORKFLOW_RUN_BIN_DIR}:${WORKFLOW_SHARE_BIN_DIR}:${ROSE_APP_BIN_DIR}" + +# If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the +# Python environment. +if [[ ! -z ${PYTHONPATH_PREPEND:-} ]]; then + echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}" + export PYTHONPATH=${PYTHONPATH_PREPEND}:${PYTHONPATH:-} +fi + +if [[ -z ${QUIET_MODE:-} ]]; then + echo "[INFO] Using the ${ENV_NAME} container" +fi + +singularity_command="singularity-env singularity -q exec ${CONTAINER_PATH} $@" +command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time ${singularity_command}" +exec ${command} diff --git a/esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env b/esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env new file mode 100755 index 0000000000..3e74cd9b4e --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/site/dkrz-singularity-env @@ -0,0 +1,34 @@ +#!/bin/bash +# +# USAGE dkrz-singularity-env COMMAND +# +# OPTIONS +# COMMAND The command to execute with options +# +# Since DKRZ uses the module environment to dynamically modify a user's +# environment via modulefiles (more details are available at +# https://docs.dkrz.de/doc/levante/access-and-environment.html#module-environment) +# this additional environment file is required to enable access to singularity +# on DKRZ. This file is used directly in the 'get_esmval' app for DKRZ, as well +# as in the 'dkrz-env' environment file. +set -eu + +module_count(){ + module list -t 2>&1 | wc -l +} + +safe_load(){ + PRE_LOAD_COUNT=$(module_count) + + module load "${1}" + # Check module count to determine whether module load was successful. + + if (( PRE_LOAD_COUNT == $(module_count) )); then + echo "[ERROR] Failed to load: ${1}" + exit 1 + fi +} +safe_load "singularity" + +command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time $@" +exec ${command} diff --git a/esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc b/esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc new file mode 100644 index 0000000000..f04252d071 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/site/dkrz.cylc @@ -0,0 +1,102 @@ +#!jinja2 +[runtime] + [[root]] + [[[environment]]] + CONTAINER_DIR = ${ROSE_DATAC}/container + CONTAINER_FILE = esmvaltool.sif + CONTAINER_PATH = ${CONTAINER_DIR}/${CONTAINER_FILE} + # Warning: fragile assumption of location of esmvaltool code, + # issue #3437 contains more details. + ESMVALTOOL_DIR = /opt/conda/envs/esmvaltool/lib/python3.12/site-packages + + # COMPUTE provides defaults for computation-heavy tasks. + # Specific tasks below override some defaults, e.g. time & memory. + [[COMPUTE]] + platform = levante + execution time limit = PT3M + [[[directives]]] + --wckey = RTW + --account = bk1088 + --partition = interactive + --mem = 15G + + [[install_env_file]] + [[[environment]]] + ROSE_APP_OPT_CONF_KEYS = {{ SITE }} + SINGULARITY_ENV_FILE=singularity-env + SINGULARITY_ENV_FILE_SITE_PATH=${CYLC_WORKFLOW_RUN_DIR}/site/${SITE}-singularity-env + + [[get_esmval]] + platform = localhost + execution time limit = PT10M # Actual: 3m06s on 2024-12-18. + [[[environment]]] + # Move the location of the singularity cache from the user's + # home area to the directory where the container is saved. + # This variable must be specified only for the 'get_esmval' + # task, otherwise, for some reason, subsequent steps take much + # longer to run than the resources specified. + SINGULARITY_CACHEDIR = ${CONTAINER_DIR} + + [[configure]] + platform = localhost + execution time limit = PT2M # Actual: 0m14s on 2024-12-18. + [[[directives]]] + --mem = 2G + + # Resources for recipes that need more than the default. Both time and + # memory should be specified, in case the default changes. + # Variable (fast, medium) must be consistent with flow.cylc. + # Comment indicates example recorded usage on DKRZ. + [[process_examples--recipe_python]] + # Actual: 0m07s 2.1 GB on 2024-12-18. + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_albedolandcover]] + # Actual: 0m10s 3.2 GB on 2024-12-18. + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_autoassess_landsurface_soilmoisture]] + # Actual: 0m13s 3.8 GB on 2024-12-18. + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_consecdrydays]] + # Actual: 0m08s 2.3 GB on 2024-12-18. + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_ensclus]] + # Actual: 0m35s 2.0 GB on 2024-12-18. + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_heatwaves_coldwaves]] + # Actual: 0m30s 1.6 GB on 2024-12-18. + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_ocean_amoc]] + # Actual: Currently broken: No input files found for Dataset + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_ocean_multimap]] + # Actual: Currently broken: No input files found for Dataset + execution time limit = PT2M + [[[directives]]] + --mem = 15G + + [[process_recipe_radiation_budget]] + # Actual: Currently broken: No input files found for Dataset + execution time limit = PT2M + [[[directives]]] + --mem =15G diff --git a/esmvaltool/utils/recipe_test_workflow/site/jasmin.cylc b/esmvaltool/utils/recipe_test_workflow/site/jasmin.cylc index 5eb1b083ad..56c70f9df7 100644 --- a/esmvaltool/utils/recipe_test_workflow/site/jasmin.cylc +++ b/esmvaltool/utils/recipe_test_workflow/site/jasmin.cylc @@ -2,7 +2,8 @@ [runtime] [[root]] [[[environment]]] - # Warning: fragile assumption of location of esmvaltool code, see #3437 + # Warning: fragile assumption of location of esmvaltool code, + # issue #3437 contains more details. ESMVALTOOL_DIR = /opt/conda/envs/esmvaltool/lib/python3.11/site-packages # COMPUTE provides defaults for computation-heavy tasks.