From 568ac951246e81c226ce545cc143bae4fa9d699c Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Mon, 29 Jul 2024 10:56:44 +1000 Subject: [PATCH 01/13] Updated to state tests to work in-memory for true temporary testing --- tests/test_state.py | 47 ++++++++++++++++++++++++--------------------- tests/test_utils.py | 21 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/tests/test_state.py b/tests/test_state.py index 51738bd..7c77f33 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -3,37 +3,40 @@ import pytest from benchcab.utils.state import State, StateAttributeError +from tempfile import TemporaryDirectory -@pytest.fixture() -def state(): - """Return a State object.""" - return State(state_dir=Path("my_state")) - - -def test_state_is_set(state): +def test_state_is_set(): """Success case: test state is set.""" - state.set("foo") - assert state.is_set("foo") + with TemporaryDirectory() as tmp_dir: + state = State(state_dir=Path(tmp_dir)) + state.set("foo") + assert state.is_set("foo") -def test_state_reset(state): +def test_state_reset(): """Success case: test state is reset.""" - state.set("foo") - state.reset() - assert not state.is_set("foo") + with TemporaryDirectory() as tmp_dir: + state = State(state_dir=Path(tmp_dir)) + state.set("foo") + state.reset() + assert not state.is_set("foo") -def test_state_get(state): +def test_state_get(): """Success case: test get() returns the most recent state attribute.""" - state.set("foo") - # This is done so that time stamps can be resolved between state attributes - time.sleep(0.01) - state.set("bar") - assert state.get() == "bar" + with TemporaryDirectory() as tmp_dir: + state = State(state_dir=Path(tmp_dir)) + state.set("foo") + # This is done so that time stamps can be resolved between state attributes + time.sleep(1) + state.set("bar") + assert state.get() == "bar" -def test_state_get_raises_exception(state): +def test_state_get_raises_exception(): """Failure case: test get() raises an exception when no attributes are set.""" - with pytest.raises(StateAttributeError): - state.get() + with TemporaryDirectory() as tmp_dir: + state = State(state_dir=Path(tmp_dir)) + with pytest.raises(StateAttributeError): + state.get() \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index a99fadd..37284ba 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,7 @@ import pytest import benchcab.utils as bu +from benchcab.comparison import ComparisonTask def test_get_installed_root(): @@ -60,3 +61,23 @@ def test_get_logger_singleton_fail(): logger2 = bu.get_logger(name="benchcab2") assert logger1 is not logger2 + + +def test_task_summary(): + + # Create some mocked tasks + t1 = ComparisonTask(files=(), task_name="t1") + t2 = ComparisonTask(files=(), task_name="t2") + + # Inject success/fail cases + t1.is_done = lambda: True + t2.is_done = lambda: False + + # Run the function + n_tasks, n_success, n_failed, all_complete = bu.task_summary([t1, t2]) + + # Check correct results + assert n_tasks == 2 + assert n_success == 1 + assert n_failed == 1 + assert all_complete == False \ No newline at end of file From 7ee1b7b1df0eebd2a20cb278118dea66926ecf8b Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Mon, 29 Jul 2024 10:57:57 +1000 Subject: [PATCH 02/13] Added task summary helper for meorg_client integration. Fixes #300 --- src/benchcab/benchcab.py | 12 +++++------- src/benchcab/utils/__init__.py | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/benchcab/benchcab.py b/src/benchcab/benchcab.py index 81b5574..273e753 100644 --- a/src/benchcab/benchcab.py +++ b/src/benchcab/benchcab.py @@ -22,7 +22,7 @@ from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface from benchcab.internal import get_met_forcing_file_names from benchcab.model import Model -from benchcab.utils import is_verbose +from benchcab.utils import is_verbose, task_summary from benchcab.utils.fs import mkdir, next_path from benchcab.utils.pbs import render_job_script from benchcab.utils.repo import create_repo @@ -347,8 +347,7 @@ def fluxsite_run_tasks(self, config_path: str): else: fluxsite.run_tasks(tasks) - tasks_failed = [task for task in tasks if not task.is_done()] - n_failed, n_success = len(tasks_failed), len(tasks) - len(tasks_failed) + n_tasks, n_success, n_failed, all_complete = task_summary(tasks) logger.info(f"{n_failed} failed, {n_success} passed") def fluxsite_bitwise_cmp(self, config_path: str): @@ -374,10 +373,9 @@ def fluxsite_bitwise_cmp(self, config_path: str): ncpus = config["fluxsite"]["pbs"]["ncpus"] run_comparisons_in_parallel(comparisons, n_processes=ncpus) else: - run_comparisons(comparisons) - - tasks_failed = [task for task in comparisons if not task.is_done()] - n_failed, n_success = len(tasks_failed), len(comparisons) - len(tasks_failed) + run_comparisons(comparisons) + + n_tasks, n_success, n_failed, all_complete = task_summary(comparisons) logger.info(f"{n_failed} failed, {n_success} passed") def fluxsite(self, config_path: str, no_submit: bool, skip: list[str]): diff --git a/src/benchcab/utils/__init__.py b/src/benchcab/utils/__init__.py index db8c427..b652c4f 100644 --- a/src/benchcab/utils/__init__.py +++ b/src/benchcab/utils/__init__.py @@ -11,7 +11,7 @@ import sys from importlib import resources from pathlib import Path -from typing import Union +from typing import Union, Iterable import yaml from jinja2 import BaseLoader, Environment @@ -148,3 +148,23 @@ def get_logger(name="benchcab", level="debug"): def is_verbose(): """Return True if verbose output is enabled, False otherwise.""" return get_logger().getEffectiveLevel() == logging.DEBUG + + +def task_summary(tasks: Iterable) -> tuple: + """Return a summary of task completions. + + Parameters + ---------- + tasks : Iterable + Iterable of tasks with an .is_done() method available. + + Returns + ------- + tuple + num_tasks, num_complete, num_failed, all_complete + """ + num_tasks = len(tasks) + num_complete = len([task for task in tasks if task.is_done()]) + num_failed = num_tasks - num_complete + + return num_tasks, num_complete, num_failed, num_complete == num_tasks \ No newline at end of file From 26122842d33e13bbf4089e99c1662218084c7706 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Wed, 28 Aug 2024 15:30:54 +1000 Subject: [PATCH 03/13] Added scaffolding for ME.org integration. --- .conda/benchcab-dev.yaml | 2 + src/benchcab/benchcab.py | 14 +++- src/benchcab/data/config-schema.yml | 3 + src/benchcab/data/meorg_jobscript.j2 | 43 +++++++++++++ src/benchcab/internal.py | 9 +++ src/benchcab/utils/meorg.py | 95 ++++++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 src/benchcab/data/meorg_jobscript.j2 create mode 100644 src/benchcab/utils/meorg.py diff --git a/.conda/benchcab-dev.yaml b/.conda/benchcab-dev.yaml index b304a3b..ddf3047 100644 --- a/.conda/benchcab-dev.yaml +++ b/.conda/benchcab-dev.yaml @@ -16,6 +16,8 @@ dependencies: - cerberus>=1.3.5 - gitpython - jinja2 + - hpcpy>=0.3.0 + - meorg_client # CI - pytest-cov # Dev Dependencies diff --git a/src/benchcab/benchcab.py b/src/benchcab/benchcab.py index 273e753..457b9b8 100644 --- a/src/benchcab/benchcab.py +++ b/src/benchcab/benchcab.py @@ -24,6 +24,7 @@ from benchcab.model import Model from benchcab.utils import is_verbose, task_summary from benchcab.utils.fs import mkdir, next_path +import benchcab.utils.meorg as bm from benchcab.utils.pbs import render_job_script from benchcab.utils.repo import create_repo from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface @@ -234,13 +235,24 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None: logger.error(exc.output) raise - logger.info(f"PBS job submitted: {proc.stdout.strip()}") + # Get the job ID + job_id = proc.stdout.strip() + + logger.info(f"PBS job submitted: {job_id}") logger.info("CABLE log file for each task is written to:") logger.info(f"{internal.FLUXSITE_DIRS['LOG']}/_log.txt") logger.info("The CABLE standard output for each task is written to:") logger.info(f"{internal.FLUXSITE_DIRS['TASKS']}//out.txt") logger.info("The NetCDF output for each task is written to:") logger.info(f"{internal.FLUXSITE_DIRS['OUTPUT']}/_out.nc") + + # Upload to meorg by default + bm.do_meorg( + config, + upload_dir=internal.FLUXSITE_DIRS['OUTPUT'], + benchcab_bin=str(self.benchcab_exe_path) + ) + def gen_codecov(self, config_path: str): """Endpoint for `benchcab codecov`.""" diff --git a/src/benchcab/data/config-schema.yml b/src/benchcab/data/config-schema.yml index 07a55f4..ee83fdf 100644 --- a/src/benchcab/data/config-schema.yml +++ b/src/benchcab/data/config-schema.yml @@ -107,6 +107,9 @@ fluxsite: schema: type: "string" required: false + meorg_model_output_id: + type: "string" + required: false spatial: type: "dict" diff --git a/src/benchcab/data/meorg_jobscript.j2 b/src/benchcab/data/meorg_jobscript.j2 new file mode 100644 index 0000000..1c89caa --- /dev/null +++ b/src/benchcab/data/meorg_jobscript.j2 @@ -0,0 +1,43 @@ +#!/bin/bash +#PBS -l wd +#PBS -l ncpus={{num_threads}} +#PBS -l mem={{mem}} +#PBS -l walltime={{walltime}} +#PBS -q copyq +#PBS -P {{project}} +#PBS -j oe +#PBS -m e +#PBS -l storage={{storage_str}} + +module purge +{% for module in modules -%} +module load {{module}} +{% endfor %} +set -ev + +# Set some things +DATA_DIR={{data_dir}} +NUM_THREADS={{num_threads}} +MODEL_OUTPUT_ID={{model_output_id}} +CACHE_DELAY={{cache_delay}} +MEORG_BIN={{meorg_bin}} + +{% if purge_outputs %} +# Purge existing model outputs +echo "Purging existing outputs from $MODEL_OUTPUT_ID" +$MEORG_BIN file detach_all $MODEL_OUTPUT_ID +{% endif %} + +# Upload the data +echo "Uploading data to $MODEL_OUTPUT_ID" +$MEORG_BIN file upload $DATA_DIR/*.nc -n $NUM_THREADS --attach_to $MODEL_OUTPUT_ID + +# Wait for the cache to transfer to the object store. +echo "Waiting for object store transfer ($CACHE_DELAY sec)" +sleep $CACHE_DELAY + +# Trigger the analysis +echo "Triggering analysis on $MODEL_OUTPUT_ID" +$MEORG_BIN analysis start $MODEL_OUTPUT_ID + +echo "DONE" \ No newline at end of file diff --git a/src/benchcab/internal.py b/src/benchcab/internal.py index 5c064a8..58dd479 100644 --- a/src/benchcab/internal.py +++ b/src/benchcab/internal.py @@ -274,3 +274,12 @@ def get_met_forcing_file_names(experiment: str) -> list[str]: ] return file_names + +# Configuration for the client upload +MEORG_CLIENT = dict( + num_threads=4, # Parallel uploads over 4 cores + cache_delay=60*5, # 5mins between upload and analysis triggering + mem="8G", + walltime="01:00:00", + storage=["gdata/ks32", "gdata/hh5", "gdata/wd9"] +) \ No newline at end of file diff --git a/src/benchcab/utils/meorg.py b/src/benchcab/utils/meorg.py new file mode 100644 index 0000000..66aa44f --- /dev/null +++ b/src/benchcab/utils/meorg.py @@ -0,0 +1,95 @@ +"""Utility methods for interacting with the ME.org client.""" +from benchcab.internal import MEORG_CLIENT +from meorg_client.client import Client as MeorgClient +from hpcpy.client import PBSClient +import benchcab.utils as bu +import os + +def do_meorg(config: dict, upload_dir: str, benchcab_bin: str): + """Perform the upload of model outputs to modelevaluation.org + + Parameters + ---------- + config : dict + The master config dictionary + upload_dir : str + Absolute path to the data dir for upload + benchcab_bin : str + Path to the benchcab bin, from which to infer the client bin + + Returns + ------- + bool + True if successful, False otherwise + """ + + logger = bu.get_logger() + + model_output_id = config.get("fluxsite").get("meorg_model_output_id", False) + num_threads = MEORG_CLIENT["num_threads"] + + # Only run if upload is enabled and a model output id is configured + if config.get("meorg_upload", True) == False: + logger.debug("meorg_upload is disabled") + return False + + # Check if a model output id has been assigned + if model_output_id == False: + logger.error("meorg_upload is set to True, but no meorg_model_output_id key found.") + logger.error("NOT uploading to modelevaluation.org") + return False + + # Allow the user to specify an absolute path to the meorg bin + meorg_bin = config.get("meorg_bin", False) + + # Otherwise infer the path from the benchcab installation + if meorg_bin == False: + bin_segments = benchcab_bin.split("/") + bin_segments[-1] = "meorg" + meorg_bin = "/".join(bin_segments) + + # Now check if that actually exists + if os.path.isfile(meorg_bin) == False: + logger.error(f"No meorg_client executable found at {meorg_bin}") + logger.error("NOT uploading to modelevaluation.org") + return False + + # Also only run if the client is initialised + if MeorgClient().is_initialised() == False: + + logger.warn("meorg_upload is set to True, but the client is not initialised.") + logger.warn("To initialise, run `meorg initialise` in the installation environment.") + logger.warn("Once initialised, the outputs from this run can be uploaded with the following command:") + logger.warn(f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {model_output_id}") + logger.warn("Then the analysis can be triggered with:") + logger.warn(f"meorg analysis start {model_output_id}") + return False + + # Finally, attempt the upload! + else: + + logger.info("Uploading outputs to modelevaluation.org") + + # Submit the outputs + client = PBSClient() + meorg_jobid = client.submit( + + bu.get_installed_root() / "data" / "meorg_jobscript.j2", + render=True, + dry_run=False, + + model_output_id=model_output_id, + data_dir=upload_dir, + cache_delay=MEORG_CLIENT["cache_delay"], + mem=MEORG_CLIENT["mem"], + + walltime=MEORG_CLIENT["walltime"], + storage=MEORG_CLIENT['storage'], + project=config['project'], + modules=config['modules'], + purge_outputs=True, + meorg_bin=meorg_bin + ) + + logger.info(f"Upload job submitted: {meorg_jobid}") + return True \ No newline at end of file From f6fadbc48caec4bd074522ceb449caa119e56243 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Fri, 4 Oct 2024 15:59:53 +1000 Subject: [PATCH 04/13] Updated integration code. Fixes #300 --- src/benchcab/benchcab.py | 3 +- src/benchcab/data/test/integration_meorg.sh | 52 +++++++++++++++++++++ src/benchcab/internal.py | 4 +- src/benchcab/utils/meorg.py | 29 ++++++------ 4 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 src/benchcab/data/test/integration_meorg.sh diff --git a/src/benchcab/benchcab.py b/src/benchcab/benchcab.py index 457b9b8..4eec1af 100644 --- a/src/benchcab/benchcab.py +++ b/src/benchcab/benchcab.py @@ -250,7 +250,8 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None: bm.do_meorg( config, upload_dir=internal.FLUXSITE_DIRS['OUTPUT'], - benchcab_bin=str(self.benchcab_exe_path) + benchcab_bin=str(self.benchcab_exe_path), + benchcab_job_id=job_id ) diff --git a/src/benchcab/data/test/integration_meorg.sh b/src/benchcab/data/test/integration_meorg.sh new file mode 100644 index 0000000..26b1d1e --- /dev/null +++ b/src/benchcab/data/test/integration_meorg.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -ex + +CABLE_REPO="git@github.com:CABLE-LSM/CABLE.git" +CABLE_DIR=/scratch/$PROJECT/$USER/benchcab/CABLE + +TEST_DIR=/scratch/$PROJECT/$USER/benchcab/integration +EXAMPLE_REPO="git@github.com:CABLE-LSM/bench_example.git" + +# Remove CABLE and test work space, then recreate +rm -rf $CABLE_DIR +mkdir -p $CABLE_DIR + +rm -rf $TEST_DIR +mkdir -p $TEST_DIR + +# Clone local checkout for CABLE +git clone $CABLE_REPO $CABLE_DIR +cd $CABLE_DIR + +# Clone the example repo +git clone $EXAMPLE_REPO $TEST_DIR +cd $TEST_DIR +git reset --hard 9bfba54ee8bf23141d95b1abe4b7207b0f3498e2 + +cat > config.yaml << EOL +project: $PROJECT + +realisations: + - repo: + local: + path: $CABLE_DIR + - repo: + git: + branch: main +modules: [ + intel-compiler/2021.1.1, + netcdf/4.7.4, + openmpi/4.1.0 +] + +fluxsite: + experiment: AU-Tum + pbs: + storage: + - scratch/$PROJECT + - gdata/$PROJECT + meorg_model_output_id: Sss7qupAHEZ8ovbCv +EOL + +benchcab run -v diff --git a/src/benchcab/internal.py b/src/benchcab/internal.py index 58dd479..be56982 100644 --- a/src/benchcab/internal.py +++ b/src/benchcab/internal.py @@ -277,9 +277,9 @@ def get_met_forcing_file_names(experiment: str) -> list[str]: # Configuration for the client upload MEORG_CLIENT = dict( - num_threads=4, # Parallel uploads over 4 cores + num_threads=1, # Parallel uploads over 4 cores cache_delay=60*5, # 5mins between upload and analysis triggering mem="8G", walltime="01:00:00", - storage=["gdata/ks32", "gdata/hh5", "gdata/wd9"] + storage=["gdata/ks32", "gdata/hh5", "gdata/wd9", "gdata/rp23"] ) \ No newline at end of file diff --git a/src/benchcab/utils/meorg.py b/src/benchcab/utils/meorg.py index 66aa44f..9359ccb 100644 --- a/src/benchcab/utils/meorg.py +++ b/src/benchcab/utils/meorg.py @@ -1,11 +1,12 @@ """Utility methods for interacting with the ME.org client.""" from benchcab.internal import MEORG_CLIENT from meorg_client.client import Client as MeorgClient -from hpcpy.client import PBSClient +from hpcpy import get_client import benchcab.utils as bu import os +from glob import glob -def do_meorg(config: dict, upload_dir: str, benchcab_bin: str): +def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: str): """Perform the upload of model outputs to modelevaluation.org Parameters @@ -27,28 +28,26 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str): model_output_id = config.get("fluxsite").get("meorg_model_output_id", False) num_threads = MEORG_CLIENT["num_threads"] - - # Only run if upload is enabled and a model output id is configured - if config.get("meorg_upload", True) == False: - logger.debug("meorg_upload is disabled") - return False # Check if a model output id has been assigned if model_output_id == False: - logger.error("meorg_upload is set to True, but no meorg_model_output_id key found.") - logger.error("NOT uploading to modelevaluation.org") + logger.info("No model_output_id found in fluxsite configuration.") + logger.info("NOT uploading to modelevaluation.org") return False - # Allow the user to specify an absolute path to the meorg bin + # Allow the user to specify an absolute path to the meorg bin in config meorg_bin = config.get("meorg_bin", False) # Otherwise infer the path from the benchcab installation if meorg_bin == False: + logger.debug(f"Inferring meorg bin from {benchcab_bin}") bin_segments = benchcab_bin.split("/") bin_segments[-1] = "meorg" meorg_bin = "/".join(bin_segments) + + logger.debug(f"meorg_bin = {meorg_bin}") - # Now check if that actually exists + # Now, check if that actually exists if os.path.isfile(meorg_bin) == False: logger.error(f"No meorg_client executable found at {meorg_bin}") logger.error("NOT uploading to modelevaluation.org") @@ -57,7 +56,7 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str): # Also only run if the client is initialised if MeorgClient().is_initialised() == False: - logger.warn("meorg_upload is set to True, but the client is not initialised.") + logger.warn("A model_output_id has been supplied, but the meorg_client is not initialised.") logger.warn("To initialise, run `meorg initialise` in the installation environment.") logger.warn("Once initialised, the outputs from this run can be uploaded with the following command:") logger.warn(f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {model_output_id}") @@ -71,18 +70,20 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str): logger.info("Uploading outputs to modelevaluation.org") # Submit the outputs - client = PBSClient() + client = get_client() meorg_jobid = client.submit( bu.get_installed_root() / "data" / "meorg_jobscript.j2", render=True, dry_run=False, + depends_on=benchcab_job_id, + # Interpolate into the job script model_output_id=model_output_id, data_dir=upload_dir, cache_delay=MEORG_CLIENT["cache_delay"], mem=MEORG_CLIENT["mem"], - + num_threads=MEORG_CLIENT["num_threads"], walltime=MEORG_CLIENT["walltime"], storage=MEORG_CLIENT['storage'], project=config['project'], From 2534bc414468cadd24f5612129f632b93de7ab9e Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Fri, 4 Oct 2024 16:21:25 +1000 Subject: [PATCH 05/13] Updated meta.yaml --- .conda/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 3b42d15..abc97f5 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -29,3 +29,5 @@ requirements: - cerberus >=1.3.5 - gitpython - jinja2 + - hpcpy>=0.3.0 + - meorg_client From 0508cdbb799e194ec4060b23d824ee17e5db847d Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Sun, 6 Oct 2024 22:23:31 +1100 Subject: [PATCH 06/13] Update benchcab-dev.yaml Updated hpcpy version. --- .conda/benchcab-dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.conda/benchcab-dev.yaml b/.conda/benchcab-dev.yaml index ddf3047..8cbae41 100644 --- a/.conda/benchcab-dev.yaml +++ b/.conda/benchcab-dev.yaml @@ -16,7 +16,7 @@ dependencies: - cerberus>=1.3.5 - gitpython - jinja2 - - hpcpy>=0.3.0 + - hpcpy>=0.5.0 - meorg_client # CI - pytest-cov @@ -26,4 +26,4 @@ dependencies: - black - ruff - pip: - - -r mkdocs-requirements.txt \ No newline at end of file + - -r mkdocs-requirements.txt From 88056b06fdb42c56bfdbd2bc9b2609d89e6114f4 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Wed, 9 Oct 2024 11:54:47 +1100 Subject: [PATCH 07/13] Review changes, black/ruff formats. Updated tests. Fixes #300 --- .conda/meta.yaml | 2 +- docs/user_guide/config_options.md | 23 +- src/benchcab/benchcab.py | 17 +- src/benchcab/config.py | 3 + src/benchcab/data/config-schema.yml | 14 +- src/benchcab/data/test/config-optional.yml | 1 + src/benchcab/data/test/integration_meorg.sh | 1 + src/benchcab/internal.py | 10 +- src/benchcab/utils/__init__.py | 7 +- src/benchcab/utils/meorg.py | 53 ++-- tests/test_comparison.py | 1 + tests/test_config.py | 2 + tests/test_fluxsite.py | 1 + tests/test_fs.py | 1 + tests/test_spatial.py | 1 + tests/test_state.py | 5 +- tests/test_utils.py | 6 +- versioneer.py | 315 ++++++++++++-------- 18 files changed, 285 insertions(+), 178 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index abc97f5..529ba01 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -29,5 +29,5 @@ requirements: - cerberus >=1.3.5 - gitpython - jinja2 - - hpcpy>=0.3.0 + - hpcpy>=0.5.0 - meorg_client diff --git a/docs/user_guide/config_options.md b/docs/user_guide/config_options.md index fdc76b5..b8d7515 100644 --- a/docs/user_guide/config_options.md +++ b/docs/user_guide/config_options.md @@ -68,6 +68,7 @@ fluxsite: walltime: 06:00:00 storage: [scratch/a00, gdata/xy11] multiprocess: True + meorg_model_output_id: XXXXXXXX ``` ### [experiment](#experiment) @@ -154,7 +155,7 @@ fluxsite: ### [multiprocess](#multiprocess) -: **Default:** True, _optional key_. :octicons-dash-24: Enables or disables multiprocessing for executing embarrassingly parallel tasks. + ```yaml @@ -163,6 +164,14 @@ fluxsites: ``` +### [meorg_model_output_id](#meorg_model_output_id) + +: **Default:** False, _optional key_. :octicons-dash-24: The unique Model Output ID from modelevaluation.org to which output files will be automatically uploaded for analysis. + +A separate upload job will be submitted at the successful completion of benchcab tasks if this key is present, however, the validity is not checked by benchcab at this stage. + +Note: It is the user's responsbility to ensure the model output is configured on modelevaluation.org. + ## spatial Contains settings specific to spatial tests. @@ -493,4 +502,14 @@ codecov: [f90nml-github]: https://github.com/marshallward/f90nml [environment-modules]: https://modules.sourceforge.net/ [nci-pbs-directives]: https://opus.nci.org.au/display/Help/PBS+Directives+Explained -[cable-github]: https://github.com/CABLE-LSM/CABLE \ No newline at end of file +[cable-github]: https://github.com/CABLE-LSM/CABLE + +## meorg_bin + +: **Default:** False, _optional key. :octicons-dash-24: Specifies the absolute system path to the ME.org client executable. In the absence of this key it will be inferred from the same directory as benchcab should `meorg_model_output_id` be set in `fluxsite` above. + +``` yaml + +meorg_bin: /path/to/meorg + +``` \ No newline at end of file diff --git a/src/benchcab/benchcab.py b/src/benchcab/benchcab.py index 4eec1af..c4d1b9b 100644 --- a/src/benchcab/benchcab.py +++ b/src/benchcab/benchcab.py @@ -11,6 +11,7 @@ from subprocess import CalledProcessError from typing import Optional +import benchcab.utils.meorg as bm from benchcab import fluxsite, internal, spatial from benchcab.comparison import run_comparisons, run_comparisons_in_parallel from benchcab.config import read_config @@ -24,7 +25,6 @@ from benchcab.model import Model from benchcab.utils import is_verbose, task_summary from benchcab.utils.fs import mkdir, next_path -import benchcab.utils.meorg as bm from benchcab.utils.pbs import render_job_script from benchcab.utils.repo import create_repo from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface @@ -245,15 +245,14 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None: logger.info(f"{internal.FLUXSITE_DIRS['TASKS']}//out.txt") logger.info("The NetCDF output for each task is written to:") logger.info(f"{internal.FLUXSITE_DIRS['OUTPUT']}/_out.nc") - + # Upload to meorg by default bm.do_meorg( config, - upload_dir=internal.FLUXSITE_DIRS['OUTPUT'], + upload_dir=internal.FLUXSITE_DIRS["OUTPUT"], benchcab_bin=str(self.benchcab_exe_path), - benchcab_job_id=job_id + benchcab_job_id=job_id, ) - def gen_codecov(self, config_path: str): """Endpoint for `benchcab codecov`.""" @@ -360,7 +359,7 @@ def fluxsite_run_tasks(self, config_path: str): else: fluxsite.run_tasks(tasks) - n_tasks, n_success, n_failed, all_complete = task_summary(tasks) + _, n_success, n_failed, _ = task_summary(tasks) logger.info(f"{n_failed} failed, {n_success} passed") def fluxsite_bitwise_cmp(self, config_path: str): @@ -386,9 +385,9 @@ def fluxsite_bitwise_cmp(self, config_path: str): ncpus = config["fluxsite"]["pbs"]["ncpus"] run_comparisons_in_parallel(comparisons, n_processes=ncpus) else: - run_comparisons(comparisons) - - n_tasks, n_success, n_failed, all_complete = task_summary(comparisons) + run_comparisons(comparisons) + + _, n_success, n_failed, _ = task_summary(comparisons) logger.info(f"{n_failed} failed, {n_success} passed") def fluxsite(self, config_path: str, no_submit: bool, skip: list[str]): diff --git a/src/benchcab/config.py b/src/benchcab/config.py index 102a9bb..14766e1 100644 --- a/src/benchcab/config.py +++ b/src/benchcab/config.py @@ -119,6 +119,9 @@ def read_optional_key(config: dict): config["fluxsite"]["pbs"] = internal.FLUXSITE_DEFAULT_PBS | config["fluxsite"].get( "pbs", {} ) + config["fluxsite"]["meorg_model_output_id"] = config["fluxsite"].get( + "meorg_model_output_id", internal.FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID + ) config["codecov"] = config.get("codecov", False) diff --git a/src/benchcab/data/config-schema.yml b/src/benchcab/data/config-schema.yml index ee83fdf..654d1a5 100644 --- a/src/benchcab/data/config-schema.yml +++ b/src/benchcab/data/config-schema.yml @@ -108,8 +108,11 @@ fluxsite: type: "string" required: false meorg_model_output_id: - type: "string" + type: + - "boolean" + - "string" required: false + default: false spatial: type: "dict" @@ -137,4 +140,11 @@ spatial: codecov: type: "boolean" - required: false \ No newline at end of file + required: false + +meorg_bin: + type: + - "boolean" + - "string" + required: False + default: False \ No newline at end of file diff --git a/src/benchcab/data/test/config-optional.yml b/src/benchcab/data/test/config-optional.yml index f4605e8..e36c436 100644 --- a/src/benchcab/data/test/config-optional.yml +++ b/src/benchcab/data/test/config-optional.yml @@ -3,6 +3,7 @@ project: hh5 fluxsite: experiment: AU-Tum + meorg_model_output_id: False multiprocess: False pbs: ncpus: 6 diff --git a/src/benchcab/data/test/integration_meorg.sh b/src/benchcab/data/test/integration_meorg.sh index 26b1d1e..e90c531 100644 --- a/src/benchcab/data/test/integration_meorg.sh +++ b/src/benchcab/data/test/integration_meorg.sh @@ -46,6 +46,7 @@ fluxsite: storage: - scratch/$PROJECT - gdata/$PROJECT + # This ID is currently configured on the me.org server. meorg_model_output_id: Sss7qupAHEZ8ovbCv EOL diff --git a/src/benchcab/internal.py b/src/benchcab/internal.py index be56982..66af72d 100644 --- a/src/benchcab/internal.py +++ b/src/benchcab/internal.py @@ -252,6 +252,7 @@ } FLUXSITE_DEFAULT_EXPERIMENT = "forty-two-site-test" +FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID = False OPTIONAL_COMMANDS = ["fluxsite-bitwise-cmp", "gen_codecov"] @@ -275,11 +276,12 @@ def get_met_forcing_file_names(experiment: str) -> list[str]: return file_names + # Configuration for the client upload MEORG_CLIENT = dict( - num_threads=1, # Parallel uploads over 4 cores - cache_delay=60*5, # 5mins between upload and analysis triggering + num_threads=1, # Parallel uploads over 4 cores + cache_delay=60 * 5, # 5mins between upload and analysis triggering mem="8G", walltime="01:00:00", - storage=["gdata/ks32", "gdata/hh5", "gdata/wd9", "gdata/rp23"] -) \ No newline at end of file + storage=["gdata/ks32", "gdata/hh5", "gdata/wd9", "gdata/rp23"], +) diff --git a/src/benchcab/utils/__init__.py b/src/benchcab/utils/__init__.py index b652c4f..b628e6e 100644 --- a/src/benchcab/utils/__init__.py +++ b/src/benchcab/utils/__init__.py @@ -11,7 +11,7 @@ import sys from importlib import resources from pathlib import Path -from typing import Union, Iterable +from typing import Iterable, Union import yaml from jinja2 import BaseLoader, Environment @@ -162,9 +162,10 @@ def task_summary(tasks: Iterable) -> tuple: ------- tuple num_tasks, num_complete, num_failed, all_complete + """ num_tasks = len(tasks) num_complete = len([task for task in tasks if task.is_done()]) num_failed = num_tasks - num_complete - - return num_tasks, num_complete, num_failed, num_complete == num_tasks \ No newline at end of file + + return num_tasks, num_complete, num_failed, num_complete == num_tasks diff --git a/src/benchcab/utils/meorg.py b/src/benchcab/utils/meorg.py index 9359ccb..9ee35df 100644 --- a/src/benchcab/utils/meorg.py +++ b/src/benchcab/utils/meorg.py @@ -1,10 +1,13 @@ """Utility methods for interacting with the ME.org client.""" -from benchcab.internal import MEORG_CLIENT -from meorg_client.client import Client as MeorgClient + +import os + from hpcpy import get_client +from meorg_client.client import Client as MeorgClient + import benchcab.utils as bu -import os -from glob import glob +from benchcab.internal import MEORG_CLIENT + def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: str): """Perform the upload of model outputs to modelevaluation.org @@ -22,29 +25,29 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: ------- bool True if successful, False otherwise - """ + """ logger = bu.get_logger() - model_output_id = config.get("fluxsite").get("meorg_model_output_id", False) + model_output_id = config["fluxsite"]["meorg_model_output_id"] num_threads = MEORG_CLIENT["num_threads"] - + # Check if a model output id has been assigned if model_output_id == False: logger.info("No model_output_id found in fluxsite configuration.") logger.info("NOT uploading to modelevaluation.org") return False - + # Allow the user to specify an absolute path to the meorg bin in config meorg_bin = config.get("meorg_bin", False) - + # Otherwise infer the path from the benchcab installation if meorg_bin == False: logger.debug(f"Inferring meorg bin from {benchcab_bin}") bin_segments = benchcab_bin.split("/") bin_segments[-1] = "meorg" meorg_bin = "/".join(bin_segments) - + logger.debug(f"meorg_bin = {meorg_bin}") # Now, check if that actually exists @@ -56,28 +59,34 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: # Also only run if the client is initialised if MeorgClient().is_initialised() == False: - logger.warn("A model_output_id has been supplied, but the meorg_client is not initialised.") - logger.warn("To initialise, run `meorg initialise` in the installation environment.") - logger.warn("Once initialised, the outputs from this run can be uploaded with the following command:") - logger.warn(f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {model_output_id}") + logger.warn( + "A model_output_id has been supplied, but the meorg_client is not initialised." + ) + logger.warn( + "To initialise, run `meorg initialise` in the installation environment." + ) + logger.warn( + "Once initialised, the outputs from this run can be uploaded with the following command:" + ) + logger.warn( + f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {model_output_id}" + ) logger.warn("Then the analysis can be triggered with:") logger.warn(f"meorg analysis start {model_output_id}") return False # Finally, attempt the upload! else: - + logger.info("Uploading outputs to modelevaluation.org") # Submit the outputs client = get_client() meorg_jobid = client.submit( - bu.get_installed_root() / "data" / "meorg_jobscript.j2", render=True, dry_run=False, depends_on=benchcab_job_id, - # Interpolate into the job script model_output_id=model_output_id, data_dir=upload_dir, @@ -85,12 +94,12 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: mem=MEORG_CLIENT["mem"], num_threads=MEORG_CLIENT["num_threads"], walltime=MEORG_CLIENT["walltime"], - storage=MEORG_CLIENT['storage'], - project=config['project'], - modules=config['modules'], + storage=MEORG_CLIENT["storage"], + project=config["project"], + modules=config["modules"], purge_outputs=True, - meorg_bin=meorg_bin + meorg_bin=meorg_bin, ) logger.info(f"Upload job submitted: {meorg_jobid}") - return True \ No newline at end of file + return True diff --git a/tests/test_comparison.py b/tests/test_comparison.py index cec5125..568b1c2 100644 --- a/tests/test_comparison.py +++ b/tests/test_comparison.py @@ -8,6 +8,7 @@ from pathlib import Path import pytest + from benchcab import internal from benchcab.comparison import ComparisonTask diff --git a/tests/test_config.py b/tests/test_config.py index 5f741be..e995d99 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -75,6 +75,7 @@ def all_optional_default_config(no_optional_config) -> dict: "experiment": bi.FLUXSITE_DEFAULT_EXPERIMENT, "multiprocess": bi.FLUXSITE_DEFAULT_MULTIPROCESS, "pbs": bi.FLUXSITE_DEFAULT_PBS, + "meorg_model_output_id": bi.FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID }, "science_configurations": bi.DEFAULT_SCIENCE_CONFIGURATIONS, "spatial": { @@ -106,6 +107,7 @@ def all_optional_custom_config(no_optional_config) -> dict: "walltime": "10:00:00", "storage": ["scratch/$PROJECT"], }, + "meorg_model_output_id": False }, "science_configurations": [ { diff --git a/tests/test_fluxsite.py b/tests/test_fluxsite.py index e4732ba..a7558d5 100644 --- a/tests/test_fluxsite.py +++ b/tests/test_fluxsite.py @@ -10,6 +10,7 @@ import f90nml import netCDF4 import pytest + from benchcab import __version__, internal from benchcab.fluxsite import ( CableError, diff --git a/tests/test_fs.py b/tests/test_fs.py index 23d95d0..3101699 100644 --- a/tests/test_fs.py +++ b/tests/test_fs.py @@ -10,6 +10,7 @@ from pathlib import Path import pytest + from benchcab.utils.fs import chdir, mkdir, next_path, prepend_path diff --git a/tests/test_spatial.py b/tests/test_spatial.py index aed4283..7a0d950 100644 --- a/tests/test_spatial.py +++ b/tests/test_spatial.py @@ -10,6 +10,7 @@ import f90nml import pytest import yaml + from benchcab import internal from benchcab.model import Model from benchcab.spatial import SpatialTask, get_spatial_tasks diff --git a/tests/test_state.py b/tests/test_state.py index 7c77f33..cda2e73 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1,9 +1,10 @@ import time from pathlib import Path +from tempfile import TemporaryDirectory import pytest + from benchcab.utils.state import State, StateAttributeError -from tempfile import TemporaryDirectory def test_state_is_set(): @@ -39,4 +40,4 @@ def test_state_get_raises_exception(): with TemporaryDirectory() as tmp_dir: state = State(state_dir=Path(tmp_dir)) with pytest.raises(StateAttributeError): - state.get() \ No newline at end of file + state.get() diff --git a/tests/test_utils.py b/tests/test_utils.py index 37284ba..d6ce264 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -64,7 +64,7 @@ def test_get_logger_singleton_fail(): def test_task_summary(): - + # Create some mocked tasks t1 = ComparisonTask(files=(), task_name="t1") t2 = ComparisonTask(files=(), task_name="t2") @@ -72,7 +72,7 @@ def test_task_summary(): # Inject success/fail cases t1.is_done = lambda: True t2.is_done = lambda: False - + # Run the function n_tasks, n_success, n_failed, all_complete = bu.task_summary([t1, t2]) @@ -80,4 +80,4 @@ def test_task_summary(): assert n_tasks == 2 assert n_success == 1 assert n_failed == 1 - assert all_complete == False \ No newline at end of file + assert all_complete == False diff --git a/versioneer.py b/versioneer.py index 8851eaa..0ae83db 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,4 +1,3 @@ - # Version: 0.29 """The Versioneer - like a rocketeer, but for versions. @@ -310,15 +309,14 @@ import configparser import errno +import functools import json import os import re import subprocess import sys from pathlib import Path -from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union -from typing import NoReturn -import functools +from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, Union, cast have_tomllib = True if sys.version_info >= (3, 11): @@ -367,11 +365,13 @@ def get_root() -> str: or os.path.exists(pyproject_toml) or os.path.exists(versioneer_py) ): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -384,8 +384,10 @@ def get_root() -> str: me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(my_path), versioneer_py) + ) except NameError: pass return root @@ -403,9 +405,9 @@ def get_config_from_root(root: str) -> VersioneerConfig: section: Union[Dict[str, Any], configparser.SectionProxy, None] = None if pyproject_toml.exists() and have_tomllib: try: - with open(pyproject_toml, 'rb') as fobj: + with open(pyproject_toml, "rb") as fobj: pp = tomllib.load(fobj) - section = pp['tool']['versioneer'] + section = pp["tool"]["versioneer"] except (tomllib.TOMLDecodeError, KeyError) as e: print(f"Failed to load config from {pyproject_toml}: {e}") print("Try to load it from setup.cfg") @@ -422,7 +424,7 @@ def get_config_from_root(root: str) -> VersioneerConfig: # `None` values elsewhere where it matters cfg = VersioneerConfig() - cfg.VCS = section['VCS'] + cfg.VCS = section["VCS"] cfg.style = section.get("style", "") cfg.versionfile_source = cast(str, section.get("versionfile_source")) cfg.versionfile_build = section.get("versionfile_build") @@ -450,10 +452,12 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f + return decorate @@ -480,10 +484,14 @@ def run_command( try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) + process = subprocess.Popen( + [command] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + **popen_kwargs, + ) break except OSError as e: if e.errno == errno.ENOENT: @@ -505,7 +513,9 @@ def run_command( return stdout, process.returncode -LONG_VERSION_PY['git'] = r''' +LONG_VERSION_PY[ + "git" +] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -1250,7 +1260,7 @@ def git_versions_from_keywords( # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1259,7 +1269,7 @@ def git_versions_from_keywords( # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1267,32 +1277,36 @@ def git_versions_from_keywords( for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') - if not re.match(r'\d', r): + if not re.match(r"\d", r): continue if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( - tag_prefix: str, - root: str, - verbose: bool, - runner: Callable = run_command + tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. @@ -1311,8 +1325,7 @@ def git_pieces_from_vcs( env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1320,10 +1333,19 @@ def git_pieces_from_vcs( # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) + describe_out, rc = runner( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + f"{tag_prefix}[[:digit:]]*", + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1338,8 +1360,7 @@ def git_pieces_from_vcs( pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") @@ -1379,17 +1400,16 @@ def git_pieces_from_vcs( dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -1398,10 +1418,12 @@ def git_pieces_from_vcs( if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1479,15 +1501,21 @@ def versions_from_parentdir( for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1516,11 +1544,13 @@ def versions_from_file(filename: str) -> Dict[str, Any]: contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1528,8 +1558,7 @@ def versions_from_file(filename: str) -> Dict[str, Any]: def write_to_version_file(filename: str, versions: Dict[str, Any]) -> None: """Write the given version number to the given _version.py file.""" - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1561,8 +1590,7 @@ def render_pep440(pieces: Dict[str, Any]) -> str: rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1591,8 +1619,7 @@ def render_pep440_branch(pieces: Dict[str, Any]) -> str: rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1753,11 +1780,13 @@ def render_git_describe_long(pieces: Dict[str, Any]) -> str: def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1781,9 +1810,13 @@ def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1806,8 +1839,9 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or bool(cfg.verbose) # `bool()` used to avoid `None` - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1861,9 +1895,13 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version() -> str: @@ -1916,6 +1954,7 @@ def run(self) -> None: print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in setuptools @@ -1937,8 +1976,8 @@ def run(self) -> None: # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments - if 'build_py' in cmds: - _build_py: Any = cmds['build_py'] + if "build_py" in cmds: + _build_py: Any = cmds["build_py"] else: from setuptools.command.build_py import build_py as _build_py @@ -1955,14 +1994,14 @@ def run(self) -> None: # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py - if 'build_ext' in cmds: - _build_ext: Any = cmds['build_ext'] + if "build_ext" in cmds: + _build_ext: Any = cmds["build_ext"] else: from setuptools.command.build_ext import build_ext as _build_ext @@ -1982,19 +2021,22 @@ def run(self) -> None: # it with an updated value if not cfg.versionfile_build: return - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) if not os.path.exists(target_versionfile): - print(f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running build_ext " - "without first running build_py.") + print( + f"Warning: {target_versionfile} does not exist, skipping " + "version update. This can happen if you are running build_ext " + "without first running build_py." + ) return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # type: ignore + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -2015,17 +2057,21 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.setuptools_buildexe import py2exe as _py2exe # type: ignore except ImportError: @@ -2044,18 +2090,22 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info - if 'egg_info' in cmds: - _egg_info: Any = cmds['egg_info'] + if "egg_info" in cmds: + _egg_info: Any = cmds["egg_info"] else: from setuptools.command.egg_info import egg_info as _egg_info @@ -2068,7 +2118,7 @@ def find_sources(self) -> None: # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) - self.filelist.append('versioneer.py') + self.filelist.append("versioneer.py") if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit @@ -2081,18 +2131,21 @@ def find_sources(self) -> None: # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils - normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') - for f in self.filelist.files] - manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') - with open(manifest_filename, 'w') as fobj: - fobj.write('\n'.join(normalized)) + normalized = [ + unicode_utils.filesys_decode(f).replace(os.sep, "/") + for f in self.filelist.files + ] - cmds['egg_info'] = cmd_egg_info + manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") + with open(manifest_filename, "w") as fobj: + fobj.write("\n".join(normalized)) + + cmds["egg_info"] = cmd_egg_info # we override different "sdist" commands for both environments - if 'sdist' in cmds: - _sdist: Any = cmds['sdist'] + if "sdist" in cmds: + _sdist: Any = cmds["sdist"] else: from setuptools.command.sdist import sdist as _sdist @@ -2114,8 +2167,10 @@ def make_release_tree(self, base_dir: str, files: List[str]) -> None: # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -2175,11 +2230,9 @@ def do_setup() -> int: root = get_root() try: cfg = get_config_from_root(root) - except (OSError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -2188,15 +2241,18 @@ def do_setup() -> int: print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") maybe_ipy: Optional[str] = ipy if os.path.exists(ipy): try: @@ -2275,4 +2331,3 @@ def setup_command() -> NoReturn: cmd = sys.argv[1] if cmd == "setup": setup_command() - From f6197587023fdc3476469da83616d93a4bd5485b Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Wed, 9 Oct 2024 12:02:23 +1100 Subject: [PATCH 08/13] Update config_options.md Copy/paste error. --- docs/user_guide/config_options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/config_options.md b/docs/user_guide/config_options.md index b8d7515..b5803fe 100644 --- a/docs/user_guide/config_options.md +++ b/docs/user_guide/config_options.md @@ -155,7 +155,7 @@ fluxsite: ### [multiprocess](#multiprocess) - +: **Default:** True, _optional key_. :octicons-dash-24: Enables or disables multiprocessing for executing embarrassingly parallel tasks. ```yaml @@ -512,4 +512,4 @@ codecov: meorg_bin: /path/to/meorg -``` \ No newline at end of file +``` From 7face51735d35f300297aa25d1fbc8cae39435fe Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Thu, 10 Oct 2024 09:27:30 +1100 Subject: [PATCH 09/13] Pin python version to <3.13 to avoid setuptools issue but get higher version. Fixes #300 --- .conda/benchcab-dev.yaml | 2 +- .conda/meta.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.conda/benchcab-dev.yaml b/.conda/benchcab-dev.yaml index 8cbae41..21bb6c7 100644 --- a/.conda/benchcab-dev.yaml +++ b/.conda/benchcab-dev.yaml @@ -6,7 +6,7 @@ channels: - defaults dependencies: - - python=3.9 + - python<3.13 - payu>=1.0.30 - pip - f90nml diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 529ba01..1da97fb 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -17,10 +17,10 @@ build: requirements: host: - - python >=3.9 + - python <3.13 - pip run: - - python >=3.9 + - python <3.13 - payu >=1.0.30 - netCDF4 - PyYAML From 63caf104f6ee613d903b96f34c963f981923c269 Mon Sep 17 00:00:00 2001 From: Sean Bryan <39685865+SeanBryan51@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:26:23 +1100 Subject: [PATCH 10/13] Update .conda/benchcab-dev.yaml --- .conda/benchcab-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.conda/benchcab-dev.yaml b/.conda/benchcab-dev.yaml index 21bb6c7..3e97206 100644 --- a/.conda/benchcab-dev.yaml +++ b/.conda/benchcab-dev.yaml @@ -6,7 +6,7 @@ channels: - defaults dependencies: - - python<3.13 + - python=3.13 - payu>=1.0.30 - pip - f90nml From fdf9a047bc3cfab520acb888e2d58a25f4d1c1ae Mon Sep 17 00:00:00 2001 From: Sean Bryan <39685865+SeanBryan51@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:26:35 +1100 Subject: [PATCH 11/13] Update .conda/meta.yaml --- .conda/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 1da97fb..cb27338 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -17,7 +17,7 @@ build: requirements: host: - - python <3.13 + - python >=3.9,<3.13 - pip run: - python <3.13 From f7dafd09efe53095134a2738a5dcda2948c8cf08 Mon Sep 17 00:00:00 2001 From: Sean Bryan <39685865+SeanBryan51@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:26:42 +1100 Subject: [PATCH 12/13] Update .conda/meta.yaml --- .conda/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index cb27338..6944735 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -20,7 +20,7 @@ requirements: - python >=3.9,<3.13 - pip run: - - python <3.13 + - python >=3.9,<3.13 - payu >=1.0.30 - netCDF4 - PyYAML From 8e0d58f0dc04486cb3665bf98403da19fdd7c8a8 Mon Sep 17 00:00:00 2001 From: Sean Bryan Date: Thu, 10 Oct 2024 14:29:15 +1100 Subject: [PATCH 13/13] Restore versioneer.py --- versioneer.py | 315 +++++++++++++++++++++----------------------------- 1 file changed, 130 insertions(+), 185 deletions(-) diff --git a/versioneer.py b/versioneer.py index 0ae83db..8851eaa 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,3 +1,4 @@ + # Version: 0.29 """The Versioneer - like a rocketeer, but for versions. @@ -309,14 +310,15 @@ import configparser import errno -import functools import json import os import re import subprocess import sys from pathlib import Path -from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, Union, cast +from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union +from typing import NoReturn +import functools have_tomllib = True if sys.version_info >= (3, 11): @@ -365,13 +367,11 @@ def get_root() -> str: or os.path.exists(pyproject_toml) or os.path.exists(versioneer_py) ): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -384,10 +384,8 @@ def get_root() -> str: me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py) - ) + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root @@ -405,9 +403,9 @@ def get_config_from_root(root: str) -> VersioneerConfig: section: Union[Dict[str, Any], configparser.SectionProxy, None] = None if pyproject_toml.exists() and have_tomllib: try: - with open(pyproject_toml, "rb") as fobj: + with open(pyproject_toml, 'rb') as fobj: pp = tomllib.load(fobj) - section = pp["tool"]["versioneer"] + section = pp['tool']['versioneer'] except (tomllib.TOMLDecodeError, KeyError) as e: print(f"Failed to load config from {pyproject_toml}: {e}") print("Try to load it from setup.cfg") @@ -424,7 +422,7 @@ def get_config_from_root(root: str) -> VersioneerConfig: # `None` values elsewhere where it matters cfg = VersioneerConfig() - cfg.VCS = section["VCS"] + cfg.VCS = section['VCS'] cfg.style = section.get("style", "") cfg.versionfile_source = cast(str, section.get("versionfile_source")) cfg.versionfile_build = section.get("versionfile_build") @@ -452,12 +450,10 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f - return decorate @@ -484,14 +480,10 @@ def run_command( try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen( - [command] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - **popen_kwargs, - ) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) break except OSError as e: if e.errno == errno.ENOENT: @@ -513,9 +505,7 @@ def run_command( return stdout, process.returncode -LONG_VERSION_PY[ - "git" -] = r''' +LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -1260,7 +1250,7 @@ def git_versions_from_keywords( # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1269,7 +1259,7 @@ def git_versions_from_keywords( # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1277,36 +1267,32 @@ def git_versions_from_keywords( for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] + r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') - if not re.match(r"\d", r): + if not re.match(r'\d', r): continue if verbose: print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( - tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command + tag_prefix: str, + root: str, + verbose: bool, + runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. @@ -1325,7 +1311,8 @@ def git_pieces_from_vcs( env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1333,19 +1320,10 @@ def git_pieces_from_vcs( # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - f"{tag_prefix}[[:digit:]]*", - ], - cwd=root, - ) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1360,7 +1338,8 @@ def git_pieces_from_vcs( pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") @@ -1400,16 +1379,17 @@ def git_pieces_from_vcs( dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] + git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) return pieces # tag @@ -1418,12 +1398,10 @@ def git_pieces_from_vcs( if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] + pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1501,21 +1479,15 @@ def versions_from_parentdir( for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1544,13 +1516,11 @@ def versions_from_file(filename: str) -> Dict[str, Any]: contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) + mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1558,7 +1528,8 @@ def versions_from_file(filename: str) -> Dict[str, Any]: def write_to_version_file(filename: str, versions: Dict[str, Any]) -> None: """Write the given version number to the given _version.py file.""" - contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1590,7 +1561,8 @@ def render_pep440(pieces: Dict[str, Any]) -> str: rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1619,7 +1591,8 @@ def render_pep440_branch(pieces: Dict[str, Any]) -> str: rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1780,13 +1753,11 @@ def render_git_describe_long(pieces: Dict[str, Any]) -> str: def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default @@ -1810,13 +1781,9 @@ def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: else: raise ValueError("unknown style '%s'" % style) - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} class VersioneerBadRootError(Exception): @@ -1839,9 +1806,8 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or bool(cfg.verbose) # `bool()` used to avoid `None` - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1895,13 +1861,9 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: if verbose: print("unable to compute version") - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version", + "date": None} def get_version() -> str: @@ -1954,7 +1916,6 @@ def run(self) -> None: print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) - cmds["version"] = cmd_version # we override "build_py" in setuptools @@ -1976,8 +1937,8 @@ def run(self) -> None: # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments - if "build_py" in cmds: - _build_py: Any = cmds["build_py"] + if 'build_py' in cmds: + _build_py: Any = cmds['build_py'] else: from setuptools.command.build_py import build_py as _build_py @@ -1994,14 +1955,14 @@ def run(self) -> None: # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) - cmds["build_py"] = cmd_build_py - if "build_ext" in cmds: - _build_ext: Any = cmds["build_ext"] + if 'build_ext' in cmds: + _build_ext: Any = cmds['build_ext'] else: from setuptools.command.build_ext import build_ext as _build_ext @@ -2021,22 +1982,19 @@ def run(self) -> None: # it with an updated value if not cfg.versionfile_build: return - target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) if not os.path.exists(target_versionfile): - print( - f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running build_ext " - "without first running build_py." - ) + print(f"Warning: {target_versionfile} does not exist, skipping " + "version update. This can happen if you are running build_ext " + "without first running build_py.") return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) - cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # type: ignore - # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -2057,21 +2015,17 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if "py2exe" in sys.modules: # py2exe enabled? + if 'py2exe' in sys.modules: # py2exe enabled? try: from py2exe.setuptools_buildexe import py2exe as _py2exe # type: ignore except ImportError: @@ -2090,22 +2044,18 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info - if "egg_info" in cmds: - _egg_info: Any = cmds["egg_info"] + if 'egg_info' in cmds: + _egg_info: Any = cmds['egg_info'] else: from setuptools.command.egg_info import egg_info as _egg_info @@ -2118,7 +2068,7 @@ def find_sources(self) -> None: # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) - self.filelist.append("versioneer.py") + self.filelist.append('versioneer.py') if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit @@ -2131,21 +2081,18 @@ def find_sources(self) -> None: # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils + normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') + for f in self.filelist.files] - normalized = [ - unicode_utils.filesys_decode(f).replace(os.sep, "/") - for f in self.filelist.files - ] + manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') + with open(manifest_filename, 'w') as fobj: + fobj.write('\n'.join(normalized)) - manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") - with open(manifest_filename, "w") as fobj: - fobj.write("\n".join(normalized)) - - cmds["egg_info"] = cmd_egg_info + cmds['egg_info'] = cmd_egg_info # we override different "sdist" commands for both environments - if "sdist" in cmds: - _sdist: Any = cmds["sdist"] + if 'sdist' in cmds: + _sdist: Any = cmds['sdist'] else: from setuptools.command.sdist import sdist as _sdist @@ -2167,10 +2114,8 @@ def make_release_tree(self, base_dir: str, files: List[str]) -> None: # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) cmds["sdist"] = cmd_sdist return cmds @@ -2230,9 +2175,11 @@ def do_setup() -> int: root = get_root() try: cfg = get_config_from_root(root) - except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: + except (OSError, configparser.NoSectionError, + configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", file=sys.stderr) + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -2241,18 +2188,15 @@ def do_setup() -> int: print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") maybe_ipy: Optional[str] = ipy if os.path.exists(ipy): try: @@ -2331,3 +2275,4 @@ def setup_command() -> NoReturn: cmd = sys.argv[1] if cmd == "setup": setup_command() +