From fe832c143ac7ee418b42013ac57f864ee4910190 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:44:10 -0700 Subject: [PATCH 1/6] Omnibus 2024-11-27 (#663) --- .github/workflows/test.yaml | 4 +- .../cli/tools/config/realize-verbose.out | 56 +++++++------- .../cli/tools/config/validate-verbose.out | 42 +++++----- docs/sections/user_guide/yaml/tags.rst | 77 +++++++++++++------ recipe/meta.json | 3 +- recipe/meta.yaml | 1 + src/uwtools/config/formats/yaml.py | 12 +-- src/uwtools/config/jinja2.py | 24 +++--- src/uwtools/config/support.py | 11 +++ src/uwtools/tests/config/test_jinja2.py | 11 ++- src/uwtools/tests/config/test_tools.py | 62 +++++++++++++++ 11 files changed, 207 insertions(+), 96 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3351ba97d..c34875b49 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,10 +2,10 @@ name: Test on: pull_request: branches: - - main + - '**' push: branches: - - main + - '**' workflow_dispatch: branches: - '**' diff --git a/docs/sections/user_guide/cli/tools/config/realize-verbose.out b/docs/sections/user_guide/cli/tools/config/realize-verbose.out index 3371e2070..cac8e0711 100644 --- a/docs/sections/user_guide/cli/tools/config/realize-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/realize-verbose.out @@ -1,30 +1,30 @@ -[2024-05-23T19:39:16] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose -[2024-05-23T19:39:16] DEBUG Reading input from stdin -[2024-05-23T19:39:16] DEBUG Dereferencing, current value: -[2024-05-23T19:39:16] DEBUG hello: '{{ recipient }}' -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: {{ recipient }} -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: recipient -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: recipient -[2024-05-23T19:39:16] DEBUG Dereferencing, current value: -[2024-05-23T19:39:16] DEBUG hello: world -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: recipient -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: recipient -[2024-05-23T19:39:16] DEBUG Dereferencing, final value: -[2024-05-23T19:39:16] DEBUG hello: world -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG Writing output to stdout +[2024-11-27T05:24:34] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose +[2024-11-27T05:24:34] DEBUG Reading input from stdin +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG hello: '{{ recipient }}' +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: {{ recipient }} +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG hello: world +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG Dereferencing, final value: +[2024-11-27T05:24:34] DEBUG hello: world +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG Writing output to stdout hello: world recipient: world diff --git a/docs/sections/user_guide/cli/tools/config/validate-verbose.out b/docs/sections/user_guide/cli/tools/config/validate-verbose.out index 17dc2a651..d48c7ea14 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/validate-verbose.out @@ -1,21 +1,21 @@ -[2024-08-26T22:54:28] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose -[2024-08-26T22:54:28] DEBUG Using schema file: schema.jsonschema -[2024-08-26T22:54:28] DEBUG Dereferencing, current value: -[2024-08-26T22:54:28] DEBUG values: -[2024-08-26T22:54:28] DEBUG greeting: Hello -[2024-08-26T22:54:28] DEBUG recipient: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: Hello -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: Hello -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: greeting -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: greeting -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: recipient -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: recipient -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: values -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: values -[2024-08-26T22:54:28] DEBUG Dereferencing, final value: -[2024-08-26T22:54:28] DEBUG values: -[2024-08-26T22:54:28] DEBUG greeting: Hello -[2024-08-26T22:54:28] DEBUG recipient: World -[2024-08-26T22:54:29] INFO 0 UW schema-validation errors found in config +[2024-11-27T05:24:34] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose +[2024-11-27T05:24:34] DEBUG Using schema file: schema.jsonschema +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG values: +[2024-11-27T05:24:34] DEBUG greeting: Hello +[2024-11-27T05:24:34] DEBUG recipient: World +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: values +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: values +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: greeting +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: greeting +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: Hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: Hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: World +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: World +[2024-11-27T05:24:34] DEBUG Dereferencing, final value: +[2024-11-27T05:24:34] DEBUG values: +[2024-11-27T05:24:34] DEBUG greeting: Hello +[2024-11-27T05:24:34] DEBUG recipient: World +[2024-11-27T05:24:34] INFO 0 UW schema-validation errors found in config diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index 8acd99caa..e768d74e5 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -26,7 +26,7 @@ Additionally, UW defines the following tags to support use cases not covered by ``!bool`` ^^^^^^^^^ -Converts the tagged node to a Python ``boolean`` object. For example, given ``input.yaml``: +Converts the tagged node to a Python ``bool`` object. For example, given ``input.yaml``: .. code-block:: yaml @@ -35,7 +35,7 @@ Converts the tagged node to a Python ``boolean`` object. For example, given ``in .. code-block:: text - % uw config realize -i ../input.yaml --output-format yaml + $ uw config realize -i ../input.yaml --output-format yaml flag1: True flag2: True @@ -52,7 +52,7 @@ Converts the tagged node to a Python ``datetime`` object. For example, given ``i .. code-block:: text - % uw config realize -i ../input.yaml --output-format yaml + $ uw config realize -i ../input.yaml --output-format yaml date1: 2024-09-01 date2: 2024-09-01 00:00:00 @@ -69,49 +69,76 @@ Converts the tagged node to a Python ``float`` value. For example, given ``input .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml + $ uw config realize --input-file input.yaml --output-format yaml f2: 5.859 -``!int`` -^^^^^^^^ +``!include`` +^^^^^^^^^^^^ -Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``: +Load and parse the files specified in the tagged sequence value and insert their contents here. For example, given ``numbers.yaml``: .. code-block:: yaml - f1: 3 - f2: 11 - f3: !int "{{ (f1 + f2) * 10 }}" + values: !include [constants.yaml] + +and ``constants.yaml``: + +.. code-block:: yaml + + e: 2.718 + pi: 3.141 .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml - f1: 3 - f2: 11 - f2: 140 + $ uw config realize --input-file numbers.yaml --output-format yaml + values: + e: 2.718 + pi: 3.141 -``!include`` -^^^^^^^^^^^^ +Values from files later in the sequence overwrite their predecessors, and full-value replacement, not structural merging, is performed. For example, given ``numbers.yaml``: + +.. code-block:: yaml + + values: !include [e.yaml, pi.yaml] -Parse the tagged file and include its tags. For example, given ``input.yaml``: +``e.yaml``: .. code-block:: yaml - values: !include [./supplemental.yaml] + constants: + e: 2.718 -and ``supplemental.yaml``: +and ``pi.yaml``: .. code-block:: yaml - e: 2.718 - pi: 3.141 + constants: + pi: 3.141 .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml + $ uw config realize --input-file numbers.yaml --output-format yaml values: - e: 2.718 - pi: 3.141 + constants: + pi: 3.141 + +``!int`` +^^^^^^^^ + +Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``: + +.. code-block:: yaml + + f1: 3 + f2: 11 + f3: !int "{{ (f1 + f2) * 10 }}" + +.. code-block:: text + + $ uw config realize --input-file input.yaml --output-format yaml + f1: 3 + f2: 11 + f2: 140 ``!remove`` ^^^^^^^^^^^ @@ -131,5 +158,5 @@ and ``update.yaml``: .. code-block:: text - % uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml + $ uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml pi: 3.141 diff --git a/recipe/meta.json b/recipe/meta.json index 282173e37..8cfa7a58f 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -21,7 +21,8 @@ "pytest-cov =5.0.*", "pytest-xdist =3.6.*", "python >=3.9,<3.13", - "pyyaml =6.0.*" + "pyyaml =6.0.*", + "setuptools" ], "run": [ "f90nml =1.4.*", diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 479b49330..9a9dfd0d9 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -12,6 +12,7 @@ build: requirements: build: - pip + - setuptools run: - f90nml 1.4.* - iotaa 0.8.* diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index 99c5b32a6..71bc870f8 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -10,9 +10,9 @@ from uwtools.config.support import ( INCLUDE_TAG, UWYAMLConvert, - UWYAMLRemove, from_od, log_and_error, + uw_yaml_loader, yaml_to_str, ) from uwtools.exceptions import UWConfigError @@ -94,10 +94,9 @@ def _load(self, config_file: Optional[Path]) -> dict: :param config_file: Path to config file to load. """ - loader = self._yaml_loader with readable(config_file) as f: try: - config = yaml.load(f.read(), Loader=loader) + config = yaml.load(f.read(), Loader=self._yaml_loader) if isinstance(config, dict): return config t = type(config).__name__ @@ -157,13 +156,10 @@ def _yaml_include(self, loader: yaml.Loader, node: yaml.SequenceNode) -> dict: @property def _yaml_loader(self) -> type[yaml.SafeLoader]: """ - The loader, with appropriate constructors added. + A loader with all UW constructors added. """ - loader = yaml.SafeLoader + loader = uw_yaml_loader() loader.add_constructor(INCLUDE_TAG, self._yaml_include) - for tag_class in (UWYAMLConvert, UWYAMLRemove): - for tag in getattr(tag_class, "TAGS"): - loader.add_constructor(tag, tag_class) return loader # Public methods diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 8115e248b..cba03bae0 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -8,10 +8,12 @@ from pathlib import Path from typing import Optional, Union +import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined, Undefined, meta from jinja2.exceptions import UndefinedError -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config, uw_yaml_loader +from uwtools.exceptions import UWConfigRealizeError from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable @@ -122,19 +124,19 @@ def dereference( :param keys: The dict keys leading to this value. :return: The input value, with Jinja2 syntax rendered. """ - rendered: _ConfigVal = val # fall-back value + rendered: _ConfigVal if isinstance(val, dict): keys = keys or [] - new = {} + rendered = {} for k, v in val.items(): if isinstance(v, UWYAMLRemove): - _deref_debug("Removing value at", " > ".join([*keys, k])) + _deref_debug("Removing value at", ".".join([*keys, k])) else: - new[dereference(k, context)] = dereference(v, context, local=val, keys=[*keys, k]) - return new - if isinstance(val, list): - return [dereference(v, context) for v in val] - if isinstance(val, str): + kd, vd = [dereference(x, context, val, [*keys, k]) for x in (k, v)] + rendered[kd] = vd + elif isinstance(val, list): + rendered = [dereference(v, context) for v in val] + elif isinstance(val, str): _deref_debug("Rendering", val) rendered = _deref_render(val, context, local) elif isinstance(val, UWYAMLConvert): @@ -143,6 +145,7 @@ def dereference( rendered = _deref_convert(val) else: _deref_debug("Accepting", val) + rendered = val return rendered @@ -266,6 +269,9 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: context = {**(local or {}), **context} try: rendered = _register_filters(env).from_string(val).render(context) + if isinstance(yaml.load(rendered, Loader=uw_yaml_loader()), UWYAMLConvert): + _deref_debug("Held", rendered) + raise UWConfigRealizeError() _deref_debug("Rendered", rendered) except Exception as e: # pylint: disable=broad-exception-caught rendered = val diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index f6337853c..393ed1a44 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -68,6 +68,17 @@ def log_and_error(msg: str) -> Exception: return UWConfigError(msg) +def uw_yaml_loader() -> type[yaml.SafeLoader]: + """ + A loader with basic UW constructors added. + """ + loader = yaml.SafeLoader + for tag_class in (UWYAMLConvert, UWYAMLRemove): + for tag in getattr(tag_class, "TAGS"): + loader.add_constructor(tag, tag_class) + return loader + + def yaml_to_str(cfg: dict) -> str: """ Return a uwtools-conventional YAML representation of the given dict. diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index 407e49f9c..2985addc1 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -17,7 +17,7 @@ from uwtools.config import jinja2 from uwtools.config.jinja2 import J2Template -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, uw_yaml_loader from uwtools.logging import log from uwtools.tests.support import logged, regex_logged @@ -141,7 +141,7 @@ def test_dereference_remove(caplog): remove = UWYAMLRemove(yaml.SafeLoader(""), yaml.ScalarNode(tag="!remove", value="")) val = {"a": {"b": {"c": "cherry", "d": remove}}} assert jinja2.dereference(val=val, context={}) == {"a": {"b": {"c": "cherry"}}} - assert regex_logged(caplog, "Removing value at: a > b > d") + assert regex_logged(caplog, "Removing value at: a.b.d") def test_dereference_str_expression_rendered(): @@ -315,6 +315,13 @@ def test__deref_debug(caplog): assert logged(caplog, "[dereference] Frobnicated: foo") +def test__deref_render_held(caplog): + val, context = "!int '{{ a }}'", yaml.load("a: !int '{{ 42 }}'", Loader=uw_yaml_loader()) + assert jinja2._deref_render(val=val, context=context) == val + assert not regex_logged(caplog, "Rendered") + assert regex_logged(caplog, "Held") + + def test__deref_render_no(caplog, deref_render_assets): val, context, _ = deref_render_assets assert jinja2._deref_render(val=val, context=context) == val diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index f7103628a..793af23db 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -59,6 +59,16 @@ def realize_config_yaml_input(tmp_path): # Helpers +def help_realize_config_double_tag(config, expected, tmp_path): + path_in = tmp_path / "in.yaml" + path_out = tmp_path / "out.yaml" + with open(path_in, "w", encoding="utf-8") as f: + print(dedent(config).strip(), file=f) + tools.realize_config(input_config=path_in, output_file=path_out) + with open(path_out, "r", encoding="utf-8") as f: + assert f.read().strip() == dedent(expected).strip() + + def help_realize_config_fmt2fmt(input_file, input_format, update_file, update_format, tmpdir): input_file = fixture_path(input_file) update_file = fixture_path(update_file) @@ -235,6 +245,58 @@ def test_realize_config_depth_mismatch_to_sh(realize_config_yaml_input): ) +def test_realize_config_double_tag_flat(tmp_path): + config = """ + a: 1 + b: 2 + foo: !int "{{ a + b }}" + bar: !int "{{ foo }}" + """ + expected = """ + a: 1 + b: 2 + foo: 3 + bar: 3 + """ + help_realize_config_double_tag(config, expected, tmp_path) + + +def test_realize_config_double_tag_nest(tmp_path): + config = """ + a: 1.0 + b: 2.0 + qux: + foo: !float "{{ a + b }}" + bar: !float "{{ foo }}" + """ + expected = """ + a: 1.0 + b: 2.0 + qux: + foo: 3.0 + bar: 3.0 + """ + help_realize_config_double_tag(config, expected, tmp_path) + + +def test_realize_config_double_tag_nest_forwrad_reference(tmp_path): + config = """ + a: true + b: false + bar: !bool "{{ qux.foo }}" + qux: + foo: !bool "{{ a or b }}" + """ + expected = """ + a: true + b: false + bar: true + qux: + foo: true + """ + help_realize_config_double_tag(config, expected, tmp_path) + + def test_realize_config_dry_run(caplog): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file. From 94980a7a673266bdc045077765b14656d4b3e2bf Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:58:28 -0700 Subject: [PATCH 2/6] Update CODEOWNERS (#666) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 30ce63d6a..d24f98cf1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners -* @NaureenBharwaniNOAA @christinaholtNOAA @elcarpenterNOAA @fgabelmannjr @maddenp-noaa @weirae @Byrnetp +* @Byrnetp @NaureenBharwaniNOAA @christinaholtNOAA @elcarpenterNOAA @maddenp-noaa From a95fa0b3f7c24de7e4a65bd0050f562968c3c94f Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:14:40 -0700 Subject: [PATCH 3/6] Update fs mode to use --key-path CLI switch / key_path API argument (#670) --- docs/environment.yml | 2 +- .../user_guide/cli/drivers/cdeps/run-help.out | 3 +- .../cli/drivers/chgres_cube/run-help.out | 3 +- .../cli/drivers/esg_grid/run-help.out | 3 +- .../cli/drivers/filter_topo/run-help.out | 3 +- .../user_guide/cli/drivers/fv3/run-help.out | 3 +- .../drivers/global_equiv_resol/run-help.out | 3 +- .../user_guide/cli/drivers/ioda/run-help.out | 3 +- .../user_guide/cli/drivers/jedi/run-help.out | 3 +- .../cli/drivers/make_hgrid/run-help.out | 3 +- .../cli/drivers/make_solo_mosaic/run-help.out | 3 +- .../user_guide/cli/drivers/mpas/run-help.out | 3 +- .../cli/drivers/mpas_init/run-help.out | 3 +- .../user_guide/cli/drivers/orog/run-help.out | 3 +- .../cli/drivers/orog_gsl/run-help.out | 3 +- .../cli/drivers/sfc_climo_gen/run-help.out | 3 +- .../user_guide/cli/drivers/shave/run-help.out | 3 +- .../cli/drivers/ungrib/run-help.out | 3 +- .../user_guide/cli/drivers/upp/run-help.out | 3 +- .../user_guide/cli/tools/execute/help.out | 3 +- .../tools/fs/copy-exec-no-target-dir-err.cmd | 2 +- .../cli/tools/fs/copy-exec-timedep.cmd | 2 +- .../user_guide/cli/tools/fs/copy-exec.cmd | 2 +- .../user_guide/cli/tools/fs/copy-help.out | 9 +- .../tools/fs/link-exec-no-target-dir-err.cmd | 2 +- .../cli/tools/fs/link-exec-timedep.cmd | 2 +- .../user_guide/cli/tools/fs/link-exec.cmd | 2 +- .../user_guide/cli/tools/fs/link-help.out | 9 +- .../fs/makedirs-exec-no-target-dir-err.cmd | 2 +- .../cli/tools/fs/makedirs-exec-timedep.cmd | 2 +- .../user_guide/cli/tools/fs/makedirs-exec.cmd | 2 +- .../user_guide/cli/tools/fs/makedirs-help.out | 8 +- notebooks/config.ipynb | 2 +- notebooks/fs.ipynb | 446 +++++++++--------- notebooks/install-deps | 2 +- src/uwtools/api/config.py | 7 +- src/uwtools/api/execute.py | 5 +- src/uwtools/api/fs.py | 19 +- src/uwtools/cli.py | 26 +- src/uwtools/config/support.py | 2 +- src/uwtools/config/tools.py | 32 +- src/uwtools/drivers/driver.py | 39 +- src/uwtools/fs.py | 30 +- src/uwtools/strings.py | 1 - src/uwtools/tests/api/test_fs.py | 2 +- src/uwtools/tests/config/test_tools.py | 59 --- src/uwtools/tests/test_cli.py | 4 +- src/uwtools/tests/test_fs.py | 18 +- src/uwtools/utils/api.py | 11 +- 49 files changed, 341 insertions(+), 467 deletions(-) diff --git a/docs/environment.yml b/docs/environment.yml index 86e4aa822..8b0384a49 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -6,4 +6,4 @@ dependencies: - python >=3.9,<3.13 # keep in sync with meta.yaml run req - sphinx_rtd_theme 3.0.* - sphinxcontrib-bibtex 2.6.* - - tree + - tree 2.2.* diff --git a/docs/sections/user_guide/cli/drivers/cdeps/run-help.out b/docs/sections/user_guide/cli/drivers/cdeps/run-help.out index abe05b455..67131c0ab 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps/run-help.out +++ b/docs/sections/user_guide/cli/drivers/cdeps/run-help.out @@ -20,8 +20,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out b/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out index 3f0eca53e..218bf5612 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out @@ -25,8 +25,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out b/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out index 94eeee88b..2a993a489 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out +++ b/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out @@ -19,8 +19,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out b/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out index 95604d511..721b937a8 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out +++ b/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out @@ -19,8 +19,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/fv3/run-help.out b/docs/sections/user_guide/cli/drivers/fv3/run-help.out index e56710a4d..8d4a13bca 100644 --- a/docs/sections/user_guide/cli/drivers/fv3/run-help.out +++ b/docs/sections/user_guide/cli/drivers/fv3/run-help.out @@ -23,8 +23,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out b/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out index 5e6e9a84e..d4974f5ca 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out @@ -19,8 +19,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.out b/docs/sections/user_guide/cli/drivers/ioda/run-help.out index 43e297ce0..a6b938c86 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/run-help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.out @@ -23,8 +23,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/jedi/run-help.out b/docs/sections/user_guide/cli/drivers/jedi/run-help.out index aa75b918a..4ef5237ff 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/run-help.out +++ b/docs/sections/user_guide/cli/drivers/jedi/run-help.out @@ -23,8 +23,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out b/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out index 76ad2d63b..ab39a59d2 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out @@ -19,8 +19,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out index 203357701..e3fd0cde8 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out @@ -19,8 +19,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/mpas/run-help.out b/docs/sections/user_guide/cli/drivers/mpas/run-help.out index 54d66f963..4af661a79 100644 --- a/docs/sections/user_guide/cli/drivers/mpas/run-help.out +++ b/docs/sections/user_guide/cli/drivers/mpas/run-help.out @@ -23,8 +23,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out b/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out index c9682821e..0b1c7509b 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out +++ b/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out @@ -23,8 +23,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/orog/run-help.out b/docs/sections/user_guide/cli/drivers/orog/run-help.out index 3a2bd47d8..7d60a6504 100644 --- a/docs/sections/user_guide/cli/drivers/orog/run-help.out +++ b/docs/sections/user_guide/cli/drivers/orog/run-help.out @@ -18,8 +18,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out b/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out index ff7a8e7fe..3b12edfdb 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out @@ -19,8 +19,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out index 7b6b718ad..8d0660ced 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out @@ -19,8 +19,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/shave/run-help.out b/docs/sections/user_guide/cli/drivers/shave/run-help.out index 085279196..81b6359e2 100644 --- a/docs/sections/user_guide/cli/drivers/shave/run-help.out +++ b/docs/sections/user_guide/cli/drivers/shave/run-help.out @@ -18,8 +18,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/ungrib/run-help.out b/docs/sections/user_guide/cli/drivers/ungrib/run-help.out index d73f89464..c7132634c 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib/run-help.out +++ b/docs/sections/user_guide/cli/drivers/ungrib/run-help.out @@ -23,8 +23,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/drivers/upp/run-help.out b/docs/sections/user_guide/cli/drivers/upp/run-help.out index 106254c93..da5c4d6b6 100644 --- a/docs/sections/user_guide/cli/drivers/upp/run-help.out +++ b/docs/sections/user_guide/cli/drivers/upp/run-help.out @@ -25,8 +25,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --schema-file PATH Path to schema file to use for validation --quiet, -q diff --git a/docs/sections/user_guide/cli/tools/execute/help.out b/docs/sections/user_guide/cli/tools/execute/help.out index 265292176..75c7260fe 100644 --- a/docs/sections/user_guide/cli/tools/execute/help.out +++ b/docs/sections/user_guide/cli/tools/execute/help.out @@ -34,8 +34,7 @@ Optional arguments: --graph-file PATH Path to Graphviz DOT output [experimental] --key-path KEY[.KEY...] - Dot-separated path of keys leading through the config to the driver's - configuration block + Dot-separated path of keys to driver config block --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd index fb3d8b83f..d89130592 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd @@ -1 +1 @@ -uw fs copy --config-file copy-config.yaml config files +uw fs copy --config-file copy-config.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd index 0f2a1911b..6c3c956af 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd @@ -1,4 +1,4 @@ rm -rf copy-dst-timedep -uw fs copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files +uw fs copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files echo tree copy-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd index 175451030..f0579f20e 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd @@ -1,4 +1,4 @@ rm -rf copy-dst -uw fs copy --target-dir copy-dst --config-file copy-config.yaml config files +uw fs copy --target-dir copy-dst --config-file copy-config.yaml --key-path config.files echo tree copy-dst diff --git a/docs/sections/user_guide/cli/tools/fs/copy-help.out b/docs/sections/user_guide/cli/tools/fs/copy-help.out index ce73007a6..b53b814f2 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-help.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-help.out @@ -1,7 +1,6 @@ usage: uw fs copy [-h] [--version] [--config-file PATH] [--target-dir PATH] - [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] [--quiet] - [--verbose] - [KEY ...] + [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] + [--key-path KEY[.KEY...]] [--quiet] [--verbose] Copy files @@ -20,9 +19,9 @@ Optional arguments: The leadtime as hours[:minutes[:seconds]] --dry-run Only log info, making no changes + --key-path KEY[.KEY...] + Dot-separated path of keys to config block to use --quiet, -q Print no logging messages --verbose, -v Print all logging messages - KEY - YAML key leading to file dst/src block diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd index 8446a4e8b..d1d03c79b 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd @@ -1 +1 @@ -uw fs link --config-file link-config.yaml config files +uw fs link --config-file link-config.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd index f3e240397..d815b7d38 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd @@ -1,4 +1,4 @@ rm -rf link-dst-timedep -uw fs link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files +uw fs link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files echo tree link-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd index f4f14059b..e1ca6bcda 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd @@ -1,4 +1,4 @@ rm -rf link-dst -uw fs link --target-dir link-dst --config-file link-config.yaml config files +uw fs link --target-dir link-dst --config-file link-config.yaml --key-path config.files echo tree link-dst diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.out b/docs/sections/user_guide/cli/tools/fs/link-help.out index be07a9eec..f3a706638 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-help.out +++ b/docs/sections/user_guide/cli/tools/fs/link-help.out @@ -1,7 +1,6 @@ usage: uw fs link [-h] [--version] [--config-file PATH] [--target-dir PATH] - [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] [--quiet] - [--verbose] - [KEY ...] + [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] + [--key-path KEY[.KEY...]] [--quiet] [--verbose] Link files @@ -20,9 +19,9 @@ Optional arguments: The leadtime as hours[:minutes[:seconds]] --dry-run Only log info, making no changes + --key-path KEY[.KEY...] + Dot-separated path of keys to config block to use --quiet, -q Print no logging messages --verbose, -v Print all logging messages - KEY - YAML key leading to file dst/src block diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd index 80ff4150e..8c2d83cc9 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd @@ -1 +1 @@ -uw fs makedirs --config-file makedirs-config.yaml config +uw fs makedirs --config-file makedirs-config.yaml --key-path config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd index fa30614ec..221bac642 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent-timedep -uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config +uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config echo tree -F makedirs-parent-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd index b99d8c3f4..4ca49133b 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent -uw fs makedirs --target-dir makedirs-parent --config-file makedirs-config.yaml config +uw fs makedirs --target-dir makedirs-parent --config-file makedirs-config.yaml --key-path config echo tree -F makedirs-parent diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-help.out b/docs/sections/user_guide/cli/tools/fs/makedirs-help.out index 0e80d2f16..335752133 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-help.out +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-help.out @@ -1,7 +1,7 @@ usage: uw fs makedirs [-h] [--version] [--config-file PATH] [--target-dir PATH] [--cycle CYCLE] - [--leadtime LEADTIME] [--dry-run] [--quiet] [--verbose] - [KEY ...] + [--leadtime LEADTIME] [--dry-run] + [--key-path KEY[.KEY...]] [--quiet] [--verbose] Make directories @@ -20,9 +20,9 @@ Optional arguments: The leadtime as hours[:minutes[:seconds]] --dry-run Only log info, making no changes + --key-path KEY[.KEY...] + Dot-separated path of keys to config block to use --quiet, -q Print no logging messages --verbose, -v Print all logging messages - KEY - YAML key leading to file dst/src block diff --git a/notebooks/config.ipynb b/notebooks/config.ipynb index 257e1ee31..64551f2ea 100644 --- a/notebooks/config.ipynb +++ b/notebooks/config.ipynb @@ -367,7 +367,7 @@ " :param update_format: Format of the update config (optional if file's extension is recognized).\n", " :param output_file: Output config file (``None`` => write to ``stdout``).\n", " :param output_format: Format of the output config (optional if file's extension is recognized).\n", - " :param key_path: Path through keys to the desired output block.\n", + " :param key_path: Path of keys to the desired output block.\n", " :param values_needed: Report complete, missing, and template values.\n", " :param total: Require rendering of all Jinja2 variables/expressions.\n", " :param dry_run: Log output instead of writing to output.\n", diff --git a/notebooks/fs.ipynb b/notebooks/fs.ipynb index 4060181a7..a69311b61 100644 --- a/notebooks/fs.ipynb +++ b/notebooks/fs.ipynb @@ -64,14 +64,14 @@ "text": [ "Help on function copy in module uwtools.api.fs:\n", "\n", - "copy(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", + "copy(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", " Copy files.\n", "\n", " :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " :param target_dir: Path to target directory.\n", " :param cycle: A datetime object to make available for use in the config.\n", " :param leadtime: A timedelta object to make available for use in the config.\n", - " :param keys: YAML keys leading to file dst/src block.\n", + " :param key_path: Path of keys to config block to use.\n", " :param dry_run: Do not copy files.\n", " :param stdin_ok: OK to read from ``stdin``?\n", " :return: ``True`` if all copies were created.\n", @@ -130,26 +130,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO File copies: Final state: Ready\n" + "[2024-12-09T21:56:27] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:56:27] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:56:27] INFO File copies: Initial state: Not Ready\n", + "[2024-12-09T21:56:27] INFO File copies: Checking requirements\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Initial state: Not Ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Checking requirements\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Executing\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Final state: Ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Initial state: Not Ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Checking requirements\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Executing\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Initial state: Not Ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Checking requirements\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Executing\n", + "[2024-12-09T21:56:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-12-09T21:56:27] INFO File copies: Final state: Ready\n" ] }, { @@ -195,7 +195,7 @@ "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -224,16 +224,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Checking requirements\n", - "[2024-11-19T23:14:42] WARNING File fixtures/fs/missing-file.nml: State: Not Ready (external asset)\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Requirement(s) not ready\n", - "[2024-11-19T23:14:42] WARNING Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Final state: Not Ready\n", - "[2024-11-19T23:14:42] WARNING File copies: Final state: Not Ready\n" + "[2024-12-09T21:56:33] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:56:33] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:56:33] INFO File copies: Initial state: Not Ready\n", + "[2024-12-09T21:56:33] INFO File copies: Checking requirements\n", + "[2024-12-09T21:56:33] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Initial state: Not Ready\n", + "[2024-12-09T21:56:33] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Checking requirements\n", + "[2024-12-09T21:56:33] WARNING File fixtures/fs/missing-file.nml: State: Not Ready (external asset)\n", + "[2024-12-09T21:56:33] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Requirement(s) not ready\n", + "[2024-12-09T21:56:33] WARNING Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Final state: Not Ready\n", + "[2024-12-09T21:56:33] WARNING File copies: Final state: Not Ready\n" ] }, { @@ -278,7 +278,7 @@ "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -292,7 +292,7 @@ "id": "b2527839-c217-428d-a686-c684a682c0e8", "metadata": {}, "source": [ - "### Using the `keys` parameter\n", + "### Using the `key_path` argument\n", "\n", "Consider the following configuration, in which the destination/source mapping is not located at the top level of the configuration:" ] @@ -326,7 +326,7 @@ "id": "5311866f-a1f5-4243-81a8-2c52172e091a", "metadata": {}, "source": [ - "Without additional information, `copy()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `keys` parameter:" + "Without additional information, `copy()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `key_path` argument:" ] }, { @@ -339,26 +339,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO File copies: Final state: Ready\n" + "[2024-12-09T21:57:25] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:57:25] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:57:25] INFO File copies: Initial state: Not Ready\n", + "[2024-12-09T21:57:25] INFO File copies: Checking requirements\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Initial state: Not Ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Checking requirements\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Executing\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Final state: Ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Initial state: Not Ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Checking requirements\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Executing\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Initial state: Not Ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Checking requirements\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Executing\n", + "[2024-12-09T21:57:25] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-12-09T21:57:25] INFO File copies: Final state: Ready\n" ] }, { @@ -377,7 +377,7 @@ "fs.copy(\n", " config=\"fixtures/fs/copy-keys-config.yaml\",\n", " target_dir=\"tmp/copy-keys-target\",\n", - " keys=[\"files\",\"to\",\"copy\"]\n", + " key_path=[\"files\",\"to\",\"copy\"]\n", ")" ] }, @@ -405,7 +405,7 @@ "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -437,7 +437,7 @@ "Help on class Copier in module uwtools.fs:\n", "\n", "class Copier(FileStager)\n", - " | Copier(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | Copier(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False) -> None\n", " |\n", " | Stage files by copying.\n", " |\n", @@ -461,14 +461,14 @@ " | ----------------------------------------------------------------------\n", " | Methods inherited from Stager:\n", " |\n", - " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False) -> None\n", " | Stage files and directories.\n", " |\n", " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", " | :param cycle: A ``datetime`` object to make available for use in the config.\n", " | :param leadtime: A ``timedelta`` object to make available for use in the config.\n", - " | :param keys: YAML keys leading to file dst/src block.\n", + " | :param key_path: Path of keys to config block to use.\n", " | :param dry_run: Do not copy files.\n", " | :raises: ``UWConfigError`` if config fails validation.\n", " |\n", @@ -506,26 +506,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Executing\n", - "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO File copies: Final state: Ready\n" + "[2024-12-09T21:57:38] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:57:38] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:57:38] INFO File copies: Initial state: Not Ready\n", + "[2024-12-09T21:57:38] INFO File copies: Checking requirements\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Initial state: Not Ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Checking requirements\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Executing\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Final state: Ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Initial state: Not Ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Checking requirements\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Executing\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Initial state: Not Ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Checking requirements\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Executing\n", + "[2024-12-09T21:57:38] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-12-09T21:57:38] INFO File copies: Final state: Ready\n" ] }, { @@ -574,7 +574,7 @@ "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -605,14 +605,14 @@ "text": [ "Help on function link in module uwtools.api.fs:\n", "\n", - "link(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", + "link(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", " Link files.\n", "\n", " :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " :param target_dir: Path to target directory.\n", " :param cycle: A datetime object to make available for use in the config.\n", " :param leadtime: A timedelta object to make available for use in the config.\n", - " :param keys: YAML keys leading to file dst/src block.\n", + " :param key_path: Path of keys to config block to use.\n", " :param dry_run: Do not link files.\n", " :param stdin_ok: OK to read from ``stdin``?\n", " :return: ``True`` if all links were created.\n", @@ -671,26 +671,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO File links: Final state: Ready\n" + "[2024-12-09T21:57:45] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:57:45] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:57:45] INFO File links: Initial state: Not Ready\n", + "[2024-12-09T21:57:45] INFO File links: Checking requirements\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-12-09T21:57:45] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-12-09T21:57:45] INFO File links: Final state: Ready\n" ] }, { @@ -736,7 +736,7 @@ "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -765,16 +765,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Checking requirements\n", - "[2024-11-19T23:14:42] WARNING Filesystem item fixtures/fs/missing-file.nml: State: Not Ready (external asset)\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Requirement(s) not ready\n", - "[2024-11-19T23:14:42] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Final state: Not Ready\n", - "[2024-11-19T23:14:42] WARNING File links: Final state: Not Ready\n" + "[2024-12-09T21:57:49] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:57:49] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:57:49] INFO File links: Initial state: Not Ready\n", + "[2024-12-09T21:57:49] INFO File links: Checking requirements\n", + "[2024-12-09T21:57:49] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Initial state: Not Ready\n", + "[2024-12-09T21:57:49] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Checking requirements\n", + "[2024-12-09T21:57:49] WARNING Filesystem item fixtures/fs/missing-file.nml: State: Not Ready (external asset)\n", + "[2024-12-09T21:57:49] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Requirement(s) not ready\n", + "[2024-12-09T21:57:49] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Final state: Not Ready\n", + "[2024-12-09T21:57:49] WARNING File links: Final state: Not Ready\n" ] }, { @@ -819,7 +819,7 @@ "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -833,7 +833,7 @@ "id": "b887c95e-f71f-4a26-b709-d410a3c30c2e", "metadata": {}, "source": [ - "### Using the `keys` parameter \n", + "### Using the `key_path` argument \n", "\n", "Consider the following configuration, in which the destination/source mapping is not located at the top level of the configuration:" ] @@ -867,7 +867,7 @@ "id": "9977ee46-17da-419e-821b-a32fac5139f8", "metadata": {}, "source": [ - "Without additional information, `link()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `keys` parameter:" + "Without additional information, `link()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `Key_path` argument:" ] }, { @@ -880,26 +880,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO File links: Final state: Ready\n" + "[2024-12-09T21:58:19] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:58:19] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:58:19] INFO File links: Initial state: Not Ready\n", + "[2024-12-09T21:58:19] INFO File links: Checking requirements\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-12-09T21:58:19] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-12-09T21:58:19] INFO File links: Final state: Ready\n" ] }, { @@ -918,7 +918,7 @@ "fs.link(\n", " config=\"fixtures/fs/link-keys-config.yaml\",\n", " target_dir=\"tmp/link-keys-target\",\n", - " keys=[\"files\",\"to\",\"link\"]\n", + " key_path=[\"files\",\"to\",\"link\"]\n", ")" ] }, @@ -946,7 +946,7 @@ "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -978,7 +978,7 @@ "Help on class Linker in module uwtools.fs:\n", "\n", "class Linker(FileStager)\n", - " | Linker(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | Linker(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False) -> None\n", " |\n", " | Stage files by linking.\n", " |\n", @@ -1002,14 +1002,14 @@ " | ----------------------------------------------------------------------\n", " | Methods inherited from Stager:\n", " |\n", - " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False) -> None\n", " | Stage files and directories.\n", " |\n", " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", " | :param cycle: A ``datetime`` object to make available for use in the config.\n", " | :param leadtime: A ``timedelta`` object to make available for use in the config.\n", - " | :param keys: YAML keys leading to file dst/src block.\n", + " | :param key_path: Path of keys to config block to use.\n", " | :param dry_run: Do not copy files.\n", " | :raises: ``UWConfigError`` if config fails validation.\n", " |\n", @@ -1047,26 +1047,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", - "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO File links: Final state: Ready\n" + "[2024-12-09T21:58:29] INFO Validating config against internal schema: files-to-stage\n", + "[2024-12-09T21:58:29] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:58:29] INFO File links: Initial state: Not Ready\n", + "[2024-12-09T21:58:29] INFO File links: Checking requirements\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-12-09T21:58:29] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-12-09T21:58:29] INFO File links: Final state: Ready\n" ] }, { @@ -1115,7 +1115,7 @@ "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "1 directory, 3 files\n" + "2 directories, 3 files\n" ] } ], @@ -1146,14 +1146,14 @@ "text": [ "Help on function makedirs in module uwtools.api.fs:\n", "\n", - "makedirs(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", + "makedirs(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", " Make directories.\n", "\n", " :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " :param target_dir: Path to target directory.\n", " :param cycle: A datetime object to make available for use in the config.\n", " :param leadtime: A timedelta object to make available for use in the config.\n", - " :param keys: YAML keys leading to file dst/src block.\n", + " :param key_path: Path of keys to config block to use.\n", " :param dry_run: Do not link files.\n", " :param stdin_ok: OK to read from ``stdin``?\n", " :return: ``True`` if all directories were made.\n", @@ -1212,21 +1212,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: makedirs\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO Directories: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directories: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Executing\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Executing\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Directories: Final state: Ready\n" + "[2024-12-09T21:58:35] INFO Validating config against internal schema: makedirs\n", + "[2024-12-09T21:58:35] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:58:35] INFO Directories: Initial state: Not Ready\n", + "[2024-12-09T21:58:35] INFO Directories: Checking requirements\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/foo: Initial state: Not Ready\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/foo: Checking requirements\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/foo: Requirement(s) ready\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/foo: Executing\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/foo: Final state: Ready\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/bar/baz: Initial state: Not Ready\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/bar/baz: Checking requirements\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/bar/baz: Requirement(s) ready\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/bar/baz: Executing\n", + "[2024-12-09T21:58:35] INFO Directory tmp/dir-target/bar/baz: Final state: Ready\n", + "[2024-12-09T21:58:35] INFO Directories: Final state: Ready\n" ] }, { @@ -1271,7 +1271,7 @@ "│   └── \u001b[01;34mbaz\u001b[0m\n", "└── \u001b[01;34mfoo\u001b[0m\n", "\n", - "3 directories, 0 files\n" + "4 directories, 0 files\n" ] } ], @@ -1285,7 +1285,7 @@ "id": "329e939d-0f6d-412c-a36a-4682fe99609a", "metadata": {}, "source": [ - "### Using the `keys` parameter \n", + "### Using the `key_path` argument \n", "\n", "Consider the following configuration, in which the destination/source mapping is not located at the top level of the configuration:" ] @@ -1319,7 +1319,7 @@ "id": "909ff3ea-8577-4b91-94fc-6ce6effe4bec", "metadata": {}, "source": [ - "Without additional information, `makedirs()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `keys` parameter:" + "Without additional information, `makedirs()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `key_path` argument:" ] }, { @@ -1332,21 +1332,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: makedirs\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO Directories: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directories: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Executing\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Executing\n", - "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Directories: Final state: Ready\n" + "[2024-12-09T21:59:02] INFO Validating config against internal schema: makedirs\n", + "[2024-12-09T21:59:02] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:59:02] INFO Directories: Initial state: Not Ready\n", + "[2024-12-09T21:59:02] INFO Directories: Checking requirements\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/foo/bar: Initial state: Not Ready\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/foo/bar: Checking requirements\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/foo/bar: Requirement(s) ready\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/foo/bar: Executing\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/foo/bar: Final state: Ready\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/baz: Initial state: Not Ready\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/baz: Checking requirements\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/baz: Requirement(s) ready\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/baz: Executing\n", + "[2024-12-09T21:59:02] INFO Directory tmp/dir-keys-target/baz: Final state: Ready\n", + "[2024-12-09T21:59:02] INFO Directories: Final state: Ready\n" ] }, { @@ -1365,7 +1365,7 @@ "fs.makedirs(\n", " config=\"fixtures/fs/dir-keys-config.yaml\",\n", " target_dir=\"tmp/dir-keys-target\",\n", - " keys=[\"path\",\"to\",\"dirs\"]\n", + " key_path=[\"path\",\"to\",\"dirs\"]\n", ")" ] }, @@ -1392,7 +1392,7 @@ "└── \u001b[01;34mfoo\u001b[0m\n", " └── \u001b[01;34mbar\u001b[0m\n", "\n", - "3 directories, 0 files\n" + "4 directories, 0 files\n" ] } ], @@ -1424,7 +1424,7 @@ "Help on class MakeDirs in module uwtools.fs:\n", "\n", "class MakeDirs(Stager)\n", - " | MakeDirs(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | MakeDirs(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False) -> None\n", " |\n", " | Make directories.\n", " |\n", @@ -1447,14 +1447,14 @@ " | ----------------------------------------------------------------------\n", " | Methods inherited from Stager:\n", " |\n", - " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, dry_run: bool = False) -> None\n", " | Stage files and directories.\n", " |\n", " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", " | :param cycle: A ``datetime`` object to make available for use in the config.\n", " | :param leadtime: A ``timedelta`` object to make available for use in the config.\n", - " | :param keys: YAML keys leading to file dst/src block.\n", + " | :param key_path: Path of keys to config block to use.\n", " | :param dry_run: Do not copy files.\n", " | :raises: ``UWConfigError`` if config fails validation.\n", " |\n", @@ -1492,21 +1492,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:14:42] INFO Validating config against internal schema: makedirs\n", - "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", - "[2024-11-19T23:14:42] INFO Directories: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directories: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Executing\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Initial state: Not Ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Checking requirements\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Requirement(s) ready\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Executing\n", - "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Final state: Ready\n", - "[2024-11-19T23:14:42] INFO Directories: Final state: Ready\n" + "[2024-12-09T21:59:07] INFO Validating config against internal schema: makedirs\n", + "[2024-12-09T21:59:07] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-12-09T21:59:07] INFO Directories: Initial state: Not Ready\n", + "[2024-12-09T21:59:07] INFO Directories: Checking requirements\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/foo: Initial state: Not Ready\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/foo: Checking requirements\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/foo: Requirement(s) ready\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/foo: Executing\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/foo: Final state: Ready\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/bar/baz: Initial state: Not Ready\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/bar/baz: Checking requirements\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/bar/baz: Requirement(s) ready\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/bar/baz: Executing\n", + "[2024-12-09T21:59:07] INFO Directory tmp/makedirs-target/bar/baz: Final state: Ready\n", + "[2024-12-09T21:59:07] INFO Directories: Final state: Ready\n" ] }, { @@ -1553,7 +1553,7 @@ "│   └── \u001b[01;34mbaz\u001b[0m\n", "└── \u001b[01;34mfoo\u001b[0m\n", "\n", - "3 directories, 0 files\n" + "4 directories, 0 files\n" ] } ], @@ -1565,9 +1565,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:DEV-uwtools] *", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DEV-uwtools-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1579,7 +1579,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.8" } }, "nbformat": 4, diff --git a/notebooks/install-deps b/notebooks/install-deps index 54a214b38..37283c00c 100644 --- a/notebooks/install-deps +++ b/notebooks/install-deps @@ -1 +1 @@ -conda install -q -y --repodata-fn repodata.json "jupyterlab<4.4" "testbook<0.5" +conda install -q -y --repodata-fn repodata.json "jupyterlab<4.4" "testbook<0.5" "tree 2.2.*" diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index bbdd7c517..5df802939 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -12,6 +12,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.config.formats.sh import SHConfig from uwtools.config.formats.yaml import YAMLConfig +from uwtools.config.support import YAMLKey from uwtools.config.tools import compare_configs as _compare from uwtools.config.tools import realize_config as _realize from uwtools.config.validator import validate_external as _validate_external @@ -116,7 +117,7 @@ def realize( update_format: Optional[str] = None, output_file: Optional[Union[Path, str]] = None, output_format: Optional[str] = None, - key_path: Optional[list[Union[str, int]]] = None, + key_path: Optional[list[YAMLKey]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, @@ -146,7 +147,7 @@ def realize_to_dict( # pylint: disable=unused-argument input_format: Optional[str] = None, update_config: Optional[Union[dict, _Config, Path, str]] = None, update_format: Optional[str] = None, - key_path: Optional[list[Union[str, int]]] = None, + key_path: Optional[list[YAMLKey]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, @@ -239,7 +240,7 @@ def validate( :param update_format: Format of the update config (optional if file's extension is recognized). :param output_file: Output config file (``None`` => write to ``stdout``). :param output_format: Format of the output config (optional if file's extension is recognized). -:param key_path: Path through keys to the desired output block. +:param key_path: Path of keys to the desired output block. :param values_needed: Report complete, missing, and template values. :param total: Require rendering of all Jinja2 variables/expressions. :param dry_run: Log output instead of writing to output. diff --git a/src/uwtools/api/execute.py b/src/uwtools/api/execute.py index 6ab629db0..f6546a017 100644 --- a/src/uwtools/api/execute.py +++ b/src/uwtools/api/execute.py @@ -10,6 +10,7 @@ from types import ModuleType from typing import Optional, Type, Union +from uwtools.config.support import YAMLKey from uwtools.drivers.support import graph from uwtools.drivers.support import tasks as _tasks from uwtools.logging import log @@ -28,7 +29,7 @@ def execute( batch: Optional[bool] = False, # pylint: disable=unused-argument dry_run: Optional[bool] = False, graph_file: Optional[Union[Path, str]] = None, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, stdin_ok: Optional[bool] = False, ) -> bool: """ @@ -48,7 +49,7 @@ def execute( :param batch: Submit run to the batch system? :param dry_run: Do not run the executable, just report what would have been done. :param graph_file: Write Graphviz DOT output here. - :param key_path: Path of keys to subsection of config file. + :param key_path: Path of keys to config block to use. :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ diff --git a/src/uwtools/api/fs.py b/src/uwtools/api/fs.py index 5b7d6102f..cb0343b85 100644 --- a/src/uwtools/api/fs.py +++ b/src/uwtools/api/fs.py @@ -8,6 +8,7 @@ from iotaa import Asset +from uwtools.config.support import YAMLKey from uwtools.fs import Copier, Linker, MakeDirs from uwtools.utils.api import ensure_data_source as _ensure_data_source @@ -17,7 +18,7 @@ def copy( target_dir: Optional[Union[Path, str]] = None, cycle: Optional[dt.datetime] = None, leadtime: Optional[dt.timedelta] = None, - keys: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, dry_run: bool = False, stdin_ok: bool = False, ) -> bool: @@ -28,7 +29,7 @@ def copy( :param target_dir: Path to target directory. :param cycle: A datetime object to make available for use in the config. :param leadtime: A timedelta object to make available for use in the config. - :param keys: YAML keys leading to file dst/src block. + :param key_path: Path of keys to config block to use. :param dry_run: Do not copy files. :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if all copies were created. @@ -38,7 +39,7 @@ def copy( config=_ensure_data_source(config, stdin_ok), cycle=cycle, leadtime=leadtime, - keys=keys, + key_path=key_path, dry_run=dry_run, ) assets: list[Asset] = stager.go() # type: ignore @@ -50,7 +51,7 @@ def link( target_dir: Optional[Union[Path, str]] = None, cycle: Optional[dt.datetime] = None, leadtime: Optional[dt.timedelta] = None, - keys: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, dry_run: bool = False, stdin_ok: bool = False, ) -> bool: @@ -61,7 +62,7 @@ def link( :param target_dir: Path to target directory. :param cycle: A datetime object to make available for use in the config. :param leadtime: A timedelta object to make available for use in the config. - :param keys: YAML keys leading to file dst/src block. + :param key_path: Path of keys to config block to use. :param dry_run: Do not link files. :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if all links were created. @@ -71,7 +72,7 @@ def link( config=_ensure_data_source(config, stdin_ok), cycle=cycle, leadtime=leadtime, - keys=keys, + key_path=key_path, dry_run=dry_run, ) assets: list[Asset] = stager.go() # type: ignore @@ -83,7 +84,7 @@ def makedirs( target_dir: Optional[Union[Path, str]] = None, cycle: Optional[dt.datetime] = None, leadtime: Optional[dt.timedelta] = None, - keys: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, dry_run: bool = False, stdin_ok: bool = False, ) -> bool: @@ -94,7 +95,7 @@ def makedirs( :param target_dir: Path to target directory. :param cycle: A datetime object to make available for use in the config. :param leadtime: A timedelta object to make available for use in the config. - :param keys: YAML keys leading to file dst/src block. + :param key_path: Path of keys to config block to use. :param dry_run: Do not link files. :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if all directories were made. @@ -104,7 +105,7 @@ def makedirs( config=_ensure_data_source(config, stdin_ok), cycle=cycle, leadtime=leadtime, - keys=keys, + key_path=key_path, dry_run=dry_run, ) assets: list[Asset] = stager.go() # type: ignore diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index e268a2e9b..036b04cbf 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -278,11 +278,7 @@ def _add_subparser_execute(subparsers: Subparsers) -> ModeChecks: _add_arg_batch(optional) _add_arg_dry_run(optional) _add_arg_graph_file(optional) - _add_arg_key_path( - optional, - helpmsg="Dot-separated path of keys leading through the config " - "to the driver's configuration block", - ) + _add_arg_key_path(optional, helpmsg="Dot-separated path of keys to driver config block") return {STR.execute: _add_args_verbosity(optional)} @@ -339,8 +335,8 @@ def _add_subparser_fs_common(parser: Parser) -> ActionChecks: _add_arg_cycle(optional) _add_arg_leadtime(optional) _add_arg_dry_run(optional) + _add_arg_key_path(optional, helpmsg="Dot-separated path of keys to config block to use") checks = _add_args_verbosity(optional) - _add_arg_keys(optional) return checks @@ -399,7 +395,7 @@ def _dispatch_fs_copy(args: Args) -> bool: config=args[STR.cfgfile], cycle=args[STR.cycle], leadtime=args[STR.leadtime], - keys=args[STR.keys], + key_path=args[STR.keypath], dry_run=args[STR.dryrun], stdin_ok=True, ) @@ -416,7 +412,7 @@ def _dispatch_fs_link(args: Args) -> bool: config=args[STR.cfgfile], cycle=args[STR.cycle], leadtime=args[STR.leadtime], - keys=args[STR.keys], + key_path=args[STR.keypath], dry_run=args[STR.dryrun], stdin_ok=True, ) @@ -433,7 +429,7 @@ def _dispatch_fs_makedirs(args: Args) -> bool: config=args[STR.cfgfile], cycle=args[STR.cycle], leadtime=args[STR.leadtime], - keys=args[STR.keys], + key_path=args[STR.keypath], dry_run=args[STR.dryrun], stdin_ok=True, ) @@ -756,15 +752,6 @@ def _add_arg_key_path(group: Group, helpmsg: str) -> None: ) -def _add_arg_keys(group: Group) -> None: - group.add_argument( - STR.keys, - help="YAML key leading to file dst/src block", - metavar="KEY", - nargs="*", - ) - - def _add_arg_leadtime(group: Group, required: bool = False) -> None: group.add_argument( _switch(STR.leadtime), @@ -1031,8 +1018,7 @@ def _add_subparser_for_driver_task( _add_arg_graph_file(optional) _add_arg_key_path( optional, - helpmsg="Dot-separated path of keys leading through the config " - "to the driver's configuration block", + helpmsg="Dot-separated path of keys to driver config block", ) _add_arg_schema_file(optional) checks = _add_args_verbosity(optional) diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index 393ed1a44..713c12419 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -13,7 +13,7 @@ from uwtools.strings import FORMAT INCLUDE_TAG = "!include" - +YAMLKey = Union[bool, float, int, str] # Public functions diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index 305e4808a..e200dee89 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -7,7 +7,7 @@ from uwtools.config.formats.base import Config from uwtools.config.jinja2 import unrendered -from uwtools.config.support import depth, format_to_config, log_and_error +from uwtools.config.support import YAMLKey, depth, format_to_config, log_and_error from uwtools.exceptions import UWConfigError, UWConfigRealizeError, UWError from uwtools.logging import log from uwtools.strings import FORMAT @@ -81,7 +81,7 @@ def realize_config( update_format: Optional[str] = None, output_file: Optional[Path] = None, output_format: Optional[str] = None, - key_path: Optional[list[Union[str, int]]] = None, + key_path: Optional[list[YAMLKey]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, @@ -109,19 +109,19 @@ def realize_config( return input_obj.data -def walk_key_path(config: dict, key_path: list[str]) -> tuple[dict, str]: +def walk_key_path(config: dict, key_path: list[YAMLKey]) -> tuple[dict, str]: """ Navigate to the sub-config at the end of the path of given keys. :param config: A config. - :param key_path: Path of keys to subsection of config file. + :param key_path: Path of keys to config block to use. :return: The sub-config and a string representation of the key path. """ keys = [] pathstr = "" for key in key_path: keys.append(key) - pathstr = ".".join(keys) + pathstr = ".".join(str(key) for key in keys) try: subconfig = config[key] except KeyError as e: @@ -158,22 +158,6 @@ def _ensure_format( return fmt -def _print_config_section(config: dict, key_path: list[str]) -> None: - """ - Print the contents of the located subtree as key=value pairs, one per line. - - :param config: A config. - :param key_path: Path of keys to subsection of config file. - """ - config, pathstr = walk_key_path(config, key_path) - output_lines = [] - for key, value in config.items(): - if type(value) not in (bool, float, int, str): - raise log_and_error(f"Non-scalar value {value} found at {pathstr}") - output_lines.append(f"{key}={value}") - print("\n".join(sorted(output_lines))) - - def _realize_config_input_setup( input_config: Optional[Union[Config, Path, dict]] = None, input_format: Optional[str] = None ) -> Config: @@ -197,7 +181,7 @@ def _realize_config_output_setup( input_obj: Config, output_file: Optional[Path] = None, output_format: Optional[str] = None, - key_path: Optional[list[Union[str, int]]] = None, + key_path: Optional[list[YAMLKey]] = None, ) -> tuple[dict, str]: """ Set up config-realize output. @@ -205,7 +189,7 @@ def _realize_config_output_setup( :param input_obj: The input Config object. :param output_file: Output config destination (None => write to stdout). :param output_format: Format of the output config. - :param key_path: Path through keys to the desired output block. + :param key_path: Path of keys to config block to use. :return: The unrealized data to output and the output format name. """ output_format = _ensure_format("output", output_format, output_file) @@ -344,7 +328,7 @@ def _validate_format(other_fmt_desc: str, other_fmt: str, input_fmt: str) -> Non :param update_format: Format of the update config. :param output_file: Output config destination (None => write to ``stdout``). :param output_format: Format of the output config. -:param key_path: Path through keys to the desired output block. +:param key_path: Path of keys to the desired output block. :param values_needed: Report complete, missing, and template values. :param total: Require rendering of all Jinja2 variables/expressions. :param dry_run: Log output instead of writing to output. diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 878e44e8b..28bc8badd 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -18,6 +18,7 @@ from uwtools.config.formats.base import Config from uwtools.config.formats.yaml import YAMLConfig +from uwtools.config.support import YAMLKey from uwtools.config.tools import walk_key_path from uwtools.config.validator import ( bundle, @@ -47,9 +48,9 @@ def __init__( leadtime: Optional[timedelta] = None, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ) -> None: config_input = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) config_input.dereference( @@ -166,7 +167,7 @@ def _create_user_updated_config( else: log.debug(f"Failed to validate {path}") - def _delegate(self, controller: Optional[list[str]], config_key: str) -> None: + def _delegate(self, controller: Optional[list[YAMLKey]], config_key: str) -> None: """ Selectively delegate config to controller. @@ -189,7 +190,7 @@ def driver_name(cls) -> str: """ def namelist_schema( - self, config_keys: Optional[list[str]] = None, schema_keys: Optional[list[str]] = None + self, config_keys: Optional[list[YAMLKey]] = None, schema_keys: Optional[list[str]] = None ) -> dict: """ Return the (sub)schema for validating the driver's namelist content. @@ -251,9 +252,9 @@ def __init__( cycle: datetime, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ): super().__init__( cycle=cycle, @@ -284,9 +285,9 @@ def __init__( leadtime: timedelta, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ): super().__init__( cycle=cycle, @@ -324,9 +325,9 @@ def __init__( self, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ): super().__init__( config=config, @@ -348,10 +349,10 @@ def __init__( leadtime: Optional[timedelta] = None, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ): super().__init__( cycle=cycle, @@ -570,10 +571,10 @@ def __init__( cycle: datetime, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ): super().__init__( cycle=cycle, @@ -605,10 +606,10 @@ def __init__( leadtime: timedelta, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ): super().__init__( cycle=cycle, @@ -647,10 +648,10 @@ def __init__( self, config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[list[str]] = None, + controller: Optional[list[YAMLKey]] = None, ): super().__init__( config=config, @@ -679,7 +680,7 @@ def _add_docstring(class_: type, omit: Optional[list[str]] = None) -> None: :param leadtime: The leadtime. :param config: Path to config file (read stdin if missing or None). :param dry_run: Run in dry-run mode? - :param key_path: Keys leading through the config to the driver's configuration block. + :param key_path: Keys of keys to driver config block. :param batch: Run component via the batch system? :param schema_file: Path to schema file to use to validate an external driver. :param controller: Key(s) leading to block in config controlling run-time values. diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 46bf68244..c492cee4d 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -10,9 +10,10 @@ from iotaa import dryrun, tasks from uwtools.config.formats.yaml import YAMLConfig +from uwtools.config.support import YAMLKey +from uwtools.config.tools import walk_key_path from uwtools.config.validator import validate_internal from uwtools.exceptions import UWConfigError -from uwtools.logging import log from uwtools.strings import STR from uwtools.utils.api import str2path from uwtools.utils.tasks import directory, filecopy, symlink @@ -29,7 +30,7 @@ def __init__( target_dir: Optional[Union[str, Path]] = None, cycle: Optional[dt.datetime] = None, leadtime: Optional[dt.timedelta] = None, - keys: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, dry_run: bool = False, ) -> None: """ @@ -39,12 +40,11 @@ def __init__( :param target_dir: Path to target directory. :param cycle: A ``datetime`` object to make available for use in the config. :param leadtime: A ``timedelta`` object to make available for use in the config. - :param keys: YAML keys leading to file dst/src block. + :param key_path: Path of keys to config block to use. :param dry_run: Do not copy files. :raises: ``UWConfigError`` if config fails validation. """ dryrun(enable=dry_run) - self._keys = keys or [] self._target_dir = str2path(target_dir) yaml_config = YAMLConfig(config=str2path(config)) yaml_config.dereference( @@ -54,8 +54,7 @@ def __init__( **yaml_config.data, } ) - self._config = yaml_config.data - self._set_config_block() + self._config, _ = walk_key_path(yaml_config.data, key_path or []) self._validate() self._check_paths() @@ -72,25 +71,6 @@ def _check_paths(self) -> None: if not Path(dst).is_absolute(): raise UWConfigError(errmsg % dst) - def _set_config_block(self) -> None: - """ - Navigate keys to a config block. - - :raises: UWConfigError if no target directory is specified and a relative path is. - """ - cfg = self._config - nav = [] - for key in self._keys: - nav.append(key) - if key not in cfg: - raise UWConfigError("Failed following YAML key(s): %s" % ".".join(nav)) - log.debug("Following config key '%s'", key) - cfg = cfg[key] - if not isinstance(cfg, dict): - msg = "Expected block not found at key path: %s" % ".".join(self._keys) - raise UWConfigError(msg) - self._config = cfg - @property @abstractmethod def _dst_paths(self) -> list[str]: diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 234ef728a..6e281c457 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -96,7 +96,6 @@ class STR: ioda: str = "ioda" jedi: str = "jedi" keypath: str = "key_path" - keys: str = "keys" keyvalpairs: str = "key_eq_val_pairs" leadtime: str = "leadtime" link: str = "link" diff --git a/src/uwtools/tests/api/test_fs.py b/src/uwtools/tests/api/test_fs.py index 71d48b5cb..858b7b724 100644 --- a/src/uwtools/tests/api/test_fs.py +++ b/src/uwtools/tests/api/test_fs.py @@ -21,7 +21,7 @@ def kwargs(tmp_path): "config": config, "cycle": dt.datetime.now(), "leadtime": dt.timedelta(hours=6), - "keys": ["a", "b"], + "key_path": ["a", "b"], "dry_run": False, } diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 793af23db..6747b6f0b 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -666,65 +666,6 @@ def test__ensure_format_explicitly_specified_with_path(): ) -def test__print_config_section_ini(capsys): - config_obj = INIConfig(fixture_path("simple3.ini")) - section = ["dessert"] - tools._print_config_section(config_obj.data, section) - actual = capsys.readouterr().out - expected = """ - flavor={{ flavor }} - servings=0 - side=False - type=pie - """ - assert actual.strip() == dedent(expected).strip() - - -def test__print_config_section_ini_missing_section(): - config_obj = INIConfig(fixture_path("simple3.ini")) - section = ["sandwich"] - msg = "Bad config path: sandwich" - with raises(UWConfigError) as e: - tools._print_config_section(config_obj.data, section) - assert msg in str(e.value) - - -def test__print_config_section_yaml(capsys): - config_obj = YAMLConfig(fixture_path("FV3_GFS_v16.yaml")) - section = ["sgs_tke", "profile_type"] - tools._print_config_section(config_obj.data, section) - actual = capsys.readouterr().out - expected = """ - name=fixed - surface_value=0.0 - """ - assert actual.strip() == dedent(expected).strip() - - -def test__print_config_section_yaml_for_nonscalar(): - config_obj = YAMLConfig(fixture_path("FV3_GFS_v16.yaml")) - section = ["o3mr"] - with raises(UWConfigError) as e: - tools._print_config_section(config_obj.data, section) - assert "Non-scalar value" in str(e.value) - - -def test__print_config_section_yaml_list(): - config_obj = YAMLConfig(fixture_path("srw_example.yaml")) - section = ["FV3GFS", "nomads", "file_names", "grib2", "anl"] - with raises(UWConfigError) as e: - tools._print_config_section(config_obj.data, section) - assert "must be a dictionary" in str(e.value) - - -def test__print_config_section_yaml_not_dict(): - config_obj = YAMLConfig(fixture_path("FV3_GFS_v16.yaml")) - section = ["sgs_tke", "units"] - with raises(UWConfigError) as e: - tools._print_config_section(config_obj.data, section) - assert "must be a dictionary" in str(e.value) - - def test__realize_config_input_setup_ini_cfgobj(): data = {"section": {"foo": "bar"}} cfgobj = INIConfig(config=data) diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index d64b458c2..6a5248ff5 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -59,7 +59,7 @@ def args_dispatch_fs(): "config_file": "/config/file", "cycle": dt.datetime.now(), "leadtime": dt.timedelta(hours=6), - "keys": ["a", "b"], + "key_path": ["a", "b"], "dry_run": False, "stdin_ok": True, } @@ -401,7 +401,7 @@ def test__dispatch_fs_action(action, args_dispatch_fs): config=args["config_file"], cycle=args["cycle"], leadtime=args["leadtime"], - keys=args["keys"], + key_path=args["key_path"], dry_run=args["dry_run"], stdin_ok=args["stdin_ok"], ) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 6e54dd07d..bb0caf4d7 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -54,7 +54,7 @@ def test_Copier(assets, source): config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() - fs.Copier(target_dir=dstdir, config=config, keys=["a", "b"]).go() + fs.Copier(target_dir=dstdir, config=config, key_path=["a", "b"]).go() assert (dstdir / "foo").is_file() assert (dstdir / "subdir" / "bar").is_file() @@ -63,7 +63,7 @@ def test_Copier_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() - fs.Copier(target_dir=dstdir, config=cfgdict, keys=["a", "b"], dry_run=True).go() + fs.Copier(target_dir=dstdir, config=cfgdict, key_path=["a", "b"], dry_run=True).go() assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() iotaa.dryrun(False) @@ -80,7 +80,7 @@ def test_Copier_no_targetdir_abspath_pass(assets): def test_Copier_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: - fs.Copier(config=cfgdict, keys=["a", "b"]).go() + fs.Copier(config=cfgdict, key_path=["a", "b"]).go() errmsg = "Relative path '%s' requires the target directory to be specified" assert errmsg % "foo" in str(e.value) @@ -89,7 +89,7 @@ def test_Copier_no_targetdir_relpath_fail(assets): def test_FilerStager(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile - assert fs.FileStager(target_dir=dstdir, config=config, keys=["a", "b"]) + assert fs.FileStager(target_dir=dstdir, config=config, key_path=["a", "b"]) @mark.parametrize("source", ("dict", "file")) @@ -98,7 +98,7 @@ def test_Linker(assets, source): config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() - fs.Linker(target_dir=dstdir, config=config, keys=["a", "b"]).go() + fs.Linker(target_dir=dstdir, config=config, key_path=["a", "b"]).go() assert (dstdir / "foo").is_symlink() assert (dstdir / "subdir" / "bar").is_symlink() @@ -108,8 +108,8 @@ def test_Stager__config_block_fail_bad_key_path(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile with raises(UWConfigError) as e: - ConcreteStager(target_dir=dstdir, config=config, keys=["a", "x"]) - assert str(e.value) == "Failed following YAML key(s): a.x" + ConcreteStager(target_dir=dstdir, config=config, key_path=["a", "x"]) + assert str(e.value) == "Bad config path: a.x" @mark.parametrize("val", [None, True, False, "str", 42, 3.14, [], tuple()]) @@ -117,5 +117,5 @@ def test_Stager__config_block_fails_bad_type(assets, val): dstdir, cfgdict, _ = assets cfgdict["a"]["b"] = val with raises(UWConfigError) as e: - ConcreteStager(target_dir=dstdir, config=cfgdict, keys=["a", "b"]) - assert str(e.value) == "Expected block not found at key path: a.b" + ConcreteStager(target_dir=dstdir, config=cfgdict, key_path=["a", "b"]) + assert str(e.value) == "Value at a.b must be a dictionary" diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 38ac2a28f..8ad82ccfa 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Callable, Optional, TypeVar, Union +from uwtools.config.support import YAMLKey from uwtools.drivers.driver import DriverT from uwtools.drivers.support import graph from uwtools.exceptions import UWError @@ -51,7 +52,7 @@ def execute( batch: bool = False, dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: @@ -76,7 +77,7 @@ def execute_cycle( batch: bool = False, dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: @@ -102,7 +103,7 @@ def execute_cycle_leadtime( batch: bool = False, dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: @@ -151,7 +152,7 @@ def _execute( batch: bool = False, # pylint: disable=unused-argument dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, - key_path: Optional[list[str]] = None, + key_path: Optional[list[YAMLKey]] = None, schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: @@ -169,7 +170,7 @@ def _execute( :param batch: Submit run to the batch system? :param dry_run: Do not run the executable, just report what would have been done. :param graph_file: Write Graphviz DOT output here. - :param key_path: Path of keys to subsection of config file. + :param key_path: Path of keys to config block to use. :param schema_file: The JSON Schema file to use for validation. :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. From 2eacebf2047c756992883319c53d22e56555ab57 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Tue, 10 Dec 2024 08:17:55 -0700 Subject: [PATCH 4/6] Implement fs copy from HTTP (#669) --- docs/sections/user_guide/cli/tools/fs.rst | 6 +- .../user_guide/cli/tools/fs/copy-config.yaml | 1 + .../tools/fs/copy-exec-no-target-dir-err.out | 6 +- .../user_guide/cli/tools/fs/copy-exec.out | 39 +++-- .../tools/fs/link-exec-no-target-dir-err.out | 6 +- .../fs/makedirs-exec-no-target-dir-err.out | 6 +- recipe/meta.json | 4 +- recipe/meta.yaml | 1 + src/uwtools/cli.py | 3 +- src/uwtools/fs.py | 60 +++++-- src/uwtools/strings.py | 1 + src/uwtools/tests/api/test_fs.py | 2 +- src/uwtools/tests/test_fs.py | 89 ++++++++++- src/uwtools/tests/utils/test_tasks.py | 148 ++++++++++++++++-- src/uwtools/utils/tasks.py | 90 +++++++++-- 15 files changed, 383 insertions(+), 79 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index cb61dd7bf..153404e98 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -15,6 +15,8 @@ The ``uw`` mode for handling filesystem items (files and directories). The ``copy`` action stages files in a target directory by copying files. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. +Source paths prefixed with ``http://`` or ``https://`` will be copied from their upstream network locations to the local filesystem. + .. literalinclude:: fs/copy-help.cmd :emphasize-lines: 1 .. literalinclude:: fs/copy-help.out @@ -23,7 +25,7 @@ The ``copy`` action stages files in a target directory by copying files. Any ``K Examples ^^^^^^^^ -Given ``copy-config.yaml`` containing +Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths .. literalinclude:: fs/copy-config.yaml :language: yaml @@ -32,7 +34,7 @@ Given ``copy-config.yaml`` containing .. literalinclude:: fs/copy-exec.out :language: text -Here, ``foo`` and ``bar`` are copies of their respective source files. +Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source. The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-config.yaml index 17c45a3e6..98d02f202 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-config.yaml +++ b/docs/sections/user_guide/cli/tools/fs/copy-config.yaml @@ -1,4 +1,5 @@ config: files: foo: src/foo + licenses/gpl: https://www.gnu.org/licenses/gpl-3.0.txt subdir/bar: src/bar diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out index 71b4ca3a1..17a9f799f 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-26T23:03:40] INFO Validating config against internal schema: files-to-stage -[2024-08-26T23:03:40] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:40] ERROR Relative path 'foo' requires the target directory to be specified +[2024-12-07T01:01:51] INFO Validating config against internal schema: files-to-stage +[2024-12-07T01:01:53] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:53] ERROR Relative path 'foo' requires target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-exec.out index 57221e756..46a06048e 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec.out @@ -1,22 +1,29 @@ -[2024-08-26T23:03:41] INFO Validating config against internal schema: files-to-stage -[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:41] INFO File copies: Initial state: Not Ready -[2024-08-26T23:03:41] INFO File copies: Checking requirements -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Checking requirements -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Executing -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Final state: Ready -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Executing -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready -[2024-08-26T23:03:41] INFO File copies: Final state: Ready +[2024-12-07T01:01:56] INFO Validating config against internal schema: files-to-stage +[2024-12-07T01:01:56] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:56] INFO File copies: Initial state: Not Ready +[2024-12-07T01:01:56] INFO File copies: Checking requirements +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Checking requirements +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Executing +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Final state: Ready +[2024-12-07T01:01:56] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Initial state: Not Ready +[2024-12-07T01:01:56] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Checking requirements +[2024-12-07T01:01:58] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Requirement(s) ready +[2024-12-07T01:01:58] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Executing +[2024-12-07T01:01:58] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Final state: Ready +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Executing +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready +[2024-12-07T01:01:58] INFO File copies: Final state: Ready copy-dst ├── foo +├── licenses +│   └── gpl └── subdir └── bar -2 directories, 2 files +3 directories, 3 files diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out index dcb5593ed..7118df558 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-26T23:03:41] INFO Validating config against internal schema: files-to-stage -[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:41] ERROR Relative path 'foo' requires the target directory to be specified +[2024-12-07T01:01:55] INFO Validating config against internal schema: files-to-stage +[2024-12-07T01:01:55] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:55] ERROR Relative path 'foo' requires target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out index 84c7710bf..63c47798e 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-26T23:03:44] INFO Validating config against internal schema: makedirs -[2024-08-26T23:03:45] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:45] ERROR Relative path 'foo' requires the target directory to be specified +[2024-12-07T01:01:55] INFO Validating config against internal schema: makedirs +[2024-12-07T01:01:55] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:55] ERROR Relative path 'foo' requires target directory to be specified diff --git a/recipe/meta.json b/recipe/meta.json index 8cfa7a58f..cdccd2d82 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -22,6 +22,7 @@ "pytest-xdist =3.6.*", "python >=3.9,<3.13", "pyyaml =6.0.*", + "requests =2.32.*", "setuptools" ], "run": [ @@ -31,7 +32,8 @@ "jsonschema >=4.18,<4.24", "lxml =5.3.*", "python >=3.9,<3.13", - "pyyaml =6.0.*" + "pyyaml =6.0.*", + "requests =2.32.*" ] }, "version": "2.5.0" diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 9a9dfd0d9..83e95675d 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -21,6 +21,7 @@ requirements: - lxml 5.3.* - python >=3.9,<3.13 - pyyaml 6.0.* + - requests 2.32.* test: requires: - black 24.8.* diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 036b04cbf..84c162f1c 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -94,7 +94,8 @@ def main() -> None: modes = {**tools, **drivers} sys.exit(0 if modes[args[STR.mode]](args) else 1) except UWError as e: - log.error(str(e)) + for line in str(e).split("\n"): + log.error(line) sys.exit(1) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index c492cee4d..f7d214a49 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import Optional, Union +from urllib.parse import urlparse from iotaa import dryrun, tasks @@ -56,20 +57,44 @@ def __init__( ) self._config, _ = walk_key_path(yaml_config.data, key_path or []) self._validate() - self._check_paths() + self._check_target_dir() + self._check_destination_paths() - def _check_paths(self) -> None: + def _check_destination_paths(self) -> None: """ - Check that all paths are absolute if no target directory is specified. + Check that destination paths are valid. - :parm paths: The paths to check. - :raises: UWConfigError if no target directory is specified and a relative path is. + :raises: UWConfigError when a bad path is detected. """ - if not self._target_dir: - errmsg = "Relative path '%s' requires the target directory to be specified" - for dst in self._dst_paths: - if not Path(dst).is_absolute(): - raise UWConfigError(errmsg % dst) + for dst in self._dst_paths: + scheme = urlparse(dst).scheme + absolute = scheme or Path(dst).is_absolute() + if scheme and scheme != STR.url_scheme_file: + msg = "Non-filesystem destination path '%s' not currently supported" + raise UWConfigError(msg % dst) + if self._target_dir and scheme: + msg = "Non-filesystem path '%s' invalid when target directory is specified" + raise UWConfigError(msg % dst) + if self._target_dir and absolute: + msg = "Path '%s' must be relative when target directory is specified" + raise UWConfigError(msg % dst) + if not self._target_dir and not absolute: + msg = "Relative path '%s' requires target directory to be specified" + raise UWConfigError(msg % dst) + + def _check_target_dir(self) -> None: + """ + Check that target directory is valid. + + :raises: UWConfigError when a bad path is detected. + """ + if ( + self._target_dir + and (scheme := urlparse(str(self._target_dir)).scheme) + and scheme != STR.url_scheme_file + ): + msg = "Non-filesystem path '%s' invalid as target directory" + raise UWConfigError(msg % self._target_dir) @property @abstractmethod @@ -124,9 +149,20 @@ def go(self): """ Copy files. """ - dst = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File copies" - yield [filecopy(src=Path(v), dst=dst(k)) for k, v in self._config.items()] + yield [ + filecopy(src=src, dst=self._simple(self._target_dir) / self._simple(dst)) + for dst, src in self._config.items() + ] + + @staticmethod + def _simple(path: Union[Path, str]) -> Path: + """ + Convert a path, potentially prefixed with scheme file://, into a simple filesystem path. + + :param path: The path to convert. + """ + return Path(urlparse(str(path)).path) class Linker(FileStager): diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 6e281c457..fb941a547 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -142,6 +142,7 @@ class STR: updatefmt: str = "update_format" updatevalues: str = "update_values" upp: str = "upp" + url_scheme_file: str = "file" validate: str = "validate" valsfile: str = "values_file" valsfmt: str = "values_format" diff --git a/src/uwtools/tests/api/test_fs.py b/src/uwtools/tests/api/test_fs.py index 858b7b724..643d1ce4a 100644 --- a/src/uwtools/tests/api/test_fs.py +++ b/src/uwtools/tests/api/test_fs.py @@ -17,7 +17,7 @@ def kwargs(tmp_path): f.touch() config = {"a": {"b": {str(dstdir / "f1"): str(srcfile1), str(dstdir / "f2"): str(srcfile2)}}} return { - "target_dir": dstdir, + "target_dir": None, "config": config, "cycle": dt.datetime.now(), "leadtime": dt.timedelta(hours=6), diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index bb0caf4d7..227c52764 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -1,7 +1,11 @@ # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring +# pylint: disable=protected-access # pylint: disable=redefined-outer-name +from pathlib import Path +from unittest.mock import Mock, patch + import iotaa import yaml from pytest import fixture, mark, raises @@ -48,8 +52,19 @@ def _schema(self): # Tests +@mark.parametrize("src_fn", [str, Path]) +@mark.parametrize("dst_fn", [str, Path]) +@mark.parametrize("td_fn", [str, Path]) +def test_fs_Copier_go(src_fn, dst_fn, td_fn): + src, td, dst = src_fn("/src/file"), td_fn("/dst"), dst_fn("file") + obj = Mock(_config={dst: src}, _simple=fs.Copier._simple, _target_dir=td) + with patch.object(fs, "filecopy") as filecopy: + fs.Copier.go(obj) + filecopy.assert_called_once_with(src=src, dst=Path("/dst/file")) + + @mark.parametrize("source", ("dict", "file")) -def test_Copier(assets, source): +def test_fs_Copier_go_live(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() @@ -59,7 +74,7 @@ def test_Copier(assets, source): assert (dstdir / "subdir" / "bar").is_file() -def test_Copier_config_file_dry_run(assets): +def test_fs_Copier_go_live_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() @@ -69,7 +84,7 @@ def test_Copier_config_file_dry_run(assets): iotaa.dryrun(False) -def test_Copier_no_targetdir_abspath_pass(assets): +def test_fs_Copier_go_live_no_targetdir_abspath_pass(assets): dstdir, cfgdict, _ = assets old = cfgdict["a"]["b"] cfgdict = {str(dstdir / "foo"): old["foo"], str(dstdir / "bar"): old["subdir/bar"]} @@ -81,19 +96,26 @@ def test_Copier_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: fs.Copier(config=cfgdict, key_path=["a", "b"]).go() - errmsg = "Relative path '%s' requires the target directory to be specified" + errmsg = "Relative path '%s' requires target directory to be specified" assert errmsg % "foo" in str(e.value) +def test_fs_Copier__simple(): + assert fs.Copier._simple("relative/path") == Path("relative/path") + assert fs.Copier._simple("/absolute/path") == Path("/absolute/path") + assert fs.Copier._simple("file:///absolute/path") == Path("/absolute/path") + assert fs.Copier._simple("") == Path("") + + @mark.parametrize("source", ("dict", "file")) -def test_FilerStager(assets, source): +def test_fs_FilerStager(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert fs.FileStager(target_dir=dstdir, config=config, key_path=["a", "b"]) @mark.parametrize("source", ("dict", "file")) -def test_Linker(assets, source): +def test_fs_Linker(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() @@ -103,8 +125,59 @@ def test_Linker(assets, source): assert (dstdir / "subdir" / "bar").is_symlink() +@mark.parametrize( + "path,target_dir,msg,fail_expected", + [ + ( + "/other/path", + "/some/path", + "Path '%s' must be relative when target directory is specified", + True, + ), + ( + "foo://bucket/a/b", + None, + "Non-filesystem destination path '%s' not currently supported", + True, + ), + ( + "relpath", + None, + "Relative path '%s' requires target directory to be specified", + True, + ), + ( + "file://foo.com/a/b", + "/some/path", + "Non-filesystem path '%s' invalid when target directory is specified", + True, + ), + ("other/path", "/some/path", None, False), + ("other/path", "file:///some/path", None, False), + ], +) +def test_fs_Stager__check_destination_paths_fail(path, target_dir, msg, fail_expected): + obj = Mock(_dst_paths=[path], _target_dir=target_dir) + if fail_expected: + with raises(UWConfigError) as e: + fs.Stager._check_destination_paths(obj) + assert str(e.value) == msg % path + + +@mark.parametrize( + "path,fail_expected", + [("foo://bucket/a/b", True), ("/some/path", False), ("file:///some/path", False)], +) +def test_fs_Stager__check_target_dir_fail_bad_scheme(path, fail_expected): + obj = Mock(_target_dir="foo://bucket/a/b") + if fail_expected: + with raises(UWConfigError) as e: + fs.Stager._check_target_dir(obj) + assert str(e.value) == "Non-filesystem path '%s' invalid as target directory" % path + + @mark.parametrize("source", ("dict", "file")) -def test_Stager__config_block_fail_bad_key_path(assets, source): +def test_fs_Stager__config_block_fail_bad_key_path(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile with raises(UWConfigError) as e: @@ -113,7 +186,7 @@ def test_Stager__config_block_fail_bad_key_path(assets, source): @mark.parametrize("val", [None, True, False, "str", 42, 3.14, [], tuple()]) -def test_Stager__config_block_fails_bad_type(assets, val): +def test_fs_Stager__config_block_fails_bad_type(assets, val): dstdir, cfgdict, _ = assets cfgdict["a"]["b"] = val with raises(UWConfigError) as e: diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index aa606ed09..1fe044bb4 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -1,14 +1,29 @@ -# pylint: disable=missing-function-docstring +# pylint: disable=missing-function-docstring,protected-access +import logging import os import stat -from unittest.mock import patch +from pathlib import Path +from typing import Union +from unittest.mock import Mock, patch +from iotaa import asset, external +from pytest import mark, raises + +from uwtools.exceptions import UWConfigError +from uwtools.logging import log +from uwtools.tests.support import logged from uwtools.utils import tasks # Helpers +@external +def exists(x): + yield x + yield asset(x, lambda: True) + + def ready(taskval): return taskval.ready() @@ -37,37 +52,75 @@ def test_tasks_executable(tmp_path): assert ready(tasks.executable(program=p)) -def test_tasks_existing_missing(tmp_path): - path = tmp_path / "x" +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_existing_local_missing(caplog, prefix, tmp_path): + log.setLevel(logging.INFO) + base = tmp_path / "x" + path = prefix + str(base) if prefix else base assert not ready(tasks.existing(path=path)) + assert logged(caplog, "Filesystem item %s: State: Not Ready (external asset)" % base) -def test_tasks_existing_present_directory(tmp_path): +def test_tasks_existing_local_present_directory(caplog, tmp_path): + log.setLevel(logging.INFO) path = tmp_path / "directory" path.mkdir() assert ready(tasks.existing(path=path)) + assert logged(caplog, "Filesystem item %s: State: Ready" % path) -def test_tasks_existing_present_file(tmp_path): - path = tmp_path / "file" - path.touch() +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_existing_local_present_file(caplog, prefix, tmp_path): + log.setLevel(logging.INFO) + base = tmp_path / "file" + base.touch() + path = prefix + str(base) if prefix else base assert ready(tasks.existing(path=path)) + assert logged(caplog, "Filesystem item %s: State: Ready" % base) -def test_tasks_existing_present_symlink(tmp_path): - path = tmp_path / "symlink" - path.symlink_to(os.devnull) +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_existing_local_present_symlink(caplog, prefix, tmp_path): + log.setLevel(logging.INFO) + base = tmp_path / "symlink" + base.symlink_to(os.devnull) + path = prefix + str(base) if prefix else base assert ready(tasks.existing(path=path)) + assert logged(caplog, "Filesystem item %s: State: Ready" % base) + + +@mark.parametrize("scheme", ["http", "https"]) +@mark.parametrize("code,expected", [(200, True), (404, False)]) +def test_tasks_existing_remote(caplog, code, expected, scheme): + log.setLevel(logging.INFO) + path = f"{scheme}://foo.com/obj" + with patch.object(tasks.requests, "head", return_value=Mock(status_code=code)) as head: + state = ready(tasks.existing(path=path)) + assert state is expected + head.assert_called_with(path, allow_redirects=True, timeout=3) + msg = "Remote object %s: State: %s" % (path, "Ready" if state else "Not Ready (external asset)") + assert logged(caplog, msg) + +def test_tasks_existing_bad_scheme(): + path = "foo://bucket/a/b" + with raises(UWConfigError) as e: + tasks.existing(path=path) + assert str(e.value) == f"Scheme 'foo' in '{path}' not supported" -def test_tasks_file_missing(tmp_path): + +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_file_missing(prefix, tmp_path): path = tmp_path / "file" + path = "%s%s" % (prefix, path) if prefix else path assert not ready(tasks.file(path=path)) -def test_tasks_file_present(tmp_path): +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_file_present(prefix, tmp_path): path = tmp_path / "file" path.touch() + path = "%s%s" % (prefix, path) if prefix else path assert ready(tasks.file(path=path)) @@ -89,19 +142,80 @@ def test_tasks_filecopy_directory_hierarchy(tmp_path): assert dst.is_file() -def test_tasks_symlink_simple(tmp_path): +@mark.parametrize("code,expected", [(200, True), (404, False)]) +@mark.parametrize("src", ["http://foo.com/obj", "https://foo.com/obj"]) +def test_tasks_filecopy_source_http(code, expected, src, tmp_path): + log.setLevel(logging.INFO) + dst = tmp_path / "a-file" + assert not dst.is_file() + with patch.object(tasks, "existing", exists): + with patch.object(tasks, "requests") as requests: + response = requests.get() + response.status_code = code + response.content = "data".encode("utf-8") + tasks.filecopy(src=src, dst=dst) + requests.get.assert_called_with(src, allow_redirects=True, timeout=3) + assert dst.is_file() is expected + + +@mark.parametrize( + "src,ok", + [("/src/file", True), ("file:///src/file", True), ("foo://bucket/a/b", False)], +) +def test_tasks_filecopy_source_local(src, ok): + dst = "/dst/file" + with patch.object(tasks.Path, "mkdir") as mkdir: + if ok: + with patch.object(tasks, "file", exists): + with patch.object(tasks, "copy") as copy: + tasks.filecopy(src=src, dst=dst) + mkdir.assert_called_once_with(parents=True, exist_ok=True) + copy.assert_called_once_with(Path("/src/file"), Path(dst)) + else: + with raises(UWConfigError) as e: + tasks.filecopy(src=src, dst=dst) + assert str(e.value) == f"Scheme 'foo' in '{src}' not supported" + + +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_symlink_simple(prefix, tmp_path): target = tmp_path / "target" link = tmp_path / "link" target.touch() assert not link.is_file() - tasks.symlink(target=target, linkname=link) + t2, l2 = ["%s%s" % (prefix, x) if prefix else x for x in (target, link)] + tasks.symlink(target=t2, linkname=l2) assert link.is_symlink() -def test_tasks_symlink_directory_hierarchy(tmp_path): +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_symlink_directory_hierarchy(prefix, tmp_path): target = tmp_path / "target" link = tmp_path / "foo" / "bar" / "link" target.touch() assert not link.is_file() - tasks.symlink(target=target, linkname=link) + t2, l2 = ["%s%s" % (prefix, x) if prefix else x for x in (target, link)] + tasks.symlink(target=t2, linkname=l2) assert link.is_symlink() + + +def test__bad_scheme(): + path = "foo://bucket/a/b" + with raises(UWConfigError) as e: + tasks.existing(path=path) + assert str(e.value) == f"Scheme 'foo' in '{path}' not supported" + + +def test__local_path_fail(): + path = "foo://bucket/a/b" + with patch.object(tasks, "_bad_scheme") as _bad_scheme: + tasks._local_path(path) + _bad_scheme.assert_called_once_with(path, "foo") + + +@mark.parametrize("prefix", ["", "file://"]) +@mark.parametrize("wrapper", [str, Path]) +def test__local_path_pass(prefix, wrapper): + path = "/some/file" + p2: Union[str, Path] = str(f"{prefix}{path}") if wrapper == str else Path(path) + assert tasks._local_path(p2) == Path(path) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 7659fe2be..de66e3c9b 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -5,10 +5,18 @@ import os from pathlib import Path from shutil import copy, which -from typing import Union +from types import SimpleNamespace as ns +from typing import NoReturn, Union +from urllib.parse import urlparse +import requests from iotaa import asset, external, task +from uwtools.exceptions import UWConfigError +from uwtools.logging import log + +SCHEMES = ns(http=("http", "https"), local=("", "file")) + @task def directory(path: Path): @@ -35,52 +43,83 @@ def executable(program: Union[Path, str]): @external -def existing(path: Path): +def existing(path: Union[Path, str]): """ - An existing filesystem item (file, directory, or symlink). + An existing file, directory, symlink, or remote object. :param path: Path to the item. + :raises: UWConfigError for unsupported URL schemes. """ - yield "Filesystem item %s" % path - yield asset(path, path.exists) + info = urlparse(str(path)) + scheme = info.scheme + if scheme in SCHEMES.local: + path = _local_path(path) + yield "Filesystem item %s" % path + yield asset(path, path.exists) + elif scheme in SCHEMES.http: + path = str(path) + ready = lambda: requests.head(path, allow_redirects=True, timeout=3).status_code == 200 + yield "Remote object %s" % path + yield asset(path, ready) + else: + _bad_scheme(path, scheme) @external -def file(path: Path, context: str = ""): +def file(path: Union[Path, str], context: str = ""): """ An existing file or symlink to an existing file. :param path: Path to the file. :param context: Optional additional context for the file. """ + path = _local_path(path) suffix = f" ({context})" if context else "" yield "File %s%s" % (path, suffix) yield asset(path, path.is_file) @task -def filecopy(src: Path, dst: Path): +def filecopy(src: Union[Path, str], dst: Union[Path, str]): """ A copy of an existing file. :param src: Path to the source file. :param dst: Path to the destination file to create. + :raises: UWConfigError for unsupported URL schemes. """ yield "Copy %s -> %s" % (src, dst) - yield asset(dst, dst.is_file) - yield file(src) - dst.parent.mkdir(parents=True, exist_ok=True) - copy(src, dst) + yield asset(Path(dst), Path(dst).is_file) + dst = _local_path(dst) # currently no support for remote destinations + src_scheme = urlparse(str(src)).scheme + if src_scheme in SCHEMES.local: + src = _local_path(src) + yield file(src) + dst.parent.mkdir(parents=True, exist_ok=True) + copy(src, dst) + elif src_scheme in SCHEMES.http: + src = str(src) + yield existing(src) + dst.parent.mkdir(parents=True, exist_ok=True) + response = requests.get(src, allow_redirects=True, timeout=3) + if (code := response.status_code) == 200: + with open(dst, "wb") as f: + f.write(response.content) + else: + log.error("Could not get '%s', HTTP status was: %s", src, code) + else: + _bad_scheme(src, src_scheme) @task -def symlink(target: Path, linkname: Path): +def symlink(target: Union[Path, str], linkname: Union[Path, str]): """ A symbolic link. :param target: The existing file or directory. :param linkname: The symlink to create. """ + target, linkname = map(_local_path, [target, linkname]) yield "Link %s -> %s" % (linkname, target) yield asset(linkname, linkname.exists) yield existing(target) @@ -89,3 +128,30 @@ def symlink(target: Path, linkname: Path): src=target if target.is_absolute() else os.path.relpath(target, linkname.parent), dst=linkname, ) + + +# Private helpers + + +def _bad_scheme(path: Union[Path, str], scheme: str) -> NoReturn: + """ + Fail on an unsupported URL scheme. + + :param path: The path with a bad scheme. + :param scheme: The scheme. + :raises: UWConfigError. + """ + raise UWConfigError(f"Scheme '{scheme}' in '{path}' not supported") + + +def _local_path(path: Union[Path, str]) -> Path: + """ + Ensure path is local and return simple version. + + :param path: The local path to check. + :raises: UWConfigError if a non-local scheme is specified. + """ + info = urlparse(str(path)) + if info.scheme and info.scheme not in SCHEMES.local: + _bad_scheme(path, info.scheme) + return Path(info.path) From 0a9813d54751ca2064361694bf6153b15789311b Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:13:41 -0700 Subject: [PATCH 5/6] Render fix (#671) --- src/uwtools/config/formats/base.py | 4 +- src/uwtools/config/jinja2.py | 54 ++++++++++++++----------- src/uwtools/tests/config/test_jinja2.py | 27 +++++++++---- 3 files changed, 53 insertions(+), 32 deletions(-) diff --git a/src/uwtools/config/formats/base.py b/src/uwtools/config/formats/base.py index 436a3ea98..8c64b7cc0 100644 --- a/src/uwtools/config/formats/base.py +++ b/src/uwtools/config/formats/base.py @@ -216,9 +216,9 @@ def dereference(self, context: Optional[dict] = None) -> None: """ def logstate(state: str) -> None: - log.debug("Dereferencing, %s value:", state) + jinja2.deref_debug("Dereferencing, %s value:" % state) for line in yaml_to_str(self.data).split("\n"): - log.debug("%s%s", INDENT, line) + jinja2.deref_debug("%s%s" % (INDENT, line)) while True: logstate("current") diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index cba03bae0..0bce1dea2 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -13,7 +13,6 @@ from jinja2.exceptions import UndefinedError from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config, uw_yaml_loader -from uwtools.exceptions import UWConfigRealizeError from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable @@ -130,25 +129,35 @@ def dereference( rendered = {} for k, v in val.items(): if isinstance(v, UWYAMLRemove): - _deref_debug("Removing value at", ".".join([*keys, k])) + deref_debug("Removing value at", ".".join([*keys, k])) else: kd, vd = [dereference(x, context, val, [*keys, k]) for x in (k, v)] rendered[kd] = vd elif isinstance(val, list): rendered = [dereference(v, context) for v in val] elif isinstance(val, str): - _deref_debug("Rendering", val) + deref_debug("Rendering", val) rendered = _deref_render(val, context, local) elif isinstance(val, UWYAMLConvert): - _deref_debug("Rendering", val.value) + deref_debug("Rendering", val.value) val.value = _deref_render(val.value, context, local) rendered = _deref_convert(val) else: - _deref_debug("Accepting", val) + deref_debug("Accepting", val) rendered = val return rendered +def deref_debug(action: str, val: Optional[_ConfigVal] = "") -> None: + """ + Log a debug-level message related to dereferencing. + + :param action: The dereferencing activity being performed. + :param val: The value being dereferenced. + """ + log.debug("[dereference] %s: %s", action, val) + + def render( values_src: Optional[Union[dict, Path]] = None, values_format: Optional[str] = None, @@ -234,25 +243,15 @@ def _deref_convert(val: UWYAMLConvert) -> _ConfigVal: :return: The value translated to the specified type. """ converted: _ConfigVal = val # fall-back value - _deref_debug("Converting", val.value) + deref_debug("Converting", val.value) try: converted = val.convert() - _deref_debug("Converted", converted) + deref_debug("Converted", converted) except Exception as e: # pylint: disable=broad-exception-caught - _deref_debug("Conversion failed", str(e)) + deref_debug("Conversion failed", str(e)) return converted -def _deref_debug(action: str, val: _ConfigVal) -> None: - """ - Log a debug-level message related to dereferencing. - - :param action: The dereferencing activity being performed. - :param val: The value being dereferenced. - """ - log.debug("[dereference] %s: %s", action, val) - - def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: """ Render a Jinja2 variable/expression as part of dereferencing. @@ -269,13 +268,22 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: context = {**(local or {}), **context} try: rendered = _register_filters(env).from_string(val).render(context) - if isinstance(yaml.load(rendered, Loader=uw_yaml_loader()), UWYAMLConvert): - _deref_debug("Held", rendered) - raise UWConfigRealizeError() - _deref_debug("Rendered", rendered) + deref_debug("Rendered", rendered) + except Exception as e: # pylint: disable=broad-exception-caught + rendered = val + deref_debug("Rendering failed", val) + for line in str(e).split("\n"): + deref_debug(line) + try: + loaded = yaml.load(rendered, Loader=uw_yaml_loader()) except Exception as e: # pylint: disable=broad-exception-caught + loaded = None + deref_debug("Loading rendered value as YAML", rendered) + for line in str(e).split("\n"): + deref_debug(line) + if isinstance(loaded, UWYAMLConvert): rendered = val - _deref_debug("Rendering failed", str(e)) + deref_debug("Held", rendered) return rendered diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index 2985addc1..10c3c0f7c 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -167,6 +167,12 @@ def test_dereference_str_variable_rendered_str(): assert jinja2.dereference(val=val, context={"greeting": "hello"}) == "hello" +def test_deref_debug(caplog): + log.setLevel(logging.DEBUG) + jinja2.deref_debug(action="Frobnicated", val="foo") + assert logged(caplog, "[dereference] Frobnicated: foo") + + def test_register_filters_env(): s = "hello {{ 'RECIPIENT' | env }}" template = jinja2._register_filters(Environment(undefined=DebugUndefined)).from_string(s) @@ -309,20 +315,16 @@ def test__deref_convert_ok(caplog, converted, tag, value): assert not regex_logged(caplog, "Conversion failed") -def test__deref_debug(caplog): - log.setLevel(logging.DEBUG) - jinja2._deref_debug(action="Frobnicated", val="foo") - assert logged(caplog, "[dereference] Frobnicated: foo") - - def test__deref_render_held(caplog): + log.setLevel(logging.DEBUG) val, context = "!int '{{ a }}'", yaml.load("a: !int '{{ 42 }}'", Loader=uw_yaml_loader()) assert jinja2._deref_render(val=val, context=context) == val - assert not regex_logged(caplog, "Rendered") + assert regex_logged(caplog, "Rendered") assert regex_logged(caplog, "Held") def test__deref_render_no(caplog, deref_render_assets): + log.setLevel(logging.DEBUG) val, context, _ = deref_render_assets assert jinja2._deref_render(val=val, context=context) == val assert not regex_logged(caplog, "Rendered") @@ -330,19 +332,30 @@ def test__deref_render_no(caplog, deref_render_assets): def test__deref_render_ok(caplog, deref_render_assets): + log.setLevel(logging.DEBUG) val, context, local = deref_render_assets assert jinja2._deref_render(val=val, context=context, local=local) == "hello world" assert regex_logged(caplog, "Rendered") assert not regex_logged(caplog, "Rendering failed") +def test__deref_render_unloadable_val(caplog): + log.setLevel(logging.DEBUG) + val = "&XMLENTITY;" + assert jinja2._deref_render(val='{{ "%s" if True }}' % val, context={}) == val + assert regex_logged(caplog, "Rendered") + assert not regex_logged(caplog, "Rendering failed") + + def test__dry_run_template(caplog): + log.setLevel(logging.DEBUG) jinja2._dry_run_template("roses are red\nviolets are blue") assert logged(caplog, "roses are red") assert logged(caplog, "violets are blue") def test__log_missing_values(caplog): + log.setLevel(logging.DEBUG) missing = ["roses_color", "violets_color"] jinja2._log_missing_values(missing) assert logged(caplog, "Value(s) required to render template not provided:") From 05dea64b354bf91b401c67170499483134e3f452 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Wed, 11 Dec 2024 22:49:55 -0700 Subject: [PATCH 6/6] Refresh Anaconda badges on GitHub page (#673) --- .github/scripts/install-conda.sh | 2 +- .github/scripts/publish.sh | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/scripts/install-conda.sh b/.github/scripts/install-conda.sh index ae338fda2..b7562179b 100755 --- a/.github/scripts/install-conda.sh +++ b/.github/scripts/install-conda.sh @@ -6,4 +6,4 @@ wget --no-verbose -O $installer $url bash $installer -bfp $CI_CONDA_DIR set +ux ci_conda_activate -conda install --quiet --yes --channel maddenp --repodata-fn repodata.json anaconda-client condev +conda install --quiet --yes --channel maddenp --repodata-fn repodata.json anaconda-client condev jq diff --git a/.github/scripts/publish.sh b/.github/scripts/publish.sh index e86831ab9..ef39948e5 100755 --- a/.github/scripts/publish.sh +++ b/.github/scripts/publish.sh @@ -7,3 +7,11 @@ glob="$(jq -r .name $f)-$(jq -r .version $f)-*_$(jq -r .buildnum $f).tar.bz2" for x in $(find $CI_CONDA_DIR/conda-bld -type f -name "$glob"); do anaconda -t $ANACONDA_TOKEN upload $x done +# Refresh Anaconda badges on GitHub. +ids=( + 6c9ed105fbe1d674e82460d5e7fa6c7eb8e2eb6eb3640c763a8189f407e2a9a2/68747470733a2f2f616e61636f6e64612e6f72672f7566732d636f6d6d756e6974792f7577746f6f6c732f6261646765732f76657273696f6e2e737667 + a6f1f3ab481647dc492ab577cb7e60522efded549caf0544ba863d0a72958179/68747470733a2f2f616e61636f6e64612e6f72672f7566732d636f6d6d756e6974792f7577746f6f6c732f6261646765732f6c61746573745f72656c656173655f646174652e737667 +) +for id in ${ids[*]}; do + curl -s -X PURGE https://camo.githubusercontent.com/$id | jq . +done