From 18034480cc32f73f61c3e2208fdbc4c854066be2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 04:59:08 +0000 Subject: [PATCH 01/25] Extend schema to permit UWYAMLGlob objects as fs src values --- src/uwtools/config/validator.py | 12 +++++++++--- .../resources/jsonschema/files-to-stage.jsonschema | 2 +- src/uwtools/tests/test_schemas.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 0e8910ded..31b378dff 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -7,12 +7,13 @@ from pathlib import Path from typing import Optional, Union -from jsonschema import Draft202012Validator +from jsonschema import Draft202012Validator, validators from jsonschema.exceptions import ValidationError from referencing import Registry, Resource from referencing.jsonschema import DRAFT202012 from uwtools.config.formats.yaml import YAMLConfig +from uwtools.config.support import UWYAMLGlob from uwtools.exceptions import UWConfigError from uwtools.logging import INDENT, log from uwtools.utils.file import resource_path @@ -150,7 +151,7 @@ def validate_external( else: config = config_data if not str(schema_file).startswith(str(resource_path())): - log.debug("Using schema file: %s", schema_file) + log.debug("Validating config against external schema file: %s", schema_file) with open(schema_file, "r", encoding="utf-8") as f: schema = json.load(f) if not validate(schema=schema, desc=desc, config=config): @@ -184,5 +185,10 @@ def _validation_errors(config: JSONValueT, schema: dict) -> list[ValidationError :param schema: JSON Schema to validate the config against. :return: Any validation errors. """ - validator = Draft202012Validator(schema, registry=_registry()) + base = Draft202012Validator + type_checker = base.TYPE_CHECKER.redefine( + "fs_src", lambda _, x: any(isinstance(x, t) for t in [str, UWYAMLGlob]) + ) + UWValidator = validators.extend(base, type_checker=type_checker) + validator = UWValidator(schema, registry=_registry()) return list(validator.iter_errors(config)) diff --git a/src/uwtools/resources/jsonschema/files-to-stage.jsonschema b/src/uwtools/resources/jsonschema/files-to-stage.jsonschema index 7c83c61f3..26ca12529 100644 --- a/src/uwtools/resources/jsonschema/files-to-stage.jsonschema +++ b/src/uwtools/resources/jsonschema/files-to-stage.jsonschema @@ -1,6 +1,6 @@ { "additionalProperties": { - "type": "string" + "type": "fs_src" }, "minProperties": 1, "type": "object" diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 8c0fefbf7..751d0e21e 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -768,7 +768,7 @@ def test_schema_stage_files(): # An empty dict is not allowed: assert non_empty_dict(errors({})) # Non-string values are not allowed: - assert "True is not of type 'string'\n" in errors({"file1": True}) + assert "True is not of type 'fs_src'\n" in errors({"file1": True}) # filter-topo From 35576960032129e6bbde1f038763d825b46a067b Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 05:24:16 +0000 Subject: [PATCH 02/25] WIP, works but tests fail --- src/uwtools/fs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 1edeeb12f..9350cbcb2 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -4,6 +4,7 @@ import datetime as dt from abc import ABC, abstractmethod +from glob import glob from pathlib import Path from typing import Optional, Union from urllib.parse import urlparse @@ -11,7 +12,7 @@ from iotaa import tasks from uwtools.config.formats.yaml import YAMLConfig -from uwtools.config.support import YAMLKey +from uwtools.config.support import UWYAMLGlob, YAMLKey from uwtools.config.tools import walk_key_path from uwtools.config.validator import validate_internal from uwtools.exceptions import UWConfigError @@ -136,6 +137,18 @@ def _dst_paths(self) -> list[str]: """ return list(self._config.keys()) + def _expand_wildcards(self) -> list[tuple[str, str]]: + items = [] + for dst, src in self._config.items(): + assert isinstance(src, (str, UWYAMLGlob)) + if isinstance(src, UWYAMLGlob): + d = Path(dst).parent + for path in map(Path, glob(src.value)): + items.append((str(d / path.name), str(path))) + else: + items.append((dst, src)) + return items + @property def _schema(self) -> str: """ @@ -157,7 +170,7 @@ def go(self): yield "File copies" yield [ filecopy(src=src, dst=self._simple(self._target_dir) / self._simple(dst)) - for dst, src in self._config.items() + for dst, src in self._expand_wildcards() ] @staticmethod From 8b63493b80e4c7d171f76f6964fc4242b71b4453 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 05:31:31 +0000 Subject: [PATCH 03/25] Tests pass, at 99% --- src/uwtools/tests/test_fs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index f923520bd..3a84db539 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -53,12 +53,13 @@ def _schema(self): # Tests -@mark.parametrize("src_fn", [str, Path]) -@mark.parametrize("dst_fn", [str, Path]) -@mark.parametrize("td_fn", [str, Path]) -def test_fs_Copier_go(src_fn, dst_fn, td_fn): - src, td, dst = src_fn("/src/file"), td_fn("/dst"), dst_fn("file") - obj = Mock(_config={dst: src}, _simple=fs.Copier._simple, _target_dir=td) +@mark.parametrize("src_func", [str, Path]) +@mark.parametrize("dst_func", [str, Path]) +@mark.parametrize("tgt_func", [str, Path]) +def test_fs_Copier_go(src_func, dst_func, tgt_func): + src, dst, tgt = src_func("/src/file"), dst_func("file"), tgt_func("/dst") + obj = Mock(_simple=fs.Copier._simple, _target_dir=tgt) + obj._expand_wildcards.return_value = [(dst, src)] with patch.object(fs, "filecopy") as filecopy: filecopy.return_value = iotaa.NodeExternal( taskname="test", threads=0, logger=getLogger(), assets_=None From 89ad7db313a18b06dafa8b4387ddfaa116867c0a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 05:47:46 +0000 Subject: [PATCH 04/25] Tests @ 100% --- src/uwtools/tests/test_fs.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 3a84db539..c17131b85 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -5,6 +5,7 @@ from logging import getLogger from pathlib import Path +from textwrap import dedent from unittest.mock import Mock, patch import iotaa @@ -12,6 +13,7 @@ from pytest import fixture, mark, raises from uwtools import fs +from uwtools.config.support import uw_yaml_loader from uwtools.exceptions import UWConfigError # Fixtures @@ -97,7 +99,7 @@ def test_fs_Copier_go_live_no_targetdir_abspath_pass(assets): assert all(path.is_file() for path in [dstdir / "foo", dstdir / "bar"]) -def test_Copier_no_targetdir_relpath_fail(assets): +def test_Copier_go_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: fs.Copier(config=cfgdict, key_path=["a", "b"]).go() @@ -119,6 +121,21 @@ def test_fs_FilerStager(assets, source): assert fs.FileStager(target_dir=dstdir, config=config, key_path=["a", "b"]) +def test_fs_FileStager__expand_wildcards(): + config = """ + /dst/a1: /src/a1 + /dst/: !glob /src/b* + """ + obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) + with patch.object(fs, "glob", return_value=["/src/b1", "/src/b2"]) as glob: + assert fs.FileStager._expand_wildcards(obj) == [ + ("/dst/a1", "/src/a1"), + ("/dst/b1", "/src/b1"), + ("/dst/b2", "/src/b2"), + ] + glob.assert_called_once_with("/src/b*") + + @mark.parametrize("source", ("dict", "file")) def test_fs_Linker(assets, source): dstdir, cfgdict, cfgfile = assets From 47810dddfdcffa74cc1bc966846239bfdc352269 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 05:49:24 +0000 Subject: [PATCH 05/25] Ditto for fs link --- src/uwtools/fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 9350cbcb2..80e4d4717 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -195,7 +195,7 @@ def go(self): """ linkname = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File links" - yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._config.items()] + yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._expand_wildcards()] class MakeDirs(Stager): From 4b75b7c89be54af14104c989eddf56e5d7b88ca3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 06:05:36 +0000 Subject: [PATCH 06/25] Error checking --- src/uwtools/fs.py | 7 +++++-- src/uwtools/tests/test_fs.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 80e4d4717..0987cf82d 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -142,9 +142,12 @@ def _expand_wildcards(self) -> list[tuple[str, str]]: for dst, src in self._config.items(): assert isinstance(src, (str, UWYAMLGlob)) if isinstance(src, UWYAMLGlob): - d = Path(dst).parent + attrs = urlparse(src.value) + if attrs.scheme not in ["", "file"]: + msg = "URL scheme '%s' incompatible with %s tag" % (attrs.scheme, src.tag) + raise UWConfigError(msg) for path in map(Path, glob(src.value)): - items.append((str(d / path.name), str(path))) + items.append((str(Path(dst).parent / path.name), str(path))) else: items.append((dst, src)) return items diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index c17131b85..1293ec937 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -136,6 +136,16 @@ def test_fs_FileStager__expand_wildcards(): glob.assert_called_once_with("/src/b*") +def test_fs_FileStager__expand_wildcards_bad_scheme(): + config = """ + /dst/: !glob https://foo.com/obj/* + """ + obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) + with raises(UWConfigError) as e: + fs.FileStager._expand_wildcards(obj) + assert str(e.value) == "URL scheme 'https' incompatible with !glob tag" + + @mark.parametrize("source", ("dict", "file")) def test_fs_Linker(assets, source): dstdir, cfgdict, cfgfile = assets From 5bf9ec4c93019ca43065634b6838f42ba8bd2c04 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 06:22:34 +0000 Subject: [PATCH 07/25] Error checking --- src/uwtools/fs.py | 5 +++-- src/uwtools/tests/test_fs.py | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 0987cf82d..6b8afa805 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -16,6 +16,7 @@ from uwtools.config.tools import walk_key_path from uwtools.config.validator import validate_internal from uwtools.exceptions import UWConfigError +from uwtools.logging import log from uwtools.strings import STR from uwtools.utils.api import str2path from uwtools.utils.tasks import directory, filecopy, symlink @@ -144,8 +145,8 @@ def _expand_wildcards(self) -> list[tuple[str, str]]: if isinstance(src, UWYAMLGlob): attrs = urlparse(src.value) if attrs.scheme not in ["", "file"]: - msg = "URL scheme '%s' incompatible with %s tag" % (attrs.scheme, src.tag) - raise UWConfigError(msg) + msg = "URL scheme '%s' incompatible with tag %s in: %s" + log.error(msg, attrs.scheme, src.tag, src) for path in map(Path, glob(src.value)): items.append((str(Path(dst).parent / path.name), str(path))) else: diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 1293ec937..807d812f8 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -15,6 +15,7 @@ from uwtools import fs from uwtools.config.support import uw_yaml_loader from uwtools.exceptions import UWConfigError +from uwtools.tests.support import logged # Fixtures @@ -136,14 +137,14 @@ def test_fs_FileStager__expand_wildcards(): glob.assert_called_once_with("/src/b*") -def test_fs_FileStager__expand_wildcards_bad_scheme(): +def test_fs_FileStager__expand_wildcards_bad_scheme(caplog): config = """ /dst/: !glob https://foo.com/obj/* """ obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) - with raises(UWConfigError) as e: - fs.FileStager._expand_wildcards(obj) - assert str(e.value) == "URL scheme 'https' incompatible with !glob tag" + assert not fs.FileStager._expand_wildcards(obj) + msg = "URL scheme 'https' incompatible with tag !glob in: !glob https://foo.com/obj/*" + assert logged(caplog, msg) @mark.parametrize("source", ("dict", "file")) From a2b8d87779c4b865f61a747e5ce474a2c46739f2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 06:34:48 +0000 Subject: [PATCH 08/25] Collapse file:// scheme --- src/uwtools/fs.py | 7 +++++-- src/uwtools/tests/test_fs.py | 23 ++++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 6b8afa805..c19bbe369 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -143,11 +143,14 @@ def _expand_wildcards(self) -> list[tuple[str, str]]: for dst, src in self._config.items(): assert isinstance(src, (str, UWYAMLGlob)) if isinstance(src, UWYAMLGlob): - attrs = urlparse(src.value) + pattern = src.value + attrs = urlparse(pattern) if attrs.scheme not in ["", "file"]: msg = "URL scheme '%s' incompatible with tag %s in: %s" log.error(msg, attrs.scheme, src.tag, src) - for path in map(Path, glob(src.value)): + if attrs.scheme == "file": + pattern = attrs.path + for path in map(Path, glob(pattern)): items.append((str(Path(dst).parent / path.name), str(path))) else: items.append((dst, src)) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 807d812f8..b0618cf47 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -124,17 +124,17 @@ def test_fs_FilerStager(assets, source): def test_fs_FileStager__expand_wildcards(): config = """ - /dst/a1: /src/a1 - /dst/: !glob /src/b* + /dst/: !glob /src/a* + /dst/b1: /src/b1 """ obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) - with patch.object(fs, "glob", return_value=["/src/b1", "/src/b2"]) as glob: + with patch.object(fs, "glob", return_value=["/src/a1", "/src/a2"]) as glob: assert fs.FileStager._expand_wildcards(obj) == [ ("/dst/a1", "/src/a1"), + ("/dst/a2", "/src/a2"), ("/dst/b1", "/src/b1"), - ("/dst/b2", "/src/b2"), ] - glob.assert_called_once_with("/src/b*") + glob.assert_called_once_with("/src/a*") def test_fs_FileStager__expand_wildcards_bad_scheme(caplog): @@ -147,6 +147,19 @@ def test_fs_FileStager__expand_wildcards_bad_scheme(caplog): assert logged(caplog, msg) +def test_fs_FileStager__expand_wildcards_file_scheme(): + config = """ + /dst/: !glob file:///src/a* + """ + obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) + with patch.object(fs, "glob", return_value=["/src/a1", "/src/a2"]) as glob: + assert fs.FileStager._expand_wildcards(obj) == [ + ("/dst/a1", "/src/a1"), + ("/dst/a2", "/src/a2"), + ] + glob.assert_called_once_with("/src/a*") + + @mark.parametrize("source", ("dict", "file")) def test_fs_Linker(assets, source): dstdir, cfgdict, cfgfile = assets From 74dfbd5f93b62328ab0816c601c66480f6f718ef Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 07:06:18 +0000 Subject: [PATCH 09/25] Doc updates --- .../cli/tools/config/validate-verbose.out | 42 +++++------ docs/sections/user_guide/yaml/files.rst | 69 ++++++++++++++++++- 2 files changed, 87 insertions(+), 24 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/config/validate-verbose.out b/docs/sections/user_guide/cli/tools/config/validate-verbose.out index 01d841518..4f07e4e73 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/validate-verbose.out @@ -1,21 +1,21 @@ -[2025-01-15T20:22:34] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose -[2025-01-15T20:22:34] DEBUG [dereference] Dereferencing, current value: -[2025-01-15T20:22:34] DEBUG [dereference] values: -[2025-01-15T20:22:34] DEBUG [dereference] greeting: Hello -[2025-01-15T20:22:34] DEBUG [dereference] recipient: World -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: values -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: values -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: greeting -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: greeting -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: Hello -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: Hello -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: recipient -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: recipient -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: World -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: World -[2025-01-15T20:22:34] DEBUG [dereference] Dereferencing, final value: -[2025-01-15T20:22:34] DEBUG [dereference] values: -[2025-01-15T20:22:34] DEBUG [dereference] greeting: Hello -[2025-01-15T20:22:34] DEBUG [dereference] recipient: World -[2025-01-15T20:22:34] DEBUG Using schema file: schema.jsonschema -[2025-01-15T20:22:34] INFO 0 schema-validation errors found in config +[2025-02-15T07:02:49] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose +[2025-02-15T07:02:49] DEBUG [dereference] Dereferencing, current value: +[2025-02-15T07:02:49] DEBUG [dereference] values: +[2025-02-15T07:02:49] DEBUG [dereference] greeting: Hello +[2025-02-15T07:02:49] DEBUG [dereference] recipient: World +[2025-02-15T07:02:49] DEBUG [dereference] Rendering: values +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: values +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: greeting +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: greeting +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: Hello +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: Hello +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: recipient +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: recipient +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: World +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: World +[2025-02-15T07:02:50] DEBUG [dereference] Dereferencing, final value: +[2025-02-15T07:02:50] DEBUG [dereference] values: +[2025-02-15T07:02:50] DEBUG [dereference] greeting: Hello +[2025-02-15T07:02:50] DEBUG [dereference] recipient: World +[2025-02-15T07:02:50] DEBUG Validating config against external schema file: schema.jsonschema +[2025-02-15T07:02:50] INFO 0 schema-validation errors found in config diff --git a/docs/sections/user_guide/yaml/files.rst b/docs/sections/user_guide/yaml/files.rst index 0d34ba38d..bc8dafd1f 100644 --- a/docs/sections/user_guide/yaml/files.rst +++ b/docs/sections/user_guide/yaml/files.rst @@ -3,6 +3,9 @@ File Blocks =========== +Basics +------ + File blocks define files to be staged in a target directory as copies or symbolic links. Keys in such blocks specify either absolute destination paths, or destination paths relative to the target directory. Values specify source paths. Example block: @@ -12,18 +15,18 @@ Example block: foo: /path/to/foo subdir/bar: /path/to/bar -* Result when copying: +* Result when copying to target directory ``target/``: .. code-block:: text - target/ + target ├── foo └── subdir └── bar where ``foo`` and ``bar`` are copies of their respective source files. -* Result when linking: +* Result when linking to target directory ``target/``: .. code-block:: text @@ -33,3 +36,63 @@ where ``foo`` and ``bar`` are copies of their respective source files. └── bar -> /path/to/bar where ``foo`` and ``bar`` are symbolic links. + +HTTP(S) Support +--------------- + +Sources values may be ``http://`` or ``https://`` URLs when copying. + +Example block: + +.. code-block:: yaml + + index: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20241001/conus/hrrr.t01z.wrfprsf00.grib2.idx + +* Result when copying to target directory ``target/``: + +.. code-block:: text + + target + └── index + +HTTP(S) sources are not supported when linking. + +Wildcard Support +---------------- + +Glob-style wildcard patterns are supported when copying and linking. + +Example block: + +.. code-block:: yaml + + a/: !glob /src/a* + b/: !glob /src/b* + +Given ``src/`` directory + +.. code-block:: text + + src + ├── aardvark + ├── apple + ├── banana + ├── bear + ├── cheetah + └── cherry + +* Result when copying to target directory ``target/``: + +.. code-block:: text + + target/ + ├── a + │ ├── aardvark + │ └── apple + └── b + ├── banana + └── bear + +Behavior is similar when linking. + +Wildcards are not supported in combination with HTTP(S) sources. From 268c763952b03b6d6f70e9913a285ef91c668502 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 07:08:17 +0000 Subject: [PATCH 10/25] Doc updates --- docs/sections/user_guide/yaml/files.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/yaml/files.rst b/docs/sections/user_guide/yaml/files.rst index bc8dafd1f..70c9261cf 100644 --- a/docs/sections/user_guide/yaml/files.rst +++ b/docs/sections/user_guide/yaml/files.rst @@ -60,7 +60,7 @@ HTTP(S) sources are not supported when linking. Wildcard Support ---------------- -Glob-style wildcard patterns are supported when copying and linking. +Glob-style wildcard patterns are supported when copying and linking and are recognized as such when tagged ``!glob``. Example block: From 933975675d326b78cd917d17ed98862e8d80eef6 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 09:32:58 +0000 Subject: [PATCH 11/25] Doc updates --- docs/sections/user_guide/yaml/files.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/sections/user_guide/yaml/files.rst b/docs/sections/user_guide/yaml/files.rst index 70c9261cf..e54e76463 100644 --- a/docs/sections/user_guide/yaml/files.rst +++ b/docs/sections/user_guide/yaml/files.rst @@ -60,7 +60,7 @@ HTTP(S) sources are not supported when linking. Wildcard Support ---------------- -Glob-style wildcard patterns are supported when copying and linking and are recognized as such when tagged ``!glob``. +Use the ``!glob`` tag to specify that a source-path value should be treated as a glob pattern: Example block: @@ -96,3 +96,12 @@ Given ``src/`` directory Behavior is similar when linking. Wildcards are not supported in combination with HTTP(S) sources. + +Note that the destination-path key is treated as a template, with the rightmost component (```` and ```` above) discarded and replaced with actual filenames. Since YAML Mapping / Python ``dict`` keys must be unique, this supports the case where the same directory is the target of multiple copies, e.g. + +.. code-block:: yaml + + /media/: !glob /some/path/*.jpg + /media/: !glob /another/path/*.mp4 + +A useful convention, adopted here, is to bracket the rightmost component between ``<`` and ``>`` characters as a visual reminder that the component is a placeholder, but this is arbitrary and the brackets have no special meaning. From 5ec30b7783a99e8125fb80fd4853ec5b1d7b4d97 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 09:41:32 +0000 Subject: [PATCH 12/25] Remove link-* doc assets --- docs/sections/user_guide/cli/tools/fs.rst | 61 +------------------ .../cli/tools/fs/link-config-report.yaml | 2 - .../cli/tools/fs/link-config-timedep.yaml | 7 --- .../user_guide/cli/tools/fs/link-config.yaml | 4 -- .../tools/fs/link-exec-no-target-dir-err.cmd | 1 - .../tools/fs/link-exec-no-target-dir-err.out | 3 - .../cli/tools/fs/link-exec-report-jq.cmd | 2 - .../cli/tools/fs/link-exec-report-jq.out | 1 - .../cli/tools/fs/link-exec-report.cmd | 2 - .../cli/tools/fs/link-exec-report.out | 21 ------- .../cli/tools/fs/link-exec-timedep.cmd | 4 -- .../cli/tools/fs/link-exec-timedep.out | 11 ---- .../user_guide/cli/tools/fs/link-exec.cmd | 4 -- .../user_guide/cli/tools/fs/link-exec.out | 16 ----- .../user_guide/cli/tools/fs/link-help.cmd | 1 - .../user_guide/cli/tools/fs/link-help.out | 29 --------- 16 files changed, 2 insertions(+), 167 deletions(-) delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-config-report.yaml delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-config.yaml delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.out delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-report.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-report.out delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec.out delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-help.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/link-help.out diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index ad4321d5c..3bb3c46cf 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -80,66 +80,9 @@ Since ``uwtools`` logs to ``stderr``, log and report output can be separated and ``link`` -------- -The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. +The ``link`` action stages files in a target directory by creating symbolic links to files, directories, or other symbolic links. It otherwise behaves identically to ``copy`` (see above) with the following caveats: -.. literalinclude:: fs/link-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: fs/link-help.out - :language: text - -Examples -^^^^^^^^ - -Given ``link-config.yaml`` containing - -.. literalinclude:: fs/link-config.yaml - :language: yaml -.. literalinclude:: fs/link-exec.cmd - :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/link-exec.out - :language: text - -Here, ``foo`` and ``bar`` are symbolic links. - -The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: - -.. literalinclude:: fs/link-config-timedep.yaml - :language: yaml -.. literalinclude:: fs/link-exec-timedep.cmd - :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/link-exec-timedep.out - :language: text - -The ``--target-dir`` option need not be specified when all linkname paths are absolute, and will never be applied to absolute linkname paths. If any linkname paths are relative, however, it is an error not to provide a target directory: - -.. literalinclude:: fs/link-config.yaml - :language: yaml -.. literalinclude:: fs/link-exec-no-target-dir-err.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: fs/link-exec-no-target-dir-err.out - :language: text - -When the ``--report`` option is specified, a report of files not linked ("not-ready") and linked ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files: - -.. literalinclude:: fs/link-config-report.yaml - :language: yaml -.. literalinclude:: fs/link-exec-report.cmd - :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/link-exec-report.out - :language: text - -Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``: - -.. literalinclude:: fs/link-exec-report-jq.cmd - :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/link-exec-report-jq.out - :language: text +* HTTP(S) sources are not supported. ``makedirs`` ------------ diff --git a/docs/sections/user_guide/cli/tools/fs/link-config-report.yaml b/docs/sections/user_guide/cli/tools/fs/link-config-report.yaml deleted file mode 100644 index 30679bb0a..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-config-report.yaml +++ /dev/null @@ -1,2 +0,0 @@ -foo: src/foo -qux: src/qux diff --git a/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml deleted file mode 100644 index 2a16449d7..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml +++ /dev/null @@ -1,7 +0,0 @@ -config: - files: - baz-{{ validtime }}: src/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/baz -yyyymmdd: "{{ cycle.strftime('%Y%m%d') }}" -hh: "{{ cycle.strftime('%H') }}" -nnn: "{{ '%03d' % (leadtime.total_seconds() // 3600) }}" -validtime: "{{ (cycle + leadtime).strftime('%Y-%m-%dT%H') }}" diff --git a/docs/sections/user_guide/cli/tools/fs/link-config.yaml b/docs/sections/user_guide/cli/tools/fs/link-config.yaml deleted file mode 100644 index 17c45a3e6..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -config: - files: - foo: src/foo - subdir/bar: src/bar diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd deleted file mode 100644 index d1d03c79b..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw fs link --config-file link-config.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out deleted file mode 100644 index 13a0a6e63..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out +++ /dev/null @@ -1,3 +0,0 @@ -[2024-12-07T01:01:55] INFO Validating config against internal schema: files-to-stage -[2024-12-07T01:01:55] INFO 0 schema-validation errors found in fs config -[2024-12-07T01:01:55] ERROR Relative path 'foo' requires target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.cmd deleted file mode 100644 index 1b2f07d3f..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.cmd +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf link-dst-report-jq -uw fs link --report --target-dir link-dst-report-jq --config-file link-config-report.yaml 2>/dev/null | jq -r .ready[] diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.out b/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.out deleted file mode 100644 index de75e07d1..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.out +++ /dev/null @@ -1 +0,0 @@ -link-dst-report-jq/foo diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-report.cmd deleted file mode 100644 index bafa22fe8..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report.cmd +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf link-dst-report -uw fs link --report --target-dir link-dst-report --config-file link-config-report.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report.out b/docs/sections/user_guide/cli/tools/fs/link-exec-report.out deleted file mode 100644 index e15034975..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report.out +++ /dev/null @@ -1,21 +0,0 @@ -[2025-01-30T06:36:37] INFO Validating config against internal schema: files-to-stage -[2025-01-30T06:36:38] INFO 0 schema-validation errors found in fs config -[2025-01-30T06:36:40] INFO Filesystem item src/foo: Ready -[2025-01-30T06:36:40] WARNING Filesystem item src/qux: Not ready [external asset] -[2025-01-30T06:36:40] INFO Link link-dst-report/foo -> src/foo: Executing -[2025-01-30T06:36:40] INFO Link link-dst-report/foo -> src/foo: Ready -[2025-01-30T06:36:41] WARNING Link link-dst-report/qux -> src/qux: Not ready -[2025-01-30T06:36:41] WARNING Link link-dst-report/qux -> src/qux: Requires: -[2025-01-30T06:36:41] WARNING Link link-dst-report/qux -> src/qux: ✖ Filesystem item src/qux -[2025-01-30T06:36:41] WARNING File links: Not ready -[2025-01-30T06:36:41] WARNING File links: Requires: -[2025-01-30T06:36:41] WARNING File links: ✔ Link link-dst-report/foo -> src/foo -[2025-01-30T06:36:41] WARNING File links: ✖ Link link-dst-report/qux -> src/qux -{ - "not-ready": [ - "link-dst-report/qux" - ], - "ready": [ - "link-dst-report/foo" - ] -} diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd deleted file mode 100644 index d815b7d38..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst-timedep -uw fs link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files -echo -tree link-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out deleted file mode 100644 index 6a6649751..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out +++ /dev/null @@ -1,11 +0,0 @@ -[2024-11-23T00:30:44] INFO Validating config against internal schema: files-to-stage -[2024-11-23T00:30:44] INFO 0 schema-validation errors found in fs config -[2024-11-23T00:30:44] INFO Filesystem item src/20240529/12/006/baz: Ready -[2024-11-23T00:30:44] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing -[2024-11-23T00:30:44] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Ready -[2024-11-23T00:30:44] INFO File links: Ready - -link-dst-timedep -└── baz-2024-05-29T18 -> ../src/20240529/12/006/baz - -1 directory, 1 file diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd deleted file mode 100644 index e1ca6bcda..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst -uw fs link --target-dir link-dst --config-file link-config.yaml --key-path config.files -echo -tree link-dst diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.out b/docs/sections/user_guide/cli/tools/fs/link-exec.out deleted file mode 100644 index 18738bc39..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec.out +++ /dev/null @@ -1,16 +0,0 @@ -[2024-11-23T00:30:43] INFO Validating config against internal schema: files-to-stage -[2024-11-23T00:30:43] INFO 0 schema-validation errors found in fs config -[2024-11-23T00:30:44] INFO Filesystem item src/foo: Ready -[2024-11-23T00:30:44] INFO Filesystem item src/bar: Ready -[2024-11-23T00:30:44] INFO Link link-dst/foo -> src/foo: Executing -[2024-11-23T00:30:44] INFO Link link-dst/foo -> src/foo: Ready -[2024-11-23T00:30:44] INFO Link link-dst/subdir/bar -> src/bar: Executing -[2024-11-23T00:30:44] INFO Link link-dst/subdir/bar -> src/bar: Ready -[2024-11-23T00:30:44] INFO File links: Ready - -link-dst -├── foo -> ../src/foo -└── subdir - └── bar -> ../../src/bar - -2 directories, 2 files diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.cmd b/docs/sections/user_guide/cli/tools/fs/link-help.cmd deleted file mode 100644 index b36d8d6d2..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw fs link --help diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.out b/docs/sections/user_guide/cli/tools/fs/link-help.out deleted file mode 100644 index 69319e3b6..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-help.out +++ /dev/null @@ -1,29 +0,0 @@ -usage: uw fs link [-h] [--version] [--config-file PATH] [--target-dir PATH] - [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] - [--key-path KEY[.KEY...]] [--report] [--quiet] [--verbose] - -Link files - -Optional arguments: - -h, --help - Show help and exit - --version - Show version info and exit - --config-file PATH, -c PATH - Path to UW YAML config file (default: read from stdin) - --target-dir PATH - Root directory for relative destination paths - --cycle CYCLE - The cycle in ISO8601 format (e.g. 2024-05-29T12) - --leadtime LEADTIME - The leadtime as hours[:minutes[:seconds]] - --dry-run - Only log info, making no changes - --key-path KEY[.KEY...] - Dot-separated path of keys to config block to use - --report - Show JSON report on [non]ready assets - --quiet, -q - Print no logging messages - --verbose, -v - Print all logging messages From 9edf6a3c5386e385e7d695bbaea3bad4b9838521 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 17:04:33 +0000 Subject: [PATCH 13/25] Simplify makedirs docs wrt --report (refer to copy section) --- docs/sections/user_guide/cli/tools/fs.rst | 26 +++++------------ .../user_guide/cli/tools/fs/link-help.cmd | 1 + .../user_guide/cli/tools/fs/link-help.out | 29 +++++++++++++++++++ .../cli/tools/fs/makedirs-config-report.yaml | 3 -- .../cli/tools/fs/makedirs-exec-report-jq.cmd | 6 ---- .../cli/tools/fs/makedirs-exec-report-jq.out | 1 - .../cli/tools/fs/makedirs-exec-report.cmd | 6 ---- .../cli/tools/fs/makedirs-exec-report.out | 19 ------------ 8 files changed, 38 insertions(+), 53 deletions(-) create mode 100644 docs/sections/user_guide/cli/tools/fs/link-help.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/link-help.out delete mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-config-report.yaml delete mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.out delete mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.cmd delete mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.out diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 3bb3c46cf..79e12761a 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -84,10 +84,16 @@ The ``link`` action stages files in a target directory by creating symbolic link * HTTP(S) sources are not supported. +.. literalinclude:: fs/link-help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: fs/link-help.out + :language: text + ``makedirs`` ------------ -The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block `. +The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block `, which must nest under a ``makedirs:`` key. .. literalinclude:: fs/makedirs-help.cmd :language: text @@ -128,20 +134,4 @@ The ``--target-dir`` option need not be specified when all directory paths are a .. literalinclude:: fs/makedirs-exec-no-target-dir-err.out :language: text -When the ``--report`` option is specified, a report of directories not created ("not-ready") and created ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files: - -.. literalinclude:: fs/makedirs-config-report.yaml - :language: yaml -.. literalinclude:: fs/makedirs-exec-report.cmd - :language: text - :emphasize-lines: 5 -.. literalinclude:: fs/makedirs-exec-report.out - :language: text - -Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``: - -.. literalinclude:: fs/makedirs-exec-report-jq.cmd - :language: text - :emphasize-lines: 5 -.. literalinclude:: fs/makedirs-exec-report-jq.out - :language: text +The ``--report`` option behaves the same as for ``link``, see above. diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.cmd b/docs/sections/user_guide/cli/tools/fs/link-help.cmd new file mode 100644 index 000000000..b36d8d6d2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-help.cmd @@ -0,0 +1 @@ +uw fs link --help diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.out b/docs/sections/user_guide/cli/tools/fs/link-help.out new file mode 100644 index 000000000..efe789d10 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-help.out @@ -0,0 +1,29 @@ +usage: uw fs link [-h] [--version] [--config-file PATH] [--target-dir PATH] + [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] + [--key-path KEY[.KEY...]] [--report] [--quiet] [--verbose] + +Link files + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --target-dir PATH + Root directory for relative destination paths + --cycle CYCLE + The cycle in ISO8601 format (e.g. 2025-02-15T12) + --leadtime LEADTIME + The leadtime as hours[:minutes[:seconds]] + --dry-run + Only log info, making no changes + --key-path KEY[.KEY...] + Dot-separated path of keys to config block to use + --report + Show JSON report on [non]ready assets + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config-report.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-config-report.yaml deleted file mode 100644 index 0fb268290..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-config-report.yaml +++ /dev/null @@ -1,3 +0,0 @@ -makedirs: - - foo - - subdir/bar diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.cmd deleted file mode 100644 index 99a4b194b..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.cmd +++ /dev/null @@ -1,6 +0,0 @@ -d=makedirs-parent-report-jq -rm -rf $d -mkdir -p $d/subdir -chmod 550 $d/subdir # read-only -uw fs makedirs --report --target-dir $d --config-file makedirs-config-report.yaml 2>/dev/null | jq -r .ready[] -chmod 750 $d/subdir # read-write diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.out deleted file mode 100644 index 596ea9e41..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.out +++ /dev/null @@ -1 +0,0 @@ -makedirs-parent-report-jq/foo diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.cmd deleted file mode 100644 index 6842f2662..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.cmd +++ /dev/null @@ -1,6 +0,0 @@ -d=makedirs-parent-report -rm -rf $d -mkdir -p $d/subdir -chmod 550 $d/subdir # read-only -uw fs makedirs --report --target-dir $d --config-file makedirs-config-report.yaml -chmod 750 $d/subdir # read-write diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.out deleted file mode 100644 index f7022e576..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.out +++ /dev/null @@ -1,19 +0,0 @@ -[2025-02-13T17:19:49] INFO Validating config against internal schema: makedirs -[2025-02-13T17:19:49] INFO 0 schema-validation errors found in fs config -[2025-02-13T17:19:50] INFO Directory makedirs-parent-report/foo: Executing -[2025-02-13T17:19:50] INFO Directory makedirs-parent-report/foo: Ready -[2025-02-13T17:19:50] INFO Directory makedirs-parent-report/subdir/bar: Executing -[2025-02-13T17:19:50] ERROR [Errno 13] Permission denied: 'makedirs-parent-report/subdir/bar' -[2025-02-13T17:19:50] WARNING Directory makedirs-parent-report/subdir/bar: Not ready -[2025-02-13T17:19:50] WARNING Directories: Not ready -[2025-02-13T17:19:50] WARNING Directories: Requires: -[2025-02-13T17:19:50] WARNING Directories: ✔ Directory makedirs-parent-report/foo -[2025-02-13T17:19:50] WARNING Directories: ✖ Directory makedirs-parent-report/subdir/bar -{ - "not-ready": [ - "makedirs-parent-report/subdir/bar" - ], - "ready": [ - "makedirs-parent-report/foo" - ] -} From 2726976815b4cfebdaef2ceffb77fd73c678d8aa Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 17:28:04 +0000 Subject: [PATCH 14/25] Rename fs doc assets for better grouping/listing --- docs/sections/user_guide/cli/tools/fs.rst | 58 +++++++++++-------- ...opy-config.yaml => copy-basic-config.yaml} | 0 .../fs/{copy-exec.cmd => copy-basic-exec.cmd} | 0 .../fs/{copy-exec.out => copy-basic-exec.out} | 0 ...rr.cmd => copy-no-target-dir-err-exec.cmd} | 0 ...rr.out => copy-no-target-dir-err-exec.out} | 0 ...ig-report.yaml => copy-report-config.yaml} | 0 ...y-exec-report.cmd => copy-report-exec.cmd} | 0 ...y-exec-report.out => copy-report-exec.out} | 0 ...-report-jq.cmd => copy-report-jq-exec.cmd} | 0 ...-report-jq.out => copy-report-jq-exec.out} | 0 ...-timedep.yaml => copy-timedep-config.yaml} | 0 ...exec-timedep.cmd => copy-timedep-exec.cmd} | 0 ...exec-timedep.out => copy-timedep-exec.out} | 0 ...config.yaml => makedirs-basic-config.yaml} | 0 ...edirs-exec.cmd => makedirs-basic-exec.cmd} | 0 ...edirs-exec.out => makedirs-basic-exec.out} | 0 ...md => makedirs-no-target-dir-err-exec.cmd} | 0 ...ut => makedirs-no-target-dir-err-exec.out} | 0 ...edep.yaml => makedirs-timedep-config.yaml} | 0 ...-timedep.cmd => makedirs-timedep-exec.cmd} | 0 ...-timedep.out => makedirs-timedep-exec.out} | 0 22 files changed, 34 insertions(+), 24 deletions(-) rename docs/sections/user_guide/cli/tools/fs/{copy-config.yaml => copy-basic-config.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec.cmd => copy-basic-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec.out => copy-basic-exec.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-no-target-dir-err.cmd => copy-no-target-dir-err-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-no-target-dir-err.out => copy-no-target-dir-err-exec.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-config-report.yaml => copy-report-config.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-report.cmd => copy-report-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-report.out => copy-report-exec.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-report-jq.cmd => copy-report-jq-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-report-jq.out => copy-report-jq-exec.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-config-timedep.yaml => copy-timedep-config.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-timedep.cmd => copy-timedep-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-exec-timedep.out => copy-timedep-exec.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-config.yaml => makedirs-basic-config.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-exec.cmd => makedirs-basic-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-exec.out => makedirs-basic-exec.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-exec-no-target-dir-err.cmd => makedirs-no-target-dir-err-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-exec-no-target-dir-err.out => makedirs-no-target-dir-err-exec.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-config-timedep.yaml => makedirs-timedep-config.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-exec-timedep.cmd => makedirs-timedep-exec.cmd} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-exec-timedep.out => makedirs-timedep-exec.out} (100%) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 79e12761a..ccf76af39 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -29,52 +29,62 @@ Examples Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths -.. literalinclude:: fs/copy-config.yaml +.. literalinclude:: fs/copy-basic-config.yaml :language: yaml -.. literalinclude:: fs/copy-exec.cmd +.. literalinclude:: fs/copy-basic-exec.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec.out +.. literalinclude:: fs/copy-basic-exec.out :language: text Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source. The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: -.. literalinclude:: fs/copy-config-timedep.yaml +.. literalinclude:: fs/copy-timedep-config.yaml :language: yaml -.. literalinclude:: fs/copy-exec-timedep.cmd +.. literalinclude:: fs/copy-timedep-exec.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec-timedep.out +.. literalinclude:: fs/copy-timedep-exec.out :language: text The ``--target-dir`` option need not be specified when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/copy-config.yaml +.. literalinclude:: fs/copy-basic-config.yaml :language: yaml -.. literalinclude:: fs/copy-exec-no-target-dir-err.cmd +.. literalinclude:: fs/copy-no-target-dir-err-exec.cmd :language: text :emphasize-lines: 1 -.. literalinclude:: fs/copy-exec-no-target-dir-err.out +.. literalinclude:: fs/copy-no-target-dir-err-exec.out :language: text When the ``--report`` option is specified, a report of files not copied ("not-ready") and copied ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files: -.. literalinclude:: fs/copy-config-report.yaml +.. literalinclude:: fs/copy-report-config.yaml :language: yaml -.. literalinclude:: fs/copy-exec-report.cmd +.. literalinclude:: fs/copy-report-exec.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec-report.out +.. literalinclude:: fs/copy-report-exec.out :language: text Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``: -.. literalinclude:: fs/copy-exec-report-jq.cmd +.. literalinclude:: fs/copy-report-jq-exec.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec-report-jq.out +.. literalinclude:: fs/copy-report-jq-exec.out + :language: text + +HTTP(S) URLs are supported in source values. For example: + +.. literalinclude:: fs/copy-url-config.yaml + :language: yaml +.. literalinclude:: fs/copy-url-exec.cmd + :language: text + :emphasize-lines: 2 +.. literalinclude:: fs/copy-url-exec.out :language: text ``link`` @@ -106,32 +116,32 @@ Examples Given ``makedirs-config.yaml`` containing -.. literalinclude:: fs/makedirs-config.yaml +.. literalinclude:: fs/makedirs-basic-config.yaml :language: yaml -.. literalinclude:: fs/makedirs-exec.cmd +.. literalinclude:: fs/makedirs-basic-exec.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/makedirs-exec.out +.. literalinclude:: fs/makedirs-basic-exec.out :language: text The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: -.. literalinclude:: fs/makedirs-config-timedep.yaml +.. literalinclude:: fs/makedirs-timedep-config.yaml :language: yaml -.. literalinclude:: fs/makedirs-exec-timedep.cmd +.. literalinclude:: fs/makedirs-timedep-exec.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/makedirs-exec-timedep.out +.. literalinclude:: fs/makedirs-timedep-exec.out :language: text The ``--target-dir`` option need not be specified when all directory paths are absolute, and will never be applied to absolute paths. If any paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/makedirs-config.yaml +.. literalinclude:: fs/makedirs-basic-config.yaml :language: yaml -.. literalinclude:: fs/makedirs-exec-no-target-dir-err.cmd +.. literalinclude:: fs/makedirs-no-target-dir-err-exec.cmd :language: text :emphasize-lines: 1 -.. literalinclude:: fs/makedirs-exec-no-target-dir-err.out +.. literalinclude:: fs/makedirs-no-target-dir-err-exec.out :language: text -The ``--report`` option behaves the same as for ``link``, see above. +The ``--report`` option behaves the same as for ``link`` (see above). diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-basic-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-basic-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out rename to docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config-report.yaml b/docs/sections/user_guide/cli/tools/fs/copy-report-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-config-report.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-report-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-report.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report.out b/docs/sections/user_guide/cli/tools/fs/copy-report-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-report.out rename to docs/sections/user_guide/cli/tools/fs/copy-report-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.out b/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.out rename to docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/copy-timedep-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-timedep-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out rename to docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-basic-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml rename to docs/sections/user_guide/cli/tools/fs/makedirs-basic-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.out From 3b71492fa718434c9f5383961a5632e86d3608d2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 17:56:44 +0000 Subject: [PATCH 15/25] Doc updates --- docs/sections/user_guide/cli/tools/fs.rst | 10 ------- .../user_guide/cli/tools/fs/.gitignore | 8 +---- .../cli/tools/fs/copy-basic-exec.cmd | 6 ++-- .../cli/tools/fs/copy-basic-exec.out | 26 ++++++++-------- .../tools/fs/copy-no-target-dir-err-exec.cmd | 2 +- .../cli/tools/fs/copy-report-exec.cmd | 4 +-- .../cli/tools/fs/copy-report-exec.out | 30 +++++++++---------- .../cli/tools/fs/copy-report-jq-exec.cmd | 4 +-- .../cli/tools/fs/copy-report-jq-exec.out | 2 +- .../cli/tools/fs/copy-timedep-exec.cmd | 6 ++-- .../cli/tools/fs/copy-timedep-exec.out | 14 ++++----- .../cli/tools/fs/makedirs-basic-exec.cmd | 2 +- .../fs/makedirs-no-target-dir-err-exec.cmd | 2 +- .../cli/tools/fs/makedirs-timedep-exec.cmd | 2 +- 14 files changed, 51 insertions(+), 67 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index ccf76af39..11b6e1a8c 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -77,16 +77,6 @@ Since ``uwtools`` logs to ``stderr``, log and report output can be separated and .. literalinclude:: fs/copy-report-jq-exec.out :language: text -HTTP(S) URLs are supported in source values. For example: - -.. literalinclude:: fs/copy-url-config.yaml - :language: yaml -.. literalinclude:: fs/copy-url-exec.cmd - :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/copy-url-exec.out - :language: text - ``link`` -------- diff --git a/docs/sections/user_guide/cli/tools/fs/.gitignore b/docs/sections/user_guide/cli/tools/fs/.gitignore index e49682489..3a41be37d 100644 --- a/docs/sections/user_guide/cli/tools/fs/.gitignore +++ b/docs/sections/user_guide/cli/tools/fs/.gitignore @@ -1,7 +1 @@ -copy-dst -copy-dst-report-jq -copy-dst-timedep -link-dst -link-dst-report -link-dst-report-jq -link-dst-timedep +dst/ diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd index f0579f20e..e355c2f5a 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd @@ -1,4 +1,4 @@ -rm -rf copy-dst -uw fs copy --target-dir copy-dst --config-file copy-config.yaml --key-path config.files +rm -rf dst/copy +uw fs copy --target-dir dst/copy --config-file copy-basic-config.yaml --key-path config.files echo -tree copy-dst +tree dst/copy diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out index bd0fee1c0..f3f202c82 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out @@ -1,17 +1,17 @@ -[2025-01-18T07:29:30] INFO Validating config against internal schema: files-to-stage -[2025-01-18T07:29:30] INFO 0 schema-validation errors found in fs config -[2025-01-18T07:29:32] INFO File src/foo: Ready -[2025-01-18T07:29:33] INFO Remote object https://www.gnu.org/licenses/gpl-3.0.txt: Ready -[2025-01-18T07:29:33] INFO File src/bar: Ready -[2025-01-18T07:29:33] INFO Copy src/foo -> copy-dst/foo: Executing -[2025-01-18T07:29:33] INFO Copy src/foo -> copy-dst/foo: Ready -[2025-01-18T07:29:33] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Executing -[2025-01-18T07:29:33] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Ready -[2025-01-18T07:29:33] INFO Copy src/bar -> copy-dst/subdir/bar: Executing -[2025-01-18T07:29:33] INFO Copy src/bar -> copy-dst/subdir/bar: Ready -[2025-01-18T07:29:33] INFO File copies: Ready +[2025-02-15T17:50:51] INFO Validating config against internal schema: files-to-stage +[2025-02-15T17:50:51] INFO 0 schema-validation errors found in fs config +[2025-02-15T17:50:54] INFO File src/foo: Ready +[2025-02-15T17:50:55] INFO Remote object https://www.gnu.org/licenses/gpl-3.0.txt: Ready +[2025-02-15T17:50:55] INFO File src/bar: Ready +[2025-02-15T17:50:55] INFO Copy src/foo -> dst/copy/foo: Executing +[2025-02-15T17:50:55] INFO Copy src/foo -> dst/copy/foo: Ready +[2025-02-15T17:50:55] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Executing +[2025-02-15T17:50:55] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Ready +[2025-02-15T17:50:55] INFO Copy src/bar -> dst/copy/subdir/bar: Executing +[2025-02-15T17:50:55] INFO Copy src/bar -> dst/copy/subdir/bar: Ready +[2025-02-15T17:50:55] INFO File copies: Ready -copy-dst +dst/copy ├── foo ├── licenses │   └── gpl diff --git a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd index d89130592..532d54377 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd @@ -1 +1 @@ -uw fs copy --config-file copy-config.yaml --key-path config.files +uw fs copy --config-file copy-basic-config.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd index 7f1b614a5..8c2893a53 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd @@ -1,2 +1,2 @@ -rm -rf copy-dst-report-jq -uw fs copy --report --target-dir copy-dst-report-jq --config-file copy-config-report.yaml +rm -rf dst/copy-report-jq +uw fs copy --report --target-dir dst/copy-report-jq --config-file copy-report-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-report-exec.out index 6377f3b55..0367d954a 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-report-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-report-exec.out @@ -1,21 +1,21 @@ -[2025-01-30T06:36:44] INFO Validating config against internal schema: files-to-stage -[2025-01-30T06:36:44] INFO 0 schema-validation errors found in fs config -[2025-01-30T06:36:48] INFO File src/foo: Ready -[2025-01-30T06:36:48] WARNING File src/qux: Not ready [external asset] -[2025-01-30T06:36:48] INFO Copy src/foo -> copy-dst-report-jq/foo: Executing -[2025-01-30T06:36:48] INFO Copy src/foo -> copy-dst-report-jq/foo: Ready -[2025-01-30T06:36:48] WARNING Copy src/qux -> copy-dst-report-jq/qux: Not ready -[2025-01-30T06:36:48] WARNING Copy src/qux -> copy-dst-report-jq/qux: Requires: -[2025-01-30T06:36:48] WARNING Copy src/qux -> copy-dst-report-jq/qux: ✖ File src/qux -[2025-01-30T06:36:48] WARNING File copies: Not ready -[2025-01-30T06:36:48] WARNING File copies: Requires: -[2025-01-30T06:36:48] WARNING File copies: ✔ Copy src/foo -> copy-dst-report-jq/foo -[2025-01-30T06:36:48] WARNING File copies: ✖ Copy src/qux -> copy-dst-report-jq/qux +[2025-02-15T17:50:50] INFO Validating config against internal schema: files-to-stage +[2025-02-15T17:50:51] INFO 0 schema-validation errors found in fs config +[2025-02-15T17:50:53] INFO File src/foo: Ready +[2025-02-15T17:50:53] WARNING File src/qux: Not ready [external asset] +[2025-02-15T17:50:53] INFO Copy src/foo -> dst/copy-report-jq/foo: Executing +[2025-02-15T17:50:53] INFO Copy src/foo -> dst/copy-report-jq/foo: Ready +[2025-02-15T17:50:53] WARNING Copy src/qux -> dst/copy-report-jq/qux: Not ready +[2025-02-15T17:50:53] WARNING Copy src/qux -> dst/copy-report-jq/qux: Requires: +[2025-02-15T17:50:53] WARNING Copy src/qux -> dst/copy-report-jq/qux: ✖ File src/qux +[2025-02-15T17:50:53] WARNING File copies: Not ready +[2025-02-15T17:50:53] WARNING File copies: Requires: +[2025-02-15T17:50:53] WARNING File copies: ✔ Copy src/foo -> dst/copy-report-jq/foo +[2025-02-15T17:50:53] WARNING File copies: ✖ Copy src/qux -> dst/copy-report-jq/qux { "not-ready": [ - "copy-dst-report-jq/qux" + "dst/copy-report-jq/qux" ], "ready": [ - "copy-dst-report-jq/foo" + "dst/copy-report-jq/foo" ] } diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd index d1362690c..982928f97 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd @@ -1,2 +1,2 @@ -rm -rf copy-dst -uw fs copy --report --target-dir copy-dst --config-file copy-config-report.yaml 2>/dev/null | jq -r .ready[] +rm -rf dst/copy +uw fs copy --report --target-dir dst/copy --config-file copy-report-config.yaml 2>/dev/null | jq -r .ready[] diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out index 7082d6a3a..04ec58c5d 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out @@ -1 +1 @@ -copy-dst/foo +dst/copy/foo diff --git a/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd index 6c3c956af..e28947c68 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd @@ -1,4 +1,4 @@ -rm -rf copy-dst-timedep -uw fs copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files +rm -rf dst/copy-timedep +uw fs copy --target-dir dst/copy-timedep --config-file copy-timedep-config.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files echo -tree copy-dst-timedep +tree dst/copy-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out index c7a4e8ff6..8be3e9ead 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out @@ -1,11 +1,11 @@ -[2024-11-23T00:30:43] INFO Validating config against internal schema: files-to-stage -[2024-11-23T00:30:43] INFO 0 schema-validation errors found in fs config -[2024-11-23T00:30:44] INFO File src/20240529/12/006/baz: Ready -[2024-11-23T00:30:44] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing -[2024-11-23T00:30:44] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Ready -[2024-11-23T00:30:44] INFO File copies: Ready +[2025-02-15T17:50:54] INFO Validating config against internal schema: files-to-stage +[2025-02-15T17:50:54] INFO 0 schema-validation errors found in fs config +[2025-02-15T17:50:56] INFO File src/20240529/12/006/baz: Ready +[2025-02-15T17:50:56] INFO Copy src/20240529/12/006/baz -> dst/copy-timedep/baz-2024-05-29T18: Executing +[2025-02-15T17:50:56] INFO Copy src/20240529/12/006/baz -> dst/copy-timedep/baz-2024-05-29T18: Ready +[2025-02-15T17:50:56] INFO File copies: Ready -copy-dst-timedep +dst/copy-timedep └── baz-2024-05-29T18 1 directory, 1 file diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd index 4ca49133b..c2243a8bd 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent -uw fs makedirs --target-dir makedirs-parent --config-file makedirs-config.yaml --key-path config +uw fs makedirs --target-dir makedirs-parent --config-file makedirs-basic-config.yaml --key-path config echo tree -F makedirs-parent diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd index 8c2d83cc9..f2896635e 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd @@ -1 +1 @@ -uw fs makedirs --config-file makedirs-config.yaml --key-path config +uw fs makedirs --config-file makedirs-basic-config.yaml --key-path config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd index 221bac642..acea90ef3 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent-timedep -uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config +uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-timedep-config.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config echo tree -F makedirs-parent-timedep From fb1faefaa1f024c3b1c38bbc3a63bd58ea808917 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 18:18:52 +0000 Subject: [PATCH 16/25] Doc updates --- docs/sections/user_guide/cli/tools/fs.rst | 16 ++++++++++++- .../cli/tools/fs/copy-glob-config.yaml | 1 + .../cli/tools/fs/copy-glob-exec.cmd | 4 ++++ .../cli/tools/fs/copy-glob-exec.out | 23 +++++++++++++++++++ src/uwtools/fs.py | 5 +++- 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-glob-config.yaml create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-glob-exec.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-glob-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 11b6e1a8c..95f33e3e1 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -77,12 +77,26 @@ Since ``uwtools`` logs to ``stderr``, log and report output can be separated and .. literalinclude:: fs/copy-report-jq-exec.out :language: text +Use the ``!glob`` tag to specify that a source-path value should be treated as a glob pattern: + +.. literalinclude:: fs/copy-glob-config.yaml + :language: yaml +.. literalinclude:: fs/copy-glob-exec.cmd + :language: text + :emphasize-lines: 2 +.. literalinclude:: fs/copy-glob-exec.out + :language: text + +See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. Note that directories are excluded, and recursion (deep copies) are not supported. + ``link`` -------- -The ``link`` action stages files in a target directory by creating symbolic links to files, directories, or other symbolic links. It otherwise behaves identically to ``copy`` (see above) with the following caveats: +The ``link`` action stages items in a target directory by creating symbolic links to files, directories, or other symbolic links. It otherwise behaves similarly to ``copy`` (see above), but note the following: +* In addition to file, directories and other symbolic links can be linked. * HTTP(S) sources are not supported. +* Support for wildcard source values is the same as for ``link``. .. literalinclude:: fs/link-help.cmd :language: text diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-glob-config.yaml new file mode 100644 index 000000000..c3f256dd0 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob-config.yaml @@ -0,0 +1 @@ +dst/: !glob src/* diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.cmd new file mode 100644 index 000000000..065c34735 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.cmd @@ -0,0 +1,4 @@ +rm -rf dst/copy-glob +uw fs copy --report --target-dir dst/copy-glob --config-file copy-glob-config.yaml +echo +tree dst/copy-glob diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.out new file mode 100644 index 000000000..d8159c05c --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.out @@ -0,0 +1,23 @@ +[2025-02-15T18:17:55] INFO Validating config against internal schema: files-to-stage +[2025-02-15T18:17:55] INFO 0 schema-validation errors found in fs config +[2025-02-15T18:17:55] INFO File src/foo: Ready +[2025-02-15T18:17:55] INFO File src/bar: Ready +[2025-02-15T18:17:55] INFO Copy src/foo -> dst/copy-glob/dst/foo: Executing +[2025-02-15T18:17:55] INFO Copy src/foo -> dst/copy-glob/dst/foo: Ready +[2025-02-15T18:17:55] INFO Copy src/bar -> dst/copy-glob/dst/bar: Executing +[2025-02-15T18:17:55] INFO Copy src/bar -> dst/copy-glob/dst/bar: Ready +[2025-02-15T18:17:55] INFO File copies: Ready +{ + "not-ready": [], + "ready": [ + "dst/copy-glob/dst/foo", + "dst/copy-glob/dst/bar" + ] +} + +dst/copy-glob +└── dst + ├── bar + └── foo + +2 directories, 2 files diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index c19bbe369..8ec84b4d8 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -151,7 +151,10 @@ def _expand_wildcards(self) -> list[tuple[str, str]]: if attrs.scheme == "file": pattern = attrs.path for path in map(Path, glob(pattern)): - items.append((str(Path(dst).parent / path.name), str(path))) + if not path.is_dir(): + items.append((str(Path(dst).parent / path.name), str(path))) + else: + log.debug("Skipping directory %s", path) else: items.append((dst, src)) return items From a2f389c19eed00961c073723079fa85cfbe84380 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 18:19:48 +0000 Subject: [PATCH 17/25] Doc updates --- docs/sections/user_guide/cli/tools/fs.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 95f33e3e1..593d3bdb5 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -87,7 +87,9 @@ Use the ``!glob`` tag to specify that a source-path value should be treated as a .. literalinclude:: fs/copy-glob-exec.out :language: text -See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. Note that directories are excluded, and recursion (deep copies) are not supported. +See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. + +Note that directories are excluded, and recursion (deep copies) are not supported. ``link`` -------- From 7893bed055a8609e9f53b4dc343c9682f690d93d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 18:27:07 +0000 Subject: [PATCH 18/25] Doc updates --- docs/sections/user_guide/cli/tools/fs.rst | 56 +++++++++---------- .../cli/tools/fs/copy-basic-exec.cmd | 4 -- .../user_guide/cli/tools/fs/copy-basic.cmd | 4 ++ .../{copy-basic-exec.out => copy-basic.out} | 0 ...copy-basic-config.yaml => copy-basic.yaml} | 0 .../fs/{copy-glob-exec.cmd => copy-glob.cmd} | 2 +- .../fs/{copy-glob-exec.out => copy-glob.out} | 0 .../{copy-glob-config.yaml => copy-glob.yaml} | 0 .../tools/fs/copy-no-target-dir-err-exec.cmd | 1 - .../cli/tools/fs/copy-no-target-dir-err.cmd | 1 + ...rr-exec.out => copy-no-target-dir-err.out} | 0 ...-report-jq-exec.cmd => copy-report-jq.cmd} | 2 +- ...-report-jq-exec.out => copy-report-jq.out} | 0 .../{copy-report-exec.cmd => copy-report.cmd} | 2 +- .../{copy-report-exec.out => copy-report.out} | 0 ...py-report-config.yaml => copy-report.yaml} | 0 ...copy-timedep-exec.cmd => copy-timedep.cmd} | 2 +- ...copy-timedep-exec.out => copy-timedep.out} | 0 ...-timedep-config.yaml => copy-timedep.yaml} | 0 ...dirs-basic-exec.cmd => makedirs-basic.cmd} | 2 +- ...dirs-basic-exec.out => makedirs-basic.out} | 0 ...-basic-config.yaml => makedirs-basic.yaml} | 0 .../fs/makedirs-no-target-dir-err-exec.cmd | 1 - .../tools/fs/makedirs-no-target-dir-err.cmd | 1 + ...xec.out => makedirs-no-target-dir-err.out} | 0 ...-timedep-exec.cmd => makedirs-timedep.cmd} | 2 +- ...-timedep-exec.out => makedirs-timedep.out} | 0 ...edep-config.yaml => makedirs-timedep.yaml} | 0 28 files changed, 40 insertions(+), 40 deletions(-) delete mode 100644 docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-basic.cmd rename docs/sections/user_guide/cli/tools/fs/{copy-basic-exec.out => copy-basic.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-basic-config.yaml => copy-basic.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-glob-exec.cmd => copy-glob.cmd} (85%) rename docs/sections/user_guide/cli/tools/fs/{copy-glob-exec.out => copy-glob.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-glob-config.yaml => copy-glob.yaml} (100%) delete mode 100644 docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd rename docs/sections/user_guide/cli/tools/fs/{copy-no-target-dir-err-exec.out => copy-no-target-dir-err.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-report-jq-exec.cmd => copy-report-jq.cmd} (64%) rename docs/sections/user_guide/cli/tools/fs/{copy-report-jq-exec.out => copy-report-jq.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-report-exec.cmd => copy-report.cmd} (77%) rename docs/sections/user_guide/cli/tools/fs/{copy-report-exec.out => copy-report.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-report-config.yaml => copy-report.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-timedep-exec.cmd => copy-timedep.cmd} (60%) rename docs/sections/user_guide/cli/tools/fs/{copy-timedep-exec.out => copy-timedep.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-timedep-config.yaml => copy-timedep.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-basic-exec.cmd => makedirs-basic.cmd} (74%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-basic-exec.out => makedirs-basic.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-basic-config.yaml => makedirs-basic.yaml} (100%) delete mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd rename docs/sections/user_guide/cli/tools/fs/{makedirs-no-target-dir-err-exec.out => makedirs-no-target-dir-err.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-timedep-exec.cmd => makedirs-timedep.cmd} (61%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-timedep-exec.out => makedirs-timedep.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-timedep-config.yaml => makedirs-timedep.yaml} (100%) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 593d3bdb5..7591fdc63 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -27,64 +27,64 @@ Source paths prefixed with ``http://`` or ``https://`` will be copied from their Examples ^^^^^^^^ -Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths +Given a config containing a mapping from local-filesystem destination paths to source paths -.. literalinclude:: fs/copy-basic-config.yaml +.. literalinclude:: fs/copy-basic.yaml :language: yaml -.. literalinclude:: fs/copy-basic-exec.cmd +.. literalinclude:: fs/copy-basic.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-basic-exec.out +.. literalinclude:: fs/copy-basic.out :language: text Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source. The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: -.. literalinclude:: fs/copy-timedep-config.yaml +.. literalinclude:: fs/copy-timedep.yaml :language: yaml -.. literalinclude:: fs/copy-timedep-exec.cmd +.. literalinclude:: fs/copy-timedep.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-timedep-exec.out +.. literalinclude:: fs/copy-timedep.out :language: text The ``--target-dir`` option need not be specified when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/copy-basic-config.yaml +.. literalinclude:: fs/copy-basic.yaml :language: yaml -.. literalinclude:: fs/copy-no-target-dir-err-exec.cmd +.. literalinclude:: fs/copy-no-target-dir-err.cmd :language: text :emphasize-lines: 1 -.. literalinclude:: fs/copy-no-target-dir-err-exec.out +.. literalinclude:: fs/copy-no-target-dir-err.out :language: text When the ``--report`` option is specified, a report of files not copied ("not-ready") and copied ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files: -.. literalinclude:: fs/copy-report-config.yaml +.. literalinclude:: fs/copy-report.yaml :language: yaml -.. literalinclude:: fs/copy-report-exec.cmd +.. literalinclude:: fs/copy-report.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-report-exec.out +.. literalinclude:: fs/copy-report.out :language: text Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``: -.. literalinclude:: fs/copy-report-jq-exec.cmd +.. literalinclude:: fs/copy-report-jq.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-report-jq-exec.out +.. literalinclude:: fs/copy-report-jq.out :language: text Use the ``!glob`` tag to specify that a source-path value should be treated as a glob pattern: -.. literalinclude:: fs/copy-glob-config.yaml +.. literalinclude:: fs/copy-glob.yaml :language: yaml -.. literalinclude:: fs/copy-glob-exec.cmd +.. literalinclude:: fs/copy-glob.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-glob-exec.out +.. literalinclude:: fs/copy-glob.out :language: text See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. @@ -120,34 +120,34 @@ The ``makedirs`` action creates directories. Any ``KEY`` positional arguments ar Examples ^^^^^^^^ -Given ``makedirs-config.yaml`` containing +Given a config containing -.. literalinclude:: fs/makedirs-basic-config.yaml +.. literalinclude:: fs/makedirs-basic.yaml :language: yaml -.. literalinclude:: fs/makedirs-basic-exec.cmd +.. literalinclude:: fs/makedirs-basic.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/makedirs-basic-exec.out +.. literalinclude:: fs/makedirs-basic.out :language: text The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: -.. literalinclude:: fs/makedirs-timedep-config.yaml +.. literalinclude:: fs/makedirs-timedep.yaml :language: yaml -.. literalinclude:: fs/makedirs-timedep-exec.cmd +.. literalinclude:: fs/makedirs-timedep.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/makedirs-timedep-exec.out +.. literalinclude:: fs/makedirs-timedep.out :language: text The ``--target-dir`` option need not be specified when all directory paths are absolute, and will never be applied to absolute paths. If any paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/makedirs-basic-config.yaml +.. literalinclude:: fs/makedirs-basic.yaml :language: yaml -.. literalinclude:: fs/makedirs-no-target-dir-err-exec.cmd +.. literalinclude:: fs/makedirs-no-target-dir-err.cmd :language: text :emphasize-lines: 1 -.. literalinclude:: fs/makedirs-no-target-dir-err-exec.out +.. literalinclude:: fs/makedirs-no-target-dir-err.out :language: text The ``--report`` option behaves the same as for ``link`` (see above). diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd deleted file mode 100644 index e355c2f5a..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf dst/copy -uw fs copy --target-dir dst/copy --config-file copy-basic-config.yaml --key-path config.files -echo -tree dst/copy diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic.cmd b/docs/sections/user_guide/cli/tools/fs/copy-basic.cmd new file mode 100644 index 000000000..5a1369b93 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-basic.cmd @@ -0,0 +1,4 @@ +rm -rf dst/copy +uw fs copy --target-dir dst/copy --config-file copy-basic.yaml --key-path config.files +echo +tree dst/copy diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-basic.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-basic-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-basic.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-basic.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-basic-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-basic.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-glob.cmd similarity index 85% rename from docs/sections/user_guide/cli/tools/fs/copy-glob-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-glob.cmd index 065c34735..ce00e70f2 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.cmd @@ -1,4 +1,4 @@ rm -rf dst/copy-glob -uw fs copy --report --target-dir dst/copy-glob --config-file copy-glob-config.yaml +uw fs copy --report --target-dir dst/copy-glob --config-file copy-glob.yaml echo tree dst/copy-glob diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-glob.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-glob-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-glob.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-glob.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-glob-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-glob.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd deleted file mode 100644 index 532d54377..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.cmd +++ /dev/null @@ -1 +0,0 @@ -uw fs copy --config-file copy-basic-config.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd new file mode 100644 index 000000000..9c189bd9d --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs copy --config-file copy-basic.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report-jq.cmd similarity index 64% rename from docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-report-jq.cmd index 982928f97..d1d1f51a5 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-report-jq.cmd @@ -1,2 +1,2 @@ rm -rf dst/copy -uw fs copy --report --target-dir dst/copy --config-file copy-report-config.yaml 2>/dev/null | jq -r .ready[] +uw fs copy --report --target-dir dst/copy --config-file copy-report.yaml 2>/dev/null | jq -r .ready[] diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-report-jq.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-report-jq-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-report-jq.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report.cmd similarity index 77% rename from docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-report.cmd index 8c2893a53..018eab7c9 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-report-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-report.cmd @@ -1,2 +1,2 @@ rm -rf dst/copy-report-jq -uw fs copy --report --target-dir dst/copy-report-jq --config-file copy-report-config.yaml +uw fs copy --report --target-dir dst/copy-report-jq --config-file copy-report.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-report.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-report-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-report.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-report.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-report-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-report.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-timedep.cmd similarity index 60% rename from docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/copy-timedep.cmd index e28947c68..f1e0ea729 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-timedep.cmd @@ -1,4 +1,4 @@ rm -rf dst/copy-timedep -uw fs copy --target-dir dst/copy-timedep --config-file copy-timedep-config.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files +uw fs copy --target-dir dst/copy-timedep --config-file copy-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files echo tree dst/copy-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-timedep.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-timedep-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-timedep.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-timedep-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-timedep-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-timedep.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-basic.cmd similarity index 74% rename from docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs-basic.cmd index c2243a8bd..18a85c70d 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-basic.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent -uw fs makedirs --target-dir makedirs-parent --config-file makedirs-basic-config.yaml --key-path config +uw fs makedirs --target-dir makedirs-parent --config-file makedirs-basic.yaml --key-path config echo tree -F makedirs-parent diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs-basic.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-basic-exec.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-basic.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-basic-config.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-basic.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-basic-config.yaml rename to docs/sections/user_guide/cli/tools/fs/makedirs-basic.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd deleted file mode 100644 index f2896635e..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.cmd +++ /dev/null @@ -1 +0,0 @@ -uw fs makedirs --config-file makedirs-basic-config.yaml --key-path config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd new file mode 100644 index 000000000..66d3c5c8f --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs makedirs --config-file makedirs-basic.yaml --key-path config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err-exec.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.cmd similarity index 61% rename from docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep.cmd index acea90ef3..1ebaaff7c 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent-timedep -uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-timedep-config.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config +uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config echo tree -F makedirs-parent-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-timedep-exec.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-timedep-config.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-timedep-config.yaml rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep.yaml From aec58d523d7cac3e2dc61f4ee9550e066f348de2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 18:53:07 +0000 Subject: [PATCH 19/25] Doc updates --- docs/sections/user_guide/cli/tools/fs.rst | 6 ++++-- .../user_guide/cli/tools/fs/copy-glob.out | 19 ++++++++++--------- src/uwtools/fs.py | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 7591fdc63..91ff9f481 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -87,9 +87,11 @@ Use the ``!glob`` tag to specify that a source-path value should be treated as a .. literalinclude:: fs/copy-glob.out :language: text -See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. +The ``--report`` output can be especially useful in combination with wildcards to allow downstream logic to process a set of copied files whose identity is not known in advance. + +Note that directories are excluded, and recursive copies are not supported. To recursively copy from a shell script, where you might otherwise use ``uw fs copy``, consider using ``cp -r``; and from Python code where you might otherwise call ``uwtools.api.fs.copy()``, consider using the standard-library `shutil.copytree()` `_. For more control, including file-grained include and exclude, consider the unrivaled `rsync` `_, which can be installed from conda in case your system does not already provide it. It can be called from shell scripts, or via `subprocess` `_ from Python. -Note that directories are excluded, and recursion (deep copies) are not supported. +See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. ``link`` -------- diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.out b/docs/sections/user_guide/cli/tools/fs/copy-glob.out index d8159c05c..49c3b7f45 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-glob.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.out @@ -1,12 +1,13 @@ -[2025-02-15T18:17:55] INFO Validating config against internal schema: files-to-stage -[2025-02-15T18:17:55] INFO 0 schema-validation errors found in fs config -[2025-02-15T18:17:55] INFO File src/foo: Ready -[2025-02-15T18:17:55] INFO File src/bar: Ready -[2025-02-15T18:17:55] INFO Copy src/foo -> dst/copy-glob/dst/foo: Executing -[2025-02-15T18:17:55] INFO Copy src/foo -> dst/copy-glob/dst/foo: Ready -[2025-02-15T18:17:55] INFO Copy src/bar -> dst/copy-glob/dst/bar: Executing -[2025-02-15T18:17:55] INFO Copy src/bar -> dst/copy-glob/dst/bar: Ready -[2025-02-15T18:17:55] INFO File copies: Ready +[2025-02-15T18:52:47] INFO Validating config against internal schema: files-to-stage +[2025-02-15T18:52:47] INFO 0 schema-validation errors found in fs config +[2025-02-15T18:52:47] WARNING Ignoring directory src/20240529 +[2025-02-15T18:52:47] INFO File src/foo: Ready +[2025-02-15T18:52:47] INFO File src/bar: Ready +[2025-02-15T18:52:47] INFO Copy src/foo -> dst/copy-glob/dst/foo: Executing +[2025-02-15T18:52:47] INFO Copy src/foo -> dst/copy-glob/dst/foo: Ready +[2025-02-15T18:52:47] INFO Copy src/bar -> dst/copy-glob/dst/bar: Executing +[2025-02-15T18:52:47] INFO Copy src/bar -> dst/copy-glob/dst/bar: Ready +[2025-02-15T18:52:47] INFO File copies: Ready { "not-ready": [], "ready": [ diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 8ec84b4d8..0964ac98a 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -154,7 +154,7 @@ def _expand_wildcards(self) -> list[tuple[str, str]]: if not path.is_dir(): items.append((str(Path(dst).parent / path.name), str(path))) else: - log.debug("Skipping directory %s", path) + log.warning("Ignoring directory %s", path) else: items.append((dst, src)) return items From 0c1a6a6cf330736f03cc03fca6d8a3393ccf51c1 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 19:05:25 +0000 Subject: [PATCH 20/25] Update _expand_wildcards() test --- src/uwtools/tests/test_fs.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index b0618cf47..4d590e536 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -3,6 +3,7 @@ # pylint: disable=protected-access # pylint: disable=redefined-outer-name +import logging from logging import getLogger from pathlib import Path from textwrap import dedent @@ -15,6 +16,7 @@ from uwtools import fs from uwtools.config.support import uw_yaml_loader from uwtools.exceptions import UWConfigError +from uwtools.logging import log from uwtools.tests.support import logged # Fixtures @@ -122,19 +124,23 @@ def test_fs_FilerStager(assets, source): assert fs.FileStager(target_dir=dstdir, config=config, key_path=["a", "b"]) -def test_fs_FileStager__expand_wildcards(): - config = """ - /dst/: !glob /src/a* - /dst/b1: /src/b1 +def test_fs_FileStager__expand_wildcards(caplog, tmp_path): + log.setLevel(logging.WARNING) + d = tmp_path + for fn in ["a1", "a2", "b1"]: + (d / fn).touch() + (d / "a3").mkdir() + config = f""" + /dst/: !glob {d}/a* + /dst/b1: {d}/b1 """ obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) - with patch.object(fs, "glob", return_value=["/src/a1", "/src/a2"]) as glob: - assert fs.FileStager._expand_wildcards(obj) == [ - ("/dst/a1", "/src/a1"), - ("/dst/a2", "/src/a2"), - ("/dst/b1", "/src/b1"), - ] - glob.assert_called_once_with("/src/a*") + assert sorted(fs.FileStager._expand_wildcards(obj)) == [ + ("/dst/a1", str(d / "a1")), + ("/dst/a2", str(d / "a2")), + ("/dst/b1", str(d / "b1")), + ] + assert logged(caplog, f"Ignoring directory {d}/a3") def test_fs_FileStager__expand_wildcards_bad_scheme(caplog): From dca62d117bbd15fd5ab03f2a1ba2470a7b66544d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 19:18:46 +0000 Subject: [PATCH 21/25] Doc updates --- docs/sections/user_guide/cli/tools/fs.rst | 2 +- .../user_guide/cli/tools/fs/copy-basic.out | 22 +++++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 91ff9f481..a7a0030e5 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -89,7 +89,7 @@ Use the ``!glob`` tag to specify that a source-path value should be treated as a The ``--report`` output can be especially useful in combination with wildcards to allow downstream logic to process a set of copied files whose identity is not known in advance. -Note that directories are excluded, and recursive copies are not supported. To recursively copy from a shell script, where you might otherwise use ``uw fs copy``, consider using ``cp -r``; and from Python code where you might otherwise call ``uwtools.api.fs.copy()``, consider using the standard-library `shutil.copytree()` `_. For more control, including file-grained include and exclude, consider the unrivaled `rsync` `_, which can be installed from conda in case your system does not already provide it. It can be called from shell scripts, or via `subprocess` `_ from Python. +Note that directories are excluded, and recursive copies are not supported. To recursively copy from a shell script, where you might otherwise use ``uw fs copy``, consider using ``cp -r``; and from Python code where you might otherwise call ``uwtools.api.fs.copy()``, consider using the standard-library :python:`shutil.html#shutil.copytree`. For more control, including file-grained include and exclude, consider the unrivaled `rsync` `_, which can be installed from conda in case your system does not already provide it. It can be called from shell scripts, or via :python:`subprocess.html` from Python. See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic.out b/docs/sections/user_guide/cli/tools/fs/copy-basic.out index f3f202c82..165d61afe 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-basic.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-basic.out @@ -1,15 +1,13 @@ -[2025-02-15T17:50:51] INFO Validating config against internal schema: files-to-stage -[2025-02-15T17:50:51] INFO 0 schema-validation errors found in fs config -[2025-02-15T17:50:54] INFO File src/foo: Ready -[2025-02-15T17:50:55] INFO Remote object https://www.gnu.org/licenses/gpl-3.0.txt: Ready -[2025-02-15T17:50:55] INFO File src/bar: Ready -[2025-02-15T17:50:55] INFO Copy src/foo -> dst/copy/foo: Executing -[2025-02-15T17:50:55] INFO Copy src/foo -> dst/copy/foo: Ready -[2025-02-15T17:50:55] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Executing -[2025-02-15T17:50:55] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Ready -[2025-02-15T17:50:55] INFO Copy src/bar -> dst/copy/subdir/bar: Executing -[2025-02-15T17:50:55] INFO Copy src/bar -> dst/copy/subdir/bar: Ready -[2025-02-15T17:50:55] INFO File copies: Ready +[2025-02-15T19:08:19] INFO Validating config against internal schema: files-to-stage +[2025-02-15T19:08:19] INFO 0 schema-validation errors found in fs config +[2025-02-15T19:08:21] INFO Copy src/foo -> dst/copy/foo: Ready +[2025-02-15T19:08:21] INFO Remote object https://www.gnu.org/licenses/gpl-3.0.txt: Ready +[2025-02-15T19:08:21] INFO File src/bar: Ready +[2025-02-15T19:08:21] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Executing +[2025-02-15T19:08:21] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Ready +[2025-02-15T19:08:21] INFO Copy src/bar -> dst/copy/subdir/bar: Executing +[2025-02-15T19:08:21] INFO Copy src/bar -> dst/copy/subdir/bar: Ready +[2025-02-15T19:08:21] INFO File copies: Ready dst/copy ├── foo From 4fcb4a5cf69a74b4311ad9a6fac4594c7c3a95fb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 19:33:11 +0000 Subject: [PATCH 22/25] Doc updates --- docs/sections/user_guide/cli/tools/fs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index a7a0030e5..2507c7e9c 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -89,9 +89,9 @@ Use the ``!glob`` tag to specify that a source-path value should be treated as a The ``--report`` output can be especially useful in combination with wildcards to allow downstream logic to process a set of copied files whose identity is not known in advance. -Note that directories are excluded, and recursive copies are not supported. To recursively copy from a shell script, where you might otherwise use ``uw fs copy``, consider using ``cp -r``; and from Python code where you might otherwise call ``uwtools.api.fs.copy()``, consider using the standard-library :python:`shutil.html#shutil.copytree`. For more control, including file-grained include and exclude, consider the unrivaled `rsync` `_, which can be installed from conda in case your system does not already provide it. It can be called from shell scripts, or via :python:`subprocess.html` from Python. +Note that directories are excluded, and recursive copies are not supported. To recursively copy from a shell script, where you might otherwise use ``uw fs copy``, consider using ``cp -r``; and from Python code where you might otherwise call ``uwtools.api.fs.copy()``, consider using the standard-library :python:`shutil.copytree `. For more control, including file-grained include and exclude, consider the unrivaled `rsync `_, which can be installed from conda in case your system does not already provide it. It can be called from shell scripts, or via :python:`subprocess ` from Python. -See :ref:`files_yaml` for more information on the semantics of the `!glob` tag and wildcard copies. +See :ref:`files_yaml` for more information on the semantics of the ``!glob`` tag and wildcard copies. ``link`` -------- From 20a8bdac2af766ef74f0ff2c16ffd85b860a9a21 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 19:59:52 +0000 Subject: [PATCH 23/25] Remove -basic from filenames --- docs/sections/user_guide/cli/tools/fs.rst | 16 ++++++++-------- .../user_guide/cli/tools/fs/copy-basic.cmd | 4 ---- .../cli/tools/fs/copy-no-target-dir-err.cmd | 2 +- docs/sections/user_guide/cli/tools/fs/copy.cmd | 4 ++++ .../cli/tools/fs/{copy-basic.out => copy.out} | 0 .../cli/tools/fs/{copy-basic.yaml => copy.yaml} | 0 .../cli/tools/fs/makedirs-no-target-dir-err.cmd | 2 +- .../fs/{makedirs-basic.cmd => makedirs.cmd} | 2 +- .../fs/{makedirs-basic.out => makedirs.out} | 0 .../fs/{makedirs-basic.yaml => makedirs.yaml} | 0 10 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 docs/sections/user_guide/cli/tools/fs/copy-basic.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/copy.cmd rename docs/sections/user_guide/cli/tools/fs/{copy-basic.out => copy.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{copy-basic.yaml => copy.yaml} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-basic.cmd => makedirs.cmd} (78%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-basic.out => makedirs.out} (100%) rename docs/sections/user_guide/cli/tools/fs/{makedirs-basic.yaml => makedirs.yaml} (100%) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index 2507c7e9c..58e9ae705 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -29,12 +29,12 @@ Examples Given a config containing a mapping from local-filesystem destination paths to source paths -.. literalinclude:: fs/copy-basic.yaml +.. literalinclude:: fs/copy.yaml :language: yaml -.. literalinclude:: fs/copy-basic.cmd +.. literalinclude:: fs/copy.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-basic.out +.. literalinclude:: fs/copy.out :language: text Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source. @@ -51,7 +51,7 @@ The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime The ``--target-dir`` option need not be specified when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/copy-basic.yaml +.. literalinclude:: fs/copy.yaml :language: yaml .. literalinclude:: fs/copy-no-target-dir-err.cmd :language: text @@ -124,12 +124,12 @@ Examples Given a config containing -.. literalinclude:: fs/makedirs-basic.yaml +.. literalinclude:: fs/makedirs.yaml :language: yaml -.. literalinclude:: fs/makedirs-basic.cmd +.. literalinclude:: fs/makedirs.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/makedirs-basic.out +.. literalinclude:: fs/makedirs.out :language: text The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: @@ -144,7 +144,7 @@ The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime The ``--target-dir`` option need not be specified when all directory paths are absolute, and will never be applied to absolute paths. If any paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/makedirs-basic.yaml +.. literalinclude:: fs/makedirs.yaml :language: yaml .. literalinclude:: fs/makedirs-no-target-dir-err.cmd :language: text diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic.cmd b/docs/sections/user_guide/cli/tools/fs/copy-basic.cmd deleted file mode 100644 index 5a1369b93..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-basic.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf dst/copy -uw fs copy --target-dir dst/copy --config-file copy-basic.yaml --key-path config.files -echo -tree dst/copy diff --git a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd index 9c189bd9d..90e82985d 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd +++ b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd @@ -1 +1 @@ -uw fs copy --config-file copy-basic.yaml --key-path config.files +uw fs copy --config-file copy.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/copy.cmd b/docs/sections/user_guide/cli/tools/fs/copy.cmd new file mode 100644 index 000000000..02dacbcc0 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy.cmd @@ -0,0 +1,4 @@ +rm -rf dst/copy +uw fs copy --target-dir dst/copy --config-file copy.yaml --key-path config.files +echo +tree dst/copy diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic.out b/docs/sections/user_guide/cli/tools/fs/copy.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-basic.out rename to docs/sections/user_guide/cli/tools/fs/copy.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-basic.yaml b/docs/sections/user_guide/cli/tools/fs/copy.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-basic.yaml rename to docs/sections/user_guide/cli/tools/fs/copy.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd index 66d3c5c8f..28fc0a702 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd @@ -1 +1 @@ -uw fs makedirs --config-file makedirs-basic.yaml --key-path config +uw fs makedirs --config-file makedirs.yaml --key-path config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-basic.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs.cmd similarity index 78% rename from docs/sections/user_guide/cli/tools/fs/makedirs-basic.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs.cmd index 18a85c70d..1afd2c1a4 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-basic.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent -uw fs makedirs --target-dir makedirs-parent --config-file makedirs-basic.yaml --key-path config +uw fs makedirs --target-dir makedirs-parent --config-file makedirs.yaml --key-path config echo tree -F makedirs-parent diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-basic.out b/docs/sections/user_guide/cli/tools/fs/makedirs.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-basic.out rename to docs/sections/user_guide/cli/tools/fs/makedirs.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-basic.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-basic.yaml rename to docs/sections/user_guide/cli/tools/fs/makedirs.yaml From 27cde474a7e09566f5ee0c2607ff7cb3b5c67a67 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 20:19:26 +0000 Subject: [PATCH 24/25] Revert change to link-help.out --- docs/sections/user_guide/cli/tools/fs/link-help.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.out b/docs/sections/user_guide/cli/tools/fs/link-help.out index efe789d10..69319e3b6 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-help.out +++ b/docs/sections/user_guide/cli/tools/fs/link-help.out @@ -14,7 +14,7 @@ Optional arguments: --target-dir PATH Root directory for relative destination paths --cycle CYCLE - The cycle in ISO8601 format (e.g. 2025-02-15T12) + The cycle in ISO8601 format (e.g. 2024-05-29T12) --leadtime LEADTIME The leadtime as hours[:minutes[:seconds]] --dry-run From b2c56e74bd3e69ad97662472e15d92d7c0e9551b Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 15 Feb 2025 20:44:02 +0000 Subject: [PATCH 25/25] Doc update --- docs/sections/user_guide/yaml/files.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/yaml/files.rst b/docs/sections/user_guide/yaml/files.rst index e54e76463..3f98114db 100644 --- a/docs/sections/user_guide/yaml/files.rst +++ b/docs/sections/user_guide/yaml/files.rst @@ -69,7 +69,7 @@ Example block: a/: !glob /src/a* b/: !glob /src/b* -Given ``src/`` directory +Given ``/src/`` directory .. code-block:: text