diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2b885939d..2113b3b9e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,12 +3,15 @@ on: pull_request: branches: - develop + - main push: branches: - develop + - main workflow_dispatch: branches: - develop + - main jobs: test: name: Test diff --git a/docs/conf.py b/docs/conf.py index 124309e1f..5130b4045 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,6 +29,7 @@ "anaconda": ("https://www.anaconda.com/%s", "%s"), "anaconda-condev": ("https://anaconda.org/maddenp/condev/%s", "%s"), "black": ("https://black.readthedocs.io/en/stable/%s", "%s"), + "cmeps": ("https://escomp.github.io/CMEPS/versions/master/html/esmflds.html#%s", "%s"), "conda": ("https://docs.conda.io/en/latest/%s", "%s"), "conda-forge": ("https://conda-forge.org/%s", "%s"), "condev": ("https://github.com/maddenp/condev/%s", "%s"), @@ -50,7 +51,9 @@ "rst": ("https://www.sphinx-doc.org/en/master/usage/restructuredtext/%s", "%s"), "rtd": ("https://readthedocs.org/projects/uwtools/%s", "%s"), "ufs": ("https://ufscommunity.org/%s", "%s"), + "ufs-weather-model": ("https://github.com/ufs-community/ufs-weather-model/%s", "%s"), "uwtools": ("https://github.com/ufs-community/uwtools/%s", "%s"), + "weather-model-io": ("https://ufs-weather-model.readthedocs.io/en/latest/InputsOutputs.html#%s", "%s"), } def setup(app): diff --git a/docs/sections/contributor_guide/documentation.rst b/docs/sections/contributor_guide/documentation.rst index 5bba6641b..800f0b950 100644 --- a/docs/sections/contributor_guide/documentation.rst +++ b/docs/sections/contributor_guide/documentation.rst @@ -40,10 +40,12 @@ Please follow these guidelines when contributing to the documentation: * Use one blank line between documentation elements (headers, paragraphs, code blocks, etc.) unless additional lines are necessary to achieve correctly formatted HTML. * Remove all trailing whitespace. * In general, avoid pronouns like "we" and "you". (Using "we" may be appropriate when synonymous with "The UW Team", "The UFS Community", etc., when the context is clear.) Prefer direct, factual statements about what the code does, requires, etc. -* The synopsis information printed by ``uw [mode [submode]] --help`` is automatically wrapped and indented based on current terminal size. For visual consistency, please set your terminal width to 100 columns when running such commands to produce output to copy into the docs. +* Use the `Oxford Comma `_. +* The synopsis information printed by ``uw [mode [action]] --help`` is automatically wrapped and indented based on current terminal size. For visual consistency, please set your terminal width to 100 columns when running such commands to produce output to copy into the docs. * Follow the :rst:`RST Sections` guidelines, underlining section headings with = characters, subsections with - characters, and subsubsections with ^ characters. If a further level of refinement is needed, use " to underline paragraph headers. * In [[sub]sub]section titles, capitalize all "principal" words. In practice this usually means all words but articles (a, an, the), logicals (and, etc.), and prepositions (for, of, etc.). Always fully capitalize acronyms (e.g., YAML). * Never capitalize proper names when their owners do not (e.g., write `"pandas" `_, not "Pandas", even at the start of a sentence) or when referring to a software artifact (e.g., write ``numpy`` when referring to the library, and "NumPy" when referring to the project). +* When referring to YAML constructs, `block` refers to an entry whose values is a nested collection of key/value pairs, while `entry` is a single key/value pair. * When using the ``.. code-block::`` directive, align the actual code with the word ``code``. Also, when ``.. code-block::`` directives appear in bulleted or numberd lists, align them with the text following the space to the right of the bullet/number. For example: .. code-block:: text diff --git a/docs/sections/user_guide/cli/mode_config.rst b/docs/sections/user_guide/cli/mode_config.rst index 1f64cf50d..60bc061c2 100644 --- a/docs/sections/user_guide/cli/mode_config.rst +++ b/docs/sections/user_guide/cli/mode_config.rst @@ -41,21 +41,23 @@ The ``compare`` mode lets users compare two config files. Required arguments: --file-1-path PATH - Path to file 1 + Path to file 1 --file-2-path PATH - Path to file 2 + Path to file 2 Optional arguments: -h, --help - Show help and exit + Show help and exit --file-1-format {ini,nml,sh,yaml} - Format of file 1 + Format of file 1 --file-2-format {ini,nml,sh,yaml} - Format of file 2 + Format of file 2 + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) --quiet, -q - Print no logging messages + Print no logging messages --verbose, -v - Print all logging messages + Print all logging messages Examples ^^^^^^^^ @@ -118,6 +120,8 @@ Users have several options: [2024-01-08T16:57:28] INFO --------------------------------------------------------------------- [2024-01-08T16:57:28] INFO values: recipient: - None + World + If additional information is needed, ``--debug`` can be used which will return the stack trace from any unhandled exception as well. + Note that ``uw`` logs to ``stderr``, so the stream can be redirected: .. code-block:: text @@ -134,6 +138,13 @@ Users have several options: [2024-01-08T16:59:20] INFO --------------------------------------------------------------------- [2024-01-08T16:59:20] INFO values: recipient: - None + World +.. note:: Comparisons are supported only for configs of the same format, e.g. YAML vs YAML, Fortran namelist vs Fortran namelist, etc. ``uw`` will flag invalid comparisons: + + .. code-block:: text + + $ uw config compare --file-1-path a.yaml --file-2-path b.nml + [2024-01-23T23:21:37] ERROR Formats do not match: yaml vs nml + .. _cli_config_realize_examples: ``realize`` @@ -143,39 +154,37 @@ In ``uw`` terminology, to realize a configuration file is to transform it from i .. code-block:: text - $ uw config realize --help - usage: uw config realize --values-file PATH [-h] [--input-file PATH] - [--input-format {ini,nml,sh,yaml}] [--output-file PATH] - [--output-format {ini,nml,sh,yaml}] [--values-format {ini,nml,sh,yaml}] - [--values-needed] [--dry-run] [--quiet] [--verbose] - - Realize config - - Required arguments: - --values-file PATH - Path to file providing override or interpolation values - - Optional arguments: - -h, --help - Show help and exit - --input-file PATH, -i PATH - Path to input file (defaults to stdin) - --input-format {ini,nml,sh,yaml} - Input format - --output-file PATH, -o PATH - Path to output file (defaults to stdout) - --output-format {ini,nml,sh,yaml} - Output format - --values-format {ini,nml,sh,yaml} - Values format - --values-needed - Print report of values needed to render template - --dry-run - Only log info, making no changes - --quiet, -q - Print no logging messages - --verbose, -v - Print all logging messages + $ uw config realize --help + usage: uw config realize [-h] [--input-file PATH] [--input-format {ini,nml,sh,yaml}] + [--output-file PATH] [--output-format {ini,nml,sh,yaml}] + [--values-needed] [--dry-run] [--quiet] [--verbose] + [PATH ...] + + Realize config + + Optional arguments: + -h, --help + Show help and exit + --input-file PATH, -i PATH + Path to input file (defaults to stdin) + --input-format {ini,nml,sh,yaml} + Input format + --output-file PATH, -o PATH + Path to output file (defaults to stdout) + --output-format {ini,nml,sh,yaml} + Output format + --values-needed + Print report of values needed to render template + --dry-run + Only log info, making no changes + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages + PATH + Additional files to supplement primary input Examples ^^^^^^^^ @@ -215,18 +224,18 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con .. code-block:: text $ uw config realize --input-file config.yaml --output-format yaml --values-needed - [2024-01-10T21:29:20] INFO Keys that are complete: - [2024-01-10T21:29:20] INFO values - [2024-01-10T21:29:20] INFO values.greeting - [2024-01-10T21:29:20] INFO values.message - [2024-01-10T21:29:20] INFO values.recipient - [2024-01-10T21:29:20] INFO values.repeat - [2024-01-10T21:29:20] INFO - [2024-01-10T21:29:20] INFO Keys with unrendered Jinja2 variables/expressions: - [2024-01-10T21:29:20] INFO values.date: {{ yyyymmdd }} - [2024-01-10T21:29:20] INFO - [2024-01-10T21:29:20] INFO Keys that are set to empty: - [2024-01-10T21:29:20] INFO values.empty + [2024-01-23T22:28:40] INFO Keys that are complete: + [2024-01-23T22:28:40] INFO values + [2024-01-23T22:28:40] INFO values.greeting + [2024-01-23T22:28:40] INFO values.message + [2024-01-23T22:28:40] INFO values.recipient + [2024-01-23T22:28:40] INFO values.repeat + [2024-01-23T22:28:40] INFO + [2024-01-23T22:28:40] INFO Keys with unrendered Jinja2 variables/expressions: + [2024-01-23T22:28:40] INFO values.date: {{ yyyymmdd }} + [2024-01-23T22:28:40] INFO + [2024-01-23T22:28:40] INFO Keys that are set to empty: + [2024-01-23T22:28:40] INFO values.empty * To realize the config to ``stdout``, a target output format must be explicitly specified: @@ -237,7 +246,7 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: '{{ yyyymmdd }}' empty: null greeting: Hello - message: Hello World + message: 'Hello World ' recipient: World repeat: 1 @@ -252,7 +261,7 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: 20240105 empty: false greeting: Good Night - message: Good Night Sun Good Night Sun Good Night Sun + message: 'Good Night Sun Good Night Sun Good Night Sun ' recipient: Moon repeat: 3 @@ -270,7 +279,7 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: 20240105 empty: null greeting: Good Night - message: Good Night Moon Good Night Moon + message: 'Good Night Moon Good Night Moon ' recipient: Moon repeat: 2 @@ -279,13 +288,13 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con .. code-block:: text $ uw config realize --input-file config.yaml --output-file realized.yaml --dry-run values1.yaml - [2024-01-10T21:38:32] INFO values: - [2024-01-10T21:38:32] INFO date: 20240105 - [2024-01-10T21:38:32] INFO empty: null - [2024-01-10T21:38:32] INFO greeting: Good Night - [2024-01-10T21:38:32] INFO message: Good Night Moon Good Night Moon - [2024-01-10T21:38:32] INFO recipient: Moon - [2024-01-10T21:38:32] INFO repeat: 2 + [2024-01-23T22:31:08] INFO values: + [2024-01-23T22:31:08] INFO date: 20240105 + [2024-01-23T22:31:08] INFO empty: null + [2024-01-23T22:31:08] INFO greeting: Good Night + [2024-01-23T22:31:08] INFO message: 'Good Night Moon Good Night Moon ' + [2024-01-23T22:31:08] INFO recipient: Moon + [2024-01-23T22:31:08] INFO repeat: 2 * If an input file is read alone from ``stdin``, ``uw`` will not know how to parse its contents: @@ -303,7 +312,7 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: 20240105 empty: null greeting: Good Night - message: Good Night Moon Good Night Moon + message: 'Good Night Moon Good Night Moon ' recipient: Moon repeat: 2 @@ -323,7 +332,7 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: 20240105 empty: null greeting: Good Night - message: Good Night Moon Good Night Moon + message: 'Good Night Moon Good Night Moon ' recipient: Moon repeat: 2 @@ -332,40 +341,32 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con .. code-block:: text $ uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml - [2024-01-10T21:42:17] DEBUG Command: uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml - [2024-01-10T21:42:17] DEBUG Before update, config has depth 2 - [2024-01-10T21:42:17] DEBUG Supplemental config has depth 2 - [2024-01-10T21:42:17] DEBUG After update, config has depth 2 - [2024-01-10T21:42:17] DEBUG Dereferencing, initial value: {'values': {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': '{{ (greeting + " " + recipient + " ") * repeat }}', 'recipient': 'Moon', 'repeat': 2}} - [2024-01-10T21:42:17] DEBUG Rendering: {'values': {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': '{{ (greeting + " " + recipient + " ") * repeat }}', 'recipient': 'Moon', 'repeat': 2}} - [2024-01-10T21:42:17] DEBUG Rendering: {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': '{{ (greeting + " " + recipient + " ") * repeat }}', 'recipient': 'Moon', 'repeat': 2} - [2024-01-10T21:42:17] DEBUG Rendering: 20240105 - [2024-01-10T21:42:17] DEBUG Rendered: 20240105 - [2024-01-10T21:42:17] DEBUG Rendering: None - [2024-01-10T21:42:17] DEBUG Rendered: None - [2024-01-10T21:42:17] DEBUG Rendering: Good Night - [2024-01-10T21:42:17] DEBUG Rendering: {{ (greeting + " " + recipient + " ") * repeat }} - [2024-01-10T21:42:17] DEBUG Rendering: Moon - [2024-01-10T21:42:17] DEBUG Rendering: 2 - [2024-01-10T21:42:17] DEBUG Rendered: 2 - [2024-01-10T21:42:17] DEBUG Dereferencing, current value: {'values': {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': '{{ (greeting + " " + recipient + " ") * repeat }}', 'recipient': 'Moon', 'repeat': 2}} - [2024-01-10T21:42:17] DEBUG Rendering: {'values': {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': 'Good Night Moon Good Night Moon', 'recipient': 'Moon', 'repeat': 2}} - [2024-01-10T21:42:17] DEBUG Rendering: {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': 'Good Night Moon Good Night Moon', 'recipient': 'Moon', 'repeat': 2} - [2024-01-10T21:42:17] DEBUG Rendering: 20240105 - [2024-01-10T21:42:17] DEBUG Rendered: 20240105 - [2024-01-10T21:42:17] DEBUG Rendering: None - [2024-01-10T21:42:17] DEBUG Rendered: None - [2024-01-10T21:42:17] DEBUG Rendering: Good Night - [2024-01-10T21:42:17] DEBUG Rendering: Good Night Moon Good Night Moon - [2024-01-10T21:42:17] DEBUG Rendering: Moon - [2024-01-10T21:42:17] DEBUG Rendering: 2 - [2024-01-10T21:42:17] DEBUG Rendered: 2 - [2024-01-10T21:42:17] DEBUG Dereferencing, final value: {'values': {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': 'Good Night Moon Good Night Moon', 'recipient': 'Moon', 'repeat': 2}} + [2024-01-23T22:59:58] DEBUG Command: uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml + [2024-01-23T22:59:58] DEBUG Before update, config has depth 2 + [2024-01-23T22:59:58] DEBUG Supplemental config has depth 2 + [2024-01-23T22:59:58] DEBUG After update, config has depth 2 + [2024-01-23T22:59:58] DEBUG Dereferencing, current value: + [2024-01-23T22:59:58] DEBUG values: + [2024-01-23T22:59:58] DEBUG date: 20240105 + [2024-01-23T22:59:58] DEBUG empty: null + [2024-01-23T22:59:58] DEBUG greeting: Good Night + [2024-01-23T22:59:58] DEBUG message: '{{ (greeting + " " + recipient + " ") * repeat }}' + [2024-01-23T22:59:58] DEBUG recipient: Moon + [2024-01-23T22:59:58] DEBUG repeat: 2 + ... + [2024-01-23T22:59:58] DEBUG Dereferencing, final value: + [2024-01-23T22:59:58] DEBUG values: + [2024-01-23T22:59:58] DEBUG date: 20240105 + [2024-01-23T22:59:58] DEBUG empty: null + [2024-01-23T22:59:58] DEBUG greeting: Good Night + [2024-01-23T22:59:58] DEBUG message: 'Good Night Moon Good Night Moon ' + [2024-01-23T22:59:58] DEBUG recipient: Moon + [2024-01-23T22:59:58] DEBUG repeat: 2 values: date: 20240105 empty: null greeting: Good Night - message: Good Night Moon Good Night Moon + message: 'Good Night Moon Good Night Moon ' recipient: Moon repeat: 2 @@ -383,7 +384,7 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: 20240105 empty: null greeting: Good Night - message: Good Night Moon Good Night Moon + message: 'Good Night Moon Good Night Moon ' recipient: Moon repeat: 2 @@ -391,25 +392,65 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con .. code-block:: text - [2024-01-10T21:43:58] DEBUG Command: uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml - [2024-01-10T21:43:58] DEBUG Before update, config has depth 2 - [2024-01-10T21:43:58] DEBUG Supplemental config has depth 2 + [2024-01-23T23:01:23] DEBUG Command: uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml + [2024-01-23T23:01:23] DEBUG Before update, config has depth 2 + [2024-01-23T23:01:23] DEBUG Supplemental config has depth 2 + [2024-01-23T23:01:23] DEBUG After update, config has depth 2 + [2024-01-23T23:01:23] DEBUG Dereferencing, current value: + [2024-01-23T23:01:23] DEBUG values: + [2024-01-23T23:01:23] DEBUG date: 20240105 + [2024-01-23T23:01:23] DEBUG empty: null + [2024-01-23T23:01:23] DEBUG greeting: Good Night + [2024-01-23T23:01:23] DEBUG message: '{{ (greeting + " " + recipient + " ") * repeat }}' + [2024-01-23T23:01:23] DEBUG recipient: Moon + [2024-01-23T23:01:23] DEBUG repeat: 2 + [2024-01-23T23:01:23] DEBUG [dereference] Accepting: 20240105 + [2024-01-23T23:01:23] DEBUG [dereference] Accepting: None + [2024-01-23T23:01:23] DEBUG [dereference] Rendering: Good Night + [2024-01-23T23:01:23] DEBUG [dereference] Rendered: Good Night + [2024-01-23T23:01:23] DEBUG [dereference] Rendering: {{ (greeting + " " + recipient + " ") * repeat }} + [2024-01-23T23:01:23] DEBUG [dereference] Rendered: Good Night Moon Good Night Moon + [2024-01-23T23:01:23] DEBUG [dereference] Rendering: Moon + [2024-01-23T23:01:23] DEBUG [dereference] Rendered: Moon + [2024-01-23T23:01:23] DEBUG [dereference] Accepting: 2 ... - [2024-01-10T21:43:58] DEBUG Rendering: 2 - [2024-01-10T21:43:58] DEBUG Rendered: 2 - [2024-01-10T21:43:58] DEBUG Dereferencing, final value: {'values': {'date': 20240105, 'empty': None, 'greeting': 'Good Night', 'message': 'Good Night Moon Good Night Moon', 'recipient': 'Moon', 'repeat': 2}} + [2024-01-23T23:01:23] DEBUG Dereferencing, final value: + [2024-01-23T23:01:23] DEBUG values: + [2024-01-23T23:01:23] DEBUG date: 20240105 + [2024-01-23T23:01:23] DEBUG empty: null + [2024-01-23T23:01:23] DEBUG greeting: Good Night + [2024-01-23T23:01:23] DEBUG message: 'Good Night Moon Good Night Moon ' + [2024-01-23T23:01:23] DEBUG recipient: Moon + [2024-01-23T23:01:23] DEBUG repeat: 2 -.. note:: +.. note:: Combining configs with incompatible depths is not supported. ``ini`` and ``nml`` configs are depth-2, as they organize their key-value pairs (one level) under top-level sections or namelists (a second level). ``sh`` configs are depth-1, and ``yaml`` configs have arbitrary depth. - Note that ``uw`` does not allow invalid conversions. For example, when attempting to generate a ``sh`` config from a depth-2 ``yaml``: + For example, when attempting to generate a ``sh`` config from a depth-2 ``yaml``: .. code-block:: text $ uw config realize --input-file config.yaml --output-format sh - [2024-01-10T21:46:00] ERROR Cannot realize depth-2 config to type-'sh' config + [2024-01-23T23:02:42] ERROR Cannot realize depth-2 config to type-'sh' config Cannot realize depth-2 config to type-'sh' config - By definition, ``ini`` and ``nml`` configs are depth-2 configs, while ``sh`` configs are depth-1 and ``yaml`` configs have arbitrary depth. +.. note:: In recognition of the different sets of value types representable in each config format, ``uw`` supports two format-combination schemes: + + 1. Output matches input: The format of the output config matches that of the input config. + 2. Input is YAML: If the input config is YAML, any output format may be requested. In the worst case, values always have a string representation, but note that, for example, the string representation of a YAML sequence (Python ``list``) in an INI output config may not be useful. + + In all cases, any supplemental configs must be in the same format as the input config and must have recognized extensions. + + ``uw`` considers invalid combination requests errors: + + .. code-block:: text + + $ uw config realize --input-file b.nml --output-file a.yaml + Output format yaml must match input format nml + + .. code-block:: text + + $ uw config realize --input-file a.yaml --output-file c.yaml b.nml + Supplemental config #1 format nml must match input format yaml .. _cli_config_validate_examples: @@ -434,6 +475,8 @@ The ``validate`` mode ensures that a given config file is structured properly. Show help and exit --input-file PATH, -i PATH Path to input file (defaults to stdin) + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/mode_forecast.rst b/docs/sections/user_guide/cli/mode_forecast.rst index 6eafe3c90..bb3d9c854 100644 --- a/docs/sections/user_guide/cli/mode_forecast.rst +++ b/docs/sections/user_guide/cli/mode_forecast.rst @@ -45,6 +45,8 @@ The ``uw`` mode for configuring and running forecasts. Path to output batch file (defaults to stdout) --dry-run Only log info, making no changes + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) --quiet, -q Print no logging messages --verbose, -v @@ -55,4 +57,32 @@ The ``uw`` mode for configuring and running forecasts. Examples ^^^^^^^^ -TBD +The examples use a configuration file named ``config.yaml``. Its contents are described in depth in Section :ref:`forecast_yaml`. + +* Run an FV3 forecast on an interactive node + + .. code-block:: sh + + $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 + + The forecast will run on the node where you have invoked this command. Optionally, capture the output in a log file using shell redirection. + +* Run an FV3 forecast using a batch system + + .. code-block:: sh + + $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 --batch-script submit_fv3.sh + + This command writes a file named ``submit_fv3.sh`` and submits it to the batch system. + +* With the ``--dry-run`` flag specified, nothing is written to ``stdout``, but a report of all the directories, files, symlinks, etc., that would have been created are logged to ``stderr``. None of these artifacts will actually be created and no jobs will be executed or submitted to the batch system. + + .. code-block:: sh + + $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 --batch-script --dry-run + +* Request verbose log output + + .. code-block:: sh + + $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 -v diff --git a/docs/sections/user_guide/cli/mode_rocoto.rst b/docs/sections/user_guide/cli/mode_rocoto.rst index 934c44260..ae3c97be2 100644 --- a/docs/sections/user_guide/cli/mode_rocoto.rst +++ b/docs/sections/user_guide/cli/mode_rocoto.rst @@ -44,6 +44,8 @@ More information about the structured UW YAML file for Rocoto can be found :any: Path to input file (defaults to stdin) --output-file PATH, -o PATH Path to output file (defaults to stdout) + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) --quiet, -q Print no logging messages --verbose, -v @@ -216,6 +218,8 @@ The examples in this section use a UW YAML file called ``rocoto.yaml`` with the Show help and exit --input-file PATH, -i PATH Path to input file (defaults to stdin) + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/mode_template.rst b/docs/sections/user_guide/cli/mode_template.rst index 41fa35031..acd0f84fe 100644 --- a/docs/sections/user_guide/cli/mode_template.rst +++ b/docs/sections/user_guide/cli/mode_template.rst @@ -51,6 +51,8 @@ The ``uw`` mode for handling :jinja2:`Jinja2 templates`. Print report of values needed to render template --dry-run Only log info, making no changes + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) --quiet, -q Print no logging messages --verbose, -v @@ -173,6 +175,8 @@ and a YAML file called ``values.yaml`` with the following contents: [2023-12-18T23:25:01] DEBUG Read initial values from values.yaml Hello, World! + If additional information is needed, ``--debug`` can be used which will return the stack trace from any unhandled exception as well. + Note that ``uw`` logs to ``stderr`` and writes non-log output to ``stdout``, so the streams can be redirected separately: .. code-block:: text @@ -248,6 +252,8 @@ and a YAML file called ``values.yaml`` with the following contents: Path to output file (defaults to stdout) --dry-run Only log info, making no changes + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/uw_yaml/field_table_yaml.rst b/docs/sections/user_guide/uw_yaml/field_table_yaml.rst new file mode 100644 index 000000000..43888ac5e --- /dev/null +++ b/docs/sections/user_guide/uw_yaml/field_table_yaml.rst @@ -0,0 +1,52 @@ +.. _defining_a_field_table: + +Defining a ``field_table`` +========================== + +The :ufs-weather-model:`UFS Weather Model<>` requires as one of its inputs (:weather-model-io:`documented here`) a ``field_table`` file in a :weather-model-io:`custom format`. Its contents can be defined in a UW YAML, then created with ``uwtools``. + +To generate a given ``field_table`` entry with the form: + +.. code-block:: text + + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=3.e-6" / + +Entries can be represented in YAML with the form: + +.. code-block:: text + + sphum: + longname: specific humidity + units: kg/kg + profile_type: + name: fixed + surface_value: 3.e-6 + + +Additional tracers may be added at the same level of YAML indentation as ``sphum:`` in the above example. + +UW YAML Keys +------------ + +``sphum:`` +^^^^^^^^^^ + +The short name of the tracer, to create the corresponding ``field_table`` entry's first line. + +``longname:`` +^^^^^^^^^^^^^ + +The descriptive name of the tracer. + +``units:`` +^^^^^^^^^^ + +The units of the tracer, to create the corresponding ``field_table`` entry's ``"units"`` line. + +``profile_type:`` +^^^^^^^^^^^^^^^^^ + +This block requires a ``name:`` entry that describes the profile type. Acceptable values (per the ``field_table`` spec) are ``fixed`` or ``profile``. The ``surface_value:`` is required in both cases, while the ``top_value:`` is required for only the ``profile`` option. diff --git a/docs/sections/user_guide/uw_yaml/forecast_yaml.rst b/docs/sections/user_guide/uw_yaml/forecast_yaml.rst new file mode 100644 index 000000000..a046d7fa1 --- /dev/null +++ b/docs/sections/user_guide/uw_yaml/forecast_yaml.rst @@ -0,0 +1,240 @@ +.. _forecast_yaml: + +Forecast YAML +============= + +The structured YAML to run a forecast is described below. It is enforced via JSON Schema. + +In this block, there are entries that define where the forecast will run (``run_directory:``), what will run (``executable:``), the data files that should be staged (``cycle_dependent:`` and ``static:``), and blocks that correspond to the configuration files required by the forecast model (``fd_ufs:``, ``field_table:``, ``namelist:``, etc.). Each of the configuration file blocks will allow the user to set a template file as input, or to define a configuration file in its native key/value pair format with an option to update the values (``update_values:``) contained in the original input file (``base_file:``). + +The configuration files required by the UFS Weather Model are documented :weather-model-io:`here`. + +The ``forecast:`` block +----------------------- + +This section describes the specifics of the FV3 atmosphere forecast component. + +.. code-block:: yaml + + forecast: + cycle_dependent: + INPUT/gfs_data.nc: /path/to/gfs_data.nc + INPUT/sfc_data.nc: /path/to/sfc_data.nc + INPUT/gfs_ctrl.nc: /path/to/gfs_ctrl.nc + diag_table: /path/to/diag_table/template/file + domain: regional + executable: fv3.exe + fd_ufs: + base_file: /path/to/base/fd_ufs.yaml + field_table: + base_file: /path/to/field_table.yaml + update_values: + liq_wat: + longname: cloud water mixing ratio + units: kg/kg + profile_type: + name: fixed + surface_value: 2.0 + length: 12 + model_configure: + base_file: /path/to/base/model_configure + update_values: + write_dopost: .false. + namelist: + base_file: /path/to/base/input.nml + update_values: + fv_core_nml: + k_split: 2 + n_split: 6 + run_dir: /path/to/forecast/run/directory/{yyyymmddhh} + static: + fv3.exe: /path/to/executable/ufs_model + INPUT/grid_spec.nc: /path/to/grid_spec.nc + ... + data_table: /path/to/data_table + eta_micro_lookup.data: /path/to/noahmptable.dat + noahmptable.tbl: /path/to/noahmptable.tbl + ufs_configure: /path/to/template/ufs.configure + +.. _updating_values: + +Updating Values +^^^^^^^^^^^^^^^ + +Many of the blocks describe configuration files needed by the UFS Weather Model, i.e. ``namelist:``, ``fd_ufs:``, ``model_configure:``. The ``base_file:`` entry in a given block is required to initially stage the file; it can then be modified via an ``update_values:`` block. + +To ensure the correct values are updated, the hierarchy of entries in the base file must be mirrored under the ``update_values:`` block. Multiple entries within a block may be updated and they need not follow the same order as those in the base file. For example, the base file named ``people.yaml`` may contain: + +.. code-block:: yaml + + person: + age: 19 + address: + city: Boston + number: 12 + state: MA + street: Acorn St + name: Jane + +Then the entries in the ``update_values:`` YAML block would override this base file with the entries: + +.. code-block:: yaml + + base_file: people.yaml + update_values: + person: + address: + street: Main St + number: 99 + +The contents of the staged ``people.yaml`` that results: + +.. code-block:: yaml + + person: + age: 19 + address: + city: Boston + number: 99 + state: MA + street: Main St + name: Jane + + +UW YAML Keys +^^^^^^^^^^^^ + +``cycle_dependent:`` +"""""""""""""""""""" + +This block contains a set of files to stage in the run directory: File names as they appear in the run directory are keys and their source paths are the values. Source paths can be provided as a single string path, or a list of paths to be staged in a common directory under their original names. + + .. warning:: The current version does not support adding cycle information to the content of the files, and this information must be hard-coded in the YAML file. + +``diag_table:`` +""""""""""""""" + +The path to the input Jinja2 template for the ``diag_table`` file. + +The diag_table is described :weather-model-io:`here`. + +``domain:`` +""""""""""" + +A switch to differentiate between a global or regional configuration. Accepted values are ``global`` and ``regional``. + +``executable:`` +""""""""""""""" + +The path to the compiled executable. + +``fd_ufs:`` +"""""""""""" + +This block requires a ``base_file:`` entry that contains the path to the YAML file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. + +The ``fd_ufs.yaml`` file is a structured YAML used by the FV3 weather model. The tested version can be found in the :ufs-weather-model:`ufs-weather-model repository`. The naming convention for the dictionary entries are documented :cmeps:`here<>`. + +``field_table:`` +"""""""""""""""" + +The block requires a ``base_file:`` entry that contains the path to the YAML file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. + +If a predefined field table (i.e., not a configurable YAML) is to be used, include it in the ``static:`` block. + +The documentation for the ``field_table`` file is :weather-model-io:`here`. Information on how to structure the UW YAML for configuring a ``field_table`` is in the :ref:`defining_a_field_table` Section. + +``length:`` +""""""""""" + +The length of the forecast in hours. + +``model_configure:`` +"""""""""""""""""""" + +The block requires a ``base_file:`` entry that contains the path to the YAML file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. + +The documentation for the ``model_configure`` file is :weather-model-io:`here`. + +``namelist:`` +""""""""""""" + +The block requires a ``base_file:`` entry that contains the path to the namelist file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. + +The documentation for the FV3 namelist, ``input.nml`` is :weather-model-io:`here`. + +``run_dir:`` +"""""""""""" + +The path where the forecast input data will be staged and output data will appear after a successful forecast. + +``static:`` +""""""""""" + +This block contains a set of files to stage in the run directory: file names as they appear in the run directory are keys and their source paths are the values. Source paths can be provided as a single string path, or a list of paths to be staged in a common directory under their original names. + +``ufs_configure:`` + +""""""""""""""""""" + +The path to the input Jinja2 template for the ``ufs.configure`` file. + +The documentation for the ``ufs.configure`` file is :weather-model-io:`here`. + +The ``platform:`` block +----------------------- + +This block describes necessary facts about the computational platform. + +.. code-block:: yaml + + platform: + mpicmd: srun # required + scheduler: slurm + +``mpicmd:`` +^^^^^^^^^^^ +The MPI command used to run the model executable. Typical options are ``srun``, ``mpirun``, ``mpiexec``, etc. System administrators should be able to advise the appropriate choice, if needed. + +``scheduler:`` +^^^^^^^^^^^^^^ +The name of the batch system. Supported options are ``lfs``, ``pbs``, and ``slurm``. + +The ``preprocessing:`` block +---------------------------- + +.. code-block:: yaml + + preprocessing: + lateral_boundary_conditions: + interval_hours: 3 # optional, default + offset: 0 # optional, default + output_file_path: # required + +``lateral_boundary_conditions:`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The optional block describes how the lateral boundary conditions have been prepared for a limited-area configuration of the model forecast. It is required for a limited-area forecast. The following entries in its subtree are used for the forecast: + +``interval_hours:`` +""""""""""""""""""" +The integer number of hours setting how frequently the lateral boundary conditions will be used in the model forecast. + +``offset:`` +""""""""""" +The integer number of hours setting how many hours earlier the external model used for boundary conditions started compared to the desired forecast cycle. + +``output_file_path:`` +""""""""""""""""""""""""" +The path to the lateral boundary conditions files prepared for the forecast. It accepts the integer ``forecast_hour`` as a Python template, e.g., ``/path/to/srw.t00z.gfs_bndy.tile7.f{forecast_hour:03d}.nc``. + +The ``user:`` block +------------------- + +.. code-block:: yaml + + user: + account: my_account # optional + +``account:`` +^^^^^^^^^^^^ +The user account associated with the batch system. diff --git a/docs/sections/user_guide/uw_yaml/index.rst b/docs/sections/user_guide/uw_yaml/index.rst index 58882e841..a0fe1a7de 100644 --- a/docs/sections/user_guide/uw_yaml/index.rst +++ b/docs/sections/user_guide/uw_yaml/index.rst @@ -4,4 +4,6 @@ UW YAML .. toctree:: :maxdepth: 1 + field_table_yaml + forecast_yaml rocoto_yaml diff --git a/src/pyproject.toml b/src/pyproject.toml index c04b89d42..4e6036746 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -22,9 +22,8 @@ profile = "black" [tool.mypy] check_untyped_defs = true -follow_imports = "skip" -ignore_missing_imports = true pretty = true +warn_return_any = true [tool.pylint.messages_control] disable = [ diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index 6323469e5..4dc645a60 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -17,25 +17,19 @@ def compare( - config_a_path: DefinitePath, - config_a_format: str, - config_b_path: DefinitePath, - config_b_format: str, + config_1_path: DefinitePath, + config_2_path: DefinitePath, + config_1_format: Optional[str] = None, + config_2_format: Optional[str] = None, ) -> bool: """ - Compare two config files. - - :param config_a_path: Path to first config file - :param config_a_format: Format of first config file - :param config_b_path: Path to second config file - :param config_b_format: Format of second config file - :return: ``False`` if config files had differences, otherwise ``True`` + NB: This docstring is dynamically replaced: See compare.__doc__ definition below. """ return _compare( - config_a_path=config_a_path, - config_a_format=config_a_format, - config_b_path=config_b_path, - config_b_format=config_b_format, + config_1_path=config_1_path, + config_2_path=config_2_path, + config_1_format=config_1_format, + config_2_format=config_2_format, ) @@ -103,26 +97,7 @@ def realize( dry_run: bool = False, ) -> bool: """ - Realize an output config based on an input config and an optional values-providing config. - - If no input is specified, ``stdin`` is read. If no output is specified, ``stdout`` is written - to. A ``dict`` may also be provided as an input value. When a filename is specified for an input - or output, its format will be deduced from its extension, if possible. This can be overriden by - specifying the format explicitly, and it is required to do so for reads from ``stdin`` or writes - to ``stdout``, as no attempt is made to deduce the format of streamed data. - - If ``values_needed`` is ``True``, a report of values needed to realize the config is logged. In - ``dry_run`` mode, output is written to ``stderr``. - - :param input_config: Input config file (``None`` or unspecified => read ``stdin``) - :param input_format: Format of the input config - :param output_file: Output config file (``None`` or unspecified => write to ``stdout``) - :param output_format: Format of the output config - :param values: Source of values used to modify input - :param values_format: Format of values when sourced from file - :param values_needed: Report complete, missing, and template values - :param dry_run: Log output instead of writing to output - :return: ``True`` + NB: This docstring is dynamically replaced: See realize.__doc__ definition below. """ _realize( input_config=_ensure_config_arg_type(input_config), @@ -144,23 +119,7 @@ def realize_to_dict( dry_run: bool = False, ) -> dict: """ - Realize an output config based on an input config and an optional values-providing config. - - If no input is specified, ``stdin`` is read. When a filename is specified for an input, its - format will be deduced from its extension, if possible. This can be overriden by specifying the - format explicitly, and it is required to do so for reads from ``stdin``, as no attempt is made - to deduce the format of streamed data. A ``dict`` may also be provided as an input value. - - If ``values_needed`` is ``True``, a report of values needed to realize the config is logged. In - ``dry_run`` mode, output is written to ``stderr``. - - :param input_config: Input config file (``None`` or unspecified => read ``stdin``) - :param input_format: Format of the input config - :param values: Source of values used to modify input - :param values_format: Format of values when sourced from file - :param values_needed: Report complete, missing, and template values - :param dry_run: Log output instead of writing to output - :return: A ``dict`` representing the realized config + NB: This docstring is dynamically replaced: See realize_to_dict.__doc__ definition below. """ return _realize( input_config=_ensure_config_arg_type(input_config), @@ -203,3 +162,88 @@ def _ensure_config_arg_type( if isinstance(config, dict): return _YAMLConfig(config=config) return config + + +# Import-time code + +# pylint: disable=duplicate-code + +# The following statements dynamically interpolate values into functions' docstrings, which will not +# work if the docstrings are inlined in the functions. They must remain separate statements to avoid +# hardcoding values into them. + +compare.__doc__ = """ +Compare two config files. + +Recognized file extensions are: {extensions} + +:param config_1_path: Path to 1st config file +:param config_2_path: Path to 2nd config file +:param config_1_format: Format of 1st config file (optional if file's extension is recognized) +:param config_2_format: Format of 2nd config file (optional if file's extension is recognized) +:return: ``False`` if config files had differences, otherwise ``True`` +""".format( + extensions=", ".join(_FORMAT.extensions()) +).strip() + + +realize.__doc__ = """ +Realize an output config based on an input config and optional supplemental configs. + +If no input is specified, ``stdin`` is read. A ``dict`` or ``Config`` object may also be provided as +input. If no output is specified, ``stdout`` is written to. When an input or output filename is +specified, its format will be deduced from its extension, if possible. This can be overridden by +specifying the format explicitly, and it is required to do so for reads from ``stdin`` or writes to +``stdout``, as no attempt is made to deduce the format of streamed data. + +If optional supplemental configs (which may likewise be file paths or ``Config`` / ``dict`` objects) +are provided, they will be merged, in the order specified, onto the input config. The format of all +input configs must match. + +If the input-config format is YAML, any supported output format may be specified. For all other +input formats, the output format must match the input. + +If ``values_needed`` is ``True``, a report of values needed to realize the config is logged. In +``dry_run`` mode, output is written to ``stderr``. + +Recognized file extensions are: {extensions} + +:param input_config: Input config file (``None`` or unspecified => read ``stdin``) +:param input_format: Format of the input config (optional if file's extension is recognized) +:param output_file: Output config file (``None`` or unspecified => write to ``stdout``) +:param output_format: Format of the output config (optional if file's extension is recognized) +:param supplemental_configs: Configs to merge, in order, onto the input +:param values_needed: Report complete, missing, and template values +:param dry_run: Log output instead of writing to output +:return: ``True`` +""".format( + extensions=", ".join(_FORMAT.extensions()) +).strip() + + +realize_to_dict.__doc__ = """ +Realize an output config based on an input config and optional supplemental configs. + +If no input is specified, ``stdin`` is read. A ``dict`` or ``Config`` object may also be provided as +input. When an input filename is specified, its format will be deduced from its extension, if +possible. This can be overridden by specifying the format explicitly, and it is required to do so for +reads from ``stdin``, as no attempt is made to deduce the format of streamed data. + +If optional supplemental configs (which may likewise be file paths or ``Config`` / ``dict`` objects) +are provided, they will be merged, in the order specified, onto the input config. The format of all +input configs must match. + +If ``values_needed`` is ``True``, a report of values needed to realize the config is logged. In +``dry_run`` mode, output is written to ``stderr``. + +Recognized file extensions are: {extensions} + +:param input_config: Input config file (``None`` or unspecified => read ``stdin``) +:param input_format: Format of the input config (optional if file's extension is recognized) +:param supplemental_configs: Configs to merge, in order, onto the input +:param values_needed: Report complete, missing, and template values +:param dry_run: Log output instead of writing to output +:return: A ``dict`` representing the realized config +""".format( + extensions=", ".join(_FORMAT.extensions()) +).strip() diff --git a/src/uwtools/api/template.py b/src/uwtools/api/template.py index 9ca735681..10c6736af 100644 --- a/src/uwtools/api/template.py +++ b/src/uwtools/api/template.py @@ -18,7 +18,7 @@ def render( Render a Jinja2 template based on specified values. Primary values used to render the template are taken from the specified file. The format of the - values source will be deduced from the filename extension, if possible. This can be overriden + values source will be deduced from the filename extension, if possible. This can be overridden via the ``values_format`` argument. A ``dict`` object may alternatively be provided as the primary values source. If no input file is specified, ``stdin`` is read. If no output file is specified, ``stdout`` is written to. diff --git a/src/uwtools/apps/srw.py b/src/uwtools/apps/srw.py index 902ab890e..5941e8d1d 100644 --- a/src/uwtools/apps/srw.py +++ b/src/uwtools/apps/srw.py @@ -40,8 +40,8 @@ def load_config(self, config_file: str) -> None: ">%s" % f, ] cmd = " ".join(cmd_components) - result = execute(cmd=cmd) - if not result.success: + _, success = execute(cmd=cmd) + if not success: raise UWError(f"Command failed: {cmd}") elif file_type == FORMAT.yaml: shutil.copy2(config_file, "config.yaml") @@ -64,8 +64,8 @@ def create_experiment(self) -> None: """ # Note: This is a temporary path until parsing the SRW directory is implemented. cmd = "python generate_FV3LAM_wflow.py" - result = execute(cmd=cmd) - if not result.success: + _, success = execute(cmd=cmd) + if not success: raise UWError(f"Command failed: {cmd}") def create_manager_files(self) -> None: diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index e762d4e32..ad25977f0 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -23,12 +23,12 @@ from uwtools.logging import log, setup_logging from uwtools.utils.file import FORMAT, get_file_format -FORMATS = [FORMAT.ini, FORMAT.nml, FORMAT.sh, FORMAT.yaml] +FORMATS = list(FORMAT.formats().keys()) TITLE_REQ_ARG = "Required arguments" Args = Dict[str, Any] -SubmodeChecks = List[Callable[[Args], Args]] -ModeChecks = Dict[str, SubmodeChecks] +ActionChecks = List[Callable[[Args], Args]] +ModeChecks = Dict[str, ActionChecks] Checks = Dict[str, ModeChecks] @@ -44,7 +44,7 @@ def main() -> None: setup_logging(quiet=True) try: args, checks = _parse_args(sys.argv[1:]) - for check in checks[args[STR.mode]][args[STR.submode]]: + for check in checks[args[STR.mode]][args[STR.action]]: check(args) setup_logging(quiet=args[STR.quiet], verbose=args[STR.verbose]) log.debug("Command: %s %s", Path(sys.argv[0]).name, " ".join(sys.argv[1:])) @@ -56,6 +56,8 @@ def main() -> None: } sys.exit(0 if modes[args[STR.mode]](args) else 1) except Exception as e: # pylint: disable=broad-exception-caught + if _switch(STR.debug) in sys.argv: + log.exception(str(e)) _abort(str(e)) @@ -70,7 +72,7 @@ def _add_subparser_config(subparsers: Subparsers) -> ModeChecks: """ parser = _add_subparser(subparsers, STR.config, "Handle configs") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.submode) + subparsers = _add_subparsers(parser, STR.action) return { STR.compare: _add_subparser_config_compare(subparsers), STR.realize: _add_subparser_config_realize(subparsers), @@ -78,7 +80,7 @@ def _add_subparser_config(subparsers: Subparsers) -> ModeChecks: } -def _add_subparser_config_compare(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_config_compare(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: config compare @@ -101,14 +103,14 @@ def _add_subparser_config_compare(subparsers: Subparsers) -> SubmodeChecks: helpmsg="Format of file 2", choices=FORMATS, ) - checks = _add_args_quiet_and_verbose(optional) + checks = _add_args_verbosity(optional) return checks + [ partial(_check_file_vs_format, STR.file1path, STR.file1fmt), partial(_check_file_vs_format, STR.file2path, STR.file2fmt), ] -def _add_subparser_config_realize(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_config_realize(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: config realize @@ -122,7 +124,7 @@ def _add_subparser_config_realize(subparsers: Subparsers) -> SubmodeChecks: _add_arg_output_format(optional, choices=FORMATS) _add_arg_values_needed(optional) _add_arg_dry_run(optional) - checks = _add_args_quiet_and_verbose(optional) + checks = _add_args_verbosity(optional) _add_arg_supplemental_files(optional) return checks + [ partial(_check_file_vs_format, STR.infile, STR.infmt), @@ -130,7 +132,7 @@ def _add_subparser_config_realize(subparsers: Subparsers) -> SubmodeChecks: ] -def _add_subparser_config_validate(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_config_validate(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: config validate @@ -141,7 +143,7 @@ def _add_subparser_config_validate(subparsers: Subparsers) -> SubmodeChecks: _add_arg_schema_file(required) optional = _basic_setup(parser) _add_arg_input_file(optional) - return _add_args_quiet_and_verbose(optional) + return _add_args_verbosity(optional) def _dispatch_config(args: Args) -> bool: @@ -154,26 +156,26 @@ def _dispatch_config(args: Args) -> bool: STR.compare: _dispatch_config_compare, STR.realize: _dispatch_config_realize, STR.validate: _dispatch_config_validate, - }[args[STR.submode]](args) + }[args[STR.action]](args) def _dispatch_config_compare(args: Args) -> bool: """ - Dispatch logic for config compare submode. + Dispatch logic for config compare action. :param args: Parsed command-line args. """ return uwtools.api.config.compare( - config_a_path=args[STR.file1path], - config_a_format=args[STR.file1fmt], - config_b_path=args[STR.file2path], - config_b_format=args[STR.file2fmt], + config_1_path=args[STR.file1path], + config_1_format=args[STR.file1fmt], + config_2_path=args[STR.file2path], + config_2_format=args[STR.file2fmt], ) def _dispatch_config_realize(args: Args) -> bool: """ - Dispatch logic for config realize submode. + Dispatch logic for config realize action. :param args: Parsed command-line args. """ @@ -190,7 +192,7 @@ def _dispatch_config_realize(args: Args) -> bool: def _dispatch_config_validate(args: Args) -> bool: """ - Dispatch logic for config validate submode. + Dispatch logic for config validate action. :param args: Parsed command-line args. """ @@ -208,13 +210,13 @@ def _add_subparser_forecast(subparsers: Subparsers) -> ModeChecks: """ parser = _add_subparser(subparsers, STR.forecast, "Configure and run forecasts") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.submode) + subparsers = _add_subparsers(parser, STR.action) return { STR.run: _add_subparser_forecast_run(subparsers), } -def _add_subparser_forecast_run(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_forecast_run(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: forecast run @@ -228,7 +230,7 @@ def _add_subparser_forecast_run(subparsers: Subparsers) -> SubmodeChecks: optional = _basic_setup(parser) _add_arg_batch_script(optional) _add_arg_dry_run(optional) - checks = _add_args_quiet_and_verbose(optional) + checks = _add_args_verbosity(optional) return checks @@ -238,12 +240,12 @@ def _dispatch_forecast(args: Args) -> bool: :param args: Parsed command-line args. """ - return {STR.run: _dispatch_forecast_run}[args[STR.submode]](args) + return {STR.run: _dispatch_forecast_run}[args[STR.action]](args) def _dispatch_forecast_run(args: Args) -> bool: """ - Dispatch logic for forecast run submode. + Dispatch logic for forecast run action. :param args: Parsed command-line args. """ @@ -267,14 +269,14 @@ def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks: """ parser = _add_subparser(subparsers, STR.rocoto, "Realize and validate Rocoto XML Documents") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.submode) + subparsers = _add_subparsers(parser, STR.action) return { STR.realize: _add_subparser_rocoto_realize(subparsers), STR.validate: _add_subparser_rocoto_validate(subparsers), } -def _add_subparser_rocoto_realize(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_rocoto_realize(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: rocoto realize @@ -284,11 +286,11 @@ def _add_subparser_rocoto_realize(subparsers: Subparsers) -> SubmodeChecks: optional = _basic_setup(parser) _add_arg_input_file(optional) _add_arg_output_file(optional) - checks = _add_args_quiet_and_verbose(optional) + checks = _add_args_verbosity(optional) return checks -def _add_subparser_rocoto_validate(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_rocoto_validate(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: rocoto validate @@ -297,7 +299,7 @@ def _add_subparser_rocoto_validate(subparsers: Subparsers) -> SubmodeChecks: parser = _add_subparser(subparsers, STR.validate, "Validate Rocoto XML") optional = _basic_setup(parser) _add_arg_input_file(optional) - checks = _add_args_quiet_and_verbose(optional) + checks = _add_args_verbosity(optional) return checks @@ -311,13 +313,13 @@ def _dispatch_rocoto(args: Args) -> bool: STR.realize: _dispatch_rocoto_realize, STR.validate: _dispatch_rocoto_validate, }[ - args[STR.submode] + args[STR.action] ](args) def _dispatch_rocoto_realize(args: Args) -> bool: """ - Dispatch logic for rocoto realize submode. Validate input and output. + Dispatch logic for rocoto realize action. Validate input and output. :param args: Parsed command-line args. """ @@ -326,7 +328,7 @@ def _dispatch_rocoto_realize(args: Args) -> bool: def _dispatch_rocoto_validate(args: Args) -> bool: """ - Dispatch logic for rocoto validate submode. + Dispatch logic for rocoto validate action. :param args: Parsed command-line args. """ @@ -344,14 +346,14 @@ def _add_subparser_template(subparsers: Subparsers) -> ModeChecks: """ parser = _add_subparser(subparsers, STR.template, "Handle templates") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.submode) + subparsers = _add_subparsers(parser, STR.action) return { STR.render: _add_subparser_template_render(subparsers), STR.translate: _add_subparser_template_translate(subparsers), } -def _add_subparser_template_translate(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_template_translate(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: template translate @@ -362,10 +364,10 @@ def _add_subparser_template_translate(subparsers: Subparsers) -> SubmodeChecks: _add_arg_input_file(optional) _add_arg_output_file(optional) _add_arg_dry_run(optional) - return _add_args_quiet_and_verbose(optional) + return _add_args_verbosity(optional) -def _add_subparser_template_render(subparsers: Subparsers) -> SubmodeChecks: +def _add_subparser_template_render(subparsers: Subparsers) -> ActionChecks: """ Subparser for mode: template render @@ -379,7 +381,7 @@ def _add_subparser_template_render(subparsers: Subparsers) -> SubmodeChecks: _add_arg_values_format(optional, choices=FORMATS) _add_arg_values_needed(optional) _add_arg_dry_run(optional) - checks = _add_args_quiet_and_verbose(optional) + checks = _add_args_verbosity(optional) _add_arg_key_eq_val_pairs(optional) return checks + [_check_template_render_vals_args] @@ -394,13 +396,13 @@ def _dispatch_template(args: Args) -> bool: STR.render: _dispatch_template_render, STR.translate: _dispatch_template_translate, }[ - args[STR.submode] + args[STR.action] ](args) def _dispatch_template_render(args: Args) -> bool: """ - Dispatch logic for template render submode. + Dispatch logic for template render action. :param args: Parsed command-line args. """ @@ -417,7 +419,7 @@ def _dispatch_template_render(args: Args) -> bool: def _dispatch_template_translate(args: Args) -> bool: """ - Dispatch logic for template translate submode. + Dispatch logic for template translate action. :param args: Parsed command-line args. """ @@ -464,6 +466,16 @@ def _add_arg_cycle(group: Group) -> None: ) +def _add_arg_debug(group: Group) -> None: + group.add_argument( + "--debug", + action="store_true", + help=""" + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) + """, + ) + + def _add_arg_dry_run(group: Group) -> None: group.add_argument( _switch(STR.dryrun), @@ -636,16 +648,17 @@ def _abort(msg: str) -> None: sys.exit(1) -def _add_args_quiet_and_verbose(group: Group) -> SubmodeChecks: +def _add_args_verbosity(group: Group) -> ActionChecks: """ - Add quiet and verbose arguments. + Add debug, quiet, and verbose arguments. :param group: The group to add the arguments to. :return: Check for mutual exclusivity of quiet/verbose arguments. """ + _add_arg_debug(group) _add_arg_quiet(group) _add_arg_verbose(group) - return [_check_quiet_vs_verbose] + return [_check_verbosity] def _add_subparser(subparsers: Subparsers, name: str, helpmsg: str) -> Parser: @@ -657,9 +670,10 @@ def _add_subparser(subparsers: Subparsers, name: str, helpmsg: str) -> Parser: :param helpmsg: The help message for the subparser. :return: The new subparser. """ - return subparsers.add_parser( + parser: Parser = subparsers.add_parser( name, add_help=False, help=helpmsg, formatter_class=_formatter, description=helpmsg ) + return parser def _add_subparsers(parser: Parser, dest: str) -> Subparsers: @@ -693,12 +707,6 @@ def _check_file_vs_format(file_arg: str, format_arg: str, args: Args) -> Args: return args -def _check_quiet_vs_verbose(args) -> Args: - if args.get(STR.quiet) and args.get(STR.verbose): - _abort("Specify at most one of %s, %s" % (_switch(STR.quiet), _switch(STR.verbose))) - return args - - def _check_template_render_vals_args(args: Args) -> Args: # In "template render" mode, a values file is optional, as values used to render the template # will be taken from the environment or from key=value command-line pairs by default. But if a @@ -710,6 +718,15 @@ def _check_template_render_vals_args(args: Args) -> Args: return args +def _check_verbosity(args: Args) -> Args: + if args.get(STR.quiet) and (args.get(STR.debug) or args.get(STR.verbose)): + _abort( + "%s may not be used with %s or %s" + % (_switch(STR.quiet), _switch(STR.debug), _switch(STR.verbose)) + ) + return args + + def _dict_from_key_eq_val_strings(config_items: List[str]) -> Dict[str, str]: """ Given a list of key=value strings, return a dictionary of key/value pairs. @@ -765,11 +782,13 @@ class STR: A lookup map for CLI-related strings. """ + action: str = "action" batch_script: str = "batch_script" cfgfile: str = "config_file" compare: str = "compare" config: str = "config" cycle: str = "cycle" + debug: str = "debug" dryrun: str = "dry_run" file1fmt: str = "file_1_format" file1path: str = "file_1_path" @@ -790,7 +809,6 @@ class STR: rocoto: str = "rocoto" run: str = "run" schemafile: str = "schema_file" - submode: str = "submode" suppfiles: str = "supplemental_files" template: str = "template" translate: str = "translate" diff --git a/src/uwtools/config/formats/nml.py b/src/uwtools/config/formats/nml.py index 2d8aa3574..f48844859 100644 --- a/src/uwtools/config/formats/nml.py +++ b/src/uwtools/config/formats/nml.py @@ -1,7 +1,7 @@ from collections import OrderedDict from typing import Optional, Union -import f90nml +import f90nml # type: ignore from f90nml import Namelist from uwtools.config.formats.base import Config @@ -34,7 +34,8 @@ def _load(self, config_file: OptionalPath) -> dict: :param config_file: Path to config file to load. """ with readable(config_file) as f: - return f90nml.read(f) + config: dict = f90nml.read(f) + return config # Public methods diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index 23c6990b6..7bb698c04 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -59,7 +59,9 @@ def _load(self, config_file: OptionalPath) -> dict: loader = self._yaml_loader with readable(config_file) as f: try: - return yaml.load(f.read(), Loader=loader) + config = yaml.load(f.read(), Loader=loader) + assert isinstance(config, dict) + return config except yaml.constructor.ConstructorError as e: if e.problem: if "unhashable" in e.problem: diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index e03d045a1..fc9abc14f 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -337,7 +337,7 @@ def _set_up_values_obj( if values_format is None: values_format = get_file_format(values_file) values_class = format_to_config(values_format) - values = values_class(values_file).data + values: dict = values_class(values_file).data log.debug("Read initial values from %s", values_file) else: values = dict(os.environ) # Do not modify os.environ: Make a copy. diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index b78c595e9..aba3bbd63 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -38,7 +38,8 @@ def format_to_config(fmt: str) -> Type: } if not fmt in lookup: raise log_and_error("Format '%s' should be one of: %s" % (fmt, ", ".join(lookup))) - return getattr(import_module(f"uwtools.config.formats.{fmt}"), lookup[fmt]) + cfgclass: Type = getattr(import_module(f"uwtools.config.formats.{fmt}"), lookup[fmt]) + return cfgclass def log_and_error(msg: str) -> Exception: @@ -60,7 +61,7 @@ class TaggedString: method. See the pyyaml documentation for details. """ - TAGS = ("!float", "!int") + TAGS: Dict[str, type] = {"!float": float, "!int": int} def __init__(self, _: yaml.SafeLoader, node: yaml.nodes.ScalarNode) -> None: self.tag: str = node.tag @@ -75,7 +76,7 @@ def convert(self) -> Union[float, int]: Will raise an exception if the value cannot be represented as the specified type. """ - converters: Dict[str, type] = dict(zip(self.TAGS, [float, int])) + converters: Dict[str, Union[Type[float], Type[int]]] = dict(zip(self.TAGS, [float, int])) return converters[self.tag](self.value) @staticmethod diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index db6bc178f..7921cb6b3 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -16,27 +16,25 @@ def compare_configs( - config_a_path: DefinitePath, - config_a_format: str, - config_b_path: DefinitePath, - config_b_format: str, + config_1_path: DefinitePath, + config_2_path: DefinitePath, + config_1_format: Optional[str] = None, + config_2_format: Optional[str] = None, ) -> bool: """ - Compare two config files. - - :param config_a_path: Path to first config file. - :param config_a_format: Format of first config file. - :param config_b_path: Path to second config file. - :param config_b_format: Format of second config file. - :return: False if config files had differences, otherwise True. + NB: This docstring is dynamically replaced: See compare_configs.__doc__ definition below. """ - - cfg_a = format_to_config(config_a_format)(config_a_path) - cfg_b = format_to_config(config_b_format)(config_b_path) - log.info("- %s", config_a_path) - log.info("+ %s", config_b_path) + config_1_format = _ensure_format("1st config file", config_1_format, config_1_path) + config_2_format = _ensure_format("2nd config file", config_2_format, config_2_path) + if config_1_format != config_2_format: + log.error("Formats do not match: %s vs %s", config_1_format, config_2_format) + return False + cfg_1: Config = format_to_config(config_1_format)(config_1_path) + cfg_2: Config = format_to_config(config_2_format)(config_2_path) + log.info("- %s", config_1_path) + log.info("+ %s", config_2_path) log.info("-" * MSGWIDTH) - return cfg_a.compare_config(cfg_b.data) + return cfg_1.compare_config(cfg_2.data) def config_check_depths_dump(config_obj: Union[Config, dict], target_format: str) -> None: @@ -86,27 +84,19 @@ def realize_config( dry_run: bool = False, ) -> dict: """ - Realize an output config based on an input config and optional values-providing configs. - - :param input_config: Input config file (None => read stdin). - :param input_format: Format of the input config. - :param output_file: Output config file (None => write to stdout). - :param output_format: Format of the output config. - :param supplemental_configs: Sources of values used to modify input. - :param values_needed: Report complete, missing, and template values. - :param dry_run: Log output instead of writing to output. - :return: The realized config (or an empty-dict for no-op modes). + NB: This docstring is dynamically replaced: See realize_config.__doc__ definition below. """ input_format = _ensure_format("input", input_format, input_config) - input_obj = ( + output_format = _ensure_format("output", output_format, output_file) + _validate_format_output(input_format, output_format) + input_obj: Config = ( input_config if isinstance(input_config, Config) else format_to_config(input_format)(config=input_config) ) if supplemental_configs: - input_obj = _realize_config_update(input_obj, supplemental_configs) + input_obj = _realize_config_update(input_obj, input_format, supplemental_configs) input_obj.dereference() - output_format = _ensure_format("output", output_format, output_file) config_check_depths_realize(input_obj, output_format) if dry_run: for line in str(input_obj).strip().split("\n"): @@ -172,26 +162,29 @@ def _print_config_section(config: dict, key_path: List[str]) -> None: def _realize_config_update( config_obj: Config, + config_fmt: str, supplemental_configs: Optional[List[Union[dict, Config, DefinitePath]]] = None, ) -> Config: """ Update config with values from other configs, if given. :param config_obj: The config to update. + :param config_fmt: Format of config's source. :param supplemental_configs: Sources of values to modify input. :return: The input config, possibly updated. """ if supplemental_configs: log.debug("Before update, config has depth %s", config_obj.depth) supplemental_obj: Config - for config in supplemental_configs: - if isinstance(config, dict): - supplemental_obj = YAMLConfig(config=config) - elif isinstance(config, Config): - supplemental_obj = config + for idx, supplemental_config in enumerate(supplemental_configs): + _validate_format_supplemental(config_fmt, supplemental_config, idx) + if isinstance(supplemental_config, dict): + supplemental_obj = YAMLConfig(config=supplemental_config) + elif isinstance(supplemental_config, Config): + supplemental_obj = supplemental_config else: - supplemental_format = get_file_format(config) - supplemental_obj = format_to_config(supplemental_format)(config=config) + supplemental_format = get_file_format(supplemental_config) + supplemental_obj = format_to_config(supplemental_format)(config=supplemental_config) log.debug("Supplemental config has depth %s", supplemental_obj.depth) config_check_depths_update(supplemental_obj, config_obj.get_format()) config_obj.update_values(supplemental_obj) @@ -248,3 +241,80 @@ def _validate_depth( raise log_and_error( "Cannot %s depth-%s config to type-'%s' config" % (action, depth(config), target_format) ) + + +def _validate_format_output(input_fmt: str, output_fmt: str) -> None: + """ + Ensure output format agrees with input. + + :param input_fmt: Input format. + :param output_fmt: Output format. + :raises: UWError if output format is incompatible. + """ + if not input_fmt in (FORMAT.yaml, output_fmt): + raise UWError("Output format %s must match input format %s" % (output_fmt, input_fmt)) + + +def _validate_format_supplemental( + config_fmt: str, supplemental_cfg: Union[dict, Config, DefinitePath], idx: int +) -> None: + """ + Ensure supplemental config format agrees with base config format. + + :param config_fmt: Base config format. + :param supplemental_cfg: Supplemental config to check. + :param idx: Index of supplemental config for identification purposes. + :raises: UWError if supplemental config format is incompatible. + """ + pre = f"Supplemental config #{idx + 1}" + if isinstance(supplemental_cfg, dict): + log.debug("%s is a dict: Cannot validate its format vs %s", pre, config_fmt) + return + sc_fmt = ( + supplemental_cfg.get_format() + if isinstance(supplemental_cfg, Config) + else _ensure_format(desc=pre, config=supplemental_cfg) + ) + if sc_fmt != config_fmt: + raise UWError("%s format %s must match input format %s" % (pre, sc_fmt, config_fmt)) + + +# Import-time code + +# pylint: disable=duplicate-code + +# The following statements dynamically interpolate values into functions' docstrings, which will not +# work if the docstrings are inlined in the functions. They must remain separate statements to avoid +# hardcoding values into them. + +compare_configs.__doc__ = """ +Compare two config files. + +Recognized file extensions are: {extensions} + +:param config_1_path: Path to 1st config file +:param config_2_path: Path to 2nd config file +:param config_1_format: Format of 1st config file (optional if file's extension is recognized) +:param config_2_format: Format of 2nd config file (optional if file's extension is recognized) +:return: ``False`` if config files had differences, otherwise ``True`` +""".format( + extensions=", ".join(FORMAT.extensions()) +).strip() + + +realize_config.__doc__ = """ +Realize an output config based on an input config and optional values-providing configs. + +Recognized file extensions are: {extensions} + +:param input_config: Input config source (None => read stdin). +:param input_format: Format of the input config. +:param output_file: Output config destination (None => write to stdout). +:param output_format: Format of the output config. +:param supplemental_configs: Sources of values used to modify input. +:param values_needed: Report complete, missing, and template values. +:param dry_run: Log output instead of writing to output. +:return: The realized config (or an empty-dict for no-op modes). +""".format( + extensions=", ".join(FORMAT.extensions()) +).strip() diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 633216565..347f19daf 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -87,7 +87,7 @@ def run_cmd(self) -> str: components = [ self._platform_config.get("mpicmd"), # MPI run program *[str(x) for x in self._config["runtime_info"].get("mpi_args", [])], # MPI arguments - self._config["exec_name"], # NWP tool executable name + self._config["executable"], # NWP tool executable name ] return " ".join(filter(None, components)) @@ -109,7 +109,7 @@ def schema_file(self) -> Path: @staticmethod def stage_files( - run_directory: str, + run_directory: Path, files_to_stage: Dict[str, Union[list, str]], link_files: bool = False, dry_run: bool = False, @@ -126,7 +126,7 @@ def stage_files( """ link_or_copy = os.symlink if link_files else shutil.copyfile for dst_rel_path, src_path_or_paths in files_to_stage.items(): - dst_path = os.path.join(run_directory, dst_rel_path) + dst_path = run_directory / dst_rel_path if isinstance(src_path_or_paths, list): Driver.stage_files( dst_path, diff --git a/src/uwtools/drivers/forecast.py b/src/uwtools/drivers/forecast.py index b01175488..3de4111d5 100644 --- a/src/uwtools/drivers/forecast.py +++ b/src/uwtools/drivers/forecast.py @@ -2,7 +2,6 @@ Drivers for forecast models. """ - import sys from collections.abc import Mapping from datetime import datetime @@ -142,11 +141,11 @@ def prepare_directories(self) -> Path: :return: Path to the run directory. """ - run_directory = self._config["run_dir"] + run_directory = Path(self._config["run_dir"]) self.create_directory_structure(run_directory, ExistAct.delete, dry_run=self._dry_run) - self._prepare_config_files(Path(run_directory)) - self._config["cycle-dependent"].update(self._define_boundary_files()) - for file_category in ["static", "cycle-dependent"]: + self._prepare_config_files(run_directory) + self._config["cycle_dependent"].update(self._define_boundary_files()) + for file_category in ["static", "cycle_dependent"]: self.stage_files( run_directory, self._config[file_category], link_files=True, dry_run=self._dry_run ) @@ -216,13 +215,14 @@ def _define_boundary_files(self) -> Dict[str, str]: """ boundary_files = {} lbcs_config = self._experiment_config["preprocessing"]["lateral_boundary_conditions"] - boundary_file_template = lbcs_config["output_file_template"] + boundary_file_path = lbcs_config["output_file_path"] offset, interval, endhour = self._boundary_hours(lbcs_config) - for tile in self._config["tiles"]: + tiles = [7] if self._config["domain"] == "global" else range(1, 7) + for tile in tiles: for boundary_hour in range(offset, endhour, interval): forecast_hour = boundary_hour - offset link_name = f"INPUT/gfs_bndy.tile{tile}.{forecast_hour:03d}.nc" - boundary_file_path = boundary_file_template.format( + boundary_file_path = boundary_file_path.format( tile=tile, forecast_hour=boundary_hour, ) @@ -282,13 +282,14 @@ def _run_via_local_execution(self) -> Tuple[bool, List[str]]: :return: A tuple containing a boolean of the success status of the FV3 run and a list of strings that make up the full command line. """ + run_directory = self.prepare_directories() pre_run = self._mpi_env_variables(" ") full_cmd = f"{pre_run} {self.run_cmd()}" command_lines = ["Command:", *full_cmd.split("\n")] if self._dry_run: return True, command_lines - result = execute(cmd=full_cmd) - return result.success, command_lines + success, _ = execute(cmd=full_cmd, cwd=run_directory, log_output=True) + return success, command_lines CLASSES = {"FV3": FV3Forecast} diff --git a/src/uwtools/logging.py b/src/uwtools/logging.py index 34978bfd4..788bfdf21 100644 --- a/src/uwtools/logging.py +++ b/src/uwtools/logging.py @@ -49,7 +49,7 @@ def setup_logging(quiet: bool = False, verbose: bool = False) -> None: for handler in logger.handlers: logger.removeHandler(handler) if quiet and verbose: - print("Specify at most one of 'quiet' and 'verbose'", file=sys.stderr) + print("--quiet may not be used with --debug or --verbose", file=sys.stderr) sys.exit(1) kwargs: dict = { "datefmt": "%Y-%m-%dT%H:%M:%S", diff --git a/src/uwtools/resources/FV3Forecast.jsonschema b/src/uwtools/resources/FV3Forecast.jsonschema index 2e070696e..fac796dec 100644 --- a/src/uwtools/resources/FV3Forecast.jsonschema +++ b/src/uwtools/resources/FV3Forecast.jsonschema @@ -1,79 +1,77 @@ { + "$defs": { + "updatable_config": { + "additionalProperties": false, + "properties": { + "properties": { + "base_file": { + "format": "uri", + "type": "string" + }, + "update_values": { + "type": "object" + } + }, + "required": [ + "base_file" + ], + "type": "object" + } + } + }, "description": "This document is to validate user-defined FV3 forecast config files", "properties": { "forecast": { + "additionalProperties": false, "description": "parameters of the forecast", "properties": { - "cycle-dependent": { + "cycle_dependent|static": { "propertyNames": { "type": "string" }, "type": "object" }, - "exec_name": { + "diag_table": { + "format": "uri", + "type": "string" + }, + "domain": { + "enum": [ + "global", + "regional" + ], + "type": "string" + }, + "executable": { + "format": "uri", "type": "string" }, + "fd_ufs": { + "$ref": "#/$defs/updatable_config" + }, "field_table": { - "properties": { - "base_file": { - "format": "uri", - "type": "string" - }, - "update_values": { - "type": "object" - } - }, - "type": "object" + "$ref": "#/$defs/updatable_config" }, "length": { "minimum": 1, "type": "integer" }, - "model": { - "type": "string" - }, "model_configure": { - "properties": { - "base_file": { - "format": "uri", - "type": "string" - }, - "update_values": { - "type": "object" - } - }, - "type": "object" + "$ref": "#/$defs/updatable_config" }, "mpicmd": { "type": "string" }, "namelist": { - "properties": { - "base_file": { - "format": "uri", - "type": "string" - }, - "update_values": { - "type": "object" - } - }, - "type": "object" + "$ref": "#/$defs/updatable_config" }, "run_dir": { "format": "uri", "type": "string" }, - "static": { - "propertyNames": { - "type": "string" - }, - "type": "object" - }, - "tiles": { - "items": { - "type": "integer" - }, - "type": "array" + "ufs_configure": { + "format": "uri", + "type": "string" } }, "type": "object" @@ -108,7 +106,7 @@ "minimum": 0, "type": "number" }, - "output_file_template": { + "output_file_path": { "format": "uri", "type": "string" } diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index a916001ba..822790a3c 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -8,7 +8,7 @@ from typing import Any, List, Optional, Tuple, Union from lxml import etree -from lxml.etree import Element, SubElement +from lxml.etree import Element, SubElement, _Element from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.validator import validate_yaml @@ -59,7 +59,7 @@ def validate_rocoto_xml_string(xml: str) -> bool: tree = etree.fromstring(xml.encode("utf-8")) with open(resource_pathobj("schema_with_metatasks.rng"), "r", encoding="utf-8") as f: schema = etree.RelaxNG(etree.parse(f)) - valid = schema.validate(tree) + valid: bool = schema.validate(tree) nerr = len(schema.error_log) log_method = log.info if valid else log.error log_method("%s Rocoto validation error%s found", nerr, "" if nerr == 1 else "s") @@ -104,7 +104,7 @@ def dump(self, path: OptionalPath = None) -> None: with writable(path) as f: f.write(str(self).strip()) - def _add_compound_time_string(self, e: Element, config: Any, tag: str) -> Element: + def _add_compound_time_string(self, e: _Element, config: Any, tag: str) -> _Element: """ Add to the given element a child element possibly containing a . @@ -124,7 +124,7 @@ def _add_compound_time_string(self, e: Element, config: Any, tag: str) -> Elemen e.text = str(config) return e - def _add_metatask(self, e: Element, config: dict, name_attr: str) -> None: + def _add_metatask(self, e: _Element, config: dict, name_attr: str) -> None: """ Add a element to the . @@ -144,7 +144,7 @@ def _add_metatask(self, e: Element, config: dict, name_attr: str) -> None: for varname, value in val.items(): SubElement(e, STR.var, name=varname).text = value - def _add_task(self, e: Element, config: dict, name_attr: str) -> None: + def _add_task(self, e: _Element, config: dict, name_attr: str) -> None: """ Add a element to the . @@ -186,7 +186,7 @@ def _add_task(self, e: Element, config: dict, name_attr: str) -> None: if STR.dependency in config: self._add_task_dependency(e, config[STR.dependency]) - def _add_task_dependency(self, e: Element, config: dict) -> None: + def _add_task_dependency(self, e: _Element, config: dict) -> None: """ Add a element to the . @@ -197,7 +197,7 @@ def _add_task_dependency(self, e: Element, config: dict) -> None: for tag, subconfig in config.items(): self._add_task_dependency_child(e, subconfig, tag) - def _add_task_dependency_child(self, e: Element, config: dict, tag: str) -> None: + def _add_task_dependency_child(self, e: _Element, config: dict, tag: str) -> None: """ Add an operator/operand element to parent element. @@ -225,7 +225,7 @@ def _add_task_dependency_child(self, e: Element, config: dict, tag: str) -> None else: raise UWConfigError("Unhandled dependency type %s" % tag) - def _add_task_dependency_datadep(self, e: Element, config: dict) -> None: + def _add_task_dependency_datadep(self, e: _Element, config: dict) -> None: """ Add a element to the . @@ -236,7 +236,7 @@ def _add_task_dependency_datadep(self, e: Element, config: dict) -> None: self._set_attrs(e, config) def _add_task_dependency_sh( - self, e: Element, config: dict, name_attr: Optional[str] = None + self, e: _Element, config: dict, name_attr: Optional[str] = None ) -> None: """ :param e: The parent element to add the new element to. @@ -247,7 +247,7 @@ def _add_task_dependency_sh( config[STR.attrs][STR.name] = name_attr self._set_attrs(e, config) - def _add_task_dependency_strequality(self, e: Element, config: dict, tag: str) -> None: + def _add_task_dependency_strequality(self, e: _Element, config: dict, tag: str) -> None: """ :param e: The parent element to add the new element to. :param config: Configuration data for the tag. @@ -255,7 +255,7 @@ def _add_task_dependency_strequality(self, e: Element, config: dict, tag: str) - """ self._set_attrs(SubElement(e, tag), config) - def _add_task_dependency_taskdep(self, e: Element, config: dict) -> None: + def _add_task_dependency_taskdep(self, e: _Element, config: dict) -> None: """ Add a element to the . @@ -264,7 +264,7 @@ def _add_task_dependency_taskdep(self, e: Element, config: dict) -> None: """ self._set_attrs(SubElement(e, STR.taskdep), config) - def _add_task_dependency_taskvalid(self, e: Element, config: dict) -> None: + def _add_task_dependency_taskvalid(self, e: _Element, config: dict) -> None: """ Add a element to the . @@ -273,7 +273,7 @@ def _add_task_dependency_taskvalid(self, e: Element, config: dict) -> None: """ self._set_attrs(SubElement(e, STR.taskvalid), config) - def _add_task_dependency_timedep(self, e: Element, config: dict) -> None: + def _add_task_dependency_timedep(self, e: _Element, config: dict) -> None: """ Add a element to the . @@ -282,7 +282,7 @@ def _add_task_dependency_timedep(self, e: Element, config: dict) -> None: """ self._add_compound_time_string(e, config, STR.timedep) - def _add_task_envar(self, e: Element, name: str, value: str) -> None: + def _add_task_envar(self, e: _Element, name: str, value: str) -> None: """ Add a element to the . @@ -304,9 +304,9 @@ def _add_workflow(self, config: dict) -> None: self._add_workflow_cycledef(e, config[STR.cycledef]) self._add_workflow_log(e, config) self._add_workflow_tasks(e, config[STR.tasks]) - self._root: Element = e + self._root: _Element = e - def _add_workflow_cycledef(self, e: Element, config: List[dict]) -> None: + def _add_workflow_cycledef(self, e: _Element, config: List[dict]) -> None: """ Add element(s) to the . @@ -318,7 +318,7 @@ def _add_workflow_cycledef(self, e: Element, config: List[dict]) -> None: cycledef.text = item["spec"] self._set_attrs(cycledef, item) - def _add_workflow_log(self, e: Element, config: dict) -> None: + def _add_workflow_log(self, e: _Element, config: dict) -> None: """ Add element(s) to the . @@ -328,7 +328,7 @@ def _add_workflow_log(self, e: Element, config: dict) -> None: tag = STR.log self._add_compound_time_string(e, config[tag], tag) - def _add_workflow_tasks(self, e: Element, config: dict) -> None: + def _add_workflow_tasks(self, e: _Element, config: dict) -> None: """ Add and/or element(s) to the . @@ -383,7 +383,7 @@ def _set_and_render_jobname(self, config: dict, taskname: str) -> None: if STR.jobname not in config: config[STR.jobname] = taskname - def _set_attrs(self, e: Element, config: dict) -> None: + def _set_attrs(self, e: _Element, config: dict) -> None: """ Set attributes on an element. diff --git a/src/uwtools/scheduler.py b/src/uwtools/scheduler.py index e31b832b9..6649ab851 100644 --- a/src/uwtools/scheduler.py +++ b/src/uwtools/scheduler.py @@ -7,10 +7,11 @@ import re from collections import UserDict, UserList from collections.abc import Mapping +from pathlib import Path from typing import Any, Dict, List from uwtools.logging import log -from uwtools.types import DefinitePath, OptionalPath +from uwtools.types import OptionalPath from uwtools.utils.file import writable from uwtools.utils.memory import Memory from uwtools.utils.processing import execute @@ -191,15 +192,17 @@ def get_scheduler(props: Mapping) -> JobScheduler: ) from error return scheduler(props) - def submit_job(self, script_path: DefinitePath) -> bool: + def submit_job(self, script_path: Path) -> bool: """ Submits a job to the scheduler. :param script_path: Path to the batch script. :return: Did the run exit with a success status? """ - result = execute(cmd=f"{self.submit_command} {script_path}") - return result.success + success, _ = execute( + cmd=f"{self.submit_command} {script_path}", cwd=f"{script_path.parent}" + ) + return success class Slurm(JobScheduler): @@ -265,7 +268,7 @@ def pre_process(self) -> Dict[str, Any]: output.pop("select", None) return dict(output) - def _select(self, items) -> Dict[str, Any]: + def _select(self, items: Dict[str, Any]) -> Dict[str, Any]: """ Select logic. """ @@ -274,7 +277,6 @@ def _select(self, items) -> Dict[str, Any]: # Set default threads=1 to address job variability with PBS threads = items.get(OptionalAttribs.THREADS, 1) memory = items.get(OptionalAttribs.MEMORY, "") - select = [ f"{total_nodes}", f"{self._map[OptionalAttribs.TASKS_PER_NODE]}={tasks_per_node}", @@ -284,26 +286,17 @@ def _select(self, items) -> Dict[str, Any]: if memory not in NONEISH: select.append(f"{self._map[OptionalAttribs.MEMORY]}={memory}") items["-l select="] = ":".join(select) - return items @staticmethod - def _placement(items) -> Dict[str, Any]: + def _placement(items: Dict[str, Any]) -> Dict[str, Any]: """ Placement logic. """ - exclusive = items.get(OptionalAttribs.EXCLUSIVE, "") placement = items.get(OptionalAttribs.PLACEMENT, "") - - if all( - [ - exclusive in NONEISH, - placement in NONEISH, - ] - ): + if all([exclusive in NONEISH, placement in NONEISH]): return items - output = [] if placement not in NONEISH: output.append(str(placement)) diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py index 182b13e81..14d75faac 100644 --- a/src/uwtools/tests/api/test_config.py +++ b/src/uwtools/tests/api/test_config.py @@ -12,10 +12,10 @@ def test_compare(): kwargs: dict = { - "config_a_path": "path1", - "config_a_format": "fmt1", - "config_b_path": "path2", - "config_b_format": "fmt2", + "config_1_path": "path1", + "config_1_format": "fmt1", + "config_2_path": "path2", + "config_2_format": "fmt2", } with patch.object(config, "_compare") as _compare: config.compare(**kwargs) diff --git a/src/uwtools/tests/config/test_support.py b/src/uwtools/tests/config/test_support.py index 9afa68aa8..b41300fc1 100644 --- a/src/uwtools/tests/config/test_support.py +++ b/src/uwtools/tests/config/test_support.py @@ -21,7 +21,9 @@ from uwtools.utils.file import FORMAT -@pytest.mark.parametrize("d,n", [({1: 88}, 1), ({1: {2: 88}}, 2), ({1: {2: {3: 88}}}, 3)]) +@pytest.mark.parametrize( + "d,n", [({1: 88}, 1), ({1: {2: 88}}, 2), ({1: {2: {3: 88}}}, 3), ({1: {}}, 2)] +) def test_depth(d, n): assert support.depth(d) == n diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index a01ee9bdc..313fc7b72 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -95,7 +95,7 @@ def test_compare_configs_good(compare_configs_assets, caplog): log.setLevel(logging.INFO) _, a, b = compare_configs_assets assert tools.compare_configs( - config_a_path=a, config_a_format=FORMAT.yaml, config_b_path=b, config_b_format=FORMAT.yaml + config_1_path=a, config_1_format=FORMAT.yaml, config_2_path=b, config_2_format=FORMAT.yaml ) assert caplog.records @@ -107,7 +107,7 @@ def test_compare_configs_changed_value(compare_configs_assets, caplog): with writable(b) as f: yaml.dump(d, f) assert not tools.compare_configs( - config_a_path=a, config_a_format=FORMAT.yaml, config_b_path=b, config_b_format=FORMAT.yaml + config_1_path=a, config_1_format=FORMAT.yaml, config_2_path=b, config_2_format=FORMAT.yaml ) assert logged(caplog, "baz: qux: - 99 + 11") @@ -120,23 +120,21 @@ def test_compare_configs_missing_key(compare_configs_assets, caplog): yaml.dump(d, f) # Note that a and b are swapped: assert not tools.compare_configs( - config_a_path=b, config_a_format=FORMAT.yaml, config_b_path=a, config_b_format=FORMAT.yaml + config_1_path=b, config_1_format=FORMAT.yaml, config_2_path=a, config_2_format=FORMAT.yaml ) assert logged(caplog, "baz: qux: - None + 99") def test_compare_configs_bad_format(caplog): log.setLevel(logging.INFO) - with raises(UWConfigError) as e: - tools.compare_configs( - config_a_path="/not/used", - config_a_format="jpg", - config_b_path="/not/used", - config_b_format=FORMAT.yaml, - ) - msg = "Format 'jpg' should be one of: fieldtable, ini, nml, sh, yaml" + assert not tools.compare_configs( + config_1_path="/not/used", + config_1_format="jpg", + config_2_path="/not/used", + config_2_format=FORMAT.yaml, + ) + msg = "Formats do not match: jpg vs yaml" assert logged(caplog, msg) - assert msg in str(e.value) def test_config_check_depths_realize_fail(caplog, realize_config_testobj): @@ -270,7 +268,7 @@ def test_realize_config_incompatible_file_type(): """ Test that providing an incompatible file type for input base file will return print statement. """ - with raises(UWConfigError): + with raises(UWError): tools.realize_config( input_config=fixture_path("model_configure.sample"), input_format="sample", @@ -351,14 +349,14 @@ def test_realize_config_single_dereference(capsys, tmp_path): def test_realize_config_supp_bad_format(tmp_path): - with raises(ValueError) as e: - path = tmp_path / "a.yaml" - supplemental_path = tmp_path / "b.clj" - msg = f"Cannot deduce format of '{supplemental_path}' from unknown extension 'clj'" - with writable(path) as f: - yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) - with writable(supplemental_path) as f: - yaml.dump({"2": "b", "temporalis": "c"}, f) + path = tmp_path / "a.yaml" + supplemental_path = tmp_path / "b.clj" + msg = f"Cannot deduce format of '{supplemental_path}' from unknown extension 'clj'" + with writable(path) as f: + yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) + with writable(supplemental_path) as f: + yaml.dump({"2": "b", "temporalis": "c"}, f) + with raises(UWError) as e: tools.realize_config( input_config=path, input_format=FORMAT.yaml, @@ -451,39 +449,6 @@ def test_realize_config_values_needed_ini(caplog): assert actual == expected -def test_realize_config_values_needed_nml(caplog): - """ - Test that the values_needed flag logs keys completed, keys containing unrendered Jinja2 - variables/expressions and keys set to empty. - """ - log.setLevel(logging.INFO) - tools.realize_config( - input_config=fixture_path("simple3.nml"), - input_format=FORMAT.nml, - output_format=FORMAT.yaml, - values_needed=True, - ) - expected = """ -Keys that are complete: - salad - salad.base - salad.fruit - salad.vegetable - salad.how_many - salad.extras - salad.dessert - -Keys with unrendered Jinja2 variables/expressions: - salad.dressing: {{ dressing }} - -Keys that are set to empty: - salad.toppings - salad.appetizer -""".strip() - actual = "\n".join(record.message for record in caplog.records) - assert actual == expected - - def test_realize_config_values_needed_yaml(caplog): """ Test that the values_needed flag logs keys completed, keys containing unrendered Jinja2 @@ -606,14 +571,16 @@ def test__print_config_section_yaml_not_dict(): def test__realize_config_update(realize_config_testobj, supplemental_configs): assert realize_config_testobj[1][2][3] == 88 o = tools._realize_config_update( - config_obj=realize_config_testobj, supplemental_configs=[supplemental_configs] + config_obj=realize_config_testobj, + config_fmt="yaml", + supplemental_configs=[supplemental_configs], ) assert o[1][2][3] == 99 def test__realize_config_update_noop(realize_config_testobj): assert realize_config_testobj == tools._realize_config_update( - config_obj=realize_config_testobj, supplemental_configs=None + config_obj=realize_config_testobj, config_fmt="yaml", supplemental_configs=None ) @@ -623,7 +590,9 @@ def test__realize_config_update_file(realize_config_testobj, tmp_path): with open(path, "w", encoding="utf-8") as f: yaml.dump(values, f) assert realize_config_testobj[1][2][3] == 88 - o = tools._realize_config_update(config_obj=realize_config_testobj, supplemental_configs=[path]) + o = tools._realize_config_update( + config_obj=realize_config_testobj, config_fmt="yaml", supplemental_configs=[path] + ) assert o[1][2][3] == 99 @@ -638,7 +607,7 @@ def test__realize_config_update_list(realize_config_testobj, tmp_path): yaml.dump(values2, f) assert realize_config_testobj[1][2][3] == 88 o = tools._realize_config_update( - config_obj=realize_config_testobj, supplemental_configs=[path, path2] + config_obj=realize_config_testobj, config_fmt="yaml", supplemental_configs=[path, path2] ) assert o[1][2][3] == 77 @@ -667,3 +636,59 @@ def test__realize_config_values_needed_negative_results(caplog, tmp_path): assert "No keys are complete." in msgs assert "No keys have unrendered Jinja2 variables/expressions." in msgs assert "No keys are set to empty." in msgs + + +@pytest.mark.parametrize("input_fmt", FORMAT.extensions()) +@pytest.mark.parametrize("output_fmt", FORMAT.extensions()) +def test__validate_format_output(input_fmt, output_fmt): + call = lambda: tools._validate_format_output(input_fmt=input_fmt, output_fmt=output_fmt) + if input_fmt in (FORMAT.yaml, output_fmt): + call() # no exception raised + else: + with raises(UWError) as e: + call() + assert str(e.value) == f"Output format {output_fmt} must match input format {input_fmt}" + + +def test__validate_format_supplemental_fail_obj(): + config_fmt = FORMAT.yaml + sc = NMLConfig(config={"n": {"k": "v"}}) + with raises(UWError) as e: + tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) + assert str(e.value) == "Supplemental config #88 format %s must match input format %s" % ( + FORMAT.nml, + config_fmt, + ) + + +def test__validate_format_supplemental_fail_path(): + config_fmt = FORMAT.yaml + sc = "/path/to/config.nml" + with raises(UWError) as e: + tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) + assert str(e.value) == "Supplemental config #%s format %s must match input format %s" % ( + 88, + FORMAT.nml, + config_fmt, + ) + + +def test__validate_format_supplemental_pass_dict(caplog): + log.setLevel(logging.DEBUG) + config_fmt = FORMAT.yaml + sc: dict = {} + tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) + msg = "Supplemental config #%s is a dict: Cannot validate its format vs %s" % (88, config_fmt) + assert logged(caplog, msg) + + +def test__validate_format_supplemental_pass_match_obj(): + config_fmt = FORMAT.yaml + sc = YAMLConfig(config={}) + tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) + + +def test__validate_format_supplemental_pass_match_path(): + config_fmt = FORMAT.yaml + sc = "/path/to/config.yaml" + tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) diff --git a/src/uwtools/tests/drivers/test_forecast.py b/src/uwtools/tests/drivers/test_forecast.py index 7f4141784..4c9b8659b 100644 --- a/src/uwtools/tests/drivers/test_forecast.py +++ b/src/uwtools/tests/drivers/test_forecast.py @@ -6,7 +6,6 @@ import logging import os from pathlib import Path -from types import SimpleNamespace as ns from unittest.mock import ANY, patch import pytest @@ -269,11 +268,11 @@ def test_forecast_run_cmd(): assert mpiexec_expected == fcstobj.run_cmd() -@pytest.mark.parametrize("section", ["static", "cycle-dependent"]) +@pytest.mark.parametrize("section", ["static", "cycle_dependent"]) @pytest.mark.parametrize("link_files", [True, False]) def test_stage_files(tmp_path, section, link_files): """ - Tests that files from static or cycle-dependent sections of the config obj are being staged + Tests that files from static or cycle_dependent sections of the config obj are being staged (copied or linked) to the run directory. """ @@ -304,7 +303,7 @@ def test_stage_files(tmp_path, section, link_files): assert all(link_or_file(d_fn) for d_fn in dst_paths) else: assert link_or_file(run_directory / dst_rel_path) - if section == "cycle-dependent": + if section == "cycle_dependent": assert link_or_file(run_directory / "INPUT" / "gfs_bndy.tile7.006.nc") @@ -314,7 +313,7 @@ def fv3_run_assets(tmp_path): config_file = fixture_path("forecast.yaml") config = YAMLConfig(config_file) config["forecast"]["run_dir"] = tmp_path.as_posix() - config["forecast"]["cycle-dependent"] = {"foo-file": str(tmp_path / "foo")} + config["forecast"]["cycle_dependent"] = {"foo-file": str(tmp_path / "foo")} config["forecast"]["static"] = {"static-foo-file": str(tmp_path / "foo")} return batch_script, config_file, config.data["forecast"] @@ -336,10 +335,11 @@ def test_run_direct(fv3_mpi_assets, fv3_run_assets): expected_command = " ".join(fv3_mpi_assets) with patch.object(FV3Forecast, "_validate", return_value=True): with patch.object(forecast, "execute") as execute: + execute.return_value = (True, "") fcstobj = FV3Forecast(config_file=config_file) with patch.object(fcstobj, "_config", config): fcstobj.run(cycle=dt.datetime.now()) - execute.assert_called_once_with(cmd=expected_command) + execute.assert_called_once_with(cmd=expected_command, cwd=ANY, log_output=True) @pytest.mark.parametrize("with_batch_script", [True, False]) @@ -387,11 +387,11 @@ def test_FV3Forecast__run_via_batch_submission(fv3_run_assets): with patch.object(fcstobj, "_config", config): with patch.object(scheduler, "execute") as execute: with patch.object(Driver, "_create_user_updated_config"): - execute.return_value = ns(success=True) + execute.return_value = (True, "") success, lines = fcstobj._run_via_batch_submission() assert success is True assert lines[0] == "Batch script:" - execute.assert_called_once_with(cmd=ANY) + execute.assert_called_once_with(cmd=ANY, cwd=ANY) def test_FV3Forecast__run_via_local_execution(fv3_run_assets): @@ -399,8 +399,8 @@ def test_FV3Forecast__run_via_local_execution(fv3_run_assets): fcstobj = FV3Forecast(config_file=config_file) with patch.object(fcstobj, "_config", config): with patch.object(forecast, "execute") as execute: - execute.return_value = ns(success=True) + execute.return_value = (True, "") success, lines = fcstobj._run_via_local_execution() assert success is True assert lines[0] == "Command:" - execute.assert_called_once_with(cmd=ANY) + execute.assert_called_once_with(cmd=ANY, cwd=ANY, log_output=True) diff --git a/src/uwtools/tests/fixtures/expt_dir.yaml b/src/uwtools/tests/fixtures/expt_dir.yaml index be3738553..77ffab919 100644 --- a/src/uwtools/tests/fixtures/expt_dir.yaml +++ b/src/uwtools/tests/fixtures/expt_dir.yaml @@ -1,4 +1,4 @@ -cycle-dependent: +cycle_dependent: INPUT/gfs_data.nc: path/to/gfs_data.tile7.halo0.nc INPUT/sfc_data.nc: path/to/sfc_data.tile7.halo0.nc INPUT/gfs_bndy.tile7.000.nc: path/to/gfs_bndy.tile7.000.nc diff --git a/src/uwtools/tests/fixtures/forecast.yaml b/src/uwtools/tests/fixtures/forecast.yaml index 120cbcbb1..945ac4c7f 100644 --- a/src/uwtools/tests/fixtures/forecast.yaml +++ b/src/uwtools/tests/fixtures/forecast.yaml @@ -7,13 +7,12 @@ preprocessing: lateral_boundary_conditions: interval_hours: 3 offset: 0 - output_file_template: "gfs_bndy.tile{tile}.f{forecast_hour}.nc" + output_file_path: "gfs_bndy.tile{tile}.f{forecast_hour}.nc" forecast: model: FV3 - exec_name: test_exec.py + executable: test_exec.py run_dir: some/path - tiles: - - 7 + domain: regional length: 12 jobinfo: nodes: 1 @@ -25,7 +24,7 @@ forecast: stacksize: 512m mpi_args: - "--export=NONE" - cycle-dependent: + cycle_dependent: INPUT/gfs_data.nc: path/to/gfs_data.tile7.halo0.nc INPUT/sfc_data.nc: path/to/sfc_data.tile7.halo0.nc INPUT/gfs_bndy.tile7.000.nc: path/to/gfs_bndy.tile7.000.nc diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index d0c94e139..eb3909a2e 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -17,7 +17,9 @@ import uwtools.drivers.forecast from uwtools import cli from uwtools.cli import STR +from uwtools.exceptions import UWError from uwtools.logging import log +from uwtools.tests.support import logged from uwtools.utils.file import FORMAT # Test functions @@ -32,7 +34,7 @@ def test__abort(capsys): def test__add_subparser_config(subparsers): cli._add_subparser_config(subparsers) - assert submodes(subparsers.choices[STR.config]) == [STR.compare, STR.realize, STR.validate] + assert actions(subparsers.choices[STR.config]) == [STR.compare, STR.realize, STR.validate] def test__add_subparser_config_compare(subparsers): @@ -52,7 +54,7 @@ def test__add_subparser_config_validate(subparsers): def test__add_subparser_forecast(subparsers): cli._add_subparser_forecast(subparsers) - assert submodes(subparsers.choices[STR.forecast]) == [STR.run] + assert actions(subparsers.choices[STR.forecast]) == [STR.run] def test__add_subparser_forecast_run(subparsers): @@ -62,7 +64,7 @@ def test__add_subparser_forecast_run(subparsers): def test__add_subparser_template(subparsers): cli._add_subparser_template(subparsers) - assert submodes(subparsers.choices[STR.template]) == [STR.render, STR.translate] + assert actions(subparsers.choices[STR.template]) == [STR.render, STR.translate] def test__add_subparser_template_render(subparsers): @@ -110,7 +112,7 @@ def test__check_file_vs_format_pass_explicit(): assert args[STR.infmt] == fmt -@pytest.mark.parametrize("fmt", vars(FORMAT).keys()) +@pytest.mark.parametrize("fmt", FORMAT.formats()) def test__check_file_vs_format_pass_implicit(fmt): # The format is correctly deduced for a file with a known extension. args = {STR.infile: f"/path/to/input.{fmt}", STR.infmt: None} @@ -122,26 +124,10 @@ def test__check_file_vs_format_pass_implicit(fmt): assert args[STR.infmt] == vars(FORMAT)[fmt] -def test__check_quiet_vs_verbose_fail(capsys): - log.setLevel(logging.INFO) - args = {STR.quiet: True, STR.verbose: True} - with raises(SystemExit): - cli._check_quiet_vs_verbose(args) - assert ( - "Specify at most one of %s, %s" % (cli._switch(STR.quiet), cli._switch(STR.verbose)) - in capsys.readouterr().err - ) - - -def test__check_quiet_vs_verbose_ok(): - args = {"foo": 88} - assert cli._check_quiet_vs_verbose(args) == args - - def test__check_template_render_vals_args_implicit_fail(): # The values-file format cannot be deduced from the filename. args = {STR.valsfile: "a.jpg"} - with raises(ValueError) as e: + with raises(UWError) as e: cli._check_template_render_vals_args(args) assert "Cannot deduce format" in str(e.value) @@ -165,6 +151,23 @@ def test__check_template_render_vals_args_noop_explicit_valsfmt(): assert cli._check_template_render_vals_args(args) == args +@pytest.mark.parametrize("flag", (STR.debug, STR.verbose)) +def test__check_verbosity_fail(capsys, flag): + log.setLevel(logging.INFO) + args = {STR.quiet: True, flag: True} + with raises(SystemExit): + cli._check_verbosity(args) + assert "--quiet may not be used with --debug or --verbose" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "flags", ([STR.debug], [STR.quiet], [STR.verbose], [STR.debug, STR.verbose]) +) +def test__check_verbosity_ok(flags): + args = {flag: True for flag in flags} + assert cli._check_verbosity(args) == args + + def test__dict_from_key_eq_val_strings(): assert not cli._dict_from_key_eq_val_strings([]) assert cli._dict_from_key_eq_val_strings(["a=1", "b=2"]) == {"a": "1", "b": "2"} @@ -179,8 +182,8 @@ def test__dict_from_key_eq_val_strings(): ], ) def test__dispatch_config(params): - submode, funcname = params - args = {STR.submode: submode} + action, funcname = params + args = {STR.action: action} with patch.object(cli, funcname) as func: cli._dispatch_config(args) func.assert_called_once_with(args) @@ -191,10 +194,10 @@ def test__dispatch_config_compare(): with patch.object(cli.uwtools.api.config, "compare") as compare: cli._dispatch_config_compare(args) compare.assert_called_once_with( - config_a_path=args[STR.file1path], - config_a_format=args[STR.file1fmt], - config_b_path=args[STR.file2path], - config_b_format=args[STR.file2fmt], + config_1_path=args[STR.file1path], + config_1_format=args[STR.file1fmt], + config_2_path=args[STR.file2path], + config_2_format=args[STR.file2fmt], ) @@ -255,8 +258,8 @@ def test__dispatch_config_validate_config_obj(): @pytest.mark.parametrize("params", [(STR.run, "_dispatch_forecast_run")]) def test__dispatch_forecast(params): - submode, funcname = params - args = {STR.submode: submode} + action, funcname = params + args = {STR.action: action} with patch.object(cli, funcname) as module: cli._dispatch_forecast(args) module.assert_called_once_with(args) @@ -286,8 +289,8 @@ def test__dispatch_forecast_run(): ], ) def test__dispatch_rocoto(params): - submode, funcname = params - args = {STR.submode: submode} + action, funcname = params + args = {STR.action: action} with patch.object(cli, funcname) as module: cli._dispatch_rocoto(args) module.assert_called_once_with(args) @@ -332,8 +335,8 @@ def test__dispatch_rocoto_validate_xml_no_optional(): [(STR.render, "_dispatch_template_render"), (STR.translate, "_dispatch_template_translate")], ) def test__dispatch_template(params): - submode, funcname = params - args = {STR.submode: submode} + action, funcname = params + args = {STR.action: action} with patch.object(cli, funcname) as func: cli._dispatch_template(args) func.assert_called_once_with(args) @@ -413,11 +416,24 @@ def test__dispatch_template_translate_no_optional(): ) -@pytest.mark.parametrize("quiet", [True]) -@pytest.mark.parametrize("verbose", [False]) -def test_main_fail_checks(capsys, quiet, verbose): +def test_main_debug_logs_stacktrace(caplog): + log.setLevel(logging.DEBUG) + msg = "Test failed intentionally" + with patch.object(cli, "_parse_args", side_effect=Exception(msg)): + with patch.object(sys, "argv", cli._switch(STR.debug)): + with raises(SystemExit): + cli.main() + assert logged(caplog, "Traceback (most recent call last):") + + +@pytest.mark.parametrize("debug", [False, True]) +@pytest.mark.parametrize("quiet", [False, True]) +@pytest.mark.parametrize("verbose", [False, True]) +def test_main_fail_checks(capsys, debug, quiet, verbose): # Using mode 'template render' for testing. raw_args = ["testing", STR.template, STR.render] + if debug: + raw_args.append(cli._switch(STR.debug)) if quiet: raw_args.append(cli._switch(STR.quiet)) if verbose: @@ -426,9 +442,11 @@ def test_main_fail_checks(capsys, quiet, verbose): with patch.object(cli, "_dispatch_template", return_value=True): with raises(SystemExit) as e: cli.main() - if quiet and verbose: + if quiet and (debug or verbose): assert e.value.code == 1 - assert "Specify at most one of" in capsys.readouterr().err + assert ( + "--quiet may not be used with --debug or --verbose" in capsys.readouterr().err + ) else: assert e.value.code == 0 @@ -465,9 +483,8 @@ def test__parse_args(): # Helper functions -def submodes(parser: Parser) -> List[str]: - # Return submodes (named subparsers) belonging to the given parser. For some background, see - # https://stackoverflow.com/questions/43688450. +def actions(parser: Parser) -> List[str]: + # Return actions (named subparsers) belonging to the given parser. if actions := [x for x in parser._actions if isinstance(x, _SubParsersAction)]: return list(actions[0].choices.keys()) return [] diff --git a/src/uwtools/tests/test_scheduler.py b/src/uwtools/tests/test_scheduler.py index 115abe6a2..32c1e6c8a 100644 --- a/src/uwtools/tests/test_scheduler.py +++ b/src/uwtools/tests/test_scheduler.py @@ -3,6 +3,8 @@ Tests for uwtools.scheduler module. """ +import os +from pathlib import Path from unittest.mock import patch from pytest import fixture, raises @@ -307,8 +309,9 @@ def test_scheduler_submit_job(pbs_props): pbs_config, _ = pbs_props js = JobScheduler.get_scheduler(pbs_config) submit_command = js.submit_command - outpath = "/path/to/batch/script" + outpath = Path("/path/to/batch/script") expected_command = f"{submit_command} {outpath}" with patch.object(scheduler, "execute") as execute: + execute.return_value = (True, "") js.submit_job(outpath) - execute.assert_called_once_with(cmd=expected_command) + execute.assert_called_once_with(cmd=expected_command, cwd=os.path.dirname(outpath)) diff --git a/src/uwtools/tests/utils/test_file.py b/src/uwtools/tests/utils/test_file.py index 0ee74f49f..2aad322e7 100644 --- a/src/uwtools/tests/utils/test_file.py +++ b/src/uwtools/tests/utils/test_file.py @@ -11,6 +11,7 @@ import pytest from pytest import fixture, raises +from uwtools.exceptions import UWError from uwtools.types import ExistAct from uwtools.utils import file @@ -75,7 +76,7 @@ def test_get_file_format(): def test_get_file_format_unrecognized(): - with raises(ValueError): + with raises(UWError): file.get_file_format("a.jpg") diff --git a/src/uwtools/tests/utils/test_processing.py b/src/uwtools/tests/utils/test_processing.py index e082b4399..cbabefdda 100644 --- a/src/uwtools/tests/utils/test_processing.py +++ b/src/uwtools/tests/utils/test_processing.py @@ -12,9 +12,9 @@ def test_run_failure(caplog): processing.log.setLevel(logging.INFO) cmd = "expr 1 / 0" - result = processing.execute(cmd=cmd) - assert "division by zero" in result.output - assert result.success is False + success, output = processing.execute(cmd=cmd) + assert "division by zero" in output + assert success is False assert logged(caplog, "Executing: %s" % cmd) assert logged(caplog, " Failed with status: 2") assert logged(caplog, " Output:") @@ -24,7 +24,8 @@ def test_run_failure(caplog): def test_run_success(caplog, tmp_path): processing.log.setLevel(logging.INFO) cmd = "echo hello $FOO" - assert processing.execute(cmd=cmd, cwd=tmp_path, env={"FOO": "bar"}, log_output=True) + success, _ = processing.execute(cmd=cmd, cwd=tmp_path, env={"FOO": "bar"}, log_output=True) + assert success assert logged(caplog, "Executing: %s" % cmd) assert logged(caplog, " in %s" % tmp_path) assert logged(caplog, " with environment variables:") diff --git a/src/uwtools/utils/file.py b/src/uwtools/utils/file.py index ed865ba87..2f83183b8 100644 --- a/src/uwtools/utils/file.py +++ b/src/uwtools/utils/file.py @@ -5,14 +5,15 @@ import shutil import sys from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, fields from datetime import datetime as dt from functools import cache from importlib import resources from io import StringIO from pathlib import Path -from typing import IO, Any, Generator, List, Union +from typing import IO, Any, Dict, Generator, List, Union +from uwtools.exceptions import UWError from uwtools.logging import log from uwtools.types import DefinitePath, ExistAct, OptionalPath @@ -47,6 +48,24 @@ class FORMAT: yaml: str = _yaml yml: str = _yaml + @staticmethod + def extensions() -> List[str]: + """ + Returns recognized filename extensions. + """ + return [FORMAT.ini, FORMAT.nml, FORMAT.sh, FORMAT.yaml] + + @staticmethod + def formats() -> Dict[str, str]: + """ + Returns the recognized format names. + """ + return { + field.name: str(getattr(FORMAT, field.name)) + for field in fields(FORMAT) + if not field.name.startswith("_") + } + class StdinProxy: """ @@ -84,11 +103,12 @@ def get_file_format(path: DefinitePath) -> str: :raises: ValueError if the path/filename suffix is unrecognized. """ suffix = Path(path).suffix.replace(".", "") - if fmt := vars(FORMAT).get(suffix): - return fmt - msg = f"Cannot deduce format of '{path}' from unknown extension '{suffix}'" - log.critical(msg) - raise ValueError(msg) + try: + return FORMAT.formats()[suffix] + except KeyError as e: + msg = f"Cannot deduce format of '{path}' from unknown extension '{suffix}'" + log.critical(msg) + raise UWError(msg) from e def handle_existing(directory: DefinitePath, exist_act: str) -> None: diff --git a/src/uwtools/utils/processing.py b/src/uwtools/utils/processing.py index bbb54ca34..3be731525 100644 --- a/src/uwtools/utils/processing.py +++ b/src/uwtools/utils/processing.py @@ -4,8 +4,7 @@ from pathlib import Path from subprocess import STDOUT, CalledProcessError, check_output -from types import SimpleNamespace as ns -from typing import Dict, Optional, Union +from typing import Dict, Optional, Tuple, Union from uwtools.logging import log @@ -15,7 +14,7 @@ def execute( cwd: Optional[Union[Path, str]] = None, env: Optional[Dict[str, str]] = None, log_output: Optional[bool] = False, -) -> ns: +) -> Tuple[bool, str]: """ Execute a command in a subshell. @@ -49,4 +48,4 @@ def execute( logfunc("%sOutput:", indent) for line in output.split("\n"): logfunc("%s%s", indent * 2, line) - return ns(output=output, success=success) + return success, output