diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..bc5c85ce9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,61 @@ +name: Bug Report +description: File a bug report to help us improve e3sm_diags +title: "[Bug]: " +labels: ["bug"] +assignees: [] +body: + - type: textarea + id: what-happened + attributes: + label: What happened? + description: | + Thanks for reporting a bug! Please describe what you were trying to get done. + Tell us what happened, what went wrong. + validations: + required: true + + - type: textarea + id: what-did-you-expect-to-happen + attributes: + label: What did you expect to happen? Are there are possible answers you came across? + description: | + Describe what you expected to happen. Include links to pages you've researched (e.g., software docs, Stack Overflow posts). + validations: + required: false + + - type: textarea + id: sample-code + attributes: + label: Minimal Complete Verifiable Example (MVCE) + description: | + Minimal, self-contained copy-pastable example that generates the issue if possible. Please be concise with code posted (e.g., module imports, publicly accessible files). + Bug reports that follow these guidelines are easier to diagnose, and so are often handled much more quickly. This section will be automatically formatted into code, so no need for markdown backticks. + + See guidelines below on how to provide a good MCVE: + + - [Minimal Complete Verifiable Examples](https://stackoverflow.com/help/mcve) + - [Craft Minimal Bug Reports](http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) + render: python + + - type: textarea + id: log-output + attributes: + label: Relevant log output + description: Please copy and paste any relevant output. This will be automatically formatted into code, so no need for markdown backticks. + render: python + + - type: textarea + id: extra + attributes: + label: Anything else we need to know? + description: | + Please describe any other information you want to share. + + - type: textarea + id: show-versions + attributes: + label: Environment + description: | + Paste your conda environment here (`conda info`). + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..f6d25a7a0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,10 @@ +blank_issues_enabled: true +contact_links: + - name: Questions + url: https://github.com/E3SM-Project/e3sm_diags/discussions + about: | + Ask questions and discuss with other e3sm_diags community members here. Please + browse the e3sm_diags Discussions Forum or e3sm_diags documentation first before asking a + question to make sure it is not already answered. If you can't find an + answer, please include a self-contained reproducible example with your + question if possible. Thanks! diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 000000000..5166f5098 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,15 @@ +name: Documentation Update +description: Update e3sm_diags documentation +title: "[Doc]: " +labels: ["documentation"] +assignees: [] +body: + - type: textarea + id: description + attributes: + label: Describe your documentation update + description: | + Concise description of why the documentation is being updated (e.g., missing content for new feature, typo) + If this is related to an issue or PR, please mention it. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..f8e97ad36 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,42 @@ +name: Feature Request +description: Suggest an idea for e3sm_diags +title: "[Feature]: " +labels: ["enhancement"] +assignees: [] +body: + - type: textarea + id: description + attributes: + label: Is your feature request related to a problem? + description: | + Please do a quick search of existing issues to make sure that this has not been asked before. + Please provide a clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Describe the solution you'd like + description: | + A clear and concise description of what you want to happen. + validations: + required: false + + - type: textarea + id: alternatives + attributes: + label: Describe alternatives you've considered + description: | + A clear and concise description of any alternative solutions or features you've considered. + validations: + required: false + + - type: textarea + id: additional-context + attributes: + label: Additional context + description: | + Add any other context about the feature request here. + validations: + required: false diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..9868a618e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,24 @@ +## Description + + + +- Closes # + +## Checklist + +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my own code +- [ ] My changes generate no new warnings +- [ ] Any dependent changes have been merged and published in downstream modules + +If applicable: + +- [ ] New and existing unit tests pass with my changes (locally and CI/CD build) +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] I have noted that this is a breaking change for a major release (fix or feature that would cause existing functionality to not work as expected) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index df255151a..cbe3de14e 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -11,7 +11,7 @@ on: env: CANCEL_OTHERS: true - PATHS_IGNORE: '["**/README.md", "**/docs/**", "**/examples/**", "**/misc/**", "**/.vscode/**"]' + PATHS_IGNORE: '["**/README.md", "**/docs/**", "**/examples/**", "**/misc/**", "**/.vscode/**", "**/.github/pull_request_template.md", "**/.github/ISSUE_TEMPLATE"]' jobs: pre-commit-hooks: @@ -99,10 +99,18 @@ jobs: mamba info - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} - name: Run Tests + name: Run Unit Tests + run: pytest tests/e3sm_diags + + - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} + name: Download Integration Test Data + run: python -m tests.integration.download_data + + - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} + name: Run Integration Tests env: CHECK_IMAGES: False - run: bash tests/test.sh + run: pytest tests/integration publish-docs: if: ${{ github.event_name == 'push' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 140dce44f..05513b719 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,14 +4,14 @@ fail_fast: true repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 23.9.1 hooks: - id: black @@ -23,15 +23,15 @@ repos: # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab # https://github.com/pre-commit/pre-commit/issues/1206 - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 - args: ["--config=setup.cfg"] - additional_dependencies: [flake8-isort] + args: [--config=setup.cfg] + additional_dependencies: [flake8-isort==6.1.0] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 + rev: v1.5.1 hooks: - id: mypy - args: ["--config=setup.cfg"] - additional_dependencies: [types-pyYAML==6.0.12.6] + args: [--config=pyproject.toml] + additional_dependencies: [dask, numpy>=1.23.0, types-PyYAML] diff --git a/.vscode/e3sm_diags.code-workspace b/.vscode/e3sm_diags.code-workspace index 31c07ce29..c7d968567 100644 --- a/.vscode/e3sm_diags.code-workspace +++ b/.vscode/e3sm_diags.code-workspace @@ -8,6 +8,9 @@ "path": ".." } ], + // =========================== + // VS Code Workspace Settings. + // =========================== "settings": { // =================== // Editor settings @@ -22,17 +25,14 @@ "editor.rulers": [80, 88, 120], "editor.wordWrap": "wordWrapColumn", "editor.wordWrapColumn": 120, - "editor.defaultFormatter": "ms-python.python" + "editor.defaultFormatter": "ms-python.black-formatter" }, // Code Formatting and Linting // --------------------------- - "python.formatting.provider": "black", - "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--config=setup.cfg"], + "flake8.args": ["--config=setup.cfg"], // Type checking // --------------------------- - "python.linting.mypyEnabled": true, - "python.linting.mypyArgs": ["--config=setup.cfg"], + "mypy-type-checker.args": ["--config=pyproject.toml"], // Testing // --------------------------- // NOTE: Debugger doesn't work if pytest-cov is enabled, so set "--no-cov" @@ -49,5 +49,24 @@ "editor.wordWrap": "wordWrapColumn", "editor.wordWrapColumn": 120 } + }, + // ===================================== + // VS Code Python Debugger Configuration + // ===================================== + "launch": { + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": true, + "env": { + "PYTHONPATH": "${workspaceFolder}" + } + } + ] } } diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..9736f7704 --- /dev/null +++ b/Makefile @@ -0,0 +1,90 @@ +.PHONY: clean clean-test clean-pyc clean-build docs help +.DEFAULT_GOAL := help + +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python -c "$$BROWSER_PYSCRIPT" + +# To run these commands: make +# ================================================== + +help: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +# Clean local repository +# ---------------------- +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr conda-build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr tests_coverage_reports/ + rm -f .coverage + rm -fr htmlcov/ + rm -f coverage.xml + rm -fr .pytest_cache + rm -rf .mypy_cache + +clean-test-int-res: ## remove integration test results and image check failures + rm -rf tests/integration/all_sets_results_test + rm -rf tests/integration/image_check_failures + +clean-test-int-data: # remove integration test data and images (expected) -- useful when they are updated + rm -rf tests/integration/integration_test_data + rm -rf tests/integration/integration_test_images + +# Quality Assurance +# ---------------------- +pre-commit: # run pre-commit quality assurance checks + pre-commit run --all-files + +lint: ## check style with flake8 + flake8 e3sm_diags tests + +test: ## run tests quickly with the default Python and produces code coverage report + pytest + $(BROWSER) tests_coverage_reports/htmlcov/index.html + +# Documentation +# ---------------------- +docs: ## generate Sphinx HTML documentation, including API docs + rm -rf docs/generated + cd docs && make html + $(MAKE) -C docs clean + $(MAKE) -C docs html + $(BROWSER) docs/_build/html/index.html + +# Build +# ---------------------- +install: clean ## install the package to the active Python's site-packages + python -m pip install . diff --git a/analysis_data_preprocess/create_OMI-MLS_climo.sh b/analysis_data_preprocess/create_OMI-MLS_climo.sh index eccb5c404..849080d2e 100644 --- a/analysis_data_preprocess/create_OMI-MLS_climo.sh +++ b/analysis_data_preprocess/create_OMI-MLS_climo.sh @@ -18,7 +18,7 @@ mkdir $climo_output_path mkdir $tmp -#Add lon dimention to zonel mean +#Add lon dimention to zonal mean ncap2 -s 'SCO=O3strat' ${original_data_path}O3strat_ZMK.nc ${time_series_output_path}SCO_200410_201712.nc #cp ${original_data_path}O3strat_ZMK.nc ${time_series_output_path}SCO_200410_201712.nc cdo splityear ${time_series_output_path}SCO_200410_201712.nc ${tmp}sco diff --git a/conda-env/ci.yml b/conda-env/ci.yml index cfb172672..46f88d448 100644 --- a/conda-env/ci.yml +++ b/conda-env/ci.yml @@ -13,7 +13,6 @@ dependencies: - beautifulsoup4 - cartopy >=0.17.0 - cartopy_offlinedata - - cdp 1.7.0 - cdms2 3.1.5 - cdutil 8.2.1 - dask @@ -23,6 +22,7 @@ dependencies: - mache >=0.15.0 - matplotlib-base - netcdf4 + - output_viewer >=1.3.0 - numpy >=1.23.0 - shapely >=2.0.0,<3.0.0 - xarray >=2023.02.0 @@ -36,4 +36,3 @@ dependencies: - sphinx - sphinx_rtd_theme - sphinx-multiversion -prefix: /opt/miniconda3/envs/e3sm_diags_ci diff --git a/conda-env/dev-nompi.yml b/conda-env/dev-nompi.yml new file mode 100644 index 000000000..b73bc33bc --- /dev/null +++ b/conda-env/dev-nompi.yml @@ -0,0 +1,56 @@ +# Conda development environment for testing local source code changes to `e3sm_diags` before merging them to production (`master` branch). +# This version contains the no MPI version of `esmf` as a workaround for allowing VS Code's testing API to work. +# The MPI version of `esmf` is usually installed by default, but it breaks VS Code's testing API because it throws a mysterious +# `yaksa` warning. +# More info: https://github.com/E3SM-Project/e3sm_diags/issues/737 +name: e3sm_diags_dev_nompi +channels: + - conda-forge + - defaults +dependencies: + # Base + # ================= + - python >=3.9 + - pip + - beautifulsoup4 + - cartopy >=0.17.0 + - cartopy_offlinedata + - cdms2 3.1.5 + - cdutil 8.2.1 + - dask + - esmf >=8.4.0 nompi* + - esmpy >=8.4.0 + - genutil 8.2.1 + - lxml + - mache >=0.15.0 + - matplotlib-base + - netcdf4 + - output_viewer >=1.3.0 + - numpy >=1.23.0 + - shapely >=2.0.0,<3.0.0 + - xarray >=2023.02.0 + # Testing + # ======================= + - scipy + - pytest + - pytest-cov + # Documentation + # ======================= + - sphinx + - sphinx_rtd_theme + - sphinx-multiversion + # Quality Assurance Tools + # ======================= + # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in + # `.pre-commit.config.yaml`, then update the pinned versions here. + - black=23.9.1 + - flake8=6.1.0 + - flake8-isort=6.1.0 + - isort=5.12.0 + - mypy=1.5.1 + - pre-commit >=3.0.0 + - types-PyYAML >=6.0.0 + # Developer Tools + # ======================= + - tbump=6.9.0 + - ipykernel diff --git a/conda-env/dev.yml b/conda-env/dev.yml index 002b858df..1d07735c1 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -11,7 +11,6 @@ dependencies: - beautifulsoup4 - cartopy >=0.17.0 - cartopy_offlinedata - - cdp 1.7.0 - cdms2 3.1.5 - cdutil 8.2.1 - dask @@ -21,32 +20,32 @@ dependencies: - mache >=0.15.0 - matplotlib-base - netcdf4 + - output_viewer >=1.3.0 - numpy >=1.23.0 - shapely >=2.0.0,<3.0.0 - xarray >=2023.02.0 # Testing - # ================== + # ======================= - scipy - pytest - pytest-cov # Documentation - # ================= + # ======================= - sphinx - sphinx_rtd_theme - sphinx-multiversion + # Quality Assurance Tools + # ======================= + # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in + # `.pre-commit.config.yaml`, then update the pinned versions here. + - black=23.9.1 + - flake8=6.1.0 + - flake8-isort=6.1.0 + - isort=5.12.0 + - mypy=1.5.1 + - pre-commit >=3.0.0 + - types-PyYAML >=6.0.0 # Developer Tools - # ================= - # If versions are updated, also update 'rev' in `.pre-commit.config.yaml` - - black=22.10.0 - - flake8=6.0.0 - - flake8-isort=5.0.3 - - isort=5.11.3 - - mypy=0.991 - - pre-commit=2.20.0 - - pytest=7.2.0 - - pytest-cov=4.0.0 - - types-PyYAML=6.0.12.6 - # Developer Tools - # ================= + # ======================= - tbump=6.9.0 -prefix: /opt/miniconda3/envs/e3sm_diags_dev + - ipykernel diff --git a/docs/source/dev_guide/adding-new-diags-sets.rst b/docs/source/dev_guide/adding-new-diags-sets.rst index 39764dd09..480af91f1 100644 --- a/docs/source/dev_guide/adding-new-diags-sets.rst +++ b/docs/source/dev_guide/adding-new-diags-sets.rst @@ -616,7 +616,7 @@ create a file ``diff_diags_viewer.py`` paste in the below code. import os from .utils import add_header, h1_to_h3 from .default_viewer import create_metadata - from cdp.cdp_viewer import OutputViewer + from e3sm_diags.viewer.core_viewer import OutputViewer def create_viewer(root_dir, parameters): diff --git a/docs/source/dev_guide/testing.rst b/docs/source/dev_guide/testing.rst index c61a41042..aa21ed3e4 100644 --- a/docs/source/dev_guide/testing.rst +++ b/docs/source/dev_guide/testing.rst @@ -56,7 +56,7 @@ The unit and integration tests are run automatically as part of this. Complete run test ----------------- -``tests/complete_run.py`` checks the images generated by all diagnostics to +``tests/integration/complete_run.py`` checks the images generated by all diagnostics to see if any differ from expected. This test is not run as part of the unit test suite, because it relies on a large quantity of data found on LCRC (Anvil/Chrysalis). @@ -78,7 +78,7 @@ Now that you have your changes on LCRC, enter your development environment. Then .. code:: pip install . # Install your changes - python -m unittest tests/complete_run.py + pytest tests/integration/complete_run.py If this test passes, you're done. If it fails however, that means your code has changed what the output looks like. @@ -108,11 +108,11 @@ then you need to update the expected images. find . -type f -name '*.png' > ../image_list_all_sets.txt cd .. -Run ``python -m unittest tests/complete_run.py`` again. Now, the test should pass. +Run ``pytest tests/integration/complete_run.py`` again. Now, the test should pass. After merging your pull request, edit ``README.md``. The version should be the version of E3SM Diags you ran -``python -m unittest tests/complete_run.py`` with, -the date should be the date you ran ``python -m unittest tests/complete_run.py`` on, +``pytest tests/integration/complete_run.py`` with, +the date should be the date you ran ``pytest tests/integration/complete_run.py`` on, and the hash should be for the top commit shown by ``git log`` or on https://github.com/E3SM-Project/e3sm_diags/commits/main. diff --git a/docs/source/dev_guide/using-cdp-output-viewer.rst b/docs/source/dev_guide/using-cdp-output-viewer.rst index c48418e47..2bc076161 100644 --- a/docs/source/dev_guide/using-cdp-output-viewer.rst +++ b/docs/source/dev_guide/using-cdp-output-viewer.rst @@ -99,7 +99,7 @@ The code below was used to create the figures above. .. code:: python - from cdp.cdp_viewer import OutputViewer + from e3sm_diags.viewer.core_viewer import OutputViewer viewer = OutputViewer(index_name='My Cool Results') viewer.add_page("My Results", ['Description', 'Generated File']) diff --git a/docs/source/index.rst b/docs/source/index.rst index 24222d264..54297ee35 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -143,7 +143,7 @@ Additional back-ends could be implemented if the need arose. | :align: center | :align: center | | :target: _static/index/fig21.png | :target: _static/index/fig22.png | | | | -| Tropical Cyclone Track Density | Annual Cycle Zonel Mean plot | +| Tropical Cyclone Track Density | Annual Cycle Zonal Mean plot | +--------------------------------------------------------+------------------------------------------------------+ | .. figure:: _static/index/fig23.png | .. figure:: _static/index/fig24.png | | :align: center | :align: center | diff --git a/e3sm_diags/__init__.py b/e3sm_diags/__init__.py index dd5651c2b..51beb8d1a 100644 --- a/e3sm_diags/__init__.py +++ b/e3sm_diags/__init__.py @@ -6,7 +6,7 @@ # issue with dask when using ESMF with system compilers. import shapely -__version__ = "v2.9.0rc3" +__version__ = "v2.10.1" INSTALL_PATH = os.path.join(sys.prefix, "share/e3sm_diags/") # Disable MPI in cdms2, which is not currently supported by E3SM-unified @@ -20,3 +20,4 @@ # Used by numpy, causes too many threads to spawn otherwise. os.environ["OPENBLAS_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" diff --git a/e3sm_diags/derivations/acme.py b/e3sm_diags/derivations/acme.py index b7c4f020a..52e602e64 100644 --- a/e3sm_diags/derivations/acme.py +++ b/e3sm_diags/derivations/acme.py @@ -465,7 +465,8 @@ def cosp_bin_sum( tau_high0: Optional[float], ): """sum of cosp bins to calculate cloud fraction in specified cloud top pressure / height and - cloud thickness bins, input variable has dimension (cosp_prs,cosp_tau,lat,lon)/(cosp_ht,cosp_tau,lat,lon)""" + cloud thickness bins, input variable has dimension (cosp_prs,cosp_tau,lat,lon)/(cosp_ht,cosp_tau,lat,lon) + """ prs: FileAxis = cld.getAxis(0) tau: FileAxis = cld.getAxis(1) @@ -2073,3 +2074,392 @@ def cosp_histogram_standardize(cld: "FileVariable"): "sitemptop": OrderedDict([(("sitemptop",), rename)]), "siv": OrderedDict([(("siv",), rename)]), } + +# Names of 2D aerosol burdens, including cloud-borne aerosols +aero_burden_list = [ + "ABURDENDUST", + "ABURDENSO4", + "ABURDENSO4_STR", + "ABURDENSO4_TRO", + "ABURDENPOM", + "ABURDENMOM", + "ABURDENSOA", + "ABURDENBC", + "ABURDENSEASALT", +] + + +def aero_burden_fxn(var): + """ + Scale the aerosol burden by 1e6. + + Parameters: + var (cdms2.TransientVariable): The input burden in kg/m2. + + Returns: + burden (cdms2.TransientVariable): The output burden in 1e-6 kg/m2. + """ + burden = var * 1e6 + burden.units = "1e-6 kg/m2" + return burden + + +# Add burden vars to derived_variables +for aero_burden_item in aero_burden_list: + derived_variables[aero_burden_item] = OrderedDict( + [((aero_burden_item,), aero_burden_fxn)] + ) + + +# Names of 2D mass slices of aerosol species +# Also add 3D masses while at it (if available) +aero_mass_list = [] +for aero_name in ["dst", "mom", "pom", "so4", "soa", "ncl", "bc"]: + for aero_lev in ["_srf", "_200", "_330", "_500", "_850", ""]: + # Note that the empty string (last entry) will get the 3D mass fields + aero_mass_list.append(f"Mass_{aero_name}{aero_lev}") + + +def aero_mass_fxn(var): + """ + Scale the given mass by 1e12. + + Parameters: + var (cdms2.TransientVariable): The input mass in kg/kg. + + Returns: + cdms2.TransientVariable: The aerosol mass concentration in 1e-12 kg/kg units. + """ + mass = var * 1e12 + mass.units = "1e-12 kg/kg" + return mass + + +# Add burden vars to derived_variables +for aero_mass_item in aero_mass_list: + derived_variables[aero_mass_item] = OrderedDict( + [((aero_mass_item,), aero_mass_fxn)] + ) + +# Add all the output_aerocom_aie.F90 variables to aero_rename_list +# components/eam/src/physics/cam/output_aerocom_aie.F90 +aero_aerocom_list = [ + "angstrm", + "aerindex", + "cdr", + "cdnc", + "cdnum", + "icnum", + "clt", + "lcc", + "lwp", + "iwp", + "icr", + "icc", + "cod", + "ccn", + "ttop", + "htop", + "ptop", + "autoconv", + "accretn", + "icnc", + "rh700", + "rwp", + "intccn", + "colrv", + "lwp2", + "iwp2", + "lwpbf", + "iwpbf", + "cdnumbf", + "icnumbf", + "aod400", + "aod700", + "colccn.1", + "colccn.3", + "ccn.1bl", + "ccn.3bl", +] + +# Add aerocom vars to derived_variables +for aero_aerocom_item in aero_aerocom_list: + derived_variables[aero_aerocom_item] = OrderedDict([((aero_aerocom_item,), rename)]) + + +def incldtop_cdnc(cdnc, lcc): + """ + Return the in-cloud cloud droplet number concentration at cloud top. + + Parameters: + cdnc (cdms2.TransientVariable): Cloud droplet number concentration in 1/m3. + lcc (cdms2.TransientVariable): Liquid cloud fraction. + + Returns: + var (cdms2.TransientVariable): In-cloud cdnc at cloud top in 1/cm3. + """ + var = cdnc * 1e-6 / lcc + var.units = "1/cm3" + var.long_name = "In-cloud-top CDNC" + return var + + +def cldtop_cdnc(cdnc): + """ + Return the in-grid cloud droplet number concentration at cloud top. + + Args: + cdnc (cdms2.TransientVariable): Cloud droplet number concentration in 1/m3. + + Returns: + var (cdms2.TransientVariable): In-grid cdnc at cloud top in 1/cm3. + """ + var = cdnc * 1e-6 + var.units = "1/cm3" + var.long_name = "In-grid cloud-top CDNC" + return var + + +def incldtop_icnc(icnc, icc): + """ + Return the in-cloud ice crystal number concentration at cloud top. + + Parameters: + icnc (cdms2.TransientVariable): ice crystal number concentration in 1/m3. + icc (cdms2.TransientVariable): ice cloud fraction. + + Returns: + var (cdms2.TransientVariable): In-cloud cdnc at cloud top in 1/cm3. + """ + var = icnc * 1e-6 / icc + var.units = "1/cm3" + var.long_name = "In-cloud-top ICNC" + return var + + +def cldtop_icnc(icnc): + """ + Return the in-grid ice crystal number concentration at cloud top. + + Args: + icnc (cdms2.TransientVariable): Cloud crystal number concentration in 1/m3. + + Returns: + var (cdms2.TransientVariable): In-grid icnc at cloud top in 1/cm3. + """ + var = icnc * 1e-6 + var.units = "1/cm3" + var.long_name = "In-grid cloud-top ICNC" + return var + + +def incld_lwp(lwp, lcc): + """ + Return the in-cloud liquid water path (LWP). + + Parameters: + lwp (cdms2.TransientVariable): Liquid water path in kg/m2. + lcc (cdms2.TransientVariable): Liquid cloud fraction. + + Returns: + cdms2.TransientVariable: In-cloud liquid water path in g/cm3. + """ + var = 1e3 * lwp / lcc + var.units = "g/cm3" + var.long_name = "In-cloud LWP" + return var + + +def cld_lwp(lwp): + """ + Return the grid-mean-cloud LWP in g/cm3. + + Parameters: + lwp (cdms2.TransientVariable): Liquid Water Path (LWP) value. + + Returns: + cdms2.TransientVariable: Grid-mean-cloud LWP in g/cm3. + """ + var = 1e3 * lwp + var.units = "g/cm3" + var.long_name = "In-grid LWP" + return var + + +def incld_iwp(iwp, icc): + """ + Return the in-cloud ice water path (IWP). + + Parameters: + iwp (cdms2.TransientVariable): Ice water path in kg/m2. + icc (cdms2.TransientVariable): Ice cloud fraction. + + Returns: + cdms2.TransientVariable: In-cloud IWP in g/cm3. + """ + var = 1e3 * iwp / icc + var.units = "g/cm3" + var.long_name = "In-cloud IWP" + return var + + +def cld_iwp(iwp): + """ + Return the in-grid ice water path (IWP). + + Parameters: + iwp (cdms2.TransientVariable): Ice water path in kg/m2. + + Returns: + cdms2.TransientVariable: In-grid IWP in g/cm3. + """ + var = 1e3 * iwp + var.units = "g/cm3" + var.long_name = "In-grid IWP" + return var + + +# add cdnc, icnc, lwp, iwp to derived_variables +derived_variables.update( + { + "in_cloud_cdnc": OrderedDict([(("cdnc", "lcc"), incldtop_cdnc)]), + "in_grid_cdnc": OrderedDict([(("cdnc",), cldtop_cdnc)]), + "in_cloud_icnc": OrderedDict([(("icnc", "icc"), incldtop_icnc)]), + "in_grid_icnc": OrderedDict([(("icnc",), cldtop_icnc)]), + "in_cloud_lwp": OrderedDict([(("lwp", "lcc"), incld_lwp)]), + "in_grid_lwp": OrderedDict([(("lwp",), cld_lwp)]), + "in_cloud_iwp": OrderedDict([(("iwp", "icc"), incld_iwp)]), + "in_grid_iwp": OrderedDict([(("iwp",), cld_iwp)]), + } +) + + +def erf_tot(fsnt, flnt): + """ + Calculate the total effective radiative forcing (ERFtot). + + Args: + fsnt (cdms2.TransientVariable): The incoming sw radiation at the top of the atmosphere. + flnt (cdms2.TransientVariable): The outgoing lw radiation at the top of the atmosphere. + + Returns: + var (cdms2.TransientVariable): The ERFtot which represents the total erf. + + See Ghan 2013 for derivation of ERF decomposition: https://doi.org/10.5194/acp-13-9971-2013 + """ + var = fsnt - flnt + var.units = "W/m2" + var.long_name = "ERFtot: total effect" + return var + + +def erf_ari(fsnt, flnt, fsnt_d1, flnt_d1): + """ + Calculate aerosol--radiation interactions (ARI) part of effective radiative forcing (ERF). + + Parameters: + fsnt (cdms2.TransientVariable): Net solar flux at the top of the atmosphere. + flnt (cdms2.TransientVariable): Net longwave flux at the top of the atmosphere. + fsnt_d1 (cdms2.TransientVariable): fsnt without aerosols. + flnt_d1 (cdms2.TransientVariable): flnt without aerosols. + + Returns: + var (cdms2.TransientVariable): ERFari (aka, direct effect) in W/m2. + + See Ghan 2013 for derivation of ERF decomposition: https://doi.org/10.5194/acp-13-9971-2013 + """ + var = (fsnt - flnt) - (fsnt_d1 - flnt_d1) + var.units = "W/m2" + var.long_name = "ERFari: direct effect" + return var + + +def erf_aci(fsnt_d1, flnt_d1, fsntc_d1, flntc_d1): + """ + Calculate aerosol--cloud interactions (ACI) part of effectie radiative forcing (ERF) + + Parameters: + fsnt_d1 (cdms2.TransientVariable): Downward shortwave radiation toa without aerosols. + flnt_d1 (cdms2.TransientVariable): Upward longwave radiation toa without aerosols. + fsntc_d1 (cdms2.TransientVariable): fsnt_d1 without clouds. + flntc_d1 (cdms2.TransientVariable): flnt_d1 without clouds. + + Returns: + var (cdms2.TransientVariable): ERFaci (aka, indirect effect) in W/m2. + + See Ghan 2013 for derivation of ERF decomposition: https://doi.org/10.5194/acp-13-9971-2013 + """ + var = (fsnt_d1 - flnt_d1) - (fsntc_d1 - flntc_d1) + var.units = "W/m2" + var.long_name = "ERFaci: indirect effect" + return var + + +def erf_res(fsntc_d1, flntc_d1): + """ + Calculate the residual effect (RES) part of effective radiative forcin g. + + Parameters: + fsntc_d1 (cdms2.TransientVariable): Downward solar radiation at the top of the atmosphere + with neither clouds nor aerosols. + flntc_d1 (cdms2.TransientVariable): Upward longwave radiation at the top of the atmosphere + with neither clouds nor aerosols. + + Returns: + var (cdms2.TransientVariable): ERFres (aka, surface effect) in W/m2. + + See Ghan 2013 for derivation of ERF decomposition: https://doi.org/10.5194/acp-13-9971-2013 + """ + var = fsntc_d1 - flntc_d1 + var.units = "W/m2" + var.long_name = "ERFres: residual effect" + return var + + +derived_variables.update( + { + "ERFtot": OrderedDict([(("FSNT", "FLNT"), erf_tot)]), + "ERFari": OrderedDict([(("FSNT", "FLNT", "FSNT_d1", "FLNT_d1"), erf_ari)]), + "ERFaci": OrderedDict( + [(("FSNT_d1", "FLNT_d1", "FSNTC_d1", "FLNTC_d1"), erf_aci)] + ), + "ERFres": OrderedDict([(("FSNTC_d1", "FLNTC_d1"), erf_res)]), + } +) + +# Add more AOD terms +# Note that AODVIS and AODDUST are already added elsewhere +aero_aod_list = [ + "AODBC", + "AODPOM", + "AODMOM", + "AODSO4", + "AODSO4_STR", + "AODSO4_TRO", + "AODSS", + "AODSOA", +] + +# Add aod vars to derived_variables +for aero_aod_item in aero_aod_list: + derived_variables[aero_aod_item] = OrderedDict([((aero_aod_item,), rename)]) + +# Add 3D variables related to aerosols and chemistry +# Note that O3 is already added above +# Note that 3D mass vars are already added by the empty string above "" +# Note that it is possible to create on-the-fly slices from these variables with +# a function of the form: +# def aero_3d_slice(var, lev): +# return var[lev, :, :] +aero_chem_list = [ + "DMS", + "H2O2", + "H2SO4", + "NO3", + "OH", + "SO2", +] + +# Add aero/chem vars to derived_variables +for aero_chem_item in aero_chem_list: + derived_variables[aero_chem_item] = OrderedDict([((aero_chem_item,), rename)]) diff --git a/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py b/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py index 9b5f2c775..1fcb75743 100755 --- a/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py +++ b/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py @@ -101,7 +101,7 @@ def run_diag(parameter: ACzonalmeanParameter) -> ACzonalmeanParameter: parameter.var_id = var parameter.output_file = "-".join([ref_name, var, "Annual-Cycle"]) - parameter.main_title = str(" ".join([var, "Zonel Mean Annual Cycle"])) + parameter.main_title = str(" ".join([var, "Zonal Mean Annual Cycle"])) parameter.viewer_descr[var] = ( test_ac.long_name diff --git a/e3sm_diags/driver/arm_diags_driver.py b/e3sm_diags/driver/arm_diags_driver.py index 633c23dcd..56d5f6399 100644 --- a/e3sm_diags/driver/arm_diags_driver.py +++ b/e3sm_diags/driver/arm_diags_driver.py @@ -423,7 +423,6 @@ def run_diag_aerosol_activation(parameter: ARMDiagsParameter) -> ARMDiagsParamet logger.info("Selected region: {}".format(region)) # Possible variables are ccn01, ccn02, ccn05 for variable in variables: - test_data = utils.dataset.Dataset(parameter, test=True) test_a_num = test_data.get_timeseries_variable("a_num", single_point=True)[ @@ -441,7 +440,6 @@ def run_diag_aerosol_activation(parameter: ARMDiagsParameter) -> ARMDiagsParamet ) if "armdiags" in ref_name: - ref_file = os.path.join( ref_path, region[:3] + "armdiagsaciactivate" + region[3:5].upper() + ".c1.nc", @@ -568,7 +566,6 @@ def run_diag_pdf_daily(parameter: ARMDiagsParameter): def run_diag(parameter: ARMDiagsParameter) -> ARMDiagsParameter: - if parameter.diags_set == "annual_cycle": return run_diag_annual_cycle(parameter) elif parameter.diags_set == "diurnal_cycle": diff --git a/e3sm_diags/driver/cosp_histogram_driver.py b/e3sm_diags/driver/cosp_histogram_driver.py index ab42bdffe..5018ad389 100755 --- a/e3sm_diags/driver/cosp_histogram_driver.py +++ b/e3sm_diags/driver/cosp_histogram_driver.py @@ -121,8 +121,8 @@ def run_diag(parameter: CoreParameter) -> CoreParameter: ) utils.general.save_ncfiles( parameter.current_set, - mv1_domain, - mv2_domain, + mv1_domain_mean, + mv2_domain_mean, diff, parameter, ) diff --git a/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_model.cfg b/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_model.cfg index d0ea991a7..d79624e6a 100644 --- a/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_model.cfg +++ b/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_model.cfg @@ -253,6 +253,16 @@ diff_colormap = "BrBG_r" contour_levels = [0., 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2] diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.02, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["AODDUST", "AODBC", "AODSS", "AODPOM", "AODMOM", "AODSOA", "AODSO4", "AODSO4_STR", "AODSO4_TRO"] +test_colormap = "Oranges" +reference_colormap = "Oranges" +diff_colormap = "BrBG_r" +contour_levels = [0, 0.01, 0.02, 0.03, 0.06, 0.09, 0.12, 0.15, 0.18, 0.21, 0.24, 0.27, 0.30] +diff_levels = [-0.10, -0.08, -0.06, -0.04, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.10] + [#] sets = ["annual_cycle_zonal_mean"] case_id = "model_vs_model" @@ -290,3 +300,180 @@ reference_colormap = "WhiteBlueGreenYellowRed.rgb" diff_colormap = "diverging_bwr.rgb" contour_levels = [12,16,20,24,28,32,36,40,44] diff_levels = [-20,-15,-10,-5,-2,2,5,10,15,20] + + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["ERFari"] +contour_levels = [-5, -4, -3, -2, -1, -0.5, -0.25, 0.25, 0.5, 1, 2, 3, 4, 5] +diff_levels = [-2.5, -2, -1.5, -1, -0.5, -0.25, -0.10, 0.10, 0.25, 0.5, 1, 1.5, 2, 2.5] + + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["ERFaci", "ERFres", "ERFtot"] +contour_levels = [-120, -100, -80, -60, -40, -20, -10, -5, -2.5, 2.5, 5, 10, 20, 40, 60, 80, 100, 120] +diff_levels = [-10, -8, -6, -4, -2, -1, -0.5, -0.25, 0.25, 0.5, 1, 2, 4, 6, 8, 10] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_bc_srf"] +contour_levels = [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550] +diff_levels = [-600, -400, -200, -100, -50, -25, -10, -5, 5, 10, 25, 50, 100, 200, 400, 600] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_ncl_srf"] +contour_levels = [0, 50, 100, 200, 500, 1000, 5000, 10000, 20000] +diff_levels = [-1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_so4_srf"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500] +diff_levels = [-2000, -1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500, 2000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_dst_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 20000, 40000, 60000, 80000, 100000] +diff_levels = [-15000, -10000, -5000, -2500, -1000, -500, -250, -100, -50, -25, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 15000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_pom_srf"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500] +diff_levels = [-2000, -1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500, 2000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_soa_srf"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500, 3000, 4000] +diff_levels = [-3000, -2000, -1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500, 2000, 3000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_mom_srf"] +contour_levels = [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550, 650, 750] +diff_levels = [-100, -50, -25, -10, -5, 5, 10, 25, 50, 100] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_bc_850"] +contour_levels = [0, 5, 10, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200] +diff_levels = [-150, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 150] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_ncl_850"] +contour_levels = [0, 50, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_so4_850"] +contour_levels = [0, 10, 20, 50, 100, 150, 200, 250, 500, 750, 1000, 1500, 2000] +diff_levels = [-1500, -1000, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 1000, 1500] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_dst_850"] +contour_levels = [0, 1000, 2000, 5000, 10000, 20000, 30000, 40000, 50000, 70000, 90000, 120000] +diff_levels = [-10000, -8000, -6000, -4000, -2000, -1000, -500, -300, -200, -100, -50, -20, -10, 10, 20,50, 100, 200, 300, 500, 1000, 2000, 4000, 6000, 8000, 10000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_pom_850"] +contour_levels = [0, 10, 20, 50, 100, 150, 200, 250, 500, 750, 1000, 1500, 2000] +diff_levels = [-900, -600, -400, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 600, 900] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_soa_850"] +contour_levels = [0, 50, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-3000, -1500, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 1500, 3000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_mom_850"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 150, 200, 300] +diff_levels = [-30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_bc_330"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 60] +diff_levels = [-50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_ncl_330"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 500] +diff_levels = [-50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_so4_330"] +contour_levels = [0, 10, 20, 50, 100, 200, 300, 400, 500, 700, 1000] +diff_levels = [-700, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 700] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_dst_330"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000, 10000, 13000] +diff_levels = [-1000, -700, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 700, 1000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_pom_330"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 150, 200, 300] +diff_levels = [-200, -150, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100, 150, 200] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_soa_330"] +contour_levels = [0, 10, 20, 50, 100, 200, 300, 400, 500, 700, 1000, 1200] +diff_levels = [-700, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 700] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["Mass_mom_330"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 3, 4, 5, 7, 10] +diff_levels = [-1, -0.7, -0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.02, -0.01, -0.005, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["in_cloud_cdnc", "in_cloud_lwp"] +contour_levels = [0, 20, 40, 60, 80, 100, 125, 150, 175, 200, 250] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "model_vs_model" +variables = ["in_grid_cdnc", "in_grid_lwp"] +contour_levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] diff --git a/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_obs.cfg b/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_obs.cfg index 110e35513..ea44f5272 100644 --- a/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_obs.cfg +++ b/e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_model_vs_obs.cfg @@ -394,6 +394,15 @@ diff_colormap = "BrBG_r" contour_levels = [0., 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2] diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.02, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["AODDUST", "AODBC", "AODSS", "AODPOM", "AODMOM", "AODSOA", "AODSO4", "AODSO4_STR", "AODSO4_TRO"] +test_colormap = "Oranges" +reference_colormap = "Oranges" +diff_colormap = "BrBG_r" +contour_levels = [0, 0.01, 0.02, 0.03, 0.06, 0.09, 0.12, 0.15, 0.18, 0.21, 0.24, 0.27, 0.30] +diff_levels = [-0.10, -0.08, -0.06, -0.04, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.10] [#] sets = ["annual_cycle_zonal_mean"] @@ -461,3 +470,164 @@ reference_colormap = "WhiteBlueGreenYellowRed.rgb" diff_colormap = "diverging_bwr.rgb" contour_levels = [12,16,20,24,28,32,36,40,44] diff_levels = [-20,-15,-10,-5,-2,2,5,10,15,20] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_srf"] +contour_levels = [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550] +diff_levels = [-600, -400, -200, -100, -50, -25, -10, -5, 5, 10, 25, 50, 100, 200, 400, 600] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_srf"] +contour_levels = [0, 50, 100, 200, 500, 1000, 5000, 10000, 20000] +diff_levels = [-1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_srf"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500] +diff_levels = [-2000, -1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500, 2000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 20000, 40000, 60000, 80000, 100000] +diff_levels = [-15000, -10000, -5000, -2500, -1000, -500, -250, -100, -50, -25, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 15000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_srf"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500] +diff_levels = [-2000, -1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500, 2000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_soa_srf"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500, 3000, 4000] +diff_levels = [-3000, -2000, -1500, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1500, 2000, 3000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_srf"] +contour_levels = [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550, 650, 750] +diff_levels = [-100, -50, -25, -10, -5, 5, 10, 25, 50, 100] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_850"] +contour_levels = [0, 5, 10, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200] +diff_levels = [-150, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 150] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_850"] +contour_levels = [0, 50, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_850"] +contour_levels = [0, 10, 20, 50, 100, 150, 200, 250, 500, 750, 1000, 1500, 2000] +diff_levels = [-1500, -1000, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 1000, 1500] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_850"] +contour_levels = [0, 1000, 2000, 5000, 10000, 20000, 30000, 40000, 50000, 70000, 90000, 120000] +diff_levels = [-10000, -8000, -6000, -4000, -2000, -1000, -500, -300, -200, -100, -50, -20, -10, 10, 20,50, 100, 200, 300, 500, 1000, 2000, 4000, 6000, 8000, 10000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_850"] +contour_levels = [0, 10, 20, 50, 100, 150, 200, 250, 500, 750, 1000, 1500, 2000] +diff_levels = [-900, -600, -400, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 600, 900] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_soa_850"] +contour_levels = [0, 50, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-3000, -1500, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 1500, 3000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_850"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 150, 200, 300] +diff_levels = [-30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_330"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 60] +diff_levels = [-50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_330"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 500] +diff_levels = [-50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_330"] +contour_levels = [0, 10, 20, 50, 100, 200, 300, 400, 500, 700, 1000] +diff_levels = [-700, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 700] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_330"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000, 10000, 13000] +diff_levels = [-1000, -700, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 700, 1000] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_330"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 150, 200, 300] +diff_levels = [-200, -150, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100, 150, 200] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_soa_330"] +contour_levels = [0, 10, 20, 50, 100, 200, 300, 400, 500, 700, 1000, 1200] +diff_levels = [-700, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 700] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_330"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 3, 4, 5, 7, 10] +diff_levels = [-1, -0.7, -0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.02, -0.01, -0.005, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["in_cloud_cdnc", "in_cloud_lwp"] +contour_levels = [0, 20, 40, 60, 80, 100, 125, 150, 175, 200, 250] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["annual_cycle_zonal_mean"] +case_id = "aero-no-ref-data" +variables = ["in_grid_cdnc", "in_grid_lwp"] +contour_levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] diff --git a/e3sm_diags/driver/default_diags/lat_lon_model_vs_model.cfg b/e3sm_diags/driver/default_diags/lat_lon_model_vs_model.cfg index d0ecd47a8..d0e883b04 100644 --- a/e3sm_diags/driver/default_diags/lat_lon_model_vs_model.cfg +++ b/e3sm_diags/driver/default_diags/lat_lon_model_vs_model.cfg @@ -734,3 +734,297 @@ reference_colormap = "WhiteBlueGreenYellowRed.rgb" diff_colormap = "diverging_bwr.rgb" contour_levels = [12,16,20,24,28,32,36,40,44] diff_levels = [-20,-15,-10,-5,-2,2,5,10,15,20] + +[#] +sets = ["lat_lon"] +case_id = "model_vs_model" +variables = ['ABURDENDUST'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 10, 50, 100, 200, 500, 1000] +diff_levels = [-1000, -500, -200, -100, -50, -10, 10, 50, 100, 200, 500, 1000] + +[#] +sets = ["lat_lon"] +case_id = "model_vs_model" +variables = ['ABURDENBC', 'ABURDENSOA'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 1, 2, 4, 8] +diff_levels = [-8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8] + +[#] +sets = ["lat_lon"] +case_id = "model_vs_model" +variables = ['ABURDENPOM', 'ABURDENSEASALT'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 1, 2, 4, 8, 16, 32] +diff_levels = [-8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8] + +[#] +sets = ["lat_lon"] +case_id = "model_vs_model" +variables = ['ABURDENSO4', 'ABURDENSO4_STR', 'ABURDENSO4_TRO'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 1, 2, 4, 8, 16, 32] +diff_levels = [-32, -16, -8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8, 16, 32] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["ERFari"] +contour_levels = [-10, -8, -6, -4, -2, -1, 1, 2, 4, 6, 8, 10] +diff_levels = [-3, -2.5, -2, -1.5, -1, -0.5, 0.5, 1, 1.5, 2, 2.5, 3] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["ERFaci", "ERFres", "ERFtot"] +contour_levels = [-100, -80, -60, -40, -20, -10, 10, 20, 40, 60, 80, 100] +diff_levels = [-3, -2.5, -2, -1.5, -1, -0.5, 0.5, 1, 1.5, 2, 2.5, 3] + + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_bc_srf"] +contour_levels = [0, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000] +diff_levels = [-5000, -4000, -3000, -2000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 2000, 3000, 4000, 5000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_ncl_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 15000, 20000, 25000] +diff_levels = [-1200, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1200] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_so4_srf"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-7000, -5000, -4000, -2000, -1000, -500, -100, -50, 50, 100, 500, 1000, 2000, 4000, 5000, 7000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_dst_srf"] +contour_levels = [0, 1000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000] +diff_levels = [-40000, -20000, -10000, -5000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 5000, 10000, 20000, 40000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_pom_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 15000, 20000, 25000] +diff_levels = [-15000, -10000, -5000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 5000, 10000, 15000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_soa_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 15000, 20000, 25000, 30000] +diff_levels = [-30000, -25000, -20000, -15000, -10000, -5000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 5000, 10000, 15000, 20000, 25000, 30000] + +[srfmom_mass_diags_lat_lon] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_mom_srf"] +contour_levels = [0, 100, 200, 300, 400, 500, 600, 700, 800] +diff_levels = [-50, -25, -10, -5, -2, -1, 1, 2, 5, 10, 25, 50] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_bc_850"] +contour_levels = [0, 500, 1000, 2000, 4000, 6000, 8000, 10000, 12000, 14000] +diff_levels = [-1200, -1000, -800, -600, -400, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 400, 600, 800, 1000, 1200] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_ncl_850"] +contour_levels = [0, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000] +diff_levels = [-600, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 600] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_so4_850"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 5000] +diff_levels = [-5000, -3000, -1000, -500, -300, -100, -50, 50, 100, 300, 500, 1000, 3000, 5000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_dst_850"] +contour_levels = [0, 10000, 50000, 100000, 200000, 300000, 400000, 500000] +diff_levels = [-23000, -20000, -15000, -10000, -5000, -4000, -3000, -2000, -1000, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 1000, 2000, 3000, 4000, 5000, 10000, 20000, 23000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_pom_850"] +contour_levels = [0, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000, 7000] +diff_levels = [-3500, -3000, -2500, -2000, -1500, -1000, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 1000, 1500, 2000, 2500, 3000, 3500] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_soa_850"] +contour_levels = [0, 1000, 5000, 10000, 15000, 20000, 25000, 30000] +diff_levels = [-15000, -10000, -5000, -4000, -3000, -2000, -1000, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 1000, 4000, 5000, 10000, 15000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_mom_850"] +contour_levels = [0, 5, 10, 20, 50, 100, 150, 200] +diff_levels = [-15, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 15] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_bc_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_ncl_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 350] +diff_levels = [-30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_so4_330"] +contour_levels = [0, 10, 50, 100, 200, 300, 400, 500, 600, 700] +diff_levels = [-600, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 600] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_dst_330"] +contour_levels = [0, 500, 1000, 2000, 5000, 10000, 15000] +diff_levels = [-2000, -1000, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100, 500, 1000, 2000] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_pom_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 300] +diff_levels = [-250, -200, -150, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_soa_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 2500] +diff_levels = [-1500, -1000, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100, 500, 1000, 1500] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_mom_330"] +contour_levels = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 10] +diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.04, -0.03, -0.02, -0.01, -0.005, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["AODBC"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09] +diff_levels = [-0.08, -0.06, -0.04, -0.02, -0.01, -0.005, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["AODSS"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.10, 0.11] +diff_levels = [-0.01, -0.008, -0.006, -0.004, -0.002, -0.001, 0.001, 0.002, 0.004, 0.006, 0.008, 0.01] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["AODSOA"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.55] +diff_levels = [-0.45, -0.35, -0.25, -0.15, -0.05, -0.025, 0.025, 0.05, 0.15, 0.25, 0.35, 0.45] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["AODSO4", "AODSO4_STR", "AODSO4_TRO"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16] +diff_levels = [-0.15, -0.12, -0.09, -0.06, -0.03, -0.015, -0.005, 0.005, 0.015, 0.03, 0.06, 0.09, 0.12, 0.15] + + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["AODPOM"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09] +diff_levels = [-0.06, -0.04, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.04, 0.06] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["AODMOM"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09] +diff_levels = [-0.06, -0.04, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.04, 0.06] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["in_cloud_cdnc", "in_cloud_lwp"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 20, 40, 60, 80, 100, 125, 150, 175, 200, 250] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] + +[#] +sets = ['lat_lon'] +case_id = "model_vs_model" +variables = ["in_grid_cdnc", "in_grid_lwp"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] diff --git a/e3sm_diags/driver/default_diags/lat_lon_model_vs_obs.cfg b/e3sm_diags/driver/default_diags/lat_lon_model_vs_obs.cfg index d9e1c40e3..f9d303075 100644 --- a/e3sm_diags/driver/default_diags/lat_lon_model_vs_obs.cfg +++ b/e3sm_diags/driver/default_diags/lat_lon_model_vs_obs.cfg @@ -1453,3 +1453,280 @@ reference_colormap = "WhiteBlueGreenYellowRed.rgb" diff_colormap = "diverging_bwr.rgb" contour_levels = [12,16,20,24,28,32,36,40,44] diff_levels = [-20,-15,-10,-5,-2,2,5,10,15,20] + +[#] +sets = ["lat_lon"] +case_id = "aero-no-ref-data" +variables = ['ABURDENDUST'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 10, 50, 100, 200, 500, 1000] +diff_levels = [-1000, -500, -200, -100, -50, -10, 10, 50, 100, 200, 500, 1000] + +[#] +sets = ["lat_lon"] +case_id = "aero-no-ref-data" +variables = ['ABURDENBC', 'ABURDENSOA'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 1, 2, 4, 8] +diff_levels = [-8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8] + +[#] +sets = ["lat_lon"] +case_id = "aero-no-ref-data" +variables = ['ABURDENPOM', 'ABURDENSEASALT'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 1, 2, 4, 8, 16, 32] +diff_levels = [-8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8] + +[#] +sets = ["lat_lon"] +case_id = "aero-no-ref-data" +variables = ['ABURDENSO4', 'ABURDENSO4_STR', 'ABURDENSO4_TRO'] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "diverging_bwr.rgb" +contour_levels = [0, 1, 2, 4, 8, 16, 32] +diff_levels = [-32, -16, -8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8, 16, 32] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_bc_srf"] +contour_levels = [0, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000] +diff_levels = [-5000, -4000, -3000, -2000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 2000, 3000, 4000, 5000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_ncl_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 15000, 20000, 25000] +diff_levels = [-1200, -1000, -500, -250, -100, -50, -25, -10, 10, 25, 50, 100, 250, 500, 1000, 1200] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_so4_srf"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-7000, -5000, -4000, -2000, -1000, -500, -100, -50, 50, 100, 500, 1000, 2000, 4000, 5000, 7000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_dst_srf"] +contour_levels = [0, 1000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000] +diff_levels = [-40000, -20000, -10000, -5000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 5000, 10000, 20000, 40000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_pom_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 15000, 20000, 25000] +diff_levels = [-15000, -10000, -5000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 5000, 10000, 15000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_soa_srf"] +contour_levels = [0, 1000, 2000, 5000, 10000, 15000, 20000, 25000, 30000] +diff_levels = [-30000, -25000, -20000, -15000, -10000, -5000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 5000, 10000, 15000, 20000, 25000, 30000] + +[srfmom_mass_diags_lat_lon] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_mom_srf"] +contour_levels = [0, 100, 200, 300, 400, 500, 600, 700, 800] +diff_levels = [-50, -25, -10, -5, -2, -1, 1, 2, 5, 10, 25, 50] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_bc_850"] +contour_levels = [0, 500, 1000, 2000, 4000, 6000, 8000, 10000, 12000, 14000] +diff_levels = [-1200, -1000, -800, -600, -400, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 400, 600, 800, 1000, 1200] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_ncl_850"] +contour_levels = [0, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000] +diff_levels = [-600, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 600] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_so4_850"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 5000] +diff_levels = [-5000, -3000, -1000, -500, -300, -100, -50, 50, 100, 300, 500, 1000, 3000, 5000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_dst_850"] +contour_levels = [0, 10000, 50000, 100000, 200000, 300000, 400000, 500000] +diff_levels = [-23000, -20000, -15000, -10000, -5000, -4000, -3000, -2000, -1000, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 1000, 2000, 3000, 4000, 5000, 10000, 20000, 23000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_pom_850"] +contour_levels = [0, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000, 7000] +diff_levels = [-3500, -3000, -2500, -2000, -1500, -1000, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 1000, 1500, 2000, 2500, 3000, 3500] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_soa_850"] +contour_levels = [0, 1000, 5000, 10000, 15000, 20000, 25000, 30000] +diff_levels = [-15000, -10000, -5000, -4000, -3000, -2000, -1000, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 1000, 4000, 5000, 10000, 15000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_mom_850"] +contour_levels = [0, 5, 10, 20, 50, 100, 150, 200] +diff_levels = [-15, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 15] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_bc_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_ncl_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 350] +diff_levels = [-30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_so4_330"] +contour_levels = [0, 10, 50, 100, 200, 300, 400, 500, 600, 700] +diff_levels = [-600, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 600] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_dst_330"] +contour_levels = [0, 500, 1000, 2000, 5000, 10000, 15000] +diff_levels = [-2000, -1000, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100, 500, 1000, 2000] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_pom_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 300] +diff_levels = [-250, -200, -150, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_soa_330"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 70, 2500] +diff_levels = [-1500, -1000, -500, -400, -300, -200, -100, -50, -40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100, 500, 1000, 1500] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +variables = ["Mass_mom_330"] +contour_levels = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 10] +diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.04, -0.03, -0.02, -0.01, -0.005, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["AODBC"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09] +diff_levels = [-0.08, -0.06, -0.04, -0.02, -0.01, -0.005, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["AODSS"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.10, 0.11] +diff_levels = [-0.01, -0.008, -0.006, -0.004, -0.002, -0.001, 0.001, 0.002, 0.004, 0.006, 0.008, 0.01] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["AODSOA"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.55] +diff_levels = [-0.45, -0.35, -0.25, -0.15, -0.05, -0.025, 0.025, 0.05, 0.15, 0.25, 0.35, 0.45] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["AODSO4", "AODSO4_STR", "AODSO4_TRO"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16] +diff_levels = [-0.15, -0.12, -0.09, -0.06, -0.03, -0.015, -0.005, 0.005, 0.015, 0.03, 0.06, 0.09, 0.12, 0.15] + + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["AODPOM"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09] +diff_levels = [-0.06, -0.04, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.04, 0.06] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["AODMOM"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09] +diff_levels = [-0.06, -0.04, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.04, 0.06] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["in_cloud_cdnc", "in_cloud_lwp"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 20, 40, 60, 80, 100, 125, 150, 175, 200, 250] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] + +[#] +sets = ['lat_lon'] +case_id = "aero-no-ref-data" +variables = ["in_grid_cdnc", "in_grid_lwp"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +contour_levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50] diff --git a/e3sm_diags/driver/default_diags/polar_model_vs_model.cfg b/e3sm_diags/driver/default_diags/polar_model_vs_model.cfg index 730ec93c2..72a2b1ac9 100644 --- a/e3sm_diags/driver/default_diags/polar_model_vs_model.cfg +++ b/e3sm_diags/driver/default_diags/polar_model_vs_model.cfg @@ -394,3 +394,384 @@ regions = ["polar_S", "polar_N"] plevs = [200.0] contour_levels = [200, 205, 210, 215, 220, 225, 230, 235, 240] diff_levels = [-10, -7.5, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 7.5, 10] + + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["ERFari"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S", "polar_N"] +contour_levels = [-2, -1.5, -1, -0.75, -0.5, -0.25, -0.10, 0.10, 0.25, 0.5, 0.75, 1, 1.5, 2.0] +diff_levels = [-1.5, -1, -0.75, -0.5, -0.25, -0.10, -0.05, 0.05, 0.10, 0.25, 0.5, 0.75, 1, 1.5] + + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["ERFaci", "ERFres", "ERFtot"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S", "polar_N"] +contour_levels = [-60, -50, -40, -30, -20, -10, -5, -2.5, 2.5, 5, 10, 20, 30, 40, 50, 60] +diff_levels = [-3, -2.5, -2, -1.5, -1, -0.5, -0.25, 0.25, 0.5, 1, 1.5, 2, 2.5, 3] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_bc_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 500, 600, 700] +diff_levels = [-700, -600, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 600, 700] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_bc_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 3, 4, 5] +diff_levels = [-5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_ncl_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N", "polar_S"] +contour_levels = [0, 100, 500, 1000, 2000, 5000, 10000, 15000, 20000] +diff_levels = [-800, -600, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 600, 800] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_so4_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600] +diff_levels = [-1200, -1000, -800, -600, -400, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 400, 600, 800, 1000, 1200] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_so4_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200] +diff_levels = [-20, -10, -5, -2, -1, 1, 2, 5, 10, 20] + + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_dst_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N", "polar_S"] +contour_levels = [0, 100, 500, 1000, 2000, 5000, 10000, 15000, 20000, 30000] +diff_levels = [-6000, -5000, -4000, -3000, -2000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_pom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000, 2400] +diff_levels = [-1500, -1200, -1000, -800, -600, -400, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 400, 600, 800, 1000, 1200, 1500] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_pom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +diff_levels = [-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_mom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 1, 2, 4, 6, 8, 10, 12, 14, 20, 30, 40, 50] +diff_levels = [-25, -20, -15, -10, -5, -2, -1, 1, 2, 5, 10, 15, 20, 25] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_mom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 25, 50, 100, 150, 200, 300, 400, 500] +diff_levels = [-40, -30, -20, -10, -5, -2, -1, 1, 2, 5, 10, 20, 30, 40] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_bc_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 3, 3.5] +diff_levels = [-3, -2, -1, -0.5, -0.2, -0.1, -0.05, -0.02, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 3] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_ncl_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-250, -200, -150, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 150, 200, 250] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_so4_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 150] +diff_levels = [-25, -20, -15, -10, -5, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 5, 10, 15, 20, 25] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_dst_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 50, 100, 200, 300, 400, 500, 700, 1000] +diff_levels = [-100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_pom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20] +diff_levels = [-10, -7, -5, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 5, 7, 10] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_soa_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_mom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 4, 6, 8, 10, 20, 40, 60, 80] +diff_levels = [-8, -6, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 6, 8] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_bc_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 4, 6, 8, 10, 12, 15] +diff_levels = [-10, -8, -6, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 6, 8, 10] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_ncl_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 5, 10, 20, 40, 60, 80, 100, 150, 200, 250, 300, 350] +diff_levels = [-30, -25, -20, -15, -10, -8, -6, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 6, 8, 10, 15, 20, 25, 30] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_so4_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150, 200, 250, 300] +diff_levels = [-150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_dst_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150, 200, 300, 400, 500, 600, 800, 1000, 1200] +diff_levels = [-200, -150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_pom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 2, 4, 6, 8, 10, 20, 30, 40, 50, 60, 80, 100] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, -1, -0.5, 0.5, 1, 2, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_soa_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150, 200, 250, 300, 350] +diff_levels = [-150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_mom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 4, 6, 8, 10] +diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.02, -0.01, -0.005, -0.002, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_bc_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150] +diff_levels = [-100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_ncl_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500, 3000, 4000, 5000] +diff_levels = [-250, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 250] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_so4_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 20, 40, 60, 80, 100, 150, 200, 300, 400, 500, 600, 700, 800, 1000] +diff_levels = [-900, -700, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 700, 800, 900] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_dst_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 1000, 2000, 4000, 6000, 8000, 10000, 15000, 20000] +diff_levels = [-2000, -1500, -1000, -800, -600, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 600, 800, 1000, 1500, 2000] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_pom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 600, 800, 1000, 1200] +diff_levels = [-300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_soa_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500, 3000, 3600] +diff_levels = [-700, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 700] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_mom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 2, 4, 6, 8, 10, 20, 30, 40, 50, 60, 80, 100] +diff_levels = [-10, -8, -6, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 6, 8, 10] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_bc_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 1, 2, 4, 6, 8, 10, 20, 30, 40, 50] +diff_levels = [-50, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_ncl_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 80, 100, 120, 150] +diff_levels = [-30, -20, -15, -10, -8, -6, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 6, 8, 10, 15, 20, 30] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_so4_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 500, 600, 700] +diff_levels = [-600, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 600] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_dst_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 500, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 9000] +diff_levels = [-1000, -700, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 700, 1000] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_pom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 10, 20, 30, 40, 50, 60, 80, 100, 120, 150, 200] +diff_levels = [-120, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 120] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_soa_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 500, 600, 700, 1000] +diff_levels = [-400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["Mass_mom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 0.1, 0.2, 0.4, 0.6, 0.8, 1, 2, 3, 4] +diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["AODDUST", "AODBC", "AODSS", "AODPOM", "AODMOM", "AODSOA", "AODSO4", "AODSO4_STR", "AODSO4_TRO"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S", "polar_N"] +contour_levels = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12] +diff_levels = [-0.02, -0.01, -0.005, -0.0025, -0.0010, -0.0005, 0.0005, 0.0010, 0.0025, 0.005, 0.01, 0.02] + +[#] +sets = ["polar"] +case_id = "model_vs_model" +variables = ["AODVIS"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S", "polar_N"] +contour_levels = [0, 0.03, 0.06, 0.09, 0.12, 0.15, 0.18] +diff_levels = [-0.05, -0.04, -0.03, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05] diff --git a/e3sm_diags/driver/default_diags/polar_model_vs_obs.cfg b/e3sm_diags/driver/default_diags/polar_model_vs_obs.cfg index 13adc3616..bcec76a04 100644 --- a/e3sm_diags/driver/default_diags/polar_model_vs_obs.cfg +++ b/e3sm_diags/driver/default_diags/polar_model_vs_obs.cfg @@ -692,3 +692,364 @@ regions = ["polar_S", "polar_N"] seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] contour_levels = [955, 965, 975,980, 985, 990, 995, 1000, 1005, 1010, 1015, 1020, 1025, 1035] diff_levels = [ -16, -12, -8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8, 12, 16] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 500, 600, 700] +diff_levels = [-700, -600, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 600, 700] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 3, 4, 5] +diff_levels = [-5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N", "polar_S"] +contour_levels = [0, 100, 500, 1000, 2000, 5000, 10000, 15000, 20000] +diff_levels = [-800, -600, -500, -400, -300, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 300, 400, 500, 600, 800] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600] +diff_levels = [-1200, -1000, -800, -600, -400, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 400, 600, 800, 1000, 1200] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200] +diff_levels = [-20, -10, -5, -2, -1, 1, 2, 5, 10, 20] + + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N", "polar_S"] +contour_levels = [0, 100, 500, 1000, 2000, 5000, 10000, 15000, 20000, 30000] +diff_levels = [-6000, -5000, -4000, -3000, -2000, -1000, -500, -100, -50, -25, -10, 10, 25, 50, 100, 500, 1000, 2000, 3000, 4000, 5000, 6000] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000, 2400] +diff_levels = [-1500, -1200, -1000, -800, -600, -400, -200, -100, -50, -25, -10, 10, 25, 50, 100, 200, 400, 600, 800, 1000, 1200, 1500] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +diff_levels = [-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 1, 2, 4, 6, 8, 10, 12, 14, 20, 30, 40, 50] +diff_levels = [-25, -20, -15, -10, -5, -2, -1, 1, 2, 5, 10, 15, 20, 25] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_srf"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 25, 50, 100, 150, 200, 300, 400, 500] +diff_levels = [-40, -30, -20, -10, -5, -2, -1, 1, 2, 5, 10, 20, 30, 40] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 3, 3.5] +diff_levels = [-3, -2, -1, -0.5, -0.2, -0.1, -0.05, -0.02, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 3] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 7000] +diff_levels = [-250, -200, -150, -100, -50, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 50, 100, 150, 200, 250] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 100, 150] +diff_levels = [-25, -20, -15, -10, -5, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 5, 10, 15, 20, 25] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 50, 100, 200, 300, 400, 500, 700, 1000] +diff_levels = [-100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20] +diff_levels = [-10, -7, -5, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 5, 7, 10] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_soa_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 1, 2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] +diff_levels = [-40, -30, -20, -10, -5, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 5, 10, 20, 30, 40] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.5, 1, 2, 4, 6, 8, 10, 20, 40, 60, 80] +diff_levels = [-8, -6, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 6, 8] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 4, 6, 8, 10, 12, 15] +diff_levels = [-10, -8, -6, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 6, 8, 10] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 5, 10, 20, 40, 60, 80, 100, 150, 200, 250, 300, 350] +diff_levels = [-30, -25, -20, -15, -10, -8, -6, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 6, 8, 10, 15, 20, 25, 30] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150, 200, 250, 300] +diff_levels = [-150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150, 200, 300, 400, 500, 600, 800, 1000, 1200] +diff_levels = [-200, -150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 2, 4, 6, 8, 10, 20, 30, 40, 50, 60, 80, 100] +diff_levels = [-50, -40, -30, -20, -10, -5, -2, -1, -0.5, 0.5, 1, 2, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_soa_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150, 200, 250, 300, 350] +diff_levels = [-150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +contour_levels = [0, 0.1, 0.2, 0.5, 1, 2, 4, 6, 8, 10] +diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, -0.02, -0.01, -0.005, -0.002, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 10, 20, 40, 60, 80, 100, 150] +diff_levels = [-100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500, 3000, 4000, 5000] +diff_levels = [-250, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, -2, 2, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 250] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 20, 40, 60, 80, 100, 150, 200, 300, 400, 500, 600, 700, 800, 1000] +diff_levels = [-900, -700, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 700, 800, 900] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 1000, 2000, 4000, 6000, 8000, 10000, 15000, 20000] +diff_levels = [-2000, -1500, -1000, -800, -600, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 600, 800, 1000, 1500, 2000] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 600, 800, 1000, 1200] +diff_levels = [-300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_soa_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 100, 200, 400, 600, 800, 1000, 1500, 2000, 2500, 3000, 3600] +diff_levels = [-700, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 700] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_850"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 2, 4, 6, 8, 10, 20, 30, 40, 50, 60, 80, 100] +diff_levels = [-10, -8, -6, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 6, 8, 10] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_bc_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 1, 2, 4, 6, 8, 10, 20, 30, 40, 50] +diff_levels = [-50, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 50] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_ncl_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 5, 10, 20, 30, 40, 50, 60, 80, 100, 120, 150] +diff_levels = [-30, -20, -15, -10, -8, -6, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 6, 8, 10, 15, 20, 30] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_so4_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 500, 600, 700] +diff_levels = [-600, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 600] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_dst_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 500, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 9000] +diff_levels = [-1000, -700, -500, -400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400, 500, 700, 1000] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_pom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 10, 20, 30, 40, 50, 60, 80, 100, 120, 150, 200] +diff_levels = [-120, -100, -80, -60, -40, -30, -20, -10, -5, 5, 10, 20, 30, 40, 60, 80, 100, 120] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_soa_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 50, 100, 200, 300, 400, 500, 600, 700, 1000] +diff_levels = [-400, -300, -200, -150, -100, -80, -60, -40, -30, -20, -10, 10, 20, 30, 40, 60, 80, 100, 150, 200, 300, 400] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["Mass_mom_330"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_N"] +contour_levels = [0, 0.1, 0.2, 0.4, 0.6, 0.8, 1, 2, 3, 4] +diff_levels = [-0.5, -0.4, -0.3, -0.2, -0.1, -0.05, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["AODDUST", "AODBC", "AODSS", "AODPOM", "AODMOM", "AODSOA", "AODSO4", "AODSO4_STR", "AODSO4_TRO"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S", "polar_N"] +contour_levels = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12] +diff_levels = [-0.02, -0.01, -0.005, -0.0025, -0.0010, -0.0005, 0.0005, 0.0010, 0.0025, 0.005, 0.01, 0.02] + +[#] +sets = ["polar"] +case_id = "aero-no-ref-data" +variables = ["AODVIS"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] +regions = ["polar_S", "polar_N"] +contour_levels = [0, 0.03, 0.06, 0.09, 0.12, 0.15, 0.18] +diff_levels = [-0.05, -0.04, -0.03, -0.02, -0.01, -0.005, -0.0025, 0.0025, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05] diff --git a/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_model.cfg b/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_model.cfg index 844d93c46..e91583974 100644 --- a/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_model.cfg +++ b/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_model.cfg @@ -296,3 +296,41 @@ sets = ["zonal_mean_xy"] case_id = "model_vs_model" variables = ["TGCLDLWP_OCN"] seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] + +[#] +sets = ["zonal_mean_xy"] +case_id = "model_vs_model" +variables = ["ERFtot", "ERFari", "ERFaci", "ERFres"] +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] + +[#] +sets = ["zonal_mean_xy"] +case_id = "model_vs_model" +variables = ['Mass_bc_200', 'Mass_ncl_200', 'Mass_so4_200', 'Mass_dst_200', 'Mass_pom_200', 'Mass_soa_200', 'Mass_mom_200'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + +[#] +sets = ["zonal_mean_xy"] +case_id = "model_vs_model" +variables = ['Mass_bc_srf', 'Mass_ncl_srf', 'Mass_so4_srf', 'Mass_dst_srf', 'Mass_pom_srf', 'Mass_soa_srf', 'Mass_mom_srf'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + +[#] +sets = ["zonal_mean_xy"] +case_id = "model_vs_model" +variables = ['Mass_bc_850', 'Mass_ncl_850', 'Mass_so4_850', 'Mass_dst_850', 'Mass_pom_850', 'Mass_soa_850', 'Mass_mom_850'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + + +[#] +sets = ["zonal_mean_xy"] +case_id = "model_vs_model" +variables = ['Mass_bc_330', 'Mass_ncl_330', 'Mass_so4_330', 'Mass_dst_330', 'Mass_pom_330', 'Mass_soa_330', 'Mass_mom_330'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + + +[#] +sets = ["zonal_mean_xy"] +case_id = "model_vs_model" +variables = ["in_grid_cdnc", "in_grid_lwp", "in_cloud_cdnc", "in_cloud_lwp"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] diff --git a/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_obs.cfg b/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_obs.cfg index a9f0af873..bdf58e278 100644 --- a/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_obs.cfg +++ b/e3sm_diags/driver/default_diags/zonal_mean_xy_model_vs_obs.cfg @@ -535,3 +535,35 @@ variables = ["PminusE"] ref_name = "COREv2_Flux" reference_name = "COREv2_Flux" seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] + +[#] +sets = ["zonal_mean_xy"] +case_id = "aero-no-ref-data" +variables = ['Mass_bc_200', 'Mass_ncl_200', 'Mass_so4_200', 'Mass_dst_200', 'Mass_pom_200', 'Mass_soa_200', 'Mass_mom_200'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + +[#] +sets = ["zonal_mean_xy"] +case_id = "aero-no-ref-data" +variables = ['Mass_bc_srf', 'Mass_ncl_srf', 'Mass_so4_srf', 'Mass_dst_srf', 'Mass_pom_srf', 'Mass_soa_srf', 'Mass_mom_srf'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + +[#] +sets = ["zonal_mean_xy"] +case_id = "aero-no-ref-data" +variables = ['Mass_bc_850', 'Mass_ncl_850', 'Mass_so4_850', 'Mass_dst_850', 'Mass_pom_850', 'Mass_soa_850', 'Mass_mom_850'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + + +[#] +sets = ["zonal_mean_xy"] +case_id = "aero-no-ref-data" +variables = ['Mass_bc_330', 'Mass_ncl_330', 'Mass_so4_330', 'Mass_dst_330', 'Mass_pom_330', 'Mass_soa_330', 'Mass_mom_330'] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] + + +[#] +sets = ["zonal_mean_xy"] +case_id = "aero-no-ref-data" +variables = ["in_grid_cdnc", "in_grid_lwp", "in_cloud_cdnc", "in_cloud_lwp"] +seasons = ["ANN","DJF", "MAM", "JJA", "SON"] diff --git a/e3sm_diags/driver/enso_diags_driver.py b/e3sm_diags/driver/enso_diags_driver.py index e216b87ae..ff060110a 100644 --- a/e3sm_diags/driver/enso_diags_driver.py +++ b/e3sm_diags/driver/enso_diags_driver.py @@ -272,7 +272,11 @@ def run_diag_map(parameter: EnsoDiagsParameter) -> EnsoDiagsParameter: ) # Reference - (ref_domain, ref_reg_coe, ref_confidence_levels,) = perform_regression( + ( + ref_domain, + ref_reg_coe, + ref_confidence_levels, + ) = perform_regression( ref_data, parameter, var, diff --git a/e3sm_diags/driver/lat_lon_driver.py b/e3sm_diags/driver/lat_lon_driver.py index efc92c386..2b5e63c50 100755 --- a/e3sm_diags/driver/lat_lon_driver.py +++ b/e3sm_diags/driver/lat_lon_driver.py @@ -175,12 +175,8 @@ def run_diag(parameter: CoreParameter) -> CoreParameter: # noqa: C901 # Select plev. for ilev in range(len(plev)): - mv1 = mv1_p[ - ilev, - ] - mv2 = mv2_p[ - ilev, - ] + mv1 = mv1_p[ilev,] + mv2 = mv2_p[ilev,] for region in regions: parameter.var_region = region diff --git a/e3sm_diags/driver/polar_driver.py b/e3sm_diags/driver/polar_driver.py index 75930e7ea..32a8dfc19 100755 --- a/e3sm_diags/driver/polar_driver.py +++ b/e3sm_diags/driver/polar_driver.py @@ -133,12 +133,8 @@ def run_diag(parameter: CoreParameter) -> CoreParameter: # Select plev. for ilev in range(len(plev)): - mv1 = mv1_p[ - ilev, - ] - mv2 = mv2_p[ - ilev, - ] + mv1 = mv1_p[ilev,] + mv2 = mv2_p[ilev,] for region in regions: logger.info("Selected region: {}".format(region)) diff --git a/e3sm_diags/driver/streamflow_driver.py b/e3sm_diags/driver/streamflow_driver.py index 2cf14288c..13810780b 100644 --- a/e3sm_diags/driver/streamflow_driver.py +++ b/e3sm_diags/driver/streamflow_driver.py @@ -285,7 +285,7 @@ def setup_test(parameter, var, using_test_mat_file): if parameter.print_statements: # For edison: 720x360x600 logger.info("test_array.shape={}".format(test_array.shape)) - if type(area_upstream) == cdms2.tvariable.TransientVariable: + if isinstance(area_upstream, cdms2.tvariable.TransientVariable): area_upstream = area_upstream.getValue() return area_upstream, test_array @@ -489,7 +489,7 @@ def generate_export( else: monthly = mmat - if type(monthly) == cdms2.tvariable.TransientVariable: + if isinstance(monthly, cdms2.tvariable.TransientVariable): monthly = monthly.getValue() seasonality_index_ref, peak_month_ref = get_seasonality(monthly) @@ -515,7 +515,7 @@ def generate_export( else: monthly = mmat - if type(monthly) == cdms2.tvariable.TransientVariable: + if isinstance(monthly, cdms2.tvariable.TransientVariable): monthly = monthly.getValue() seasonality_index_test, peak_month_test = get_seasonality(monthly) diff --git a/e3sm_diags/driver/tc_analysis_driver.py b/e3sm_diags/driver/tc_analysis_driver.py index 9db1c06ed..de82b2c1b 100644 --- a/e3sm_diags/driver/tc_analysis_driver.py +++ b/e3sm_diags/driver/tc_analysis_driver.py @@ -265,9 +265,9 @@ def _get_vars_from_te_stitch( vars_dict["yearmc"][k - 1, index - 1] = float(line_split[6]) vars_dict["monthmc"][k - 1, index - 1] = float(line_split[7]) - vars_dict["year_start"] = year_start - vars_dict["year_end"] = year_end - vars_dict["num_years"] = year_end - year_start + 1 + vars_dict["year_start"] = year_start # type: ignore + vars_dict["year_end"] = year_end # type: ignore + vars_dict["num_years"] = year_end - year_start + 1 # type: ignore logger.info( f"TE Start Year: {vars_dict['year_start']}, TE End Year: {vars_dict['year_end']}, Total Years: {vars_dict['num_years']}" ) @@ -340,7 +340,6 @@ def _derive_metrics_per_basin( and lat[0] > basin_info[3] and lat[0] < basin_info[4] ): - mod_num_ocn = mod_num_ocn + 1 mod_mon.append(mon[0]) mod_wnd.append(np.max(wind)) @@ -491,7 +490,7 @@ def _calc_mean_ace(vsmc: "MaskedArray", yearic: np.ndarray, num_rows: int) -> fl wind_ts = wind[wind >= 35] ace[i] = ace[i] + np.sum(wind_ts**2) / 1e4 - return np.mean(ace) + return np.mean(ace) # type: ignore def _calc_ts_intensity_dist(wind_speeds: List[int]) -> np.ndarray: diff --git a/e3sm_diags/driver/utils/general.py b/e3sm_diags/driver/utils/general.py index c051d93c8..b4d5b07fd 100644 --- a/e3sm_diags/driver/utils/general.py +++ b/e3sm_diags/driver/utils/general.py @@ -293,7 +293,7 @@ def save_transient_variables_to_netcdf(set_num, variables_dict, label, parameter Save the transient variables to nc file. """ if parameter.save_netcdf: - for (variable_name, variable) in variables_dict.items(): + for variable_name, variable in variables_dict.items(): # Set cdms preferences - no compression, no shuffling, no complaining cdms2.setNetcdfDeflateFlag(1) # 1-9, min to max - Comes at heavy IO (read/write time cost) diff --git a/e3sm_diags/driver/zonal_mean_xy_driver.py b/e3sm_diags/driver/zonal_mean_xy_driver.py index 2069d8c28..195922f05 100755 --- a/e3sm_diags/driver/zonal_mean_xy_driver.py +++ b/e3sm_diags/driver/zonal_mean_xy_driver.py @@ -171,12 +171,8 @@ def run_diag(parameter: CoreParameter) -> CoreParameter: # Select plev. for ilev in range(len(plev)): - mv1 = mv1_p[ - ilev, - ] - mv2 = mv2_p[ - ilev, - ] + mv1 = mv1_p[ilev,] + mv2 = mv2_p[ilev,] for region in regions: logger.info(f"Selected region: {region}") diff --git a/e3sm_diags/e3sm_diags_driver.py b/e3sm_diags/e3sm_diags_driver.py index 53fe24623..57ed8aed9 100644 --- a/e3sm_diags/e3sm_diags_driver.py +++ b/e3sm_diags/e3sm_diags_driver.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # The above line is needed for `test_all_sets.test_all_sets_mpl`. # Otherwise, OSError: [Errno 8] Exec format error: 'e3sm_diags_driver.py'. - -import importlib import os import subprocess import sys @@ -10,6 +8,7 @@ from typing import Dict, List, Tuple import dask +import dask.bag as db import e3sm_diags from e3sm_diags.logger import custom_logger @@ -251,56 +250,6 @@ def create_parameter_dict(parameters): return d -def run_diag(parameter: CoreParameter) -> List[CoreParameter]: - """Run the corresponding diagnostics for a set of parameters. - - Additional CoreParameter (or CoreParameter sub-class) objects are derived - from the CoreParameter `sets` attribute, hence this function returns a - list of CoreParameter objects. - - This function loops over the specified diagnostic sets and runs the - diagnostic using the parameters. - - Parameters - ---------- - parameter : CoreParameter - The CoreParameter object to run diagnostics on. - - Returns - ------- - List[CoreParameter] - The list of CoreParameter objects with results from the diagnostic run. - """ - results = [] - - for set_name in parameter.sets: - # FIXME: The parameter and driver for a diagnostic should be mapped - # together. If this is done, the `run_diag` function should be - # accessible by the parameter class without needing to perform a static - # string reference for the module name. - parameter.current_set = set_name - mod_str = "e3sm_diags.driver.{}_driver".format(set_name) - - # Check if there is a matching driver module for the `set_name`. - try: - module = importlib.import_module(mod_str) - except ModuleNotFoundError as e: - logger.error(f"'Error with set name {set_name}'", exc_info=e) - continue - - # If the module exists, call the driver module's `run_diag` function. - try: - single_result = module.run_diag(parameter) - results.append(single_result) - except Exception: - logger.exception(f"Error in {mod_str}", exc_info=True) - - if parameter.debug: - sys.exit() - - return results - - def _run_serially(parameters: List[CoreParameter]) -> List[CoreParameter]: """Run diagnostics with the parameters serially. @@ -314,14 +263,16 @@ def _run_serially(parameters: List[CoreParameter]) -> List[CoreParameter]: List[CoreParameter] The list of CoreParameter objects with results from the diagnostic run. """ - results = [] + # A nested list of lists, where a sub-list represents the results of + # the sets related to the CoreParameter object. + nested_results: List[List[CoreParameter]] = [] - for p in parameters: - results.append(run_diag(p)) + for parameter in parameters: + nested_results.append(parameter._run_diag()) # `results` becomes a list of lists of parameters so it needs to be # collapsed a level. - collapsed_results = _collapse_results(results) + collapsed_results = _collapse_results(nested_results) return collapsed_results @@ -350,7 +301,7 @@ def _run_with_dask(parameters: List[CoreParameter]) -> List[CoreParameter]: https://docs.dask.org/en/stable/generated/dask.bag.map.html https://docs.dask.org/en/stable/generated/dask.dataframe.DataFrame.compute.html """ - bag = dask.bag.from_sequence(parameters) + bag = db.from_sequence(parameters) config = {"scheduler": "processes", "multiprocessing.context": "fork"} num_workers = getattr(parameters[0], "num_workers", None) @@ -362,7 +313,7 @@ def _run_with_dask(parameters: List[CoreParameter]) -> List[CoreParameter]: ) with dask.config.set(config): - results = bag.map(run_diag).compute(num_workers=num_workers) + results = bag.map(CoreParameter._run_diag).compute(num_workers=num_workers) # `results` becomes a list of lists of parameters so it needs to be # collapsed a level. @@ -397,15 +348,16 @@ def _collapse_results(parameters: List[List[CoreParameter]]) -> List[CoreParamet return output_parameters -def main(parameters=[]): +def main(parameters=[]) -> List[CoreParameter]: # Get the diagnostic run parameters # --------------------------------- parser = CoreParser() # If no parameters are passed, use the parser args as defaults. Otherwise, # create the dictionary of expected parameters. - if not parameters: + if len(parameters) == 0: parameters = get_parameters(parser) + expected_parameters = create_parameter_dict(parameters) if not os.path.exists(parameters[0].results_dir): @@ -458,6 +410,8 @@ def main(parameters=[]): ) raise Exception(message) + return parameters_results + if __name__ == "__main__": main() diff --git a/e3sm_diags/parameter/core_parameter.py b/e3sm_diags/parameter/core_parameter.py index e2caba989..5382c74c1 100644 --- a/e3sm_diags/parameter/core_parameter.py +++ b/e3sm_diags/parameter/core_parameter.py @@ -1,5 +1,37 @@ import copy -from typing import Dict, List +import importlib +import sys +from typing import Any, Dict, List + +from e3sm_diags.logger import custom_logger + +logger = custom_logger(__name__) + +# FIXME: There is probably a better way of defining default sets because most of +# this is repeated in SETS_TO_PARAMETERS and SETS_TO_PARSERS. +# Also integration tests will break if "mp_partition" is included because +# we did not take it into account yet. +DEFAULT_SETS = [ + "zonal_mean_xy", + "zonal_mean_2d", + "zonal_mean_2d_stratosphere", + "meridional_mean_2d", + "lat_lon", + "polar", + "area_mean_time_series", + "cosp_histogram", + "enso_diags", + "qbo", + "streamflow", + "diurnal_cycle", + "arm_diags", + "tc_analysis", + "annual_cycle_zonal_mean", + "lat_lon_land", + "lat_lon_river", + "aerosol_aeronet", + "aerosol_budget", +] class CoreParameter: @@ -52,28 +84,8 @@ def __init__(self): # 'model_vs_obs' (by default), 'model_vs_model', or 'obs_vs_obs'. self.run_type: str = "model_vs_obs" - # A list of the sets to be run. Default is all sets - self.sets: List[str] = [ - "zonal_mean_xy", - "zonal_mean_2d", - "zonal_mean_2d_stratosphere", - "meridional_mean_2d", - "lat_lon", - "polar", - "area_mean_time_series", - "cosp_histogram", - "enso_diags", - "qbo", - "streamflow", - "diurnal_cycle", - "arm_diags", - "tc_analysis", - "annual_cycle_zonal_mean", - "lat_lon_land", - "lat_lon_river", - "aerosol_aeronet", - "aerosol_budget", - ] + # A list of the sets to be run, by default all sets. + self.sets: List[str] = DEFAULT_SETS # The current set that is being ran when looping over sets in # `e3sm_diags_driver.run_diag()`. @@ -213,3 +225,48 @@ def check_values(self): ): msg = "You need to define both the 'test_start_yr' and 'test_end_yr' parameter." raise RuntimeError(msg) + + def _run_diag(self) -> List[Any]: + """Run the diagnostics for each set in the parameter. + + Additional CoreParameter (or CoreParameter sub-class) objects are derived + from the CoreParameter `sets` attribute, hence this function returns a + list of CoreParameter objects. + + This method loops over the parameter's diagnostic sets and attempts to + import and call the related `run_diags()` function. + + Returns + ------- + List[Any] + The list of CoreParameter objects with results from the diagnostic run. + NOTE: `Self` type is not yet supported by mypy. + """ + results = [] + + for set_name in self.sets: + self.current_set = set_name + # FIXME: This is a shortcut to importing `run_diag`, but can break + # easily because the module driver is statically imported via string. + # Instead, the import should be done more progammatically via + # direct Python import. + mod_str = "e3sm_diags.driver.{}_driver".format(set_name) + + # Check if there is a matching driver module for the `set_name`. + try: + module = importlib.import_module(mod_str) + except ModuleNotFoundError as e: + logger.error(f"'Error with set name {set_name}'", exc_info=e) + continue + + # If the module exists, call the driver module's `run_diag` function. + try: + single_result = module.run_diag(self) + results.append(single_result) + except Exception: + logger.exception(f"Error in {mod_str}", exc_info=True) + + if self.debug: + sys.exit() + + return results diff --git a/e3sm_diags/parser/core_parser.py b/e3sm_diags/parser/core_parser.py index 2d1defd23..d823de253 100644 --- a/e3sm_diags/parser/core_parser.py +++ b/e3sm_diags/parser/core_parser.py @@ -815,21 +815,21 @@ def _get_cfg_parameters( parameters = [] cfg_file_obj = self._create_cfg_hash_titles(cfg_file) - kwargs = ( - {"strict": False} if sys.version_info[0] >= 3 else {} - ) # 'strict' keyword doesn't work in Python 2. - config = configparser.ConfigParser( # type: ignore - **kwargs - ) # Allow for two lines to be the same. - config.readfp(cfg_file_obj) - - for section in config.sections(): + + # Setting `strict=False` enables the parser to allow for any section + # or duplicates while reading from a single source. This is required + # because .cfg diagnostic files might contain duplicate sections with + # slight tweaks based on the set. + parser = configparser.ConfigParser(strict=False) + parser.read_file(cfg_file_obj) + + for section in parser.sections(): p = self._parameter_cls() # Remove all of the variables. p.__dict__.clear() - for k, v in config.items(section): + for k, v in parser.items(section): v = yaml.safe_load(v) setattr(p, k, v) diff --git a/e3sm_diags/plot/cartopy/aerosol_aeronet_plot.py b/e3sm_diags/plot/cartopy/aerosol_aeronet_plot.py index 3e75918b6..9f43dc8df 100644 --- a/e3sm_diags/plot/cartopy/aerosol_aeronet_plot.py +++ b/e3sm_diags/plot/cartopy/aerosol_aeronet_plot.py @@ -114,7 +114,7 @@ def plot(test, test_site, ref_site, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/annual_cycle_zonal_mean_plot.py b/e3sm_diags/plot/cartopy/annual_cycle_zonal_mean_plot.py index 498b8c4fc..ab72288b4 100644 --- a/e3sm_diags/plot/cartopy/annual_cycle_zonal_mean_plot.py +++ b/e3sm_diags/plot/cartopy/annual_cycle_zonal_mean_plot.py @@ -40,7 +40,6 @@ def get_ax_size(fig, ax): def plot_panel(n, fig, var, clevels, cmap, title, parameters, stats=None): - mon = var.getTime() lat = var.getLatitude() var = np.transpose(var) @@ -172,7 +171,7 @@ def plot(reference, test, diff, metrics_dict, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/area_mean_time_series_plot.py b/e3sm_diags/plot/cartopy/area_mean_time_series_plot.py index 7e6941656..eebafc0d4 100644 --- a/e3sm_diags/plot/cartopy/area_mean_time_series_plot.py +++ b/e3sm_diags/plot/cartopy/area_mean_time_series_plot.py @@ -126,7 +126,7 @@ def plot(var, regions_to_data, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/arm_diags_plot.py b/e3sm_diags/plot/cartopy/arm_diags_plot.py index 4837eab60..cf485dd06 100644 --- a/e3sm_diags/plot/cartopy/arm_diags_plot.py +++ b/e3sm_diags/plot/cartopy/arm_diags_plot.py @@ -174,6 +174,7 @@ def plot_convection_onset_statistics( var_time_absolute = cwv.getTime().asComponentTime() time_interval = int(var_time_absolute[1].hour - var_time_absolute[0].hour) + # FIXME: UnboundLocalError: local variable 'cwv_max' referenced before assignment number_of_bins = int(np.ceil((cwv_max - cwv_min) / bin_width)) bin_center = np.arange( (cwv_min + (bin_width / 2)), diff --git a/e3sm_diags/plot/cartopy/cosp_histogram_plot.py b/e3sm_diags/plot/cartopy/cosp_histogram_plot.py index b4982275f..eab7cd328 100644 --- a/e3sm_diags/plot/cartopy/cosp_histogram_plot.py +++ b/e3sm_diags/plot/cartopy/cosp_histogram_plot.py @@ -39,7 +39,6 @@ def get_ax_size(fig, ax): def plot_panel(n, fig, _, var, clevels, cmap, title, parameters, stats=None): - # Contour levels levels = None norm = None @@ -170,7 +169,6 @@ def plot_panel(n, fig, _, var, clevels, cmap, title, parameters, stats=None): def plot(reference, test, diff, _, parameter): - # Create figure, projection fig = plt.figure(figsize=parameter.figsize, dpi=parameter.dpi) @@ -251,7 +249,7 @@ def plot(reference, test, diff, _, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/diurnal_cycle_plot.py b/e3sm_diags/plot/cartopy/diurnal_cycle_plot.py index 44560bb2d..b1073af88 100644 --- a/e3sm_diags/plot/cartopy/diurnal_cycle_plot.py +++ b/e3sm_diags/plot/cartopy/diurnal_cycle_plot.py @@ -65,7 +65,6 @@ def determine_tick_step(degrees_covered): def plot_panel(n, fig, proj, var, amp, amp_ref, title, parameter): - normalize_test_amp = parameter.normalize_test_amp specified_max_amp = parameter.normalize_amp_int @@ -129,7 +128,7 @@ def plot_panel(n, fig, proj, var, amp, amp_ref, title, parameter): # If less than 0.50 is subtracted, then 0 W will overlap 0 E on the left side of the plot. # If a number is added, then the value won't show up at all. if global_domain or full_lon: - xticks = [0, 60, 120, 180, 240, 300, 359.99] + xticks = [0, 60, 120, 180, 240, 300, 359.99] # type: ignore else: xticks = np.append(xticks, lon_east) proj = ccrs.PlateCarree() @@ -309,7 +308,7 @@ def plot(test_tmax, test_amp, ref_tmax, ref_amp, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot subplot_suffix = ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/enso_diags_plot.py b/e3sm_diags/plot/cartopy/enso_diags_plot.py index 02c4c3795..457c8a1dc 100644 --- a/e3sm_diags/plot/cartopy/enso_diags_plot.py +++ b/e3sm_diags/plot/cartopy/enso_diags_plot.py @@ -63,7 +63,6 @@ def determine_tick_step(degrees_covered): def plot_panel_map( n, fig, proj, var, clevels, cmap, title, parameter, conf=None, stats={} ): - var = add_cyclic(var) lon = var.getLongitude() lat = var.getLatitude() @@ -330,7 +329,7 @@ def plot_map( subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot subplot_suffix = ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/lat_lon_land_plot.py b/e3sm_diags/plot/cartopy/lat_lon_land_plot.py index 1ce44a16d..c7da4782d 100644 --- a/e3sm_diags/plot/cartopy/lat_lon_land_plot.py +++ b/e3sm_diags/plot/cartopy/lat_lon_land_plot.py @@ -4,5 +4,4 @@ def plot(reference, test, diff, metrics_dict, parameter): - return base_plot(reference, test, diff, metrics_dict, parameter) diff --git a/e3sm_diags/plot/cartopy/lat_lon_plot.py b/e3sm_diags/plot/cartopy/lat_lon_plot.py index 1a20ae783..117be5889 100644 --- a/e3sm_diags/plot/cartopy/lat_lon_plot.py +++ b/e3sm_diags/plot/cartopy/lat_lon_plot.py @@ -65,7 +65,6 @@ def determine_tick_step(degrees_covered): def plot_panel( # noqa: C901 n, fig, proj, var, clevels, cmap, title, parameters, stats=None ): - var = add_cyclic(var) lon = var.getLongitude() lat = var.getLatitude() @@ -249,7 +248,6 @@ def plot_panel( # noqa: C901 def plot(reference, test, diff, metrics_dict, parameter): - # Create figure, projection fig = plt.figure(figsize=parameter.figsize, dpi=parameter.dpi) proj = ccrs.PlateCarree() @@ -337,7 +335,7 @@ def plot(reference, test, diff, metrics_dict, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/lat_lon_river_plot.py b/e3sm_diags/plot/cartopy/lat_lon_river_plot.py index 1ce44a16d..c7da4782d 100644 --- a/e3sm_diags/plot/cartopy/lat_lon_river_plot.py +++ b/e3sm_diags/plot/cartopy/lat_lon_river_plot.py @@ -4,5 +4,4 @@ def plot(reference, test, diff, metrics_dict, parameter): - return base_plot(reference, test, diff, metrics_dict, parameter) diff --git a/e3sm_diags/plot/cartopy/meridional_mean_2d_plot.py b/e3sm_diags/plot/cartopy/meridional_mean_2d_plot.py index 855c35317..09d3ed258 100644 --- a/e3sm_diags/plot/cartopy/meridional_mean_2d_plot.py +++ b/e3sm_diags/plot/cartopy/meridional_mean_2d_plot.py @@ -45,7 +45,6 @@ def get_ax_size(fig, ax): def plot_panel(n, fig, proj, var, clevels, cmap, title, parameters, stats=None): - # var_min = float(var.min()) # var_max = float(var.max()) # var_mean = cdutil.averager(var, axis='xy', weights='generate') @@ -167,7 +166,6 @@ def plot_panel(n, fig, proj, var, clevels, cmap, title, parameters, stats=None): def plot(reference, test, diff, metrics_dict, parameter): - # Create figure, projection fig = plt.figure(figsize=parameter.figsize, dpi=parameter.dpi) # proj = ccrs.PlateCarree(central_longitude=180) @@ -255,7 +253,7 @@ def plot(reference, test, diff, metrics_dict, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/polar_plot.py b/e3sm_diags/plot/cartopy/polar_plot.py index b7439e184..7ec12ad80 100644 --- a/e3sm_diags/plot/cartopy/polar_plot.py +++ b/e3sm_diags/plot/cartopy/polar_plot.py @@ -47,7 +47,6 @@ def get_ax_size(fig, ax): def plot_panel(n, fig, proj, pole, var, clevels, cmap, title, parameters, stats=None): - var = add_cyclic(var) lon = var.getLongitude() lat = var.getLatitude() @@ -158,7 +157,6 @@ def plot_panel(n, fig, proj, pole, var, clevels, cmap, title, parameters, stats= def plot(reference, test, diff, metrics_dict, parameter): - # Create figure, projection fig = plt.figure(figsize=parameter.figsize, dpi=parameter.dpi) @@ -262,7 +260,7 @@ def plot(reference, test, diff, metrics_dict, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/qbo_plot.py b/e3sm_diags/plot/cartopy/qbo_plot.py index e7a591d49..773a21a17 100644 --- a/e3sm_diags/plot/cartopy/qbo_plot.py +++ b/e3sm_diags/plot/cartopy/qbo_plot.py @@ -231,7 +231,7 @@ def plot(parameter, test, ref): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot subplot_suffix = (".%i." % i) + f diff --git a/e3sm_diags/plot/cartopy/streamflow_plot.py b/e3sm_diags/plot/cartopy/streamflow_plot.py index 2c0a18649..a1aad2240 100644 --- a/e3sm_diags/plot/cartopy/streamflow_plot.py +++ b/e3sm_diags/plot/cartopy/streamflow_plot.py @@ -341,7 +341,7 @@ def plot_seasonality_map(export, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list((subpage * page).flatten()) + subpage = list((subpage * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot subplot_suffix = ".%i." % i + f @@ -623,7 +623,7 @@ def plot_annual_map(export, bias, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list((subpage * page).flatten()) + subpage = list((subpage * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot subplot_suffix = ".%i." % i + f diff --git a/e3sm_diags/plot/cartopy/taylor_diagram.py b/e3sm_diags/plot/cartopy/taylor_diagram.py index 6feee0b2b..3a87658b7 100644 --- a/e3sm_diags/plot/cartopy/taylor_diagram.py +++ b/e3sm_diags/plot/cartopy/taylor_diagram.py @@ -35,7 +35,7 @@ def __init__(self, refstd, fig=None, rect=111, label="_"): tr = PolarAxes.PolarTransform() # Correlation labels - rlocs = np.concatenate((np.arange(10) / 10.0, [0.95, 0.99])) + rlocs = np.concatenate((np.arange(10) / 10.0, [0.95, 0.99])) # type: ignore tlocs = np.arccos(rlocs) # Conversion to polar angles gl1 = GF.FixedLocator(tlocs) # Positions gl2_num = np.linspace(0, 1.5, 7) diff --git a/e3sm_diags/plot/cartopy/tc_analysis_plot.py b/e3sm_diags/plot/cartopy/tc_analysis_plot.py index a9d90a08a..2b3a6b357 100644 --- a/e3sm_diags/plot/cartopy/tc_analysis_plot.py +++ b/e3sm_diags/plot/cartopy/tc_analysis_plot.py @@ -63,7 +63,6 @@ def get_ax_size(fig, ax): def plot_panel(n, fig, proj, var, var_num_years, region, title): - ax = fig.add_axes(panel[n], projection=proj) ax.set_extent(plot_info[region][0], ccrs.PlateCarree()) diff --git a/e3sm_diags/plot/cartopy/zonal_mean_2d_plot.py b/e3sm_diags/plot/cartopy/zonal_mean_2d_plot.py index 785caf15d..83a9b48a8 100644 --- a/e3sm_diags/plot/cartopy/zonal_mean_2d_plot.py +++ b/e3sm_diags/plot/cartopy/zonal_mean_2d_plot.py @@ -46,7 +46,6 @@ def get_ax_size(fig, ax): def plot_panel(n, fig, proj, var, clevels, cmap, title, parameters, stats=None): - # var_min = float(var.min()) # var_max = float(var.max()) # var_mean = cdutil.averager(var, axis='xy', weights='generate') @@ -187,7 +186,6 @@ def plot_panel(n, fig, proj, var, clevels, cmap, title, parameters, stats=None): def plot(reference, test, diff, metrics_dict, parameter): - # Create figure, projection fig = plt.figure(figsize=parameter.figsize, dpi=parameter.dpi) # proj = ccrs.PlateCarree(central_longitude=180) @@ -277,7 +275,7 @@ def plot(reference, test, diff, metrics_dict, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/plot/cartopy/zonal_mean_2d_stratosphere_plot.py b/e3sm_diags/plot/cartopy/zonal_mean_2d_stratosphere_plot.py index b600cb021..e9bab3713 100644 --- a/e3sm_diags/plot/cartopy/zonal_mean_2d_stratosphere_plot.py +++ b/e3sm_diags/plot/cartopy/zonal_mean_2d_stratosphere_plot.py @@ -4,5 +4,4 @@ def plot(reference, test, diff, metrics_dict, parameter): - return base_plot(reference, test, diff, metrics_dict, parameter) diff --git a/e3sm_diags/plot/cartopy/zonal_mean_xy_plot.py b/e3sm_diags/plot/cartopy/zonal_mean_xy_plot.py index 1bff35dd7..938faab48 100644 --- a/e3sm_diags/plot/cartopy/zonal_mean_xy_plot.py +++ b/e3sm_diags/plot/cartopy/zonal_mean_xy_plot.py @@ -37,7 +37,6 @@ def get_ax_size(fig, ax): def plot(reference, test, diff, metrics_dict, parameter): - # Create figure fig = plt.figure(figsize=parameter.figsize, dpi=parameter.dpi) @@ -121,7 +120,7 @@ def plot(reference, test, diff, metrics_dict, parameter): subpage = np.array(p).reshape(2, 2) subpage[1, :] = subpage[0, :] + subpage[1, :] subpage = subpage + np.array(border).reshape(2, 2) - subpage = list(((subpage) * page).flatten()) + subpage = list(((subpage) * page).flatten()) # type: ignore extent = matplotlib.transforms.Bbox.from_extents(*subpage) # Save subplot fname = fnm + ".%i." % (i) + f diff --git a/e3sm_diags/run.py b/e3sm_diags/run.py index d02eb40c0..5589795d6 100644 --- a/e3sm_diags/run.py +++ b/e3sm_diags/run.py @@ -1,9 +1,12 @@ import copy +from itertools import chain +from typing import List, Union +import e3sm_diags # noqa: F401 from e3sm_diags.e3sm_diags_driver import get_default_diags_path, main from e3sm_diags.logger import custom_logger, move_log_to_prov_dir from e3sm_diags.parameter import SET_TO_PARAMETERS -from e3sm_diags.parameter.core_parameter import CoreParameter +from e3sm_diags.parameter.core_parameter import DEFAULT_SETS, CoreParameter from e3sm_diags.parser.core_parser import CoreParser logger = custom_logger(__name__) @@ -17,79 +20,297 @@ class Run: """ def __init__(self): - self.sets_to_run = CoreParameter().sets self.parser = CoreParser() - def run_diags(self, parameters): + # The list of sets to run based on diagnostic parameters. + self.sets_to_run = [] + + # The path to the user-specified `.cfg` file using `-d/--diags` or + # the default diagnostics `.cfg` file. + self.cfg_path = None + + @property + def is_cfg_file_arg_set(self): + """A property to check if `-d/--diags` was set to a `.cfg` filepath. + + Returns + ------- + bool + True if list contains more than one path, else False. """ - Based on sets_to_run, run the diags with the list of parameters. + args = self.parser.view_args() + self.cfg_path = args.other_parameters + + is_set = len(self.cfg_path) > 0 + + return is_set + + def run_diags( + self, parameters: List[CoreParameter], use_cfg: bool = True + ) -> Union[List[CoreParameter], None]: + """Run a set of diagnostics with a list of parameters. + + Parameters + ---------- + parameters : List[CoreParameter] + A list of parameters defined through the Python API. + use_cfg : bool, optional + Also run diagnostics using a `.cfg` file, by default True. + + * If True, run all sets using the list of parameters passed in + this function and parameters defined in a .cfg file (if + defined), or use the .cfg file(s) for default diagnostics. This + is the default option. + * If False, only the parameters passed via ``parameters`` will be + run. The sets to run are based on the sets defined by the + parameters. This makes it easy to debug a few sets instead of + all of the debug sets too. + + Returns + ------- + Union[List[CoreParameter], None] + A list of parameter objects with their results (if successful). + + Raises + ------ + RuntimeError + If a diagnostic run using a parameter fails for any reason. """ - final_params = self.get_final_parameters(parameters) - if not final_params: - msg = "No parameters we able to be extracted." - msg += " Please check the parameters you defined." - raise RuntimeError(msg) + params = self.get_run_parameters(parameters, use_cfg) + params_results = None + + if params is None or len(params) == 0: + raise RuntimeError( + "No parameters we able to be extracted. Please " + "check the parameters you defined." + ) try: - main(final_params) + params_results = main(params) except Exception: logger.exception("Error traceback:", exc_info=True) - move_log_to_prov_dir(final_params[0].results_dir) - def get_final_parameters(self, parameters): - """ - Based on sets_to_run and the list of parameters, - get the final list of paremeters to run the diags on. - """ - if not parameters or not isinstance(parameters, list): - msg = "You must pass in a list of parameter objects." - raise RuntimeError(msg) + # param_results might be None because the run(s) failed, so move + # the log using the `params[0].results_dir` instead. + move_log_to_prov_dir(params[0].results_dir) + + return params_results + + def get_run_parameters( + self, parameters: List[CoreParameter], use_cfg: bool = True + ) -> List[CoreParameter]: + """Get the run parameters. - # For each of the passed in parameters, we can only have one of - # each type. - types = set([p.__class__ for p in parameters]) - if len(types) != len(parameters): - msg = "You passed in two or more parameters of the same type." - raise RuntimeError(msg) + Parameters + ---------- + parameters : List[CoreParameter] + A list of parameters defined through the Python API. + use_cfg : bool, optional + Use parameters defined in a .cfg file too, by default True. + Returns + ------- + List[CoreParameter] + A list of run parameters. + """ + self._validate_parameters(parameters) + + # FIXME: This line produces some unintended side-effects. For example, + # let's say we have two objects: 1. CoreParameter, 2. ZonalMean2DParameter. + # If object 1 has `plevs=[200]`, this value will get copied to object 2. + # Object 2 has a check to make sure plevs has more than 1 value. This + # breaks the diagnostic run as a result. The workaround is to loop + # over `run_diags()` function and run one parameter at a time. self._add_parent_attrs_to_children(parameters) - final_params = [] + if use_cfg: + run_params = self._get_parameters_with_api_and_cfg(parameters) + else: + run_params = self._get_parameters_with_api_only(parameters) + + self.parser.check_values_of_params(run_params) + + return run_params + + def _validate_parameters(self, parameters: List[CoreParameter]): + if parameters is None or not isinstance(parameters, list): + raise RuntimeError("You must pass in a list of parameter objects.") + + param_types_list = [ + p.__class__ for p in parameters if p.__class__ != CoreParameter + ] + param_types_set = set(param_types_list) + + if len(param_types_set) != len(param_types_list): + raise RuntimeError( + "You passed in two or more non-CoreParameter objects of the same type." + ) + + def _get_parameters_with_api_and_cfg( + self, parameters: List[CoreParameter] + ) -> List[CoreParameter]: + """Get the run parameters using the Python API and .cfg file. + + Parameters + ---------- + parameters : List[CoreParameter] + A list of parameter objects defined by the Python API. + + Returns + ------- + List[CoreParameter] + A list of run parameters, including ones defined in a .cfg file. + Any non-CoreParameter objects will be replaced by a sub-class + based on the set (``SETS_TO_PARAMETERS``). + """ + run_params = [] + + if len(self.sets_to_run) == 0: + self.sets_to_run = DEFAULT_SETS for set_name in self.sets_to_run: - other_params = self._get_other_diags(parameters[0].run_type) + if self.is_cfg_file_arg_set: + cfg_params = self._get_custom_params_from_cfg_file() + else: + run_type = parameters[0].run_type + cfg_params = self._get_default_params_from_cfg_file(run_type) - # For each of the set_names, get the corresponding parameter. param = self._get_instance_of_param_class( SET_TO_PARAMETERS[set_name], parameters ) - # Since each parameter will have lots of default values, we want to remove them. - # Otherwise when calling get_parameters(), these default values - # will take precedence over values defined in other_params. + # Since each parameter will have lots of default values, we want to + # remove them. Otherwise when calling get_parameters(), these + # default values will take precedence over values defined in + # other_params. self._remove_attrs_with_default_values(param) param.sets = [set_name] params = self.parser.get_parameters( orig_parameters=param, - other_parameters=other_params, + other_parameters=cfg_params, cmd_default_vars=False, argparse_vals_only=False, ) - # Makes sure that any parameters that are selectors - # will be in param. + # Makes sure that any parameters that are selectors will be in param. self._add_attrs_with_default_values(param) + # The select() call in get_parameters() was made for the original - # command-line way of using CDP. - # We just call it manually with the parameter object param. + # command-line way of using CDP. We just call it manually with the + # parameter object param. params = self.parser.select(param, params) - final_params.extend(params) + run_params.extend(params) + + return run_params + + def _get_custom_params_from_cfg_file(self) -> List[CoreParameter]: + """Get parameters using the cfg file set by `-d`/`--diags`. + + Returns + ------- + List[CoreParameter] + A list of parameter objects. + """ + params = self.parser.get_cfg_parameters(argparse_vals_only=False) + + params_final = self._convert_params_to_subclass(params) + + return params_final + + def _get_default_params_from_cfg_file(self, run_type: str) -> List[CoreParameter]: + """Get parameters using the default diagnostic .cfg file(s). + + Parameters + ---------- + run_type : str + The run type used to check for which .cfg file(s) to reference. + + Returns + ------- + List[CoreParameter] + A list of parameter objects. + """ + # Get the paths to the default diags .cfg file(s). + paths = [] + for set_name in self.sets_to_run: + path = get_default_diags_path(set_name, run_type, False) + paths.append(path) + + self.cfg_path = paths + + # Convert the .cfg file(s) to parameter objects. + params = self.parser.get_cfg_parameters( + files_to_open=paths, argparse_vals_only=False + ) - self.parser.check_values_of_params(final_params) + # Update parameter objects using subclass with default values. + params_final = self._convert_params_to_subclass(params) - return final_params + return params_final + + def _convert_params_to_subclass( + self, parameters: List[CoreParameter] + ) -> List[CoreParameter]: + new_params: List[CoreParameter] = [] + + # For each of the params, add in the default values using the parameter + # classes in SET_TO_PARAMETERS. + for param in parameters: + set_key = param.sets[0] + new_param = SET_TO_PARAMETERS[set_key]() + param + + new_params.append(new_param) + + return new_params + + def _get_parameters_with_api_only( + self, parameters: List[CoreParameter] + ) -> List[CoreParameter]: + """Get the parameters defined through the Python API only. + + This method replaces CoreParameter objects with the related sub-class + for the specified set. + + Parameters + ---------- + parameters : List[CoreParameter] + A list of parameter objects. + + Returns + ------- + List[CoreParameter] + A list of parameter objects defined through the Python API only. + Any non-CoreParameter objects will be replaced by a sub-class based + on the set (``SETS_TO_PARAMETERS``). + """ + params = [] + + if len(self.sets_to_run) == 0: + sets_to_run = [param.sets for param in parameters] + self.sets_to_run = list(chain.from_iterable(sets_to_run)) + + for set_name in self.sets_to_run: + # For each of the set_names, get the corresponding parameter. + param = self._get_instance_of_param_class( + SET_TO_PARAMETERS[set_name], parameters + ) + + # Since each parameter will have lots of default values, we want to remove them. + # Otherwise when calling get_parameters(), these default values + # will take precedence over values defined in other_params. + self._remove_attrs_with_default_values(param) + param.sets = [set_name] + + # Makes sure that any parameters that are selectors will be in param. + self._add_attrs_with_default_values(param) + + params.append(param) + + self.parser.check_values_of_params(params) + + return params def _add_parent_attrs_to_children(self, parameters): """ @@ -226,37 +447,14 @@ def _get_instance_of_param_class(self, cls, parameters): for cls_type in class_types: for p in parameters: - if type(p) == cls_type: + # NOTE: This conditional is used instead of + # `isinstance(p, cls_type)` because we want to check for exact + # type matching and exclude sub-class matching. + if type(p) is cls_type: return p msg = "There's weren't any class of types {} in your parameters." raise RuntimeError(msg.format(class_types)) - def _get_other_diags(self, run_type): - """ - If the user has ran the script with a -d, get the diags for that. - If not, load the default diags based on sets_to_run and run_type. - """ - args = self.parser.view_args() - - # If the user has passed in args with -d. - if args.other_parameters: - params = self.parser.get_cfg_parameters(argparse_vals_only=False) - else: - default_diags_paths = [ - get_default_diags_path(set_name, run_type, False) - for set_name in self.sets_to_run - ] - params = self.parser.get_cfg_parameters( - files_to_open=default_diags_paths, argparse_vals_only=False - ) - - # For each of the params, add in the default values - # using the parameter classes in SET_TO_PARAMETERS. - for i in range(len(params)): - params[i] = SET_TO_PARAMETERS[params[i].sets[0]]() + params[i] - - return params - runner = Run() diff --git a/e3sm_diags/viewer/aerosol_budget_viewer.py b/e3sm_diags/viewer/aerosol_budget_viewer.py index e1f3f900a..271c83093 100644 --- a/e3sm_diags/viewer/aerosol_budget_viewer.py +++ b/e3sm_diags/viewer/aerosol_budget_viewer.py @@ -1,7 +1,7 @@ import os import shutil -from cdp.cdp_viewer import OutputViewer +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import seasons_used from .lat_lon_viewer import _cvs_to_html diff --git a/e3sm_diags/viewer/annual_cycle_zonal_mean_viewer.py b/e3sm_diags/viewer/annual_cycle_zonal_mean_viewer.py index fcc76d899..83e7d8753 100644 --- a/e3sm_diags/viewer/annual_cycle_zonal_mean_viewer.py +++ b/e3sm_diags/viewer/annual_cycle_zonal_mean_viewer.py @@ -1,6 +1,6 @@ import os -from cdp.cdp_viewer import OutputViewer +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import create_metadata from .utils import add_header, h1_to_h3 diff --git a/e3sm_diags/viewer/area_mean_time_series_viewer.py b/e3sm_diags/viewer/area_mean_time_series_viewer.py index bae423521..a9cb5d45e 100644 --- a/e3sm_diags/viewer/area_mean_time_series_viewer.py +++ b/e3sm_diags/viewer/area_mean_time_series_viewer.py @@ -1,6 +1,6 @@ import os -from cdp.cdp_viewer import OutputViewer +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import create_metadata from .utils import add_header, h1_to_h3 diff --git a/e3sm_diags/viewer/arm_diags_viewer.py b/e3sm_diags/viewer/arm_diags_viewer.py index 59ea0e75c..1a1ed82c2 100644 --- a/e3sm_diags/viewer/arm_diags_viewer.py +++ b/e3sm_diags/viewer/arm_diags_viewer.py @@ -1,6 +1,6 @@ import os -from cdp.cdp_viewer import OutputViewer +from e3sm_diags.viewer.core_viewer import OutputViewer from .utils import add_header, h1_to_h3 diff --git a/e3sm_diags/viewer/core_viewer.py b/e3sm_diags/viewer/core_viewer.py new file mode 100644 index 000000000..1e29930b7 --- /dev/null +++ b/e3sm_diags/viewer/core_viewer.py @@ -0,0 +1,133 @@ +import os +import stat + +from output_viewer.build import build_page, build_viewer +from output_viewer.index import ( + OutputFile, + OutputGroup, + OutputIndex, + OutputPage, + OutputRow, +) +from output_viewer.utils import rechmod + + +class OutputViewer(object): + def __init__(self, path=".", index_name="Results"): + self.path = os.path.abspath(path) + self.index = OutputIndex(index_name) + self.cache = {} # dict of { OutputPage: { OutputGroup: [OutputRow] } } + self.page = None + self.group = None + self.row = None + + def add_page(self, page_title, *args, **kwargs): + """Add a page to the viewer's index""" + self.page = OutputPage(page_title, *args, **kwargs) + self.cache[self.page] = {} + self.index.addPage(self.page) + + def set_page(self, page_title): + """Sets the page with the title name as the current page""" + for output_page in self.cache: + if page_title == output_page.title: + self.page = output_page + return + raise RuntimeError("There is no page titled: %s" % page_title) + + def add_group(self, group_name): + """Add a group to the current page""" + if self.page is None: + raise RuntimeError("You must first insert a page with add_page()") + self.group = OutputGroup(group_name) + if self.group not in self.cache[self.page]: + self.cache[self.page][self.group] = [] # group doesn't have any rows yet + self.page.addGroup(self.group) + + def set_group(self, group_name): + """Sets the group with the title name as the current group""" + for output_group in self.cache[self.page]: + if group_name == output_group.title: + self.group = output_group + return + raise RuntimeError("There is no group titled: %s" % group_name) + + def add_row(self, row_name): + """Add a row with the title name to the current group""" + if self.group is None: + raise RuntimeError("You must first insert a group with add_group()") + self.row = OutputRow(row_name, []) + if self.row not in self.cache[self.page][self.group]: + self.cache[self.page][self.group].append(self.row) + self.page.addRow(self.row, len(self.page.groups) - 1) # type: ignore + + def set_row(self, row_name): + """Sets the row with the title name as the current row""" + for output_row in self.cache[self.page][self.group]: + if row_name == output_row.title: + self.row = output_row + return + raise RuntimeError("There is no row titled: %s" % row_name) + + def add_cols(self, cols): + """Add multiple string cols to the current row""" + self.row.columns.append(cols) # type: ignore + + def add_col(self, col, is_file=False, **kwargs): + """Add a single col to the current row. Set is_file to True if the col is a file path.""" + if is_file: + self.row.columns.append(OutputFile(col, **kwargs)) # type: ignore + else: + self.row.columns.append(col) # type: ignore + + def generate_page(self): + """ + Generate and return the location of the current HTML page. + """ + self.index.toJSON(os.path.join(self.path, "index.json")) + + default_mask = stat.S_IMODE(os.stat(self.path).st_mode) + rechmod(self.path, default_mask) + + if os.access(self.path, os.W_OK): + default_mask = stat.S_IMODE( + os.stat(self.path).st_mode + ) # mode of files to be included + url = build_page( + self.page, + os.path.join(self.path, "index.json"), + default_mask=default_mask, + ) + return url + + raise RuntimeError("Error geneating the page.") + + def generate_viewer(self, prompt_user=True): + """Generate the webpage and ask the user if they want to see it.""" + self.index.toJSON(os.path.join(self.path, "index.json")) + + default_mask = stat.S_IMODE(os.stat(self.path).st_mode) + rechmod(self.path, default_mask) + + if os.access(self.path, os.W_OK): + default_mask = stat.S_IMODE( + os.stat(self.path).st_mode + ) # mode of files to be included + build_viewer( + os.path.join(self.path, "index.json"), + diag_name="My Diagnostics", + default_mask=default_mask, + ) + + if os.path.exists(os.path.join(self.path, "index.html")): + if prompt_user: + user_prompt = "Would you like to open in a browser? y/[n]: " + should_open = input(user_prompt) + if should_open and should_open.lower()[0] == "y": + import webbrowser + + index_path = os.path.join(self.path, "index.html") + url = "file://{path}".format(path=index_path) + webbrowser.open(url) + else: + raise RuntimeError("Failed to generate the viewer.") diff --git a/e3sm_diags/viewer/default_viewer.py b/e3sm_diags/viewer/default_viewer.py index 6a01d4d09..069746ef8 100644 --- a/e3sm_diags/viewer/default_viewer.py +++ b/e3sm_diags/viewer/default_viewer.py @@ -10,10 +10,10 @@ from typing import Dict import numpy -from cdp.cdp_viewer import OutputViewer from e3sm_diags.logger import custom_logger from e3sm_diags.parser import SET_TO_PARSER +from e3sm_diags.viewer.core_viewer import OutputViewer from . import lat_lon_viewer, utils diff --git a/e3sm_diags/viewer/enso_diags_viewer.py b/e3sm_diags/viewer/enso_diags_viewer.py index b8e941dee..72db8c731 100644 --- a/e3sm_diags/viewer/enso_diags_viewer.py +++ b/e3sm_diags/viewer/enso_diags_viewer.py @@ -1,9 +1,8 @@ import os from typing import Dict, List -from cdp.cdp_viewer import OutputViewer - from e3sm_diags.logger import custom_logger +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import create_metadata from .utils import add_header, h1_to_h3 diff --git a/e3sm_diags/viewer/lat_lon_viewer.py b/e3sm_diags/viewer/lat_lon_viewer.py index 28ef759a4..678c68748 100644 --- a/e3sm_diags/viewer/lat_lon_viewer.py +++ b/e3sm_diags/viewer/lat_lon_viewer.py @@ -253,7 +253,6 @@ def _create_lat_lon_table_index( # --- Function to read CMIP6 model metrics --- def read_cmip6_metrics_from_csv(path, variables, seasons): - models = [] with open(path, "r") as fin: @@ -292,7 +291,6 @@ def read_cmip6_metrics_from_csv(path, variables, seasons): # --- Function to read E3SM Diags metrics --- def read_e3sm_diags_metrics(path, variables, seasons, names=None): - # List of available models models = [] paths = [] @@ -674,7 +672,6 @@ def _create_csv_from_dict_taylor_diag( # Add samples for baseline simulation. if run_type == "model_vs_obs": - # Read the control run data. # Example base line csv file name: JJA_metrics_table_taylor_diag_historical_1985-2014_E3SMv1.csv for ibase, base_line_csv_path in enumerate(base_line_csv_paths): diff --git a/e3sm_diags/viewer/mean_2d_viewer.py b/e3sm_diags/viewer/mean_2d_viewer.py index 2ba6ba0e5..e5e717834 100644 --- a/e3sm_diags/viewer/mean_2d_viewer.py +++ b/e3sm_diags/viewer/mean_2d_viewer.py @@ -1,6 +1,6 @@ import os -from cdp.cdp_viewer import OutputViewer +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import SEASONS, create_metadata, seasons_used from .utils import add_header, h1_to_h3 @@ -39,7 +39,6 @@ def create_viewer(root_dir, parameters): for var in param.variables: for season in param.seasons: for region in param.regions: - try: viewer.set_group(param.case_id) except RuntimeError: diff --git a/e3sm_diags/viewer/mp_partition_viewer.py b/e3sm_diags/viewer/mp_partition_viewer.py index de67b4466..4ceeba0e1 100644 --- a/e3sm_diags/viewer/mp_partition_viewer.py +++ b/e3sm_diags/viewer/mp_partition_viewer.py @@ -1,8 +1,7 @@ import os -from cdp.cdp_viewer import OutputViewer - from e3sm_diags.logger import custom_logger +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import create_metadata from .utils import add_header, h1_to_h3 diff --git a/e3sm_diags/viewer/qbo_viewer.py b/e3sm_diags/viewer/qbo_viewer.py index ee33ddbb8..7e5a9b4a8 100644 --- a/e3sm_diags/viewer/qbo_viewer.py +++ b/e3sm_diags/viewer/qbo_viewer.py @@ -1,8 +1,7 @@ import os -from cdp.cdp_viewer import OutputViewer - from e3sm_diags.logger import custom_logger +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import create_metadata from .utils import add_header, h1_to_h3 diff --git a/e3sm_diags/viewer/streamflow_viewer.py b/e3sm_diags/viewer/streamflow_viewer.py index 2e9801cdf..525b50ac0 100644 --- a/e3sm_diags/viewer/streamflow_viewer.py +++ b/e3sm_diags/viewer/streamflow_viewer.py @@ -1,6 +1,6 @@ import os -from cdp.cdp_viewer import OutputViewer +from e3sm_diags.viewer.core_viewer import OutputViewer from .default_viewer import create_metadata from .utils import add_header, h1_to_h3 diff --git a/e3sm_diags/viewer/tc_analysis_viewer.py b/e3sm_diags/viewer/tc_analysis_viewer.py index 97f199de2..2e3ec7269 100644 --- a/e3sm_diags/viewer/tc_analysis_viewer.py +++ b/e3sm_diags/viewer/tc_analysis_viewer.py @@ -1,6 +1,6 @@ import os -from cdp.cdp_viewer import OutputViewer +from e3sm_diags.viewer.core_viewer import OutputViewer from .utils import add_header, h1_to_h3 diff --git a/pyproject.toml b/pyproject.toml index 6835efe21..09d466ab5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,7 @@ -# This config file is required for Black. -# Unforunately, black does not support setup.cfg (refer to link). -# https://github.com/psf/black/issues/683#issuecomment-490236406 - [tool.black] +# Docs: https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html line-length = 88 -target-version = ['py36'] +target-version = ["py39", "py310"] include = '\.pyi?$' exclude = ''' /( @@ -25,3 +22,34 @@ exclude = ''' | analysis_data_preprocess )/ ''' + +[tool.isort] +# Docs: https://pycqa.github.io/isort/docs/configuration/options.html#example-pyprojecttoml_4 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 88 +skip = "e3sm_diags/e3sm_diags_driver.py" + +[tool.pytest.ini_options] +# Docs: https://docs.pytest.org/en/7.2.x/reference/customize.html#configuration +junit_family = "xunit2" +addopts = "--cov=e3sm_diags --cov-report term --cov-report html:tests_coverage_reports/htmlcov --cov-report xml:tests_coverage_reports/coverage.xml -s" +python_files = ["tests.py", "test_*.py"] +# Only run the unit tests because integration tests take a long time. +# Integration tests can be executed manually with `test.sh` or `pytest tests/integration`. +testpaths = "tests/e3sm_diags" + +[tool.mypy] +# Docs: https://mypy.readthedocs.io/en/stable/config_file.html +python_version = "3.10" +check_untyped_defs = true +ignore_missing_imports = true +warn_unused_ignores = true +warn_redundant_casts = true +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = ["analysis_data_preprocess.*", "model_data_preprocess.*"] +ignore_errors = true diff --git a/setup.cfg b/setup.cfg index 036c04f66..50b647306 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,5 @@ +# This file is used for configuring flake8, which does not currently support pyproject.toml + [flake8] # https://pep8.readthedocs.io/en/latest/intro.html#error-codes ignore = @@ -26,49 +28,3 @@ exclude = venv, analysis_data_preprocess model_data_preprocess - -[isort] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -skip= - e3sm_diags/e3sm_diags_driver.py - -[pycodestyle] -max-line-length = 119 -exclude = - .tox - .git - */migrations/* - */static/CACHE/* - docs - node_modules - .idea - .mypy_cache - .pytest_cache - *__init__.py - venv - analysis_data_preprocess - model_data_preprocess - -[mypy] -python_version = 3.10 -check_untyped_defs = True -ignore_missing_imports = True -warn_unused_ignores = True -warn_redundant_casts = True -warn_unused_configs = True - -[mypy-analysis_data_preprocess.*] -ignore_errors = True - -[mypy-model_data_preprocess.*] -ignore_errors = True - -[tool:pytest] -junit_family=xunit2 -addopts = --cov=e3sm_diags --cov-report term --cov-report html:tests_coverage_reports/htmlcov --cov-report xml:tests_coverage_reports/coverage.xml -s --ignore=tests/integration -python_files = tests.py test_*.py -testpaths = tests/e3sm_diags diff --git a/setup.py b/setup.py index 9f6e86ebd..a76b7de7c 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def get_all_files_in_dir(directory, pattern): "Programming Language :: Python :: 3.10", ], name="e3sm_diags", - version="2.9.0rc3", + version="2.10.1", author="Chengzhu (Jill) Zhang, Tom Vo, Ryan Forsyth, Chris Golaz and Zeshawn Shaheen", author_email="zhang40@llnl.gov", description="E3SM Diagnostics", diff --git a/tbump.toml b/tbump.toml index bb3098713..3e4f9ff71 100644 --- a/tbump.toml +++ b/tbump.toml @@ -2,7 +2,7 @@ github_url = "https://github.com/E3SM-Project/e3sm_diags" [version] -current = "2.9.0rc3" +current = "2.10.1" # Example of a semver regexp. # Make sure this matches current_version before diff --git a/tests/e3sm_diags/test_e3sm_diags_driver.py b/tests/e3sm_diags/test_e3sm_diags_driver.py index 9aa2e2b6b..a06220a35 100644 --- a/tests/e3sm_diags/test_e3sm_diags_driver.py +++ b/tests/e3sm_diags/test_e3sm_diags_driver.py @@ -1,6 +1,6 @@ import pytest -from e3sm_diags.e3sm_diags_driver import _run_serially, _run_with_dask, run_diag +from e3sm_diags.e3sm_diags_driver import _run_serially, _run_with_dask from e3sm_diags.logger import custom_logger from e3sm_diags.parameter.core_parameter import CoreParameter @@ -8,44 +8,6 @@ class TestRunDiag: - def test_returns_parameter_with_results(self): - parameter = CoreParameter() - parameter.sets = ["lat_lon"] - - results = run_diag(parameter) - expected = [parameter] - - # NOTE: We are only testing that the function returns a list of - # parameter objects, not the results themselves. There are integration - # tests validates the results. - assert results == expected - - def test_logs_error_if_driver_module_for_set_not_found(self, caplog): - parameter = CoreParameter() - parameter.sets = ["invalid_set"] - - run_diag(parameter) - - assert ( - "ModuleNotFoundError: No module named 'e3sm_diags.driver.invalid_set_driver'" - in caplog.text - ) - - @pytest.mark.xfail - def test_logs_exception_if_driver_run_diag_function_fails(self, caplog): - # TODO: Need to implement this test by raising an exception through - # the driver's `run_diag` function - parameter = CoreParameter() - parameter.sets = ["lat_lon"] - - # Make this attribute an invalid value to test exception is thrown. - parameter.seasons = None # type: ignore - - # NOTE: Comment out temporarily to avoid polluting the test output log. - # run_diag(parameter) - - assert "TypeError: 'NoneType' object is not iterable" in caplog.text - def test_run_diag_serially_returns_parameters_with_results(self): parameter = CoreParameter() parameter.sets = ["lat_lon"] diff --git a/tests/e3sm_diags/test_parameters.py b/tests/e3sm_diags/test_parameters.py index 9b30fa3d3..83162ad8b 100644 --- a/tests/e3sm_diags/test_parameters.py +++ b/tests/e3sm_diags/test_parameters.py @@ -79,6 +79,44 @@ def test_check_values_raises_error_if_test_timeseries_input_and_no_test_start_an with pytest.raises(RuntimeError): param.check_values() + def test_returns_parameter_with_results(self): + parameter = CoreParameter() + parameter.sets = ["lat_lon"] + + results = parameter._run_diag() + expected = [parameter] + + # NOTE: We are only testing that the function returns a list of + # parameter objects, not the results themselves. There are integration + # tests validates the results. + assert results == expected + + def test_logs_error_if_driver_module_for_set_not_found(self, caplog): + parameter = CoreParameter() + parameter.sets = ["invalid_set"] + + parameter._run_diag() + + assert ( + "ModuleNotFoundError: No module named 'e3sm_diags.driver.invalid_set_driver'" + in caplog.text + ) + + @pytest.mark.xfail + def test_logs_exception_if_driver_run_diag_function_fails(self, caplog): + # TODO: Need to implement this test by raising an exception through + # the driver's `run_diag` function + parameter = CoreParameter() + parameter.sets = ["lat_lon"] + + # Make this attribute an invalid value to test exception is thrown. + parameter.seasons = None # type: ignore + + # NOTE: Comment out temporarily to avoid polluting the test output log. + # parameter._run_diag() + + assert "TypeError: 'NoneType' object is not iterable" in caplog.text + def test_ac_zonal_mean_parameter(): param = ACzonalmeanParameter() diff --git a/tests/integration/test_run.py b/tests/e3sm_diags/test_run.py similarity index 88% rename from tests/integration/test_run.py rename to tests/e3sm_diags/test_run.py index f1ea689de..3421b0aaf 100644 --- a/tests/integration/test_run.py +++ b/tests/e3sm_diags/test_run.py @@ -35,7 +35,7 @@ def setUp(self): def test_lat_lon_ann(self): self.core_param.seasons = ["ANN"] self.runner.sets_to_run = ["lat_lon"] - parameters = self.runner.get_final_parameters([self.core_param]) + parameters = self.runner.get_run_parameters([self.core_param]) for param in parameters: bad_seasons = ["DJF", "MAM", "JJA", "SON"] @@ -64,8 +64,8 @@ def test_all_sets_and_all_seasons(self): "streamflow", ] - parameters = self.runner.get_final_parameters( - [self.core_param, ts_param, enso_param, streamflow_param] + parameters = self.runner.get_run_parameters( + [self.core_param, ts_param, enso_param, streamflow_param], use_cfg=True ) # Counts the number of each set and each seasons to run the diags on. set_counter, season_counter = ( @@ -88,23 +88,25 @@ def test_all_sets_and_all_seasons(self): # So, reduce the ANN count by the number of times these appear season_counter["ANN"] -= set_counter["enso_diags"] season_counter["ANN"] -= set_counter["streamflow"] - if not all(season_counter["ANN"] == count for count in season_counter.values()): - self.fail( - "In .cfg files, at least one season does not match the count for ANN: {}".format( - season_counter + + for season, count in season_counter.items(): + if count != season_counter["ANN"]: + self.fail( + "In .cfg files, at least one season does not match the count for ANN: {}".format( + season_counter + ) ) - ) def test_zonal_mean_2d(self): # Running zonal_mean_2d with the core param only. self.runner.sets_to_run = ["zonal_mean_2d"] - core_only_results = self.runner.get_final_parameters([self.core_param]) + core_only_results = self.runner.get_run_parameters([self.core_param]) # Running zonal_mean_2d with a set-specific param. # We pass in both the core and this parameter. zonal_mean_2d_param = ZonalMean2dParameter() zonal_mean_2d_param.plevs = [10.0, 20.0, 30.0] - both_results = self.runner.get_final_parameters( + both_results = self.runner.get_run_parameters( [self.core_param, zonal_mean_2d_param] ) @@ -121,7 +123,7 @@ def test_zonal_mean_2d(self): another_zonal_mean_2d_param.reference_data_path = "/something" another_zonal_mean_2d_param.test_data_path = "/something/else" another_zonal_mean_2d_param.results_dir = "/something/else/too" - zm_2d_only_results = self.runner.get_final_parameters( + zm_2d_only_results = self.runner.get_run_parameters( [another_zonal_mean_2d_param] ) diff --git a/tests/integration/all_sets.cfg b/tests/integration/all_sets.cfg index 059155714..06277d1e0 100644 --- a/tests/integration/all_sets.cfg +++ b/tests/integration/all_sets.cfg @@ -17,7 +17,6 @@ test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" results_dir = "tests/integration/all_sets_results_test" debug = True - [zonal_mean_2d] sets = ["zonal_mean_2d"] case_id = "ERA-Interim" @@ -38,7 +37,6 @@ test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" results_dir = "tests/integration/all_sets_results_test" debug = True - [meridional_mean_2d] sets = ["meridional_mean_2d"] case_id = "ERA-Interim" @@ -59,7 +57,6 @@ test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" results_dir = "tests/integration/all_sets_results_test" debug = True - [lat_lon] sets = ["lat_lon"] case_id = "ERA-Interim" @@ -82,8 +79,6 @@ results_dir = "tests/integration/all_sets_results_test" debug = True regions=["CONUS_RRM","global"] - - [polar] sets = ["polar"] case_id = "ERA-Interim" @@ -106,7 +101,6 @@ test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" results_dir = "tests/integration/all_sets_results_test" debug = True - [cosp_histogram] sets = ["cosp_histogram"] case_id = "MISR-COSP" @@ -127,7 +121,6 @@ test_file = "CLD_MISR_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" results_dir = "tests/integration/all_sets_results_test" debug = True - [area_mean_time_series] sets = ["area_mean_time_series"] variables = ["TREFHT"] @@ -158,7 +151,6 @@ reference_data_path = 'tests/integration/integration_test_data' ref_file = 'TREFHT_201201_201312.nc' test_name = "system tests" variables = ["TREFHT"] -print_statements = True [#] sets = ["enso_diags"] @@ -177,7 +169,6 @@ reference_data_path = 'tests/integration/integration_test_data' ref_file = 'TREFHT_201201_201312.nc' test_name = "system tests" variables = ["TREFHT"] -print_statements = True [#] sets = ["enso_diags"] @@ -197,7 +188,6 @@ reference_data_path = 'tests/integration/integration_test_data' ref_file = 'TREFHT_201201_201312.nc' test_name = "system tests" variables = ["TREFHT"] -print_statements = True [#] sets = ["enso_diags"] @@ -214,7 +204,6 @@ reference_data_path = 'tests/integration/integration_test_data' ref_file = 'TREFHT_201201_201312.nc' test_name = "system tests" variables = ["TREFHT"] -print_statements = True [#] sets = ["enso_diags"] @@ -232,7 +221,6 @@ reference_data_path = 'tests/integration/integration_test_data' ref_file = 'TREFHT_201201_201312.nc' test_name = "system tests" variables = ["TREFHT"] -print_statements = True [#] sets = ["enso_diags"] @@ -251,7 +239,6 @@ reference_data_path = 'tests/integration/integration_test_data' ref_file = 'TREFHT_201201_201312.nc' test_name = "system tests" variables = ["TREFHT"] -print_statements = True [qbo] sets = ["qbo"] @@ -283,7 +270,6 @@ test_start_yr = '1959' test_end_yr = '1961' results_dir = "tests/integration/all_sets_results_test" test_name = "system tests" -print_statements = True [diurnal_cycle] sets = ["diurnal_cycle"] @@ -304,7 +290,6 @@ test_file = "20180215.DECKv1b_H1.ne30_oEC.edison.cam.h4_JJA_200006_200908_climo. results_dir = "tests/integration/all_sets_results_test" debug = True - [arm_diags1] sets = ["arm_diags"] diags_set = "annual_cycle" diff --git a/tests/integration/all_sets.py b/tests/integration/all_sets.py deleted file mode 100644 index 943de7f89..000000000 --- a/tests/integration/all_sets.py +++ /dev/null @@ -1,30 +0,0 @@ -# Running the software with the API: -# python all_sets.py -d all_sets.py -from e3sm_diags.parameter.area_mean_time_series_parameter import ( - AreaMeanTimeSeriesParameter, -) -from e3sm_diags.parameter.core_parameter import CoreParameter -from e3sm_diags.parameter.enso_diags_parameter import EnsoDiagsParameter -from e3sm_diags.parameter.meridional_mean_2d_parameter import MeridionalMean2dParameter -from e3sm_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter -from e3sm_diags.run import Run - -run_object = Run() -param = CoreParameter() -ts_param = AreaMeanTimeSeriesParameter() - -m2d_param = MeridionalMean2dParameter() -m2d_param.plevs = [ - 200.0, - 500.0, -] -z2d_param = ZonalMean2dParameter() -z2d_param.plevs = [ - 200.0, - 300.0, -] - -enso_param = EnsoDiagsParameter() -enso_param.test_name = "e3sm_v1" - -run_object.run_diags([param, ts_param, m2d_param, z2d_param, enso_param]) diff --git a/tests/integration/all_sets_modified.cfg b/tests/integration/all_sets_modified.cfg deleted file mode 100644 index b4fc5e472..000000000 --- a/tests/integration/all_sets_modified.cfg +++ /dev/null @@ -1,187 +0,0 @@ -[zonal_mean_xy] -sets = ["zonal_mean_xy"] -case_id = "ERA-Interim" -variables = ["T"] -seasons = ["ANN"] -plevs = [200.0] - -test_name = "system tests" -short_test_name = "short_system tests" -ref_name = "ERA-Interim" -reference_name = "ERA-Interim Reanalysis 1979-2015" -reference_data_path = "tests/integration" -ref_file = "ta_ERA-Interim_ANN_198001_201401_climo.nc" -test_data_path = "tests/integration" -test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" - -backend = "mpl" -results_dir = "tests/integration/all_sets_results" -debug = True - - -[zonal_mean_2d] -sets = ["zonal_mean_2d"] -case_id = "ERA-Interim" -variables = ["T"] -seasons = ["ANN"] -contour_levels = [180,185,190,200,210,220,230,240,250,260,270,280,290,295,300] -diff_levels = [-7,-6,-5,-4,-3,-2,-1,1,2,3,4,5,6,7] - -test_name = "system tests" -short_test_name = "short_system tests" -ref_name = "ERA-Interim" -reference_name = "ERA-Interim Reanalysis 1989-2005" -reference_data_path = "tests/integration" -ref_file = "ta_ERA-Interim_ANN_198001_201401_climo.nc" -test_data_path = "tests/integration" -test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" - -backend = "mpl" -results_dir = "tests/integration/all_sets_results" -debug = True -plot_log_plevs = False -plot_plevs = False - - -[meridional_mean_2d] -sets = ["meridional_mean_2d"] -case_id = "ERA-Interim" -variables = ["T"] -seasons = ["ANN"] -contour_levels = [180,185,190,200,210,220,230,240,250,260,270,280,290,295,300] -diff_levels = [-7,-6,-5,-4,-3,-2,-1,1,2,3,4,5,6,7] - -test_name = "system tests" -short_test_name = "short_system tests" -ref_name = "ERA-Interim" -reference_name = "ERA-Interim Reanalysis 1989-2005" -reference_data_path = "tests/integration" -ref_file = "ta_ERA-Interim_ANN_198001_201401_climo.nc" -test_data_path = "tests/integration" -test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" - -backend = "mpl" -results_dir = "tests/integration/all_sets_results" -debug = True -plot_log_plevs = False -plot_plevs = False - - -[lat_lon] -sets = ["lat_lon"] -case_id = "ERA-Interim" -variables = ["T"] -seasons = ["ANN"] -plevs = [850.0] -contour_levels = [240, 245, 250, 255, 260, 265, 270, 275, 280, 285, 290, 295] -diff_levels = [-10, -7.5, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 7.5, 10] - -test_name = "system tests" -short_test_name = "short_system tests" -ref_name = "ERA-Interim" -reference_name = "ERA-Interim Reanalysis 1979-2015" -reference_data_path = "tests/integration" -ref_file = "ta_ERA-Interim_ANN_198001_201401_climo.nc" -test_data_path = "tests/integration" -test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" - -backend = "mpl" -results_dir = "tests/integration/all_sets_results" -debug = True - - -[polar] -sets = ["polar"] -case_id = "ERA-Interim" -variables = ["T"] -seasons = ["ANN"] -regions = ["polar_S"] -plevs = [850.0] -contour_levels = [240, 244, 248, 252, 256, 260, 264, 268, 272] -diff_levels = [-15, -10, -7.5, -5, -2.5, -1, 1, 2.5, 5, 7.5, 10, 15] - -test_name = "system tests" -short_test_name = "short_system tests" -ref_name = "ERA-Interim" -reference_name = "ERA-Interim Reanalysis 1979-2015" -reference_data_path = "tests/integration" -ref_file = "ta_ERA-Interim_ANN_198001_201401_climo.nc" -test_data_path = "tests/integration" -test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" - -backend = "mpl" -results_dir = "tests/integration/all_sets_results" -debug = True - - -[cosp_histogram] -sets = ["cosp_histogram"] -case_id = "MISR-COSP" -variables = ["COSP_HISTOGRAM_MISR"] -seasons = ["ANN"] -contour_levels = [0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5] -diff_levels = [-3.0,-2.5,-2.0,-1.5,-1.0,-0.5,0,0.5,1.0,1.5,2.0,2.5,3.0] - -test_name = "system tests" -short_test_name = "short_system tests" -ref_name = "MISRCOSP" -reference_name = "MISR COSP (2000-2009)" -reference_data_path = "tests/integration" -ref_file = "CLDMISR_ERA-Interim_ANN_198001_201401_climo.nc" -test_data_path = "tests/integration" -test_file = "CLD_MISR_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc" - -backend = "mpl" -results_dir = "tests/integration/all_sets_results" -debug = True - - -[area_mean_time_series] -sets = ["area_mean_time_series"] -variables = ["TREFHT"] - -test_name = "system tests" -short_test_name = "short_system tests" -test_data_path = "tests/integration" -test_file = "TREFHT_201201_201312.nc" -start_yr = '2012' -end_yr = '2013' - -backend = "mpl" -results_dir = "tests/integration/all_sets_results" -debug = True -ref_names = [] -ref_timeseries_input = True -test_timeseries_input = True -ref_start_yr='2012' -ref_end_yr='2013' -test_start_yr='2012' -test_end_yr='2013' - - -[enso_diags] -sets = ["enso_diags"] -case_id = 'TREFHT-response' -debug = True -diff_colormap = "BrBG" -regions = ["20S20N"] -results_dir = "tests/integration/all_sets_results" -seasons = ["ANN"] -start_yr = '2012' -end_yr = '2013' -test_data_path = '.' -test_file = 'TREFHT_201201_201312.nc' -reference_data_path = '.' -ref_file = 'TREFHT_201201_201312.nc' -test_name = "system tests" -variables = ["TREFHT"] -print_statements = True -nino_region = "NINO34" -ref_timeseries_input = True -test_timeseries_input = True -ref_start_yr='2012' -ref_end_yr='2013' -test_start_yr='2012' -test_end_yr='2013' -backend = "mpl" -plot_type="map" diff --git a/tests/complete_run.py b/tests/integration/complete_run.py similarity index 62% rename from tests/complete_run.py rename to tests/integration/complete_run.py index 68b6627f9..cd8af9f7e 100644 --- a/tests/complete_run.py +++ b/tests/integration/complete_run.py @@ -1,15 +1,21 @@ +"""The complete E3SM Diagnostic run. + +Due to the large amount of data required to run, this test will be run manually +on Anvil (rather than as part of the CI tests). + +Run the following first: + - srun --pty --nodes=1 --time=01:00:00 /bin/bash + - source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh + - Or: source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_anvil.sh +""" import os -import unittest # This test should be run with the latest E3SM Diags tutorial code. from examples.run_v2_6_0_all_sets_E3SM_machines import run_lcrc -from tests.integration.test_diags import compare_images - -# Due to the large amount of data required to run, this test will be run manually on Anvil -# (rather than as part of the CI tests). +from tests.integration.utils import _compare_images -class TestCompleteRun(unittest.TestCase): +class TestCompleteRun: def test_complete_run(self): actual_images_dir = run_lcrc(".") @@ -25,20 +31,11 @@ def test_complete_run(self): path_to_actual_png = os.path.join(actual_images_dir, image_name) path_to_expected_png = os.path.join(expected_images_dir, image_name) - compare_images( - self, + mismatched_images = _compare_images( mismatched_images, image_name, path_to_actual_png, path_to_expected_png, ) - self.assertEqual(mismatched_images, []) - - -if __name__ == "__main__": - # Run the following first: - # srun --pty --nodes=1 --time=01:00:00 /bin/bash - # source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh - # Or: source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_anvil.sh - unittest.main() + assert len(mismatched_images) == 0 diff --git a/tests/integration/config.py b/tests/integration/config.py index f90f8cb51..7e26cc667 100644 --- a/tests/integration/config.py +++ b/tests/integration/config.py @@ -1,5 +1,5 @@ -# Paths and directories used in the integration test. Configurations in -# `all_sets.cfg` and `all_sets_modified.cfg` should match these paths. +# Paths and directories used in the integration test. All `.cfg` paths +# should align with these (e.g., `all_sets.cfg`). TEST_ROOT_PATH = "tests/integration/" TEST_DATA_DIR = "integration_test_data" TEST_IMAGES_DIR = "integration_test_images" diff --git a/tests/integration/test_all_sets.py b/tests/integration/test_all_sets.py deleted file mode 100644 index 6a06c54d2..000000000 --- a/tests/integration/test_all_sets.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import re -import shutil -import unittest - -from tests.integration.config import TEST_DATA_DIR -from tests.integration.utils import run_cmd_and_pipe_stderr - - -def count_images(directory, file_type="png"): - """Count the number of images of type file_type in directory""" - count = 0 - for _, __, files in os.walk(directory): - for f in files: - if f.endswith(file_type): - count += 1 - return count - - -class TestAllSets(unittest.TestCase): - def get_results_dir(self, output): - """Given output from e3sm_diags_driver, extract the path to results_dir.""" - for line in output: - match = re.search("Viewer HTML generated at (.*)viewer.*.html", line) - if match: - results_dir = match.group(1) - return results_dir - self.fail("No viewer directory listed in output: {}".format(output)) - - def run_test(self, backend): - pth = os.path.dirname(__file__) - test_pth = os.path.join(pth, TEST_DATA_DIR) - cfg_pth = os.path.join(pth, "all_sets_modified.cfg") - cfg_pth = os.path.abspath(cfg_pth) - - if backend == "mpl": - backend_option = "" - # Note: remember to increase expected_num_diags as adding new images. - expected_num_diags = 12 - else: - raise RuntimeError("Invalid backend: {}".format(backend)) - # *_data_path needs to be added b/c the tests runs the diags from a different location - cmd = ( - "e3sm_diags_driver.py -d {}{} --reference_data_path {} --test_data_path {}" - ) - cmd = cmd.format(cfg_pth, backend_option, test_pth, test_pth) - stderr = run_cmd_and_pipe_stderr(cmd) - # count the number of pngs in viewer_dir - results_dir = self.get_results_dir(stderr) - count = count_images(results_dir) - # -1 is needed because of the E3SM logo in the viewer html - self.assertEqual(count - 1, expected_num_diags) - - shutil.rmtree(results_dir) # remove all generated results from the diags - - def test_all_sets_mpl(self): - self.run_test("mpl") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/integration/test_all_sets_image_diffs.py b/tests/integration/test_all_sets_image_diffs.py new file mode 100644 index 000000000..0e15be4e8 --- /dev/null +++ b/tests/integration/test_all_sets_image_diffs.py @@ -0,0 +1,379 @@ +import os +import re +import sys +from typing import List + +import pytest + +from e3sm_diags.logger import custom_logger +from e3sm_diags.run import runner +from tests.integration.config import TEST_IMAGES_PATH, TEST_ROOT_PATH +from tests.integration.utils import _compare_images, _get_test_params + +CFG_PATH = os.path.join(TEST_ROOT_PATH, "all_sets.cfg") + + +logger = custom_logger(__name__) + + +@pytest.fixture(scope="module") +def run_diags_and_get_results_dir() -> str: + """Run the diagnostics and get the results directory containing the images. + + The scope of this fixture is at the module level so that it only runs + once, then each individual test can reference the result directory. + + Returns + ------- + str + The path to the results directory. + """ + # Set -d flag to use the .cfg file for running additional diagnostic sets. + sys.argv.extend(["-d", CFG_PATH]) + + params = _get_test_params() + results = runner.run_diags(params) + + # If results is None then that means some/all diagnostic set(s) failed. + # We use params[0].results_dir to check if any diagnostic sets passed. + if results is not None: + results_dir = results[0].results_dir + else: + results_dir = params[0].results_dir + + logger.info(f"results_dir={results_dir}") + return results_dir + + +class TestAllSetsImageDiffs: + @pytest.fixture(autouse=True) + def setup(self, run_diags_and_get_results_dir): + self.results_dir = run_diags_and_get_results_dir + + def test_num_images_is_the_same_as_the_expected(self): + actual_num_images, actual_images = self._count_images_in_dir( + f"{TEST_ROOT_PATH}/all_sets_results_test" + ) + expected_num_images, expected_images = self._count_images_in_dir( + TEST_IMAGES_PATH + ) + + assert actual_images == expected_images + assert actual_num_images == expected_num_images + + def test_area_mean_time_series_plot_diffs(self): + set_name = "area_mean_time_series" + variables = ["TREFHT"] + for variable in variables: + variable_lower = variable.lower() + + # Check PNG path is the same as the expected. + png_path = "{}/{}.png".format(set_name, variable) + full_png_path = os.path.join(self.results_dir, png_path) + path_exists = os.path.exists(full_png_path) + + assert path_exists + + # Check full HTML path is the same as the expected. + filename = "viewer/{}/variable/{}/plot.html".format( + set_name, variable_lower + ) + html_path = os.path.join(self.results_dir, filename) + self._check_html_image(html_path, png_path, full_png_path) + + def test_cosp_histogram_plot_diffs(self): + self._check_plots_generic( + set_name="cosp_histogram", + case_id="MISR-COSP", + ref_name="MISRCOSP", + variables=["COSP_HISTOGRAM_MISR"], + region="global", + ) + + def test_enso_diags_map_diffs(self): + case_id = "TREFHT-response-map" + self._check_enso_map_plots(case_id) + + def test_enso_diags_map_with_start_yrs_diffs(self): + case_id = "TREFHT-response-map-start-yrs" + self._check_enso_map_plots(case_id) + + def test_enso_diags_map_test_with_ref_yrs_diffs(self): + case_id = "TREFHT-response-map-test-ref-yrs" + self._check_enso_map_plots(case_id) + + def test_enso_diags_scatter_plot_diffs(self): + case_id = "TREFHT-response-scatter" + self._check_enso_scatter_plots(case_id) + + def test_enso_diags_scatter_with_start_yrs_plot_diffs(self): + case_id = "TREFHT-response-scatter-start-yrs" + self._check_enso_scatter_plots(case_id) + + def test_enso_diags_scatter_with_test_ref_yrs_plot_diffs(self): + case_id = "TREFHT-response-scatter-test-ref-yrs" + self._check_enso_scatter_plots(case_id) + + def test_lat_lon_plot_diffs(self): + self._check_plots_plevs("lat_lon", "global", [850.0]) + + def test_lat_lon_regional_plot_diffs(self): + self._check_plots_plevs("lat_lon", "CONUS_RRM", [850.0]) + + def test_meridional_mean_2d_plot_diffs(self): + self._check_plots_2d("meridional_mean_2d") + + def test_polar_plot_diffs(self): + self._check_plots_plevs("polar", "polar_S", [850.0]) + + def test_qbo_plot_diffs(self): + case_id = "qbo-test" + case_id_lower = case_id.lower() + set_name = "qbo" + + # Check PNG path is the same as the expected. + png_path = "{}/{}/qbo_diags.png".format(set_name, case_id) + full_png_path = os.path.join(self.results_dir, png_path) + path_exists = os.path.exists(full_png_path) + + assert path_exists + + # Check full HTML path is the same as the expected. + # viewer/qbo/variable/era-interim/plot.html + filename = "viewer/{}/variable/{}/plot.html".format(set_name, case_id_lower) + html_path = os.path.join(self.results_dir, filename) + self._check_html_image(html_path, png_path, full_png_path) + + def test_streamflow_plot_diffs(self): + self._check_streamflow_plots() + + def test_zonal_mean_2d_plot_diffs(self): + self._check_plots_2d("zonal_mean_2d") + + def test_zonal_mean_xy_plot_diffs(self): + self._check_plots_plevs("zonal_mean_xy", "global", [200.0]) + + # Utility methods. + # -------------------------------------------------------------------------- + def _count_images_in_dir(self, directory): + images = [] + for root, _, filenames in os.walk(directory): + # download_data.py won't download files in the viewer directory + # because the webpage is more than a simple page of links. + if "viewer" not in root: + for file in filenames: + if file.endswith(".png"): + images.append(file) + return len(images), images + + def _check_html_image(self, html_path, png_path, full_png_path): + # Check HTML image tags exist. + img_src = None + option_value = None + href = None + with open(html_path, "r") as html: + for line in html: + # If `img_src` is not defined yet: + if not img_src: + re_str = ''.format(png_path) + img_src = re.search(re_str, line) + # If `option_value` is not defined yet: + if not option_value: + re_str = '