Merge branch 'main' into update_zppy_use_perlmutter

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,61 @@
+name: Bug Report
+description: File a bug report to help us improve e3sm_diags
+title: "[Bug]: "
+labels: ["bug"]
+assignees: []
+body:
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: |
+        Thanks for reporting a bug! Please describe what you were trying to get done.
+        Tell us what happened, what went wrong.
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-did-you-expect-to-happen
+    attributes:
+      label: What did you expect to happen? Are there are possible answers you came across?
+      description: |
+        Describe what you expected to happen. Include links to pages you've researched (e.g., software docs, Stack Overflow posts).
+    validations:
+      required: false
+
+  - type: textarea
+    id: sample-code
+    attributes:
+      label: Minimal Complete Verifiable Example (MVCE)
+      description: |
+        Minimal, self-contained copy-pastable example that generates the issue if possible. Please be concise with code posted (e.g., module imports, publicly accessible files).
+        Bug reports that follow these guidelines are easier to diagnose, and so are often handled much more quickly. This section will be automatically formatted into code, so no need for markdown backticks.
+
+        See guidelines below on how to provide a good MCVE:
+
+          - [Minimal Complete Verifiable Examples](https://stackoverflow.com/help/mcve)
+          - [Craft Minimal Bug Reports](http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports)
+      render: python
+
+  - type: textarea
+    id: log-output
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant output. This will be automatically formatted into code, so no need for markdown backticks.
+      render: python
+
+  - type: textarea
+    id: extra
+    attributes:
+      label: Anything else we need to know?
+      description: |
+        Please describe any other information you want to share.
+
+  - type: textarea
+    id: show-versions
+    attributes:
+      label: Environment
+      description: |
+        Paste your conda environment here (`conda info`).
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,10 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Questions
+    url: https://github.com/E3SM-Project/e3sm_diags/discussions
+    about: |
+      Ask questions and discuss with other e3sm_diags community members here. Please
+      browse the e3sm_diags Discussions Forum or e3sm_diags documentation first before asking a
+      question to make sure it is not already answered. If you can't find an
+      answer, please include a self-contained reproducible example with your
+      question if possible. Thanks!

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,15 @@
+name: Documentation Update
+description: Update e3sm_diags documentation
+title: "[Doc]: "
+labels: ["documentation"]
+assignees: []
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe your documentation update
+      description: |
+        Concise description of why the documentation is being updated (e.g., missing content for new feature, typo)
+        If this is related to an issue or PR, please mention it.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,42 @@
+name: Feature Request
+description: Suggest an idea for e3sm_diags
+title: "[Feature]: "
+labels: ["enhancement"]
+assignees: []
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem?
+      description: |
+        Please do a quick search of existing issues to make sure that this has not been asked before.
+        Please provide a clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: |
+        A clear and concise description of what you want to happen.
+    validations:
+      required: false
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Describe alternatives you've considered
+      description: |
+        A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional context
+      description: |
+        Add any other context about the feature request here.
+    validations:
+      required: false

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+## Description
+
+<!--
+  Please include a summary of the change and which issue is fixed.
+  Please also include relevant motivation and context.
+  List any dependencies that are required for this change.
+-->
+
+- Closes #<ISSUE_NUMBER_HERE>
+
+## Checklist
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] My changes generate no new warnings
+- [ ] Any dependent changes have been merged and published in downstream modules
+
+If applicable:
+
+- [ ] New and existing unit tests pass with my changes (locally and CI/CD build)
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] I have noted that this is a breaking change for a major release (fix or feature that would cause existing functionality to not work as expected)

.github/workflows/build_workflow.yml

@@ -11,7 +11,7 @@ on:
 
 env:
   CANCEL_OTHERS: true
-  PATHS_IGNORE: '["**/README.md", "**/docs/**", "**/examples/**", "**/misc/**", "**/.vscode/**"]'
+  PATHS_IGNORE: '["**/README.md", "**/docs/**", "**/examples/**", "**/misc/**", "**/.vscode/**", "**/.github/pull_request_template.md", "**/.github/ISSUE_TEMPLATE"]'
 
 jobs:
   pre-commit-hooks:
@@ -99,10 +99,18 @@ jobs:
           mamba info
 
       - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
-        name: Run Tests
+        name: Run Unit Tests
+        run: pytest tests/e3sm_diags
+
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Download Integration Test Data
+        run: python -m tests.integration.download_data
+
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Run Integration Tests
         env:
           CHECK_IMAGES: False
-        run: bash tests/test.sh
+        run: pytest tests/integration
 
   publish-docs:
     if: ${{ github.event_name == 'push' }}

.pre-commit-config.yaml

@@ -4,14 +4,14 @@ fail_fast: true
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
 
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 23.9.1
     hooks:
       - id: black
 
@@ -23,15 +23,15 @@ repos:
   # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab
   # https://github.com/pre-commit/pre-commit/issues/1206
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
-        args: ["--config=setup.cfg"]
-        additional_dependencies: [flake8-isort]
+        args: [--config=setup.cfg]
+        additional_dependencies: [flake8-isort==6.1.0]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.5.1
     hooks:
       - id: mypy
-        args: ["--config=setup.cfg"]
-        additional_dependencies: [types-pyYAML==6.0.12.6]
+        args: [--config=pyproject.toml]
+        additional_dependencies: [dask, numpy>=1.23.0, types-PyYAML]

.vscode/e3sm_diags.code-workspace

@@ -8,6 +8,9 @@
       "path": ".."
     }
   ],
+  // ===========================
+  // VS Code Workspace Settings.
+  // ===========================
   "settings": {
     // ===================
     // Editor settings
@@ -22,17 +25,14 @@
       "editor.rulers": [80, 88, 120],
       "editor.wordWrap": "wordWrapColumn",
       "editor.wordWrapColumn": 120,
-      "editor.defaultFormatter": "ms-python.python"
+      "editor.defaultFormatter": "ms-python.black-formatter"
     },
     // Code Formatting and Linting
     // ---------------------------
-    "python.formatting.provider": "black",
-    "python.linting.flake8Enabled": true,
-    "python.linting.flake8Args": ["--config=setup.cfg"],
+    "flake8.args": ["--config=setup.cfg"],
     // Type checking
     // ---------------------------
-    "python.linting.mypyEnabled": true,
-    "python.linting.mypyArgs": ["--config=setup.cfg"],
+    "mypy-type-checker.args": ["--config=pyproject.toml"],
     // Testing
     // ---------------------------
     // NOTE: Debugger doesn't work if pytest-cov is enabled, so set "--no-cov"
@@ -49,5 +49,24 @@
       "editor.wordWrap": "wordWrapColumn",
       "editor.wordWrapColumn": 120
     }
+  },
+  // =====================================
+  // VS Code Python Debugger Configuration
+  // =====================================
+  "launch": {
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Python: Current File",
+        "type": "python",
+        "request": "launch",
+        "program": "${file}",
+        "console": "integratedTerminal",
+        "justMyCode": true,
+        "env": {
+          "PYTHONPATH": "${workspaceFolder}"
+        }
+      }
+    ]
   }
 }

Makefile

@@ -0,0 +1,90 @@
+.PHONY: clean clean-test clean-pyc clean-build docs help
+.DEFAULT_GOAL := help
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+
+from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+
+define PRINT_HELP_PYSCRIPT
+import re, sys
+
+for line in sys.stdin:
+	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
+	if match:
+		target, help = match.groups()
+		print("%-20s %s" % (target, help))
+endef
+export PRINT_HELP_PYSCRIPT
+
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+# To run these commands: make <COMMAND>
+# ==================================================
+
+help:
+	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
+
+# Clean local repository
+# ----------------------
+clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+
+clean-build: ## remove build artifacts
+	rm -fr build/
+	rm -fr conda-build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc: ## remove Python file artifacts
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## remove test and coverage artifacts
+	rm -fr tests_coverage_reports/
+	rm -f .coverage
+	rm -fr htmlcov/
+	rm -f coverage.xml
+	rm -fr .pytest_cache
+	rm -rf .mypy_cache
+
+clean-test-int-res: ## remove integration test results and image check failures
+	rm -rf tests/integration/all_sets_results_test
+	rm -rf tests/integration/image_check_failures
+
+clean-test-int-data:  # remove integration test data and images (expected) -- useful when they are updated
+	rm -rf tests/integration/integration_test_data
+	rm -rf tests/integration/integration_test_images
+
+# Quality Assurance
+# ----------------------
+pre-commit:  # run pre-commit quality assurance checks
+	pre-commit run --all-files
+
+lint: ## check style with flake8
+	flake8 e3sm_diags tests
+
+test: ## run tests quickly with the default Python and produces code coverage report
+	pytest
+	$(BROWSER) tests_coverage_reports/htmlcov/index.html
+
+# Documentation
+# ----------------------
+docs: ## generate Sphinx HTML documentation, including API docs
+	rm -rf docs/generated
+	cd docs && make html
+	$(MAKE) -C docs clean
+	$(MAKE) -C docs html
+	$(BROWSER) docs/_build/html/index.html
+
+# Build
+# ----------------------
+install: clean ## install the package to the active Python's site-packages
+	python -m pip install .

analysis_data_preprocess/create_OMI-MLS_climo.sh

@@ -18,7 +18,7 @@ mkdir $climo_output_path
 mkdir $tmp
 
 
-#Add lon dimention to zonel mean
+#Add lon dimention to zonal mean
 ncap2 -s 'SCO=O3strat' ${original_data_path}O3strat_ZMK.nc ${time_series_output_path}SCO_200410_201712.nc
 #cp ${original_data_path}O3strat_ZMK.nc ${time_series_output_path}SCO_200410_201712.nc
 cdo splityear ${time_series_output_path}SCO_200410_201712.nc ${tmp}sco