Update DevOps configs (#755)

- Update conda env dependencies - Add `dev-nompi.yml` which includes the nompi version of esmf to avoid the yaksa leak error that breaks VS Code testing API - Update pre-commit dependency versions and fix new issues for `black`, `flake8`, and `mypy` - Move most of `setup.cfg` to `pyproject.toml` - Add `Makefile` with convenient developer commands - Update `test_diags.py` and `test_allsets.py` to use `pytest`
E3SM-Project · Nov 29, 2023 · 633b52c · 633b52c
1 parent a0a3561
commit 633b52c
Show file tree

Hide file tree

Showing 40 changed files with 577 additions and 384 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,14 +4,14 @@ fail_fast: true
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
 
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 23.9.1
     hooks:
       - id: black
 
@@ -23,15 +23,15 @@ repos:
   # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab
   # https://github.com/pre-commit/pre-commit/issues/1206
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
-        args: ["--config=setup.cfg"]
-        additional_dependencies: [flake8-isort]
+        args: [--config=setup.cfg]
+        additional_dependencies: [flake8-isort==6.1.0]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.5.1
     hooks:
       - id: mypy
-        args: ["--config=setup.cfg"]
-        additional_dependencies: [types-pyYAML==6.0.12.6]
+        args: [--config=pyproject.toml]
+        additional_dependencies: [dask, numpy>=1.23.0, types-PyYAML]
diff --git a/Makefile b/Makefile
@@ -0,0 +1,82 @@
+.PHONY: clean clean-test clean-pyc clean-build docs help
+.DEFAULT_GOAL := help
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+
+from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+
+define PRINT_HELP_PYSCRIPT
+import re, sys
+
+for line in sys.stdin:
+	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
+	if match:
+		target, help = match.groups()
+		print("%-20s %s" % (target, help))
+endef
+export PRINT_HELP_PYSCRIPT
+
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+# To run these commands: make <COMMAND>
+# ==================================================
+
+help:
+	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
+
+# Clean local repository
+# ----------------------
+clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+
+clean-build: ## remove build artifacts
+	rm -fr build/
+	rm -fr conda-build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc: ## remove Python file artifacts
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## remove test and coverage artifacts
+	rm -fr tests_coverage_reports/
+	rm -f .coverage
+	rm -fr htmlcov/
+	rm -f coverage.xml
+	rm -fr .pytest_cache
+	rm -rf .mypy_cache
+
+# Quality Assurance
+# ----------------------
+pre-commit:  # run pre-commit quality assurance checks
+	pre-commit run --all-files
+
+lint: ## check style with flake8
+	flake8 e3sm_diags tests
+
+test: ## run tests quickly with the default Python and produces code coverage report
+	pytest
+	$(BROWSER) tests_coverage_reports/htmlcov/index.html
+
+# Documentation
+# ----------------------
+docs: ## generate Sphinx HTML documentation, including API docs
+	rm -rf docs/generated
+	cd docs && make html
+	$(MAKE) -C docs clean
+	$(MAKE) -C docs html
+	$(BROWSER) docs/_build/html/index.html
+
+# Build
+# ----------------------
+install: clean ## install the package to the active Python's site-packages
+	python -m pip install .
diff --git a/conda-env/ci.yml b/conda-env/ci.yml
@@ -36,4 +36,3 @@ dependencies:
   - sphinx
   - sphinx_rtd_theme
   - sphinx-multiversion
-prefix: /opt/miniconda3/envs/e3sm_diags_ci
diff --git a/conda-env/dev-nompi.yml b/conda-env/dev-nompi.yml
@@ -0,0 +1,56 @@
+# Conda development environment for testing local source code changes to `e3sm_diags` before merging them to production (`master` branch).
+# This version contains the no MPI version of `esmf` as a workaround for allowing VS Code's testing API to work.
+# The MPI version of `esmf` is usually installed by default, but it breaks VS Code's testing API because it throws a mysterious
+# `yaksa` warning.
+# More info: https://github.com/E3SM-Project/e3sm_diags/issues/737
+name: e3sm_diags_dev_nompi
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # =================
+  - python >=3.9
+  - pip
+  - beautifulsoup4
+  - cartopy >=0.17.0
+  - cartopy_offlinedata
+  - cdp 1.7.0
+  - cdms2 3.1.5
+  - cdutil 8.2.1
+  - dask
+  - esmf >=8.4.0 nompi*
+  - esmpy >=8.4.0
+  - genutil 8.2.1
+  - lxml
+  - mache >=0.15.0
+  - matplotlib-base
+  - netcdf4
+  - numpy >=1.23.0
+  - shapely >=2.0.0,<3.0.0
+  - xarray >=2023.02.0
+  # Testing
+  # =======================
+  - scipy
+  - pytest
+  - pytest-cov
+  # Documentation
+  # =======================
+  - sphinx
+  - sphinx_rtd_theme
+  - sphinx-multiversion
+  # Quality Assurance Tools
+  # =======================
+  # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in
+  # `.pre-commit.config.yaml`, then update the pinned versions here.
+  - black=23.9.1
+  - flake8=6.1.0
+  - flake8-isort=6.1.0
+  - isort=5.12.0
+  - mypy=1.5.1
+  - pre-commit >=3.0.0
+  - types-PyYAML >=6.0.0
+  # Developer Tools
+  # =======================
+  - tbump=6.9.0
+  - ipykernel
diff --git a/conda-env/dev.yml b/conda-env/dev.yml
@@ -25,28 +25,27 @@ dependencies:
   - shapely >=2.0.0,<3.0.0
   - xarray >=2023.02.0
   # Testing
-  # ==================
+  # =======================
   - scipy
   - pytest
   - pytest-cov
   # Documentation
-  # =================
+  # =======================
   - sphinx
   - sphinx_rtd_theme
   - sphinx-multiversion
+  # Quality Assurance Tools
+  # =======================
+  # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in
+  # `.pre-commit.config.yaml`, then update the pinned versions here.
+  - black=23.9.1
+  - flake8=6.1.0
+  - flake8-isort=6.1.0
+  - isort=5.12.0
+  - mypy=1.5.1
+  - pre-commit >=3.0.0
+  - types-PyYAML >=6.0.0
   # Developer Tools
-  # =================
-  # If versions are updated, also update 'rev' in `.pre-commit.config.yaml`
-  - black=22.10.0
-  - flake8=6.0.0
-  - flake8-isort=5.0.3
-  - isort=5.11.3
-  - mypy=0.991
-  - pre-commit=2.20.0
-  - pytest=7.2.0
-  - pytest-cov=4.0.0
-  - types-PyYAML=6.0.12.6
-  # Developer Tools
-  # =================
+  # =======================
   - tbump=6.9.0
-prefix: /opt/miniconda3/envs/e3sm_diags_dev
+  - ipykernel
diff --git a/e3sm_diags/derivations/acme.py b/e3sm_diags/derivations/acme.py
@@ -465,7 +465,8 @@ def cosp_bin_sum(
     tau_high0: Optional[float],
 ):
     """sum of cosp bins to calculate cloud fraction in specified cloud top pressure / height and
-    cloud thickness bins, input variable has dimension (cosp_prs,cosp_tau,lat,lon)/(cosp_ht,cosp_tau,lat,lon)"""
+    cloud thickness bins, input variable has dimension (cosp_prs,cosp_tau,lat,lon)/(cosp_ht,cosp_tau,lat,lon)
+    """
     prs: FileAxis = cld.getAxis(0)
     tau: FileAxis = cld.getAxis(1)
 

diff --git a/e3sm_diags/driver/arm_diags_driver.py b/e3sm_diags/driver/arm_diags_driver.py
@@ -423,7 +423,6 @@ def run_diag_aerosol_activation(parameter: ARMDiagsParameter) -> ARMDiagsParamet
         logger.info("Selected region: {}".format(region))
         # Possible variables are ccn01, ccn02, ccn05
         for variable in variables:
-
             test_data = utils.dataset.Dataset(parameter, test=True)
 
             test_a_num = test_data.get_timeseries_variable("a_num", single_point=True)[
@@ -441,7 +440,6 @@ def run_diag_aerosol_activation(parameter: ARMDiagsParameter) -> ARMDiagsParamet
             )
 
             if "armdiags" in ref_name:
-
                 ref_file = os.path.join(
                     ref_path,
                     region[:3] + "armdiagsaciactivate" + region[3:5].upper() + ".c1.nc",
@@ -568,7 +566,6 @@ def run_diag_pdf_daily(parameter: ARMDiagsParameter):
 
 
 def run_diag(parameter: ARMDiagsParameter) -> ARMDiagsParameter:
-
     if parameter.diags_set == "annual_cycle":
         return run_diag_annual_cycle(parameter)
     elif parameter.diags_set == "diurnal_cycle":

diff --git a/e3sm_diags/driver/enso_diags_driver.py b/e3sm_diags/driver/enso_diags_driver.py
@@ -272,7 +272,11 @@ def run_diag_map(parameter: EnsoDiagsParameter) -> EnsoDiagsParameter:
                 )
 
                 # Reference
-                (ref_domain, ref_reg_coe, ref_confidence_levels,) = perform_regression(
+                (
+                    ref_domain,
+                    ref_reg_coe,
+                    ref_confidence_levels,
+                ) = perform_regression(
                     ref_data,
                     parameter,
                     var,

diff --git a/e3sm_diags/driver/lat_lon_driver.py b/e3sm_diags/driver/lat_lon_driver.py
@@ -175,12 +175,8 @@ def run_diag(parameter: CoreParameter) -> CoreParameter:  # noqa: C901
 
                 # Select plev.
                 for ilev in range(len(plev)):
-                    mv1 = mv1_p[
-                        ilev,
-                    ]
-                    mv2 = mv2_p[
-                        ilev,
-                    ]
+                    mv1 = mv1_p[ilev,]
+                    mv2 = mv2_p[ilev,]
 
                     for region in regions:
                         parameter.var_region = region

diff --git a/e3sm_diags/driver/polar_driver.py b/e3sm_diags/driver/polar_driver.py
@@ -133,12 +133,8 @@ def run_diag(parameter: CoreParameter) -> CoreParameter:
 
                 # Select plev.
                 for ilev in range(len(plev)):
-                    mv1 = mv1_p[
-                        ilev,
-                    ]
-                    mv2 = mv2_p[
-                        ilev,
-                    ]
+                    mv1 = mv1_p[ilev,]
+                    mv2 = mv2_p[ilev,]
 
                     for region in regions:
                         logger.info("Selected region: {}".format(region))

diff --git a/e3sm_diags/driver/streamflow_driver.py b/e3sm_diags/driver/streamflow_driver.py
@@ -285,7 +285,7 @@ def setup_test(parameter, var, using_test_mat_file):
     if parameter.print_statements:
         # For edison: 720x360x600
         logger.info("test_array.shape={}".format(test_array.shape))
-    if type(area_upstream) == cdms2.tvariable.TransientVariable:
+    if isinstance(area_upstream, cdms2.tvariable.TransientVariable):
         area_upstream = area_upstream.getValue()
 
     return area_upstream, test_array
@@ -489,7 +489,7 @@ def generate_export(
                 else:
                     monthly = mmat
 
-                if type(monthly) == cdms2.tvariable.TransientVariable:
+                if isinstance(monthly, cdms2.tvariable.TransientVariable):
                     monthly = monthly.getValue()
 
                 seasonality_index_ref, peak_month_ref = get_seasonality(monthly)
@@ -515,7 +515,7 @@ def generate_export(
             else:
                 monthly = mmat
 
-            if type(monthly) == cdms2.tvariable.TransientVariable:
+            if isinstance(monthly, cdms2.tvariable.TransientVariable):
                 monthly = monthly.getValue()
 
             seasonality_index_test, peak_month_test = get_seasonality(monthly)

diff --git a/e3sm_diags/driver/tc_analysis_driver.py b/e3sm_diags/driver/tc_analysis_driver.py
@@ -265,9 +265,9 @@ def _get_vars_from_te_stitch(
             vars_dict["yearmc"][k - 1, index - 1] = float(line_split[6])
             vars_dict["monthmc"][k - 1, index - 1] = float(line_split[7])
 
-    vars_dict["year_start"] = year_start
-    vars_dict["year_end"] = year_end
-    vars_dict["num_years"] = year_end - year_start + 1
+    vars_dict["year_start"] = year_start  # type: ignore
+    vars_dict["year_end"] = year_end  # type: ignore
+    vars_dict["num_years"] = year_end - year_start + 1  # type: ignore
     logger.info(
         f"TE Start Year: {vars_dict['year_start']}, TE End Year: {vars_dict['year_end']}, Total Years: {vars_dict['num_years']}"
     )
@@ -340,7 +340,6 @@ def _derive_metrics_per_basin(
             and lat[0] > basin_info[3]
             and lat[0] < basin_info[4]
         ):
-
             mod_num_ocn = mod_num_ocn + 1
             mod_mon.append(mon[0])
             mod_wnd.append(np.max(wind))
@@ -491,7 +490,7 @@ def _calc_mean_ace(vsmc: "MaskedArray", yearic: np.ndarray, num_rows: int) -> fl
                 wind_ts = wind[wind >= 35]
                 ace[i] = ace[i] + np.sum(wind_ts**2) / 1e4
 
-    return np.mean(ace)
+    return np.mean(ace)  # type: ignore
 
 
 def _calc_ts_intensity_dist(wind_speeds: List[int]) -> np.ndarray:

diff --git a/e3sm_diags/driver/utils/general.py b/e3sm_diags/driver/utils/general.py
@@ -293,7 +293,7 @@ def save_transient_variables_to_netcdf(set_num, variables_dict, label, parameter
     Save the transient variables to nc file.
     """
     if parameter.save_netcdf:
-        for (variable_name, variable) in variables_dict.items():
+        for variable_name, variable in variables_dict.items():
             # Set cdms preferences - no compression, no shuffling, no complaining
             cdms2.setNetcdfDeflateFlag(1)
             # 1-9, min to max - Comes at heavy IO (read/write time cost)

diff --git a/e3sm_diags/driver/zonal_mean_xy_driver.py b/e3sm_diags/driver/zonal_mean_xy_driver.py
@@ -171,12 +171,8 @@ def run_diag(parameter: CoreParameter) -> CoreParameter:
 
                 # Select plev.
                 for ilev in range(len(plev)):
-                    mv1 = mv1_p[
-                        ilev,
-                    ]
-                    mv2 = mv2_p[
-                        ilev,
-                    ]
+                    mv1 = mv1_p[ilev,]
+                    mv2 = mv2_p[ilev,]
 
                     for region in regions:
                         logger.info(f"Selected region: {region}")

diff --git a/e3sm_diags/plot/cartopy/aerosol_aeronet_plot.py b/e3sm_diags/plot/cartopy/aerosol_aeronet_plot.py
@@ -114,7 +114,7 @@ def plot(test, test_site, ref_site, parameter):
             subpage = np.array(p).reshape(2, 2)
             subpage[1, :] = subpage[0, :] + subpage[1, :]
             subpage = subpage + np.array(border).reshape(2, 2)
-            subpage = list(((subpage) * page).flatten())
+            subpage = list(((subpage) * page).flatten())  # type: ignore
             extent = matplotlib.transforms.Bbox.from_extents(*subpage)
             # Save subplot
             fname = fnm + ".%i." % (i) + f