From 0afdfee060a4c884797baea4779d9a4cc128c8c6 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Mon, 13 May 2024 18:48:06 +0200 Subject: [PATCH 01/13] wip on simplifying pre-commit setup --- .flake8 | 3 ++ .github/workflows/pre-commit.yml | 5 ++- .pre-commit-config.yaml | 58 ++++++++++++++++---------------- requirements.txt | 6 +--- 4 files changed, 35 insertions(+), 37 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..b02dd545 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +ignore = E203, F811, I002, W503 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a6ad84f1..4828afc9 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -28,6 +28,5 @@ jobs: pip install pyg-lib==0.2.0 torch-scatter==2.1.1 torch-sparse==0.6.17 \ torch-cluster==1.6.1 torch-geometric==2.3.1 \ -f https://pytorch-geometric.com/whl/torch-2.0.1+cpu.html - - name: Run pre-commit hooks - run: | - pre-commit run --all-files + - uses: pre-commit/action@v2.0.3 + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f48eca67..106cb64b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,51 +1,51 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - - id: check-ast - - id: check-case-conflict - - id: check-docstring-first - - id: check-symlinks - - id: check-toml - - id: check-yaml - - id: debug-statements - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: local + - id: check-ast + - id: check-case-conflict + - id: check-docstring-first + - id: check-symlinks + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/codespell-project/codespell + rev: 2.0.0 hooks: - - id: codespell - name: codespell + - id: codespell description: Check for spelling errors language: system - entry: codespell -- repo: local + + - repo: https://github.com/psf/black + rev: 22.3.0 hooks: - - id: black - name: black + - id: black description: Format Python code language: system - entry: black types_or: [python, pyi] -- repo: local + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 hooks: - - id: isort - name: isort + - id: isort description: Group and sort Python imports language: system - entry: isort types_or: [python, pyi, cython] -- repo: local + + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 hooks: - - id: flake8 - name: flake8 + - id: flake8 description: Check Python code for correctness, consistency and adherence to best practices language: system - entry: flake8 --max-line-length=80 --ignore=E203,F811,I002,W503 types: [python] -- repo: local + + - repo: https://github.com/pylint-dev/pylint hooks: - - id: pylint - name: pylint + - id: pylint entry: pylint -rn -sn language: system types: [python] diff --git a/requirements.txt b/requirements.txt index 5a2111b2..f381d54f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,6 @@ Cartopy>=0.22.0 pyproj>=3.4.1 tueplots>=0.0.8 plotly>=5.15.0 + # for dev -codespell>=2.0.0 -black>=21.9b0 -isort>=5.9.3 -flake8>=4.0.1 -pylint>=3.0.3 pre-commit>=2.15.0 From 28118a6e82775a4cb6d0d3d7e89e50c86541d320 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Mon, 13 May 2024 18:51:46 +0200 Subject: [PATCH 02/13] setup pylint version --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 106cb64b..28cd91b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,6 +44,7 @@ repos: types: [python] - repo: https://github.com/pylint-dev/pylint + rev: 2.0.0 hooks: - id: pylint entry: pylint -rn -sn From 3da310860f209bcaec6387fbebc0468505aa6331 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Mon, 13 May 2024 19:09:09 +0200 Subject: [PATCH 03/13] remove external deps install in cicd linting --- .github/workflows/pre-commit.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 4828afc9..0ff792a3 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -11,22 +11,11 @@ on: jobs: pre-commit-job: runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install pre-commit hooks - run: | - pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 \ - --index-url https://download.pytorch.org/whl/cpu - pip install -r requirements.txt - pip install pyg-lib==0.2.0 torch-scatter==2.1.1 torch-sparse==0.6.17 \ - torch-cluster==1.6.1 torch-geometric==2.3.1 \ - -f https://pytorch-geometric.com/whl/torch-2.0.1+cpu.html - uses: pre-commit/action@v2.0.3 From 1d12b0d917e87cf0d834dc603da00a727f72c332 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Mon, 13 May 2024 21:38:49 +0200 Subject: [PATCH 04/13] minor fixes --- .github/workflows/pre-commit.yml | 1 - .pre-commit-config.yaml | 17 +---------------- neural_lam/models/base_hi_graph_model.py | 12 ++++++------ plot_graph.py | 6 +----- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 0ff792a3..29203557 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -18,4 +18,3 @@ jobs: with: python-version: 3.9 - uses: pre-commit/action@v2.0.3 - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28cd91b9..0547c6b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,40 +13,25 @@ repos: - id: trailing-whitespace - repo: https://github.com/codespell-project/codespell - rev: 2.0.0 + rev: v2.2.6 hooks: - id: codespell description: Check for spelling errors - language: system - repo: https://github.com/psf/black rev: 22.3.0 hooks: - id: black description: Format Python code - language: system - types_or: [python, pyi] - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: - id: isort description: Group and sort Python imports - language: system - types_or: [python, pyi, cython] - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: - id: flake8 description: Check Python code for correctness, consistency and adherence to best practices - language: system - types: [python] - - - repo: https://github.com/pylint-dev/pylint - rev: 2.0.0 - hooks: - - id: pylint - entry: pylint -rn -sn - language: system - types: [python] diff --git a/neural_lam/models/base_hi_graph_model.py b/neural_lam/models/base_hi_graph_model.py index 8ce87030..d9a4c676 100644 --- a/neural_lam/models/base_hi_graph_model.py +++ b/neural_lam/models/base_hi_graph_model.py @@ -179,9 +179,9 @@ def process_step(self, mesh_rep): ) # Update node and edge vectors in lists - mesh_rep_levels[level_l] = ( - new_node_rep # (B, num_mesh_nodes[l], d_h) - ) + mesh_rep_levels[ + level_l + ] = new_node_rep # (B, num_mesh_nodes[l], d_h) mesh_up_rep[level_l - 1] = new_edge_rep # (B, M_up[l-1], d_h) # - PROCESSOR - @@ -207,9 +207,9 @@ def process_step(self, mesh_rep): new_node_rep = gnn(send_node_rep, rec_node_rep, edge_rep) # Update node and edge vectors in lists - mesh_rep_levels[level_l] = ( - new_node_rep # (B, num_mesh_nodes[l], d_h) - ) + mesh_rep_levels[ + level_l + ] = new_node_rep # (B, num_mesh_nodes[l], d_h) # Return only bottom level representation return mesh_rep_levels[0] # (B, num_mesh_nodes[0], d_h) diff --git a/plot_graph.py b/plot_graph.py index 48427d5c..27e230e7 100644 --- a/plot_graph.py +++ b/plot_graph.py @@ -47,11 +47,7 @@ def main(): # Load graph data hierarchical, graph_ldict = utils.load_graph(args.graph) - ( - g2m_edge_index, - m2g_edge_index, - m2m_edge_index, - ) = ( + (g2m_edge_index, m2g_edge_index, m2m_edge_index,) = ( graph_ldict["g2m_edge_index"], graph_ldict["m2g_edge_index"], graph_ldict["m2m_edge_index"], From ad0accc6dafc1bcdbbb7833654b707d997553071 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Mon, 13 May 2024 21:41:49 +0200 Subject: [PATCH 05/13] run on all branch pushes --- .github/workflows/pre-commit.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 29203557..2f665631 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,15 +1,17 @@ name: Run pre-commit job on: - push: + # trigger on pushes to any branch, but not main + push: + branches-ignore: + - master + # and also on PRs to main + pull_request: branches: - - main - pull_request: - branches: - - main + - main jobs: - pre-commit-job: + pre-commit-job: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 3e69502eb3c5b890b61552c9a6c1e8fe49f5149b Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Mon, 13 May 2024 21:42:54 +0200 Subject: [PATCH 06/13] rename action to "lint" --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 2f665631..14ab3c3b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,4 +1,4 @@ -name: Run pre-commit job +name: lint on: # trigger on pushes to any branch, but not main From d732b2223159918468d1e40aba94fb49bcf0c2d6 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 14 May 2024 15:20:37 +0200 Subject: [PATCH 07/13] Update .github/workflows/pre-commit.yml Co-authored-by: sadamov <45732287+sadamov@users.noreply.github.com> --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 14ab3c3b..d80a3e65 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -4,7 +4,7 @@ on: # trigger on pushes to any branch, but not main push: branches-ignore: - - master + - main # and also on PRs to main pull_request: branches: From ba424ce3d9db9dee284ef45922f26cda43b26858 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 14 May 2024 15:39:21 +0200 Subject: [PATCH 08/13] use pre-commit with multiple python versions --- .github/workflows/pre-commit.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d80a3e65..d0ee6cc4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,14 +9,17 @@ on: pull_request: branches: - main - + jobs: pre-commit-job: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9, 3.10, 3.11, 3.12] steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 - - uses: pre-commit/action@v2.0.3 + python-version: ${{ matrix.python-version }} + - uses: pre-commit/action@v2.0.3 \ No newline at end of file From 659c72dd1aefc975eda3c692c6c14089bbf977fd Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 14 May 2024 15:40:27 +0200 Subject: [PATCH 09/13] quote versions --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d0ee6cc4..69342955 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9, 3.10, 3.11, 3.12] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 - name: Set up Python From f50bc19f1fe957454044e7f94928c0807c50a8ad Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 14 May 2024 15:41:39 +0200 Subject: [PATCH 10/13] fix linting --- .github/workflows/pre-commit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 69342955..dc519e5b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,7 +9,7 @@ on: pull_request: branches: - main - + jobs: pre-commit-job: runs-on: ubuntu-latest @@ -22,4 +22,4 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: pre-commit/action@v2.0.3 \ No newline at end of file + - uses: pre-commit/action@v2.0.3 From db1976fc79b1ae1a0c8270f01ae133cb4b3077b1 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Wed, 15 May 2024 15:19:56 +0200 Subject: [PATCH 11/13] Bump flake8 to v7.0.0 to address py3.12 issues --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0547c6b9..815a92e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: description: Group and sort Python imports - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 description: Check Python code for correctness, consistency and adherence to best practices From cc119187270e8b9ef7d8549df0ae7e595d1b1224 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Wed, 22 May 2024 12:11:30 +0200 Subject: [PATCH 12/13] fix flake8 issues for py3.12 --- neural_lam/models/ar_model.py | 4 ++-- neural_lam/models/base_hi_graph_model.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/neural_lam/models/ar_model.py b/neural_lam/models/ar_model.py index 7d0a8320..6d160526 100644 --- a/neural_lam/models/ar_model.py +++ b/neural_lam/models/ar_model.py @@ -400,7 +400,7 @@ def plot_examples(self, batch, n_examples, prediction=None): target_t[:, var_i], self.interior_mask[:, 0], title=f"{var_name} ({var_unit}), " - f"t={t_i} ({self.step_length*t_i} h)", + f"t={t_i} ({self.step_length * t_i} h)", vrange=var_vrange, ) for var_i, (var_name, var_unit, var_vrange) in enumerate( @@ -542,7 +542,7 @@ def on_test_epoch_end(self): vis.plot_spatial_error( loss_map, self.interior_mask[:, 0], - title=f"Test loss, t={t_i} ({self.step_length*t_i} h)", + title=f"Test loss, t={t_i} ({self.step_length * t_i} h)", ) for t_i, loss_map in zip( constants.VAL_STEP_LOG_ERRORS, mean_spatial_loss diff --git a/neural_lam/models/base_hi_graph_model.py b/neural_lam/models/base_hi_graph_model.py index d9a4c676..3fd30579 100644 --- a/neural_lam/models/base_hi_graph_model.py +++ b/neural_lam/models/base_hi_graph_model.py @@ -36,7 +36,7 @@ def __init__(self, args): if level_index < (self.num_levels - 1): up_edges = self.mesh_up_features[level_index].shape[0] down_edges = self.mesh_down_features[level_index].shape[0] - print(f" {level_index}<->{level_index+1}") + print(f" {level_index}<->{level_index + 1}") print(f" - {up_edges} up edges, {down_edges} down edges") # Embedders # Assume all levels have same static feature dimensionality From aff1036f0bbd373537db0095972d08384ff12292 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Wed, 22 May 2024 12:19:38 +0200 Subject: [PATCH 13/13] changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 823ac8b1..63feff96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [\#13](https://github.com/joeloskarsson/neural-lam/pull/13) @joeloskarsson +### Maintenance + +- simplify pre-commit setup by 1) reducing linting to only cover static + analysis excluding imports from external dependencies (this will be handled + in build/test cicd action introduced later), 2) pinning versions of linting + tools in pre-commit config (and remove from `requirements.txt`) and 3) using + github action to run pre-commit. + [\#29](https://github.com/mllam/neural-lam/pull/29) + @leifdenby + + ## [v0.1.0](https://github.com/joeloskarsson/neural-lam/releases/tag/v0.1.0) First tagged release of `neural-lam`, matching Oskarsson et al 2023 publication