Extract positron-ipykernel CI and test requirements, re-add ruff and …

…prettier checks (posit-dev/positron-python#416) * unpin pytest our reason for pinning has been fixed upstream in microsoft/vscode-python#22799. also bump pytest-asyncio to a compatible version. * extract positron-ipykernel ci job and test requirements * remove pillow - it's another snyk pin * remove unusued test requirements * re-add pyright job * re-add ruff to ci lint job * re-add format-check; downgrade prettier to match upstream; fix format errors * checkout before setting up pip cache
posit-dev · Mar 28, 2024 · f5b8fcc · f5b8fcc
1 parent 3a7eac3
commit f5b8fcc
Show file tree

Hide file tree

Showing 40 changed files with 324 additions and 307 deletions.
diff --git a/extensions/positron-python/.github/workflows/positron-ci.yml b/extensions/positron-python/.github/workflows/positron-ci.yml
@@ -1,4 +1,4 @@
-name: "Positron Python CI"
+name: 'Positron Python CI'
 
 on:
   push:
@@ -33,6 +33,12 @@ jobs:
       - name: Install Yarn
         run: npm install -g yarn
 
+      - name: Install Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
       - name: Install Node dependencies
         run: yarn install --immutable --network-timeout 120000 --prefer-offline
 
@@ -45,68 +51,159 @@ jobs:
       - name: Lint TypeScript code
         run: yarn lint
 
-      - name: Install Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
+      - name: Check TypeScript format
+        run: yarn format-check
 
       - name: Check Python format
         run: |
           python -m pip install -U black
-          python -m black . --check --include "positron\/.*.py$|positron_language_server.py$|tests\/test_positron_.*.py$"
+          python -m black . --check
         working-directory: pythonFiles
 
+      - name: Run Ruff
+        run: |
+          python -m pip install -U ruff
+          python -m ruff check .
+        working-directory: pythonFiles
+
+  check-types:
+    name: Check Python types
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Use Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
+      - name: Install base Python requirements
+        uses: brettcannon/pip-secure-install@v1
+        with:
+          options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py'
+
+      - name: Install Positron IPyKernel requirements
+        run: python scripts/vendor.py
+
+      - name: Install other Python requirements
+        run: |
+          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
+          python -m pip install --upgrade -r build/test-requirements.txt
+          python -m pip install --upgrade -r pythonFiles/positron/pinned-test-requirements.txt
+
+      - name: Run Pyright
+        uses: jakebailey/pyright-action@v2
+        with:
+          version: 1.1.352
+          working-directory: 'pythonFiles'
+
   python-tests:
-    name: Test Python
+    name: Python Tests
+    # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
     runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: ${{ env.special-working-directory }}
     strategy:
+      fail-fast: false
       matrix:
+        # TODO: Decide whether we want to match upstream matrix.
+        # # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
+        # # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case.
+        # os: [ubuntu-latest, windows-latest]
+        # # Run the tests on the oldest and most recent versions of Python.
+        # python: ['3.8', '3.x'] # run for 3 pytest versions, most recent stable, oldest version supported and pre-release
+        # pytest-version: ['pytest', 'pytest@pre-release', 'pytest==6.2.0']
         include:
           - os: 'ubuntu-latest'
-            python-version: '3.8'
+            python: '3.8'
+            pytest-version: 'pytest'
           - os: 'macos-latest'
-            python-version: '3.9'
+            python: '3.9'
+            pytest-version: 'pytest'
           - os: 'windows-latest'
-            python-version: '3.10'
+            python: '3.10'
+            pytest-version: 'pytest'
           - os: 'ubuntu-latest'
-            python-version: '3.11'
+            python: '3.11'
+            pytest-version: 'pytest'
           - os: 'ubuntu-latest'
-            python-version: '3.12'
+            python: '3.12'
+            pytest-version: 'pytest'
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-
-      - name: Use Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          path: ${{ env.special-working-directory-relative }}
 
-      - name: Install Python requirements
-        uses: brettcannon/pip-secure-install@v1
+      - name: Use Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
         with:
-          options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py'
+          python-version: ${{ matrix.python }}
 
-      - name: Install Jedi requirements
-        run: python scripts/vendor.py
+      - name: Install specific pytest version
+        if: matrix.pytest-version == 'pytest@pre-release'
+        run: |
+          python -m pip install --pre pytest
 
-      - name: Install test requirements
+      - name: Install specific pytest version
+        if: matrix.pytest-version != 'pytest@pre-release'
         run: |
-          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
-          python -m pip install --prefer-binary --upgrade -r build/pinned-test-requirements.txt
+          python -m pip install "${{ matrix.pytest-version }}"
 
-      - name: Run Pyright
-        uses: jakebailey/pyright-action@v1
+      - name: Install specific pytest version
+        run: python -m pytest --version
+      - name: Install base Python requirements
+        uses: brettcannon/pip-secure-install@v1
         with:
-          working-directory: 'pythonFiles'
+          requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"'
+          options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py'
 
-      - name: Run Positron unit tests
-        run: pytest pythonFiles/positron/
+      - name: Install test requirements
+        run: python -m pip install --upgrade -r build/test-requirements.txt
 
-      - name: Run Microsoft unit tests
+      - name: Run Python unit tests
         run: python pythonFiles/tests/run_all.py
 
+  positron-ipykernel-tests:
+    name: Test Positron IPyKernel
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: 'ubuntu-latest'
+            python: '3.8'
+          - os: 'macos-latest'
+            python: '3.9'
+          - os: 'windows-latest'
+            python: '3.10'
+          - os: 'ubuntu-latest'
+            python: '3.11'
+          - os: 'ubuntu-latest'
+            python: '3.12'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Use Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: 'pip'
+
+      - name: Install Positron IPyKernel requirements
+        run: python scripts/vendor.py
+
+      - name: Install Positron IPyKernel test requirements
+        run: python -m pip install --prefer-binary --upgrade -r pythonFiles/positron/pinned-test-requirements.txt
+
+      - name: Run Positron IPyKernel unit tests
+        run: pytest pythonFiles/positron
+
   typescript-tests:
     name: Test TypeScript
     runs-on: ${{ matrix.os }}
@@ -139,7 +236,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-            path: ${{ env.special-working-directory-relative }}
+          path: ${{ env.special-working-directory-relative }}
 
       - name: Install Node ${{ env.NODE_VERSION }}
         uses: actions/setup-node@v3
@@ -167,7 +264,7 @@ jobs:
         run: npx @vscode/l10n-dev@latest export ./src
 
       - name: Install test requirements
-        run: python -m pip install --upgrade -r ./build/pinned-test-requirements.txt
+        run: python -m pip install --upgrade -r ./build/test-requirements.txt
 
       - name: Install functional test requirements
         run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt

diff --git a/extensions/positron-python/build/constants.js b/extensions/positron-python/build/constants.js
@@ -11,5 +11,6 @@ exports.existingFiles = util.getListOfFiles('existingFiles.json');
 exports.contributedFiles = util.getListOfFiles('contributedFiles.json');
 exports.isWindows = /^win/.test(process.platform);
 // --- Start Positron ---
-exports.isCI = process.env.GITHUB_ACTIONS === 'true' || process.env.TRAVIS === 'true' || process.env.TF_BUILD !== undefined;
+exports.isCI =
+    process.env.GITHUB_ACTIONS === 'true' || process.env.TRAVIS === 'true' || process.env.TF_BUILD !== undefined;
 // --- End Positron ---
diff --git a/extensions/positron-python/build/pinned-test-requirements.txt b/extensions/positron-python/build/pinned-test-requirements.txt
diff --git a/extensions/positron-python/build/test-requirements.txt b/extensions/positron-python/build/test-requirements.txt
@@ -2,50 +2,20 @@
 setoptconf==0.3.0
 
 flake8
-# --- Start Positron ---
-# Require bandit >= 1.7.7 to avoid CWE-89
-bandit>=1.7.7
-# --- End Positron ---
+bandit
 pylint
 pycodestyle
 pydocstyle
-pygls
 prospector
 pytest
 flask
-# --- Start Positron ---
-# Require fastapi >= 0.109.1 to avoid CVE-2024-24762
-fastapi>=0.109.1
-# --- End Positron ---
+fastapi
 uvicorn
 django
 
 # Integrated TensorBoard tests
-# --- Start Positron ---
-# Require wheel >= 0.38.0 to avoid CVE-2022-40898
-wheel>=0.38.0
-# --- End Positron ---
 tensorboard
 torch-tb-profiler
 
 # extension build tests
 freezegun
-
-# --- Start Positron ---
-# iPython Kernel tests
-fastcore
-# Require fonttools >= 4.43.0 to avoid CVE-2023-45139
-fonttools>=4.43.0
-ipykernel
-ipywidgets
-matplotlib
-numpy
-pandas
-# Require pillow >= 10.0.1 to avoid CVE-2023-4863
-pillow>=10.0.1
-pyright
-pytest-asyncio
-pytest-mock
-polars
-torch; python_version < '3.12'
-# --- End Positron ---
diff --git a/extensions/positron-python/build/webpack/webpack.extension.browser.config.js b/extensions/positron-python/build/webpack/webpack.extension.browser.config.js
@@ -45,7 +45,7 @@ const nodeConfig = (_, { mode }) => ({
     externals: {
         vscode: 'commonjs vscode',
         // --- Start Positron ---
-        'positron': 'commonjs positron', // ignored because we inject positron via module loader
+        positron: 'commonjs positron', // ignored because we inject positron via module loader
         // --- End Positron ---
         // These dependencies are ignored because we don't use them, and App Insights has try-catch protecting their loading if they don't exist
         // See: https://github.com/microsoft/vscode-extension-telemetry/issues/41#issuecomment-598852991
@@ -54,7 +54,7 @@ const nodeConfig = (_, { mode }) => ({
         // --- Start Positron ---
         '@opentelemetry/instrumentation': 'commonjs @opentelemetry/instrumentation',
         '@azure/opentelemetry-instrumentation-azure-sdk': 'commonjs @azure/opentelemetry-instrumentation-azure-sdk',
-        '@azure/functions-core': 'commonjs @azure/functions-core'
+        '@azure/functions-core': 'commonjs @azure/functions-core',
         // --- End Positron ---
     },
     module: {

diff --git a/extensions/positron-python/package.json b/extensions/positron-python/package.json
@@ -1700,7 +1700,7 @@
         "node-loader": "^1.0.2",
         "node-polyfill-webpack-plugin": "^1.1.4",
         "nyc": "^15.0.0",
-        "prettier": "^2.0.2",
+        "prettier": "2.0.2",
         "rewiremock": "^3.13.0",
         "rimraf": "^3.0.2",
         "shortid": "^2.2.8",

diff --git a/extensions/positron-python/pythonFiles/positron/pinned-test-requirements.txt b/extensions/positron-python/pythonFiles/positron/pinned-test-requirements.txt
@@ -0,0 +1,14 @@
+fastcore==1.5.29
+ipykernel==6.29.3
+ipywidgets==8.1.2
+matplotlib==3.8.2; python_version >= '3.9'
+matplotlib==3.7.4; python_version < '3.9'
+numpy==1.26.3; python_version >= '3.9'
+numpy==1.24.4; python_version < '3.9'
+pandas==2.2.0; python_version >= '3.9'
+pandas==2.0.3; python_version < '3.9'
+pytest==8.0.2
+pytest-asyncio==0.23.5
+pytest-mock==3.12.0
+polars==0.20.13
+torch==2.1.2; python_version < '3.12'
diff --git a/extensions/positron-python/pythonFiles/positron/positron_ipykernel/data_explorer.py b/extensions/positron-python/pythonFiles/positron/positron_ipykernel/data_explorer.py
@@ -686,7 +686,7 @@ def _fire_schema_update(discard_state=False):
             msg = SchemaUpdateParams(discard_state=discard_state)
             comm.send_event(DataExplorerFrontendEvent.SchemaUpdate.value, msg.dict())
 
-        if type(new_table) is not type(table_view.table):
+        if type(new_table) is not type(table_view.table):  # noqa: E721
             # Data type has changed. For now, we will signal the UI to
             # reset its entire state: sorting keys, filters, etc. and
             # start over. At some point we can return here and

diff --git a/extensions/positron-python/pythonFiles/positron/positron_ipykernel/docstrings/epytext.py b/extensions/positron-python/pythonFiles/positron/positron_ipykernel/docstrings/epytext.py
@@ -3,7 +3,7 @@
 #
 
 import re
-from textwrap import dedent, indent
+from textwrap import dedent
 from typing import List
 
 # from markdown_to_docstring.google.ESCAPE_RULES