Add code coverage to CI (#655)

* Add code coverage to CI

* Exclude framework-specific directories

* Add coverage tracking to full workflows

* Add [toml] dependency because 3.9 doesn't have builtin toml

* Trying to install coverage correctly..

* Add coverage explicitly

* Remove toml optional dependnecy again

* Add tomli as explicit dependency instead

* Revert "Add tomli as explicit dependencyt instead"

This reverts commit a3590c9.

* Add pip list output for debugging purposes

* Explicitly call coverage as Python module

* Cache entire venv instead

.github/workflows/run_all_frameworks.yml

@@ -33,7 +33,7 @@ jobs:
           ".github/runbenchmark/action.yml "\
           "runbenchmark.py "\
           "requirements.txt"
-          
+
           echo Common files: $common_files
           echo Changed files: $changed_files
 
@@ -100,7 +100,7 @@ jobs:
       - uses: actions/cache@v3
         id: cache
         with:
-          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
+          path: /home/runner/work/automlbenchmark/automlbenchmark/venv
           key: pip-v3-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             pip-v3-
@@ -109,12 +109,18 @@ jobs:
         run: |
           source venv/bin/activate
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip show openml
+          python -m pip install -r requirements.txt
+          python -m pip install "coverage[toml]"
+          python -m pip show openml
       - name: Run constantpredictor on openml iris
         run: |
           source venv/bin/activate
-          python runbenchmark.py ${{ matrix.framework }} -t ${{ matrix.task }} -f 0 -e
+          coverage run -m runbenchmark ${{ matrix.framework }} -t ${{ matrix.task }} -f 0 -e
+          coverage xml
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   run_frameworks:
     name: ${{ matrix.framework }}/${{ matrix.task }}
@@ -143,7 +149,7 @@ jobs:
     - uses: actions/cache@v3
       id: cache
       with:
-        path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
+        path: /home/runner/work/automlbenchmark/automlbenchmark/venv
         key: pip-v3-${{ hashFiles('**/requirements.txt') }}
         restore-keys: |
           pip-v3-
@@ -153,9 +159,15 @@ jobs:
         source venv/bin/activate
         python -m pip install --upgrade pip
         python -m pip install -r requirements.txt
+        python -m pip install "coverage[toml]"
     - name: Run ${{ matrix.framework }} on ${{ matrix.task }}
       run: |
         source venv/bin/activate
-        python runbenchmark.py ${{ matrix.framework }} ${{ matrix.benchmark }} test -f 0 -t ${{ matrix.task }} -e
+        coverage run -m runbenchmark ${{ matrix.framework }} ${{ matrix.benchmark }} test -f 0 -t ${{ matrix.task }} -e
+        coverage xml
       env:
         GITHUB_PAT: ${{ secrets.PUBLIC_ACCESS_GITHUB_PAT }}
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/run_pytest.yaml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/cache@v3
         id: cache
         with:
-          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
+          path: /home/runner/work/automlbenchmark/automlbenchmark/venv
           key: pip-v3-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-dev.txt') }}
           restore-keys: |
             pip-v3-
@@ -32,4 +32,9 @@ jobs:
       - name: Run unit tests
         run: |
           source venv/bin/activate
-          python -m pytest -vv -m "not stress and not openml_upload"
+          coverage run -m pytest -vv -m "not stress and not openml_upload"
+          coverage xml
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

pyproject.toml

@@ -9,6 +9,13 @@ disable_error_code = "attr-defined"
 #  problem with ruamel: https://github.com/python/mypy/issues/12664
 incremental = false
 
+[tool.coverage.run]
+include=[
+    "tests/**",
+    "amlb/**",
+    "frameworks/shared/**"
+]
+
 # 3rd party libraries without type hints or stubs:
 [[tool.mypy.overrides]]
 module=[
@@ -23,4 +30,4 @@ module = "amlb.benchmark"
 disable_error_code = "union-attr"
 # special run-mode script makes some attributes optional,
 # TODO: refactor so that recovery script does not need special runtime or that it doesnt
-# abuse the benchmark class.
+# abuse the benchmark class.

requirements-dev.txt

@@ -1,6 +1,7 @@
 pytest
 pytest-mock
 pip-tools
+coverage[toml]
 
 types-psutil
 types-xmltodict