Run Array API tests weekly (#88)

Co-authored-by: Aditya Goel <48102515+adityagoel4512@users.noreply.github.com>
Quantco · Jan 2, 2025 · 9621586 · 9621586
1 parent 210251c
commit 9621586
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 4 deletions.
diff --git a/.github/workflows/array-api.yml b/.github/workflows/array-api.yml
@@ -1,5 +1,9 @@
 name: Array API coverage tests
-on: [push]
+
+on:
+  push: # Run on all branches
+  schedule:
+    - cron: "0 8 * * 0" # Runs weekly at 08:00 UTC on Sunday
 
 # Automatically stop old builds on the same branch/PR
 concurrency:
@@ -23,10 +27,36 @@ jobs:
         uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659
       - name: Install repository
         run: pixi run postinstall
-      - name: Run Array API tests
+      - name: Run Array API tests (Scheduled)
+        if: ${{ github.event_name == 'schedule' && github.ref == 'refs/heads/main' }}
         run: pixi run arrayapitests
+      - name: Run Array API tests (Push)
+        if: ${{ github.event_name == 'push' }}
+        run: pixi run arrayapitests --max-examples 16 --hypothesis-seed=0
       - name: Upload Array API tests report
         uses: actions/upload-artifact@v4
         with:
           name: api-coverage-tests
           path: api-coverage-tests.json
+      - name: Issue on failure
+        uses: actions/github-script@v7
+        if: ${{ failure() && github.event_name == 'schedule' && github.ref == 'refs/heads/main' }}
+        with:
+          script: |
+            github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: "open",
+              labels: "[bot] Weekly run"
+            }).then((issues) => {
+              if (issues.data.length === 0){
+                github.rest.issues.create({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  title: "Weekly run failure: Array API coverage",
+                  body: "The weekly run of the Array API test suite failed. See https://github.com/Quantco/ndonnx/actions/runs/${{ github.run_id }} for details.",
+                  assignees: ["adityagoel4512", "cbourjau", "neNasko1"],
+                  labels: ["[bot] Weekly run"]
+                })
+              }
+            });
diff --git a/README.md b/README.md
@@ -104,5 +104,5 @@ Summary(1119 total):
 Run the tests with:
 
 ```bash
-pixi run arrayapitests
+pixi run arrayapitests --max-examples 16 --hypothesis-seed=0
 ```
diff --git a/pixi.toml b/pixi.toml
@@ -49,7 +49,7 @@ test = "pytest"
 test-coverage = "pytest --cov=ndonnx --cov-report=xml --cov-report=term-missing"
 
 [feature.test.tasks.arrayapitests]
-cmd = "pytest --max-examples 16 api-coverage-tests/array_api_tests/  -v -rfX --json-report --json-report-file=api-coverage-tests.json -n auto --disable-deadline --disable-extension linalg --skips-file=skips.txt --xfails-file=xfails.txt --hypothesis-seed=0"
+cmd = "pytest api-coverage-tests/array_api_tests/ -v -rfX --json-report --json-report-file=api-coverage-tests.json -n auto --disable-deadline --disable-extension linalg --skips-file=skips.txt --xfails-file=xfails.txt"
 [feature.test.tasks.arrayapitests.env]
 ARRAY_API_TESTS_MODULE = "ndonnx"
 ARRAY_API_TESTS_VERSION = "2023.12"