From 96215868b9e82614f78b2cb3dbe1edd156df512d Mon Sep 17 00:00:00 2001
From: Atanas Dimitrov <70822030+neNasko1@users.noreply.github.com>
Date: Fri, 3 Jan 2025 00:10:35 +0200
Subject: [PATCH] Run Array API tests weekly (#88)

Co-authored-by: Aditya Goel <48102515+adityagoel4512@users.noreply.github.com>
---
 .github/workflows/array-api.yml | 34 +++++++++++++++++++++++++++++++--
 README.md                       |  2 +-
 pixi.toml                       |  2 +-
 3 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/array-api.yml b/.github/workflows/array-api.yml
index fefe240..8354195 100644
--- a/.github/workflows/array-api.yml
+++ b/.github/workflows/array-api.yml
@@ -1,5 +1,9 @@
 name: Array API coverage tests
-on: [push]
+
+on:
+  push: # Run on all branches
+  schedule:
+    - cron: "0 8 * * 0" # Runs weekly at 08:00 UTC on Sunday
 
 # Automatically stop old builds on the same branch/PR
 concurrency:
@@ -23,10 +27,36 @@ jobs:
         uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659
       - name: Install repository
         run: pixi run postinstall
-      - name: Run Array API tests
+      - name: Run Array API tests (Scheduled)
+        if: ${{ github.event_name == 'schedule' && github.ref == 'refs/heads/main' }}
         run: pixi run arrayapitests
+      - name: Run Array API tests (Push)
+        if: ${{ github.event_name == 'push' }}
+        run: pixi run arrayapitests --max-examples 16 --hypothesis-seed=0
       - name: Upload Array API tests report
         uses: actions/upload-artifact@v4
         with:
           name: api-coverage-tests
           path: api-coverage-tests.json
+      - name: Issue on failure
+        uses: actions/github-script@v7
+        if: ${{ failure() && github.event_name == 'schedule' && github.ref == 'refs/heads/main' }}
+        with:
+          script: |
+            github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: "open",
+              labels: "[bot] Weekly run"
+            }).then((issues) => {
+              if (issues.data.length === 0){
+                github.rest.issues.create({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  title: "Weekly run failure: Array API coverage",
+                  body: "The weekly run of the Array API test suite failed. See https://github.com/Quantco/ndonnx/actions/runs/${{ github.run_id }} for details.",
+                  assignees: ["adityagoel4512", "cbourjau", "neNasko1"],
+                  labels: ["[bot] Weekly run"]
+                })
+              }
+            });
diff --git a/README.md b/README.md
index cc93e55..cb0d52a 100644
--- a/README.md
+++ b/README.md
@@ -104,5 +104,5 @@ Summary(1119 total):
 Run the tests with:
 
 ```bash
-pixi run arrayapitests
+pixi run arrayapitests --max-examples 16 --hypothesis-seed=0
 ```
diff --git a/pixi.toml b/pixi.toml
index 0aee9f7..3a16c4b 100644
--- a/pixi.toml
+++ b/pixi.toml
@@ -49,7 +49,7 @@ test = "pytest"
 test-coverage = "pytest --cov=ndonnx --cov-report=xml --cov-report=term-missing"
 
 [feature.test.tasks.arrayapitests]
-cmd = "pytest --max-examples 16 api-coverage-tests/array_api_tests/  -v -rfX --json-report --json-report-file=api-coverage-tests.json -n auto --disable-deadline --disable-extension linalg --skips-file=skips.txt --xfails-file=xfails.txt --hypothesis-seed=0"
+cmd = "pytest api-coverage-tests/array_api_tests/ -v -rfX --json-report --json-report-file=api-coverage-tests.json -n auto --disable-deadline --disable-extension linalg --skips-file=skips.txt --xfails-file=xfails.txt"
 [feature.test.tasks.arrayapitests.env]
 ARRAY_API_TESTS_MODULE = "ndonnx"
 ARRAY_API_TESTS_VERSION = "2023.12"