ci: test

Wu-kung · Oct 28, 2024 · cf69112 · cf69112
1 parent e4163cf
commit cf69112
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 102 deletions.
diff --git a/.github/workflows/bench_mako_pr.yml b/.github/workflows/bench_mako_pr.yml
@@ -48,104 +48,6 @@ jobs:
       - uses: ./.github/actions/build-mako
         with:
           path: ${{ env.MAKO_DIR }}
-      - name: Setup cargo cache
-        uses: actions/cache@v3
-        with:
-          # ref: https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
-          path: |
-            ~/.cargo/.crates.toml
-            ~/.cargo/.crates2.json
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: cargo-store-${{ hashFiles('**/Cargo.lock') }}      
-      - name: Run benchmark
-        run: cargo bench --bench with_examples -- --output-format bencher | tee output.txt
-
-      - name: Store benchmark result
-        uses: benchmark-action/github-action-benchmark@v1
-        with:
-          name: examples
-          tool: 'cargo'
-          output-file-path: output.txt
-          github-token: ${{ secrets.MAKO_BOT_ACCESS_TOKEN }}
-          alert-threshold: '150%'
-          external-data-json-path: ./benchmark-data/benchmark-data.json
-          summary-always: true
-          fail-on-alert: true
-          comment-on-alert: true
 
-      - name: Comment PR
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.MAKO_BOT_ACCESS_TOKEN }}
-          script: |
-            const { owner, repo } = context.repo;
-            const prNumber = context.payload.pull_request.number;
-            const cacheIdFile = `./comment-id`;
-
-            let data = '';
-            await exec.exec('node', ['-e', `
-              const fs = require('fs');
-              const path = require('path');
-              const summaryDir = path.dirname('${process.env.GITHUB_STEP_SUMMARY}');
-              let summary;
-
-              // read benchmark summary from prev step
-              fs.readdirSync(summaryDir)
-                .filter(file => file.startsWith('step_summary_'))
-                .some(file => {
-                  const content = fs.readFileSync(path.join(summaryDir, file), 'utf-8');
-
-                  if (content.includes('Benchmark')) {
-                    summary = content
-                      // add hyperlink for commit hash
-                      .replace(
-                        /((?:Current|Previous): )"(.{8})(.+?)"/g,
-                        '$1<a href="../tree/$2$3"><code>$2</code></a>',
-                      )
-                      // convert ns to ms for duration time
-                      .replace(
-                        /\`(\\d+)\`\\s+ns|\`(±\\s+)(\\d+)\`/g,
-                        (_, d1, s, d2) => \`\${s || ''}\${((d2 || d1)/10e6).toFixed(2)} ms\`,
-                      );
-                    return true;
-                  }
-                });
-
-              // read cached comment id
-              const commentId = fs.existsSync('${cacheIdFile}') ? fs.readFileSync('${cacheIdFile}', 'utf-8') : null;
-
-              console.log(JSON.stringify({
-                summary,
-                commentId,
-              }));
-            `], {
-              listeners: {
-                stdout: (chunk) => {
-                  data += chunk;
-                },
-              },
-            });
-            data = JSON.parse(data);
-
-            const { summary, commentId } = data;
-
-            if (commentId) {
-              await github.rest.issues.updateComment({
-                owner,
-                repo,
-                comment_id: commentId,
-                body: summary,
-              });
-            } else {
-              const { data: { id } } = await github.rest.issues.createComment({
-                owner,
-                repo,
-                issue_number: prNumber,
-                body: summary,
-              });
-              await exec.exec('node', ['-e', `require('fs').writeFileSync('${cacheIdFile}', '${id}', 'utf-8')`]);
-            }
+      - name: Run benchmark
+        run: node bin/cli.js bench
diff --git a/bin/cli.js b/bin/cli.js
@@ -1,4 +1,4 @@
-import { mkdir, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
 import { join } from "node:path";
 import meow from "meow";
 import { $, cd } from "zx";
@@ -60,7 +60,32 @@ if (!command || command === "build") {
 
   await $`pnpm --version`;
   await $`pnpm install --prefer-frozen-lockfile --prefer-offline`;
-  // await $`pnpm run release:mako`;
+  await $`cargo build --release`;
 
   cd(cwd);
 }
+
+if (!command || command === "bench") {
+  cd(makoDirectory);
+
+  const baselineHash = (await $`git rev-parse --short HEAD`).stdout.trim();
+  const baselineMakoPath = `./tmp/mako-${baselineHash}`;
+  if (!existsSync(join(__dirname, `../tmp/mako-${baselineHash}`))) {
+    if (shouldBuild) {
+      await $`cargo build --release`;
+      await $`cp target/release/mako ${baselineMakoPath}`;
+    } else {
+      console.log(`Since --no-build is set, build for baseline is skipped.`);
+    }
+  }
+
+  let currentMakoPath = "./target/release/mako";
+  const currentHash = (await $`git rev-parse --short HEAD`).stdout.trim();
+  const makoCurrentName = `mako-${currentHash}`;
+  await $`cp target/release/mako ./tmp/${makoCurrentName}`;
+  currentMakoPath = `./tmp/${makoCurrentName}`;
+
+  const warmup = argv.warmup || 3;
+  const runs = argv.runs || 10;
+  await $`hyperfine --warmup ${warmup} --runs ${runs} "${currentMakoPath} ${casePath} --mode production" "${baselineMakoPath} ${casePath} --mode production"`;
+}