feat(repo): setup codspeed for benchmarking

.github/workflows/ci.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - "**"
 
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
@@ -156,3 +160,42 @@ jobs:
           else
             echo "Skip E2E tests for macOS as there are no changes in React Native projects."
           fi
+
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Fetch Master
+        run: git fetch origin master:master
+        if: ${{ github.event_name == 'pull_request' }}
+
+      - name: Set SHAs
+        uses: nrwl/nx-set-shas@v4
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ca-certificates lsof libvips-dev libglib2.0-dev libgirepository1.0-dev
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        with:
+          version: 9.8.0
+          run_install: false
+
+      - name: Install project dependencies
+        run: |
+          pnpm install --frozen-lockfile
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          run: "pnpm exec nx run-many -t benchmark"
+          token: ${{ secrets.CODSPEED_TOKEN }}

package.json

@@ -46,6 +46,7 @@
     "@babel/preset-react": "^7.22.5",
     "@babel/preset-typescript": "^7.22.5",
     "@babel/runtime": "^7.22.6",
+    "@codspeed/tinybench-plugin": "^4.0.0",
     "@eslint/compat": "^1.1.1",
     "@eslint/eslintrc": "^2.1.1",
     "@eslint/js": "^8.48.0",
@@ -377,6 +378,7 @@
     "tailwind-merge": "^2.4.0",
     "tailwindcss": "3.4.4",
     "three": "^0.166.1",
+    "tinybench": "^2.9.0",
     "tslib": "^2.3.0",
     "webpack-cli": "^5.1.4"
   },

packages/nx/project.json

@@ -105,6 +105,14 @@
     "echo": {
       "command": "echo hi"
     },
+    "benchmark": {
+      "command": "node -r ts-node/register --hash-seed=1 --random-seed=1 --no-opt --predictable --predictable-gc-schedule --interpreted-frames-native-stack --allow-natives-syntax --expose-gc --no-concurrent-sweeping ./tools/tinybench-runner {projectRoot}",
+      "options": {
+        "env": {
+          "TS_NODE_PROJECT": "./tools/tinybench-runner/tsconfig.json"
+        }
+      }
+    },
     "build": {
       "dependsOn": ["^build-client", "build-base", "build-native"],
       "inputs": [

packages/nx/src/command-line/run-many/run-many.benchmark.ts

@@ -0,0 +1,63 @@
+import type { Bench } from 'tinybench';
+
+import { projectsToRun } from './run-many';
+import { ProjectGraph } from '../../config/project-graph';
+
+let projectGraph: ProjectGraph = {
+  nodes: {
+    proj1: {
+      name: 'proj1',
+      type: 'lib',
+      data: {
+        root: 'proj1',
+        tags: ['api', 'theme1'],
+        targets: {
+          build: {},
+          test: {},
+        },
+      },
+    },
+    proj2: {
+      name: 'proj2',
+      type: 'lib',
+      data: {
+        root: 'proj2',
+        tags: ['ui', 'theme2'],
+        targets: {
+          test: {},
+        },
+      },
+    },
+  } as any,
+  dependencies: {},
+};
+
+for (let i = 0; i < 1000000; i++) {
+  projectGraph.nodes['proj' + i] = {
+    name: 'proj' + i,
+    type: 'lib',
+    data: {
+      root: 'proj' + i,
+      targets: {
+        test: {},
+      },
+    } as any,
+  };
+}
+
+export function registerBenchmarks(bench: Bench) {
+  return bench.add(
+    'should be able to select and exclude via patterns',
+    async () => {
+      projectsToRun(
+        {
+          targets: ['test'],
+          projects: ['proj1*'],
+          exclude: ['proj12*'],
+        },
+        projectGraph
+      );
+    },
+    {}
+  );
+}

packages/nx/tsconfig.benchmark.json

@@ -0,0 +1,17 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "CommonJS",
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "exclude": [
+    "**/*.spec.ts",
+    "**/*_spec.ts",
+    "jest.config.ts",
+    "**/__fixtures__/**/*.*",
+    "./src/internal-testing-utils/**/*.ts"
+  ],
+  "include": ["**/*.benchmark.ts"]
+}

packages/nx/tsconfig.lib.json

@@ -11,7 +11,8 @@
     "**/*_spec.ts",
     "jest.config.ts",
     "**/__fixtures__/**/*.*",
-    "./src/internal-testing-utils/**/*.ts"
+    "./src/internal-testing-utils/**/*.ts",
+    "**/*.benchmark.ts"
   ],
   "include": ["**/*.ts"]
 }

tools/tinybench-runner/index.ts

@@ -0,0 +1,48 @@
+import { join } from 'node:path';
+import { Bench, Fn } from 'tinybench';
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+
+import { globWithWorkspaceContext } from 'nx/src/utils/workspace-context';
+import { workspaceRoot } from '../../packages/nx/src/utils/workspace-root';
+
+export function findAllBenchmarks(root: string) {
+  return globWithWorkspaceContext(workspaceRoot, [
+    join(root, '**', '*.benchmark.ts'),
+  ]);
+}
+
+export async function runAllBenchmarks(root: string) {
+  const bench = withCodSpeed(new Bench());
+
+  const benchmarks = await findAllBenchmarks(root);
+
+  for (const benchmarkFile of benchmarks) {
+    const m = require(join('../../', benchmarkFile)) as {
+      registerBenchmarks?: (bench: Bench) => void;
+    };
+    if (m.registerBenchmarks) {
+      m.registerBenchmarks(bench);
+    } else {
+      throw new Error(`No benchmarks found in ${benchmarkFile}`);
+    }
+  }
+  const results = await bench.run();
+  console.table(bench.table());
+  const errors = results.map((s) => s.result?.error).filter(Boolean);
+  if (errors.length) {
+    throw new AggregateError(errors);
+  }
+}
+
+if (require.main === module) {
+  const root = process.argv[2] || '.';
+  runAllBenchmarks(root)
+    .then(() => {
+      console.log('All benchmarks completed.');
+      process.exit(0);
+    })
+    .catch((err) => {
+      console.error('Error running benchmarks:', err);
+      process.exit(1);
+    });
+}

tools/tinybench-runner/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "../../build/bench-out",
+    "module": "commonjs",
+    "types": ["node"]
+  },
+  "include": ["../../**/*.ts"]
+}