diff --git a/.github/workflows/bench_mako_pr.yml b/.github/workflows/bench_mako_pr.yml index 0ecbe92..6732824 100644 --- a/.github/workflows/bench_mako_pr.yml +++ b/.github/workflows/bench_mako_pr.yml @@ -31,52 +31,51 @@ jobs: }) return comment.id - # bench: - # runs-on: ubuntu-latest - # needs: [create-comment] - # strategy: - # matrix: - # shardIndex: [1] - # shardTotal: [1] - # fail-fast: false - # steps: - # - uses: actions/checkout@v4 - # - name: Init env - # uses: ./.github/actions/env - # - uses: ./.github/actions/build-mako - # with: - # path: ${{ env.MAKO_DIR }} - - # - name: Install hyperfine - # run: sudo apt-get update && sudo apt-get install -y hyperfine - - # - name: Run benchmark - # run: node bin/cli.js bench - - # - name: List files - # run: ls -la - - # - name: Upload benchmark results - # uses: actions/upload-artifact@v4 - # with: - # name: benchmark-results - # path: ./benchmark-results.md - # if-no-files-found: error + bench: + runs-on: ubuntu-latest + needs: [create-comment] + strategy: + matrix: + shardIndex: [1] + shardTotal: [1] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: Init env + uses: ./.github/actions/env + - uses: ./.github/actions/build-mako + with: + path: ${{ env.MAKO_DIR }} + + - name: Install hyperfine + run: sudo apt-get update && sudo apt-get install -y hyperfine + + - name: Run benchmark + run: node bin/cli.js bench + + - name: List files + run: ls -la + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: ./benchmark-results.md + if-no-files-found: error comment-results: runs-on: ubuntu-latest - # needs: [create-comment, bench] - needs: [create-comment] + needs: [create-comment, bench] if: always() steps: - uses: actions/checkout@v4 - name: Check current working directory run: pwd - # - name: Download benchmark results - # uses: actions/download-artifact@v4 - # with: - # name: benchmark-results - # path: ./ + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: benchmark-results + path: ./ - name: List files run: ls -la diff --git a/benchmark-results.md b/benchmark-results.md deleted file mode 100644 index a28a711..0000000 --- a/benchmark-results.md +++ /dev/null @@ -1,4 +0,0 @@ -| Command | Mean [s] | Min [s] | Max [s] | Relative | -|:---|---:|---:|---:|---:| -| `./tmp/mako-8a2f4d02 examples/with-antd --mode production` | 3.539 ± 0.050 | 3.473 | 3.599 | 1.03 ± 0.02 | -| `./tmp/mako-8a2f4d02 examples/with-antd --mode production` | 3.448 ± 0.024 | 3.399 | 3.475 | 1.00 | \ No newline at end of file diff --git a/lib/bench.js b/lib/bench.js deleted file mode 100644 index ae4d783..0000000 --- a/lib/bench.js +++ /dev/null @@ -1,46 +0,0 @@ -import path from "path"; -import { fileURLToPath } from "url"; -import { getScenario } from "./scenarios/index.js"; -import { useAddons, dirExist } from "./index.js"; - -const dirname = path.resolve(fileURLToPath(import.meta.url), ".."); - -export async function run(benchmarkName) { - const [caseName, ...addonNames] = benchmarkName.split("_"); - const scenario = getScenario(caseName); - const addons = await Promise.all( - addonNames.map(async name => { - const hasDir = await dirExist(path.resolve(dirname, `./addons/${name}/`)); - const Addon = hasDir - ? await import(`./addons/${name}/index.js`) - : await import(`./addons/${name}.js`); - return new Addon.default(); - }) - ); - - await useAddons(addons, "beforeSetup"); - const ctx = await scenario.setup(); - await useAddons(addons, "afterSetup", ctx); - - try { - await useAddons(addons, "beforeGenerate", ctx); - await scenario.generate(ctx); - await useAddons(addons, "afterGenerate", ctx); - - // warmup - await scenario.warmup(ctx); - - // run - await scenario.run(ctx); - - await useAddons(addons, "beforeStatistic", ctx); - await scenario.statistic(ctx); - await useAddons(addons, "afterStatistic", ctx); - } finally { - await useAddons(addons, "beforeTeardown", ctx); - await scenario.teardown(ctx); - await useAddons(addons, "afterTeardown", ctx); - } - - return ctx.result; -} diff --git a/lib/compare.js b/lib/compare.js deleted file mode 100644 index e69de29..0000000 diff --git a/lib/index.js b/lib/index.js index 72dc4f7..d418892 100644 --- a/lib/index.js +++ b/lib/index.js @@ -7,9 +7,3 @@ export async function dirExist(p) { return false; } } - -export async function useAddons(addons, stage, ...args) { - for (const item of addons) { - await item[stage](...args); - } -} diff --git a/lib/scenarios/build-plugin.cjs b/lib/scenarios/build-plugin.cjs deleted file mode 100644 index c2d08c5..0000000 --- a/lib/scenarios/build-plugin.cjs +++ /dev/null @@ -1,79 +0,0 @@ -const WARMUP_BUILDS = 10; -const TOTAL_BUILDS = 20; - -module.exports = class BuildPlugin { - apply(compiler) { - let counter = 0; - let isWatching = false; - compiler.hooks.watchRun.tap("BuildPlugin", () => { - isWatching = true; - }); - (compiler.hooks.afterDone || compiler.hooks.done).tap("BuildPlugin", () => { - setTimeout(() => { - if (counter === WARMUP_BUILDS) console.log("#!# start"); - if (isWatching && counter <= TOTAL_BUILDS) console.log("#!# next"); - }, 10); - }); - compiler.hooks.done.tap("BuildPlugin", stats => { - if (isWatching) { - counter++; - if (counter <= WARMUP_BUILDS) return; - if (counter > TOTAL_BUILDS) { - if (compiler.watching) { - compiler.watching.close(); - } else { - process.nextTick(() => process.exit(0)); - } - } - } - const { logging, time } = stats.toJson({ - all: false, - timings: true, - logging: "verbose" - }); - console.log(`#!# stats = ${time}`); - const memoryUsage = process.memoryUsage(); - console.log(`#!# heap memory = ${memoryUsage.heapUsed}`); - console.log(`#!# rss memory = ${memoryUsage.rss}`); - console.log(`#!# external memory = ${memoryUsage.external}`); - console.log(`#!# array buffers memory = ${memoryUsage.arrayBuffers}`); - for (const _name of Object.keys(logging)) { - const { entries } = logging[_name]; - const name = _name.replace(/^rspack\./, ""); - for (const { type, args, message } of entries) { - if (type === "time") { - if (args) { - const ms = args[1] * 1000 + args[2] / 1000000; - console.log(`#!# ${name}.${args[0]} = ${ms}`); - } else { - const [, label, msStr] = /^(.+): ([\d.]+) ms$/.exec(message); - console.log(`#!# ${name}.${label} = ${msStr}`); - } - } else if (type === "cache") { - if (args) { - const ratio = (args[1] / args[2]) * 100; - console.log(`#!# ${name}.${args[0]} = ${ratio}`); - } else { - const [, label, ratio] = - /^(.+): ([\d.]+)% \(([\d.\/]+)\/([\d.\/]+)\)$/.exec(message); - console.log(`#!# ${name}.${label} = ${ratio}`); - } - } - } - } - }); - const old = compiler.infrastructureLogger; - compiler.infrastructureLogger = (_name, type, args) => { - old(_name, type, args); - const name = _name.replace(/^webpack\./, ""); - if (type !== "time") return; - const ms = args[1] * 1000 + args[2] / 1000000; - console.log( - `#!# ${name}.${args[0].replace( - /restore cache content \d.+$/, - "restore cache content" - )} = ${ms}` - ); - }; - } -}; diff --git a/lib/scenarios/index.js b/lib/scenarios/index.js deleted file mode 100644 index 0f014ec..0000000 --- a/lib/scenarios/index.js +++ /dev/null @@ -1,156 +0,0 @@ -import path from "path"; -import { readFile, unlink, writeFile } from "fs/promises"; -import { fileURLToPath } from "url"; -import actionsCore from "@actions/core"; -import { isGitHubActions, runCommand } from "../utils.js"; -import { - getDirSizes, - calcStatistics, - clearCaches, - getHmrConfig -} from "./utils.js"; - -const rootDir = path.resolve(fileURLToPath(import.meta.url), "../../.."); - -async function runMako(ctx) { - let start = Date.now(); - - let counter = 0; - let dataSetCounter = -1; - let promise = Promise.resolve(); - const data = {}; - const processLine = line => { - if (line === "#!# start") { - start = Date.now(); - dataSetCounter = 0; - } else if (line === "#!# next") { - promise = promise.then(async () => { - counter++; - if (dataSetCounter >= 0) dataSetCounter++; - await new Promise(r => setTimeout(r, Math.max(300, 1000 / counter))); - for (const item of ctx.hmrConfig) { - const content = item.generateContent( - ctx.originalFiles[item.rebuildChangeFile], - counter - ); - await writeFile(item.rebuildChangeFile, content); - } - }); - } else if (line.startsWith("#!#")) { - const [, name, valueStr] = /^#!# (.+) = ((\d|\.|e|-)+)$/.exec(line); - data[name] = (data[name] || 0) + +valueStr; - } - }; - let remainingLine = ""; - await runCommand( - path.join(rootDir, "node_modules/@rspack/cli/bin/rspack"), - ctx.rspackArgs, - { - verbose: false, - onData: function (chunk) { - const lines = (remainingLine + chunk).split("\n"); - remainingLine = lines.pop(); - lines.forEach(processLine); - } - } - ); - - data.exec = Date.now() - start; - await promise; - if (dataSetCounter > 1) { - for (const key of Object.keys(data)) { - data[key] /= dataSetCounter; - } - } - data["dist size"] = await getDirSizes("dist"); - return data; -} - -export function getScenario(caseName) { - return { - async setup() { - const caseDir = path.resolve(rootDir, "cases", caseName); - process.chdir(caseDir); - const configFilePath = path.resolve(caseDir, "rspack.config.js"); - const config = await readFile(configFilePath); - const hmrConfig = await getHmrConfig(path.resolve(caseDir, "hmr.js")); - return { - caseDir, - originalFiles: { - [configFilePath]: config, - ...hmrConfig.originalFiles - }, - config: `${config} -module.exports.plugins = module.exports.plugins || []; -module.exports.plugins.push(new (require("../../lib/scenarios/build-plugin.cjs"))());`, - hmrConfig: hmrConfig.config, - rspackArgs: [], - runTimes: 10, - timeout: 5 * 60 * 1000, - runData: [], - result: {} - }; - }, - async generate(ctx) { - if (isGitHubActions) { - actionsCore.startGroup("Generating rspack configuration:"); - console.log(ctx.config); - actionsCore.endGroup(); - } - - await writeFile( - path.resolve(ctx.caseDir, "rspack.config.js"), - ctx.config - ); - const rspackDir = - process.env.RSPACK_DIR || path.resolve(rootDir, ".rspack"); - console.log("Create Rspack package link"); - await runCommand("mkdir", [ - "-p", - path.resolve(rootDir, "node_modules/@rspack") - ]); - await runCommand("ln", [ - "-nsf", - path.resolve(rspackDir, "packages/rspack"), - path.resolve(rootDir, "node_modules/@rspack/core") - ]); - await runCommand("ln", [ - "-nsf", - path.resolve(rspackDir, "packages/rspack-cli"), - path.resolve(rootDir, "node_modules/@rspack/cli") - ]); - }, - async warmup(ctx) { - console.log("Run Rspack with args:", ctx.rspackArgs); - await runMako({ - ...ctx, - rspackArgs: ctx.rspackArgs.filter(a => a !== "--watch") - }); - }, - async run(ctx) { - const start = Date.now(); - for (let i = 0; i < ctx.runTimes; i++) { - await clearCaches(ctx.caseDir); - - ctx.runData.push(await runMako(ctx)); - - const runtime = Date.now() - start; - if (runtime > ctx.timeout) break; - } - }, - async statistic(ctx) { - ctx.result = calcStatistics(ctx.runData); - }, - async teardown(ctx) { - Promise.all( - Object.entries(ctx.originalFiles).map(async ([path, content]) => { - if (content == null) { - return unlink(path); - } - return writeFile(path, content); - }) - ); - process.chdir(rootDir); - } - }; -} diff --git a/lib/scenarios/utils.js b/lib/scenarios/utils.js deleted file mode 100644 index 6fcae08..0000000 --- a/lib/scenarios/utils.js +++ /dev/null @@ -1,103 +0,0 @@ -import * as fs from "fs/promises"; -import { resolve, join } from "path"; - -function rmdir(p) { - if (fs.rm) { - return fs.rm(p, { - force: true, - maxRetries: 10, - recursive: true - }); - } else { - return fs.rmdir(p, { - maxRetries: 10, - recursive: true - }); - } -} -export async function clearCaches(directory) { - const paths = ["node_modules/.cache", ".cache", "dist"]; - await Promise.all(paths.map(p => rmdir(resolve(directory, p)))); -} - -export async function getDirSizes(dir) { - let totalSize = 0; - const dirContents = await fs.readdir(dir, { withFileTypes: true }); - for (const dirent of dirContents) { - if (dirent.isDirectory()) { - totalSize += await getDirSizes(join(dir, dirent.name)); - } else { - const fileStats = await fs.stat(join(dir, dirent.name)); - totalSize += fileStats.size; - } - } - return totalSize; -} - -export async function getHmrConfig(filePath) { - const config = (await import(filePath)).default; - const originalFiles = {}; - await Promise.all( - config.map(async item => { - item.rebuildChangeFile = resolve(item.rebuildChangeFile); - const content = await fs.readFile(item.rebuildChangeFile); - originalFiles[item.rebuildChangeFile] = content; - }) - ); - return { - originalFiles, - config - }; -} - -const T_TABLE = [ - 12.71, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, - 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, - 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042 -]; -const tDist95Two = n => { - if (n <= 1) return 12.71; - if (n <= 30) return T_TABLE[n - 1]; - if (n <= 40) return 2.021; - if (n <= 50) return 2.009; - if (n <= 60) return 2.0; - if (n <= 80) return 1.99; - if (n <= 100) return 1.984; - if (n <= 120) return 1.98; - return 1.96; -}; -export function calcStatistics(data) { - const stats = {}; - for (const key of Object.keys(data[0])) { - const values = data.map(r => r[key] || 0); - if (typeof values[0] === "object") { - stats[key] = calcStatistics(values); - } else { - values.sort(); - const mean = values.reduce((sum, v) => sum + v, 0) / values.length; - const variance = - values.reduce((sum, v) => sum + (mean - v) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); - const confidence = - (tDist95Two(values.length - 1) * stdDev) / Math.sqrt(values.length); - const low = Math.max(0, mean - confidence); - const high = mean + confidence; - stats[key] = { - min: Math.min(...values), - max: Math.max(...values), - mean, - median: - values.length % 2 === 0 - ? (values[values.length / 2 - 1] + values[values.length / 2]) / 2 - : values[(values.length - 1) / 2], - variance, - stdDev, - confidence: high - low, - low, - high, - count: values.length - }; - } - } - return stats; -} diff --git a/lib/utils.js b/lib/utils.js deleted file mode 100644 index 8883062..0000000 --- a/lib/utils.js +++ /dev/null @@ -1,29 +0,0 @@ -import { spawn } from "child_process"; - -export const isGitHubActions = !!process.env.GITHUB_ACTIONS; - -export async function runCommand( - command, - args, - { verbose = true, env, onData } = {} -) { - const hasOnData = typeof onData === "function"; - const stdio = verbose ? "inherit" : "ignore"; - const p = spawn(command, args, { - shell: true, - stdio: [stdio, hasOnData ? "pipe" : stdio, "inherit"], - env: env - ? { - ...process.env, - ...env - } - : undefined - }); - if (hasOnData) { - p.stdout.on("data", onData); - } - - const exitCode = await new Promise(resolve => p.once("exit", resolve)); - if (exitCode !== 0) - throw new Error(`${command} ${args.join(" ")} failed with ${exitCode}`); -} diff --git a/package.json b/package.json index 000640e..f478199 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "name": "mako-ecosystem-benchmark", "type": "module", "dependencies": { - "@actions/core": "^1.11.1", "meow": "^13.2.0", "zx": "^8.1.9" },