From 965b9c8cc0beeb8dea640d8dbf6e81b05413c2ed Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 1 Feb 2025 15:44:29 +0100 Subject: [PATCH 1/3] Log fail test to file went running tests locally --- .gitignore | 2 ++ .vscode/launch.json | 4 +++ packages/test-harness/src/runAllTests.ts | 32 ++++++++++++++++++------ packages/test-harness/src/testSubset.ts | 22 ++++++++++++++++ 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 064fef48ed..a227de4272 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ next-env.d.ts # test subset config packages/test-harness/testSubsetGrep.properties +packages/test-harness/failedTests.properties + # cursorless-neovim cursorless.nvim/node/cursorless-neovim diff --git a/.vscode/launch.json b/.vscode/launch.json index 276b40d8fb..a7a87c13da 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -31,6 +31,7 @@ "request": "launch", "env": { "CURSORLESS_MODE": "test", + "CURSORLESS_LOG_FAILED": "true", "CURSORLESS_REPO_ROOT": "${workspaceFolder}" }, "args": [ @@ -52,6 +53,7 @@ "env": { "CURSORLESS_MODE": "test", "CURSORLESS_RUN_TEST_SUBSET": "true", + "CURSORLESS_LOG_FAILED": "true", "CURSORLESS_REPO_ROOT": "${workspaceFolder}" }, "args": [ @@ -136,6 +138,7 @@ "program": "${workspaceFolder}/packages/test-harness/dist/runTalonTests.cjs", "env": { "CURSORLESS_MODE": "test", + "CURSORLESS_LOG_FAILED": "true", "CURSORLESS_REPO_ROOT": "${workspaceFolder}" }, "outFiles": ["${workspaceFolder}/**/out/**/*.js"], @@ -171,6 +174,7 @@ "program": "${workspaceFolder}/packages/test-harness/dist/runTalonJsTests.cjs", "env": { "CURSORLESS_MODE": "test", + "CURSORLESS_LOG_FAILED": "true", "CURSORLESS_REPO_ROOT": "${workspaceFolder}" }, "outFiles": ["${workspaceFolder}/**/out/**/*.js"], diff --git a/packages/test-harness/src/runAllTests.ts b/packages/test-harness/src/runAllTests.ts index 3473980aa3..ae4da9060d 100644 --- a/packages/test-harness/src/runAllTests.ts +++ b/packages/test-harness/src/runAllTests.ts @@ -1,8 +1,13 @@ -import Mocha from "mocha"; -import * as path from "node:path"; import { getCursorlessRepoRoot } from "@cursorless/node-common"; -import { runTestSubset, testSubsetGrepString } from "./testSubset"; import { glob } from "glob"; +import Mocha from "mocha"; +import * as path from "node:path"; +import { + logFailedTests, + runTestSubset, + shouldLogFailedTests, + testSubsetGrepString, +} from "./testSubset"; /** * Type of test to run, eg unit, vscode, talon @@ -24,7 +29,7 @@ export enum TestType { neovim, } -export function runAllTests(...types: TestType[]) { +export function runAllTests(...types: TestType[]): Promise { return runTestsInDir( path.join(getCursorlessRepoRoot(), "packages"), (files) => @@ -68,14 +73,25 @@ async function runTestsInDir( try { // Run the mocha test - await new Promise((c, e) => { - mocha.run((failures) => { + await new Promise((resolve, reject) => { + const failedTests: string[] = []; + + const runner = mocha.run((failures) => { if (failures > 0) { - e(new Error(`${failures} tests failed.`)); + if (shouldLogFailedTests()) { + logFailedTests(failedTests); + } + reject(`${failures} tests failed.`); } else { - c(); + resolve(); } }); + + if (shouldLogFailedTests()) { + runner.on("fail", (test) => { + failedTests.push(test.fullTitle()); + }); + } }); } catch (err) { console.error(err); diff --git a/packages/test-harness/src/testSubset.ts b/packages/test-harness/src/testSubset.ts index 7782e2ad58..6cb496c9ef 100644 --- a/packages/test-harness/src/testSubset.ts +++ b/packages/test-harness/src/testSubset.ts @@ -29,6 +29,20 @@ export function testSubsetFilePath() { ); } +function testFailedFilePath() { + return path.join( + getCursorlessRepoRoot(), + "packages", + "test-harness", + "failedTests.properties", + ); +} + +export function logFailedTests(testNames: string[]) { + const lines = [`${testNames.length} failed tests`, "", ...testNames]; + fs.writeFileSync(testFailedFilePath(), lines.join("\n")); +} + /** * Determine whether we should run just the subset of the tests specified by * {@link TEST_SUBSET_GREP_STRING}. @@ -37,3 +51,11 @@ export function testSubsetFilePath() { export function runTestSubset() { return process.env.CURSORLESS_RUN_TEST_SUBSET === "true"; } + +/** + * Determine whether we should log the failed tests. + * @returns `true` if we should log the failed tests + */ +export function shouldLogFailedTests() { + return process.env.CURSORLESS_LOG_FAILED === "true"; +} From f8c4a7d64bd125c4d8581eff39c57d5f79af4d49 Mon Sep 17 00:00:00 2001 From: Phil Cohen Date: Sat, 1 Feb 2025 10:35:53 -0800 Subject: [PATCH 2/3] Update packages/test-harness/src/runAllTests.ts --- packages/test-harness/src/runAllTests.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/test-harness/src/runAllTests.ts b/packages/test-harness/src/runAllTests.ts index ae4da9060d..72061ae3de 100644 --- a/packages/test-harness/src/runAllTests.ts +++ b/packages/test-harness/src/runAllTests.ts @@ -88,9 +88,7 @@ async function runTestsInDir( }); if (shouldLogFailedTests()) { - runner.on("fail", (test) => { - failedTests.push(test.fullTitle()); - }); + runner.on("fail", (test) => failedTests.push(test.fullTitle())); } }); } catch (err) { From f0779fec80c04dde20cf4167fbee183b1c84f6e3 Mon Sep 17 00:00:00 2001 From: Phil Cohen Date: Sat, 1 Feb 2025 10:39:44 -0800 Subject: [PATCH 3/3] Update packages/test-harness/src/testSubset.ts --- packages/test-harness/src/testSubset.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/test-harness/src/testSubset.ts b/packages/test-harness/src/testSubset.ts index 6cb496c9ef..e3d106c446 100644 --- a/packages/test-harness/src/testSubset.ts +++ b/packages/test-harness/src/testSubset.ts @@ -53,8 +53,8 @@ export function runTestSubset() { } /** - * Determine whether we should log the failed tests. - * @returns `true` if we should log the failed tests + * Determine whether we should log the failed tests to a file. This makes it easier to put them in `testSubsetGrep.properties` for faster iterating. + * @returns `true` if we should log failed tests to `packages/test-harness/failedTests.properties` */ export function shouldLogFailedTests() { return process.env.CURSORLESS_LOG_FAILED === "true";