fix(ci): split unit-fast into bounded shared-worker lanes

This commit is contained in:
Shakker 2026-03-20 05:08:39 +00:00 committed by Shakker
parent 4d9ae5899d
commit 94ab044387
3 changed files with 91 additions and 38 deletions

View File

@ -352,12 +352,40 @@ const unitFastExcludedFiles = [
const unitAutoSingletonFiles = [
...new Set([...unitSingletonIsolatedFiles, ...memoryHeavyUnitFiles]),
];
const unitFastExtraExcludeFile =
unitFastExcludedFiles.length > 0
? writeTempJsonArtifact("vitest-unit-fast-excludes", unitFastExcludedFiles)
: null;
const estimateUnitDurationMs = (file) =>
unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs;
const unitFastExcludedFileSet = new Set(unitFastExcludedFiles);
const unitFastCandidateFiles = allKnownUnitFiles.filter(
(file) => !unitFastExcludedFileSet.has(file),
);
const defaultUnitFastLaneCount = isCI && !isWindows ? 2 : 1;
const unitFastLaneCount = Math.max(
1,
parseEnvNumber("OPENCLAW_TEST_UNIT_FAST_LANES", defaultUnitFastLaneCount),
);
const unitFastBuckets =
unitFastLaneCount > 1
? packFilesByDuration(unitFastCandidateFiles, unitFastLaneCount, estimateUnitDurationMs)
: [unitFastCandidateFiles];
const unitFastEntries = unitFastBuckets
.filter((files) => files.length > 0)
.map((files, index) => ({
name: unitFastBuckets.length === 1 ? "unit-fast" : `unit-fast-${String(index + 1)}`,
env: {
OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact(
`vitest-unit-fast-include-${String(index + 1)}`,
files,
),
},
args: [
"vitest",
"run",
"--config",
"vitest.unit.config.ts",
`--pool=${useVmForks ? "vmForks" : "forks"}`,
...(disableIsolation ? ["--isolate=false"] : []),
],
}));
const heavyUnitBuckets = packFilesByDuration(
timedHeavyUnitFiles,
heavyUnitLaneCount,
@ -370,23 +398,7 @@ const unitHeavyEntries = heavyUnitBuckets.map((files, index) => ({
const baseRuns = [
...(shouldSplitUnitRuns
? [
{
name: "unit-fast",
env:
unitFastExtraExcludeFile === null
? undefined
: {
OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE: unitFastExtraExcludeFile,
},
args: [
"vitest",
"run",
"--config",
"vitest.unit.config.ts",
`--pool=${useVmForks ? "vmForks" : "forks"}`,
...(disableIsolation ? ["--isolate=false"] : []),
],
},
...unitFastEntries,
...(unitBehaviorIsolatedFiles.length > 0
? [
{
@ -1223,14 +1235,17 @@ if (passthroughRequiresSingleRun && passthroughOptionArgs.length > 0) {
}
if (isMacMiniProfile && targetedEntries.length === 0) {
const unitFastEntry = parallelRuns.find((entry) => entry.name === "unit-fast");
if (unitFastEntry) {
const unitFastCode = await run(unitFastEntry, passthroughOptionArgs);
const unitFastEntriesForMacMini = parallelRuns.filter((entry) =>
entry.name.startsWith("unit-fast"),
);
for (const entry of unitFastEntriesForMacMini) {
// eslint-disable-next-line no-await-in-loop
const unitFastCode = await run(entry, passthroughOptionArgs);
if (unitFastCode !== 0) {
process.exit(unitFastCode);
}
}
const deferredEntries = parallelRuns.filter((entry) => entry.name !== "unit-fast");
const deferredEntries = parallelRuns.filter((entry) => !entry.name.startsWith("unit-fast"));
const failedMacMiniParallel = await runEntriesWithLimit(
deferredEntries,
passthroughOptionArgs,

View File

@ -2,7 +2,10 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { loadExtraExcludePatternsFromEnv } from "../vitest.unit.config.ts";
import {
loadExtraExcludePatternsFromEnv,
loadIncludePatternsFromEnv,
} from "../vitest.unit.config.ts";
const tempDirs = new Set<string>();
@ -13,21 +16,42 @@ afterEach(() => {
tempDirs.clear();
});
const writeExcludeFile = (value: unknown) => {
const writePatternFile = (basename: string, value: unknown) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-vitest-unit-config-"));
tempDirs.add(dir);
const filePath = path.join(dir, "extra-exclude.json");
const filePath = path.join(dir, basename);
fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8");
return filePath;
};
describe("loadIncludePatternsFromEnv", () => {
it("returns null when no include file is configured", () => {
expect(loadIncludePatternsFromEnv({})).toBeNull();
});
it("loads include patterns from a JSON file", () => {
const filePath = writePatternFile("include.json", [
"src/infra/update-runner.test.ts",
42,
"",
"ui/src/ui/views/chat.test.ts",
]);
expect(
loadIncludePatternsFromEnv({
OPENCLAW_VITEST_INCLUDE_FILE: filePath,
}),
).toEqual(["src/infra/update-runner.test.ts", "ui/src/ui/views/chat.test.ts"]);
});
});
describe("loadExtraExcludePatternsFromEnv", () => {
it("returns an empty list when no extra exclude file is configured", () => {
expect(loadExtraExcludePatternsFromEnv({})).toEqual([]);
});
it("loads extra exclude patterns from a JSON file", () => {
const filePath = writeExcludeFile([
const filePath = writePatternFile("extra-exclude.json", [
"src/infra/update-runner.test.ts",
42,
"",
@ -42,7 +66,9 @@ describe("loadExtraExcludePatternsFromEnv", () => {
});
it("throws when the configured file is not a JSON array", () => {
const filePath = writeExcludeFile({ exclude: ["src/infra/update-runner.test.ts"] });
const filePath = writePatternFile("extra-exclude.json", {
exclude: ["src/infra/update-runner.test.ts"],
});
expect(() =>
loadExtraExcludePatternsFromEnv({

View File

@ -9,6 +9,24 @@ import {
const base = baseConfig as unknown as Record<string, unknown>;
const baseTest = (baseConfig as { test?: { include?: string[]; exclude?: string[] } }).test ?? {};
const exclude = baseTest.exclude ?? [];
function loadPatternListFile(filePath: string, label: string): string[] {
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown;
if (!Array.isArray(parsed)) {
throw new TypeError(`${label} must point to a JSON array: ${filePath}`);
}
return parsed.filter((value): value is string => typeof value === "string" && value.length > 0);
}
export function loadIncludePatternsFromEnv(
env: Record<string, string | undefined> = process.env,
): string[] | null {
const includeFile = env.OPENCLAW_VITEST_INCLUDE_FILE?.trim();
if (!includeFile) {
return null;
}
return loadPatternListFile(includeFile, "OPENCLAW_VITEST_INCLUDE_FILE");
}
export function loadExtraExcludePatternsFromEnv(
env: Record<string, string | undefined> = process.env,
): string[] {
@ -16,20 +34,14 @@ export function loadExtraExcludePatternsFromEnv(
if (!extraExcludeFile) {
return [];
}
const parsed = JSON.parse(fs.readFileSync(extraExcludeFile, "utf8")) as unknown;
if (!Array.isArray(parsed)) {
throw new TypeError(
`OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE must point to a JSON array: ${extraExcludeFile}`,
);
}
return parsed.filter((value): value is string => typeof value === "string" && value.length > 0);
return loadPatternListFile(extraExcludeFile, "OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE");
}
export default defineConfig({
...base,
test: {
...baseTest,
include: unitTestIncludePatterns,
include: loadIncludePatternsFromEnv() ?? unitTestIncludePatterns,
exclude: [
...new Set([
...exclude,