fix(ci): split unit-fast into bounded shared-worker lanes

This commit is contained in:
Shakker 2026-03-20 05:08:39 +00:00 committed by Shakker
parent 4d9ae5899d
commit 94ab044387
3 changed files with 91 additions and 38 deletions

View File

@ -352,12 +352,40 @@ const unitFastExcludedFiles = [
const unitAutoSingletonFiles = [ const unitAutoSingletonFiles = [
...new Set([...unitSingletonIsolatedFiles, ...memoryHeavyUnitFiles]), ...new Set([...unitSingletonIsolatedFiles, ...memoryHeavyUnitFiles]),
]; ];
const unitFastExtraExcludeFile =
unitFastExcludedFiles.length > 0
? writeTempJsonArtifact("vitest-unit-fast-excludes", unitFastExcludedFiles)
: null;
const estimateUnitDurationMs = (file) => const estimateUnitDurationMs = (file) =>
unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs; unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs;
const unitFastExcludedFileSet = new Set(unitFastExcludedFiles);
const unitFastCandidateFiles = allKnownUnitFiles.filter(
(file) => !unitFastExcludedFileSet.has(file),
);
const defaultUnitFastLaneCount = isCI && !isWindows ? 2 : 1;
const unitFastLaneCount = Math.max(
1,
parseEnvNumber("OPENCLAW_TEST_UNIT_FAST_LANES", defaultUnitFastLaneCount),
);
const unitFastBuckets =
unitFastLaneCount > 1
? packFilesByDuration(unitFastCandidateFiles, unitFastLaneCount, estimateUnitDurationMs)
: [unitFastCandidateFiles];
const unitFastEntries = unitFastBuckets
.filter((files) => files.length > 0)
.map((files, index) => ({
name: unitFastBuckets.length === 1 ? "unit-fast" : `unit-fast-${String(index + 1)}`,
env: {
OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact(
`vitest-unit-fast-include-${String(index + 1)}`,
files,
),
},
args: [
"vitest",
"run",
"--config",
"vitest.unit.config.ts",
`--pool=${useVmForks ? "vmForks" : "forks"}`,
...(disableIsolation ? ["--isolate=false"] : []),
],
}));
const heavyUnitBuckets = packFilesByDuration( const heavyUnitBuckets = packFilesByDuration(
timedHeavyUnitFiles, timedHeavyUnitFiles,
heavyUnitLaneCount, heavyUnitLaneCount,
@ -370,23 +398,7 @@ const unitHeavyEntries = heavyUnitBuckets.map((files, index) => ({
const baseRuns = [ const baseRuns = [
...(shouldSplitUnitRuns ...(shouldSplitUnitRuns
? [ ? [
{ ...unitFastEntries,
name: "unit-fast",
env:
unitFastExtraExcludeFile === null
? undefined
: {
OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE: unitFastExtraExcludeFile,
},
args: [
"vitest",
"run",
"--config",
"vitest.unit.config.ts",
`--pool=${useVmForks ? "vmForks" : "forks"}`,
...(disableIsolation ? ["--isolate=false"] : []),
],
},
...(unitBehaviorIsolatedFiles.length > 0 ...(unitBehaviorIsolatedFiles.length > 0
? [ ? [
{ {
@ -1223,14 +1235,17 @@ if (passthroughRequiresSingleRun && passthroughOptionArgs.length > 0) {
} }
if (isMacMiniProfile && targetedEntries.length === 0) { if (isMacMiniProfile && targetedEntries.length === 0) {
const unitFastEntry = parallelRuns.find((entry) => entry.name === "unit-fast"); const unitFastEntriesForMacMini = parallelRuns.filter((entry) =>
if (unitFastEntry) { entry.name.startsWith("unit-fast"),
const unitFastCode = await run(unitFastEntry, passthroughOptionArgs); );
for (const entry of unitFastEntriesForMacMini) {
// eslint-disable-next-line no-await-in-loop
const unitFastCode = await run(entry, passthroughOptionArgs);
if (unitFastCode !== 0) { if (unitFastCode !== 0) {
process.exit(unitFastCode); process.exit(unitFastCode);
} }
} }
const deferredEntries = parallelRuns.filter((entry) => entry.name !== "unit-fast"); const deferredEntries = parallelRuns.filter((entry) => !entry.name.startsWith("unit-fast"));
const failedMacMiniParallel = await runEntriesWithLimit( const failedMacMiniParallel = await runEntriesWithLimit(
deferredEntries, deferredEntries,
passthroughOptionArgs, passthroughOptionArgs,

View File

@ -2,7 +2,10 @@ import fs from "node:fs";
import os from "node:os"; import os from "node:os";
import path from "node:path"; import path from "node:path";
import { afterEach, describe, expect, it } from "vitest"; import { afterEach, describe, expect, it } from "vitest";
import { loadExtraExcludePatternsFromEnv } from "../vitest.unit.config.ts"; import {
loadExtraExcludePatternsFromEnv,
loadIncludePatternsFromEnv,
} from "../vitest.unit.config.ts";
const tempDirs = new Set<string>(); const tempDirs = new Set<string>();
@ -13,21 +16,42 @@ afterEach(() => {
tempDirs.clear(); tempDirs.clear();
}); });
const writeExcludeFile = (value: unknown) => { const writePatternFile = (basename: string, value: unknown) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-vitest-unit-config-")); const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-vitest-unit-config-"));
tempDirs.add(dir); tempDirs.add(dir);
const filePath = path.join(dir, "extra-exclude.json"); const filePath = path.join(dir, basename);
fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8"); fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8");
return filePath; return filePath;
}; };
describe("loadIncludePatternsFromEnv", () => {
it("returns null when no include file is configured", () => {
expect(loadIncludePatternsFromEnv({})).toBeNull();
});
it("loads include patterns from a JSON file", () => {
const filePath = writePatternFile("include.json", [
"src/infra/update-runner.test.ts",
42,
"",
"ui/src/ui/views/chat.test.ts",
]);
expect(
loadIncludePatternsFromEnv({
OPENCLAW_VITEST_INCLUDE_FILE: filePath,
}),
).toEqual(["src/infra/update-runner.test.ts", "ui/src/ui/views/chat.test.ts"]);
});
});
describe("loadExtraExcludePatternsFromEnv", () => { describe("loadExtraExcludePatternsFromEnv", () => {
it("returns an empty list when no extra exclude file is configured", () => { it("returns an empty list when no extra exclude file is configured", () => {
expect(loadExtraExcludePatternsFromEnv({})).toEqual([]); expect(loadExtraExcludePatternsFromEnv({})).toEqual([]);
}); });
it("loads extra exclude patterns from a JSON file", () => { it("loads extra exclude patterns from a JSON file", () => {
const filePath = writeExcludeFile([ const filePath = writePatternFile("extra-exclude.json", [
"src/infra/update-runner.test.ts", "src/infra/update-runner.test.ts",
42, 42,
"", "",
@ -42,7 +66,9 @@ describe("loadExtraExcludePatternsFromEnv", () => {
}); });
it("throws when the configured file is not a JSON array", () => { it("throws when the configured file is not a JSON array", () => {
const filePath = writeExcludeFile({ exclude: ["src/infra/update-runner.test.ts"] }); const filePath = writePatternFile("extra-exclude.json", {
exclude: ["src/infra/update-runner.test.ts"],
});
expect(() => expect(() =>
loadExtraExcludePatternsFromEnv({ loadExtraExcludePatternsFromEnv({

View File

@ -9,6 +9,24 @@ import {
const base = baseConfig as unknown as Record<string, unknown>; const base = baseConfig as unknown as Record<string, unknown>;
const baseTest = (baseConfig as { test?: { include?: string[]; exclude?: string[] } }).test ?? {}; const baseTest = (baseConfig as { test?: { include?: string[]; exclude?: string[] } }).test ?? {};
const exclude = baseTest.exclude ?? []; const exclude = baseTest.exclude ?? [];
function loadPatternListFile(filePath: string, label: string): string[] {
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown;
if (!Array.isArray(parsed)) {
throw new TypeError(`${label} must point to a JSON array: ${filePath}`);
}
return parsed.filter((value): value is string => typeof value === "string" && value.length > 0);
}
export function loadIncludePatternsFromEnv(
env: Record<string, string | undefined> = process.env,
): string[] | null {
const includeFile = env.OPENCLAW_VITEST_INCLUDE_FILE?.trim();
if (!includeFile) {
return null;
}
return loadPatternListFile(includeFile, "OPENCLAW_VITEST_INCLUDE_FILE");
}
export function loadExtraExcludePatternsFromEnv( export function loadExtraExcludePatternsFromEnv(
env: Record<string, string | undefined> = process.env, env: Record<string, string | undefined> = process.env,
): string[] { ): string[] {
@ -16,20 +34,14 @@ export function loadExtraExcludePatternsFromEnv(
if (!extraExcludeFile) { if (!extraExcludeFile) {
return []; return [];
} }
const parsed = JSON.parse(fs.readFileSync(extraExcludeFile, "utf8")) as unknown; return loadPatternListFile(extraExcludeFile, "OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE");
if (!Array.isArray(parsed)) {
throw new TypeError(
`OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE must point to a JSON array: ${extraExcludeFile}`,
);
}
return parsed.filter((value): value is string => typeof value === "string" && value.length > 0);
} }
export default defineConfig({ export default defineConfig({
...base, ...base,
test: { test: {
...baseTest, ...baseTest,
include: unitTestIncludePatterns, include: loadIncludePatternsFromEnv() ?? unitTestIncludePatterns,
exclude: [ exclude: [
...new Set([ ...new Set([
...exclude, ...exclude,