fix(ci): split unit-fast into bounded shared-worker lanes
This commit is contained in:
parent
4d9ae5899d
commit
94ab044387
@ -352,12 +352,40 @@ const unitFastExcludedFiles = [
|
|||||||
const unitAutoSingletonFiles = [
|
const unitAutoSingletonFiles = [
|
||||||
...new Set([...unitSingletonIsolatedFiles, ...memoryHeavyUnitFiles]),
|
...new Set([...unitSingletonIsolatedFiles, ...memoryHeavyUnitFiles]),
|
||||||
];
|
];
|
||||||
const unitFastExtraExcludeFile =
|
|
||||||
unitFastExcludedFiles.length > 0
|
|
||||||
? writeTempJsonArtifact("vitest-unit-fast-excludes", unitFastExcludedFiles)
|
|
||||||
: null;
|
|
||||||
const estimateUnitDurationMs = (file) =>
|
const estimateUnitDurationMs = (file) =>
|
||||||
unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs;
|
unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs;
|
||||||
|
const unitFastExcludedFileSet = new Set(unitFastExcludedFiles);
|
||||||
|
const unitFastCandidateFiles = allKnownUnitFiles.filter(
|
||||||
|
(file) => !unitFastExcludedFileSet.has(file),
|
||||||
|
);
|
||||||
|
const defaultUnitFastLaneCount = isCI && !isWindows ? 2 : 1;
|
||||||
|
const unitFastLaneCount = Math.max(
|
||||||
|
1,
|
||||||
|
parseEnvNumber("OPENCLAW_TEST_UNIT_FAST_LANES", defaultUnitFastLaneCount),
|
||||||
|
);
|
||||||
|
const unitFastBuckets =
|
||||||
|
unitFastLaneCount > 1
|
||||||
|
? packFilesByDuration(unitFastCandidateFiles, unitFastLaneCount, estimateUnitDurationMs)
|
||||||
|
: [unitFastCandidateFiles];
|
||||||
|
const unitFastEntries = unitFastBuckets
|
||||||
|
.filter((files) => files.length > 0)
|
||||||
|
.map((files, index) => ({
|
||||||
|
name: unitFastBuckets.length === 1 ? "unit-fast" : `unit-fast-${String(index + 1)}`,
|
||||||
|
env: {
|
||||||
|
OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact(
|
||||||
|
`vitest-unit-fast-include-${String(index + 1)}`,
|
||||||
|
files,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
args: [
|
||||||
|
"vitest",
|
||||||
|
"run",
|
||||||
|
"--config",
|
||||||
|
"vitest.unit.config.ts",
|
||||||
|
`--pool=${useVmForks ? "vmForks" : "forks"}`,
|
||||||
|
...(disableIsolation ? ["--isolate=false"] : []),
|
||||||
|
],
|
||||||
|
}));
|
||||||
const heavyUnitBuckets = packFilesByDuration(
|
const heavyUnitBuckets = packFilesByDuration(
|
||||||
timedHeavyUnitFiles,
|
timedHeavyUnitFiles,
|
||||||
heavyUnitLaneCount,
|
heavyUnitLaneCount,
|
||||||
@ -370,23 +398,7 @@ const unitHeavyEntries = heavyUnitBuckets.map((files, index) => ({
|
|||||||
const baseRuns = [
|
const baseRuns = [
|
||||||
...(shouldSplitUnitRuns
|
...(shouldSplitUnitRuns
|
||||||
? [
|
? [
|
||||||
{
|
...unitFastEntries,
|
||||||
name: "unit-fast",
|
|
||||||
env:
|
|
||||||
unitFastExtraExcludeFile === null
|
|
||||||
? undefined
|
|
||||||
: {
|
|
||||||
OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE: unitFastExtraExcludeFile,
|
|
||||||
},
|
|
||||||
args: [
|
|
||||||
"vitest",
|
|
||||||
"run",
|
|
||||||
"--config",
|
|
||||||
"vitest.unit.config.ts",
|
|
||||||
`--pool=${useVmForks ? "vmForks" : "forks"}`,
|
|
||||||
...(disableIsolation ? ["--isolate=false"] : []),
|
|
||||||
],
|
|
||||||
},
|
|
||||||
...(unitBehaviorIsolatedFiles.length > 0
|
...(unitBehaviorIsolatedFiles.length > 0
|
||||||
? [
|
? [
|
||||||
{
|
{
|
||||||
@ -1223,14 +1235,17 @@ if (passthroughRequiresSingleRun && passthroughOptionArgs.length > 0) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (isMacMiniProfile && targetedEntries.length === 0) {
|
if (isMacMiniProfile && targetedEntries.length === 0) {
|
||||||
const unitFastEntry = parallelRuns.find((entry) => entry.name === "unit-fast");
|
const unitFastEntriesForMacMini = parallelRuns.filter((entry) =>
|
||||||
if (unitFastEntry) {
|
entry.name.startsWith("unit-fast"),
|
||||||
const unitFastCode = await run(unitFastEntry, passthroughOptionArgs);
|
);
|
||||||
|
for (const entry of unitFastEntriesForMacMini) {
|
||||||
|
// eslint-disable-next-line no-await-in-loop
|
||||||
|
const unitFastCode = await run(entry, passthroughOptionArgs);
|
||||||
if (unitFastCode !== 0) {
|
if (unitFastCode !== 0) {
|
||||||
process.exit(unitFastCode);
|
process.exit(unitFastCode);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const deferredEntries = parallelRuns.filter((entry) => entry.name !== "unit-fast");
|
const deferredEntries = parallelRuns.filter((entry) => !entry.name.startsWith("unit-fast"));
|
||||||
const failedMacMiniParallel = await runEntriesWithLimit(
|
const failedMacMiniParallel = await runEntriesWithLimit(
|
||||||
deferredEntries,
|
deferredEntries,
|
||||||
passthroughOptionArgs,
|
passthroughOptionArgs,
|
||||||
|
|||||||
@ -2,7 +2,10 @@ import fs from "node:fs";
|
|||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { afterEach, describe, expect, it } from "vitest";
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
import { loadExtraExcludePatternsFromEnv } from "../vitest.unit.config.ts";
|
import {
|
||||||
|
loadExtraExcludePatternsFromEnv,
|
||||||
|
loadIncludePatternsFromEnv,
|
||||||
|
} from "../vitest.unit.config.ts";
|
||||||
|
|
||||||
const tempDirs = new Set<string>();
|
const tempDirs = new Set<string>();
|
||||||
|
|
||||||
@ -13,21 +16,42 @@ afterEach(() => {
|
|||||||
tempDirs.clear();
|
tempDirs.clear();
|
||||||
});
|
});
|
||||||
|
|
||||||
const writeExcludeFile = (value: unknown) => {
|
const writePatternFile = (basename: string, value: unknown) => {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-vitest-unit-config-"));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-vitest-unit-config-"));
|
||||||
tempDirs.add(dir);
|
tempDirs.add(dir);
|
||||||
const filePath = path.join(dir, "extra-exclude.json");
|
const filePath = path.join(dir, basename);
|
||||||
fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8");
|
fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8");
|
||||||
return filePath;
|
return filePath;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
describe("loadIncludePatternsFromEnv", () => {
|
||||||
|
it("returns null when no include file is configured", () => {
|
||||||
|
expect(loadIncludePatternsFromEnv({})).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("loads include patterns from a JSON file", () => {
|
||||||
|
const filePath = writePatternFile("include.json", [
|
||||||
|
"src/infra/update-runner.test.ts",
|
||||||
|
42,
|
||||||
|
"",
|
||||||
|
"ui/src/ui/views/chat.test.ts",
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
loadIncludePatternsFromEnv({
|
||||||
|
OPENCLAW_VITEST_INCLUDE_FILE: filePath,
|
||||||
|
}),
|
||||||
|
).toEqual(["src/infra/update-runner.test.ts", "ui/src/ui/views/chat.test.ts"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("loadExtraExcludePatternsFromEnv", () => {
|
describe("loadExtraExcludePatternsFromEnv", () => {
|
||||||
it("returns an empty list when no extra exclude file is configured", () => {
|
it("returns an empty list when no extra exclude file is configured", () => {
|
||||||
expect(loadExtraExcludePatternsFromEnv({})).toEqual([]);
|
expect(loadExtraExcludePatternsFromEnv({})).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("loads extra exclude patterns from a JSON file", () => {
|
it("loads extra exclude patterns from a JSON file", () => {
|
||||||
const filePath = writeExcludeFile([
|
const filePath = writePatternFile("extra-exclude.json", [
|
||||||
"src/infra/update-runner.test.ts",
|
"src/infra/update-runner.test.ts",
|
||||||
42,
|
42,
|
||||||
"",
|
"",
|
||||||
@ -42,7 +66,9 @@ describe("loadExtraExcludePatternsFromEnv", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("throws when the configured file is not a JSON array", () => {
|
it("throws when the configured file is not a JSON array", () => {
|
||||||
const filePath = writeExcludeFile({ exclude: ["src/infra/update-runner.test.ts"] });
|
const filePath = writePatternFile("extra-exclude.json", {
|
||||||
|
exclude: ["src/infra/update-runner.test.ts"],
|
||||||
|
});
|
||||||
|
|
||||||
expect(() =>
|
expect(() =>
|
||||||
loadExtraExcludePatternsFromEnv({
|
loadExtraExcludePatternsFromEnv({
|
||||||
|
|||||||
@ -9,6 +9,24 @@ import {
|
|||||||
const base = baseConfig as unknown as Record<string, unknown>;
|
const base = baseConfig as unknown as Record<string, unknown>;
|
||||||
const baseTest = (baseConfig as { test?: { include?: string[]; exclude?: string[] } }).test ?? {};
|
const baseTest = (baseConfig as { test?: { include?: string[]; exclude?: string[] } }).test ?? {};
|
||||||
const exclude = baseTest.exclude ?? [];
|
const exclude = baseTest.exclude ?? [];
|
||||||
|
function loadPatternListFile(filePath: string, label: string): string[] {
|
||||||
|
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown;
|
||||||
|
if (!Array.isArray(parsed)) {
|
||||||
|
throw new TypeError(`${label} must point to a JSON array: ${filePath}`);
|
||||||
|
}
|
||||||
|
return parsed.filter((value): value is string => typeof value === "string" && value.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function loadIncludePatternsFromEnv(
|
||||||
|
env: Record<string, string | undefined> = process.env,
|
||||||
|
): string[] | null {
|
||||||
|
const includeFile = env.OPENCLAW_VITEST_INCLUDE_FILE?.trim();
|
||||||
|
if (!includeFile) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return loadPatternListFile(includeFile, "OPENCLAW_VITEST_INCLUDE_FILE");
|
||||||
|
}
|
||||||
|
|
||||||
export function loadExtraExcludePatternsFromEnv(
|
export function loadExtraExcludePatternsFromEnv(
|
||||||
env: Record<string, string | undefined> = process.env,
|
env: Record<string, string | undefined> = process.env,
|
||||||
): string[] {
|
): string[] {
|
||||||
@ -16,20 +34,14 @@ export function loadExtraExcludePatternsFromEnv(
|
|||||||
if (!extraExcludeFile) {
|
if (!extraExcludeFile) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
const parsed = JSON.parse(fs.readFileSync(extraExcludeFile, "utf8")) as unknown;
|
return loadPatternListFile(extraExcludeFile, "OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE");
|
||||||
if (!Array.isArray(parsed)) {
|
|
||||||
throw new TypeError(
|
|
||||||
`OPENCLAW_VITEST_EXTRA_EXCLUDE_FILE must point to a JSON array: ${extraExcludeFile}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return parsed.filter((value): value is string => typeof value === "string" && value.length > 0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
...base,
|
...base,
|
||||||
test: {
|
test: {
|
||||||
...baseTest,
|
...baseTest,
|
||||||
include: unitTestIncludePatterns,
|
include: loadIncludePatternsFromEnv() ?? unitTestIncludePatterns,
|
||||||
exclude: [
|
exclude: [
|
||||||
...new Set([
|
...new Set([
|
||||||
...exclude,
|
...exclude,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user