diff --git a/AGENTS.md b/AGENTS.md index e6c5b1a5e92..daaa0b1ebd5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,6 +111,7 @@ - Agents MUST NOT modify baseline, inventory, ignore, snapshot, or expected-failure files to silence failing checks without explicit approval in this chat. - For targeted/local debugging, keep using the wrapper: `pnpm test -- [vitest args...]` (for example `pnpm test -- src/commands/onboard-search.test.ts -t "shows registered plugin providers"`); do not default to raw `pnpm vitest run ...` because it bypasses wrapper config/profile/pool routing. - Do not set test workers above 16; tried already. +- Do not switch CI `pnpm test` lanes back to Vitest `vmForks` by default without fresh green evidence on current `main`; keep CI on `forks` unless explicitly re-validated. - If local Vitest runs cause memory pressure (common on non-Mac-Studio hosts), use `OPENCLAW_TEST_PROFILE=low OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test` for land/gate runs. - Live tests (real keys): `CLAWDBOT_LIVE_TEST=1 pnpm test:live` (OpenClaw-only) or `LIVE=1 pnpm test:live` (includes provider live tests). Docker: `pnpm test:docker:live-models`, `pnpm test:docker:live-gateway`. Onboarding Docker E2E: `pnpm test:docker:onboard`. - Full kit + what’s covered: `docs/help/testing.md`. diff --git a/docs/reference/test.md b/docs/reference/test.md index e337e963e1d..08ebb2af3fc 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -11,8 +11,9 @@ title: "Tests" - `pnpm test:force`: Kills any lingering gateway process holding the default control port, then runs the full Vitest suite with an isolated gateway port so server tests don’t collide with a running instance. Use this when a prior gateway run left port 18789 occupied. - `pnpm test:coverage`: Runs the unit suite with V8 coverage (via `vitest.unit.config.ts`). Global thresholds are 70% lines/branches/functions/statements. Coverage excludes integration-heavy entrypoints (CLI wiring, gateway/telegram bridges, webchat static server) to keep the target focused on unit-testable logic. -- `pnpm test` on Node 22, 23, and 24 uses Vitest `vmForks` by default for faster startup. Node 25+ falls back to `forks` until re-validated. You can force behavior with `OPENCLAW_TEST_VM_FORKS=0|1`. +- `pnpm test` on Node 22, 23, and 24 uses Vitest `vmForks` by default for local runs with enough memory. CI stays on `forks` unless explicitly overridden. Node 25+ falls back to `forks` until re-validated. You can force behavior with `OPENCLAW_TEST_VM_FORKS=0|1`. - `pnpm test`: runs the full wrapper. It keeps only a small behavioral override manifest in git, then uses a checked-in timing snapshot to peel the heaviest measured unit files into dedicated lanes. +- Files marked `singletonIsolated` no longer spawn one fresh Vitest process each by default. The wrapper batches them into dedicated `forks` lanes with `maxWorkers=1`, which preserves isolation from `unit-fast` while cutting process startup overhead. Tune lane count with `OPENCLAW_TEST_SINGLETON_ISOLATED_LANES=`. - `pnpm test:channels`: runs channel-heavy suites. - `pnpm test:extensions`: runs extension/plugin suites. - `pnpm test:perf:update-timings`: refreshes the checked-in slow-file timing snapshot used by `scripts/test-parallel.mjs`. diff --git a/scripts/test-parallel.mjs b/scripts/test-parallel.mjs index 011211a307b..2901daad6b5 100644 --- a/scripts/test-parallel.mjs +++ b/scripts/test-parallel.mjs @@ -15,6 +15,7 @@ import { resolveTestRunExitCode, } from "./test-parallel-utils.mjs"; import { + dedupeFilesPreserveOrder, loadUnitMemoryHotspotManifest, loadTestRunnerBehavior, loadUnitTimingManifest, @@ -81,18 +82,17 @@ const testProfile = ? rawTestProfile : "normal"; const isMacMiniProfile = testProfile === "macmini"; -// vmForks is a big win for transform/import heavy suites. Node 24 is stable again -// for the default unit-fast lane after moving the known flaky files to fork-only -// isolation, but Node 25+ still falls back to process forks until re-validated. -// Keep it opt-out via OPENCLAW_TEST_VM_FORKS=0, and let users force-enable with =1. +// vmForks is still useful for local transform-heavy runs, but do not flip CI +// back to vmForks by default unless you have fresh green evidence on current +// main. The retained Vite/Vitest module graph is the OOM pattern we keep +// tripping over under CI-style pressure. Keep it opt-out via +// OPENCLAW_TEST_VM_FORKS=0, and let users force-enable with =1 for comparison. const supportsVmForks = Number.isFinite(nodeMajor) ? nodeMajor <= 24 : true; +const preferVmForksByDefault = + !isCI && !isWindows && supportsVmForks && !lowMemLocalHost && testProfile !== "low"; const useVmForks = process.env.OPENCLAW_TEST_VM_FORKS === "1" || - (process.env.OPENCLAW_TEST_VM_FORKS !== "0" && - !isWindows && - supportsVmForks && - !lowMemLocalHost && - (isCI || testProfile !== "low")); + (process.env.OPENCLAW_TEST_VM_FORKS !== "0" && preferVmForksByDefault); const disableIsolation = process.env.OPENCLAW_TEST_NO_ISOLATE === "1"; const includeGatewaySuite = process.env.OPENCLAW_TEST_INCLUDE_GATEWAY === "1"; const includeExtensionsSuite = process.env.OPENCLAW_TEST_INCLUDE_EXTENSIONS === "1"; @@ -345,15 +345,46 @@ const { memoryHeavyFiles: memoryHeavyUnitFiles, timedHeavyFiles: timedHeavyUnitF memoryHeavyFiles: [], timedHeavyFiles: [], }; +const unitSingletonBatchFiles = dedupeFilesPreserveOrder( + unitSingletonIsolatedFiles, + new Set(unitBehaviorIsolatedFiles), +); +const unitMemorySingletonFiles = dedupeFilesPreserveOrder( + memoryHeavyUnitFiles, + new Set([...unitBehaviorOverrideSet, ...unitSingletonBatchFiles]), +); const unitSchedulingOverrideSet = new Set([...unitBehaviorOverrideSet, ...memoryHeavyUnitFiles]); const unitFastExcludedFiles = [ ...new Set([...unitSchedulingOverrideSet, ...timedHeavyUnitFiles, ...channelSingletonFiles]), ]; -const unitAutoSingletonFiles = [ - ...new Set([...unitSingletonIsolatedFiles, ...memoryHeavyUnitFiles]), -]; +const defaultSingletonBatchLaneCount = + testProfile === "serial" + ? 0 + : unitSingletonBatchFiles.length === 0 + ? 0 + : isCI + ? Math.ceil(unitSingletonBatchFiles.length / 6) + : highMemLocalHost + ? Math.ceil(unitSingletonBatchFiles.length / 8) + : lowMemLocalHost + ? Math.ceil(unitSingletonBatchFiles.length / 12) + : Math.ceil(unitSingletonBatchFiles.length / 10); +const singletonBatchLaneCount = + unitSingletonBatchFiles.length === 0 + ? 0 + : Math.min( + unitSingletonBatchFiles.length, + Math.max( + 1, + parseEnvNumber("OPENCLAW_TEST_SINGLETON_ISOLATED_LANES", defaultSingletonBatchLaneCount), + ), + ); const estimateUnitDurationMs = (file) => unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs; +const unitSingletonBuckets = + singletonBatchLaneCount > 0 + ? packFilesByDuration(unitSingletonBatchFiles, singletonBatchLaneCount, estimateUnitDurationMs) + : []; const unitFastExcludedFileSet = new Set(unitFastExcludedFiles); const unitFastCandidateFiles = allKnownUnitFiles.filter( (file) => !unitFastExcludedFileSet.has(file), @@ -400,6 +431,11 @@ const unitHeavyEntries = heavyUnitBuckets.map((files, index) => ({ name: `unit-heavy-${String(index + 1)}`, args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=forks", ...files], })); +const unitSingletonEntries = unitSingletonBuckets.map((files, index) => ({ + name: + unitSingletonBuckets.length === 1 ? "unit-singleton" : `unit-singleton-${String(index + 1)}`, + args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=forks", ...files], +})); const baseRuns = [ ...(shouldSplitUnitRuns ? [ @@ -420,7 +456,8 @@ const baseRuns = [ ] : []), ...unitHeavyEntries, - ...unitAutoSingletonFiles.map((file) => ({ + ...unitSingletonEntries, + ...unitMemorySingletonFiles.map((file) => ({ name: `${path.basename(file, ".test.ts")}-isolated`, args: [ "vitest", @@ -756,6 +793,9 @@ const maxWorkersForRun = (name) => { if (resolvedOverride) { return resolvedOverride; } + if (name === "unit-singleton" || name.startsWith("unit-singleton-")) { + return 1; + } if (isCI && !isMacOS) { return null; } diff --git a/scripts/test-runner-manifest.mjs b/scripts/test-runner-manifest.mjs index 4e0ff9d0a5a..ee5644f3328 100644 --- a/scripts/test-runner-manifest.mjs +++ b/scripts/test-runner-manifest.mjs @@ -231,3 +231,18 @@ export function packFilesByDuration(files, bucketCount, estimateDurationMs) { return buckets.map((bucket) => bucket.files).filter((bucket) => bucket.length > 0); } + +export function dedupeFilesPreserveOrder(files, exclude = new Set()) { + const result = []; + const seen = new Set(); + + for (const file of files) { + if (exclude.has(file) || seen.has(file)) { + continue; + } + seen.add(file); + result.push(file); + } + + return result; +} diff --git a/test/scripts/test-runner-manifest.test.ts b/test/scripts/test-runner-manifest.test.ts index cd650ae2aad..0fac87c25e1 100644 --- a/test/scripts/test-runner-manifest.test.ts +++ b/test/scripts/test-runner-manifest.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; import { + dedupeFilesPreserveOrder, + packFilesByDuration, selectMemoryHeavyFiles, selectTimedHeavyFiles, selectUnitHeavyFileGroups, @@ -91,3 +93,44 @@ describe("scripts/test-runner-manifest memory selection", () => { }); }); }); + +describe("dedupeFilesPreserveOrder", () => { + it("removes duplicates while keeping the first-seen order", () => { + expect( + dedupeFilesPreserveOrder([ + "src/b.test.ts", + "src/a.test.ts", + "src/b.test.ts", + "src/c.test.ts", + "src/a.test.ts", + ]), + ).toEqual(["src/b.test.ts", "src/a.test.ts", "src/c.test.ts"]); + }); + + it("filters excluded files before deduping", () => { + expect( + dedupeFilesPreserveOrder( + ["src/a.test.ts", "src/b.test.ts", "src/c.test.ts", "src/b.test.ts"], + new Set(["src/b.test.ts"]), + ), + ).toEqual(["src/a.test.ts", "src/c.test.ts"]); + }); +}); + +describe("packFilesByDuration", () => { + it("packs heavier files into the lightest remaining bucket", () => { + const durationByFile = { + "src/a.test.ts": 100, + "src/b.test.ts": 90, + "src/c.test.ts": 20, + "src/d.test.ts": 10, + } satisfies Record; + + expect( + packFilesByDuration(Object.keys(durationByFile), 2, (file) => durationByFile[file] ?? 0), + ).toEqual([ + ["src/a.test.ts", "src/d.test.ts"], + ["src/b.test.ts", "src/c.test.ts"], + ]); + }); +});