2026-01-23 07:34:50 +00:00
import { spawn } from "node:child_process" ;
2026-02-07 08:27:50 +00:00
import fs from "node:fs" ;
2026-01-23 11:36:28 +00:00
import os from "node:os" ;
2026-03-13 18:36:38 -05:00
import path from "node:path" ;
2026-03-14 11:23:25 -07:00
import { channelTestPrefixes } from "../vitest.channel-paths.mjs" ;
2026-03-18 12:16:07 -07:00
import { isUnitConfigTestFile } from "../vitest.unit-paths.mjs" ;
2026-03-19 17:59:13 -04:00
import {
getProcessTreeRecords ,
parseCompletedTestFileLines ,
sampleProcessTreeRssKb ,
} from "./test-parallel-memory.mjs" ;
2026-03-19 11:01:16 -07:00
import {
appendCapturedOutput ,
hasFatalTestRunOutput ,
resolveTestRunExitCode ,
} from "./test-parallel-utils.mjs" ;
2026-03-18 16:57:27 +00:00
import {
2026-03-20 17:09:46 +00:00
dedupeFilesPreserveOrder ,
2026-03-20 04:27:49 +00:00
loadUnitMemoryHotspotManifest ,
2026-03-18 16:57:27 +00:00
loadTestRunnerBehavior ,
loadUnitTimingManifest ,
2026-03-20 04:43:09 +00:00
selectUnitHeavyFileGroups ,
2026-03-18 16:57:27 +00:00
packFilesByDuration ,
} from "./test-runner-manifest.mjs" ;
2026-01-23 07:34:50 +00:00
2026-02-15 03:35:02 +00:00
// On Windows, `.cmd` launchers can fail with `spawn EINVAL` when invoked without a shell
// (especially under GitHub Actions + Git Bash). Use `shell: true` and let the shell resolve pnpm.
const pnpm = "pnpm" ;
2026-03-18 16:57:27 +00:00
const behaviorManifest = loadTestRunnerBehavior ( ) ;
const existingFiles = ( entries ) =>
entries . map ( ( entry ) => entry . file ) . filter ( ( file ) => fs . existsSync ( file ) ) ;
2026-03-20 04:48:24 +00:00
let tempArtifactDir = null ;
const ensureTempArtifactDir = ( ) => {
if ( tempArtifactDir === null ) {
tempArtifactDir = fs . mkdtempSync ( path . join ( os . tmpdir ( ) , "openclaw-test-parallel-" ) ) ;
}
return tempArtifactDir ;
} ;
const writeTempJsonArtifact = ( name , value ) => {
const filePath = path . join ( ensureTempArtifactDir ( ) , ` ${ name } .json ` ) ;
fs . writeFileSync ( filePath , ` ${ JSON . stringify ( value ) } \n ` , "utf8" ) ;
return filePath ;
} ;
const cleanupTempArtifacts = ( ) => {
if ( tempArtifactDir === null ) {
return ;
}
fs . rmSync ( tempArtifactDir , { recursive : true , force : true } ) ;
tempArtifactDir = null ;
} ;
2026-03-18 12:16:07 -07:00
const existingUnitConfigFiles = ( entries ) => existingFiles ( entries ) . filter ( isUnitConfigTestFile ) ;
const unitBehaviorIsolatedFiles = existingUnitConfigFiles ( behaviorManifest . unit . isolated ) ;
const unitSingletonIsolatedFiles = existingUnitConfigFiles ( behaviorManifest . unit . singletonIsolated ) ;
const unitThreadSingletonFiles = existingUnitConfigFiles ( behaviorManifest . unit . threadSingleton ) ;
const unitVmForkSingletonFiles = existingUnitConfigFiles ( behaviorManifest . unit . vmForkSingleton ) ;
2026-03-18 16:57:27 +00:00
const unitBehaviorOverrideSet = new Set ( [
... unitBehaviorIsolatedFiles ,
... unitSingletonIsolatedFiles ,
... unitThreadSingletonFiles ,
... unitVmForkSingletonFiles ,
] ) ;
const channelSingletonFiles = [ ] ;
2026-02-13 04:30:39 +00:00
const children = new Set ( ) ;
const isCI = process . env . CI === "true" || process . env . GITHUB _ACTIONS === "true" ;
const isMacOS = process . platform === "darwin" || process . env . RUNNER _OS === "macOS" ;
const isWindows = process . platform === "win32" || process . env . RUNNER _OS === "Windows" ;
const isWindowsCi = isCI && isWindows ;
2026-02-25 12:16:17 +02:00
const hostCpuCount = os . cpus ( ) . length ;
const hostMemoryGiB = Math . floor ( os . totalmem ( ) / 1024 * * 3 ) ;
// Keep aggressive local defaults for high-memory workstations (Mac Studio class).
const highMemLocalHost = ! isCI && hostMemoryGiB >= 96 ;
const lowMemLocalHost = ! isCI && hostMemoryGiB < 64 ;
2026-02-13 08:15:25 -05:00
const nodeMajor = Number . parseInt ( process . versions . node . split ( "." ) [ 0 ] ? ? "" , 10 ) ;
2026-03-06 17:45:35 -05:00
const rawTestProfile = process . env . OPENCLAW _TEST _PROFILE ? . trim ( ) . toLowerCase ( ) ;
const testProfile =
rawTestProfile === "low" ||
2026-03-19 07:47:07 -05:00
rawTestProfile === "macmini" ||
2026-03-06 17:45:35 -05:00
rawTestProfile === "max" ||
rawTestProfile === "normal" ||
rawTestProfile === "serial"
? rawTestProfile
: "normal" ;
2026-03-19 07:47:07 -05:00
const isMacMiniProfile = testProfile === "macmini" ;
2026-03-20 13:15:55 -04:00
// Vitest executes Node tests through Vite's SSR/module-runner pipeline, so the
// shared unit lane still retains transformed ESM/module state even when the
// tests themselves are not "server rendering" a website. vmForks can win in
// ideal transform-heavy cases, but for this repo we measured higher aggregate
// CPU load and fatal heap OOMs on memory-constrained dev machines and CI when
// unit-fast stayed on vmForks. Keep forks as the default unless that evidence
// is re-run and replaced:
// PR: https://github.com/openclaw/openclaw/pull/51145
// OOM evidence: https://github.com/openclaw/openclaw/pull/51145#issuecomment-4099663958
// Preserve OPENCLAW_TEST_VM_FORKS=1 as the explicit override/debug escape hatch.
2026-03-19 15:02:48 -07:00
const supportsVmForks = Number . isFinite ( nodeMajor ) ? nodeMajor <= 24 : true ;
2026-03-20 13:15:55 -04:00
const useVmForks = process . env . OPENCLAW _TEST _VM _FORKS === "1" && supportsVmForks ;
2026-03-19 15:02:48 -07:00
const disableIsolation = process . env . OPENCLAW _TEST _NO _ISOLATE === "1" ;
const includeGatewaySuite = process . env . OPENCLAW _TEST _INCLUDE _GATEWAY === "1" ;
const includeExtensionsSuite = process . env . OPENCLAW _TEST _INCLUDE _EXTENSIONS === "1" ;
2026-03-08 18:39:37 +00:00
// Even on low-memory hosts, keep the isolated lane split so files like
// git-commit.test.ts still get the worker/process isolation they require.
const shouldSplitUnitRuns = testProfile !== "serial" ;
2026-03-18 16:57:27 +00:00
let runs = [ ] ;
2026-01-30 03:15:10 +01:00
const shardOverride = Number . parseInt ( process . env . OPENCLAW _TEST _SHARDS ? ? "" , 10 ) ;
2026-02-26 00:33:36 -06:00
const configuredShardCount =
Number . isFinite ( shardOverride ) && shardOverride > 1 ? shardOverride : null ;
const shardCount = configuredShardCount ? ? ( isWindowsCi ? 2 : 1 ) ;
const shardIndexOverride = ( ( ) => {
const parsed = Number . parseInt ( process . env . OPENCLAW _TEST _SHARD _INDEX ? ? "" , 10 ) ;
return Number . isFinite ( parsed ) && parsed > 0 ? parsed : null ;
} ) ( ) ;
2026-03-13 18:36:38 -05:00
const OPTION _TAKES _VALUE = new Set ( [
"-t" ,
"-c" ,
"-r" ,
"--testNamePattern" ,
"--config" ,
"--root" ,
"--dir" ,
"--reporter" ,
"--outputFile" ,
"--pool" ,
"--execArgv" ,
"--vmMemoryLimit" ,
"--maxWorkers" ,
"--environment" ,
"--shard" ,
"--changed" ,
"--sequence" ,
"--inspect" ,
"--inspectBrk" ,
"--testTimeout" ,
"--hookTimeout" ,
"--bail" ,
"--retry" ,
"--diff" ,
"--exclude" ,
"--project" ,
"--slowTestThreshold" ,
"--teardownTimeout" ,
"--attachmentsDir" ,
"--mode" ,
"--api" ,
"--browser" ,
"--maxConcurrency" ,
"--mergeReports" ,
"--configLoader" ,
"--experimental" ,
] ) ;
const SINGLE _RUN _ONLY _FLAGS = new Set ( [ "--coverage" , "--outputFile" , "--mergeReports" ] ) ;
2026-02-26 00:33:36 -06:00
if ( shardIndexOverride !== null && shardCount <= 1 ) {
console . error (
` [test-parallel] OPENCLAW_TEST_SHARD_INDEX= ${ String (
shardIndexOverride ,
) } requires OPENCLAW _TEST _SHARDS > 1. ` ,
) ;
process . exit ( 2 ) ;
}
if ( shardIndexOverride !== null && shardIndexOverride > shardCount ) {
console . error (
` [test-parallel] OPENCLAW_TEST_SHARD_INDEX= ${ String (
shardIndexOverride ,
) } exceeds OPENCLAW _TEST _SHARDS = $ { String ( shardCount ) } . ` ,
) ;
process . exit ( 2 ) ;
}
2026-02-06 23:18:19 -03:00
const windowsCiArgs = isWindowsCi ? [ "--dangerouslyIgnoreUnhandledErrors" ] : [ ] ;
2026-02-12 17:59:44 +00:00
const silentArgs =
process . env . OPENCLAW _TEST _SHOW _PASSED _LOGS === "1" ? [ ] : [ "--silent=passed-only" ] ;
2026-02-07 20:02:32 -08:00
const rawPassthroughArgs = process . argv . slice ( 2 ) ;
const passthroughArgs =
rawPassthroughArgs [ 0 ] === "--" ? rawPassthroughArgs . slice ( 1 ) : rawPassthroughArgs ;
2026-03-13 18:36:38 -05:00
const parsePassthroughArgs = ( args ) => {
const fileFilters = [ ] ;
const optionArgs = [ ] ;
let consumeNextAsOptionValue = false ;
for ( const arg of args ) {
if ( consumeNextAsOptionValue ) {
optionArgs . push ( arg ) ;
consumeNextAsOptionValue = false ;
continue ;
}
if ( arg === "--" ) {
optionArgs . push ( arg ) ;
continue ;
}
if ( arg . startsWith ( "-" ) ) {
optionArgs . push ( arg ) ;
consumeNextAsOptionValue = ! arg . includes ( "=" ) && OPTION _TAKES _VALUE . has ( arg ) ;
continue ;
}
fileFilters . push ( arg ) ;
}
return { fileFilters , optionArgs } ;
} ;
const { fileFilters : passthroughFileFilters , optionArgs : passthroughOptionArgs } =
parsePassthroughArgs ( passthroughArgs ) ;
2026-03-19 07:47:07 -05:00
const passthroughMetadataFlags = new Set ( [ "-h" , "--help" , "--listTags" , "--clearCache" ] ) ;
const passthroughMetadataOnly =
passthroughArgs . length > 0 &&
passthroughFileFilters . length === 0 &&
passthroughOptionArgs . every ( ( arg ) => {
if ( ! arg . startsWith ( "-" ) ) {
return false ;
}
const [ flag ] = arg . split ( "=" , 1 ) ;
return passthroughMetadataFlags . has ( flag ) ;
} ) ;
2026-03-18 08:58:29 -07:00
const countExplicitEntryFilters = ( entryArgs ) => {
const { fileFilters } = parsePassthroughArgs ( entryArgs . slice ( 2 ) ) ;
return fileFilters . length > 0 ? fileFilters . length : null ;
} ;
2026-03-19 14:02:19 -07:00
const getExplicitEntryFilters = ( entryArgs ) => parsePassthroughArgs ( entryArgs . slice ( 2 ) ) . fileFilters ;
2026-03-13 18:36:38 -05:00
const passthroughRequiresSingleRun = passthroughOptionArgs . some ( ( arg ) => {
if ( ! arg . startsWith ( "-" ) ) {
return false ;
}
const [ flag ] = arg . split ( "=" , 1 ) ;
return SINGLE _RUN _ONLY _FLAGS . has ( flag ) ;
} ) ;
const baseConfigPrefixes = [ "src/agents/" , "src/auto-reply/" , "src/commands/" , "test/" , "ui/" ] ;
const normalizeRepoPath = ( value ) => value . split ( path . sep ) . join ( "/" ) ;
const walkTestFiles = ( rootDir ) => {
if ( ! fs . existsSync ( rootDir ) ) {
return [ ] ;
}
const entries = fs . readdirSync ( rootDir , { withFileTypes : true } ) ;
const files = [ ] ;
for ( const entry of entries ) {
const fullPath = path . join ( rootDir , entry . name ) ;
if ( entry . isDirectory ( ) ) {
files . push ( ... walkTestFiles ( fullPath ) ) ;
continue ;
}
if ( ! entry . isFile ( ) ) {
continue ;
}
if (
fullPath . endsWith ( ".test.ts" ) ||
fullPath . endsWith ( ".live.test.ts" ) ||
fullPath . endsWith ( ".e2e.test.ts" )
) {
files . push ( normalizeRepoPath ( fullPath ) ) ;
}
}
return files ;
} ;
const allKnownTestFiles = [
... new Set ( [
... walkTestFiles ( "src" ) ,
... walkTestFiles ( "extensions" ) ,
... walkTestFiles ( "test" ) ,
... walkTestFiles ( path . join ( "ui" , "src" , "ui" ) ) ,
] ) ,
] ;
const inferTarget = ( fileFilter ) => {
2026-03-18 16:57:27 +00:00
const isolated = unitBehaviorIsolatedFiles . includes ( fileFilter ) ;
2026-03-13 18:36:38 -05:00
if ( fileFilter . endsWith ( ".live.test.ts" ) ) {
return { owner : "live" , isolated } ;
}
if ( fileFilter . endsWith ( ".e2e.test.ts" ) ) {
return { owner : "e2e" , isolated } ;
}
2026-03-14 11:23:25 -07:00
if ( channelTestPrefixes . some ( ( prefix ) => fileFilter . startsWith ( prefix ) ) ) {
return { owner : "channels" , isolated } ;
}
2026-03-13 18:36:38 -05:00
if ( fileFilter . startsWith ( "extensions/" ) ) {
return { owner : "extensions" , isolated } ;
}
if ( fileFilter . startsWith ( "src/gateway/" ) ) {
return { owner : "gateway" , isolated } ;
}
if ( baseConfigPrefixes . some ( ( prefix ) => fileFilter . startsWith ( prefix ) ) ) {
return { owner : "base" , isolated } ;
}
if ( fileFilter . startsWith ( "src/" ) ) {
return { owner : "unit" , isolated } ;
}
return { owner : "base" , isolated } ;
} ;
2026-03-18 16:57:27 +00:00
const unitTimingManifest = loadUnitTimingManifest ( ) ;
2026-03-20 04:27:49 +00:00
const unitMemoryHotspotManifest = loadUnitMemoryHotspotManifest ( ) ;
2026-03-18 16:57:27 +00:00
const parseEnvNumber = ( name , fallback ) => {
const parsed = Number . parseInt ( process . env [ name ] ? ? "" , 10 ) ;
return Number . isFinite ( parsed ) && parsed >= 0 ? parsed : fallback ;
} ;
2026-03-18 18:19:12 +00:00
const allKnownUnitFiles = allKnownTestFiles . filter ( ( file ) => {
2026-03-18 12:16:07 -07:00
return isUnitConfigTestFile ( file ) ;
2026-03-18 18:19:12 +00:00
} ) ;
2026-03-18 16:57:27 +00:00
const defaultHeavyUnitFileLimit =
2026-03-19 07:47:07 -05:00
testProfile === "serial"
? 0
: isMacMiniProfile
? 90
: testProfile === "low"
2026-03-20 20:04:25 +00:00
? 32
2026-03-19 07:47:07 -05:00
: highMemLocalHost
? 80
: 60 ;
2026-03-18 16:57:27 +00:00
const defaultHeavyUnitLaneCount =
2026-03-19 07:47:07 -05:00
testProfile === "serial"
? 0
: isMacMiniProfile
? 6
: testProfile === "low"
2026-03-20 20:04:25 +00:00
? 3
2026-03-19 07:47:07 -05:00
: highMemLocalHost
? 5
: 4 ;
2026-03-18 16:57:27 +00:00
const heavyUnitFileLimit = parseEnvNumber (
"OPENCLAW_TEST_HEAVY_UNIT_FILE_LIMIT" ,
defaultHeavyUnitFileLimit ,
) ;
const heavyUnitLaneCount = parseEnvNumber (
"OPENCLAW_TEST_HEAVY_UNIT_LANES" ,
defaultHeavyUnitLaneCount ,
) ;
const heavyUnitMinDurationMs = parseEnvNumber ( "OPENCLAW_TEST_HEAVY_UNIT_MIN_MS" , 1200 ) ;
2026-03-20 04:27:49 +00:00
const defaultMemoryHeavyUnitFileLimit =
2026-03-20 05:00:11 +00:00
testProfile === "serial" ? 0 : isCI ? 64 : testProfile === "low" ? 8 : 16 ;
2026-03-20 04:27:49 +00:00
const memoryHeavyUnitFileLimit = parseEnvNumber (
"OPENCLAW_TEST_MEMORY_HEAVY_UNIT_FILE_LIMIT" ,
defaultMemoryHeavyUnitFileLimit ,
) ;
const memoryHeavyUnitMinDeltaKb = parseEnvNumber (
"OPENCLAW_TEST_MEMORY_HEAVY_UNIT_MIN_KB" ,
unitMemoryHotspotManifest . defaultMinDeltaKb ,
) ;
2026-03-20 04:43:09 +00:00
const { memoryHeavyFiles : memoryHeavyUnitFiles , timedHeavyFiles : timedHeavyUnitFiles } =
shouldSplitUnitRuns
? selectUnitHeavyFileGroups ( {
2026-03-18 16:57:27 +00:00
candidates : allKnownUnitFiles ,
2026-03-20 04:43:09 +00:00
behaviorOverrides : unitBehaviorOverrideSet ,
timedLimit : heavyUnitFileLimit ,
timedMinDurationMs : heavyUnitMinDurationMs ,
memoryLimit : memoryHeavyUnitFileLimit ,
memoryMinDeltaKb : memoryHeavyUnitMinDeltaKb ,
2026-03-18 16:57:27 +00:00
timings : unitTimingManifest ,
2026-03-20 04:27:49 +00:00
hotspots : unitMemoryHotspotManifest ,
} )
2026-03-20 04:43:09 +00:00
: {
memoryHeavyFiles : [ ] ,
timedHeavyFiles : [ ] ,
} ;
2026-03-20 17:09:46 +00:00
const unitSingletonBatchFiles = dedupeFilesPreserveOrder (
unitSingletonIsolatedFiles ,
new Set ( unitBehaviorIsolatedFiles ) ,
) ;
const unitMemorySingletonFiles = dedupeFilesPreserveOrder (
memoryHeavyUnitFiles ,
new Set ( [ ... unitBehaviorOverrideSet , ... unitSingletonBatchFiles ] ) ,
) ;
2026-03-20 04:43:09 +00:00
const unitSchedulingOverrideSet = new Set ( [ ... unitBehaviorOverrideSet , ... memoryHeavyUnitFiles ] ) ;
2026-03-18 16:57:27 +00:00
const unitFastExcludedFiles = [
2026-03-20 04:43:09 +00:00
... new Set ( [ ... unitSchedulingOverrideSet , ... timedHeavyUnitFiles , ... channelSingletonFiles ] ) ,
2026-03-20 04:27:49 +00:00
] ;
2026-03-20 17:09:46 +00:00
const defaultSingletonBatchLaneCount =
testProfile === "serial"
? 0
: unitSingletonBatchFiles . length === 0
? 0
: isCI
? Math . ceil ( unitSingletonBatchFiles . length / 6 )
: highMemLocalHost
? Math . ceil ( unitSingletonBatchFiles . length / 8 )
: lowMemLocalHost
? Math . ceil ( unitSingletonBatchFiles . length / 12 )
: Math . ceil ( unitSingletonBatchFiles . length / 10 ) ;
const singletonBatchLaneCount =
unitSingletonBatchFiles . length === 0
? 0
: Math . min (
unitSingletonBatchFiles . length ,
Math . max (
1 ,
parseEnvNumber ( "OPENCLAW_TEST_SINGLETON_ISOLATED_LANES" , defaultSingletonBatchLaneCount ) ,
) ,
) ;
2026-03-18 16:57:27 +00:00
const estimateUnitDurationMs = ( file ) =>
unitTimingManifest . files [ file ] ? . durationMs ? ? unitTimingManifest . defaultDurationMs ;
2026-03-20 17:09:46 +00:00
const unitSingletonBuckets =
singletonBatchLaneCount > 0
? packFilesByDuration ( unitSingletonBatchFiles , singletonBatchLaneCount , estimateUnitDurationMs )
: [ ] ;
2026-03-20 05:08:39 +00:00
const unitFastExcludedFileSet = new Set ( unitFastExcludedFiles ) ;
const unitFastCandidateFiles = allKnownUnitFiles . filter (
( file ) => ! unitFastExcludedFileSet . has ( file ) ,
) ;
2026-03-19 23:29:22 -07:00
const defaultUnitFastLaneCount = isCI && ! isWindows ? 3 : 1 ;
2026-03-20 05:08:39 +00:00
const unitFastLaneCount = Math . max (
1 ,
parseEnvNumber ( "OPENCLAW_TEST_UNIT_FAST_LANES" , defaultUnitFastLaneCount ) ,
) ;
2026-03-19 23:29:22 -07:00
// Heap snapshots on current main show long-lived unit-fast workers retaining
// transformed Vitest/Vite module graphs rather than app objects. Multiple
// bounded unit-fast lanes only help if we also recycle them serially instead
// of keeping several transform-heavy workers resident at the same time.
2026-03-20 05:08:39 +00:00
const unitFastBuckets =
unitFastLaneCount > 1
? packFilesByDuration ( unitFastCandidateFiles , unitFastLaneCount , estimateUnitDurationMs )
: [ unitFastCandidateFiles ] ;
const unitFastEntries = unitFastBuckets
. filter ( ( files ) => files . length > 0 )
. map ( ( files , index ) => ( {
name : unitFastBuckets . length === 1 ? "unit-fast" : ` unit-fast- ${ String ( index + 1 ) } ` ,
2026-03-19 23:29:22 -07:00
serialPhase : "unit-fast" ,
2026-03-20 05:08:39 +00:00
env : {
OPENCLAW _VITEST _INCLUDE _FILE : writeTempJsonArtifact (
` vitest-unit-fast-include- ${ String ( index + 1 ) } ` ,
files ,
) ,
} ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
` --pool= ${ useVmForks ? "vmForks" : "forks" } ` ,
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
] ,
} ) ) ;
2026-03-18 16:57:27 +00:00
const heavyUnitBuckets = packFilesByDuration (
timedHeavyUnitFiles ,
heavyUnitLaneCount ,
estimateUnitDurationMs ,
) ;
const unitHeavyEntries = heavyUnitBuckets . map ( ( files , index ) => ( {
name : ` unit-heavy- ${ String ( index + 1 ) } ` ,
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=forks" , ... files ] ,
} ) ) ;
2026-03-20 17:09:46 +00:00
const unitSingletonEntries = unitSingletonBuckets . map ( ( files , index ) => ( {
name :
unitSingletonBuckets . length === 1 ? "unit-singleton" : ` unit-singleton- ${ String ( index + 1 ) } ` ,
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=forks" , ... files ] ,
} ) ) ;
2026-03-18 16:57:27 +00:00
const baseRuns = [
... ( shouldSplitUnitRuns
? [
2026-03-20 05:08:39 +00:00
... unitFastEntries ,
2026-03-18 16:57:27 +00:00
... ( unitBehaviorIsolatedFiles . length > 0
? [
{
name : "unit-isolated" ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
"--pool=forks" ,
... unitBehaviorIsolatedFiles ,
] ,
} ,
]
: [ ] ) ,
... unitHeavyEntries ,
2026-03-20 17:09:46 +00:00
... unitSingletonEntries ,
... unitMemorySingletonFiles . map ( ( file ) => ( {
2026-03-18 16:57:27 +00:00
name : ` ${ path . basename ( file , ".test.ts" ) } -isolated ` ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
` --pool= ${ useVmForks ? "vmForks" : "forks" } ` ,
file ,
] ,
} ) ) ,
... unitThreadSingletonFiles . map ( ( file ) => ( {
name : ` ${ path . basename ( file , ".test.ts" ) } -threads ` ,
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=threads" , file ] ,
} ) ) ,
... unitVmForkSingletonFiles . map ( ( file ) => ( {
name : ` ${ path . basename ( file , ".test.ts" ) } -vmforks ` ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
` --pool= ${ useVmForks ? "vmForks" : "forks" } ` ,
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
file ,
] ,
} ) ) ,
... channelSingletonFiles . map ( ( file ) => ( {
name : ` ${ path . basename ( file , ".test.ts" ) } -channels-isolated ` ,
args : [ "vitest" , "run" , "--config" , "vitest.channels.config.ts" , "--pool=forks" , file ] ,
} ) ) ,
]
: [
{
name : "unit" ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
` --pool= ${ useVmForks ? "vmForks" : "forks" } ` ,
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
] ,
} ,
] ) ,
... ( includeExtensionsSuite
? [
{
name : "extensions" ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.extensions.config.ts" ,
... ( useVmForks ? [ "--pool=vmForks" ] : [ ] ) ,
] ,
} ,
]
: [ ] ) ,
... ( includeGatewaySuite
? [
{
name : "gateway" ,
args : [ "vitest" , "run" , "--config" , "vitest.gateway.config.ts" , "--pool=forks" ] ,
} ,
]
: [ ] ) ,
] ;
runs = baseRuns ;
const formatEntrySummary = ( entry ) => {
const explicitFilters = countExplicitEntryFilters ( entry . args ) ? ? 0 ;
return ` ${ entry . name } filters= ${ String ( explicitFilters || "all" ) } maxWorkers= ${ String (
maxWorkersForRun ( entry . name ) ? ? "default" ,
) } ` ;
} ;
2026-03-13 18:36:38 -05:00
const resolveFilterMatches = ( fileFilter ) => {
const normalizedFilter = normalizeRepoPath ( fileFilter ) ;
if ( fs . existsSync ( fileFilter ) ) {
const stats = fs . statSync ( fileFilter ) ;
if ( stats . isFile ( ) ) {
return [ normalizedFilter ] ;
}
if ( stats . isDirectory ( ) ) {
const prefix = normalizedFilter . endsWith ( "/" ) ? normalizedFilter : ` ${ normalizedFilter } / ` ;
return allKnownTestFiles . filter ( ( file ) => file . startsWith ( prefix ) ) ;
}
}
if ( /[*?[\]{}]/ . test ( normalizedFilter ) ) {
return allKnownTestFiles . filter ( ( file ) => path . matchesGlob ( file , normalizedFilter ) ) ;
}
return allKnownTestFiles . filter ( ( file ) => file . includes ( normalizedFilter ) ) ;
} ;
2026-03-17 06:53:29 +00:00
const isVmForkSingletonUnitFile = ( fileFilter ) => unitVmForkSingletonFiles . includes ( fileFilter ) ;
2026-03-18 03:39:02 +00:00
const isThreadSingletonUnitFile = ( fileFilter ) => unitThreadSingletonFiles . includes ( fileFilter ) ;
2026-03-13 18:36:38 -05:00
const createTargetedEntry = ( owner , isolated , filters ) => {
const name = isolated ? ` ${ owner } -isolated ` : owner ;
const forceForks = isolated ;
2026-03-17 06:53:29 +00:00
if ( owner === "unit-vmforks" ) {
return {
name ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
` --pool= ${ useVmForks ? "vmForks" : "forks" } ` ,
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
... filters ,
] ,
} ;
}
2026-03-13 18:36:38 -05:00
if ( owner === "unit" ) {
return {
name ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
` --pool= ${ forceForks ? "forks" : useVmForks ? "vmForks" : "forks" } ` ,
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
... filters ,
] ,
} ;
}
2026-03-18 03:39:02 +00:00
if ( owner === "unit-threads" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=threads" , ... filters ] ,
} ;
}
2026-03-13 18:36:38 -05:00
if ( owner === "extensions" ) {
return {
name ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.extensions.config.ts" ,
... ( forceForks ? [ "--pool=forks" ] : useVmForks ? [ "--pool=vmForks" ] : [ ] ) ,
... filters ,
] ,
} ;
}
if ( owner === "gateway" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.gateway.config.ts" , "--pool=forks" , ... filters ] ,
} ;
}
if ( owner === "channels" ) {
return {
name ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.channels.config.ts" ,
2026-03-18 15:54:02 +05:30
... ( forceForks ? [ "--pool=forks" ] : useVmForks ? [ "--pool=vmForks" ] : [ ] ) ,
2026-03-13 18:36:38 -05:00
... filters ,
] ,
} ;
}
if ( owner === "live" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.live.config.ts" , ... filters ] ,
} ;
}
if ( owner === "e2e" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.e2e.config.ts" , ... filters ] ,
} ;
}
return {
name ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.config.ts" ,
... ( forceForks ? [ "--pool=forks" ] : [ ] ) ,
... filters ,
] ,
} ;
} ;
const targetedEntries = ( ( ) => {
if ( passthroughFileFilters . length === 0 ) {
return [ ] ;
}
const groups = passthroughFileFilters . reduce ( ( acc , fileFilter ) => {
const matchedFiles = resolveFilterMatches ( fileFilter ) ;
if ( matchedFiles . length === 0 ) {
2026-03-17 06:53:29 +00:00
const normalizedFile = normalizeRepoPath ( fileFilter ) ;
const target = inferTarget ( normalizedFile ) ;
2026-03-18 03:39:02 +00:00
const owner = isThreadSingletonUnitFile ( normalizedFile )
? "unit-threads"
: isVmForkSingletonUnitFile ( normalizedFile )
? "unit-vmforks"
: target . owner ;
2026-03-17 06:53:29 +00:00
const key = ` ${ owner } : ${ target . isolated ? "isolated" : "default" } ` ;
2026-03-13 18:36:38 -05:00
const files = acc . get ( key ) ? ? [ ] ;
2026-03-17 06:53:29 +00:00
files . push ( normalizedFile ) ;
2026-03-13 18:36:38 -05:00
acc . set ( key , files ) ;
return acc ;
}
for ( const matchedFile of matchedFiles ) {
const target = inferTarget ( matchedFile ) ;
2026-03-18 03:39:02 +00:00
const owner = isThreadSingletonUnitFile ( matchedFile )
? "unit-threads"
: isVmForkSingletonUnitFile ( matchedFile )
? "unit-vmforks"
: target . owner ;
2026-03-17 06:53:29 +00:00
const key = ` ${ owner } : ${ target . isolated ? "isolated" : "default" } ` ;
2026-03-13 18:36:38 -05:00
const files = acc . get ( key ) ? ? [ ] ;
files . push ( matchedFile ) ;
acc . set ( key , files ) ;
}
return acc ;
} , new Map ( ) ) ;
return Array . from ( groups , ( [ key , filters ] ) => {
const [ owner , mode ] = key . split ( ":" ) ;
return createTargetedEntry ( owner , mode === "isolated" , [ ... new Set ( filters ) ] ) ;
} ) ;
} ) ( ) ;
2026-03-18 03:39:02 +00:00
// Node 25 local runs still show cross-process worker shutdown contention even
// after moving the known heavy files into singleton lanes.
const topLevelParallelEnabled =
2026-03-19 07:47:07 -05:00
testProfile !== "low" &&
testProfile !== "serial" &&
! ( ! isCI && nodeMajor >= 25 ) &&
! isMacMiniProfile ;
2026-03-20 01:36:12 +00:00
const defaultTopLevelParallelLimit =
testProfile === "serial"
? 1
: testProfile === "low"
? 2
: testProfile === "max"
? 5
: highMemLocalHost
? 4
: lowMemLocalHost
? 2
: 3 ;
const topLevelParallelLimit = Math . max (
1 ,
parseEnvNumber ( "OPENCLAW_TEST_TOP_LEVEL_CONCURRENCY" , defaultTopLevelParallelLimit ) ,
) ;
2026-01-30 03:15:10 +01:00
const overrideWorkers = Number . parseInt ( process . env . OPENCLAW _TEST _WORKERS ? ? "" , 10 ) ;
2026-01-31 21:21:09 +09:00
const resolvedOverride =
Number . isFinite ( overrideWorkers ) && overrideWorkers > 0 ? overrideWorkers : null ;
2026-02-23 20:48:05 +02:00
const parallelGatewayEnabled =
2026-03-19 07:47:07 -05:00
! isMacMiniProfile &&
( process . env . OPENCLAW _TEST _PARALLEL _GATEWAY === "1" || ( ! isCI && highMemLocalHost ) ) ;
2026-02-23 20:48:05 +02:00
// Keep gateway serial by default except when explicitly requested or on high-memory local hosts.
2026-02-12 17:59:44 +00:00
const keepGatewaySerial =
isWindowsCi ||
process . env . OPENCLAW _TEST _SERIAL _GATEWAY === "1" ||
2026-02-15 07:40:13 -06:00
testProfile === "serial" ||
2026-02-23 20:48:05 +02:00
! parallelGatewayEnabled ;
2026-02-12 17:59:44 +00:00
const parallelRuns = keepGatewaySerial ? runs . filter ( ( entry ) => entry . name !== "gateway" ) : runs ;
const serialRuns = keepGatewaySerial ? runs . filter ( ( entry ) => entry . name === "gateway" ) : [ ] ;
2026-03-19 23:29:22 -07:00
const serialPrefixRuns = parallelRuns . filter ( ( entry ) => entry . serialPhase ) ;
const deferredParallelRuns = parallelRuns . filter ( ( entry ) => ! entry . serialPhase ) ;
2026-02-22 21:59:13 +00:00
const baseLocalWorkers = Math . max ( 4 , Math . min ( 16 , hostCpuCount ) ) ;
const loadAwareDisabledRaw = process . env . OPENCLAW _TEST _LOAD _AWARE ? . trim ( ) . toLowerCase ( ) ;
const loadAwareDisabled = loadAwareDisabledRaw === "0" || loadAwareDisabledRaw === "false" ;
const loadRatio =
! isCI && ! loadAwareDisabled && process . platform !== "win32" && hostCpuCount > 0
? os . loadavg ( ) [ 0 ] / hostCpuCount
: 0 ;
// Keep the fast-path unchanged on normal load; only throttle under extreme host pressure.
const extremeLoadScale = loadRatio >= 1.1 ? 0.75 : loadRatio >= 1 ? 0.85 : 1 ;
const localWorkers = Math . max ( 4 , Math . min ( 16 , Math . floor ( baseLocalWorkers * extremeLoadScale ) ) ) ;
2026-02-15 07:40:13 -06:00
const defaultWorkerBudget =
testProfile === "low"
? {
unit : 2 ,
unitIsolated : 1 ,
2026-02-25 12:16:17 +02:00
extensions : 4 ,
2026-02-15 07:40:13 -06:00
gateway : 1 ,
}
2026-03-19 07:47:07 -05:00
: isMacMiniProfile
2026-02-15 07:40:13 -06:00
? {
2026-03-19 07:47:07 -05:00
unit : 3 ,
2026-02-15 07:40:13 -06:00
unitIsolated : 1 ,
extensions : 1 ,
gateway : 1 ,
}
2026-03-19 07:47:07 -05:00
: testProfile === "serial"
2026-02-15 07:40:13 -06:00
? {
2026-03-19 07:47:07 -05:00
unit : 1 ,
unitIsolated : 1 ,
extensions : 1 ,
gateway : 1 ,
2026-02-15 07:40:13 -06:00
}
2026-03-19 07:47:07 -05:00
: testProfile === "max"
2026-02-23 20:48:05 +02:00
? {
2026-03-19 07:47:07 -05:00
unit : localWorkers ,
unitIsolated : Math . min ( 4 , localWorkers ) ,
extensions : Math . max ( 1 , Math . min ( 6 , Math . floor ( localWorkers / 2 ) ) ) ,
gateway : Math . max ( 1 , Math . min ( 2 , Math . floor ( localWorkers / 4 ) ) ) ,
2026-02-23 20:48:05 +02:00
}
2026-03-19 07:47:07 -05:00
: highMemLocalHost
2026-02-23 20:48:05 +02:00
? {
2026-03-19 07:47:07 -05:00
// After peeling measured hotspots into dedicated lanes, the shared
// unit-fast lane shuts down more reliably with a slightly smaller
// worker fan-out than the old "max it out" local default.
unit : Math . max ( 4 , Math . min ( 10 , Math . floor ( ( localWorkers * 5 ) / 8 ) ) ) ,
unitIsolated : Math . max ( 1 , Math . min ( 2 , Math . floor ( localWorkers / 6 ) || 1 ) ) ,
2026-02-23 20:48:05 +02:00
extensions : Math . max ( 1 , Math . min ( 4 , Math . floor ( localWorkers / 4 ) ) ) ,
2026-03-19 07:47:07 -05:00
gateway : Math . max ( 2 , Math . min ( 6 , Math . floor ( localWorkers / 2 ) ) ) ,
}
: lowMemLocalHost
? {
// Sub-64 GiB local hosts are prone to OOM with large vmFork runs.
unit : 2 ,
unitIsolated : 1 ,
extensions : 4 ,
gateway : 1 ,
}
: {
// 64-95 GiB local hosts: conservative split with some parallel headroom.
unit : Math . max ( 2 , Math . min ( 8 , Math . floor ( localWorkers / 2 ) ) ) ,
unitIsolated : 1 ,
extensions : Math . max ( 1 , Math . min ( 4 , Math . floor ( localWorkers / 4 ) ) ) ,
gateway : 1 ,
} ;
2026-02-07 07:57:50 +00:00
2026-01-25 07:22:36 -05:00
// Keep worker counts predictable for local runs; trim macOS CI workers to avoid worker crashes/OOM.
2026-01-25 10:18:51 +05:30
// In CI on linux/windows, prefer Vitest defaults to avoid cross-test interference from lower worker counts.
2026-02-07 07:57:50 +00:00
const maxWorkersForRun = ( name ) => {
if ( resolvedOverride ) {
return resolvedOverride ;
}
2026-03-20 17:09:46 +00:00
if ( name === "unit-singleton" || name . startsWith ( "unit-singleton-" ) ) {
return 1 ;
}
2026-02-07 07:57:50 +00:00
if ( isCI && ! isMacOS ) {
return null ;
}
if ( isCI && isMacOS ) {
return 1 ;
}
2026-03-18 16:57:27 +00:00
if ( name . endsWith ( "-threads" ) || name . endsWith ( "-vmforks" ) ) {
return 1 ;
}
if ( name . endsWith ( "-isolated" ) && name !== "unit-isolated" ) {
return 1 ;
}
if ( name === "unit-isolated" || name . startsWith ( "unit-heavy-" ) ) {
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . unitIsolated ;
2026-02-13 04:30:39 +00:00
}
2026-02-07 07:57:50 +00:00
if ( name === "extensions" ) {
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . extensions ;
2026-02-07 07:57:50 +00:00
}
if ( name === "gateway" ) {
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . gateway ;
2026-02-07 07:57:50 +00:00
}
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . unit ;
2026-02-07 07:57:50 +00:00
} ;
2026-01-23 07:34:50 +00:00
2026-01-24 11:16:41 +00:00
const WARNING _SUPPRESSION _FLAGS = [
"--disable-warning=ExperimentalWarning" ,
"--disable-warning=DEP0040" ,
"--disable-warning=DEP0060" ,
2026-02-13 13:28:23 +00:00
"--disable-warning=MaxListenersExceededWarning" ,
2026-01-24 11:16:41 +00:00
] ;
2026-02-15 05:06:58 +00:00
const DEFAULT _CI _MAX _OLD _SPACE _SIZE _MB = 4096 ;
const maxOldSpaceSizeMb = ( ( ) => {
// CI can hit Node heap limits (especially on large suites). Allow override, default to 4GB.
const raw = process . env . OPENCLAW _TEST _MAX _OLD _SPACE _SIZE _MB ? ? "" ;
const parsed = Number . parseInt ( raw , 10 ) ;
if ( Number . isFinite ( parsed ) && parsed > 0 ) {
return parsed ;
}
if ( isCI && ! isWindows ) {
return DEFAULT _CI _MAX _OLD _SPACE _SIZE _MB ;
}
return null ;
} ) ( ) ;
2026-03-18 16:57:27 +00:00
const formatElapsedMs = ( elapsedMs ) =>
elapsedMs >= 1000 ? ` ${ ( elapsedMs / 1000 ) . toFixed ( 1 ) } s ` : ` ${ Math . round ( elapsedMs ) } ms ` ;
2026-03-19 14:02:19 -07:00
const formatMemoryKb = ( rssKb ) =>
rssKb >= 1024 * * 2
? ` ${ ( rssKb / 1024 * * 2 ) . toFixed ( 2 ) } GiB `
: rssKb >= 1024
? ` ${ ( rssKb / 1024 ) . toFixed ( 1 ) } MiB `
: ` ${ rssKb } KiB ` ;
const formatMemoryDeltaKb = ( rssKb ) =>
` ${ rssKb >= 0 ? "+" : "-" } ${ formatMemoryKb ( Math . abs ( rssKb ) ) } ` ;
const rawMemoryTrace = process . env . OPENCLAW _TEST _MEMORY _TRACE ? . trim ( ) . toLowerCase ( ) ;
const memoryTraceEnabled =
process . platform !== "win32" &&
( rawMemoryTrace === "1" ||
rawMemoryTrace === "true" ||
( rawMemoryTrace !== "0" && rawMemoryTrace !== "false" && isCI ) ) ;
const memoryTracePollMs = Math . max ( 250 , parseEnvNumber ( "OPENCLAW_TEST_MEMORY_TRACE_POLL_MS" , 1000 ) ) ;
const memoryTraceTopCount = Math . max ( 1 , parseEnvNumber ( "OPENCLAW_TEST_MEMORY_TRACE_TOP_COUNT" , 6 ) ) ;
2026-03-19 17:59:13 -04:00
const heapSnapshotIntervalMs = Math . max (
0 ,
parseEnvNumber ( "OPENCLAW_TEST_HEAPSNAPSHOT_INTERVAL_MS" , 0 ) ,
) ;
const heapSnapshotMinIntervalMs = 5000 ;
const heapSnapshotEnabled =
2026-03-19 15:05:56 -07:00
process . platform !== "win32" && heapSnapshotIntervalMs >= heapSnapshotMinIntervalMs ;
2026-03-19 17:59:13 -04:00
const heapSnapshotSignal = process . env . OPENCLAW _TEST _HEAPSNAPSHOT _SIGNAL ? . trim ( ) || "SIGUSR2" ;
const heapSnapshotBaseDir = heapSnapshotEnabled
? path . resolve (
process . env . OPENCLAW _TEST _HEAPSNAPSHOT _DIR ? . trim ( ) ||
path . join ( os . tmpdir ( ) , ` openclaw-heapsnapshots- ${ Date . now ( ) } ` ) ,
)
: null ;
const ensureNodeOptionFlag = ( nodeOptions , flagPrefix , nextValue ) =>
nodeOptions . includes ( flagPrefix ) ? nodeOptions : ` ${ nodeOptions } ${ nextValue } ` . trim ( ) ;
const isNodeLikeProcess = ( command ) => / ( ? : ^ | \ / ) node ( ? : $ | \ . exe$ ) / iu . test ( command ) ;
2026-02-15 05:06:58 +00:00
2026-01-27 16:39:28 +00:00
const runOnce = ( entry , extraArgs = [ ] ) =>
2026-01-23 07:34:50 +00:00
new Promise ( ( resolve ) => {
2026-03-18 16:57:27 +00:00
const startedAt = Date . now ( ) ;
2026-02-07 07:57:50 +00:00
const maxWorkers = maxWorkersForRun ( entry . name ) ;
2026-02-25 12:16:17 +02:00
// vmForks with a single worker has shown cross-file leakage in extension suites.
// Fall back to process forks when we intentionally clamp that lane to one worker.
const entryArgs =
entry . name === "extensions" && maxWorkers === 1 && entry . args . includes ( "--pool=vmForks" )
? entry . args . map ( ( arg ) => ( arg === "--pool=vmForks" ? "--pool=forks" : arg ) )
: entry . args ;
2026-03-19 14:02:19 -07:00
const explicitEntryFilters = getExplicitEntryFilters ( entryArgs ) ;
2026-01-27 16:39:28 +00:00
const args = maxWorkers
2026-02-07 08:27:50 +00:00
? [
2026-02-25 12:16:17 +02:00
... entryArgs ,
2026-02-07 08:27:50 +00:00
"--maxWorkers" ,
String ( maxWorkers ) ,
2026-02-12 17:59:44 +00:00
... silentArgs ,
2026-02-07 08:27:50 +00:00
... windowsCiArgs ,
... extraArgs ,
]
2026-03-01 13:03:06 -08:00
: [ ... entryArgs , ... silentArgs , ... windowsCiArgs , ... extraArgs ] ;
2026-03-18 16:57:27 +00:00
console . log (
` [test-parallel] start ${ entry . name } workers= ${ maxWorkers ? ? "default" } filters= ${ String (
countExplicitEntryFilters ( entryArgs ) ? ? "all" ,
) } ` ,
) ;
2026-01-24 11:16:41 +00:00
const nodeOptions = process . env . NODE _OPTIONS ? ? "" ;
const nextNodeOptions = WARNING _SUPPRESSION _FLAGS . reduce (
( acc , flag ) => ( acc . includes ( flag ) ? acc : ` ${ acc } ${ flag } ` . trim ( ) ) ,
nodeOptions ,
) ;
2026-03-19 17:59:13 -04:00
const heapSnapshotDir =
heapSnapshotBaseDir === null ? null : path . join ( heapSnapshotBaseDir , entry . name ) ;
let resolvedNodeOptions =
2026-02-15 05:06:58 +00:00
maxOldSpaceSizeMb && ! nextNodeOptions . includes ( "--max-old-space-size=" )
2026-03-19 17:59:13 -04:00
? ` ${ nextNodeOptions } --max-old-space-size= ${ maxOldSpaceSizeMb } ` . trim ( )
: nextNodeOptions ;
if ( heapSnapshotEnabled && heapSnapshotDir ) {
try {
fs . mkdirSync ( heapSnapshotDir , { recursive : true } ) ;
} catch ( err ) {
2026-03-19 15:05:56 -07:00
console . error (
` [test-parallel] failed to create heap snapshot dir ${ heapSnapshotDir } : ${ String ( err ) } ` ,
) ;
2026-03-19 17:59:13 -04:00
resolve ( 1 ) ;
return ;
}
resolvedNodeOptions = ensureNodeOptionFlag (
resolvedNodeOptions ,
"--diagnostic-dir=" ,
` --diagnostic-dir= ${ heapSnapshotDir } ` ,
) ;
resolvedNodeOptions = ensureNodeOptionFlag (
resolvedNodeOptions ,
"--heapsnapshot-signal=" ,
` --heapsnapshot-signal= ${ heapSnapshotSignal } ` ,
) ;
}
2026-03-19 09:52:00 -07:00
let output = "" ;
2026-03-19 11:01:16 -07:00
let fatalSeen = false ;
let childError = null ;
2026-02-15 03:35:02 +00:00
let child ;
2026-03-19 14:02:19 -07:00
let pendingLine = "" ;
let memoryPollTimer = null ;
2026-03-19 17:59:13 -04:00
let heapSnapshotTimer = null ;
2026-03-19 14:02:19 -07:00
const memoryFileRecords = [ ] ;
let initialTreeSample = null ;
let latestTreeSample = null ;
let peakTreeSample = null ;
2026-03-19 17:59:13 -04:00
let heapSnapshotSequence = 0 ;
2026-03-19 14:02:19 -07:00
const updatePeakTreeSample = ( sample , reason ) => {
if ( ! sample ) {
return ;
}
if ( ! peakTreeSample || sample . rssKb > peakTreeSample . rssKb ) {
peakTreeSample = { ... sample , reason } ;
}
} ;
2026-03-19 17:59:13 -04:00
const triggerHeapSnapshot = ( reason ) => {
if ( ! heapSnapshotEnabled || ! child ? . pid || ! heapSnapshotDir ) {
return ;
}
const records = getProcessTreeRecords ( child . pid ) ? ? [ ] ;
const targetPids = records
. filter ( ( record ) => record . pid !== process . pid && isNodeLikeProcess ( record . command ) )
. map ( ( record ) => record . pid ) ;
if ( targetPids . length === 0 ) {
return ;
}
heapSnapshotSequence += 1 ;
let signaledCount = 0 ;
for ( const pid of targetPids ) {
try {
process . kill ( pid , heapSnapshotSignal ) ;
signaledCount += 1 ;
} catch {
// Process likely exited between ps sampling and signal delivery.
}
}
if ( signaledCount > 0 ) {
console . log (
` [test-parallel][heap] ${ entry . name } seq= ${ String ( heapSnapshotSequence ) } reason= ${ reason } signaled= ${ String (
signaledCount ,
) } / $ { String ( targetPids . length ) } dir = $ { heapSnapshotDir } ` ,
) ;
}
} ;
2026-03-19 14:02:19 -07:00
const captureTreeSample = ( reason ) => {
if ( ! memoryTraceEnabled || ! child ? . pid ) {
return null ;
}
const sample = sampleProcessTreeRssKb ( child . pid ) ;
if ( ! sample ) {
return null ;
}
latestTreeSample = sample ;
if ( ! initialTreeSample ) {
initialTreeSample = sample ;
}
updatePeakTreeSample ( sample , reason ) ;
return sample ;
} ;
const logMemoryTraceForText = ( text ) => {
if ( ! memoryTraceEnabled ) {
return ;
}
const combined = ` ${ pendingLine } ${ text } ` ;
const lines = combined . split ( /\r?\n/u ) ;
pendingLine = lines . pop ( ) ? ? "" ;
const completedFiles = parseCompletedTestFileLines ( lines . join ( "\n" ) ) ;
for ( const completedFile of completedFiles ) {
const sample = captureTreeSample ( completedFile . file ) ;
if ( ! sample ) {
continue ;
}
const previousRssKb =
memoryFileRecords . length > 0
? ( memoryFileRecords . at ( - 1 ) ? . rssKb ? ? initialTreeSample ? . rssKb ? ? sample . rssKb )
: ( initialTreeSample ? . rssKb ? ? sample . rssKb ) ;
const deltaKb = sample . rssKb - previousRssKb ;
const record = {
... completedFile ,
rssKb : sample . rssKb ,
processCount : sample . processCount ,
deltaKb ,
} ;
memoryFileRecords . push ( record ) ;
console . log (
` [test-parallel][mem] ${ entry . name } file= ${ record . file } rss= ${ formatMemoryKb (
record . rssKb ,
) } delta = $ { formatMemoryDeltaKb ( record . deltaKb ) } peak = $ { formatMemoryKb (
peakTreeSample ? . rssKb ? ? record . rssKb ,
) } procs = $ { record . processCount } $ { record . durationMs ? ` duration= ${ formatElapsedMs ( record . durationMs ) } ` : "" } ` ,
) ;
}
} ;
const logMemoryTraceSummary = ( ) => {
if ( ! memoryTraceEnabled ) {
return ;
}
captureTreeSample ( "close" ) ;
const fallbackRecord =
memoryFileRecords . length === 0 &&
explicitEntryFilters . length === 1 &&
latestTreeSample &&
initialTreeSample
? [
{
file : explicitEntryFilters [ 0 ] ,
deltaKb : latestTreeSample . rssKb - initialTreeSample . rssKb ,
} ,
]
: [ ] ;
const totalDeltaKb =
initialTreeSample && latestTreeSample
? latestTreeSample . rssKb - initialTreeSample . rssKb
: 0 ;
const topGrowthFiles = [ ... memoryFileRecords , ... fallbackRecord ]
. filter ( ( record ) => record . deltaKb > 0 && typeof record . file === "string" )
. toSorted ( ( left , right ) => right . deltaKb - left . deltaKb )
. slice ( 0 , memoryTraceTopCount )
. map ( ( record ) => ` ${ record . file } : ${ formatMemoryDeltaKb ( record . deltaKb ) } ` ) ;
console . log (
` [test-parallel][mem] summary ${ entry . name } files= ${ memoryFileRecords . length } peak= ${ formatMemoryKb (
peakTreeSample ? . rssKb ? ? 0 ,
) } totalDelta = $ { formatMemoryDeltaKb ( totalDeltaKb ) } peakAt = $ {
peakTreeSample ? . reason ? ? "n/a"
} top = $ { topGrowthFiles . length > 0 ? topGrowthFiles . join ( ", " ) : "none" } ` ,
) ;
} ;
2026-02-15 03:35:02 +00:00
try {
child = spawn ( pnpm , args , {
2026-03-19 09:52:00 -07:00
stdio : [ "inherit" , "pipe" , "pipe" ] ,
2026-03-20 04:48:24 +00:00
env : {
... process . env ,
... entry . env ,
VITEST _GROUP : entry . name ,
NODE _OPTIONS : resolvedNodeOptions ,
} ,
2026-02-15 03:35:02 +00:00
shell : isWindows ,
} ) ;
2026-03-19 14:02:19 -07:00
captureTreeSample ( "spawn" ) ;
if ( memoryTraceEnabled ) {
memoryPollTimer = setInterval ( ( ) => {
captureTreeSample ( "poll" ) ;
} , memoryTracePollMs ) ;
}
2026-03-19 17:59:13 -04:00
if ( heapSnapshotEnabled ) {
heapSnapshotTimer = setInterval ( ( ) => {
triggerHeapSnapshot ( "interval" ) ;
} , heapSnapshotIntervalMs ) ;
}
2026-02-15 03:35:02 +00:00
} catch ( err ) {
console . error ( ` [test-parallel] spawn failed: ${ String ( err ) } ` ) ;
resolve ( 1 ) ;
return ;
}
2026-01-23 07:34:50 +00:00
children . add ( child ) ;
2026-03-19 09:52:00 -07:00
child . stdout ? . on ( "data" , ( chunk ) => {
const text = chunk . toString ( ) ;
2026-03-19 11:01:16 -07:00
fatalSeen || = hasFatalTestRunOutput ( ` ${ output } ${ text } ` ) ;
2026-03-19 09:52:00 -07:00
output = appendCapturedOutput ( output , text ) ;
2026-03-19 14:02:19 -07:00
logMemoryTraceForText ( text ) ;
2026-03-19 09:52:00 -07:00
process . stdout . write ( chunk ) ;
} ) ;
child . stderr ? . on ( "data" , ( chunk ) => {
const text = chunk . toString ( ) ;
2026-03-19 11:01:16 -07:00
fatalSeen || = hasFatalTestRunOutput ( ` ${ output } ${ text } ` ) ;
2026-03-19 09:52:00 -07:00
output = appendCapturedOutput ( output , text ) ;
2026-03-19 14:02:19 -07:00
logMemoryTraceForText ( text ) ;
2026-03-19 09:52:00 -07:00
process . stderr . write ( chunk ) ;
} ) ;
2026-02-15 03:35:02 +00:00
child . on ( "error" , ( err ) => {
2026-03-19 11:01:16 -07:00
childError = err ;
2026-02-15 03:35:02 +00:00
console . error ( ` [test-parallel] child error: ${ String ( err ) } ` ) ;
} ) ;
2026-03-19 09:52:00 -07:00
child . on ( "close" , ( code , signal ) => {
2026-03-19 14:02:19 -07:00
if ( memoryPollTimer ) {
clearInterval ( memoryPollTimer ) ;
}
2026-03-19 17:59:13 -04:00
if ( heapSnapshotTimer ) {
clearInterval ( heapSnapshotTimer ) ;
}
2026-01-23 07:34:50 +00:00
children . delete ( child ) ;
2026-03-19 11:01:16 -07:00
const resolvedCode = resolveTestRunExitCode ( { code , signal , output , fatalSeen , childError } ) ;
2026-03-19 14:02:19 -07:00
logMemoryTraceSummary ( ) ;
2026-03-18 16:57:27 +00:00
console . log (
2026-03-19 09:52:00 -07:00
` [test-parallel] done ${ entry . name } code= ${ String ( resolvedCode ) } elapsed= ${ formatElapsedMs ( Date . now ( ) - startedAt ) } ` ,
2026-03-18 16:57:27 +00:00
) ;
2026-03-19 09:52:00 -07:00
resolve ( resolvedCode ) ;
2026-01-23 07:34:50 +00:00
} ) ;
} ) ;
2026-03-13 18:36:38 -05:00
const run = async ( entry , extraArgs = [ ] ) => {
2026-03-18 08:58:29 -07:00
const explicitFilterCount = countExplicitEntryFilters ( entry . args ) ;
2026-03-18 12:16:07 -07:00
// Vitest requires the shard count to stay strictly below the number of
// resolved test files, so explicit-filter lanes need a `< fileCount` cap.
2026-03-18 08:58:29 -07:00
const effectiveShardCount =
2026-03-18 12:16:07 -07:00
explicitFilterCount === null
? shardCount
: Math . min ( shardCount , Math . max ( 1 , explicitFilterCount - 1 ) ) ;
2026-03-18 08:58:29 -07:00
if ( effectiveShardCount <= 1 ) {
if ( shardIndexOverride !== null && shardIndexOverride > effectiveShardCount ) {
return 0 ;
}
2026-03-13 18:36:38 -05:00
return runOnce ( entry , extraArgs ) ;
2026-01-31 21:29:14 +09:00
}
2026-02-26 00:33:36 -06:00
if ( shardIndexOverride !== null ) {
2026-03-18 08:58:29 -07:00
if ( shardIndexOverride > effectiveShardCount ) {
return 0 ;
}
return runOnce ( entry , [
"--shard" ,
` ${ shardIndexOverride } / ${ effectiveShardCount } ` ,
... extraArgs ,
] ) ;
2026-02-26 00:33:36 -06:00
}
2026-03-18 08:58:29 -07:00
for ( let shardIndex = 1 ; shardIndex <= effectiveShardCount ; shardIndex += 1 ) {
2026-01-27 16:39:28 +00:00
// eslint-disable-next-line no-await-in-loop
2026-03-18 08:58:29 -07:00
const code = await runOnce ( entry , [
"--shard" ,
` ${ shardIndex } / ${ effectiveShardCount } ` ,
... extraArgs ,
] ) ;
2026-01-31 21:29:14 +09:00
if ( code !== 0 ) {
return code ;
}
2026-01-27 16:39:28 +00:00
}
return 0 ;
} ;
2026-03-19 07:47:07 -05:00
const runEntriesWithLimit = async ( entries , extraArgs = [ ] , concurrency = 1 ) => {
if ( entries . length === 0 ) {
return undefined ;
}
const normalizedConcurrency = Math . max ( 1 , Math . floor ( concurrency ) ) ;
if ( normalizedConcurrency <= 1 ) {
for ( const entry of entries ) {
// eslint-disable-next-line no-await-in-loop
const code = await run ( entry , extraArgs ) ;
if ( code !== 0 ) {
return code ;
}
}
return undefined ;
}
let nextIndex = 0 ;
let firstFailure ;
const worker = async ( ) => {
while ( firstFailure === undefined ) {
const entryIndex = nextIndex ;
nextIndex += 1 ;
if ( entryIndex >= entries . length ) {
return ;
}
const code = await run ( entries [ entryIndex ] , extraArgs ) ;
if ( code !== 0 && firstFailure === undefined ) {
firstFailure = code ;
}
}
} ;
const workerCount = Math . min ( normalizedConcurrency , entries . length ) ;
await Promise . all ( Array . from ( { length : workerCount } , ( ) => worker ( ) ) ) ;
return firstFailure ;
} ;
2026-03-13 18:36:38 -05:00
const runEntries = async ( entries , extraArgs = [ ] ) => {
2026-03-06 17:45:35 -05:00
if ( topLevelParallelEnabled ) {
2026-03-20 01:36:12 +00:00
// Keep a bounded number of top-level Vitest processes in flight. As the
// singleton lane list grows, unbounded Promise.all scheduling turns
// isolation into cross-process contention and can reintroduce timeouts.
return runEntriesWithLimit ( entries , extraArgs , topLevelParallelLimit ) ;
2026-03-06 17:45:35 -05:00
}
2026-03-19 07:47:07 -05:00
return runEntriesWithLimit ( entries , extraArgs ) ;
2026-03-06 17:45:35 -05:00
} ;
2026-01-23 07:34:50 +00:00
const shutdown = ( signal ) => {
for ( const child of children ) {
child . kill ( signal ) ;
}
} ;
process . on ( "SIGINT" , ( ) => shutdown ( "SIGINT" ) ) ;
process . on ( "SIGTERM" , ( ) => shutdown ( "SIGTERM" ) ) ;
2026-03-20 04:48:24 +00:00
process . on ( "exit" , cleanupTempArtifacts ) ;
2026-01-23 07:34:50 +00:00
2026-03-18 16:57:27 +00:00
if ( process . env . OPENCLAW _TEST _LIST _LANES === "1" ) {
const entriesToPrint = targetedEntries . length > 0 ? targetedEntries : runs ;
for ( const entry of entriesToPrint ) {
console . log ( formatEntrySummary ( entry ) ) ;
}
process . exit ( 0 ) ;
}
2026-03-19 07:47:07 -05:00
if ( passthroughMetadataOnly ) {
const exitCode = await runOnce (
{
name : "vitest-meta" ,
args : [ "vitest" , "run" ] ,
} ,
passthroughOptionArgs ,
) ;
process . exit ( exitCode ) ;
}
2026-03-13 18:36:38 -05:00
if ( targetedEntries . length > 0 ) {
if ( passthroughRequiresSingleRun && targetedEntries . length > 1 ) {
console . error (
"[test-parallel] The provided Vitest args require a single run, but the selected test filters span multiple wrapper configs. Run one target/config at a time." ,
) ;
process . exit ( 2 ) ;
}
const targetedParallelRuns = keepGatewaySerial
? targetedEntries . filter ( ( entry ) => entry . name !== "gateway" )
: targetedEntries ;
const targetedSerialRuns = keepGatewaySerial
? targetedEntries . filter ( ( entry ) => entry . name === "gateway" )
: [ ] ;
const failedTargetedParallel = await runEntries ( targetedParallelRuns , passthroughOptionArgs ) ;
if ( failedTargetedParallel !== undefined ) {
process . exit ( failedTargetedParallel ) ;
}
for ( const entry of targetedSerialRuns ) {
// eslint-disable-next-line no-await-in-loop
const code = await run ( entry , passthroughOptionArgs ) ;
if ( code !== 0 ) {
process . exit ( code ) ;
2026-02-15 03:35:02 +00:00
}
2026-03-13 18:36:38 -05:00
}
process . exit ( 0 ) ;
}
if ( passthroughRequiresSingleRun && passthroughOptionArgs . length > 0 ) {
console . error (
"[test-parallel] The provided Vitest args require a single run. Use the dedicated npm script for that workflow (for example `pnpm test:coverage`) or target a single test file/filter." ,
) ;
process . exit ( 2 ) ;
fix: cron scheduler reliability, store hardening, and UX improvements (#10776)
* refactor: update cron job wake mode and run mode handling
- Changed default wake mode from 'next-heartbeat' to 'now' in CronJobEditor and related CLI commands.
- Updated cron-tool tests to reflect changes in run mode, introducing 'due' and 'force' options.
- Enhanced cron-tool logic to handle new run modes and ensure compatibility with existing job structures.
- Added new tests for delivery plan consistency and job execution behavior under various conditions.
- Improved normalization functions to handle wake mode and session target casing.
This refactor aims to streamline cron job configurations and enhance the overall user experience with clearer defaults and improved functionality.
* test: enhance cron job functionality and UI
- Added tests to ensure the isolated agent correctly announces the final payload text when delivering messages via Telegram.
- Implemented a new function to pick the last deliverable payload from a list of delivery payloads.
- Enhanced the cron service to maintain legacy "every" jobs while minute cron jobs recompute schedules.
- Updated the cron store migration tests to verify the addition of anchorMs to legacy every schedules.
- Improved the UI for displaying cron job details, including job state and delivery information, with new styles and layout adjustments.
These changes aim to improve the reliability and user experience of the cron job system.
* test: enhance sessions thinking level handling
- Added tests to verify that the correct thinking levels are applied during session spawning.
- Updated the sessions-spawn-tool to include a new parameter for overriding thinking levels.
- Enhanced the UI to support additional thinking levels, including "xhigh" and "full", and improved the handling of current options in dropdowns.
These changes aim to improve the flexibility and accuracy of thinking level configurations in session management.
* feat: enhance session management and cron job functionality
- Introduced passthrough arguments in the test-parallel script to allow for flexible command-line options.
- Updated session handling to hide cron run alias session keys from the sessions list, improving clarity.
- Enhanced the cron service to accurately record job start times and durations, ensuring better tracking of job execution.
- Added tests to verify the correct behavior of the cron service under various conditions, including zero-delay timers.
These changes aim to improve the usability and reliability of session and cron job management.
* feat: implement job running state checks in cron service
- Added functionality to prevent manual job runs if a job is already in progress, enhancing job management.
- Updated the `isJobDue` function to include checks for running jobs, ensuring accurate scheduling.
- Enhanced the `run` function to return a specific reason when a job is already running.
- Introduced a new test case to verify the behavior of forced manual runs during active job execution.
These changes aim to improve the reliability and clarity of cron job execution and management.
* feat: add session ID and key to CronRunLogEntry model
- Introduced `sessionid` and `sessionkey` properties to the `CronRunLogEntry` struct for enhanced tracking of session-related information.
- Updated the initializer and Codable conformance to accommodate the new properties, ensuring proper serialization and deserialization.
These changes aim to improve the granularity of logging and session management within the cron job system.
* fix: improve session display name resolution
- Updated the `resolveSessionDisplayName` function to ensure that both label and displayName are trimmed and default to an empty string if not present.
- Enhanced the logic to prevent returning the key if it matches the label or displayName, improving clarity in session naming.
These changes aim to enhance the accuracy and usability of session display names in the UI.
* perf: skip cron store persist when idle timer tick produces no changes
recomputeNextRuns now returns a boolean indicating whether any job
state was mutated. The idle path in onTimer only persists when the
return value is true, eliminating unnecessary file writes every 60s
for far-future or idle schedules.
* fix: prep for merge - explicit delivery mode migration, docs + changelog (#10776) (thanks @tyler6204)
2026-02-06 18:03:03 -08:00
}
2026-03-19 23:29:22 -07:00
if ( serialPrefixRuns . length > 0 ) {
const failedSerialPrefix = await runEntriesWithLimit ( serialPrefixRuns , passthroughOptionArgs , 1 ) ;
if ( failedSerialPrefix !== undefined ) {
process . exit ( failedSerialPrefix ) ;
}
const failedDeferredParallel = isMacMiniProfile
? await runEntriesWithLimit ( deferredParallelRuns , passthroughOptionArgs , 3 )
: await runEntries ( deferredParallelRuns , passthroughOptionArgs ) ;
if ( failedDeferredParallel !== undefined ) {
process . exit ( failedDeferredParallel ) ;
}
} else if ( isMacMiniProfile && targetedEntries . length === 0 ) {
2026-03-20 05:08:39 +00:00
const unitFastEntriesForMacMini = parallelRuns . filter ( ( entry ) =>
entry . name . startsWith ( "unit-fast" ) ,
) ;
for ( const entry of unitFastEntriesForMacMini ) {
// eslint-disable-next-line no-await-in-loop
const unitFastCode = await run ( entry , passthroughOptionArgs ) ;
2026-03-19 07:47:07 -05:00
if ( unitFastCode !== 0 ) {
process . exit ( unitFastCode ) ;
}
}
2026-03-20 05:08:39 +00:00
const deferredEntries = parallelRuns . filter ( ( entry ) => ! entry . name . startsWith ( "unit-fast" ) ) ;
2026-03-19 07:47:07 -05:00
const failedMacMiniParallel = await runEntriesWithLimit (
deferredEntries ,
passthroughOptionArgs ,
3 ,
) ;
if ( failedMacMiniParallel !== undefined ) {
process . exit ( failedMacMiniParallel ) ;
}
} else {
const failedParallel = await runEntries ( parallelRuns , passthroughOptionArgs ) ;
if ( failedParallel !== undefined ) {
process . exit ( failedParallel ) ;
}
2026-01-23 11:36:28 +00:00
}
for ( const entry of serialRuns ) {
// eslint-disable-next-line no-await-in-loop
2026-03-13 18:36:38 -05:00
const code = await run ( entry , passthroughOptionArgs ) ;
2026-01-23 11:36:28 +00:00
if ( code !== 0 ) {
process . exit ( code ) ;
}
}
process . exit ( 0 ) ;