fix(ci): auto-isolate memory-heavy unit tests

package.json

@@ -669,6 +669,7 @@
     "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh",
     "test:perf:budget": "node scripts/test-perf-budget.mjs",
     "test:perf:hotspots": "node scripts/test-hotspots.mjs",
+    "test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs",
     "test:perf:update-timings": "node scripts/test-update-timings.mjs",
     "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",
     "test:startup:memory": "node scripts/check-cli-startup-memory.mjs",

scripts/test-parallel-memory.mjs

@@ -10,6 +10,10 @@ const ANSI_ESCAPE_PATTERN = new RegExp(
 
 const COMPLETED_TEST_FILE_LINE_PATTERN =
   /(?<file>(?:src|extensions|test|ui)\/\S+?\.(?:live\.test|e2e\.test|test)\.ts)\s+\(.*\)\s+(?<duration>\d+(?:\.\d+)?)(?<unit>ms|s)\s*$/;
+const MEMORY_TRACE_SUMMARY_PATTERN =
+  /^\[test-parallel\]\[mem\] summary (?<lane>\S+) files=(?<files>\d+) peak=(?<peak>[0-9]+(?:\.[0-9]+)?(?:GiB|MiB|KiB)) totalDelta=(?<totalDelta>[+-][0-9]+(?:\.[0-9]+)?(?:GiB|MiB|KiB)) peakAt=(?<peakAt>\S+) top=(?<top>.*)$/u;
+const MEMORY_TRACE_TOP_ENTRY_PATTERN =
+  /^(?<file>(?:src|extensions|test|ui)\/\S+?\.(?:live\.test|e2e\.test|test)\.ts):(?<delta>[+-][0-9]+(?:\.[0-9]+)?(?:GiB|MiB|KiB))$/u;
 
 const PS_COLUMNS = ["pid=", "ppid=", "rss=", "comm="];
 
@@ -21,6 +25,21 @@ function parseDurationMs(rawValue, unit) {
   return unit === "s" ? Math.round(parsed * 1000) : Math.round(parsed);
 }
 
+export function parseMemoryValueKb(rawValue) {
+  const match = rawValue.match(/^(?<sign>[+-]?)(?<value>\d+(?:\.\d+)?)(?<unit>GiB|MiB|KiB)$/u);
+  if (!match?.groups) {
+    return null;
+  }
+  const value = Number.parseFloat(match.groups.value);
+  if (!Number.isFinite(value)) {
+    return null;
+  }
+  const multiplier =
+    match.groups.unit === "GiB" ? 1024 ** 2 : match.groups.unit === "MiB" ? 1024 : 1;
+  const signed = Math.round(value * multiplier);
+  return match.groups.sign === "-" ? -signed : signed;
+}
+
 function stripAnsi(text) {
   return text.replaceAll(ANSI_ESCAPE_PATTERN, "");
 }
@@ -41,6 +60,52 @@ export function parseCompletedTestFileLines(text) {
     .filter((entry) => entry !== null);
 }
 
+export function parseMemoryTraceSummaryLines(text) {
+  return stripAnsi(text)
+    .split(/\r?\n/u)
+    .map((line) => {
+      const match = line.match(MEMORY_TRACE_SUMMARY_PATTERN);
+      if (!match?.groups) {
+        return null;
+      }
+      const peakRssKb = parseMemoryValueKb(match.groups.peak);
+      const totalDeltaKb = parseMemoryValueKb(match.groups.totalDelta);
+      const fileCount = Number.parseInt(match.groups.files, 10);
+      if (!Number.isInteger(fileCount) || peakRssKb === null || totalDeltaKb === null) {
+        return null;
+      }
+      const top =
+        match.groups.top === "none"
+          ? []
+          : match.groups.top
+              .split(/,\s+/u)
+              .map((entry) => {
+                const topMatch = entry.match(MEMORY_TRACE_TOP_ENTRY_PATTERN);
+                if (!topMatch?.groups) {
+                  return null;
+                }
+                const deltaKb = parseMemoryValueKb(topMatch.groups.delta);
+                if (deltaKb === null) {
+                  return null;
+                }
+                return {
+                  file: topMatch.groups.file,
+                  deltaKb,
+                };
+              })
+              .filter((entry) => entry !== null);
+      return {
+        lane: match.groups.lane,
+        files: fileCount,
+        peakRssKb,
+        totalDeltaKb,
+        peakAt: match.groups.peakAt,
+        top,
+      };
+    })
+    .filter((entry) => entry !== null);
+}
+
 export function getProcessTreeRecords(rootPid) {
   if (!Number.isInteger(rootPid) || rootPid <= 0 || process.platform === "win32") {
     return null;

scripts/test-parallel.mjs

@@ -15,8 +15,10 @@ import {
   resolveTestRunExitCode,
 } from "./test-parallel-utils.mjs";
 import {
+  loadUnitMemoryHotspotManifest,
   loadTestRunnerBehavior,
   loadUnitTimingManifest,
+  selectMemoryHeavyFiles,
   packFilesByDuration,
   selectTimedHeavyFiles,
 } from "./test-runner-manifest.mjs";
@@ -262,6 +264,7 @@ const inferTarget = (fileFilter) => {
   return { owner: "base", isolated };
 };
 const unitTimingManifest = loadUnitTimingManifest();
+const unitMemoryHotspotManifest = loadUnitMemoryHotspotManifest();
 const parseEnvNumber = (name, fallback) => {
   const parsed = Number.parseInt(process.env[name] ?? "", 10);
   return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
@@ -298,6 +301,16 @@ const heavyUnitLaneCount = parseEnvNumber(
   defaultHeavyUnitLaneCount,
 );
 const heavyUnitMinDurationMs = parseEnvNumber("OPENCLAW_TEST_HEAVY_UNIT_MIN_MS", 1200);
+const defaultMemoryHeavyUnitFileLimit =
+  testProfile === "serial" ? 0 : isCI ? 32 : testProfile === "low" ? 8 : 16;
+const memoryHeavyUnitFileLimit = parseEnvNumber(
+  "OPENCLAW_TEST_MEMORY_HEAVY_UNIT_FILE_LIMIT",
+  defaultMemoryHeavyUnitFileLimit,
+);
+const memoryHeavyUnitMinDeltaKb = parseEnvNumber(
+  "OPENCLAW_TEST_MEMORY_HEAVY_UNIT_MIN_KB",
+  unitMemoryHotspotManifest.defaultMinDeltaKb,
+);
 const timedHeavyUnitFiles =
   shouldSplitUnitRuns && heavyUnitFileLimit > 0
     ? selectTimedHeavyFiles({
@@ -308,8 +321,26 @@ const timedHeavyUnitFiles =
         timings: unitTimingManifest,
       })
     : [];
+const memoryHeavyUnitFiles =
+  shouldSplitUnitRuns && memoryHeavyUnitFileLimit > 0
+    ? selectMemoryHeavyFiles({
+        candidates: allKnownUnitFiles,
+        limit: memoryHeavyUnitFileLimit,
+        minDeltaKb: memoryHeavyUnitMinDeltaKb,
+        exclude: unitBehaviorOverrideSet,
+        hotspots: unitMemoryHotspotManifest,
+      })
+    : [];
 const unitFastExcludedFiles = [
-  ...new Set([...unitBehaviorOverrideSet, ...timedHeavyUnitFiles, ...channelSingletonFiles]),
+  ...new Set([
+    ...unitBehaviorOverrideSet,
+    ...timedHeavyUnitFiles,
+    ...memoryHeavyUnitFiles,
+    ...channelSingletonFiles,
+  ]),
+];
+const unitAutoSingletonFiles = [
+  ...new Set([...unitSingletonIsolatedFiles, ...memoryHeavyUnitFiles]),
 ];
 const estimateUnitDurationMs = (file) =>
   unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs;
@@ -353,7 +384,7 @@ const baseRuns = [
             ]
           : []),
         ...unitHeavyEntries,
-        ...unitSingletonIsolatedFiles.map((file) => ({
+        ...unitAutoSingletonFiles.map((file) => ({
           name: `${path.basename(file, ".test.ts")}-isolated`,
           args: [
             "vitest",

scripts/test-runner-manifest.mjs

@@ -3,12 +3,18 @@ import path from "node:path";
 
 export const behaviorManifestPath = "test/fixtures/test-parallel.behavior.json";
 export const unitTimingManifestPath = "test/fixtures/test-timings.unit.json";
+export const unitMemoryHotspotManifestPath = "test/fixtures/test-memory-hotspots.unit.json";
 
 const defaultTimingManifest = {
   config: "vitest.unit.config.ts",
   defaultDurationMs: 250,
   files: {},
 };
+const defaultMemoryHotspotManifest = {
+  config: "vitest.unit.config.ts",
+  defaultMinDeltaKb: 256 * 1024,
+  files: {},
+};
 
 const readJson = (filePath, fallback) => {
   try {
@@ -82,6 +88,46 @@ export function loadUnitTimingManifest() {
   };
 }
 
+export function loadUnitMemoryHotspotManifest() {
+  const raw = readJson(unitMemoryHotspotManifestPath, defaultMemoryHotspotManifest);
+  const defaultMinDeltaKb =
+    Number.isFinite(raw.defaultMinDeltaKb) && raw.defaultMinDeltaKb > 0
+      ? raw.defaultMinDeltaKb
+      : defaultMemoryHotspotManifest.defaultMinDeltaKb;
+  const files = Object.fromEntries(
+    Object.entries(raw.files ?? {})
+      .map(([file, value]) => {
+        const normalizedFile = normalizeRepoPath(file);
+        const deltaKb =
+          Number.isFinite(value?.deltaKb) && value.deltaKb > 0 ? Math.round(value.deltaKb) : null;
+        const sources = Array.isArray(value?.sources)
+          ? value.sources.filter((source) => typeof source === "string" && source.length > 0)
+          : [];
+        if (deltaKb === null) {
+          return [normalizedFile, null];
+        }
+        return [
+          normalizedFile,
+          {
+            deltaKb,
+            ...(sources.length > 0 ? { sources } : {}),
+          },
+        ];
+      })
+      .filter(([, value]) => value !== null),
+  );
+
+  return {
+    config:
+      typeof raw.config === "string" && raw.config
+        ? raw.config
+        : defaultMemoryHotspotManifest.config,
+    generatedAt: typeof raw.generatedAt === "string" ? raw.generatedAt : "",
+    defaultMinDeltaKb,
+    files,
+  };
+}
+
 export function selectTimedHeavyFiles({
   candidates,
   limit,
@@ -102,6 +148,26 @@ export function selectTimedHeavyFiles({
     .map((entry) => entry.file);
 }
 
+export function selectMemoryHeavyFiles({
+  candidates,
+  limit,
+  minDeltaKb,
+  exclude = new Set(),
+  hotspots,
+}) {
+  return candidates
+    .filter((file) => !exclude.has(file))
+    .map((file) => ({
+      file,
+      deltaKb: hotspots.files[file]?.deltaKb ?? 0,
+      known: Boolean(hotspots.files[file]),
+    }))
+    .filter((entry) => entry.known && entry.deltaKb >= minDeltaKb)
+    .toSorted((a, b) => b.deltaKb - a.deltaKb)
+    .slice(0, limit)
+    .map((entry) => entry.file);
+}
+
 export function packFilesByDuration(files, bucketCount, estimateDurationMs) {
   const normalizedBucketCount = Math.max(0, Math.floor(bucketCount));
   if (normalizedBucketCount <= 0 || files.length === 0) {

scripts/test-update-memory-hotspots.mjs

@@ -0,0 +1,119 @@
+import fs from "node:fs";
+import path from "node:path";
+import { parseMemoryTraceSummaryLines } from "./test-parallel-memory.mjs";
+import { unitMemoryHotspotManifestPath } from "./test-runner-manifest.mjs";
+
+function parseArgs(argv) {
+  const args = {
+    config: "vitest.unit.config.ts",
+    out: unitMemoryHotspotManifestPath,
+    lane: "unit-fast",
+    logs: [],
+    minDeltaKb: 256 * 1024,
+    limit: 64,
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--config") {
+      args.config = argv[i + 1] ?? args.config;
+      i += 1;
+      continue;
+    }
+    if (arg === "--out") {
+      args.out = argv[i + 1] ?? args.out;
+      i += 1;
+      continue;
+    }
+    if (arg === "--lane") {
+      args.lane = argv[i + 1] ?? args.lane;
+      i += 1;
+      continue;
+    }
+    if (arg === "--log") {
+      const logPath = argv[i + 1];
+      if (typeof logPath === "string" && logPath.length > 0) {
+        args.logs.push(logPath);
+      }
+      i += 1;
+      continue;
+    }
+    if (arg === "--min-delta-kb") {
+      const parsed = Number.parseInt(argv[i + 1] ?? "", 10);
+      if (Number.isFinite(parsed) && parsed > 0) {
+        args.minDeltaKb = parsed;
+      }
+      i += 1;
+      continue;
+    }
+    if (arg === "--limit") {
+      const parsed = Number.parseInt(argv[i + 1] ?? "", 10);
+      if (Number.isFinite(parsed) && parsed > 0) {
+        args.limit = parsed;
+      }
+      i += 1;
+      continue;
+    }
+  }
+  return args;
+}
+
+const opts = parseArgs(process.argv.slice(2));
+
+if (opts.logs.length === 0) {
+  console.error("[test-update-memory-hotspots] pass at least one --log <path>.");
+  process.exit(2);
+}
+
+const aggregated = new Map();
+for (const logPath of opts.logs) {
+  const text = fs.readFileSync(logPath, "utf8");
+  const summaries = parseMemoryTraceSummaryLines(text).filter(
+    (summary) => summary.lane === opts.lane,
+  );
+  for (const summary of summaries) {
+    for (const record of summary.top) {
+      if (record.deltaKb < opts.minDeltaKb) {
+        continue;
+      }
+      const nextSource = `${path.basename(logPath)}:${summary.lane}`;
+      const previous = aggregated.get(record.file);
+      if (!previous) {
+        aggregated.set(record.file, {
+          deltaKb: record.deltaKb,
+          sources: [nextSource],
+        });
+        continue;
+      }
+      previous.deltaKb = Math.max(previous.deltaKb, record.deltaKb);
+      if (!previous.sources.includes(nextSource)) {
+        previous.sources.push(nextSource);
+      }
+    }
+  }
+}
+
+const files = Object.fromEntries(
+  [...aggregated.entries()]
+    .toSorted((left, right) => right[1].deltaKb - left[1].deltaKb)
+    .slice(0, opts.limit)
+    .map(([file, value]) => [
+      file,
+      {
+        deltaKb: value.deltaKb,
+        sources: value.sources.toSorted(),
+      },
+    ]),
+);
+
+const output = {
+  config: opts.config,
+  generatedAt: new Date().toISOString(),
+  defaultMinDeltaKb: opts.minDeltaKb,
+  lane: opts.lane,
+  files,
+};
+
+fs.writeFileSync(opts.out, `${JSON.stringify(output, null, 2)}\n`);
+console.log(
+  `[test-update-memory-hotspots] wrote ${String(Object.keys(files).length)} hotspots to ${opts.out}`,
+);

test/scripts/test-parallel.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it } from "vitest";
-import { parseCompletedTestFileLines } from "../../scripts/test-parallel-memory.mjs";
+import {
+  parseCompletedTestFileLines,
+  parseMemoryTraceSummaryLines,
+  parseMemoryValueKb,
+} from "../../scripts/test-parallel-memory.mjs";
 import {
   appendCapturedOutput,
   hasFatalTestRunOutput,
@@ -76,4 +80,31 @@ describe("scripts/test-parallel memory trace parsing", () => {
       ),
     ).toEqual([]);
   });
+
+  it("parses memory trace summary lines and hotspot deltas", () => {
+    const summaries = parseMemoryTraceSummaryLines(
+      [
+        "[test-parallel][mem] summary unit-fast files=360 peak=13.22GiB totalDelta=+6.69GiB peakAt=poll top=src/config/schema.help.quality.test.ts:+1.06GiB, src/infra/update-runner.test.ts:+463.6MiB",
+      ].join("\n"),
+    );
+
+    expect(summaries).toHaveLength(1);
+    expect(summaries[0]).toEqual({
+      lane: "unit-fast",
+      files: 360,
+      peakRssKb: parseMemoryValueKb("13.22GiB"),
+      totalDeltaKb: parseMemoryValueKb("+6.69GiB"),
+      peakAt: "poll",
+      top: [
+        {
+          file: "src/config/schema.help.quality.test.ts",
+          deltaKb: parseMemoryValueKb("+1.06GiB"),
+        },
+        {
+          file: "src/infra/update-runner.test.ts",
+          deltaKb: parseMemoryValueKb("+463.6MiB"),
+        },
+      ],
+    });
+  });
 });

test/scripts/test-runner-manifest.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import {
+  selectMemoryHeavyFiles,
+  selectTimedHeavyFiles,
+} from "../../scripts/test-runner-manifest.mjs";
+
+describe("scripts/test-runner-manifest timed selection", () => {
+  it("only selects known timed heavy files above the minimum", () => {
+    expect(
+      selectTimedHeavyFiles({
+        candidates: ["a.test.ts", "b.test.ts", "c.test.ts"],
+        limit: 3,
+        minDurationMs: 1000,
+        exclude: new Set(["c.test.ts"]),
+        timings: {
+          defaultDurationMs: 250,
+          files: {
+            "a.test.ts": { durationMs: 2500 },
+            "b.test.ts": { durationMs: 900 },
+            "c.test.ts": { durationMs: 5000 },
+          },
+        },
+      }),
+    ).toEqual(["a.test.ts"]);
+  });
+});
+
+describe("scripts/test-runner-manifest memory selection", () => {
+  it("selects known memory hotspots above the minimum", () => {
+    expect(
+      selectMemoryHeavyFiles({
+        candidates: ["a.test.ts", "b.test.ts", "c.test.ts", "d.test.ts"],
+        limit: 3,
+        minDeltaKb: 256 * 1024,
+        exclude: new Set(["c.test.ts"]),
+        hotspots: {
+          files: {
+            "a.test.ts": { deltaKb: 600 * 1024 },
+            "b.test.ts": { deltaKb: 120 * 1024 },
+            "c.test.ts": { deltaKb: 900 * 1024 },
+          },
+        },
+      }),
+    ).toEqual(["a.test.ts"]);
+  });
+
+  it("orders selected memory hotspots by descending retained heap", () => {
+    expect(
+      selectMemoryHeavyFiles({
+        candidates: ["a.test.ts", "b.test.ts", "c.test.ts"],
+        limit: 2,
+        minDeltaKb: 1,
+        hotspots: {
+          files: {
+            "a.test.ts": { deltaKb: 300 },
+            "b.test.ts": { deltaKb: 700 },
+            "c.test.ts": { deltaKb: 500 },
+          },
+        },
+      }),
+    ).toEqual(["b.test.ts", "c.test.ts"]);
+  });
+});