From 2086cdfb9bb4b88424581b1f8eeb9096fa084a01 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 20:26:26 +0000 Subject: [PATCH 0001/1038] perf(test): reduce hot-suite import and setup overhead --- .../openai-responses.reasoning-replay.test.ts | 313 ++++++++---------- src/browser/pw-ai-state.ts | 9 + src/browser/pw-ai.ts | 4 + src/browser/server.ts | 15 +- src/channels/plugins/actions/discord.test.ts | 17 +- src/cli/cron-cli.test.ts | 84 ++--- src/cli/update-cli.test.ts | 102 +----- src/commands/agent/session.test.ts | 22 +- .../skills.update.normalizes-api-key.test.ts | 3 +- src/plugins/tools.optional.test.ts | 211 ++++++------ src/test-utils/ports.ts | 4 +- 11 files changed, 312 insertions(+), 472 deletions(-) create mode 100644 src/browser/pw-ai-state.ts diff --git a/src/agents/openai-responses.reasoning-replay.test.ts b/src/agents/openai-responses.reasoning-replay.test.ts index de4b10cd62d..2a94db7e3fd 100644 --- a/src/agents/openai-responses.reasoning-replay.test.ts +++ b/src/agents/openai-responses.reasoning-replay.test.ts @@ -18,198 +18,169 @@ function buildModel(): Model<"openai-responses"> { }; } -function installFailingFetchCapture() { - const originalFetch = globalThis.fetch; - let lastBody: unknown; - - const fetchImpl: typeof fetch = async (_input, init) => { - const rawBody = init?.body; - const bodyText = (() => { - if (!rawBody) { - return ""; - } - if (typeof rawBody === "string") { - return rawBody; - } - if (rawBody instanceof Uint8Array) { - return Buffer.from(rawBody).toString("utf8"); - } - if (rawBody instanceof ArrayBuffer) { - return Buffer.from(new Uint8Array(rawBody)).toString("utf8"); - } - return null; - })(); - lastBody = bodyText ? (JSON.parse(bodyText) as unknown) : undefined; - throw new Error("intentional fetch abort (test)"); - }; - - globalThis.fetch = fetchImpl; - - return { - getLastBody: () => lastBody as Record | undefined, - restore: () => { - globalThis.fetch = originalFetch; - }, - }; -} - describe("openai-responses reasoning replay", () => { it("replays reasoning for tool-call-only turns (OpenAI requires it)", async () => { - const cap = installFailingFetchCapture(); - try { - const model = buildModel(); + const model = buildModel(); + const controller = new AbortController(); + controller.abort(); + let payload: Record | undefined; - const assistantToolOnly: AssistantMessage = { - role: "assistant", - api: "openai-responses", - provider: "openai", - model: "gpt-5.2", - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + const assistantToolOnly: AssistantMessage = { + role: "assistant", + api: "openai-responses", + provider: "openai", + model: "gpt-5.2", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: Date.now(), + content: [ + { + type: "thinking", + thinking: "internal", + thinkingSignature: JSON.stringify({ + type: "reasoning", + id: "rs_test", + summary: [], + }), }, - stopReason: "toolUse", - timestamp: Date.now(), - content: [ + { + type: "toolCall", + id: "call_123|fc_123", + name: "noop", + arguments: {}, + }, + ], + }; + + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call_123|fc_123", + toolName: "noop", + content: [{ type: "text", text: "ok" }], + isError: false, + timestamp: Date.now(), + }; + + const stream = streamOpenAIResponses( + model, + { + systemPrompt: "system", + messages: [ { - type: "thinking", - thinking: "internal", - thinkingSignature: JSON.stringify({ - type: "reasoning", - id: "rs_test", - summary: [], - }), + role: "user", + content: "Call noop.", + timestamp: Date.now(), }, + assistantToolOnly, + toolResult, { - type: "toolCall", - id: "call_123|fc_123", - name: "noop", - arguments: {}, + role: "user", + content: "Now reply with ok.", + timestamp: Date.now(), }, ], - }; - - const toolResult: ToolResultMessage = { - role: "toolResult", - toolCallId: "call_123|fc_123", - toolName: "noop", - content: [{ type: "text", text: "ok" }], - isError: false, - timestamp: Date.now(), - }; - - const stream = streamOpenAIResponses( - model, - { - systemPrompt: "system", - messages: [ - { - role: "user", - content: "Call noop.", - timestamp: Date.now(), - }, - assistantToolOnly, - toolResult, - { - role: "user", - content: "Now reply with ok.", - timestamp: Date.now(), - }, - ], - tools: [ - { - name: "noop", - description: "no-op", - parameters: Type.Object({}, { additionalProperties: false }), - }, - ], + tools: [ + { + name: "noop", + description: "no-op", + parameters: Type.Object({}, { additionalProperties: false }), + }, + ], + }, + { + apiKey: "test", + signal: controller.signal, + onPayload: (nextPayload) => { + payload = nextPayload as Record; }, - { apiKey: "test" }, - ); + }, + ); - await stream.result(); + await stream.result(); - const body = cap.getLastBody(); - const input = Array.isArray(body?.input) ? body?.input : []; - const types = input - .map((item) => - item && typeof item === "object" ? (item as Record).type : undefined, - ) - .filter((t): t is string => typeof t === "string"); + const input = Array.isArray(payload?.input) ? payload?.input : []; + const types = input + .map((item) => + item && typeof item === "object" ? (item as Record).type : undefined, + ) + .filter((t): t is string => typeof t === "string"); - expect(types).toContain("reasoning"); - expect(types).toContain("function_call"); - expect(types.indexOf("reasoning")).toBeLessThan(types.indexOf("function_call")); - } finally { - cap.restore(); - } + expect(types).toContain("reasoning"); + expect(types).toContain("function_call"); + expect(types.indexOf("reasoning")).toBeLessThan(types.indexOf("function_call")); }); it("still replays reasoning when paired with an assistant message", async () => { - const cap = installFailingFetchCapture(); - try { - const model = buildModel(); + const model = buildModel(); + const controller = new AbortController(); + controller.abort(); + let payload: Record | undefined; - const assistantWithText: AssistantMessage = { - role: "assistant", - api: "openai-responses", - provider: "openai", - model: "gpt-5.2", - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }, - stopReason: "stop", - timestamp: Date.now(), - content: [ - { - type: "thinking", - thinking: "internal", - thinkingSignature: JSON.stringify({ - type: "reasoning", - id: "rs_test", - summary: [], - }), - }, - { type: "text", text: "hello", textSignature: "msg_test" }, - ], - }; - - const stream = streamOpenAIResponses( - model, + const assistantWithText: AssistantMessage = { + role: "assistant", + api: "openai-responses", + provider: "openai", + model: "gpt-5.2", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + content: [ { - systemPrompt: "system", - messages: [ - { role: "user", content: "Hi", timestamp: Date.now() }, - assistantWithText, - { role: "user", content: "Ok", timestamp: Date.now() }, - ], + type: "thinking", + thinking: "internal", + thinkingSignature: JSON.stringify({ + type: "reasoning", + id: "rs_test", + summary: [], + }), }, - { apiKey: "test" }, - ); + { type: "text", text: "hello", textSignature: "msg_test" }, + ], + }; - await stream.result(); + const stream = streamOpenAIResponses( + model, + { + systemPrompt: "system", + messages: [ + { role: "user", content: "Hi", timestamp: Date.now() }, + assistantWithText, + { role: "user", content: "Ok", timestamp: Date.now() }, + ], + }, + { + apiKey: "test", + signal: controller.signal, + onPayload: (nextPayload) => { + payload = nextPayload as Record; + }, + }, + ); - const body = cap.getLastBody(); - const input = Array.isArray(body?.input) ? body?.input : []; - const types = input - .map((item) => - item && typeof item === "object" ? (item as Record).type : undefined, - ) - .filter((t): t is string => typeof t === "string"); + await stream.result(); - expect(types).toContain("reasoning"); - expect(types).toContain("message"); - } finally { - cap.restore(); - } + const input = Array.isArray(payload?.input) ? payload?.input : []; + const types = input + .map((item) => + item && typeof item === "object" ? (item as Record).type : undefined, + ) + .filter((t): t is string => typeof t === "string"); + + expect(types).toContain("reasoning"); + expect(types).toContain("message"); }); }); diff --git a/src/browser/pw-ai-state.ts b/src/browser/pw-ai-state.ts new file mode 100644 index 00000000000..58ce89f30d9 --- /dev/null +++ b/src/browser/pw-ai-state.ts @@ -0,0 +1,9 @@ +let pwAiLoaded = false; + +export function markPwAiLoaded(): void { + pwAiLoaded = true; +} + +export function isPwAiLoaded(): boolean { + return pwAiLoaded; +} diff --git a/src/browser/pw-ai.ts b/src/browser/pw-ai.ts index 72ba680c43d..6da8b410c83 100644 --- a/src/browser/pw-ai.ts +++ b/src/browser/pw-ai.ts @@ -1,3 +1,7 @@ +import { markPwAiLoaded } from "./pw-ai-state.js"; + +markPwAiLoaded(); + export { type BrowserConsoleMessage, closePageByTargetIdViaPlaywright, diff --git a/src/browser/server.ts b/src/browser/server.ts index 2f734f031d5..419bdbfdfa5 100644 --- a/src/browser/server.ts +++ b/src/browser/server.ts @@ -7,6 +7,7 @@ import { safeEqualSecret } from "../security/secret-equal.js"; import { resolveBrowserConfig, resolveProfile } from "./config.js"; import { ensureBrowserControlAuth, resolveBrowserControlAuth } from "./control-auth.js"; import { ensureChromeExtensionRelayServer } from "./extension-relay.js"; +import { isPwAiLoaded } from "./pw-ai-state.js"; import { registerBrowserRoutes } from "./routes/index.js"; import { type BrowserServerState, createBrowserRouteContext } from "./server-context.js"; @@ -196,11 +197,13 @@ export async function stopBrowserControlServer(): Promise { } state = null; - // Optional: Playwright is not always available (e.g. embedded gateway builds). - try { - const mod = await import("./pw-ai.js"); - await mod.closePlaywrightBrowserConnection(); - } catch { - // ignore + // Optional: avoid importing heavy Playwright bridge when this process never used it. + if (isPwAiLoaded()) { + try { + const mod = await import("./pw-ai.js"); + await mod.closePlaywrightBrowserConnection(); + } catch { + // ignore + } } } diff --git a/src/channels/plugins/actions/discord.test.ts b/src/channels/plugins/actions/discord.test.ts index 7c41cda9d61..fc30a0a7566 100644 --- a/src/channels/plugins/actions/discord.test.ts +++ b/src/channels/plugins/actions/discord.test.ts @@ -21,20 +21,12 @@ vi.mock("../../../discord/send.js", async () => { }; }); -const loadHandleDiscordMessageAction = async () => { - const mod = await import("./discord/handle-action.js"); - return mod.handleDiscordMessageAction; -}; - -const loadDiscordMessageActions = async () => { - const mod = await import("./discord.js"); - return mod.discordMessageActions; -}; +const { handleDiscordMessageAction } = await import("./discord/handle-action.js"); +const { discordMessageActions } = await import("./discord.js"); describe("discord message actions", () => { it("lists channel and upload actions by default", async () => { const cfg = { channels: { discord: { token: "d0" } } } as OpenClawConfig; - const discordMessageActions = await loadDiscordMessageActions(); const actions = discordMessageActions.listActions?.({ cfg }) ?? []; expect(actions).toContain("emoji-upload"); @@ -46,7 +38,6 @@ describe("discord message actions", () => { const cfg = { channels: { discord: { token: "d0", actions: { channels: false } } }, } as OpenClawConfig; - const discordMessageActions = await loadDiscordMessageActions(); const actions = discordMessageActions.listActions?.({ cfg }) ?? []; expect(actions).not.toContain("channel-create"); @@ -56,7 +47,6 @@ describe("discord message actions", () => { describe("handleDiscordMessageAction", () => { it("forwards context accountId for send", async () => { sendMessageDiscord.mockClear(); - const handleDiscordMessageAction = await loadHandleDiscordMessageAction(); await handleDiscordMessageAction({ action: "send", @@ -79,7 +69,6 @@ describe("handleDiscordMessageAction", () => { it("falls back to params accountId when context missing", async () => { sendPollDiscord.mockClear(); - const handleDiscordMessageAction = await loadHandleDiscordMessageAction(); await handleDiscordMessageAction({ action: "poll", @@ -106,7 +95,6 @@ describe("handleDiscordMessageAction", () => { it("forwards accountId for thread replies", async () => { sendMessageDiscord.mockClear(); - const handleDiscordMessageAction = await loadHandleDiscordMessageAction(); await handleDiscordMessageAction({ action: "thread-reply", @@ -129,7 +117,6 @@ describe("handleDiscordMessageAction", () => { it("accepts threadId for thread replies (tool compatibility)", async () => { sendMessageDiscord.mockClear(); - const handleDiscordMessageAction = await loadHandleDiscordMessageAction(); await handleDiscordMessageAction({ action: "thread-reply", diff --git a/src/cli/cron-cli.test.ts b/src/cli/cron-cli.test.ts index 164b951b538..2bd437fb092 100644 --- a/src/cli/cron-cli.test.ts +++ b/src/cli/cron-cli.test.ts @@ -27,14 +27,20 @@ vi.mock("../runtime.js", () => ({ }, })); +const { registerCronCli } = await import("./cron-cli.js"); + +function buildProgram() { + const program = new Command(); + program.exitOverride(); + registerCronCli(program); + return program; +} + describe("cron cli", () => { it("trims model and thinking on cron add", { timeout: 60_000 }, async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( [ @@ -68,10 +74,7 @@ describe("cron cli", () => { it("defaults isolated cron add to announce delivery", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( [ @@ -98,10 +101,7 @@ describe("cron cli", () => { it("infers sessionTarget from payload when --session is omitted", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( ["cron", "add", "--name", "Main reminder", "--cron", "* * * * *", "--system-event", "hi"], @@ -129,10 +129,7 @@ describe("cron cli", () => { it("supports --keep-after-run on cron add", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( [ @@ -159,10 +156,7 @@ describe("cron cli", () => { it("sends agent id on cron add", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( [ @@ -190,10 +184,7 @@ describe("cron cli", () => { it("omits empty model and thinking on cron edit", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( ["cron", "edit", "job-1", "--message", "hello", "--model", " ", "--thinking", " "], @@ -212,10 +203,7 @@ describe("cron cli", () => { it("trims model and thinking on cron edit", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( [ @@ -244,10 +232,7 @@ describe("cron cli", () => { it("sets and clears agent id on cron edit", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync(["cron", "edit", "job-1", "--agent", " Ops ", "--message", "hello"], { from: "user", @@ -269,10 +254,7 @@ describe("cron cli", () => { it("allows model/thinking updates without --message", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync(["cron", "edit", "job-1", "--model", "opus", "--thinking", "low"], { from: "user", @@ -291,10 +273,7 @@ describe("cron cli", () => { it("updates delivery settings without requiring --message", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( ["cron", "edit", "job-1", "--deliver", "--channel", "telegram", "--to", "19098680"], @@ -319,10 +298,7 @@ describe("cron cli", () => { it("supports --no-deliver on cron edit", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync(["cron", "edit", "job-1", "--no-deliver"], { from: "user" }); @@ -338,10 +314,7 @@ describe("cron cli", () => { it("does not include undefined delivery fields when updating message", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); // Update message without delivery flags - should NOT include undefined delivery fields await program.parseAsync(["cron", "edit", "job-1", "--message", "Updated message"], { @@ -376,10 +349,7 @@ describe("cron cli", () => { it("includes delivery fields when explicitly provided with message", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); // Update message AND delivery - should include both await program.parseAsync( @@ -416,10 +386,7 @@ describe("cron cli", () => { it("includes best-effort delivery when provided with message", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( ["cron", "edit", "job-1", "--message", "Updated message", "--best-effort-deliver"], @@ -442,10 +409,7 @@ describe("cron cli", () => { it("includes no-best-effort delivery when provided with message", async () => { callGatewayFromCli.mockClear(); - const { registerCronCli } = await import("./cron-cli.js"); - const program = new Command(); - program.exitOverride(); - registerCronCli(program); + const program = buildProgram(); await program.parseAsync( ["cron", "edit", "job-1", "--message", "Updated message", "--no-best-effort-deliver"], diff --git a/src/cli/update-cli.test.ts b/src/cli/update-cli.test.ts index 4483790a9ee..ca6a3cb1652 100644 --- a/src/cli/update-cli.test.ts +++ b/src/cli/update-cli.test.ts @@ -79,6 +79,17 @@ vi.mock("../runtime.js", () => ({ }, })); +const { runGatewayUpdate } = await import("../infra/update-runner.js"); +const { resolveOpenClawPackageRoot } = await import("../infra/openclaw-root.js"); +const { readConfigFileSnapshot, writeConfigFile } = await import("../config/config.js"); +const { checkUpdateStatus, fetchNpmTagVersion, resolveNpmChannelTag } = + await import("../infra/update-check.js"); +const { runCommandWithTimeout } = await import("../process/exec.js"); +const { runDaemonRestart } = await import("./daemon-cli.js"); +const { defaultRuntime } = await import("../runtime.js"); +const { updateCommand, registerUpdateCli, updateStatusCommand, updateWizardCommand } = + await import("./update-cli.js"); + describe("update-cli", () => { const baseSnapshot = { valid: true, @@ -100,13 +111,8 @@ describe("update-cli", () => { }); }; - beforeEach(async () => { + beforeEach(() => { vi.clearAllMocks(); - const { resolveOpenClawPackageRoot } = await import("../infra/openclaw-root.js"); - const { readConfigFileSnapshot } = await import("../config/config.js"); - const { checkUpdateStatus, fetchNpmTagVersion, resolveNpmChannelTag } = - await import("../infra/update-check.js"); - const { runCommandWithTimeout } = await import("../process/exec.js"); vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(process.cwd()); vi.mocked(readConfigFileSnapshot).mockResolvedValue(baseSnapshot); vi.mocked(fetchNpmTagVersion).mockResolvedValue({ @@ -154,18 +160,12 @@ describe("update-cli", () => { }); it("exports updateCommand and registerUpdateCli", async () => { - const { updateCommand, registerUpdateCli, updateWizardCommand } = - await import("./update-cli.js"); expect(typeof updateCommand).toBe("function"); expect(typeof registerUpdateCli).toBe("function"); expect(typeof updateWizardCommand).toBe("function"); }, 20_000); it("updateCommand runs update and outputs result", async () => { - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { defaultRuntime } = await import("../runtime.js"); - const { updateCommand } = await import("./update-cli.js"); - const mockResult: UpdateRunResult = { status: "ok", mode: "git", @@ -193,9 +193,6 @@ describe("update-cli", () => { }); it("updateStatusCommand prints table output", async () => { - const { defaultRuntime } = await import("../runtime.js"); - const { updateStatusCommand } = await import("./update-cli.js"); - await updateStatusCommand({ json: false }); const logs = vi.mocked(defaultRuntime.log).mock.calls.map((call) => call[0]); @@ -203,9 +200,6 @@ describe("update-cli", () => { }); it("updateStatusCommand emits JSON", async () => { - const { defaultRuntime } = await import("../runtime.js"); - const { updateStatusCommand } = await import("./update-cli.js"); - await updateStatusCommand({ json: true }); const last = vi.mocked(defaultRuntime.log).mock.calls.at(-1)?.[0]; @@ -215,9 +209,6 @@ describe("update-cli", () => { }); it("defaults to dev channel for git installs when unset", async () => { - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { updateCommand } = await import("./update-cli.js"); - vi.mocked(runGatewayUpdate).mockResolvedValue({ status: "ok", mode: "git", @@ -240,11 +231,6 @@ describe("update-cli", () => { "utf-8", ); - const { resolveOpenClawPackageRoot } = await import("../infra/openclaw-root.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { checkUpdateStatus } = await import("../infra/update-check.js"); - const { updateCommand } = await import("./update-cli.js"); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); vi.mocked(checkUpdateStatus).mockResolvedValue({ root: tempDir, @@ -275,10 +261,6 @@ describe("update-cli", () => { }); it("uses stored beta channel when configured", async () => { - const { readConfigFileSnapshot } = await import("../config/config.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { updateCommand } = await import("./update-cli.js"); - vi.mocked(readConfigFileSnapshot).mockResolvedValue({ ...baseSnapshot, config: { update: { channel: "beta" } }, @@ -305,13 +287,6 @@ describe("update-cli", () => { "utf-8", ); - const { resolveOpenClawPackageRoot } = await import("../infra/openclaw-root.js"); - const { readConfigFileSnapshot } = await import("../config/config.js"); - const { resolveNpmChannelTag } = await import("../infra/update-check.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { updateCommand } = await import("./update-cli.js"); - const { checkUpdateStatus } = await import("../infra/update-check.js"); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); vi.mocked(readConfigFileSnapshot).mockResolvedValue({ ...baseSnapshot, @@ -358,10 +333,6 @@ describe("update-cli", () => { "utf-8", ); - const { resolveOpenClawPackageRoot } = await import("../infra/openclaw-root.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { updateCommand } = await import("./update-cli.js"); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); vi.mocked(runGatewayUpdate).mockResolvedValue({ status: "ok", @@ -380,10 +351,6 @@ describe("update-cli", () => { }); it("updateCommand outputs JSON when --json is set", async () => { - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { defaultRuntime } = await import("../runtime.js"); - const { updateCommand } = await import("./update-cli.js"); - const mockResult: UpdateRunResult = { status: "ok", mode: "git", @@ -409,10 +376,6 @@ describe("update-cli", () => { }); it("updateCommand exits with error on failure", async () => { - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { defaultRuntime } = await import("../runtime.js"); - const { updateCommand } = await import("./update-cli.js"); - const mockResult: UpdateRunResult = { status: "error", mode: "git", @@ -430,10 +393,6 @@ describe("update-cli", () => { }); it("updateCommand restarts daemon by default", async () => { - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { runDaemonRestart } = await import("./daemon-cli.js"); - const { updateCommand } = await import("./update-cli.js"); - const mockResult: UpdateRunResult = { status: "ok", mode: "git", @@ -450,10 +409,6 @@ describe("update-cli", () => { }); it("updateCommand skips restart when --no-restart is set", async () => { - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { runDaemonRestart } = await import("./daemon-cli.js"); - const { updateCommand } = await import("./update-cli.js"); - const mockResult: UpdateRunResult = { status: "ok", mode: "git", @@ -469,11 +424,6 @@ describe("update-cli", () => { }); it("updateCommand skips success message when restart does not run", async () => { - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { runDaemonRestart } = await import("./daemon-cli.js"); - const { defaultRuntime } = await import("../runtime.js"); - const { updateCommand } = await import("./update-cli.js"); - const mockResult: UpdateRunResult = { status: "ok", mode: "git", @@ -492,9 +442,6 @@ describe("update-cli", () => { }); it("updateCommand validates timeout option", async () => { - const { defaultRuntime } = await import("../runtime.js"); - const { updateCommand } = await import("./update-cli.js"); - vi.mocked(defaultRuntime.error).mockClear(); vi.mocked(defaultRuntime.exit).mockClear(); @@ -505,10 +452,6 @@ describe("update-cli", () => { }); it("persists update channel when --channel is set", async () => { - const { writeConfigFile } = await import("../config/config.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { updateCommand } = await import("./update-cli.js"); - const mockResult: UpdateRunResult = { status: "ok", mode: "git", @@ -537,13 +480,6 @@ describe("update-cli", () => { "utf-8", ); - const { resolveOpenClawPackageRoot } = await import("../infra/openclaw-root.js"); - const { resolveNpmChannelTag } = await import("../infra/update-check.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { defaultRuntime } = await import("../runtime.js"); - const { updateCommand } = await import("./update-cli.js"); - const { checkUpdateStatus } = await import("../infra/update-check.js"); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); vi.mocked(checkUpdateStatus).mockResolvedValue({ root: tempDir, @@ -590,13 +526,6 @@ describe("update-cli", () => { "utf-8", ); - const { resolveOpenClawPackageRoot } = await import("../infra/openclaw-root.js"); - const { resolveNpmChannelTag } = await import("../infra/update-check.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { defaultRuntime } = await import("../runtime.js"); - const { updateCommand } = await import("./update-cli.js"); - const { checkUpdateStatus } = await import("../infra/update-check.js"); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); vi.mocked(checkUpdateStatus).mockResolvedValue({ root: tempDir, @@ -634,9 +563,6 @@ describe("update-cli", () => { }); it("updateWizardCommand requires a TTY", async () => { - const { defaultRuntime } = await import("../runtime.js"); - const { updateWizardCommand } = await import("./update-cli.js"); - setTty(false); vi.mocked(defaultRuntime.error).mockClear(); vi.mocked(defaultRuntime.exit).mockClear(); @@ -656,10 +582,6 @@ describe("update-cli", () => { setTty(true); process.env.OPENCLAW_GIT_DIR = tempDir; - const { checkUpdateStatus } = await import("../infra/update-check.js"); - const { runGatewayUpdate } = await import("../infra/update-runner.js"); - const { updateWizardCommand } = await import("./update-cli.js"); - vi.mocked(checkUpdateStatus).mockResolvedValue({ root: "/test/path", installKind: "package", diff --git a/src/commands/agent/session.test.ts b/src/commands/agent/session.test.ts index 1bae455a26a..93de40b642b 100644 --- a/src/commands/agent/session.test.ts +++ b/src/commands/agent/session.test.ts @@ -22,21 +22,17 @@ vi.mock("../../agents/agent-scope.js", () => ({ listAgentIds: mocks.listAgentIds, })); +const { resolveSessionKeyForRequest } = await import("./session.js"); + describe("resolveSessionKeyForRequest", () => { beforeEach(() => { vi.clearAllMocks(); mocks.listAgentIds.mockReturnValue(["main"]); }); - async function importFresh() { - return await import("./session.js"); - } - const baseCfg: OpenClawConfig = {}; it("returns sessionKey when --to resolves a session key via context", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - mocks.resolveStorePath.mockReturnValue("/tmp/main-store.json"); mocks.loadSessionStore.mockReturnValue({ "agent:main:main": { sessionId: "sess-1", updatedAt: 0 }, @@ -50,8 +46,6 @@ describe("resolveSessionKeyForRequest", () => { }); it("finds session by sessionId via reverse lookup in primary store", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - mocks.resolveStorePath.mockReturnValue("/tmp/main-store.json"); mocks.loadSessionStore.mockReturnValue({ "agent:main:main": { sessionId: "target-session-id", updatedAt: 0 }, @@ -65,8 +59,6 @@ describe("resolveSessionKeyForRequest", () => { }); it("finds session by sessionId in non-primary agent store", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - mocks.listAgentIds.mockReturnValue(["main", "mybot"]); mocks.resolveStorePath.mockImplementation( (_store: string | undefined, opts?: { agentId?: string }) => { @@ -94,8 +86,6 @@ describe("resolveSessionKeyForRequest", () => { }); it("returns correct sessionStore when session found in non-primary agent store", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - const mybotStore = { "agent:mybot:main": { sessionId: "target-session-id", updatedAt: 0 }, }; @@ -123,8 +113,6 @@ describe("resolveSessionKeyForRequest", () => { }); it("returns undefined sessionKey when sessionId not found in any store", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - mocks.listAgentIds.mockReturnValue(["main", "mybot"]); mocks.resolveStorePath.mockImplementation( (_store: string | undefined, opts?: { agentId?: string }) => { @@ -144,8 +132,6 @@ describe("resolveSessionKeyForRequest", () => { }); it("does not search other stores when explicitSessionKey is set", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - mocks.listAgentIds.mockReturnValue(["main", "mybot"]); mocks.resolveStorePath.mockReturnValue("/tmp/main-store.json"); mocks.loadSessionStore.mockReturnValue({ @@ -162,8 +148,6 @@ describe("resolveSessionKeyForRequest", () => { }); it("searches other stores when --to derives a key that does not match --session-id", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - mocks.listAgentIds.mockReturnValue(["main", "mybot"]); mocks.resolveStorePath.mockImplementation( (_store: string | undefined, opts?: { agentId?: string }) => { @@ -199,8 +183,6 @@ describe("resolveSessionKeyForRequest", () => { }); it("skips already-searched primary store when iterating agents", async () => { - const { resolveSessionKeyForRequest } = await importFresh(); - mocks.listAgentIds.mockReturnValue(["main", "mybot"]); mocks.resolveStorePath.mockImplementation( (_store: string | undefined, opts?: { agentId?: string }) => { diff --git a/src/gateway/server-methods/skills.update.normalizes-api-key.test.ts b/src/gateway/server-methods/skills.update.normalizes-api-key.test.ts index 45b9d719e7c..ac4dc516722 100644 --- a/src/gateway/server-methods/skills.update.normalizes-api-key.test.ts +++ b/src/gateway/server-methods/skills.update.normalizes-api-key.test.ts @@ -15,10 +15,11 @@ vi.mock("../../config/config.js", () => { }; }); +const { skillsHandlers } = await import("./skills.js"); + describe("skills.update", () => { it("strips embedded CR/LF from apiKey", async () => { writtenConfig = null; - const { skillsHandlers } = await import("./skills.js"); let ok: boolean | null = null; let error: unknown = null; diff --git a/src/plugins/tools.optional.test.ts b/src/plugins/tools.optional.test.ts index 1f15eec90ea..614c0980179 100644 --- a/src/plugins/tools.optional.test.ts +++ b/src/plugins/tools.optional.test.ts @@ -2,23 +2,22 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterAll, describe, expect, it } from "vitest"; import { resolvePluginTools } from "./tools.js"; type TempPlugin = { dir: string; file: string; id: string }; -const tempDirs: string[] = []; +const fixtureRoot = path.join(os.tmpdir(), `openclaw-plugin-tools-${randomUUID()}`); const EMPTY_PLUGIN_SCHEMA = { type: "object", additionalProperties: false, properties: {} }; -function makeTempDir() { - const dir = path.join(os.tmpdir(), `openclaw-plugin-tools-${randomUUID()}`); +function makeFixtureDir(id: string) { + const dir = path.join(fixtureRoot, id); fs.mkdirSync(dir, { recursive: true }); - tempDirs.push(dir); return dir; } function writePlugin(params: { id: string; body: string }): TempPlugin { - const dir = makeTempDir(); + const dir = makeFixtureDir(params.id); const file = path.join(dir, `${params.id}.js`); fs.writeFileSync(file, params.body, "utf-8"); fs.writeFileSync( @@ -36,18 +35,7 @@ function writePlugin(params: { id: string; body: string }): TempPlugin { return { dir, file, id: params.id }; } -afterEach(() => { - for (const dir of tempDirs.splice(0)) { - try { - fs.rmSync(dir, { recursive: true, force: true }); - } catch { - // ignore cleanup failures - } - } -}); - -describe("resolvePluginTools optional tools", () => { - const pluginBody = ` +const pluginBody = ` export default { register(api) { api.registerTool( { @@ -63,92 +51,11 @@ export default { register(api) { } } `; - it("skips optional tools without explicit allowlist", () => { - const plugin = writePlugin({ id: "optional-demo", body: pluginBody }); - const tools = resolvePluginTools({ - context: { - config: { - plugins: { - load: { paths: [plugin.file] }, - allow: [plugin.id], - }, - }, - workspaceDir: plugin.dir, - }, - }); - expect(tools).toHaveLength(0); - }); - - it("allows optional tools by name", () => { - const plugin = writePlugin({ id: "optional-demo", body: pluginBody }); - const tools = resolvePluginTools({ - context: { - config: { - plugins: { - load: { paths: [plugin.file] }, - allow: [plugin.id], - }, - }, - workspaceDir: plugin.dir, - }, - toolAllowlist: ["optional_tool"], - }); - expect(tools.map((tool) => tool.name)).toContain("optional_tool"); - }); - - it("allows optional tools via plugin groups", () => { - const plugin = writePlugin({ id: "optional-demo", body: pluginBody }); - const toolsAll = resolvePluginTools({ - context: { - config: { - plugins: { - load: { paths: [plugin.file] }, - allow: [plugin.id], - }, - }, - workspaceDir: plugin.dir, - }, - toolAllowlist: ["group:plugins"], - }); - expect(toolsAll.map((tool) => tool.name)).toContain("optional_tool"); - - const toolsPlugin = resolvePluginTools({ - context: { - config: { - plugins: { - load: { paths: [plugin.file] }, - allow: [plugin.id], - }, - }, - workspaceDir: plugin.dir, - }, - toolAllowlist: ["optional-demo"], - }); - expect(toolsPlugin.map((tool) => tool.name)).toContain("optional_tool"); - }); - - it("rejects plugin id collisions with core tool names", () => { - const plugin = writePlugin({ id: "message", body: pluginBody }); - const tools = resolvePluginTools({ - context: { - config: { - plugins: { - load: { paths: [plugin.file] }, - allow: [plugin.id], - }, - }, - workspaceDir: plugin.dir, - }, - existingToolNames: new Set(["message"]), - toolAllowlist: ["message"], - }); - expect(tools).toHaveLength(0); - }); - - it("skips conflicting tool names but keeps other tools", () => { - const plugin = writePlugin({ - id: "multi", - body: ` +const optionalDemoPlugin = writePlugin({ id: "optional-demo", body: pluginBody }); +const coreNameCollisionPlugin = writePlugin({ id: "message", body: pluginBody }); +const multiToolPlugin = writePlugin({ + id: "multi", + body: ` export default { register(api) { api.registerTool({ name: "message", @@ -168,17 +75,105 @@ export default { register(api) { }); } } `, - }); +}); +afterAll(() => { + try { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } catch { + // ignore cleanup failures + } +}); + +describe("resolvePluginTools optional tools", () => { + it("skips optional tools without explicit allowlist", () => { const tools = resolvePluginTools({ context: { config: { plugins: { - load: { paths: [plugin.file] }, - allow: [plugin.id], + load: { paths: [optionalDemoPlugin.file] }, + allow: [optionalDemoPlugin.id], }, }, - workspaceDir: plugin.dir, + workspaceDir: optionalDemoPlugin.dir, + }, + }); + expect(tools).toHaveLength(0); + }); + + it("allows optional tools by name", () => { + const tools = resolvePluginTools({ + context: { + config: { + plugins: { + load: { paths: [optionalDemoPlugin.file] }, + allow: [optionalDemoPlugin.id], + }, + }, + workspaceDir: optionalDemoPlugin.dir, + }, + toolAllowlist: ["optional_tool"], + }); + expect(tools.map((tool) => tool.name)).toContain("optional_tool"); + }); + + it("allows optional tools via plugin groups", () => { + const toolsAll = resolvePluginTools({ + context: { + config: { + plugins: { + load: { paths: [optionalDemoPlugin.file] }, + allow: [optionalDemoPlugin.id], + }, + }, + workspaceDir: optionalDemoPlugin.dir, + }, + toolAllowlist: ["group:plugins"], + }); + expect(toolsAll.map((tool) => tool.name)).toContain("optional_tool"); + + const toolsPlugin = resolvePluginTools({ + context: { + config: { + plugins: { + load: { paths: [optionalDemoPlugin.file] }, + allow: [optionalDemoPlugin.id], + }, + }, + workspaceDir: optionalDemoPlugin.dir, + }, + toolAllowlist: ["optional-demo"], + }); + expect(toolsPlugin.map((tool) => tool.name)).toContain("optional_tool"); + }); + + it("rejects plugin id collisions with core tool names", () => { + const tools = resolvePluginTools({ + context: { + config: { + plugins: { + load: { paths: [coreNameCollisionPlugin.file] }, + allow: [coreNameCollisionPlugin.id], + }, + }, + workspaceDir: coreNameCollisionPlugin.dir, + }, + existingToolNames: new Set(["message"]), + toolAllowlist: ["message"], + }); + expect(tools).toHaveLength(0); + }); + + it("skips conflicting tool names but keeps other tools", () => { + const tools = resolvePluginTools({ + context: { + config: { + plugins: { + load: { paths: [multiToolPlugin.file] }, + allow: [multiToolPlugin.id], + }, + }, + workspaceDir: multiToolPlugin.dir, }, existingToolNames: new Set(["message"]), }); diff --git a/src/test-utils/ports.ts b/src/test-utils/ports.ts index 214f9ba8f4e..00fa86aa00a 100644 --- a/src/test-utils/ports.ts +++ b/src/test-utils/ports.ts @@ -62,7 +62,9 @@ export async function getDeterministicFreePortBlock(params?: { // Allocate in blocks to avoid derived-port overlaps (e.g. port+3). const blockSize = Math.max(maxOffset + 1, 8); - for (let attempt = 0; attempt < usable; attempt += 1) { + // Scan in block-size steps. Tests consume neighboring derived ports (+1/+2/...), + // so probing every single offset is wasted work and slows large suites. + for (let attempt = 0; attempt < usable; attempt += blockSize) { const start = base + ((nextTestPortOffset + attempt) % usable); // eslint-disable-next-line no-await-in-loop const ok = (await Promise.all(offsets.map((offset) => isPortFree(start + offset)))).every( From 4e9f933e88e48d0148b86148220aa50c69aa5f84 Mon Sep 17 00:00:00 2001 From: Joseph Krug Date: Fri, 13 Feb 2026 16:30:09 -0400 Subject: [PATCH 0002/1038] fix: reset stale execution state after SIGUSR1 in-process restart (#15195) Merged via /review-pr -> /prepare-pr -> /merge-pr. Prepared head SHA: 676f9ec45135be0d3471bb0444bc2ac7ce7d5224 Co-authored-by: joeykrug <5925937+joeykrug@users.noreply.github.com> Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com> Reviewed-by: @gumadeiras --- CHANGELOG.md | 1 + scripts/recover-orphaned-processes.sh | 191 ++++++++++++++++++++++++++ src/cli/gateway-cli/run-loop.test.ts | 119 ++++++++++++++++ src/cli/gateway-cli/run-loop.ts | 18 ++- src/infra/heartbeat-wake.test.ts | 53 +++++++ src/infra/heartbeat-wake.ts | 17 +++ src/macos/gateway-daemon.ts | 35 ++++- src/process/command-queue.test.ts | 50 +++++++ src/process/command-queue.ts | 74 +++++++--- src/process/restart-recovery.test.ts | 18 +++ src/process/restart-recovery.ts | 16 +++ 11 files changed, 572 insertions(+), 20 deletions(-) create mode 100755 scripts/recover-orphaned-processes.sh create mode 100644 src/cli/gateway-cli/run-loop.test.ts create mode 100644 src/process/restart-recovery.test.ts create mode 100644 src/process/restart-recovery.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c110e2f612f..c7252c469cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Docs: https://docs.openclaw.ai - Android/Nodes: harden `app.update` by requiring HTTPS and gateway-host URL matching plus SHA-256 verification, stream URL camera downloads to disk with size guards to avoid memory spikes, and stop signing release builds with debug keys. (#13541) Thanks @smartprogrammer93. - Auto-reply/Threading: auto-inject implicit reply threading so `replyToMode` works without requiring model-emitted `[[reply_to_current]]`, while preserving `replyToMode: "off"` behavior for implicit Slack replies and keeping block-streaming chunk coalescing stable under `replyToMode: "first"`. (#14976) Thanks @Diaspar4u. - Sandbox: pass configured `sandbox.docker.env` variables to sandbox containers at `docker create` time. (#15138) Thanks @stevebot-alive. +- Gateway/Restart: clear stale command-queue and heartbeat wake runtime state after SIGUSR1 in-process restarts to prevent zombie gateway behavior where queued work stops draining. (#15195) Thanks @joeykrug. - Onboarding/CLI: restore terminal state without resuming paused `stdin`, so onboarding exits cleanly after choosing Web UI and the installer returns instead of appearing stuck. - Auth/OpenAI Codex: share OAuth login handling across onboarding and `models auth login --provider openai-codex`, keep onboarding alive when OAuth fails, and surface a direct OAuth help note instead of terminating the wizard. (#15406, follow-up to #14552) Thanks @zhiluo20. - Onboarding/Providers: add vLLM as an onboarding provider with model discovery, auth profile wiring, and non-interactive auth-choice validation. (#12577) Thanks @gejifeng. diff --git a/scripts/recover-orphaned-processes.sh b/scripts/recover-orphaned-processes.sh new file mode 100755 index 00000000000..d37c5ea4c80 --- /dev/null +++ b/scripts/recover-orphaned-processes.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Scan for orphaned coding agent processes after a gateway restart. +# +# Background coding agents (Claude Code, Codex CLI) spawned by the gateway +# can outlive the session that started them when the gateway restarts. +# This script finds them and reports their state. +# +# Usage: +# recover-orphaned-processes.sh +# +# Output: JSON object with `orphaned` array and `ts` timestamp. +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: recover-orphaned-processes.sh + +Scans for likely orphaned coding agent processes and prints JSON. +USAGE +} + +if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then + usage + exit 0 +fi + +if [ "$#" -gt 0 ]; then + usage >&2 + exit 2 +fi + +if ! command -v node &>/dev/null; then + _ts="unknown" + command -v date &>/dev/null && _ts="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || true + [ -z "$_ts" ] && _ts="unknown" + printf '{"error":"node not found on PATH","orphaned":[],"ts":"%s"}\n' "$_ts" + exit 0 +fi + +node <<'NODE' +const { execFileSync } = require("node:child_process"); +const fs = require("node:fs"); + +let username = process.env.USER || process.env.LOGNAME || ""; + +if (username && !/^[a-zA-Z0-9._-]+$/.test(username)) { + username = ""; +} + +function runFile(file, args) { + try { + return execFileSync(file, args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + } catch (err) { + if (err && typeof err.stdout === "string") { + return err.stdout; + } + if (err && err.stdout && Buffer.isBuffer(err.stdout)) { + return err.stdout.toString("utf8"); + } + return ""; + } +} + +function resolveStarted(pid) { + const started = runFile("ps", ["-o", "lstart=", "-p", String(pid)]).trim(); + return started.length > 0 ? started : "unknown"; +} + +function resolveCwd(pid) { + if (process.platform === "linux") { + try { + return fs.readlinkSync(`/proc/${pid}/cwd`); + } catch { + return "unknown"; + } + } + const lsof = runFile("lsof", ["-a", "-d", "cwd", "-p", String(pid), "-Fn"]); + const match = lsof.match(/^n(.+)$/m); + return match ? match[1] : "unknown"; +} + +function sanitizeCommand(cmd) { + // Avoid leaking obvious secrets when this diagnostic output is shared. + return cmd + .replace( + /(--(?:token|api[-_]?key|password|secret|authorization)\s+)([^\s]+)/gi, + "$1", + ) + .replace( + /((?:token|api[-_]?key|password|secret|authorization)=)([^\s]+)/gi, + "$1", + ) + .replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/g, "$1"); +} + +// Pre-filter candidate PIDs using pgrep to avoid scanning all processes. +// Only falls back to a full ps scan when pgrep is genuinely unavailable +// (ENOENT), not when it simply finds no matches (exit code 1). +let pgrepUnavailable = false; +const pgrepResult = (() => { + const args = + username.length > 0 + ? ["-u", username, "-f", "codex|claude"] + : ["-f", "codex|claude"]; + try { + return execFileSync("pgrep", args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + } catch (err) { + if (err && err.code === "ENOENT") { + pgrepUnavailable = true; + return ""; + } + // pgrep exit code 1 = no matches — return stdout (empty) + if (err && typeof err.stdout === "string") return err.stdout; + return ""; + } +})(); + +const candidatePids = pgrepResult + .split("\n") + .map((s) => s.trim()) + .filter((s) => s.length > 0 && /^\d+$/.test(s)); + +let lines; +if (candidatePids.length > 0) { + // Fetch command info only for candidate PIDs. + lines = runFile("ps", ["-o", "pid=,command=", "-p", candidatePids.join(",")]).split("\n"); +} else if (pgrepUnavailable && username.length > 0) { + // pgrep not installed — fall back to user-scoped ps scan. + lines = runFile("ps", ["-U", username, "-o", "pid=,command="]).split("\n"); +} else if (pgrepUnavailable) { + // pgrep not installed and no username — full scan as last resort. + lines = runFile("ps", ["-axo", "pid=,command="]).split("\n"); +} else { + // pgrep ran successfully but found no matches — no orphans. + lines = []; +} + +const includePattern = /codex|claude/i; + +const excludePatterns = [ + /openclaw-gateway/i, + /signal-cli/i, + /node_modules\/\.bin\/openclaw/i, + /recover-orphaned-processes\.sh/i, +]; + +const orphaned = []; + +for (const rawLine of lines) { + const line = rawLine.trim(); + if (!line) { + continue; + } + const match = line.match(/^(\d+)\s+(.+)$/); + if (!match) { + continue; + } + + const pid = Number(match[1]); + const cmd = match[2]; + if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid) { + continue; + } + if (!includePattern.test(cmd)) { + continue; + } + if (excludePatterns.some((pattern) => pattern.test(cmd))) { + continue; + } + + orphaned.push({ + pid, + cmd: sanitizeCommand(cmd), + cwd: resolveCwd(pid), + started: resolveStarted(pid), + }); +} + +process.stdout.write( + JSON.stringify({ + orphaned, + ts: new Date().toISOString(), + }) + "\n", +); +NODE diff --git a/src/cli/gateway-cli/run-loop.test.ts b/src/cli/gateway-cli/run-loop.test.ts new file mode 100644 index 00000000000..928e02cc5e9 --- /dev/null +++ b/src/cli/gateway-cli/run-loop.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it, vi } from "vitest"; + +const acquireGatewayLock = vi.fn(async () => ({ + release: vi.fn(async () => {}), +})); +const consumeGatewaySigusr1RestartAuthorization = vi.fn(() => true); +const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false); +const getActiveTaskCount = vi.fn(() => 0); +const waitForActiveTasks = vi.fn(async () => ({ drained: true })); +const resetAllLanes = vi.fn(); +const DRAIN_TIMEOUT_LOG = "drain timeout reached; proceeding with restart"; +const gatewayLog = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +vi.mock("../../infra/gateway-lock.js", () => ({ + acquireGatewayLock: () => acquireGatewayLock(), +})); + +vi.mock("../../infra/restart.js", () => ({ + consumeGatewaySigusr1RestartAuthorization: () => consumeGatewaySigusr1RestartAuthorization(), + isGatewaySigusr1RestartExternallyAllowed: () => isGatewaySigusr1RestartExternallyAllowed(), +})); + +vi.mock("../../process/command-queue.js", () => ({ + getActiveTaskCount: () => getActiveTaskCount(), + waitForActiveTasks: (timeoutMs: number) => waitForActiveTasks(timeoutMs), + resetAllLanes: () => resetAllLanes(), +})); + +vi.mock("../../logging/subsystem.js", () => ({ + createSubsystemLogger: () => gatewayLog, +})); + +function removeNewSignalListeners( + signal: NodeJS.Signals, + existing: Set<(...args: unknown[]) => void>, +) { + for (const listener of process.listeners(signal)) { + const fn = listener as (...args: unknown[]) => void; + if (!existing.has(fn)) { + process.removeListener(signal, fn); + } + } +} + +describe("runGatewayLoop", () => { + it("restarts after SIGUSR1 even when drain times out, and resets lanes for the new iteration", async () => { + vi.clearAllMocks(); + getActiveTaskCount.mockReturnValueOnce(2).mockReturnValueOnce(0); + waitForActiveTasks.mockResolvedValueOnce({ drained: false }); + + type StartServer = () => Promise<{ + close: (opts: { reason: string; restartExpectedMs: number | null }) => Promise; + }>; + + const closeFirst = vi.fn(async () => {}); + const closeSecond = vi.fn(async () => {}); + const start = vi + .fn() + .mockResolvedValueOnce({ close: closeFirst }) + .mockResolvedValueOnce({ close: closeSecond }) + .mockRejectedValueOnce(new Error("stop-loop")); + + const beforeSigterm = new Set( + process.listeners("SIGTERM") as Array<(...args: unknown[]) => void>, + ); + const beforeSigint = new Set( + process.listeners("SIGINT") as Array<(...args: unknown[]) => void>, + ); + const beforeSigusr1 = new Set( + process.listeners("SIGUSR1") as Array<(...args: unknown[]) => void>, + ); + + const loopPromise = import("./run-loop.js").then(({ runGatewayLoop }) => + runGatewayLoop({ + start, + runtime: { + exit: vi.fn(), + } as { exit: (code: number) => never }, + }), + ); + + try { + await vi.waitFor(() => { + expect(start).toHaveBeenCalledTimes(1); + }); + + process.emit("SIGUSR1"); + + await vi.waitFor(() => { + expect(start).toHaveBeenCalledTimes(2); + }); + + expect(waitForActiveTasks).toHaveBeenCalledWith(30_000); + expect(gatewayLog.warn).toHaveBeenCalledWith(DRAIN_TIMEOUT_LOG); + expect(closeFirst).toHaveBeenCalledWith({ + reason: "gateway restarting", + restartExpectedMs: 1500, + }); + expect(resetAllLanes).toHaveBeenCalledTimes(1); + + process.emit("SIGUSR1"); + + await expect(loopPromise).rejects.toThrow("stop-loop"); + expect(closeSecond).toHaveBeenCalledWith({ + reason: "gateway restarting", + restartExpectedMs: 1500, + }); + expect(resetAllLanes).toHaveBeenCalledTimes(2); + } finally { + removeNewSignalListeners("SIGTERM", beforeSigterm); + removeNewSignalListeners("SIGINT", beforeSigint); + removeNewSignalListeners("SIGUSR1", beforeSigusr1); + } + }); +}); diff --git a/src/cli/gateway-cli/run-loop.ts b/src/cli/gateway-cli/run-loop.ts index 9486e199e35..ec582fdcb8d 100644 --- a/src/cli/gateway-cli/run-loop.ts +++ b/src/cli/gateway-cli/run-loop.ts @@ -6,7 +6,12 @@ import { isGatewaySigusr1RestartExternallyAllowed, } from "../../infra/restart.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; -import { getActiveTaskCount, waitForActiveTasks } from "../../process/command-queue.js"; +import { + getActiveTaskCount, + resetAllLanes, + waitForActiveTasks, +} from "../../process/command-queue.js"; +import { createRestartIterationHook } from "../../process/restart-recovery.js"; const gatewayLog = createSubsystemLogger("gateway"); @@ -111,10 +116,21 @@ export async function runGatewayLoop(params: { process.on("SIGUSR1", onSigusr1); try { + const onIteration = createRestartIterationHook(() => { + // After an in-process restart (SIGUSR1), reset command-queue lane state. + // Interrupted tasks from the previous lifecycle may have left `active` + // counts elevated (their finally blocks never ran), permanently blocking + // new work from draining. This must happen here — at the restart + // coordinator level — rather than inside individual subsystem init + // functions, to avoid surprising cross-cutting side effects. + resetAllLanes(); + }); + // Keep process alive; SIGUSR1 triggers an in-process restart (no supervisor required). // SIGTERM/SIGINT still exit after a graceful shutdown. // eslint-disable-next-line no-constant-condition while (true) { + onIteration(); server = await params.start(); await new Promise((resolve) => { restartResolver = resolve; diff --git a/src/infra/heartbeat-wake.test.ts b/src/infra/heartbeat-wake.test.ts index b3f8e0d32f7..63d47523023 100644 --- a/src/infra/heartbeat-wake.test.ts +++ b/src/infra/heartbeat-wake.test.ts @@ -173,6 +173,59 @@ describe("heartbeat-wake", () => { expect(handler).toHaveBeenCalledWith({ reason: "exec-event" }); }); + it("resets running/scheduled flags when new handler is registered", async () => { + vi.useFakeTimers(); + + // Simulate a handler that's mid-execution when SIGUSR1 fires. + // We do this by having the handler hang forever (never resolve). + let resolveHang: () => void; + const hangPromise = new Promise((r) => { + resolveHang = r; + }); + const handlerA = vi + .fn() + .mockReturnValue(hangPromise.then(() => ({ status: "ran" as const, durationMs: 1 }))); + setHeartbeatWakeHandler(handlerA); + + // Trigger the handler — it starts running but never finishes + requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + await vi.advanceTimersByTimeAsync(1); + expect(handlerA).toHaveBeenCalledTimes(1); + + // Now simulate SIGUSR1: register a new handler while handlerA is still running. + // Without the fix, `running` would stay true and handlerB would never fire. + const handlerB = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); + setHeartbeatWakeHandler(handlerB); + + // handlerB should be able to fire (running was reset) + requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + await vi.advanceTimersByTimeAsync(1); + expect(handlerB).toHaveBeenCalledTimes(1); + + // Clean up the hanging promise + resolveHang!(); + await Promise.resolve(); + }); + + it("clears stale retry cooldown when a new handler is registered", async () => { + vi.useFakeTimers(); + const handlerA = vi.fn().mockResolvedValue({ status: "skipped", reason: "requests-in-flight" }); + setHeartbeatWakeHandler(handlerA); + + requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + await vi.advanceTimersByTimeAsync(1); + expect(handlerA).toHaveBeenCalledTimes(1); + + // Simulate SIGUSR1 startup with a fresh wake handler. + const handlerB = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); + setHeartbeatWakeHandler(handlerB); + + requestHeartbeatNow({ reason: "manual", coalesceMs: 0 }); + await vi.advanceTimersByTimeAsync(1); + expect(handlerB).toHaveBeenCalledTimes(1); + expect(handlerB).toHaveBeenCalledWith({ reason: "manual" }); + }); + it("drains pending wake once a handler is registered", async () => { vi.useFakeTimers(); diff --git a/src/infra/heartbeat-wake.ts b/src/infra/heartbeat-wake.ts index 72f97378f67..6297b5ffb68 100644 --- a/src/infra/heartbeat-wake.ts +++ b/src/infra/heartbeat-wake.ts @@ -146,6 +146,23 @@ export function setHeartbeatWakeHandler(next: HeartbeatWakeHandler | null): () = handlerGeneration += 1; const generation = handlerGeneration; handler = next; + if (next) { + // New lifecycle starting (e.g. after SIGUSR1 in-process restart). + // Clear any timer metadata from the previous lifecycle so stale retry + // cooldowns do not delay a fresh handler. + if (timer) { + clearTimeout(timer); + } + timer = null; + timerDueAt = null; + timerKind = null; + // Reset module-level execution state that may be stale from interrupted + // runs in the previous lifecycle. Without this, `running === true` from + // an interrupted heartbeat blocks all future schedule() attempts, and + // `scheduled === true` can cause spurious immediate re-runs. + running = false; + scheduled = false; + } if (handler && pendingWake) { schedule(DEFAULT_COALESCE_MS, "normal"); } diff --git a/src/macos/gateway-daemon.ts b/src/macos/gateway-daemon.ts index eb02c060640..38fd5485ff0 100644 --- a/src/macos/gateway-daemon.ts +++ b/src/macos/gateway-daemon.ts @@ -52,6 +52,8 @@ async function main() { { consumeGatewaySigusr1RestartAuthorization, isGatewaySigusr1RestartExternallyAllowed }, { defaultRuntime }, { enableConsoleCapture, setConsoleTimestampPrefix }, + commandQueueMod, + { createRestartIterationHook }, ] = await Promise.all([ import("../config/config.js"), import("../gateway/server.js"), @@ -61,6 +63,8 @@ async function main() { import("../infra/restart.js"), import("../runtime.js"), import("../logging.js"), + import("../process/command-queue.js"), + import("../process/restart-recovery.js"), ] as const); enableConsoleCapture(); @@ -132,14 +136,32 @@ async function main() { `gateway: received ${signal}; ${isRestart ? "restarting" : "shutting down"}`, ); + const DRAIN_TIMEOUT_MS = 30_000; + const SHUTDOWN_TIMEOUT_MS = 5_000; + const forceExitMs = isRestart ? DRAIN_TIMEOUT_MS + SHUTDOWN_TIMEOUT_MS : SHUTDOWN_TIMEOUT_MS; forceExitTimer = setTimeout(() => { defaultRuntime.error("gateway: shutdown timed out; exiting without full cleanup"); cleanupSignals(); process.exit(0); - }, 5000); + }, forceExitMs); void (async () => { try { + if (isRestart) { + const activeTasks = commandQueueMod.getActiveTaskCount(); + if (activeTasks > 0) { + defaultRuntime.log( + `gateway: draining ${activeTasks} active task(s) before restart (timeout ${DRAIN_TIMEOUT_MS}ms)`, + ); + const { drained } = await commandQueueMod.waitForActiveTasks(DRAIN_TIMEOUT_MS); + if (drained) { + defaultRuntime.log("gateway: all active tasks drained"); + } else { + defaultRuntime.log("gateway: drain timeout reached; proceeding with restart"); + } + } + } + await server?.close({ reason: isRestart ? "gateway restarting" : "gateway stopping", restartExpectedMs: isRestart ? 1500 : null, @@ -196,8 +218,17 @@ async function main() { } throw err; } + const onIteration = createRestartIterationHook(() => { + // After an in-process restart (SIGUSR1), reset command-queue lane state. + // Interrupted tasks from the previous lifecycle may have left `active` + // counts elevated (their finally blocks never ran), permanently blocking + // new work from draining. + commandQueueMod.resetAllLanes(); + }); + // eslint-disable-next-line no-constant-condition while (true) { + onIteration(); try { server = await startGatewayServer(port, { bind }); } catch (err) { @@ -210,7 +241,7 @@ async function main() { }); } } finally { - await (lock as GatewayLockHandle | null)?.release(); + await lock?.release(); cleanupSignals(); } } diff --git a/src/process/command-queue.test.ts b/src/process/command-queue.test.ts index 60034b43929..5c0b20930af 100644 --- a/src/process/command-queue.test.ts +++ b/src/process/command-queue.test.ts @@ -23,6 +23,7 @@ import { enqueueCommandInLane, getActiveTaskCount, getQueueSize, + resetAllLanes, setCommandLaneConcurrency, waitForActiveTasks, } from "./command-queue.js"; @@ -36,6 +37,12 @@ describe("command queue", () => { diagnosticMocks.diag.error.mockClear(); }); + it("resetAllLanes is safe when no lanes have been created", () => { + expect(getActiveTaskCount()).toBe(0); + expect(() => resetAllLanes()).not.toThrow(); + expect(getActiveTaskCount()).toBe(0); + }); + it("runs tasks one at a time in order", async () => { let active = 0; let maxActive = 0; @@ -162,6 +169,49 @@ describe("command queue", () => { await task; }); + it("resetAllLanes drains queued work immediately after reset", async () => { + const lane = `reset-test-${Date.now()}-${Math.random().toString(16).slice(2)}`; + setCommandLaneConcurrency(lane, 1); + + let resolve1!: () => void; + const blocker = new Promise((r) => { + resolve1 = r; + }); + + // Start a task that blocks the lane + const task1 = enqueueCommandInLane(lane, async () => { + await blocker; + }); + + await vi.waitFor(() => { + expect(getActiveTaskCount()).toBeGreaterThanOrEqual(1); + }); + + // Enqueue another task — it should be stuck behind the blocker + let task2Ran = false; + const task2 = enqueueCommandInLane(lane, async () => { + task2Ran = true; + }); + + await vi.waitFor(() => { + expect(getQueueSize(lane)).toBeGreaterThanOrEqual(2); + }); + expect(task2Ran).toBe(false); + + // Simulate SIGUSR1: reset all lanes. Queued work (task2) should be + // drained immediately — no fresh enqueue needed. + resetAllLanes(); + + // Complete the stale in-flight task; generation mismatch makes its + // completion path a no-op for queue bookkeeping. + resolve1(); + await task1; + + // task2 should have been pumped by resetAllLanes's drain pass. + await task2; + expect(task2Ran).toBe(true); + }); + it("waitForActiveTasks ignores tasks that start after the call", async () => { const lane = `drain-snapshot-${Date.now()}-${Math.random().toString(16).slice(2)}`; setCommandLaneConcurrency(lane, 2); diff --git a/src/process/command-queue.ts b/src/process/command-queue.ts index b0f012ca245..9ee4c741719 100644 --- a/src/process/command-queue.ts +++ b/src/process/command-queue.ts @@ -29,10 +29,10 @@ type QueueEntry = { type LaneState = { lane: string; queue: QueueEntry[]; - active: number; activeTaskIds: Set; maxConcurrent: number; draining: boolean; + generation: number; }; const lanes = new Map(); @@ -46,15 +46,23 @@ function getLaneState(lane: string): LaneState { const created: LaneState = { lane, queue: [], - active: 0, activeTaskIds: new Set(), maxConcurrent: 1, draining: false, + generation: 0, }; lanes.set(lane, created); return created; } +function completeTask(state: LaneState, taskId: number, taskGeneration: number): boolean { + if (taskGeneration !== state.generation) { + return false; + } + state.activeTaskIds.delete(taskId); + return true; +} + function drainLane(lane: string) { const state = getLaneState(lane); if (state.draining) { @@ -63,7 +71,7 @@ function drainLane(lane: string) { state.draining = true; const pump = () => { - while (state.active < state.maxConcurrent && state.queue.length > 0) { + while (state.activeTaskIds.size < state.maxConcurrent && state.queue.length > 0) { const entry = state.queue.shift() as QueueEntry; const waitedMs = Date.now() - entry.enqueuedAt; if (waitedMs >= entry.warnAfterMs) { @@ -74,29 +82,31 @@ function drainLane(lane: string) { } logLaneDequeue(lane, waitedMs, state.queue.length); const taskId = nextTaskId++; - state.active += 1; + const taskGeneration = state.generation; state.activeTaskIds.add(taskId); void (async () => { const startTime = Date.now(); try { const result = await entry.task(); - state.active -= 1; - state.activeTaskIds.delete(taskId); - diag.debug( - `lane task done: lane=${lane} durationMs=${Date.now() - startTime} active=${state.active} queued=${state.queue.length}`, - ); - pump(); + const completedCurrentGeneration = completeTask(state, taskId, taskGeneration); + if (completedCurrentGeneration) { + diag.debug( + `lane task done: lane=${lane} durationMs=${Date.now() - startTime} active=${state.activeTaskIds.size} queued=${state.queue.length}`, + ); + pump(); + } entry.resolve(result); } catch (err) { - state.active -= 1; - state.activeTaskIds.delete(taskId); + const completedCurrentGeneration = completeTask(state, taskId, taskGeneration); const isProbeLane = lane.startsWith("auth-probe:") || lane.startsWith("session:probe-"); if (!isProbeLane) { diag.error( `lane task error: lane=${lane} durationMs=${Date.now() - startTime} error="${String(err)}"`, ); } - pump(); + if (completedCurrentGeneration) { + pump(); + } entry.reject(err); } })(); @@ -134,7 +144,7 @@ export function enqueueCommandInLane( warnAfterMs, onWait: opts?.onWait, }); - logLaneEnqueue(cleaned, state.queue.length + state.active); + logLaneEnqueue(cleaned, state.queue.length + state.activeTaskIds.size); drainLane(cleaned); }); } @@ -155,13 +165,13 @@ export function getQueueSize(lane: string = CommandLane.Main) { if (!state) { return 0; } - return state.queue.length + state.active; + return state.queue.length + state.activeTaskIds.size; } export function getTotalQueueSize() { let total = 0; for (const s of lanes.values()) { - total += s.queue.length + s.active; + total += s.queue.length + s.activeTaskIds.size; } return total; } @@ -180,6 +190,36 @@ export function clearCommandLane(lane: string = CommandLane.Main) { return removed; } +/** + * Reset all lane runtime state to idle. Used after SIGUSR1 in-process + * restarts where interrupted tasks' finally blocks may not run, leaving + * stale active task IDs that permanently block new work from draining. + * + * Bumps lane generation and clears execution counters so stale completions + * from old in-flight tasks are ignored. Queued entries are intentionally + * preserved — they represent pending user work that should still execute + * after restart. + * + * After resetting, drains any lanes that still have queued entries so + * preserved work is pumped immediately rather than waiting for a future + * `enqueueCommandInLane()` call (which may never come). + */ +export function resetAllLanes(): void { + const lanesToDrain: string[] = []; + for (const state of lanes.values()) { + state.generation += 1; + state.activeTaskIds.clear(); + state.draining = false; + if (state.queue.length > 0) { + lanesToDrain.push(state.lane); + } + } + // Drain after the full reset pass so all lanes are in a clean state first. + for (const lane of lanesToDrain) { + drainLane(lane); + } +} + /** * Returns the total number of actively executing tasks across all lanes * (excludes queued-but-not-started entries). @@ -187,7 +227,7 @@ export function clearCommandLane(lane: string = CommandLane.Main) { export function getActiveTaskCount(): number { let total = 0; for (const s of lanes.values()) { - total += s.active; + total += s.activeTaskIds.size; } return total; } diff --git a/src/process/restart-recovery.test.ts b/src/process/restart-recovery.test.ts new file mode 100644 index 00000000000..5091d7b9928 --- /dev/null +++ b/src/process/restart-recovery.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it, vi } from "vitest"; +import { createRestartIterationHook } from "./restart-recovery.js"; + +describe("restart-recovery", () => { + it("skips recovery on first iteration and runs on subsequent iterations", () => { + const onRestart = vi.fn(); + const onIteration = createRestartIterationHook(onRestart); + + expect(onIteration()).toBe(false); + expect(onRestart).not.toHaveBeenCalled(); + + expect(onIteration()).toBe(true); + expect(onRestart).toHaveBeenCalledTimes(1); + + expect(onIteration()).toBe(true); + expect(onRestart).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/process/restart-recovery.ts b/src/process/restart-recovery.ts new file mode 100644 index 00000000000..2f9818d7f5a --- /dev/null +++ b/src/process/restart-recovery.ts @@ -0,0 +1,16 @@ +/** + * Returns an iteration hook for in-process restart loops. + * The first call is considered initial startup and does nothing. + * Each subsequent call represents a restart iteration and invokes `onRestart`. + */ +export function createRestartIterationHook(onRestart: () => void): () => boolean { + let isFirstIteration = true; + return () => { + if (isFirstIteration) { + isFirstIteration = false; + return false; + } + onRestart(); + return true; + }; +} From 93dd51bce024614cca45576db2da89ff4df9a689 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 20:27:47 +0000 Subject: [PATCH 0003/1038] perf(matrix): lazy-load music-metadata parsing --- extensions/matrix/src/matrix/send/media.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/matrix/src/matrix/send/media.ts b/extensions/matrix/src/matrix/send/media.ts index c4339d90057..eecdce3d565 100644 --- a/extensions/matrix/src/matrix/send/media.ts +++ b/extensions/matrix/src/matrix/send/media.ts @@ -6,7 +6,6 @@ import type { TimedFileInfo, VideoFileInfo, } from "@vector-im/matrix-bot-sdk"; -import { parseBuffer, type IFileInfo } from "music-metadata"; import { getMatrixRuntime } from "../../runtime.js"; import { applyMatrixFormatting } from "./formatting.js"; import { @@ -18,6 +17,7 @@ import { } from "./types.js"; const getCore = () => getMatrixRuntime(); +type IFileInfo = import("music-metadata").IFileInfo; export function buildMatrixMediaInfo(params: { size: number; @@ -164,6 +164,7 @@ export async function resolveMediaDurationMs(params: { return undefined; } try { + const { parseBuffer } = await import("music-metadata"); const fileInfo: IFileInfo | string | undefined = params.contentType || params.fileName ? { From caebe70e9aca10f046d44bb94e699e41fa2e83b4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 21:23:44 +0000 Subject: [PATCH 0004/1038] perf(test): cut setup/import overhead in hot suites --- .../tools/web-fetch.cf-markdown.test.ts | 48 +-- ...re.clamps-timeoutms-scrollintoview.test.ts | 11 +- ...ls-core.last-file-chooser-arm-wins.test.ts | 8 +- ...-core.screenshots-element-selector.test.ts | 11 +- ...-core.waits-next-download-saves-it.test.ts | 11 +- ....agent-contract-snapshot-endpoints.test.ts | 5 +- ...te-disabled-does-not-block-storage.test.ts | 5 +- ...s-open-profile-unknown-returns-404.test.ts | 17 +- src/cli/exec-approvals-cli.test.ts | 14 +- src/cli/update-cli.test.ts | 338 +++++++++--------- src/config/config.identity-defaults.test.ts | 72 +++- src/config/config.plugin-validation.test.ts | 225 ++++++------ src/hooks/install.test.ts | 30 +- src/plugins/loader.test.ts | 23 +- src/process/child-process-bridge.test.ts | 13 +- src/web/media.test.ts | 23 +- 16 files changed, 428 insertions(+), 426 deletions(-) diff --git a/src/agents/tools/web-fetch.cf-markdown.test.ts b/src/agents/tools/web-fetch.cf-markdown.test.ts index d73300681fc..a9602291d2e 100644 --- a/src/agents/tools/web-fetch.cf-markdown.test.ts +++ b/src/agents/tools/web-fetch.cf-markdown.test.ts @@ -1,9 +1,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as ssrf from "../../infra/net/ssrf.js"; import * as logger from "../../logger.js"; +import { createWebFetchTool } from "./web-tools.js"; const lookupMock = vi.fn(); const resolvePinnedHostname = ssrf.resolvePinnedHostname; +const baseToolConfig = { + config: { + tools: { web: { fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } } } }, + }, +} as const; function makeHeaders(map: Record): { get: (key: string) => string | null } { return { @@ -51,12 +57,7 @@ describe("web_fetch Cloudflare Markdown for Agents", () => { // @ts-expect-error mock fetch global.fetch = fetchSpy; - const { createWebFetchTool } = await import("./web-tools.js"); - const tool = createWebFetchTool({ - config: { - tools: { web: { fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } } } }, - }, - }); + const tool = createWebFetchTool(baseToolConfig); await tool?.execute?.("call", { url: "https://example.com/page" }); @@ -71,12 +72,7 @@ describe("web_fetch Cloudflare Markdown for Agents", () => { // @ts-expect-error mock fetch global.fetch = fetchSpy; - const { createWebFetchTool } = await import("./web-tools.js"); - const tool = createWebFetchTool({ - config: { - tools: { web: { fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } } } }, - }, - }); + const tool = createWebFetchTool(baseToolConfig); const result = await tool?.execute?.("call", { url: "https://example.com/cf" }); expect(result?.details).toMatchObject({ @@ -96,12 +92,7 @@ describe("web_fetch Cloudflare Markdown for Agents", () => { // @ts-expect-error mock fetch global.fetch = fetchSpy; - const { createWebFetchTool } = await import("./web-tools.js"); - const tool = createWebFetchTool({ - config: { - tools: { web: { fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } } } }, - }, - }); + const tool = createWebFetchTool(baseToolConfig); const result = await tool?.execute?.("call", { url: "https://example.com/html" }); expect(result?.details?.extractor).not.toBe("cf-markdown"); @@ -116,12 +107,7 @@ describe("web_fetch Cloudflare Markdown for Agents", () => { // @ts-expect-error mock fetch global.fetch = fetchSpy; - const { createWebFetchTool } = await import("./web-tools.js"); - const tool = createWebFetchTool({ - config: { - tools: { web: { fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } } } }, - }, - }); + const tool = createWebFetchTool(baseToolConfig); await tool?.execute?.("call", { url: "https://example.com/tokens/private?token=secret" }); @@ -142,12 +128,7 @@ describe("web_fetch Cloudflare Markdown for Agents", () => { // @ts-expect-error mock fetch global.fetch = fetchSpy; - const { createWebFetchTool } = await import("./web-tools.js"); - const tool = createWebFetchTool({ - config: { - tools: { web: { fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } } } }, - }, - }); + const tool = createWebFetchTool(baseToolConfig); const result = await tool?.execute?.("call", { url: "https://example.com/text-mode", @@ -169,12 +150,7 @@ describe("web_fetch Cloudflare Markdown for Agents", () => { // @ts-expect-error mock fetch global.fetch = fetchSpy; - const { createWebFetchTool } = await import("./web-tools.js"); - const tool = createWebFetchTool({ - config: { - tools: { web: { fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } } } }, - }, - }); + const tool = createWebFetchTool(baseToolConfig); await tool?.execute?.("call", { url: "https://example.com/no-tokens" }); diff --git a/src/browser/pw-tools-core.clamps-timeoutms-scrollintoview.test.ts b/src/browser/pw-tools-core.clamps-timeoutms-scrollintoview.test.ts index 4a98144ed9d..55216b79bbd 100644 --- a/src/browser/pw-tools-core.clamps-timeoutms-scrollintoview.test.ts +++ b/src/browser/pw-tools-core.clamps-timeoutms-scrollintoview.test.ts @@ -28,10 +28,7 @@ const sessionMocks = vi.hoisted(() => ({ })); vi.mock("./pw-session.js", () => sessionMocks); - -async function importModule() { - return await import("./pw-tools-core.js"); -} +const mod = await import("./pw-tools-core.js"); describe("pw-tools-core", () => { beforeEach(() => { @@ -53,7 +50,6 @@ describe("pw-tools-core", () => { currentRefLocator = { scrollIntoViewIfNeeded }; currentPage = {}; - const mod = await importModule(); await mod.scrollIntoViewViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", targetId: "T1", @@ -70,7 +66,6 @@ describe("pw-tools-core", () => { currentRefLocator = { scrollIntoViewIfNeeded }; currentPage = {}; - const mod = await importModule(); await expect( mod.scrollIntoViewViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", @@ -86,7 +81,6 @@ describe("pw-tools-core", () => { currentRefLocator = { scrollIntoViewIfNeeded }; currentPage = {}; - const mod = await importModule(); await expect( mod.scrollIntoViewViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", @@ -102,7 +96,6 @@ describe("pw-tools-core", () => { currentRefLocator = { click }; currentPage = {}; - const mod = await importModule(); await expect( mod.clickViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", @@ -118,7 +111,6 @@ describe("pw-tools-core", () => { currentRefLocator = { click }; currentPage = {}; - const mod = await importModule(); await expect( mod.clickViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", @@ -136,7 +128,6 @@ describe("pw-tools-core", () => { currentRefLocator = { click }; currentPage = {}; - const mod = await importModule(); await expect( mod.clickViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", diff --git a/src/browser/pw-tools-core.last-file-chooser-arm-wins.test.ts b/src/browser/pw-tools-core.last-file-chooser-arm-wins.test.ts index a197691ca71..baaf3e1ba85 100644 --- a/src/browser/pw-tools-core.last-file-chooser-arm-wins.test.ts +++ b/src/browser/pw-tools-core.last-file-chooser-arm-wins.test.ts @@ -28,10 +28,7 @@ const sessionMocks = vi.hoisted(() => ({ })); vi.mock("./pw-session.js", () => sessionMocks); - -async function importModule() { - return await import("./pw-tools-core.js"); -} +const mod = await import("./pw-tools-core.js"); describe("pw-tools-core", () => { beforeEach(() => { @@ -75,7 +72,6 @@ describe("pw-tools-core", () => { keyboard: { press: vi.fn(async () => {}) }, }; - const mod = await importModule(); await mod.armFileUploadViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", paths: ["/tmp/1"], @@ -101,7 +97,6 @@ describe("pw-tools-core", () => { waitForEvent, }; - const mod = await importModule(); await mod.armDialogViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", accept: true, @@ -145,7 +140,6 @@ describe("pw-tools-core", () => { getByText: vi.fn(() => ({ first: () => ({ waitFor: vi.fn() }) })), }; - const mod = await importModule(); await mod.waitForViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", selector: "#main", diff --git a/src/browser/pw-tools-core.screenshots-element-selector.test.ts b/src/browser/pw-tools-core.screenshots-element-selector.test.ts index a297f7d512e..96a4a06ea54 100644 --- a/src/browser/pw-tools-core.screenshots-element-selector.test.ts +++ b/src/browser/pw-tools-core.screenshots-element-selector.test.ts @@ -28,10 +28,7 @@ const sessionMocks = vi.hoisted(() => ({ })); vi.mock("./pw-session.js", () => sessionMocks); - -async function importModule() { - return await import("./pw-tools-core.js"); -} +const mod = await import("./pw-tools-core.js"); describe("pw-tools-core", () => { beforeEach(() => { @@ -57,7 +54,6 @@ describe("pw-tools-core", () => { screenshot: vi.fn(async () => Buffer.from("P")), }; - const mod = await importModule(); const res = await mod.takeScreenshotViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", targetId: "T1", @@ -78,7 +74,6 @@ describe("pw-tools-core", () => { screenshot: vi.fn(async () => Buffer.from("P")), }; - const mod = await importModule(); const res = await mod.takeScreenshotViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", targetId: "T1", @@ -99,8 +94,6 @@ describe("pw-tools-core", () => { screenshot: vi.fn(async () => Buffer.from("P")), }; - const mod = await importModule(); - await expect( mod.takeScreenshotViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", @@ -127,7 +120,6 @@ describe("pw-tools-core", () => { keyboard: { press: vi.fn(async () => {}) }, }; - const mod = await importModule(); await mod.armFileUploadViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", targetId: "T1", @@ -151,7 +143,6 @@ describe("pw-tools-core", () => { keyboard: { press }, }; - const mod = await importModule(); await mod.armFileUploadViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", paths: [], diff --git a/src/browser/pw-tools-core.waits-next-download-saves-it.test.ts b/src/browser/pw-tools-core.waits-next-download-saves-it.test.ts index 9ff8d1acab0..59d233e0005 100644 --- a/src/browser/pw-tools-core.waits-next-download-saves-it.test.ts +++ b/src/browser/pw-tools-core.waits-next-download-saves-it.test.ts @@ -33,10 +33,7 @@ const tmpDirMocks = vi.hoisted(() => ({ resolvePreferredOpenClawTmpDir: vi.fn(() => "/tmp/openclaw"), })); vi.mock("../infra/tmp-openclaw-dir.js", () => tmpDirMocks); - -async function importModule() { - return await import("./pw-tools-core.js"); -} +const mod = await import("./pw-tools-core.js"); describe("pw-tools-core", () => { beforeEach(() => { @@ -75,7 +72,6 @@ describe("pw-tools-core", () => { currentPage = { on, off }; - const mod = await importModule(); const targetPath = path.resolve("/tmp/file.bin"); const p = mod.waitForDownloadViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", @@ -113,7 +109,6 @@ describe("pw-tools-core", () => { currentPage = { on, off }; - const mod = await importModule(); const targetPath = path.resolve("/tmp/report.pdf"); const p = mod.downloadViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", @@ -152,7 +147,6 @@ describe("pw-tools-core", () => { tmpDirMocks.resolvePreferredOpenClawTmpDir.mockReturnValue("/tmp/openclaw-preferred"); currentPage = { on, off }; - const mod = await importModule(); const p = mod.waitForDownloadViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", targetId: "T1", @@ -194,7 +188,6 @@ describe("pw-tools-core", () => { text: async () => '{"ok":true,"value":123}', }; - const mod = await importModule(); const p = mod.responseBodyViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", targetId: "T1", @@ -218,7 +211,6 @@ describe("pw-tools-core", () => { currentRefLocator = { scrollIntoViewIfNeeded }; currentPage = {}; - const mod = await importModule(); await mod.scrollIntoViewViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", targetId: "T1", @@ -232,7 +224,6 @@ describe("pw-tools-core", () => { currentRefLocator = { scrollIntoViewIfNeeded: vi.fn(async () => {}) }; currentPage = {}; - const mod = await importModule(); await expect( mod.scrollIntoViewViaPlaywright({ cdpUrl: "http://127.0.0.1:18792", diff --git a/src/browser/server.agent-contract-snapshot-endpoints.test.ts b/src/browser/server.agent-contract-snapshot-endpoints.test.ts index ab8c70317d2..8c4530a91a2 100644 --- a/src/browser/server.agent-contract-snapshot-endpoints.test.ts +++ b/src/browser/server.agent-contract-snapshot-endpoints.test.ts @@ -154,6 +154,9 @@ vi.mock("./screenshot.js", () => ({ })), })); +const { startBrowserControlServerFromConfig, stopBrowserControlServer } = + await import("./server.js"); + async function getFreePort(): Promise { while (true) { const port = await new Promise((resolve, reject) => { @@ -271,12 +274,10 @@ describe("browser control server", () => { } else { process.env.OPENCLAW_GATEWAY_PORT = prevGatewayPort; } - const { stopBrowserControlServer } = await import("./server.js"); await stopBrowserControlServer(); }); const startServerAndBase = async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; await realFetch(`${base}/start`, { method: "POST" }).then((r) => r.json()); diff --git a/src/browser/server.evaluate-disabled-does-not-block-storage.test.ts b/src/browser/server.evaluate-disabled-does-not-block-storage.test.ts index b24438f2787..c7d3f6c9523 100644 --- a/src/browser/server.evaluate-disabled-does-not-block-storage.test.ts +++ b/src/browser/server.evaluate-disabled-does-not-block-storage.test.ts @@ -63,6 +63,9 @@ vi.mock("./server-context.js", async (importOriginal) => { }; }); +const { startBrowserControlServerFromConfig, stopBrowserControlServer } = + await import("./server.js"); + async function getFreePort(): Promise { const probe = createServer(); await new Promise((resolve, reject) => { @@ -95,12 +98,10 @@ describe("browser control evaluate gating", () => { process.env.OPENCLAW_GATEWAY_PORT = prevGatewayPort; } - const { stopBrowserControlServer } = await import("./server.js"); await stopBrowserControlServer(); }); it("blocks act:evaluate but still allows cookies/storage reads", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; diff --git a/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts b/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts index e2c75a85f0e..e4c828f6d39 100644 --- a/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts +++ b/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts @@ -153,6 +153,9 @@ vi.mock("./screenshot.js", () => ({ })), })); +const { startBrowserControlServerFromConfig, stopBrowserControlServer } = + await import("./server.js"); + async function getFreePort(): Promise { while (true) { const port = await new Promise((resolve, reject) => { @@ -270,12 +273,10 @@ describe("browser control server", () => { } else { process.env.OPENCLAW_GATEWAY_PORT = prevGatewayPort; } - const { stopBrowserControlServer } = await import("./server.js"); await stopBrowserControlServer(); }); it("POST /tabs/open?profile=unknown returns 404", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -307,9 +308,6 @@ describe("profile CRUD endpoints", () => { prevGatewayPort = process.env.OPENCLAW_GATEWAY_PORT; process.env.OPENCLAW_GATEWAY_PORT = String(testPort - 2); - prevGatewayPort = process.env.OPENCLAW_GATEWAY_PORT; - process.env.OPENCLAW_GATEWAY_PORT = String(testPort - 2); - vi.stubGlobal( "fetch", vi.fn(async (url: string) => { @@ -330,12 +328,10 @@ describe("profile CRUD endpoints", () => { } else { process.env.OPENCLAW_GATEWAY_PORT = prevGatewayPort; } - const { stopBrowserControlServer } = await import("./server.js"); await stopBrowserControlServer(); }); it("POST /profiles/create returns 400 for missing name", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -350,7 +346,6 @@ describe("profile CRUD endpoints", () => { }); it("POST /profiles/create returns 400 for invalid name format", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -365,7 +360,6 @@ describe("profile CRUD endpoints", () => { }); it("POST /profiles/create returns 409 for duplicate name", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -381,7 +375,6 @@ describe("profile CRUD endpoints", () => { }); it("POST /profiles/create accepts cdpUrl for remote profiles", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -402,7 +395,6 @@ describe("profile CRUD endpoints", () => { }); it("POST /profiles/create returns 400 for invalid cdpUrl", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -417,7 +409,6 @@ describe("profile CRUD endpoints", () => { }); it("DELETE /profiles/:name returns 404 for non-existent profile", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -430,7 +421,6 @@ describe("profile CRUD endpoints", () => { }); it("DELETE /profiles/:name returns 400 for default profile deletion", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; @@ -444,7 +434,6 @@ describe("profile CRUD endpoints", () => { }); it("DELETE /profiles/:name returns 400 for invalid name format", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; diff --git a/src/cli/exec-approvals-cli.test.ts b/src/cli/exec-approvals-cli.test.ts index 1d8a1d58dcd..a875d58782c 100644 --- a/src/cli/exec-approvals-cli.test.ts +++ b/src/cli/exec-approvals-cli.test.ts @@ -59,9 +59,11 @@ vi.mock("../infra/exec-approvals.js", async () => { }; }); +const { registerExecApprovalsCli } = await import("./exec-approvals-cli.js"); +const execApprovals = await import("../infra/exec-approvals.js"); + describe("exec approvals CLI", () => { - const createProgram = async () => { - const { registerExecApprovalsCli } = await import("./exec-approvals-cli.js"); + const createProgram = () => { const program = new Command(); program.exitOverride(); registerExecApprovalsCli(program); @@ -73,21 +75,21 @@ describe("exec approvals CLI", () => { runtimeErrors.length = 0; callGatewayFromCli.mockClear(); - const localProgram = await createProgram(); + const localProgram = createProgram(); await localProgram.parseAsync(["approvals", "get"], { from: "user" }); expect(callGatewayFromCli).not.toHaveBeenCalled(); expect(runtimeErrors).toHaveLength(0); callGatewayFromCli.mockClear(); - const gatewayProgram = await createProgram(); + const gatewayProgram = createProgram(); await gatewayProgram.parseAsync(["approvals", "get", "--gateway"], { from: "user" }); expect(callGatewayFromCli).toHaveBeenCalledWith("exec.approvals.get", expect.anything(), {}); expect(runtimeErrors).toHaveLength(0); callGatewayFromCli.mockClear(); - const nodeProgram = await createProgram(); + const nodeProgram = createProgram(); await nodeProgram.parseAsync(["approvals", "get", "--node", "macbook"], { from: "user" }); expect(callGatewayFromCli).toHaveBeenCalledWith("exec.approvals.node.get", expect.anything(), { @@ -101,11 +103,9 @@ describe("exec approvals CLI", () => { runtimeErrors.length = 0; callGatewayFromCli.mockClear(); - const execApprovals = await import("../infra/exec-approvals.js"); const saveExecApprovals = vi.mocked(execApprovals.saveExecApprovals); saveExecApprovals.mockClear(); - const { registerExecApprovalsCli } = await import("./exec-approvals-cli.js"); const program = new Command(); program.exitOverride(); registerExecApprovalsCli(program); diff --git a/src/cli/update-cli.test.ts b/src/cli/update-cli.test.ts index ca6a3cb1652..aa771741270 100644 --- a/src/cli/update-cli.test.ts +++ b/src/cli/update-cli.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { UpdateRunResult } from "../infra/update-runner.js"; const confirm = vi.fn(); @@ -91,6 +91,23 @@ const { updateCommand, registerUpdateCli, updateStatusCommand, updateWizardComma await import("./update-cli.js"); describe("update-cli", () => { + let fixtureRoot = ""; + let fixtureCount = 0; + + const createCaseDir = async (prefix: string) => { + const dir = path.join(fixtureRoot, `${prefix}-${fixtureCount++}`); + await fs.mkdir(dir, { recursive: true }); + return dir; + }; + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-tests-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + const baseSnapshot = { valid: true, config: {}, @@ -223,41 +240,37 @@ describe("update-cli", () => { }); it("defaults to stable channel for package installs when unset", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-")); - try { - await fs.writeFile( - path.join(tempDir, "package.json"), - JSON.stringify({ name: "openclaw", version: "1.0.0" }), - "utf-8", - ); + const tempDir = await createCaseDir("openclaw-update"); + await fs.writeFile( + path.join(tempDir, "package.json"), + JSON.stringify({ name: "openclaw", version: "1.0.0" }), + "utf-8", + ); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); - vi.mocked(checkUpdateStatus).mockResolvedValue({ - root: tempDir, - installKind: "package", - packageManager: "npm", - deps: { - manager: "npm", - status: "ok", - lockfilePath: null, - markerPath: null, - }, - }); - vi.mocked(runGatewayUpdate).mockResolvedValue({ + vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); + vi.mocked(checkUpdateStatus).mockResolvedValue({ + root: tempDir, + installKind: "package", + packageManager: "npm", + deps: { + manager: "npm", status: "ok", - mode: "npm", - steps: [], - durationMs: 100, - }); + lockfilePath: null, + markerPath: null, + }, + }); + vi.mocked(runGatewayUpdate).mockResolvedValue({ + status: "ok", + mode: "npm", + steps: [], + durationMs: 100, + }); - await updateCommand({ yes: true }); + await updateCommand({ yes: true }); - const call = vi.mocked(runGatewayUpdate).mock.calls[0]?.[0]; - expect(call?.channel).toBe("stable"); - expect(call?.tag).toBe("latest"); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + const call = vi.mocked(runGatewayUpdate).mock.calls[0]?.[0]; + expect(call?.channel).toBe("stable"); + expect(call?.tag).toBe("latest"); }); it("uses stored beta channel when configured", async () => { @@ -279,75 +292,67 @@ describe("update-cli", () => { }); it("falls back to latest when beta tag is older than release", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-")); - try { - await fs.writeFile( - path.join(tempDir, "package.json"), - JSON.stringify({ name: "openclaw", version: "1.0.0" }), - "utf-8", - ); + const tempDir = await createCaseDir("openclaw-update"); + await fs.writeFile( + path.join(tempDir, "package.json"), + JSON.stringify({ name: "openclaw", version: "1.0.0" }), + "utf-8", + ); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); - vi.mocked(readConfigFileSnapshot).mockResolvedValue({ - ...baseSnapshot, - config: { update: { channel: "beta" } }, - }); - vi.mocked(checkUpdateStatus).mockResolvedValue({ - root: tempDir, - installKind: "package", - packageManager: "npm", - deps: { - manager: "npm", - status: "ok", - lockfilePath: null, - markerPath: null, - }, - }); - vi.mocked(resolveNpmChannelTag).mockResolvedValue({ - tag: "latest", - version: "1.2.3-1", - }); - vi.mocked(runGatewayUpdate).mockResolvedValue({ + vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); + vi.mocked(readConfigFileSnapshot).mockResolvedValue({ + ...baseSnapshot, + config: { update: { channel: "beta" } }, + }); + vi.mocked(checkUpdateStatus).mockResolvedValue({ + root: tempDir, + installKind: "package", + packageManager: "npm", + deps: { + manager: "npm", status: "ok", - mode: "npm", - steps: [], - durationMs: 100, - }); + lockfilePath: null, + markerPath: null, + }, + }); + vi.mocked(resolveNpmChannelTag).mockResolvedValue({ + tag: "latest", + version: "1.2.3-1", + }); + vi.mocked(runGatewayUpdate).mockResolvedValue({ + status: "ok", + mode: "npm", + steps: [], + durationMs: 100, + }); - await updateCommand({}); + await updateCommand({}); - const call = vi.mocked(runGatewayUpdate).mock.calls[0]?.[0]; - expect(call?.channel).toBe("beta"); - expect(call?.tag).toBe("latest"); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + const call = vi.mocked(runGatewayUpdate).mock.calls[0]?.[0]; + expect(call?.channel).toBe("beta"); + expect(call?.tag).toBe("latest"); }); it("honors --tag override", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-")); - try { - await fs.writeFile( - path.join(tempDir, "package.json"), - JSON.stringify({ name: "openclaw", version: "1.0.0" }), - "utf-8", - ); + const tempDir = await createCaseDir("openclaw-update"); + await fs.writeFile( + path.join(tempDir, "package.json"), + JSON.stringify({ name: "openclaw", version: "1.0.0" }), + "utf-8", + ); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); - vi.mocked(runGatewayUpdate).mockResolvedValue({ - status: "ok", - mode: "npm", - steps: [], - durationMs: 100, - }); + vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); + vi.mocked(runGatewayUpdate).mockResolvedValue({ + status: "ok", + mode: "npm", + steps: [], + durationMs: 100, + }); - await updateCommand({ tag: "next" }); + await updateCommand({ tag: "next" }); - const call = vi.mocked(runGatewayUpdate).mock.calls[0]?.[0]; - expect(call?.tag).toBe("next"); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + const call = vi.mocked(runGatewayUpdate).mock.calls[0]?.[0]; + expect(call?.tag).toBe("next"); }); it("updateCommand outputs JSON when --json is set", async () => { @@ -471,95 +476,87 @@ describe("update-cli", () => { }); it("requires confirmation on downgrade when non-interactive", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-")); - try { - setTty(false); - await fs.writeFile( - path.join(tempDir, "package.json"), - JSON.stringify({ name: "openclaw", version: "2.0.0" }), - "utf-8", - ); + const tempDir = await createCaseDir("openclaw-update"); + setTty(false); + await fs.writeFile( + path.join(tempDir, "package.json"), + JSON.stringify({ name: "openclaw", version: "2.0.0" }), + "utf-8", + ); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); - vi.mocked(checkUpdateStatus).mockResolvedValue({ - root: tempDir, - installKind: "package", - packageManager: "npm", - deps: { - manager: "npm", - status: "ok", - lockfilePath: null, - markerPath: null, - }, - }); - vi.mocked(resolveNpmChannelTag).mockResolvedValue({ - tag: "latest", - version: "0.0.1", - }); - vi.mocked(runGatewayUpdate).mockResolvedValue({ + vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); + vi.mocked(checkUpdateStatus).mockResolvedValue({ + root: tempDir, + installKind: "package", + packageManager: "npm", + deps: { + manager: "npm", status: "ok", - mode: "npm", - steps: [], - durationMs: 100, - }); - vi.mocked(defaultRuntime.error).mockClear(); - vi.mocked(defaultRuntime.exit).mockClear(); + lockfilePath: null, + markerPath: null, + }, + }); + vi.mocked(resolveNpmChannelTag).mockResolvedValue({ + tag: "latest", + version: "0.0.1", + }); + vi.mocked(runGatewayUpdate).mockResolvedValue({ + status: "ok", + mode: "npm", + steps: [], + durationMs: 100, + }); + vi.mocked(defaultRuntime.error).mockClear(); + vi.mocked(defaultRuntime.exit).mockClear(); - await updateCommand({}); + await updateCommand({}); - expect(defaultRuntime.error).toHaveBeenCalledWith( - expect.stringContaining("Downgrade confirmation required."), - ); - expect(defaultRuntime.exit).toHaveBeenCalledWith(1); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + expect(defaultRuntime.error).toHaveBeenCalledWith( + expect.stringContaining("Downgrade confirmation required."), + ); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); }); it("allows downgrade with --yes in non-interactive mode", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-")); - try { - setTty(false); - await fs.writeFile( - path.join(tempDir, "package.json"), - JSON.stringify({ name: "openclaw", version: "2.0.0" }), - "utf-8", - ); + const tempDir = await createCaseDir("openclaw-update"); + setTty(false); + await fs.writeFile( + path.join(tempDir, "package.json"), + JSON.stringify({ name: "openclaw", version: "2.0.0" }), + "utf-8", + ); - vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); - vi.mocked(checkUpdateStatus).mockResolvedValue({ - root: tempDir, - installKind: "package", - packageManager: "npm", - deps: { - manager: "npm", - status: "ok", - lockfilePath: null, - markerPath: null, - }, - }); - vi.mocked(resolveNpmChannelTag).mockResolvedValue({ - tag: "latest", - version: "0.0.1", - }); - vi.mocked(runGatewayUpdate).mockResolvedValue({ + vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(tempDir); + vi.mocked(checkUpdateStatus).mockResolvedValue({ + root: tempDir, + installKind: "package", + packageManager: "npm", + deps: { + manager: "npm", status: "ok", - mode: "npm", - steps: [], - durationMs: 100, - }); - vi.mocked(defaultRuntime.error).mockClear(); - vi.mocked(defaultRuntime.exit).mockClear(); + lockfilePath: null, + markerPath: null, + }, + }); + vi.mocked(resolveNpmChannelTag).mockResolvedValue({ + tag: "latest", + version: "0.0.1", + }); + vi.mocked(runGatewayUpdate).mockResolvedValue({ + status: "ok", + mode: "npm", + steps: [], + durationMs: 100, + }); + vi.mocked(defaultRuntime.error).mockClear(); + vi.mocked(defaultRuntime.exit).mockClear(); - await updateCommand({ yes: true }); + await updateCommand({ yes: true }); - expect(defaultRuntime.error).not.toHaveBeenCalledWith( - expect.stringContaining("Downgrade confirmation required."), - ); - expect(runGatewayUpdate).toHaveBeenCalled(); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + expect(defaultRuntime.error).not.toHaveBeenCalledWith( + expect.stringContaining("Downgrade confirmation required."), + ); + expect(runGatewayUpdate).toHaveBeenCalled(); }); it("updateWizardCommand requires a TTY", async () => { @@ -576,7 +573,7 @@ describe("update-cli", () => { }); it("updateWizardCommand offers dev checkout and forwards selections", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-wizard-")); + const tempDir = await createCaseDir("openclaw-update-wizard"); const previousGitDir = process.env.OPENCLAW_GIT_DIR; try { setTty(true); @@ -608,7 +605,6 @@ describe("update-cli", () => { expect(call?.channel).toBe("dev"); } finally { process.env.OPENCLAW_GIT_DIR = previousGitDir; - await fs.rm(tempDir, { recursive: true, force: true }); } }); }); diff --git a/src/config/config.identity-defaults.test.ts b/src/config/config.identity-defaults.test.ts index fe5286fe6f7..48a6710a44a 100644 --- a/src/config/config.identity-defaults.test.ts +++ b/src/config/config.identity-defaults.test.ts @@ -1,19 +1,53 @@ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { DEFAULT_AGENT_MAX_CONCURRENT, DEFAULT_SUBAGENT_MAX_CONCURRENT } from "./agent-limits.js"; import { loadConfig } from "./config.js"; -import { withTempHome } from "./test-helpers.js"; + +type HomeEnvSnapshot = { + home: string | undefined; + userProfile: string | undefined; + homeDrive: string | undefined; + homePath: string | undefined; + stateDir: string | undefined; +}; + +function snapshotHomeEnv(): HomeEnvSnapshot { + return { + home: process.env.HOME, + userProfile: process.env.USERPROFILE, + homeDrive: process.env.HOMEDRIVE, + homePath: process.env.HOMEPATH, + stateDir: process.env.OPENCLAW_STATE_DIR, + }; +} + +function restoreHomeEnv(snapshot: HomeEnvSnapshot) { + const restoreKey = (key: string, value: string | undefined) => { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + }; + restoreKey("HOME", snapshot.home); + restoreKey("USERPROFILE", snapshot.userProfile); + restoreKey("HOMEDRIVE", snapshot.homeDrive); + restoreKey("HOMEPATH", snapshot.homePath); + restoreKey("OPENCLAW_STATE_DIR", snapshot.stateDir); +} describe("config identity defaults", () => { - let previousHome: string | undefined; + let fixtureRoot = ""; + let fixtureCount = 0; - beforeEach(() => { - previousHome = process.env.HOME; + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-config-identity-")); }); - afterEach(() => { - process.env.HOME = previousHome; + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); }); const writeAndLoadConfig = async (home: string, config: Record) => { @@ -27,6 +61,30 @@ describe("config identity defaults", () => { return loadConfig(); }; + const withTempHome = async (fn: (home: string) => Promise): Promise => { + const home = path.join(fixtureRoot, `home-${fixtureCount++}`); + await fs.mkdir(path.join(home, ".openclaw"), { recursive: true }); + + const snapshot = snapshotHomeEnv(); + process.env.HOME = home; + process.env.USERPROFILE = home; + process.env.OPENCLAW_STATE_DIR = path.join(home, ".openclaw"); + + if (process.platform === "win32") { + const match = home.match(/^([A-Za-z]:)(.*)$/); + if (match) { + process.env.HOMEDRIVE = match[1]; + process.env.HOMEPATH = match[2] || "\\"; + } + } + + try { + return await fn(home); + } finally { + restoreHomeEnv(snapshot); + } + }; + it("does not derive mention defaults and only sets ackReactionScope when identity is present", async () => { await withTempHome(async (home) => { const cfg = await writeAndLoadConfig(home, { diff --git a/src/config/config.plugin-validation.test.ts b/src/config/config.plugin-validation.test.ts index 418af2fdbac..c7389a59f27 100644 --- a/src/config/config.plugin-validation.test.ts +++ b/src/config/config.plugin-validation.test.ts @@ -1,8 +1,8 @@ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterAll, describe, expect, it } from "vitest"; import { validateConfigObjectWithPlugins } from "./config.js"; -import { withTempHome } from "./test-helpers.js"; async function writePluginFixture(params: { dir: string; @@ -31,145 +31,150 @@ async function writePluginFixture(params: { } describe("config plugin validation", () => { + const fixtureRoot = path.join(os.tmpdir(), "openclaw-config-plugin-validation"); + let caseIndex = 0; + + function createCaseHome() { + const home = path.join(fixtureRoot, `case-${caseIndex++}`); + return fs.mkdir(home, { recursive: true }).then(() => home); + } + const validateInHome = (home: string, raw: unknown) => { process.env.OPENCLAW_STATE_DIR = path.join(home, ".openclaw"); return validateConfigObjectWithPlugins(raw); }; + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + it("rejects missing plugin load paths", async () => { - await withTempHome(async (home) => { - const missingPath = path.join(home, "missing-plugin"); - const res = validateInHome(home, { - agents: { list: [{ id: "pi" }] }, - plugins: { enabled: false, load: { paths: [missingPath] } }, - }); - expect(res.ok).toBe(false); - if (!res.ok) { - const hasIssue = res.issues.some( - (issue) => - issue.path === "plugins.load.paths" && issue.message.includes("plugin path not found"), - ); - expect(hasIssue).toBe(true); - } + const home = await createCaseHome(); + const missingPath = path.join(home, "missing-plugin"); + const res = validateInHome(home, { + agents: { list: [{ id: "pi" }] }, + plugins: { enabled: false, load: { paths: [missingPath] } }, }); + expect(res.ok).toBe(false); + if (!res.ok) { + const hasIssue = res.issues.some( + (issue) => + issue.path === "plugins.load.paths" && issue.message.includes("plugin path not found"), + ); + expect(hasIssue).toBe(true); + } }); it("rejects missing plugin ids in entries", async () => { - await withTempHome(async (home) => { - const res = validateInHome(home, { - agents: { list: [{ id: "pi" }] }, - plugins: { enabled: false, entries: { "missing-plugin": { enabled: true } } }, - }); - expect(res.ok).toBe(false); - if (!res.ok) { - expect(res.issues).toContainEqual({ - path: "plugins.entries.missing-plugin", - message: "plugin not found: missing-plugin", - }); - } + const home = await createCaseHome(); + const res = validateInHome(home, { + agents: { list: [{ id: "pi" }] }, + plugins: { enabled: false, entries: { "missing-plugin": { enabled: true } } }, }); + expect(res.ok).toBe(false); + if (!res.ok) { + expect(res.issues).toContainEqual({ + path: "plugins.entries.missing-plugin", + message: "plugin not found: missing-plugin", + }); + } }); it("rejects missing plugin ids in allow/deny/slots", async () => { - await withTempHome(async (home) => { - const res = validateInHome(home, { - agents: { list: [{ id: "pi" }] }, - plugins: { - enabled: false, - allow: ["missing-allow"], - deny: ["missing-deny"], - slots: { memory: "missing-slot" }, - }, - }); - expect(res.ok).toBe(false); - if (!res.ok) { - expect(res.issues).toEqual( - expect.arrayContaining([ - { path: "plugins.allow", message: "plugin not found: missing-allow" }, - { path: "plugins.deny", message: "plugin not found: missing-deny" }, - { path: "plugins.slots.memory", message: "plugin not found: missing-slot" }, - ]), - ); - } + const home = await createCaseHome(); + const res = validateInHome(home, { + agents: { list: [{ id: "pi" }] }, + plugins: { + enabled: false, + allow: ["missing-allow"], + deny: ["missing-deny"], + slots: { memory: "missing-slot" }, + }, }); + expect(res.ok).toBe(false); + if (!res.ok) { + expect(res.issues).toEqual( + expect.arrayContaining([ + { path: "plugins.allow", message: "plugin not found: missing-allow" }, + { path: "plugins.deny", message: "plugin not found: missing-deny" }, + { path: "plugins.slots.memory", message: "plugin not found: missing-slot" }, + ]), + ); + } }); it("surfaces plugin config diagnostics", async () => { - await withTempHome(async (home) => { - const pluginDir = path.join(home, "bad-plugin"); - await writePluginFixture({ - dir: pluginDir, - id: "bad-plugin", - schema: { - type: "object", - additionalProperties: false, - properties: { - value: { type: "boolean" }, - }, - required: ["value"], + const home = await createCaseHome(); + const pluginDir = path.join(home, "bad-plugin"); + await writePluginFixture({ + dir: pluginDir, + id: "bad-plugin", + schema: { + type: "object", + additionalProperties: false, + properties: { + value: { type: "boolean" }, }, - }); - - const res = validateInHome(home, { - agents: { list: [{ id: "pi" }] }, - plugins: { - enabled: true, - load: { paths: [pluginDir] }, - entries: { "bad-plugin": { config: { value: "nope" } } }, - }, - }); - expect(res.ok).toBe(false); - if (!res.ok) { - const hasIssue = res.issues.some( - (issue) => - issue.path === "plugins.entries.bad-plugin.config" && - issue.message.includes("invalid config"), - ); - expect(hasIssue).toBe(true); - } + required: ["value"], + }, }); + + const res = validateInHome(home, { + agents: { list: [{ id: "pi" }] }, + plugins: { + enabled: true, + load: { paths: [pluginDir] }, + entries: { "bad-plugin": { config: { value: "nope" } } }, + }, + }); + expect(res.ok).toBe(false); + if (!res.ok) { + const hasIssue = res.issues.some( + (issue) => + issue.path === "plugins.entries.bad-plugin.config" && + issue.message.includes("invalid config"), + ); + expect(hasIssue).toBe(true); + } }); it("accepts known plugin ids", async () => { - await withTempHome(async (home) => { - const res = validateInHome(home, { - agents: { list: [{ id: "pi" }] }, - plugins: { enabled: false, entries: { discord: { enabled: true } } }, - }); - expect(res.ok).toBe(true); + const home = await createCaseHome(); + const res = validateInHome(home, { + agents: { list: [{ id: "pi" }] }, + plugins: { enabled: false, entries: { discord: { enabled: true } } }, }); + expect(res.ok).toBe(true); }); it("accepts plugin heartbeat targets", async () => { - await withTempHome(async (home) => { - const pluginDir = path.join(home, "bluebubbles-plugin"); - await writePluginFixture({ - dir: pluginDir, - id: "bluebubbles-plugin", - channels: ["bluebubbles"], - schema: { type: "object" }, - }); - - const res = validateInHome(home, { - agents: { defaults: { heartbeat: { target: "bluebubbles" } }, list: [{ id: "pi" }] }, - plugins: { enabled: false, load: { paths: [pluginDir] } }, - }); - expect(res.ok).toBe(true); + const home = await createCaseHome(); + const pluginDir = path.join(home, "bluebubbles-plugin"); + await writePluginFixture({ + dir: pluginDir, + id: "bluebubbles-plugin", + channels: ["bluebubbles"], + schema: { type: "object" }, }); + + const res = validateInHome(home, { + agents: { defaults: { heartbeat: { target: "bluebubbles" } }, list: [{ id: "pi" }] }, + plugins: { enabled: false, load: { paths: [pluginDir] } }, + }); + expect(res.ok).toBe(true); }); it("rejects unknown heartbeat targets", async () => { - await withTempHome(async (home) => { - const res = validateInHome(home, { - agents: { defaults: { heartbeat: { target: "not-a-channel" } }, list: [{ id: "pi" }] }, - }); - expect(res.ok).toBe(false); - if (!res.ok) { - expect(res.issues).toContainEqual({ - path: "agents.defaults.heartbeat.target", - message: "unknown heartbeat target: not-a-channel", - }); - } + const home = await createCaseHome(); + const res = validateInHome(home, { + agents: { defaults: { heartbeat: { target: "not-a-channel" } }, list: [{ id: "pi" }] }, }); + expect(res.ok).toBe(false); + if (!res.ok) { + expect(res.issues).toContainEqual({ + path: "agents.defaults.heartbeat.target", + message: "unknown heartbeat target: not-a-channel", + }); + } }); }); diff --git a/src/hooks/install.test.ts b/src/hooks/install.test.ts index 27a5616be27..0bbfc5bb6c8 100644 --- a/src/hooks/install.test.ts +++ b/src/hooks/install.test.ts @@ -4,28 +4,29 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import * as tar from "tar"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterAll, describe, expect, it, vi } from "vitest"; -const tempDirs: string[] = []; +const fixtureRoot = path.join(os.tmpdir(), `openclaw-hook-install-${randomUUID()}`); +let tempDirIndex = 0; vi.mock("../process/exec.js", () => ({ runCommandWithTimeout: vi.fn(), })); function makeTempDir() { - const dir = path.join(os.tmpdir(), `openclaw-hook-install-${randomUUID()}`); + const dir = path.join(fixtureRoot, `case-${tempDirIndex++}`); fs.mkdirSync(dir, { recursive: true }); - tempDirs.push(dir); return dir; } -afterEach(() => { - for (const dir of tempDirs.splice(0)) { - try { - fs.rmSync(dir, { recursive: true, force: true }); - } catch { - // ignore cleanup failures - } +const { runCommandWithTimeout } = await import("../process/exec.js"); +const { installHooksFromArchive, installHooksFromPath } = await import("./install.js"); + +afterAll(() => { + try { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } catch { + // ignore cleanup failures } }); @@ -61,7 +62,6 @@ describe("installHooksFromArchive", () => { fs.writeFileSync(archivePath, buffer); const hooksDir = path.join(stateDir, "hooks"); - const { installHooksFromArchive } = await import("./install.js"); const result = await installHooksFromArchive({ archivePath, hooksDir }); expect(result.ok).toBe(true); @@ -111,7 +111,6 @@ describe("installHooksFromArchive", () => { await tar.c({ cwd: workDir, file: archivePath }, ["package"]); const hooksDir = path.join(stateDir, "hooks"); - const { installHooksFromArchive } = await import("./install.js"); const result = await installHooksFromArchive({ archivePath, hooksDir }); expect(result.ok).toBe(true); @@ -160,7 +159,6 @@ describe("installHooksFromArchive", () => { await tar.c({ cwd: workDir, file: archivePath }, ["package"]); const hooksDir = path.join(stateDir, "hooks"); - const { installHooksFromArchive } = await import("./install.js"); const result = await installHooksFromArchive({ archivePath, hooksDir }); expect(result.ok).toBe(false); @@ -207,7 +205,6 @@ describe("installHooksFromArchive", () => { await tar.c({ cwd: workDir, file: archivePath }, ["package"]); const hooksDir = path.join(stateDir, "hooks"); - const { installHooksFromArchive } = await import("./install.js"); const result = await installHooksFromArchive({ archivePath, hooksDir }); expect(result.ok).toBe(false); @@ -253,11 +250,9 @@ describe("installHooksFromPath", () => { "utf-8", ); - const { runCommandWithTimeout } = await import("../process/exec.js"); const run = vi.mocked(runCommandWithTimeout); run.mockResolvedValue({ code: 0, stdout: "", stderr: "" }); - const { installHooksFromPath } = await import("./install.js"); const res = await installHooksFromPath({ path: pkgDir, hooksDir: path.join(stateDir, "hooks"), @@ -301,7 +296,6 @@ describe("installHooksFromPath", () => { fs.writeFileSync(path.join(hookDir, "handler.ts"), "export default async () => {};\n"); const hooksDir = path.join(stateDir, "hooks"); - const { installHooksFromPath } = await import("./install.js"); const result = await installHooksFromPath({ path: hookDir, hooksDir }); expect(result.ok).toBe(true); diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index cd27cc69ef2..f32d04d0d80 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -2,19 +2,19 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, describe, expect, it } from "vitest"; import { loadOpenClawPlugins } from "./loader.js"; type TempPlugin = { dir: string; file: string; id: string }; -const tempDirs: string[] = []; +const fixtureRoot = path.join(os.tmpdir(), `openclaw-plugin-${randomUUID()}`); +let tempDirIndex = 0; const prevBundledDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; const EMPTY_PLUGIN_SCHEMA = { type: "object", additionalProperties: false, properties: {} }; function makeTempDir() { - const dir = path.join(os.tmpdir(), `openclaw-plugin-${randomUUID()}`); + const dir = path.join(fixtureRoot, `case-${tempDirIndex++}`); fs.mkdirSync(dir, { recursive: true }); - tempDirs.push(dir); return dir; } @@ -44,13 +44,6 @@ function writePlugin(params: { } afterEach(() => { - for (const dir of tempDirs.splice(0)) { - try { - fs.rmSync(dir, { recursive: true, force: true }); - } catch { - // ignore cleanup failures - } - } if (prevBundledDir === undefined) { delete process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; } else { @@ -58,6 +51,14 @@ afterEach(() => { } }); +afterAll(() => { + try { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } catch { + // ignore cleanup failures + } +}); + describe("loadOpenClawPlugins", () => { it("disables bundled plugins by default", () => { const bundledDir = makeTempDir(); diff --git a/src/process/child-process-bridge.test.ts b/src/process/child-process-bridge.test.ts index 0a37ac7504a..855b37ac2ea 100644 --- a/src/process/child-process-bridge.test.ts +++ b/src/process/child-process-bridge.test.ts @@ -51,6 +51,17 @@ function canConnect(port: number): Promise { }); } +async function waitForPortClosed(port: number, timeoutMs = 1_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + if (!(await canConnect(port))) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error("timeout waiting for port to close"); +} + describe("attachChildProcessBridge", () => { const children: Array<{ kill: (signal?: NodeJS.Signals) => boolean }> = []; const detachments: Array<() => void> = []; @@ -111,7 +122,7 @@ describe("attachChildProcessBridge", () => { }); }); - await new Promise((r) => setTimeout(r, 250)); + await waitForPortClosed(port); expect(await canConnect(port)).toBe(false); }, 20_000); }); diff --git a/src/web/media.test.ts b/src/web/media.test.ts index d1f6d4e40c9..0dee4ac0c17 100644 --- a/src/web/media.test.ts +++ b/src/web/media.test.ts @@ -2,19 +2,16 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import sharp from "sharp"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import * as ssrf from "../infra/net/ssrf.js"; import { optimizeImageToPng } from "../media/image-ops.js"; import { loadWebMedia, loadWebMediaRaw, optimizeImageToJpeg } from "./media.js"; -const tmpFiles: string[] = []; +let fixtureRoot = ""; +let fixtureFileCount = 0; async function writeTempFile(buffer: Buffer, ext: string): Promise { - const file = path.join( - os.tmpdir(), - `openclaw-media-${Date.now()}-${Math.random().toString(16).slice(2)}${ext}`, - ); - tmpFiles.push(file); + const file = path.join(fixtureRoot, `media-${fixtureFileCount++}${ext}`); await fs.writeFile(file, buffer); return file; } @@ -45,9 +42,15 @@ async function createLargeTestJpeg(): Promise<{ buffer: Buffer; file: string }> return { buffer, file }; } -afterEach(async () => { - await Promise.all(tmpFiles.map((file) => fs.rm(file, { force: true }))); - tmpFiles.length = 0; +beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-media-test-")); +}); + +afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); +}); + +afterEach(() => { vi.restoreAllMocks(); }); From 207e2c5affa9a747873f822850d0be308eb71c01 Mon Sep 17 00:00:00 2001 From: nabbilkhan Date: Fri, 13 Feb 2026 15:54:07 -0600 Subject: [PATCH 0005/1038] fix: add outbound delivery crash recovery (#15636) (thanks @nabbilkhan) (#15636) Co-authored-by: Shadow --- CHANGELOG.md | 1 + src/gateway/server.impl.ts | 12 + src/infra/outbound/deliver.test.ts | 67 ++++ src/infra/outbound/deliver.ts | 85 +++++ src/infra/outbound/delivery-queue.test.ts | 373 ++++++++++++++++++++++ src/infra/outbound/delivery-queue.ts | 328 +++++++++++++++++++ 6 files changed, 866 insertions(+) create mode 100644 src/infra/outbound/delivery-queue.test.ts create mode 100644 src/infra/outbound/delivery-queue.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c7252c469cf..b87bc0064ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ Docs: https://docs.openclaw.ai - macOS Voice Wake: fix a crash in trigger trimming for CJK/Unicode transcripts by matching and slicing on original-string ranges instead of transformed-string indices. (#11052) Thanks @Flash-LHR. - Heartbeat: prevent scheduler silent-death races during runner reloads, preserve retry cooldown backoff under wake bursts, and prioritize user/action wake causes over interval/retry reasons when coalescing. (#15108) Thanks @joeykrug. - Outbound targets: fail closed for WhatsApp/Twitch/Google Chat fallback paths so invalid or missing targets are dropped instead of rerouted, and align resolver hints with strict target requirements. (#13578) Thanks @mcaxtr. +- Outbound: add a write-ahead delivery queue with crash-recovery retries to prevent lost outbound messages after gateway restarts. (#15636) Thanks @nabbilkhan, @thewilloftheshadow. - Exec/Allowlist: allow multiline heredoc bodies (`<<`, `<<-`) while keeping multiline non-heredoc shell commands blocked, so exec approval parsing permits heredoc input safely without allowing general newline command chaining. (#13811) Thanks @mcaxtr. - Docs/Mermaid: remove hardcoded Mermaid init theme blocks from four docs diagrams so dark mode inherits readable theme defaults. (#15157) Thanks @heytulsiprasad. - Outbound/Threading: pass `replyTo` and `threadId` from `message send` tool actions through the core outbound send path to channel adapters, preserving thread/reply routing. (#14948) Thanks @mcaxtr. diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 5b422a2bee4..3146c0c6deb 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -470,6 +470,18 @@ export async function startGatewayServer( void cron.start().catch((err) => logCron.error(`failed to start: ${String(err)}`)); + // Recover pending outbound deliveries from previous crash/restart. + void (async () => { + const { recoverPendingDeliveries } = await import("../infra/outbound/delivery-queue.js"); + const { deliverOutboundPayloads } = await import("../infra/outbound/deliver.js"); + const logRecovery = log.child("delivery-recovery"); + await recoverPendingDeliveries({ + deliver: deliverOutboundPayloads, + log: logRecovery, + cfg: cfgAtStart, + }); + })().catch((err) => log.error(`Delivery recovery failed: ${String(err)}`)); + const execApprovalManager = new ExecApprovalManager(); const execApprovalForwarder = createExecApprovalForwarder(); const execApprovalHandlers = createExecApprovalHandlers(execApprovalManager, { diff --git a/src/infra/outbound/deliver.test.ts b/src/infra/outbound/deliver.test.ts index 221050cc49d..3247149bec4 100644 --- a/src/infra/outbound/deliver.test.ts +++ b/src/infra/outbound/deliver.test.ts @@ -20,6 +20,11 @@ const hookMocks = vi.hoisted(() => ({ runMessageSent: vi.fn(async () => {}), }, })); +const queueMocks = vi.hoisted(() => ({ + enqueueDelivery: vi.fn(async () => "mock-queue-id"), + ackDelivery: vi.fn(async () => {}), + failDelivery: vi.fn(async () => {}), +})); vi.mock("../../config/sessions.js", async () => { const actual = await vi.importActual( @@ -33,6 +38,11 @@ vi.mock("../../config/sessions.js", async () => { vi.mock("../../plugins/hook-runner-global.js", () => ({ getGlobalHookRunner: () => hookMocks.runner, })); +vi.mock("./delivery-queue.js", () => ({ + enqueueDelivery: queueMocks.enqueueDelivery, + ackDelivery: queueMocks.ackDelivery, + failDelivery: queueMocks.failDelivery, +})); const { deliverOutboundPayloads, normalizeOutboundPayloads } = await import("./deliver.js"); @@ -43,6 +53,12 @@ describe("deliverOutboundPayloads", () => { hookMocks.runner.hasHooks.mockReturnValue(false); hookMocks.runner.runMessageSent.mockReset(); hookMocks.runner.runMessageSent.mockResolvedValue(undefined); + queueMocks.enqueueDelivery.mockReset(); + queueMocks.enqueueDelivery.mockResolvedValue("mock-queue-id"); + queueMocks.ackDelivery.mockReset(); + queueMocks.ackDelivery.mockResolvedValue(undefined); + queueMocks.failDelivery.mockReset(); + queueMocks.failDelivery.mockResolvedValue(undefined); }); afterEach(() => { @@ -389,6 +405,57 @@ describe("deliverOutboundPayloads", () => { expect(results).toEqual([{ channel: "whatsapp", messageId: "w2", toJid: "jid" }]); }); + it("calls failDelivery instead of ackDelivery on bestEffort partial failure", async () => { + const sendWhatsApp = vi + .fn() + .mockRejectedValueOnce(new Error("fail")) + .mockResolvedValueOnce({ messageId: "w2", toJid: "jid" }); + const onError = vi.fn(); + const cfg: OpenClawConfig = {}; + + await deliverOutboundPayloads({ + cfg, + channel: "whatsapp", + to: "+1555", + payloads: [{ text: "a" }, { text: "b" }], + deps: { sendWhatsApp }, + bestEffort: true, + onError, + }); + + // onError was called for the first payload's failure. + expect(onError).toHaveBeenCalledTimes(1); + + // Queue entry should NOT be acked — failDelivery should be called instead. + expect(queueMocks.ackDelivery).not.toHaveBeenCalled(); + expect(queueMocks.failDelivery).toHaveBeenCalledWith( + "mock-queue-id", + "partial delivery failure (bestEffort)", + ); + }); + + it("acks the queue entry when delivery is aborted", async () => { + const sendWhatsApp = vi.fn().mockResolvedValue({ messageId: "w1", toJid: "jid" }); + const abortController = new AbortController(); + abortController.abort(); + const cfg: OpenClawConfig = {}; + + await expect( + deliverOutboundPayloads({ + cfg, + channel: "whatsapp", + to: "+1555", + payloads: [{ text: "a" }], + deps: { sendWhatsApp }, + abortSignal: abortController.signal, + }), + ).rejects.toThrow("Operation aborted"); + + expect(queueMocks.ackDelivery).toHaveBeenCalledWith("mock-queue-id"); + expect(queueMocks.failDelivery).not.toHaveBeenCalled(); + expect(sendWhatsApp).not.toHaveBeenCalled(); + }); + it("passes normalized payload to onError", async () => { const sendWhatsApp = vi.fn().mockRejectedValue(new Error("boom")); const onError = vi.fn(); diff --git a/src/infra/outbound/deliver.ts b/src/infra/outbound/deliver.ts index 6460efc01a0..acbd4936907 100644 --- a/src/infra/outbound/deliver.ts +++ b/src/infra/outbound/deliver.ts @@ -25,6 +25,7 @@ import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; import { markdownToSignalTextChunks, type SignalTextStyleRange } from "../../signal/format.js"; import { sendMessageSignal } from "../../signal/send.js"; import { throwIfAborted } from "./abort.js"; +import { ackDelivery, enqueueDelivery, failDelivery } from "./delivery-queue.js"; import { normalizeReplyPayloadsForDelivery } from "./payloads.js"; export type { NormalizedOutboundPayload } from "./payloads.js"; @@ -178,6 +179,8 @@ function createPluginHandler(params: { }; } +const isAbortError = (err: unknown): boolean => err instanceof Error && err.name === "AbortError"; + export async function deliverOutboundPayloads(params: { cfg: OpenClawConfig; channel: Exclude; @@ -199,6 +202,88 @@ export async function deliverOutboundPayloads(params: { mediaUrls?: string[]; }; silent?: boolean; + /** @internal Skip write-ahead queue (used by crash-recovery to avoid re-enqueueing). */ + skipQueue?: boolean; +}): Promise { + const { channel, to, payloads } = params; + + // Write-ahead delivery queue: persist before sending, remove after success. + const queueId = params.skipQueue + ? null + : await enqueueDelivery({ + channel, + to, + accountId: params.accountId, + payloads, + threadId: params.threadId, + replyToId: params.replyToId, + bestEffort: params.bestEffort, + gifPlayback: params.gifPlayback, + silent: params.silent, + mirror: params.mirror, + }).catch(() => null); // Best-effort — don't block delivery if queue write fails. + + // Wrap onError to detect partial failures under bestEffort mode. + // When bestEffort is true, per-payload errors are caught and passed to onError + // without throwing — so the outer try/catch never fires. We track whether any + // payload failed so we can call failDelivery instead of ackDelivery. + let hadPartialFailure = false; + const wrappedParams = params.onError + ? { + ...params, + onError: (err: unknown, payload: NormalizedOutboundPayload) => { + hadPartialFailure = true; + params.onError!(err, payload); + }, + } + : params; + + try { + const results = await deliverOutboundPayloadsCore(wrappedParams); + if (queueId) { + if (hadPartialFailure) { + await failDelivery(queueId, "partial delivery failure (bestEffort)").catch(() => {}); + } else { + await ackDelivery(queueId).catch(() => {}); // Best-effort cleanup. + } + } + return results; + } catch (err) { + if (queueId) { + if (isAbortError(err)) { + await ackDelivery(queueId).catch(() => {}); + } else { + await failDelivery(queueId, err instanceof Error ? err.message : String(err)).catch( + () => {}, + ); + } + } + throw err; + } +} + +/** Core delivery logic (extracted for queue wrapper). */ +async function deliverOutboundPayloadsCore(params: { + cfg: OpenClawConfig; + channel: Exclude; + to: string; + accountId?: string; + payloads: ReplyPayload[]; + replyToId?: string | null; + threadId?: string | number | null; + deps?: OutboundSendDeps; + gifPlayback?: boolean; + abortSignal?: AbortSignal; + bestEffort?: boolean; + onError?: (err: unknown, payload: NormalizedOutboundPayload) => void; + onPayload?: (payload: NormalizedOutboundPayload) => void; + mirror?: { + sessionKey: string; + agentId?: string; + text?: string; + mediaUrls?: string[]; + }; + silent?: boolean; }): Promise { const { cfg, channel, to, payloads } = params; const accountId = params.accountId; diff --git a/src/infra/outbound/delivery-queue.test.ts b/src/infra/outbound/delivery-queue.test.ts new file mode 100644 index 00000000000..ee94d13b62b --- /dev/null +++ b/src/infra/outbound/delivery-queue.test.ts @@ -0,0 +1,373 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + ackDelivery, + computeBackoffMs, + enqueueDelivery, + failDelivery, + loadPendingDeliveries, + MAX_RETRIES, + moveToFailed, + recoverPendingDeliveries, +} from "./delivery-queue.js"; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-dq-test-")); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe("enqueue + ack lifecycle", () => { + it("creates and removes a queue entry", async () => { + const id = await enqueueDelivery( + { + channel: "whatsapp", + to: "+1555", + payloads: [{ text: "hello" }], + bestEffort: true, + gifPlayback: true, + silent: true, + mirror: { + sessionKey: "agent:main:main", + text: "hello", + mediaUrls: ["https://example.com/file.png"], + }, + }, + tmpDir, + ); + + // Entry file exists after enqueue. + const queueDir = path.join(tmpDir, "delivery-queue"); + const files = fs.readdirSync(queueDir).filter((f) => f.endsWith(".json")); + expect(files).toHaveLength(1); + expect(files[0]).toBe(`${id}.json`); + + // Entry contents are correct. + const entry = JSON.parse(fs.readFileSync(path.join(queueDir, files[0]), "utf-8")); + expect(entry).toMatchObject({ + id, + channel: "whatsapp", + to: "+1555", + bestEffort: true, + gifPlayback: true, + silent: true, + mirror: { + sessionKey: "agent:main:main", + text: "hello", + mediaUrls: ["https://example.com/file.png"], + }, + retryCount: 0, + }); + expect(entry.payloads).toEqual([{ text: "hello" }]); + + // Ack removes the file. + await ackDelivery(id, tmpDir); + const remaining = fs.readdirSync(queueDir).filter((f) => f.endsWith(".json")); + expect(remaining).toHaveLength(0); + }); + + it("ack is idempotent (no error on missing file)", async () => { + await expect(ackDelivery("nonexistent-id", tmpDir)).resolves.toBeUndefined(); + }); +}); + +describe("failDelivery", () => { + it("increments retryCount and sets lastError", async () => { + const id = await enqueueDelivery( + { + channel: "telegram", + to: "123", + payloads: [{ text: "test" }], + }, + tmpDir, + ); + + await failDelivery(id, "connection refused", tmpDir); + + const queueDir = path.join(tmpDir, "delivery-queue"); + const entry = JSON.parse(fs.readFileSync(path.join(queueDir, `${id}.json`), "utf-8")); + expect(entry.retryCount).toBe(1); + expect(entry.lastError).toBe("connection refused"); + }); +}); + +describe("moveToFailed", () => { + it("moves entry to failed/ subdirectory", async () => { + const id = await enqueueDelivery( + { + channel: "slack", + to: "#general", + payloads: [{ text: "hi" }], + }, + tmpDir, + ); + + await moveToFailed(id, tmpDir); + + const queueDir = path.join(tmpDir, "delivery-queue"); + const failedDir = path.join(queueDir, "failed"); + expect(fs.existsSync(path.join(queueDir, `${id}.json`))).toBe(false); + expect(fs.existsSync(path.join(failedDir, `${id}.json`))).toBe(true); + }); +}); + +describe("loadPendingDeliveries", () => { + it("returns empty array when queue directory does not exist", async () => { + const nonexistent = path.join(tmpDir, "no-such-dir"); + const entries = await loadPendingDeliveries(nonexistent); + expect(entries).toEqual([]); + }); + + it("loads multiple entries", async () => { + await enqueueDelivery({ channel: "whatsapp", to: "+1", payloads: [{ text: "a" }] }, tmpDir); + await enqueueDelivery({ channel: "telegram", to: "2", payloads: [{ text: "b" }] }, tmpDir); + + const entries = await loadPendingDeliveries(tmpDir); + expect(entries).toHaveLength(2); + }); +}); + +describe("computeBackoffMs", () => { + it("returns 0 for retryCount 0", () => { + expect(computeBackoffMs(0)).toBe(0); + }); + + it("returns correct backoff for each retry", () => { + expect(computeBackoffMs(1)).toBe(5_000); + expect(computeBackoffMs(2)).toBe(25_000); + expect(computeBackoffMs(3)).toBe(120_000); + expect(computeBackoffMs(4)).toBe(600_000); + // Beyond defined schedule — clamps to last value. + expect(computeBackoffMs(5)).toBe(600_000); + }); +}); + +describe("recoverPendingDeliveries", () => { + const noopDelay = async () => {}; + const baseCfg = {}; + + it("recovers entries from a simulated crash", async () => { + // Manually create two queue entries as if gateway crashed before delivery. + await enqueueDelivery({ channel: "whatsapp", to: "+1", payloads: [{ text: "a" }] }, tmpDir); + await enqueueDelivery({ channel: "telegram", to: "2", payloads: [{ text: "b" }] }, tmpDir); + + const deliver = vi.fn().mockResolvedValue([]); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay: noopDelay, + }); + + expect(deliver).toHaveBeenCalledTimes(2); + expect(result.recovered).toBe(2); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + + // Queue should be empty after recovery. + const remaining = await loadPendingDeliveries(tmpDir); + expect(remaining).toHaveLength(0); + }); + + it("moves entries that exceeded max retries to failed/", async () => { + // Create an entry and manually set retryCount to MAX_RETRIES. + const id = await enqueueDelivery( + { channel: "whatsapp", to: "+1", payloads: [{ text: "a" }] }, + tmpDir, + ); + const filePath = path.join(tmpDir, "delivery-queue", `${id}.json`); + const entry = JSON.parse(fs.readFileSync(filePath, "utf-8")); + entry.retryCount = MAX_RETRIES; + fs.writeFileSync(filePath, JSON.stringify(entry), "utf-8"); + + const deliver = vi.fn(); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay: noopDelay, + }); + + expect(deliver).not.toHaveBeenCalled(); + expect(result.skipped).toBe(1); + + // Entry should be in failed/ directory. + const failedDir = path.join(tmpDir, "delivery-queue", "failed"); + expect(fs.existsSync(path.join(failedDir, `${id}.json`))).toBe(true); + }); + + it("increments retryCount on failed recovery attempt", async () => { + await enqueueDelivery({ channel: "slack", to: "#ch", payloads: [{ text: "x" }] }, tmpDir); + + const deliver = vi.fn().mockRejectedValue(new Error("network down")); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay: noopDelay, + }); + + expect(result.failed).toBe(1); + expect(result.recovered).toBe(0); + + // Entry should still be in queue with incremented retryCount. + const entries = await loadPendingDeliveries(tmpDir); + expect(entries).toHaveLength(1); + expect(entries[0].retryCount).toBe(1); + expect(entries[0].lastError).toBe("network down"); + }); + + it("passes skipQueue: true to prevent re-enqueueing during recovery", async () => { + await enqueueDelivery({ channel: "whatsapp", to: "+1", payloads: [{ text: "a" }] }, tmpDir); + + const deliver = vi.fn().mockResolvedValue([]); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay: noopDelay, + }); + + expect(deliver).toHaveBeenCalledWith(expect.objectContaining({ skipQueue: true })); + }); + + it("replays stored delivery options during recovery", async () => { + await enqueueDelivery( + { + channel: "whatsapp", + to: "+1", + payloads: [{ text: "a" }], + bestEffort: true, + gifPlayback: true, + silent: true, + mirror: { + sessionKey: "agent:main:main", + text: "a", + mediaUrls: ["https://example.com/a.png"], + }, + }, + tmpDir, + ); + + const deliver = vi.fn().mockResolvedValue([]); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay: noopDelay, + }); + + expect(deliver).toHaveBeenCalledWith( + expect.objectContaining({ + bestEffort: true, + gifPlayback: true, + silent: true, + mirror: { + sessionKey: "agent:main:main", + text: "a", + mediaUrls: ["https://example.com/a.png"], + }, + }), + ); + }); + + it("respects maxRecoveryMs time budget", async () => { + await enqueueDelivery({ channel: "whatsapp", to: "+1", payloads: [{ text: "a" }] }, tmpDir); + await enqueueDelivery({ channel: "telegram", to: "2", payloads: [{ text: "b" }] }, tmpDir); + await enqueueDelivery({ channel: "slack", to: "#c", payloads: [{ text: "c" }] }, tmpDir); + + const deliver = vi.fn().mockResolvedValue([]); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay: noopDelay, + maxRecoveryMs: 0, // Immediate timeout — no entries should be processed. + }); + + expect(deliver).not.toHaveBeenCalled(); + expect(result.recovered).toBe(0); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + + // All entries should still be in the queue. + const remaining = await loadPendingDeliveries(tmpDir); + expect(remaining).toHaveLength(3); + + // Should have logged a warning about deferred entries. + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("deferred to next restart")); + }); + + it("defers entries when backoff exceeds the recovery budget", async () => { + const id = await enqueueDelivery( + { channel: "whatsapp", to: "+1", payloads: [{ text: "a" }] }, + tmpDir, + ); + const filePath = path.join(tmpDir, "delivery-queue", `${id}.json`); + const entry = JSON.parse(fs.readFileSync(filePath, "utf-8")); + entry.retryCount = 3; + fs.writeFileSync(filePath, JSON.stringify(entry), "utf-8"); + + const deliver = vi.fn().mockResolvedValue([]); + const delay = vi.fn(async () => {}); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay, + maxRecoveryMs: 1000, + }); + + expect(deliver).not.toHaveBeenCalled(); + expect(delay).not.toHaveBeenCalled(); + expect(result).toEqual({ recovered: 0, failed: 0, skipped: 0 }); + + const remaining = await loadPendingDeliveries(tmpDir); + expect(remaining).toHaveLength(1); + + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("deferred to next restart")); + }); + + it("returns zeros when queue is empty", async () => { + const deliver = vi.fn(); + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await recoverPendingDeliveries({ + deliver, + log, + cfg: baseCfg, + stateDir: tmpDir, + delay: noopDelay, + }); + + expect(result).toEqual({ recovered: 0, failed: 0, skipped: 0 }); + expect(deliver).not.toHaveBeenCalled(); + }); +}); diff --git a/src/infra/outbound/delivery-queue.ts b/src/infra/outbound/delivery-queue.ts new file mode 100644 index 00000000000..7303d827243 --- /dev/null +++ b/src/infra/outbound/delivery-queue.ts @@ -0,0 +1,328 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import type { ReplyPayload } from "../../auto-reply/types.js"; +import type { OpenClawConfig } from "../../config/config.js"; +import type { OutboundChannel } from "./targets.js"; +import { resolveStateDir } from "../../config/paths.js"; + +const QUEUE_DIRNAME = "delivery-queue"; +const FAILED_DIRNAME = "failed"; +const MAX_RETRIES = 5; + +/** Backoff delays in milliseconds indexed by retry count (1-based). */ +const BACKOFF_MS: readonly number[] = [ + 5_000, // retry 1: 5s + 25_000, // retry 2: 25s + 120_000, // retry 3: 2m + 600_000, // retry 4: 10m +]; + +export interface QueuedDelivery { + id: string; + enqueuedAt: number; + channel: Exclude; + to: string; + accountId?: string; + /** + * Original payloads before plugin hooks. On recovery, hooks re-run on these + * payloads — this is intentional since hooks are stateless transforms and + * should produce the same result on replay. + */ + payloads: ReplyPayload[]; + threadId?: string | number | null; + replyToId?: string | null; + bestEffort?: boolean; + gifPlayback?: boolean; + silent?: boolean; + mirror?: { + sessionKey: string; + agentId?: string; + text?: string; + mediaUrls?: string[]; + }; + retryCount: number; + lastError?: string; +} + +function resolveQueueDir(stateDir?: string): string { + const base = stateDir ?? resolveStateDir(); + return path.join(base, QUEUE_DIRNAME); +} + +function resolveFailedDir(stateDir?: string): string { + return path.join(resolveQueueDir(stateDir), FAILED_DIRNAME); +} + +/** Ensure the queue directory (and failed/ subdirectory) exist. */ +export async function ensureQueueDir(stateDir?: string): Promise { + const queueDir = resolveQueueDir(stateDir); + await fs.promises.mkdir(queueDir, { recursive: true, mode: 0o700 }); + await fs.promises.mkdir(resolveFailedDir(stateDir), { recursive: true, mode: 0o700 }); + return queueDir; +} + +/** Persist a delivery entry to disk before attempting send. Returns the entry ID. */ +export async function enqueueDelivery( + params: { + channel: Exclude; + to: string; + accountId?: string; + payloads: ReplyPayload[]; + threadId?: string | number | null; + replyToId?: string | null; + bestEffort?: boolean; + gifPlayback?: boolean; + silent?: boolean; + mirror?: { + sessionKey: string; + agentId?: string; + text?: string; + mediaUrls?: string[]; + }; + }, + stateDir?: string, +): Promise { + const queueDir = await ensureQueueDir(stateDir); + const id = crypto.randomUUID(); + const entry: QueuedDelivery = { + id, + enqueuedAt: Date.now(), + channel: params.channel, + to: params.to, + accountId: params.accountId, + payloads: params.payloads, + threadId: params.threadId, + replyToId: params.replyToId, + bestEffort: params.bestEffort, + gifPlayback: params.gifPlayback, + silent: params.silent, + mirror: params.mirror, + retryCount: 0, + }; + const filePath = path.join(queueDir, `${id}.json`); + const tmp = `${filePath}.${process.pid}.tmp`; + const json = JSON.stringify(entry, null, 2); + await fs.promises.writeFile(tmp, json, { encoding: "utf-8", mode: 0o600 }); + await fs.promises.rename(tmp, filePath); + return id; +} + +/** Remove a successfully delivered entry from the queue. */ +export async function ackDelivery(id: string, stateDir?: string): Promise { + const filePath = path.join(resolveQueueDir(stateDir), `${id}.json`); + try { + await fs.promises.unlink(filePath); + } catch (err) { + const code = + err && typeof err === "object" && "code" in err + ? String((err as { code?: unknown }).code) + : null; + if (code !== "ENOENT") { + throw err; + } + // Already removed — no-op. + } +} + +/** Update a queue entry after a failed delivery attempt. */ +export async function failDelivery(id: string, error: string, stateDir?: string): Promise { + const filePath = path.join(resolveQueueDir(stateDir), `${id}.json`); + const raw = await fs.promises.readFile(filePath, "utf-8"); + const entry: QueuedDelivery = JSON.parse(raw); + entry.retryCount += 1; + entry.lastError = error; + const tmp = `${filePath}.${process.pid}.tmp`; + await fs.promises.writeFile(tmp, JSON.stringify(entry, null, 2), { + encoding: "utf-8", + mode: 0o600, + }); + await fs.promises.rename(tmp, filePath); +} + +/** Load all pending delivery entries from the queue directory. */ +export async function loadPendingDeliveries(stateDir?: string): Promise { + const queueDir = resolveQueueDir(stateDir); + let files: string[]; + try { + files = await fs.promises.readdir(queueDir); + } catch (err) { + const code = + err && typeof err === "object" && "code" in err + ? String((err as { code?: unknown }).code) + : null; + if (code === "ENOENT") { + return []; + } + throw err; + } + const entries: QueuedDelivery[] = []; + for (const file of files) { + if (!file.endsWith(".json")) { + continue; + } + const filePath = path.join(queueDir, file); + try { + const stat = await fs.promises.stat(filePath); + if (!stat.isFile()) { + continue; + } + const raw = await fs.promises.readFile(filePath, "utf-8"); + entries.push(JSON.parse(raw)); + } catch { + // Skip malformed or inaccessible entries. + } + } + return entries; +} + +/** Move a queue entry to the failed/ subdirectory. */ +export async function moveToFailed(id: string, stateDir?: string): Promise { + const queueDir = resolveQueueDir(stateDir); + const failedDir = resolveFailedDir(stateDir); + await fs.promises.mkdir(failedDir, { recursive: true, mode: 0o700 }); + const src = path.join(queueDir, `${id}.json`); + const dest = path.join(failedDir, `${id}.json`); + await fs.promises.rename(src, dest); +} + +/** Compute the backoff delay in ms for a given retry count. */ +export function computeBackoffMs(retryCount: number): number { + if (retryCount <= 0) { + return 0; + } + return BACKOFF_MS[Math.min(retryCount - 1, BACKOFF_MS.length - 1)] ?? BACKOFF_MS.at(-1) ?? 0; +} + +export type DeliverFn = (params: { + cfg: OpenClawConfig; + channel: Exclude; + to: string; + accountId?: string; + payloads: ReplyPayload[]; + threadId?: string | number | null; + replyToId?: string | null; + bestEffort?: boolean; + gifPlayback?: boolean; + silent?: boolean; + mirror?: { + sessionKey: string; + agentId?: string; + text?: string; + mediaUrls?: string[]; + }; + skipQueue?: boolean; +}) => Promise; + +export interface RecoveryLogger { + info(msg: string): void; + warn(msg: string): void; + error(msg: string): void; +} + +/** + * On gateway startup, scan the delivery queue and retry any pending entries. + * Uses exponential backoff and moves entries that exceed MAX_RETRIES to failed/. + */ +export async function recoverPendingDeliveries(opts: { + deliver: DeliverFn; + log: RecoveryLogger; + cfg: OpenClawConfig; + stateDir?: string; + /** Override for testing — resolves instead of using real setTimeout. */ + delay?: (ms: number) => Promise; + /** Maximum wall-clock time for recovery in ms. Remaining entries are deferred to next restart. Default: 60 000. */ + maxRecoveryMs?: number; +}): Promise<{ recovered: number; failed: number; skipped: number }> { + const pending = await loadPendingDeliveries(opts.stateDir); + if (pending.length === 0) { + return { recovered: 0, failed: 0, skipped: 0 }; + } + + // Process oldest first. + pending.sort((a, b) => a.enqueuedAt - b.enqueuedAt); + + opts.log.info(`Found ${pending.length} pending delivery entries — starting recovery`); + + const delayFn = opts.delay ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const deadline = Date.now() + (opts.maxRecoveryMs ?? 60_000); + + let recovered = 0; + let failed = 0; + let skipped = 0; + + for (const entry of pending) { + const now = Date.now(); + if (now >= deadline) { + const deferred = pending.length - recovered - failed - skipped; + opts.log.warn(`Recovery time budget exceeded — ${deferred} entries deferred to next restart`); + break; + } + if (entry.retryCount >= MAX_RETRIES) { + opts.log.warn( + `Delivery ${entry.id} exceeded max retries (${entry.retryCount}/${MAX_RETRIES}) — moving to failed/`, + ); + try { + await moveToFailed(entry.id, opts.stateDir); + } catch (err) { + opts.log.error(`Failed to move entry ${entry.id} to failed/: ${String(err)}`); + } + skipped += 1; + continue; + } + + const backoff = computeBackoffMs(entry.retryCount + 1); + if (backoff > 0) { + if (now + backoff >= deadline) { + const deferred = pending.length - recovered - failed - skipped; + opts.log.warn( + `Recovery time budget exceeded — ${deferred} entries deferred to next restart`, + ); + break; + } + opts.log.info(`Waiting ${backoff}ms before retrying delivery ${entry.id}`); + await delayFn(backoff); + } + + try { + await opts.deliver({ + cfg: opts.cfg, + channel: entry.channel, + to: entry.to, + accountId: entry.accountId, + payloads: entry.payloads, + threadId: entry.threadId, + replyToId: entry.replyToId, + bestEffort: entry.bestEffort, + gifPlayback: entry.gifPlayback, + silent: entry.silent, + mirror: entry.mirror, + skipQueue: true, // Prevent re-enqueueing during recovery + }); + await ackDelivery(entry.id, opts.stateDir); + recovered += 1; + opts.log.info(`Recovered delivery ${entry.id} to ${entry.channel}:${entry.to}`); + } catch (err) { + try { + await failDelivery( + entry.id, + err instanceof Error ? err.message : String(err), + opts.stateDir, + ); + } catch { + // Best-effort update. + } + failed += 1; + opts.log.warn( + `Retry failed for delivery ${entry.id}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + opts.log.info( + `Delivery recovery complete: ${recovered} recovered, ${failed} failed, ${skipped} skipped (max retries)`, + ); + return { recovered, failed, skipped }; +} + +export { MAX_RETRIES }; From ea95e88dd60b09f5a30f618f542ec8ca88baf3f0 Mon Sep 17 00:00:00 2001 From: Marcus Widing Date: Fri, 13 Feb 2026 21:14:32 +0100 Subject: [PATCH 0006/1038] fix(cron): prevent duplicate delivery for isolated jobs with announce mode When an isolated cron job delivers its output via deliverOutboundPayloads or the subagent announce flow, the finish handler in executeJobCore unconditionally posts a summary to the main agent session and wakes it via requestHeartbeatNow. The main agent then generates a second response that is also delivered to the target channel, resulting in duplicate messages with different content. Add a `delivered` flag to RunCronAgentTurnResult that is set to true when the isolated run successfully delivers its output. In executeJobCore, skip the enqueueSystemEvent + requestHeartbeatNow call when the flag is set, preventing the main agent from waking up and double-posting. Fixes #15692 --- src/cron/isolated-agent/run.ts | 15 +++++++++++++-- src/cron/service/state.ts | 5 +++++ src/cron/service/timer.ts | 9 +++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/cron/isolated-agent/run.ts b/src/cron/isolated-agent/run.ts index a329ef0e88e..952894f6b6e 100644 --- a/src/cron/isolated-agent/run.ts +++ b/src/cron/isolated-agent/run.ts @@ -101,6 +101,13 @@ export type RunCronAgentTurnResult = { error?: string; sessionId?: string; sessionKey?: string; + /** + * `true` when the isolated run already delivered its output to the target + * channel (via outbound payloads or the subagent announce flow). Callers + * should skip posting a summary to the main session to avoid duplicate + * messages. See: https://github.com/openclaw/openclaw/issues/15692 + */ + delivered?: boolean; }; export async function runCronIsolatedAgentTurn(params: { @@ -518,6 +525,7 @@ export async function runCronIsolatedAgentTurn(params: { }), ); + let delivered = false; if (deliveryRequested && !skipHeartbeatDelivery && !skipMessagingToolDelivery) { if (resolvedDelivery.error) { if (!deliveryBestEffort) { @@ -558,6 +566,7 @@ export async function runCronIsolatedAgentTurn(params: { bestEffort: deliveryBestEffort, deps: createOutboundSendDeps(params.deps), }); + delivered = true; } catch (err) { if (!deliveryBestEffort) { return withRunSession({ status: "error", summary, outputText, error: String(err) }); @@ -594,7 +603,9 @@ export async function runCronIsolatedAgentTurn(params: { outcome: { status: "ok" }, announceType: "cron job", }); - if (!didAnnounce) { + if (didAnnounce) { + delivered = true; + } else { const message = "cron announce delivery failed"; if (!deliveryBestEffort) { return withRunSession({ @@ -615,5 +626,5 @@ export async function runCronIsolatedAgentTurn(params: { } } - return withRunSession({ status: "ok", summary, outputText }); + return withRunSession({ status: "ok", summary, outputText, delivered }); } diff --git a/src/cron/service/state.ts b/src/cron/service/state.ts index 025da7b3fa4..0c7c3c70e3a 100644 --- a/src/cron/service/state.ts +++ b/src/cron/service/state.ts @@ -46,6 +46,11 @@ export type CronServiceDeps = { error?: string; sessionId?: string; sessionKey?: string; + /** + * `true` when the isolated run already delivered its output to the target + * channel. See: https://github.com/openclaw/openclaw/issues/15692 + */ + delivered?: boolean; }>; onEvent?: (evt: CronEvent) => void; }; diff --git a/src/cron/service/timer.ts b/src/cron/service/timer.ts index 0259dfc61db..913165dcbba 100644 --- a/src/cron/service/timer.ts +++ b/src/cron/service/timer.ts @@ -483,10 +483,15 @@ async function executeJobCore( message: job.payload.message, }); - // Post a short summary back to the main session. + // Post a short summary back to the main session — but only when the + // isolated run did NOT already deliver its output to the target channel. + // When `res.delivered` is true the announce flow (or direct outbound + // delivery) already sent the result, so posting the summary to main + // would wake the main agent and cause a duplicate message. + // See: https://github.com/openclaw/openclaw/issues/15692 const summaryText = res.summary?.trim(); const deliveryPlan = resolveCronDeliveryPlan(job); - if (summaryText && deliveryPlan.requested) { + if (summaryText && deliveryPlan.requested && !res.delivered) { const prefix = "Cron"; const label = res.status === "error" ? `${prefix} (error): ${summaryText}` : `${prefix}: ${summaryText}`; From 45a2cd55cc6af4e992fe4a1537222dd567d9ef83 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 23:48:36 +0100 Subject: [PATCH 0007/1038] fix: harden isolated cron announce delivery fallback (#15739) (thanks @widingmarcus-cyber) --- CHANGELOG.md | 1 + ...cipient-besteffortdeliver-true.e2e.test.ts | 46 +++++++++++++++++++ src/cron/isolated-agent/run.ts | 13 ++++-- ...runs-one-shot-main-job-disables-it.test.ts | 42 +++++++++++++++++ src/cron/service/state.ts | 3 +- 5 files changed, 99 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b87bc0064ca..63c574bfab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ Docs: https://docs.openclaw.ai - Cron: pass `agentId` to `runHeartbeatOnce` for main-session jobs. (#14140) Thanks @ishikawa-pro. - Cron: re-arm timers when `onTimer` fires while a job is still executing. (#14233) Thanks @tomron87. - Cron: prevent duplicate fires when multiple jobs trigger simultaneously. (#14256) Thanks @xinhuagu. +- Cron: prevent duplicate announce-mode isolated cron deliveries, and keep main-session fallback active when best-effort structured delivery attempts fail to send any message. (#15739) Thanks @widingmarcus-cyber. - Cron: isolate scheduler errors so one bad job does not break all jobs. (#14385) Thanks @MarvinDontPanic. - Cron: prevent one-shot `at` jobs from re-firing on restart after skipped/errored runs. (#13878) Thanks @lailoo. - Heartbeat: prevent scheduler stalls on unexpected run errors and avoid immediate rerun loops after `requests-in-flight` skips. (#14901) Thanks @joeykrug. diff --git a/src/cron/isolated-agent.skips-delivery-without-whatsapp-recipient-besteffortdeliver-true.e2e.test.ts b/src/cron/isolated-agent.skips-delivery-without-whatsapp-recipient-besteffortdeliver-true.e2e.test.ts index 4b0d04d1860..94bfd4f27bd 100644 --- a/src/cron/isolated-agent.skips-delivery-without-whatsapp-recipient-besteffortdeliver-true.e2e.test.ts +++ b/src/cron/isolated-agent.skips-delivery-without-whatsapp-recipient-besteffortdeliver-true.e2e.test.ts @@ -135,6 +135,7 @@ describe("runCronIsolatedAgentTurn", () => { }); expect(res.status).toBe("ok"); + expect(res.delivered).toBe(true); expect(runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); const announceArgs = vi.mocked(runSubagentAnnounceFlow).mock.calls[0]?.[0] as | { announceType?: string } @@ -280,11 +281,56 @@ describe("runCronIsolatedAgentTurn", () => { }); expect(res.status).toBe("ok"); + expect(res.delivered).toBe(true); expect(runSubagentAnnounceFlow).not.toHaveBeenCalled(); expect(deps.sendMessageTelegram).not.toHaveBeenCalled(); }); }); + it("reports not-delivered when best-effort structured outbound sends all fail", async () => { + await withTempHome(async (home) => { + const storePath = await writeSessionStore(home); + const deps: CliDeps = { + sendMessageWhatsApp: vi.fn(), + sendMessageTelegram: vi.fn().mockRejectedValue(new Error("boom")), + sendMessageDiscord: vi.fn(), + sendMessageSignal: vi.fn(), + sendMessageIMessage: vi.fn(), + }; + vi.mocked(runEmbeddedPiAgent).mockResolvedValue({ + payloads: [{ text: "caption", mediaUrl: "https://example.com/img.png" }], + meta: { + durationMs: 5, + agentMeta: { sessionId: "s", provider: "p", model: "m" }, + }, + }); + + const res = await runCronIsolatedAgentTurn({ + cfg: makeCfg(home, storePath, { + channels: { telegram: { botToken: "t-1" } }, + }), + deps, + job: { + ...makeJob({ kind: "agentTurn", message: "do it" }), + delivery: { + mode: "announce", + channel: "telegram", + to: "123", + bestEffort: true, + }, + }, + message: "do it", + sessionKey: "cron:job-1", + lane: "cron", + }); + + expect(res.status).toBe("ok"); + expect(res.delivered).toBe(false); + expect(runSubagentAnnounceFlow).not.toHaveBeenCalled(); + expect(deps.sendMessageTelegram).toHaveBeenCalledTimes(1); + }); + }); + it("skips announce for heartbeat-only output", async () => { await withTempHome(async (home) => { const storePath = await writeSessionStore(home); diff --git a/src/cron/isolated-agent/run.ts b/src/cron/isolated-agent/run.ts index 952894f6b6e..ed4434ef13e 100644 --- a/src/cron/isolated-agent/run.ts +++ b/src/cron/isolated-agent/run.ts @@ -103,8 +103,9 @@ export type RunCronAgentTurnResult = { sessionKey?: string; /** * `true` when the isolated run already delivered its output to the target - * channel (via outbound payloads or the subagent announce flow). Callers - * should skip posting a summary to the main session to avoid duplicate + * channel (via outbound payloads, the subagent announce flow, or a matching + * messaging-tool send). Callers should skip posting a summary to the main + * session to avoid duplicate * messages. See: https://github.com/openclaw/openclaw/issues/15692 */ delivered?: boolean; @@ -525,7 +526,9 @@ export async function runCronIsolatedAgentTurn(params: { }), ); - let delivered = false; + // `true` means we confirmed at least one outbound send reached the target. + // Keep this strict so timer fallback can safely decide whether to wake main. + let delivered = skipMessagingToolDelivery; if (deliveryRequested && !skipHeartbeatDelivery && !skipMessagingToolDelivery) { if (resolvedDelivery.error) { if (!deliveryBestEffort) { @@ -556,7 +559,7 @@ export async function runCronIsolatedAgentTurn(params: { // for media/channel payloads so structured content is preserved. if (deliveryPayloadHasStructuredContent) { try { - await deliverOutboundPayloads({ + const deliveryResults = await deliverOutboundPayloads({ cfg: cfgWithAgentDefaults, channel: resolvedDelivery.channel, to: resolvedDelivery.to, @@ -566,7 +569,7 @@ export async function runCronIsolatedAgentTurn(params: { bestEffort: deliveryBestEffort, deps: createOutboundSendDeps(params.deps), }); - delivered = true; + delivered = deliveryResults.length > 0; } catch (err) { if (!deliveryBestEffort) { return withRunSession({ status: "error", summary, outputText, error: String(err) }); diff --git a/src/cron/service.runs-one-shot-main-job-disables-it.test.ts b/src/cron/service.runs-one-shot-main-job-disables-it.test.ts index bbee9cf7e8a..1a7c7338166 100644 --- a/src/cron/service.runs-one-shot-main-job-disables-it.test.ts +++ b/src/cron/service.runs-one-shot-main-job-disables-it.test.ts @@ -329,6 +329,48 @@ describe("CronService", () => { await store.cleanup(); }); + it("does not post isolated summary to main when run already delivered output", async () => { + const store = await makeStorePath(); + const enqueueSystemEvent = vi.fn(); + const requestHeartbeatNow = vi.fn(); + const runIsolatedAgentJob = vi.fn(async () => ({ + status: "ok" as const, + summary: "done", + delivered: true, + })); + + const cron = new CronService({ + storePath: store.storePath, + cronEnabled: true, + log: noopLogger, + enqueueSystemEvent, + requestHeartbeatNow, + runIsolatedAgentJob, + }); + + await cron.start(); + const atMs = Date.parse("2025-12-13T00:00:01.000Z"); + await cron.add({ + enabled: true, + name: "weekly delivered", + schedule: { kind: "at", at: new Date(atMs).toISOString() }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { kind: "agentTurn", message: "do it" }, + delivery: { mode: "announce" }, + }); + + vi.setSystemTime(new Date("2025-12-13T00:00:01.000Z")); + await vi.runOnlyPendingTimersAsync(); + + await waitForJobs(cron, (items) => items.some((item) => item.state.lastStatus === "ok")); + expect(runIsolatedAgentJob).toHaveBeenCalledTimes(1); + expect(enqueueSystemEvent).not.toHaveBeenCalled(); + expect(requestHeartbeatNow).not.toHaveBeenCalled(); + cron.stop(); + await store.cleanup(); + }); + it("migrates legacy payload.provider to payload.channel on load", async () => { const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); diff --git a/src/cron/service/state.ts b/src/cron/service/state.ts index 0c7c3c70e3a..4dc1fffdf0a 100644 --- a/src/cron/service/state.ts +++ b/src/cron/service/state.ts @@ -48,7 +48,8 @@ export type CronServiceDeps = { sessionKey?: string; /** * `true` when the isolated run already delivered its output to the target - * channel. See: https://github.com/openclaw/openclaw/issues/15692 + * channel (including matching messaging-tool sends). See: + * https://github.com/openclaw/openclaw/issues/15692 */ delivered?: boolean; }>; From b0728e605dba07273bb1ea3d53b9b7f1a6fa902c Mon Sep 17 00:00:00 2001 From: Brandon Wise Date: Fri, 13 Feb 2026 15:09:07 -0500 Subject: [PATCH 0008/1038] fix(cron): skip relay only for explicit delivery config, not legacy payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #15692 The previous fix was too broad — it removed the relay for ALL isolated jobs. This broke backwards compatibility for jobs without explicit delivery config. The correct behavior is: - If job.delivery exists → isolated runner handles it via runSubagentAnnounceFlow - If only legacy payload.deliver fields → relay to main if requested (original behavior) This addresses Greptile's review feedback about runIsolatedAgentJob being an injected dependency that might not call runSubagentAnnounceFlow. Uses resolveCronDeliveryPlan().source to distinguish between explicit delivery config and legacy payload-only jobs. --- src/cron/service.delivery-plan.test.ts | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/cron/service.delivery-plan.test.ts b/src/cron/service.delivery-plan.test.ts index 707868cba68..15dbc873537 100644 --- a/src/cron/service.delivery-plan.test.ts +++ b/src/cron/service.delivery-plan.test.ts @@ -89,4 +89,47 @@ describe("CronService delivery plan consistency", () => { cron.stop(); await store.cleanup(); }); + + it("does not enqueue duplicate relay when isolated run marks delivery handled", async () => { + const store = await makeStorePath(); + const enqueueSystemEvent = vi.fn(); + const requestHeartbeatNow = vi.fn(); + const runIsolatedAgentJob = vi.fn(async () => ({ + status: "ok" as const, + summary: "done", + delivered: true, + })); + const cron = new CronService({ + cronEnabled: true, + storePath: store.storePath, + log: noopLogger, + enqueueSystemEvent, + requestHeartbeatNow, + runIsolatedAgentJob, + }); + await cron.start(); + const job = await cron.add({ + name: "announce-delivered", + schedule: { kind: "every", everyMs: 60_000, anchorMs: Date.now() }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { + kind: "agentTurn", + message: "hello", + }, + delivery: { channel: "telegram", to: "123" } as unknown as { + mode: "none" | "announce"; + channel?: string; + to?: string; + }, + }); + + const result = await cron.run(job.id, "force"); + expect(result).toEqual({ ok: true, ran: true }); + expect(enqueueSystemEvent).not.toHaveBeenCalled(); + expect(requestHeartbeatNow).not.toHaveBeenCalled(); + + cron.stop(); + await store.cleanup(); + }); }); From b8703546e992f0f0685f9fc5e5f197945ca8473b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 14 Feb 2026 00:00:32 +0100 Subject: [PATCH 0009/1038] docs(changelog): note cron delivered-relay regression coverage (#15737) (thanks @brandonwise) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63c574bfab2..f4c55aa8f8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Docs: https://docs.openclaw.ai - Gateway/Restart: clear stale command-queue and heartbeat wake runtime state after SIGUSR1 in-process restarts to prevent zombie gateway behavior where queued work stops draining. (#15195) Thanks @joeykrug. - Onboarding/CLI: restore terminal state without resuming paused `stdin`, so onboarding exits cleanly after choosing Web UI and the installer returns instead of appearing stuck. - Auth/OpenAI Codex: share OAuth login handling across onboarding and `models auth login --provider openai-codex`, keep onboarding alive when OAuth fails, and surface a direct OAuth help note instead of terminating the wizard. (#15406, follow-up to #14552) Thanks @zhiluo20. +- Cron: add regression coverage for announce-mode isolated jobs so runs that already report `delivered: true` do not enqueue duplicate main-session relays, including delivery configs where `mode` is omitted and defaults to announce. (#15737) Thanks @brandonwise. - Onboarding/Providers: add vLLM as an onboarding provider with model discovery, auth profile wiring, and non-interactive auth-choice validation. (#12577) Thanks @gejifeng. - Onboarding/Providers: preserve Hugging Face auth intent in auth-choice remapping (`tokenProvider=huggingface` with `authChoice=apiKey`) and skip env-override prompts when an explicit token is provided. (#13472) Thanks @Josephrp. - OpenAI Codex/Spark: implement end-to-end `gpt-5.3-codex-spark` support across fallback/thinking/model resolution and `models list` forward-compat visibility. (#14990, #15174) Thanks @L-U-C-K-Y, @loiie45e. From dac8f5ba3f5e5f1e9e6628b9e30a121ca54fbf41 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 22:28:50 +0000 Subject: [PATCH 0010/1038] perf(test): trim fixture and import overhead in hot suites --- src/auto-reply/reply.block-streaming.test.ts | 32 +++--- ...-contract-form-layout-act-commands.test.ts | 5 +- src/canvas-host/server.test.ts | 44 +++++--- src/config/io.write-config.test.ts | 72 +++++++++++- ...onse-has-heartbeat-ok-but-includes.test.ts | 19 +++- src/infra/gateway-lock.test.ts | 44 +++++--- src/memory/index.test.ts | 20 +++- src/memory/qmd-manager.test.ts | 105 ++++++++---------- src/telegram/bot.test.ts | 18 ++- src/web/media.test.ts | 69 ++++-------- 10 files changed, 262 insertions(+), 166 deletions(-) diff --git a/src/auto-reply/reply.block-streaming.test.ts b/src/auto-reply/reply.block-streaming.test.ts index 5a1f97d1d4d..21e8bdf17c2 100644 --- a/src/auto-reply/reply.block-streaming.test.ts +++ b/src/auto-reply/reply.block-streaming.test.ts @@ -39,23 +39,20 @@ describe("block streaming", () => { ]); }); - async function waitForCalls(fn: () => number, calls: number) { - const deadline = Date.now() + 5000; - while (fn() < calls) { - if (Date.now() > deadline) { - throw new Error(`Expected ${calls} call(s), got ${fn()}`); - } - await new Promise((resolve) => setTimeout(resolve, 5)); - } - } - it("waits for block replies before returning final payloads", async () => { await withTempHome(async (home) => { let releaseTyping: (() => void) | undefined; const typingGate = new Promise((resolve) => { releaseTyping = resolve; }); - const onReplyStart = vi.fn(() => typingGate); + let resolveOnReplyStart: (() => void) | undefined; + const onReplyStartCalled = new Promise((resolve) => { + resolveOnReplyStart = resolve; + }); + const onReplyStart = vi.fn(() => { + resolveOnReplyStart?.(); + return typingGate; + }); const onBlockReply = vi.fn().mockResolvedValue(undefined); const impl = async (params: RunEmbeddedPiAgentParams) => { @@ -95,7 +92,7 @@ describe("block streaming", () => { }, ); - await waitForCalls(() => onReplyStart.mock.calls.length, 1); + await onReplyStartCalled; releaseTyping?.(); const res = await replyPromise; @@ -110,7 +107,14 @@ describe("block streaming", () => { const typingGate = new Promise((resolve) => { releaseTyping = resolve; }); - const onReplyStart = vi.fn(() => typingGate); + let resolveOnReplyStart: (() => void) | undefined; + const onReplyStartCalled = new Promise((resolve) => { + resolveOnReplyStart = resolve; + }); + const onReplyStart = vi.fn(() => { + resolveOnReplyStart?.(); + return typingGate; + }); const seen: string[] = []; const onBlockReply = vi.fn(async (payload) => { seen.push(payload.text ?? ""); @@ -154,7 +158,7 @@ describe("block streaming", () => { }, ); - await waitForCalls(() => onReplyStart.mock.calls.length, 1); + await onReplyStartCalled; releaseTyping?.(); const res = await replyPromise; diff --git a/src/browser/server.agent-contract-form-layout-act-commands.test.ts b/src/browser/server.agent-contract-form-layout-act-commands.test.ts index a63eef29c19..2c5c2234740 100644 --- a/src/browser/server.agent-contract-form-layout-act-commands.test.ts +++ b/src/browser/server.agent-contract-form-layout-act-commands.test.ts @@ -156,6 +156,9 @@ vi.mock("./screenshot.js", () => ({ })), })); +const { startBrowserControlServerFromConfig, stopBrowserControlServer } = + await import("./server.js"); + async function getFreePort(): Promise { while (true) { const port = await new Promise((resolve, reject) => { @@ -274,12 +277,10 @@ describe("browser control server", () => { } else { process.env.OPENCLAW_GATEWAY_PORT = prevGatewayPort; } - const { stopBrowserControlServer } = await import("./server.js"); await stopBrowserControlServer(); }); const startServerAndBase = async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); const base = `http://127.0.0.1:${testPort}`; await realFetch(`${base}/start`, { method: "POST" }).then((r) => r.json()); diff --git a/src/canvas-host/server.test.ts b/src/canvas-host/server.test.ts index b768aa02b4d..5c360cd1c98 100644 --- a/src/canvas-host/server.test.ts +++ b/src/canvas-host/server.test.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import { createServer } from "node:http"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { WebSocket } from "ws"; import { rawDataToString } from "../infra/ws.js"; import { defaultRuntime } from "../runtime.js"; @@ -11,6 +11,23 @@ import { A2UI_PATH, CANVAS_HOST_PATH, CANVAS_WS_PATH, injectCanvasLiveReload } f import { createCanvasHostHandler, startCanvasHost } from "./server.js"; describe("canvas host", () => { + let fixtureRoot = ""; + let fixtureCount = 0; + + const createCaseDir = async () => { + const dir = path.join(fixtureRoot, `case-${fixtureCount++}`); + await fs.mkdir(dir, { recursive: true }); + return dir; + }; + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-fixtures-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + it("injects live reload script", () => { const out = injectCanvasLiveReload("Hello"); expect(out).toContain(CANVAS_WS_PATH); @@ -20,7 +37,7 @@ describe("canvas host", () => { }); it("creates a default index.html when missing", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); const server = await startCanvasHost({ runtime: defaultRuntime, @@ -39,12 +56,11 @@ describe("canvas host", () => { expect(html).toContain(CANVAS_WS_PATH); } finally { await server.close(); - await fs.rm(dir, { recursive: true, force: true }); } }); it("skips live reload injection when disabled", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); await fs.writeFile(path.join(dir, "index.html"), "no-reload", "utf8"); const server = await startCanvasHost({ @@ -67,12 +83,11 @@ describe("canvas host", () => { expect(wsRes.status).toBe(404); } finally { await server.close(); - await fs.rm(dir, { recursive: true, force: true }); } }); it("serves canvas content from the mounted base path", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); await fs.writeFile(path.join(dir, "index.html"), "v1", "utf8"); const handler = await createCanvasHostHandler({ @@ -116,12 +131,11 @@ describe("canvas host", () => { await new Promise((resolve, reject) => server.close((err) => (err ? reject(err) : resolve())), ); - await fs.rm(dir, { recursive: true, force: true }); } }); it("reuses a handler without closing it twice", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); await fs.writeFile(path.join(dir, "index.html"), "v1", "utf8"); const handler = await createCanvasHostHandler({ @@ -149,12 +163,11 @@ describe("canvas host", () => { await server.close(); expect(closeSpy).not.toHaveBeenCalled(); await originalClose(); - await fs.rm(dir, { recursive: true, force: true }); } }); it("serves HTML with injection and broadcasts reload on file changes", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); const index = path.join(dir, "index.html"); await fs.writeFile(index, "v1", "utf8"); @@ -194,18 +207,16 @@ describe("canvas host", () => { }); }); - await new Promise((resolve) => setTimeout(resolve, 100)); await fs.writeFile(index, "v2", "utf8"); expect(await msg).toBe("reload"); ws.close(); } finally { await server.close(); - await fs.rm(dir, { recursive: true, force: true }); } }, 20_000); it("serves the gateway-hosted A2UI scaffold", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); const a2uiRoot = path.resolve(process.cwd(), "src/canvas-host/a2ui"); const bundlePath = path.join(a2uiRoot, "a2ui.bundle.js"); let createdBundle = false; @@ -243,12 +254,11 @@ describe("canvas host", () => { if (createdBundle) { await fs.rm(bundlePath, { force: true }); } - await fs.rm(dir, { recursive: true, force: true }); } }); it("rejects traversal-style A2UI asset requests", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); const a2uiRoot = path.resolve(process.cwd(), "src/canvas-host/a2ui"); const bundlePath = path.join(a2uiRoot, "a2ui.bundle.js"); let createdBundle = false; @@ -277,12 +287,11 @@ describe("canvas host", () => { if (createdBundle) { await fs.rm(bundlePath, { force: true }); } - await fs.rm(dir, { recursive: true, force: true }); } }); it("rejects A2UI symlink escapes", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-")); + const dir = await createCaseDir(); const a2uiRoot = path.resolve(process.cwd(), "src/canvas-host/a2ui"); const bundlePath = path.join(a2uiRoot, "a2ui.bundle.js"); const linkName = `test-link-${Date.now()}-${Math.random().toString(16).slice(2)}.txt`; @@ -320,7 +329,6 @@ describe("canvas host", () => { if (createdBundle) { await fs.rm(bundlePath, { force: true }); } - await fs.rm(dir, { recursive: true, force: true }); } }); }); diff --git a/src/config/io.write-config.test.ts b/src/config/io.write-config.test.ts index 917a3f3f009..8bdfb7981ca 100644 --- a/src/config/io.write-config.test.ts +++ b/src/config/io.write-config.test.ts @@ -1,10 +1,78 @@ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createConfigIO } from "./io.js"; -import { withTempHome } from "./test-helpers.js"; + +type HomeEnvSnapshot = { + home: string | undefined; + userProfile: string | undefined; + homeDrive: string | undefined; + homePath: string | undefined; + stateDir: string | undefined; +}; + +function snapshotHomeEnv(): HomeEnvSnapshot { + return { + home: process.env.HOME, + userProfile: process.env.USERPROFILE, + homeDrive: process.env.HOMEDRIVE, + homePath: process.env.HOMEPATH, + stateDir: process.env.OPENCLAW_STATE_DIR, + }; +} + +function restoreHomeEnv(snapshot: HomeEnvSnapshot) { + const restoreKey = (key: string, value: string | undefined) => { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + }; + restoreKey("HOME", snapshot.home); + restoreKey("USERPROFILE", snapshot.userProfile); + restoreKey("HOMEDRIVE", snapshot.homeDrive); + restoreKey("HOMEPATH", snapshot.homePath); + restoreKey("OPENCLAW_STATE_DIR", snapshot.stateDir); +} describe("config io write", () => { + let fixtureRoot = ""; + let fixtureCount = 0; + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-config-io-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + const withTempHome = async (fn: (home: string) => Promise): Promise => { + const home = path.join(fixtureRoot, `home-${fixtureCount++}`); + await fs.mkdir(path.join(home, ".openclaw"), { recursive: true }); + + const snapshot = snapshotHomeEnv(); + process.env.HOME = home; + process.env.USERPROFILE = home; + process.env.OPENCLAW_STATE_DIR = path.join(home, ".openclaw"); + + if (process.platform === "win32") { + const match = home.match(/^([A-Za-z]:)(.*)$/); + if (match) { + process.env.HOMEDRIVE = match[1]; + process.env.HOMEPATH = match[2] || "\\"; + } + } + + try { + return await fn(home); + } finally { + restoreHomeEnv(snapshot); + } + }; + it("persists caller changes onto resolved config without leaking runtime defaults", async () => { await withTempHome(async (home) => { const configPath = path.join(home, ".openclaw", "openclaw.json"); diff --git a/src/cron/isolated-agent.delivers-response-has-heartbeat-ok-but-includes.test.ts b/src/cron/isolated-agent.delivers-response-has-heartbeat-ok-but-includes.test.ts index 674763f8e79..07965726229 100644 --- a/src/cron/isolated-agent.delivers-response-has-heartbeat-ok-but-includes.test.ts +++ b/src/cron/isolated-agent.delivers-response-has-heartbeat-ok-but-includes.test.ts @@ -1,10 +1,10 @@ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { CliDeps } from "../cli/deps.js"; import type { OpenClawConfig } from "../config/config.js"; import type { CronJob } from "./types.js"; -import { withTempHome as withTempHomeBase } from "../../test/helpers/temp-home.js"; import { telegramOutbound } from "../channels/plugins/outbound/telegram.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; import { createOutboundTestPlugin, createTestRegistry } from "../test-utils/channel-plugins.js"; @@ -26,8 +26,13 @@ import { runEmbeddedPiAgent } from "../agents/pi-embedded.js"; import { runSubagentAnnounceFlow } from "../agents/subagent-announce.js"; import { runCronIsolatedAgentTurn } from "./isolated-agent.js"; +let fixtureRoot = ""; +let fixtureCount = 0; + async function withTempHome(fn: (home: string) => Promise): Promise { - return withTempHomeBase(fn, { prefix: "openclaw-cron-" }); + const home = path.join(fixtureRoot, `home-${fixtureCount++}`); + await fs.mkdir(path.join(home, ".openclaw", "agents", "main", "sessions"), { recursive: true }); + return await fn(home); } async function writeSessionStore(home: string) { @@ -87,6 +92,14 @@ function makeJob(payload: CronJob["payload"]): CronJob { } describe("runCronIsolatedAgentTurn", () => { + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-cron-fixtures-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + beforeEach(() => { vi.mocked(runEmbeddedPiAgent).mockReset(); vi.mocked(loadModelCatalog).mockResolvedValue([]); diff --git a/src/infra/gateway-lock.test.ts b/src/infra/gateway-lock.test.ts index 12a93fd5857..3b19f25dda8 100644 --- a/src/infra/gateway-lock.test.ts +++ b/src/infra/gateway-lock.test.ts @@ -3,12 +3,16 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { resolveConfigPath, resolveGatewayLockDir, resolveStateDir } from "../config/paths.js"; import { acquireGatewayLock, GatewayLockError } from "./gateway-lock.js"; +let fixtureRoot = ""; +let fixtureCount = 0; + async function makeEnv() { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gateway-lock-")); + const dir = path.join(fixtureRoot, `case-${fixtureCount++}`); + await fs.mkdir(dir, { recursive: true }); const configPath = path.join(dir, "openclaw.json"); await fs.writeFile(configPath, "{}", "utf8"); await fs.mkdir(resolveGatewayLockDir(), { recursive: true }); @@ -18,9 +22,7 @@ async function makeEnv() { OPENCLAW_STATE_DIR: dir, OPENCLAW_CONFIG_PATH: configPath, }, - cleanup: async () => { - await fs.rm(dir, { recursive: true, force: true }); - }, + cleanup: async () => {}, }; } @@ -61,13 +63,21 @@ function makeProcStat(pid: number, startTime: number) { } describe("gateway lock", () => { + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gateway-lock-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + it("blocks concurrent acquisition until release", async () => { const { env, cleanup } = await makeEnv(); const lock = await acquireGatewayLock({ env, allowInTests: true, - timeoutMs: 200, - pollIntervalMs: 20, + timeoutMs: 80, + pollIntervalMs: 5, }); expect(lock).not.toBeNull(); @@ -75,8 +85,8 @@ describe("gateway lock", () => { acquireGatewayLock({ env, allowInTests: true, - timeoutMs: 200, - pollIntervalMs: 20, + timeoutMs: 80, + pollIntervalMs: 5, }), ).rejects.toBeInstanceOf(GatewayLockError); @@ -84,8 +94,8 @@ describe("gateway lock", () => { const lock2 = await acquireGatewayLock({ env, allowInTests: true, - timeoutMs: 200, - pollIntervalMs: 20, + timeoutMs: 80, + pollIntervalMs: 5, }); await lock2?.release(); await cleanup(); @@ -114,8 +124,8 @@ describe("gateway lock", () => { const lock = await acquireGatewayLock({ env, allowInTests: true, - timeoutMs: 200, - pollIntervalMs: 20, + timeoutMs: 80, + pollIntervalMs: 5, platform: "linux", }); expect(lock).not.toBeNull(); @@ -148,8 +158,8 @@ describe("gateway lock", () => { acquireGatewayLock({ env, allowInTests: true, - timeoutMs: 120, - pollIntervalMs: 20, + timeoutMs: 50, + pollIntervalMs: 5, staleMs: 10_000, platform: "linux", }), @@ -173,8 +183,8 @@ describe("gateway lock", () => { const lock = await acquireGatewayLock({ env, allowInTests: true, - timeoutMs: 200, - pollIntervalMs: 20, + timeoutMs: 80, + pollIntervalMs: 5, staleMs: 1, platform: "linux", }); diff --git a/src/memory/index.test.ts b/src/memory/index.test.ts index 3f01ab85593..3e319a5fd32 100644 --- a/src/memory/index.test.ts +++ b/src/memory/index.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getMemorySearchManager, type MemoryIndexManager } from "./index.js"; let embedBatchCalls = 0; @@ -34,14 +34,25 @@ vi.mock("./embeddings.js", () => { }); describe("memory index", () => { + let fixtureRoot = ""; + let fixtureCount = 0; let workspaceDir: string; let indexPath: string; let manager: MemoryIndexManager | null = null; + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + beforeEach(async () => { embedBatchCalls = 0; failEmbeddings = false; - workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-")); + workspaceDir = path.join(fixtureRoot, `case-${fixtureCount++}`); + await fs.mkdir(workspaceDir, { recursive: true }); indexPath = path.join(workspaceDir, "index.sqlite"); await fs.mkdir(path.join(workspaceDir, "memory")); await fs.writeFile( @@ -56,7 +67,6 @@ describe("memory index", () => { await manager.close(); manager = null; } - await fs.rm(workspaceDir, { recursive: true, force: true }); }); it("indexes memory files and searches by vector", async () => { @@ -270,7 +280,7 @@ describe("memory index", () => { }); it("hybrid weights can favor vector-only matches over keyword-only matches", async () => { - const manyAlpha = Array.from({ length: 200 }, () => "Alpha").join(" "); + const manyAlpha = Array.from({ length: 80 }, () => "Alpha").join(" "); await fs.writeFile( path.join(workspaceDir, "memory", "vector-only.md"), "Alpha beta. Alpha beta. Alpha beta. Alpha beta.", @@ -328,7 +338,7 @@ describe("memory index", () => { }); it("hybrid weights can favor keyword matches when text weight dominates", async () => { - const manyAlpha = Array.from({ length: 200 }, () => "Alpha").join(" "); + const manyAlpha = Array.from({ length: 80 }, () => "Alpha").join(" "); await fs.writeFile( path.join(workspaceDir, "memory", "vector-only.md"), "Alpha beta. Alpha beta. Alpha beta. Alpha beta.", diff --git a/src/memory/qmd-manager.test.ts b/src/memory/qmd-manager.test.ts index e8396802862..a4877417c23 100644 --- a/src/memory/qmd-manager.test.ts +++ b/src/memory/qmd-manager.test.ts @@ -2,7 +2,7 @@ import { EventEmitter } from "node:events"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; const { logWarnMock, logDebugMock, logInfoMock } = vi.hoisted(() => ({ logWarnMock: vi.fn(), @@ -44,6 +44,18 @@ function createMockChild(params?: { autoClose?: boolean; closeDelayMs?: number } return child; } +function emitAndClose( + child: MockChild, + stream: "stdout" | "stderr", + data: string, + code: number = 0, +) { + queueMicrotask(() => { + child[stream].emit("data", data); + child.closeWith(code); + }); +} + vi.mock("../logging/subsystem.js", () => ({ createSubsystemLogger: () => { const logger = { @@ -66,19 +78,30 @@ import { QmdMemoryManager } from "./qmd-manager.js"; const spawnMock = mockedSpawn as unknown as vi.Mock; describe("QmdMemoryManager", () => { + let fixtureRoot: string; + let fixtureCount = 0; let tmpRoot: string; let workspaceDir: string; let stateDir: string; let cfg: OpenClawConfig; const agentId = "main"; + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "qmd-manager-test-fixtures-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + beforeEach(async () => { spawnMock.mockReset(); spawnMock.mockImplementation(() => createMockChild()); logWarnMock.mockReset(); logDebugMock.mockReset(); logInfoMock.mockReset(); - tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "qmd-manager-test-")); + tmpRoot = path.join(fixtureRoot, `case-${fixtureCount++}`); + await fs.mkdir(tmpRoot, { recursive: true }); workspaceDir = path.join(tmpRoot, "workspace"); await fs.mkdir(workspaceDir, { recursive: true }); stateDir = path.join(tmpRoot, "state"); @@ -102,7 +125,6 @@ describe("QmdMemoryManager", () => { afterEach(async () => { vi.useRealTimers(); delete process.env.OPENCLAW_STATE_DIR; - await fs.rm(tmpRoot, { recursive: true, force: true }); }); it("debounces back-to-back sync calls", async () => { @@ -158,14 +180,11 @@ describe("QmdMemoryManager", () => { const createPromise = QmdMemoryManager.create({ cfg, agentId, resolved }); const race = await Promise.race([ createPromise.then(() => "created" as const), - new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 80)), + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 40)), ]); expect(race).toBe("created"); - - if (!releaseUpdate) { - throw new Error("update child missing"); - } - releaseUpdate(); + await waitForCondition(() => releaseUpdate !== null, 200); + releaseUpdate?.(); const manager = await createPromise; await manager?.close(); }); @@ -202,14 +221,11 @@ describe("QmdMemoryManager", () => { const createPromise = QmdMemoryManager.create({ cfg, agentId, resolved }); const race = await Promise.race([ createPromise.then(() => "created" as const), - new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 80)), + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 40)), ]); expect(race).toBe("timeout"); - - if (!releaseUpdate) { - throw new Error("update child missing"); - } - releaseUpdate(); + await waitForCondition(() => releaseUpdate !== null, 200); + releaseUpdate?.(); const manager = await createPromise; await manager?.close(); }); @@ -301,10 +317,7 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "search") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stdout.emit("data", "[]"); - child.closeWith(0); - }, 0); + emitAndClose(child, "stdout", "[]"); return child; } return createMockChild(); @@ -348,18 +361,12 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "search") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stderr.emit("data", "unknown flag: --json"); - child.closeWith(2); - }, 0); + emitAndClose(child, "stderr", "unknown flag: --json", 2); return child; } if (args[0] === "query") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stdout.emit("data", "[]"); - child.closeWith(0); - }, 0); + emitAndClose(child, "stdout", "[]"); return child; } return createMockChild(); @@ -435,7 +442,7 @@ describe("QmdMemoryManager", () => { const inFlight = manager.sync({ reason: "interval" }); const forced = manager.sync({ reason: "manual", force: true }); - await new Promise((resolve) => setTimeout(resolve, 20)); + await waitForCondition(() => updateCalls >= 1, 80); expect(updateCalls).toBe(1); if (!releaseFirstUpdate) { throw new Error("first update release missing"); @@ -496,14 +503,14 @@ describe("QmdMemoryManager", () => { const inFlight = manager.sync({ reason: "interval" }); const forcedOne = manager.sync({ reason: "manual", force: true }); - await new Promise((resolve) => setTimeout(resolve, 20)); + await waitForCondition(() => updateCalls >= 1, 80); expect(updateCalls).toBe(1); if (!releaseFirstUpdate) { throw new Error("first update release missing"); } releaseFirstUpdate(); - await waitForCondition(() => updateCalls >= 2, 200); + await waitForCondition(() => updateCalls >= 2, 120); const forcedTwo = manager.sync({ reason: "manual-again", force: true }); if (!releaseSecondUpdate) { @@ -535,10 +542,7 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "query") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stdout.emit("data", "[]"); - child.closeWith(0); - }, 0); + emitAndClose(child, "stdout", "[]"); return child; } return createMockChild(); @@ -805,13 +809,11 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "query") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stdout.emit( - "data", - JSON.stringify([{ docid: "abc123", score: 1, snippet: "@@ -1,1\nremember this" }]), - ); - child.closeWith(0); - }, 0); + emitAndClose( + child, + "stdout", + JSON.stringify([{ docid: "abc123", score: 1, snippet: "@@ -1,1\nremember this" }]), + ); return child; } return createMockChild(); @@ -844,10 +846,7 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "query") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stdout.emit("data", "No results found."); - child.closeWith(0); - }, 0); + emitAndClose(child, "stdout", "No results found."); return child; } return createMockChild(); @@ -870,10 +869,7 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "query") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stdout.emit("data", "No results found\n\n"); - child.closeWith(0); - }, 0); + emitAndClose(child, "stdout", "No results found\n\n"); return child; } return createMockChild(); @@ -896,10 +892,7 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "query") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { - child.stderr.emit("data", "No results found.\n"); - child.closeWith(0); - }, 0); + emitAndClose(child, "stderr", "No results found.\n"); return child; } return createMockChild(); @@ -922,11 +915,11 @@ describe("QmdMemoryManager", () => { spawnMock.mockImplementation((_cmd: string, args: string[]) => { if (args[0] === "query") { const child = createMockChild({ autoClose: false }); - setTimeout(() => { + queueMicrotask(() => { child.stdout.emit("data", " \n"); child.stderr.emit("data", "unexpected parser error"); child.closeWith(0); - }, 0); + }); return child; } return createMockChild(); @@ -1034,7 +1027,7 @@ async function waitForCondition(check: () => boolean, timeoutMs: number): Promis if (check()) { return; } - await new Promise((resolve) => setTimeout(resolve, 5)); + await new Promise((resolve) => setTimeout(resolve, 2)); } throw new Error("condition was not met in time"); } diff --git a/src/telegram/bot.test.ts b/src/telegram/bot.test.ts index 3c2c63a7d40..cb919a0237f 100644 --- a/src/telegram/bot.test.ts +++ b/src/telegram/bot.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { escapeRegExp, formatEnvelopeTimestamp } from "../../test/helpers/envelope-timestamp.js"; import { expectInboundContextContract } from "../../test/helpers/inbound-contract.js"; import { @@ -23,6 +23,13 @@ vi.mock("../auto-reply/skill-commands.js", () => ({ const { sessionStorePath } = vi.hoisted(() => ({ sessionStorePath: `/tmp/openclaw-telegram-bot-${Math.random().toString(16).slice(2)}.json`, })); +const tempDirs: string[] = []; + +function createTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} function resolveSkillCommands(config: Parameters[0]) { return listSkillCommandsForAgents({ cfg: config }); @@ -208,6 +215,13 @@ describe("createTelegramBot", () => { process.env.TZ = ORIGINAL_TZ; }); + afterAll(() => { + for (const dir of tempDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } + tempDirs.length = 0; + }); + it("installs grammY throttler", () => { createTelegramBot({ token: "tok" }); expect(throttlerSpy).toHaveBeenCalledTimes(1); @@ -1214,7 +1228,7 @@ describe("createTelegramBot", () => { onSpy.mockReset(); const replySpy = replyModule.__replySpy as unknown as ReturnType; replySpy.mockReset(); - const storeDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-telegram-")); + const storeDir = createTempDir("openclaw-telegram-"); const storePath = path.join(storeDir, "sessions.json"); fs.writeFileSync( storePath, diff --git a/src/web/media.test.ts b/src/web/media.test.ts index 0dee4ac0c17..b507b02c809 100644 --- a/src/web/media.test.ts +++ b/src/web/media.test.ts @@ -9,6 +9,8 @@ import { loadWebMedia, loadWebMediaRaw, optimizeImageToJpeg } from "./media.js"; let fixtureRoot = ""; let fixtureFileCount = 0; +let largeJpegBuffer: Buffer; +let tinyPngBuffer: Buffer; async function writeTempFile(buffer: Buffer, ext: string): Promise { const file = path.join(fixtureRoot, `media-${fixtureFileCount++}${ext}`); @@ -27,23 +29,27 @@ function buildDeterministicBytes(length: number): Buffer { } async function createLargeTestJpeg(): Promise<{ buffer: Buffer; file: string }> { - const buffer = await sharp({ + const file = await writeTempFile(largeJpegBuffer, ".jpg"); + return { buffer: largeJpegBuffer, file }; +} + +beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-media-test-")); + largeJpegBuffer = await sharp({ create: { - width: 1600, - height: 1600, + width: 1200, + height: 1200, channels: 3, background: "#ff0000", }, }) .jpeg({ quality: 95 }) .toBuffer(); - - const file = await writeTempFile(buffer, ".jpg"); - return { buffer, file }; -} - -beforeAll(async () => { - fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-media-test-")); + tinyPngBuffer = await sharp({ + create: { width: 10, height: 10, channels: 3, background: "#00ff00" }, + }) + .png() + .toBuffer(); }); afterAll(async () => { @@ -68,18 +74,7 @@ describe("web media loading", () => { }); it("compresses large local images under the provided cap", async () => { - const buffer = await sharp({ - create: { - width: 1200, - height: 1200, - channels: 3, - background: "#ff0000", - }, - }) - .jpeg({ quality: 95 }) - .toBuffer(); - - const file = await writeTempFile(buffer, ".jpg"); + const { buffer, file } = await createLargeTestJpeg(); const cap = Math.floor(buffer.length * 0.8); const result = await loadWebMedia(file, cap); @@ -109,12 +104,7 @@ describe("web media loading", () => { }); it("sniffs mime before extension when loading local files", async () => { - const pngBuffer = await sharp({ - create: { width: 2, height: 2, channels: 3, background: "#00ff00" }, - }) - .png() - .toBuffer(); - const wrongExt = await writeTempFile(pngBuffer, ".bin"); + const wrongExt = await writeTempFile(tinyPngBuffer, ".bin"); const result = await loadWebMedia(wrongExt, 1024 * 1024); @@ -292,7 +282,7 @@ describe("web media loading", () => { }); it("falls back to JPEG when PNG alpha cannot fit under cap", async () => { - const sizes = [320, 448, 640]; + const sizes = [256, 320, 448]; let pngBuffer: Buffer | null = null; let smallestPng: Awaited> | null = null; let jpegOptimized: Awaited> | null = null; @@ -333,12 +323,7 @@ describe("web media loading", () => { describe("local media root guard", () => { it("rejects local paths outside allowed roots", async () => { - const pngBuffer = await sharp({ - create: { width: 10, height: 10, channels: 3, background: "#00ff00" }, - }) - .png() - .toBuffer(); - const file = await writeTempFile(pngBuffer, ".png"); + const file = await writeTempFile(tinyPngBuffer, ".png"); // Explicit roots that don't contain the temp file. await expect( @@ -347,24 +332,14 @@ describe("local media root guard", () => { }); it("allows local paths under an explicit root", async () => { - const pngBuffer = await sharp({ - create: { width: 10, height: 10, channels: 3, background: "#00ff00" }, - }) - .png() - .toBuffer(); - const file = await writeTempFile(pngBuffer, ".png"); + const file = await writeTempFile(tinyPngBuffer, ".png"); const result = await loadWebMedia(file, 1024 * 1024, { localRoots: [os.tmpdir()] }); expect(result.kind).toBe("image"); }); it("allows any path when localRoots is 'any'", async () => { - const pngBuffer = await sharp({ - create: { width: 10, height: 10, channels: 3, background: "#00ff00" }, - }) - .png() - .toBuffer(); - const file = await writeTempFile(pngBuffer, ".png"); + const file = await writeTempFile(tinyPngBuffer, ".png"); const result = await loadWebMedia(file, 1024 * 1024, { localRoots: "any" }); expect(result.kind).toBe("image"); From e324cb5b94c24605844fdd4a9f77c297f548d008 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 22:40:20 +0000 Subject: [PATCH 0011/1038] perf(test): reduce fixture churn in hot suites --- src/auto-reply/reply.block-streaming.test.ts | 65 ++++- src/auto-reply/reply.raw-body.test.ts | 261 ++++++++++--------- src/memory/index.test.ts | 10 +- src/memory/manager.batch.test.ts | 17 +- src/memory/manager.embedding-batches.test.ts | 17 +- 5 files changed, 237 insertions(+), 133 deletions(-) diff --git a/src/auto-reply/reply.block-streaming.test.ts b/src/auto-reply/reply.block-streaming.test.ts index 21e8bdf17c2..4d4fd8d1c8e 100644 --- a/src/auto-reply/reply.block-streaming.test.ts +++ b/src/auto-reply/reply.block-streaming.test.ts @@ -1,6 +1,7 @@ +import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { withTempHome as withTempHomeBase } from "../../test/helpers/temp-home.js"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { loadModelCatalog } from "../agents/model-catalog.js"; import { getReplyFromConfig } from "./reply.js"; @@ -22,11 +23,69 @@ vi.mock("../agents/model-catalog.js", () => ({ loadModelCatalog: vi.fn(), })); +type HomeEnvSnapshot = { + HOME: string | undefined; + USERPROFILE: string | undefined; + HOMEDRIVE: string | undefined; + HOMEPATH: string | undefined; + OPENCLAW_STATE_DIR: string | undefined; +}; + +function snapshotHomeEnv(): HomeEnvSnapshot { + return { + HOME: process.env.HOME, + USERPROFILE: process.env.USERPROFILE, + HOMEDRIVE: process.env.HOMEDRIVE, + HOMEPATH: process.env.HOMEPATH, + OPENCLAW_STATE_DIR: process.env.OPENCLAW_STATE_DIR, + }; +} + +function restoreHomeEnv(snapshot: HomeEnvSnapshot) { + for (const [key, value] of Object.entries(snapshot)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} + +let fixtureRoot = ""; +let caseId = 0; + async function withTempHome(fn: (home: string) => Promise): Promise { - return withTempHomeBase(fn, { prefix: "openclaw-stream-" }); + const home = path.join(fixtureRoot, `case-${++caseId}`); + await fs.mkdir(path.join(home, ".openclaw", "agents", "main", "sessions"), { recursive: true }); + const envSnapshot = snapshotHomeEnv(); + process.env.HOME = home; + process.env.USERPROFILE = home; + process.env.OPENCLAW_STATE_DIR = path.join(home, ".openclaw"); + + if (process.platform === "win32") { + const match = home.match(/^([A-Za-z]:)(.*)$/); + if (match) { + process.env.HOMEDRIVE = match[1]; + process.env.HOMEPATH = match[2] || "\\"; + } + } + + try { + return await fn(home); + } finally { + restoreHomeEnv(envSnapshot); + } } describe("block streaming", () => { + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-stream-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + beforeEach(() => { piEmbeddedMock.abortEmbeddedPiRun.mockReset().mockReturnValue(false); piEmbeddedMock.queueEmbeddedPiMessage.mockReset().mockReturnValue(false); diff --git a/src/auto-reply/reply.raw-body.test.ts b/src/auto-reply/reply.raw-body.test.ts index 38c8b30e218..75d586bffee 100644 --- a/src/auto-reply/reply.raw-body.test.ts +++ b/src/auto-reply/reply.raw-body.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { withTempHome as withTempHomeBase } from "../../test/helpers/temp-home.js"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { loadModelCatalog } from "../agents/model-catalog.js"; import { runEmbeddedPiAgent } from "../agents/pi-embedded.js"; import { saveSessionStore } from "../config/sessions.js"; @@ -19,22 +19,78 @@ vi.mock("../agents/model-catalog.js", () => ({ loadModelCatalog: vi.fn(), })); +type HomeEnvSnapshot = { + HOME: string | undefined; + USERPROFILE: string | undefined; + HOMEDRIVE: string | undefined; + HOMEPATH: string | undefined; + OPENCLAW_STATE_DIR: string | undefined; + OPENCLAW_AGENT_DIR: string | undefined; + PI_CODING_AGENT_DIR: string | undefined; +}; + +function snapshotHomeEnv(): HomeEnvSnapshot { + return { + HOME: process.env.HOME, + USERPROFILE: process.env.USERPROFILE, + HOMEDRIVE: process.env.HOMEDRIVE, + HOMEPATH: process.env.HOMEPATH, + OPENCLAW_STATE_DIR: process.env.OPENCLAW_STATE_DIR, + OPENCLAW_AGENT_DIR: process.env.OPENCLAW_AGENT_DIR, + PI_CODING_AGENT_DIR: process.env.PI_CODING_AGENT_DIR, + }; +} + +function restoreHomeEnv(snapshot: HomeEnvSnapshot) { + for (const [key, value] of Object.entries(snapshot)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} + +let fixtureRoot = ""; +let caseId = 0; + async function withTempHome(fn: (home: string) => Promise): Promise { - return withTempHomeBase( - async (home) => { - return await fn(home); - }, - { - env: { - OPENCLAW_AGENT_DIR: (home) => path.join(home, ".openclaw", "agent"), - PI_CODING_AGENT_DIR: (home) => path.join(home, ".openclaw", "agent"), - }, - prefix: "openclaw-rawbody-", - }, - ); + const home = path.join(fixtureRoot, `case-${++caseId}`); + await fs.mkdir(path.join(home, ".openclaw", "agents", "main", "sessions"), { recursive: true }); + const envSnapshot = snapshotHomeEnv(); + process.env.HOME = home; + process.env.USERPROFILE = home; + process.env.OPENCLAW_STATE_DIR = path.join(home, ".openclaw"); + process.env.OPENCLAW_AGENT_DIR = path.join(home, ".openclaw", "agent"); + process.env.PI_CODING_AGENT_DIR = path.join(home, ".openclaw", "agent"); + + if (process.platform === "win32") { + const match = home.match(/^([A-Za-z]:)(.*)$/); + if (match) { + process.env.HOMEDRIVE = match[1]; + process.env.HOMEPATH = match[2] || "\\"; + } + } + + try { + return await fn(home); + } finally { + restoreHomeEnv(envSnapshot); + } } describe("RawBody directive parsing", () => { + type ReplyMessage = Parameters[0]; + type ReplyConfig = Parameters[2]; + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rawbody-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + beforeEach(() => { vi.mocked(runEmbeddedPiAgent).mockReset(); vi.mocked(loadModelCatalog).mockResolvedValue([ @@ -46,147 +102,116 @@ describe("RawBody directive parsing", () => { vi.clearAllMocks(); }); - it("/model, /think, /verbose directives detected from RawBody even when Body has structural wrapper", async () => { + it("detects command directives from RawBody/CommandBody in wrapped group messages", async () => { await withTempHome(async (home) => { - vi.mocked(runEmbeddedPiAgent).mockReset(); - - const groupMessageCtx = { - Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /think:high\\n[from: Jake McInteer (+6421807830)]`, - RawBody: "/think:high", - From: "+1222", - To: "+1222", - ChatType: "group", - CommandAuthorized: true, + const assertCommandReply = async (input: { + message: ReplyMessage; + config: ReplyConfig; + expectedIncludes: string[]; + }) => { + vi.mocked(runEmbeddedPiAgent).mockReset(); + const res = await getReplyFromConfig(input.message, {}, input.config); + const text = Array.isArray(res) ? res[0]?.text : res?.text; + for (const expected of input.expectedIncludes) { + expect(text).toContain(expected); + } + expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); }; - const res = await getReplyFromConfig( - groupMessageCtx, - {}, - { + await assertCommandReply({ + message: { + Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /think:high\\n[from: Jake McInteer (+6421807830)]`, + RawBody: "/think:high", + From: "+1222", + To: "+1222", + ChatType: "group", + CommandAuthorized: true, + }, + config: { agents: { defaults: { model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw"), + workspace: path.join(home, "openclaw-1"), }, }, channels: { whatsapp: { allowFrom: ["*"] } }, - session: { store: path.join(home, "sessions.json") }, + session: { store: path.join(home, "sessions-1.json") }, }, - ); + expectedIncludes: ["Thinking level set to high."], + }); - const text = Array.isArray(res) ? res[0]?.text : res?.text; - expect(text).toContain("Thinking level set to high."); - expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); - }); - }); - - it("/model status detected from RawBody", async () => { - await withTempHome(async (home) => { - vi.mocked(runEmbeddedPiAgent).mockReset(); - - const groupMessageCtx = { - Body: `[Context]\nJake: /model status\n[from: Jake]`, - RawBody: "/model status", - From: "+1222", - To: "+1222", - ChatType: "group", - CommandAuthorized: true, - }; - - const res = await getReplyFromConfig( - groupMessageCtx, - {}, - { + await assertCommandReply({ + message: { + Body: "[Context]\nJake: /model status\n[from: Jake]", + RawBody: "/model status", + From: "+1222", + To: "+1222", + ChatType: "group", + CommandAuthorized: true, + }, + config: { agents: { defaults: { model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw"), + workspace: path.join(home, "openclaw-2"), models: { "anthropic/claude-opus-4-5": {}, }, }, }, channels: { whatsapp: { allowFrom: ["*"] } }, - session: { store: path.join(home, "sessions.json") }, + session: { store: path.join(home, "sessions-2.json") }, }, - ); + expectedIncludes: ["anthropic/claude-opus-4-5"], + }); - const text = Array.isArray(res) ? res[0]?.text : res?.text; - expect(text).toContain("anthropic/claude-opus-4-5"); - expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); - }); - }); - - it("CommandBody is honored when RawBody is missing", async () => { - await withTempHome(async (home) => { - vi.mocked(runEmbeddedPiAgent).mockReset(); - - const groupMessageCtx = { - Body: `[Context]\nJake: /verbose on\n[from: Jake]`, - CommandBody: "/verbose on", - From: "+1222", - To: "+1222", - ChatType: "group", - CommandAuthorized: true, - }; - - const res = await getReplyFromConfig( - groupMessageCtx, - {}, - { + await assertCommandReply({ + message: { + Body: "[Context]\nJake: /verbose on\n[from: Jake]", + CommandBody: "/verbose on", + From: "+1222", + To: "+1222", + ChatType: "group", + CommandAuthorized: true, + }, + config: { agents: { defaults: { model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw"), + workspace: path.join(home, "openclaw-3"), }, }, channels: { whatsapp: { allowFrom: ["*"] } }, - session: { store: path.join(home, "sessions.json") }, + session: { store: path.join(home, "sessions-3.json") }, }, - ); + expectedIncludes: ["Verbose logging enabled."], + }); - const text = Array.isArray(res) ? res[0]?.text : res?.text; - expect(text).toContain("Verbose logging enabled."); - expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); - }); - }); - - it("Integration: WhatsApp group message with structural wrapper and RawBody command", async () => { - await withTempHome(async (home) => { - vi.mocked(runEmbeddedPiAgent).mockReset(); - - const groupMessageCtx = { - Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /status\\n[from: Jake McInteer (+6421807830)]`, - RawBody: "/status", - ChatType: "group", - From: "+1222", - To: "+1222", - SessionKey: "agent:main:whatsapp:group:g1", - Provider: "whatsapp", - Surface: "whatsapp", - SenderE164: "+1222", - CommandAuthorized: true, - }; - - const res = await getReplyFromConfig( - groupMessageCtx, - {}, - { + await assertCommandReply({ + message: { + Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /status\\n[from: Jake McInteer (+6421807830)]`, + RawBody: "/status", + ChatType: "group", + From: "+1222", + To: "+1222", + SessionKey: "agent:main:whatsapp:group:g1", + Provider: "whatsapp", + Surface: "whatsapp", + SenderE164: "+1222", + CommandAuthorized: true, + }, + config: { agents: { defaults: { model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw"), + workspace: path.join(home, "openclaw-4"), }, }, channels: { whatsapp: { allowFrom: ["+1222"] } }, - session: { store: path.join(home, "sessions.json") }, + session: { store: path.join(home, "sessions-4.json") }, }, - ); - - const text = Array.isArray(res) ? res[0]?.text : res?.text; - expect(text).toContain("Session: agent:main:whatsapp:group:g1"); - expect(text).toContain("anthropic/claude-opus-4-5"); - expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); + expectedIncludes: ["Session: agent:main:whatsapp:group:g1", "anthropic/claude-opus-4-5"], + }); }); }); diff --git a/src/memory/index.test.ts b/src/memory/index.test.ts index 3e319a5fd32..97c0dc0201b 100644 --- a/src/memory/index.test.ts +++ b/src/memory/index.test.ts @@ -144,6 +144,7 @@ describe("memory index", () => { throw new Error("manager missing"); } await first.manager.sync({ force: true }); + const callsAfterFirstSync = embedBatchCalls; await first.manager.close(); const second = await getMemorySearchManager({ @@ -168,8 +169,9 @@ describe("memory index", () => { } manager = second.manager; await second.manager.sync({ reason: "test" }); - const results = await second.manager.search("alpha"); - expect(results.length).toBeGreaterThan(0); + expect(embedBatchCalls).toBeGreaterThan(callsAfterFirstSync); + const status = second.manager.status(); + expect(status.files).toBeGreaterThan(0); }); it("reuses cached embeddings on forced reindex", async () => { @@ -280,7 +282,7 @@ describe("memory index", () => { }); it("hybrid weights can favor vector-only matches over keyword-only matches", async () => { - const manyAlpha = Array.from({ length: 80 }, () => "Alpha").join(" "); + const manyAlpha = Array.from({ length: 50 }, () => "Alpha").join(" "); await fs.writeFile( path.join(workspaceDir, "memory", "vector-only.md"), "Alpha beta. Alpha beta. Alpha beta. Alpha beta.", @@ -338,7 +340,7 @@ describe("memory index", () => { }); it("hybrid weights can favor keyword matches when text weight dominates", async () => { - const manyAlpha = Array.from({ length: 80 }, () => "Alpha").join(" "); + const manyAlpha = Array.from({ length: 50 }, () => "Alpha").join(" "); await fs.writeFile( path.join(workspaceDir, "memory", "vector-only.md"), "Alpha beta. Alpha beta. Alpha beta. Alpha beta.", diff --git a/src/memory/manager.batch.test.ts b/src/memory/manager.batch.test.ts index 60586d2ec58..2ac5eeb5be5 100644 --- a/src/memory/manager.batch.test.ts +++ b/src/memory/manager.batch.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getMemorySearchManager, type MemoryIndexManager } from "./index.js"; const embedBatch = vi.fn(async () => []); @@ -25,11 +25,21 @@ vi.mock("./embeddings.js", () => ({ })); describe("memory indexing with OpenAI batches", () => { + let fixtureRoot: string; + let caseId = 0; let workspaceDir: string; let indexPath: string; let manager: MemoryIndexManager | null = null; let setTimeoutSpy: ReturnType; + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-batch-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + beforeEach(async () => { embedBatch.mockClear(); embedQuery.mockClear(); @@ -48,9 +58,9 @@ describe("memory indexing with OpenAI batches", () => { } return realSetTimeout(handler, delay, ...args); }) as typeof setTimeout); - workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-batch-")); + workspaceDir = path.join(fixtureRoot, `case-${++caseId}`); indexPath = path.join(workspaceDir, "index.sqlite"); - await fs.mkdir(path.join(workspaceDir, "memory")); + await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true }); }); afterEach(async () => { @@ -60,7 +70,6 @@ describe("memory indexing with OpenAI batches", () => { await manager.close(); manager = null; } - await fs.rm(workspaceDir, { recursive: true, force: true }); }); it("uses OpenAI batch uploads when enabled", async () => { diff --git a/src/memory/manager.embedding-batches.test.ts b/src/memory/manager.embedding-batches.test.ts index 3c4019d366b..371b3e6ff17 100644 --- a/src/memory/manager.embedding-batches.test.ts +++ b/src/memory/manager.embedding-batches.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getMemorySearchManager, type MemoryIndexManager } from "./index.js"; const embedBatch = vi.fn(async (texts: string[]) => texts.map(() => [0, 1, 0])); @@ -20,16 +20,26 @@ vi.mock("./embeddings.js", () => ({ })); describe("memory embedding batches", () => { + let fixtureRoot: string; + let caseId = 0; let workspaceDir: string; let indexPath: string; let manager: MemoryIndexManager | null = null; + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-")); + }); + + afterAll(async () => { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + beforeEach(async () => { embedBatch.mockClear(); embedQuery.mockClear(); - workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-")); + workspaceDir = path.join(fixtureRoot, `case-${++caseId}`); indexPath = path.join(workspaceDir, "index.sqlite"); - await fs.mkdir(path.join(workspaceDir, "memory")); + await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true }); }); afterEach(async () => { @@ -37,7 +47,6 @@ describe("memory embedding batches", () => { await manager.close(); manager = null; } - await fs.rm(workspaceDir, { recursive: true, force: true }); }); it("splits large files across multiple embedding batches", async () => { From faeac955b5b3b7ebde51481718d1b8b4c3c4df1c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 22:42:03 +0000 Subject: [PATCH 0012/1038] perf(test): trim retry-loop work in embedding batch tests --- src/memory/manager.embedding-batches.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/memory/manager.embedding-batches.test.ts b/src/memory/manager.embedding-batches.test.ts index 371b3e6ff17..db59e21310a 100644 --- a/src/memory/manager.embedding-batches.test.ts +++ b/src/memory/manager.embedding-batches.test.ts @@ -169,7 +169,7 @@ describe("memory embedding batches", () => { let calls = 0; embedBatch.mockImplementation(async (texts: string[]) => { calls += 1; - if (calls < 3) { + if (calls < 2) { throw new Error("openai embeddings failed: 429 rate limit"); } return texts.map(() => [0, 1, 0]); @@ -217,7 +217,7 @@ describe("memory embedding batches", () => { setTimeoutSpy.mockRestore(); } - expect(calls).toBe(3); + expect(calls).toBe(2); }, 10000); it("retries embeddings on transient 5xx errors", async () => { @@ -228,7 +228,7 @@ describe("memory embedding batches", () => { let calls = 0; embedBatch.mockImplementation(async (texts: string[]) => { calls += 1; - if (calls < 3) { + if (calls < 2) { throw new Error("openai embeddings failed: 502 Bad Gateway (cloudflare)"); } return texts.map(() => [0, 1, 0]); @@ -276,7 +276,7 @@ describe("memory embedding batches", () => { setTimeoutSpy.mockRestore(); } - expect(calls).toBe(3); + expect(calls).toBe(2); }, 10000); it("skips empty chunks so embeddings input stays valid", async () => { From 1aa746f0426528504bb7defbe846352a124b61c9 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 22:43:13 +0000 Subject: [PATCH 0013/1038] perf(test): lower synthetic payload in embedding batch split case --- src/memory/manager.embedding-batches.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory/manager.embedding-batches.test.ts b/src/memory/manager.embedding-batches.test.ts index db59e21310a..d6142802fcc 100644 --- a/src/memory/manager.embedding-batches.test.ts +++ b/src/memory/manager.embedding-batches.test.ts @@ -51,7 +51,7 @@ describe("memory embedding batches", () => { it("splits large files across multiple embedding batches", async () => { const line = "a".repeat(200); - const content = Array.from({ length: 50 }, () => line).join("\n"); + const content = Array.from({ length: 40 }, () => line).join("\n"); await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-03.md"), content); const cfg = { From dc507f3dec8de780a865b865471d971312eed28b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 23:22:30 +0000 Subject: [PATCH 0014/1038] perf(test): reduce memory and port probe overhead --- src/infra/ports.ts | 8 +- src/memory/index.test.ts | 3 +- src/memory/manager.embedding-batches.test.ts | 128 +++---------------- 3 files changed, 27 insertions(+), 112 deletions(-) diff --git a/src/infra/ports.ts b/src/infra/ports.ts index f8bc799c578..1d73b7ff64e 100644 --- a/src/infra/ports.ts +++ b/src/infra/ports.ts @@ -42,8 +42,7 @@ export async function ensurePortAvailable(port: number): Promise { }); } catch (err) { if (isErrno(err) && err.code === "EADDRINUSE") { - const details = await describePortOwner(port); - throw new PortInUseError(port, details); + throw new PortInUseError(port); } throw err; } @@ -57,7 +56,10 @@ export async function handlePortError( ): Promise { // Uniform messaging for EADDRINUSE with optional owner details. if (err instanceof PortInUseError || (isErrno(err) && err.code === "EADDRINUSE")) { - const details = err instanceof PortInUseError ? err.details : await describePortOwner(port); + const details = + err instanceof PortInUseError + ? (err.details ?? (await describePortOwner(port))) + : await describePortOwner(port); runtime.error(danger(`${context} failed: port ${port} is already in use.`)); if (details) { runtime.error(info("Port listener details:")); diff --git a/src/memory/index.test.ts b/src/memory/index.test.ts index 97c0dc0201b..3030c45dbb4 100644 --- a/src/memory/index.test.ts +++ b/src/memory/index.test.ts @@ -57,9 +57,8 @@ describe("memory index", () => { await fs.mkdir(path.join(workspaceDir, "memory")); await fs.writeFile( path.join(workspaceDir, "memory", "2026-01-12.md"), - "# Log\nAlpha memory line.\nZebra memory line.\nAnother line.", + "# Log\nAlpha memory line.\nZebra memory line.", ); - await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "Beta knowledge base entry."); }); afterEach(async () => { diff --git a/src/memory/manager.embedding-batches.test.ts b/src/memory/manager.embedding-batches.test.ts index d6142802fcc..99cceee162d 100644 --- a/src/memory/manager.embedding-batches.test.ts +++ b/src/memory/manager.embedding-batches.test.ts @@ -77,12 +77,23 @@ describe("memory embedding batches", () => { throw new Error("manager missing"); } manager = result.manager; - await manager.sync({ force: true }); + const updates: Array<{ completed: number; total: number; label?: string }> = []; + await manager.sync({ + force: true, + progress: (update) => { + updates.push(update); + }, + }); const status = manager.status(); const totalTexts = embedBatch.mock.calls.reduce((sum, call) => sum + (call[0]?.length ?? 0), 0); expect(totalTexts).toBe(status.chunks); expect(embedBatch.mock.calls.length).toBeGreaterThan(1); + expect(updates.length).toBeGreaterThan(0); + expect(updates.some((update) => update.label?.includes("/"))).toBe(true); + const last = updates[updates.length - 1]; + expect(last?.total).toBeGreaterThan(0); + expect(last?.completed).toBe(last?.total); }); it("keeps small files in a single embedding batch", async () => { @@ -118,59 +129,21 @@ describe("memory embedding batches", () => { expect(embedBatch.mock.calls.length).toBe(1); }); - it("reports sync progress totals", async () => { - const line = "c".repeat(120); - const content = Array.from({ length: 8 }, () => line).join("\n"); - await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-05.md"), content); - - const cfg = { - agents: { - defaults: { - workspace: workspaceDir, - memorySearch: { - provider: "openai", - model: "mock-embed", - store: { path: indexPath }, - chunking: { tokens: 200, overlap: 0 }, - sync: { watch: false, onSessionStart: false, onSearch: false }, - query: { minScore: 0 }, - }, - }, - list: [{ id: "main", default: true }], - }, - }; - - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - expect(result.manager).not.toBeNull(); - if (!result.manager) { - throw new Error("manager missing"); - } - manager = result.manager; - const updates: Array<{ completed: number; total: number; label?: string }> = []; - await manager.sync({ - force: true, - progress: (update) => { - updates.push(update); - }, - }); - - expect(updates.length).toBeGreaterThan(0); - expect(updates.some((update) => update.label?.includes("/"))).toBe(true); - const last = updates[updates.length - 1]; - expect(last?.total).toBeGreaterThan(0); - expect(last?.completed).toBe(last?.total); - }); - - it("retries embeddings on rate limit errors", async () => { + it("retries embeddings on transient rate limit and 5xx errors", async () => { const line = "d".repeat(120); const content = Array.from({ length: 4 }, () => line).join("\n"); await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-06.md"), content); + const transientErrors = [ + "openai embeddings failed: 429 rate limit", + "openai embeddings failed: 502 Bad Gateway (cloudflare)", + ]; let calls = 0; embedBatch.mockImplementation(async (texts: string[]) => { calls += 1; - if (calls < 2) { - throw new Error("openai embeddings failed: 429 rate limit"); + const transient = transientErrors[calls - 1]; + if (transient) { + throw new Error(transient); } return texts.map(() => [0, 1, 0]); }); @@ -217,66 +190,7 @@ describe("memory embedding batches", () => { setTimeoutSpy.mockRestore(); } - expect(calls).toBe(2); - }, 10000); - - it("retries embeddings on transient 5xx errors", async () => { - const line = "e".repeat(120); - const content = Array.from({ length: 4 }, () => line).join("\n"); - await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-08.md"), content); - - let calls = 0; - embedBatch.mockImplementation(async (texts: string[]) => { - calls += 1; - if (calls < 2) { - throw new Error("openai embeddings failed: 502 Bad Gateway (cloudflare)"); - } - return texts.map(() => [0, 1, 0]); - }); - - const realSetTimeout = setTimeout; - const setTimeoutSpy = vi.spyOn(global, "setTimeout").mockImplementation((( - handler: TimerHandler, - timeout?: number, - ...args: unknown[] - ) => { - const delay = typeof timeout === "number" ? timeout : 0; - if (delay > 0 && delay <= 2000) { - return realSetTimeout(handler, 0, ...args); - } - return realSetTimeout(handler, delay, ...args); - }) as typeof setTimeout); - - const cfg = { - agents: { - defaults: { - workspace: workspaceDir, - memorySearch: { - provider: "openai", - model: "mock-embed", - store: { path: indexPath }, - chunking: { tokens: 200, overlap: 0 }, - sync: { watch: false, onSessionStart: false, onSearch: false }, - query: { minScore: 0 }, - }, - }, - list: [{ id: "main", default: true }], - }, - }; - - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - expect(result.manager).not.toBeNull(); - if (!result.manager) { - throw new Error("manager missing"); - } - manager = result.manager; - try { - await manager.sync({ force: true }); - } finally { - setTimeoutSpy.mockRestore(); - } - - expect(calls).toBe(2); + expect(calls).toBe(3); }, 10000); it("skips empty chunks so embeddings input stays valid", async () => { From ab4a08a82accc36ca8cb223c6f9a31eb8e6f72d5 Mon Sep 17 00:00:00 2001 From: Bridgerz Date: Fri, 13 Feb 2026 15:29:29 -0800 Subject: [PATCH 0015/1038] fix: defer gateway restart until all replies are sent (#12970) * fix: defer gateway restart until all replies are sent Fixes a race condition where gateway config changes (e.g., enabling plugins via iMessage) trigger an immediate SIGUSR1 restart, killing the iMessage RPC connection before replies are delivered. Both restart paths (config watcher and RPC-triggered) now defer until all queued operations, pending replies, and embedded agent runs complete (polling every 500ms, 30s timeout). A shared emitGatewayRestart() guard prevents double SIGUSR1 when both paths fire simultaneously. Key changes: - Dispatcher registry tracks active reply dispatchers globally - markComplete() called in finally block for guaranteed cleanup - Pre-restart deferral hook registered at gateway startup - Centralized extractDeliveryInfo() for session key parsing - Post-restart sentinel messages delivered directly (not via agent) - config-patch distinguished from config-apply in sentinel kind Co-Authored-By: Claude Opus 4.6 * fix: single-source gateway restart authorization --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Peter Steinberger --- src/agents/pi-embedded-runner/runs.ts | 4 + src/agents/tools/gateway-tool.ts | 36 +--- .../reply/dispatch-from-config.test.ts | 1 + src/auto-reply/reply/dispatch-from-config.ts | 4 + src/auto-reply/reply/dispatcher-registry.ts | 58 +++++ src/auto-reply/reply/reply-dispatcher.ts | 45 +++- src/auto-reply/reply/reply-routing.test.ts | 2 + src/config/sessions.ts | 1 + src/config/sessions/delivery-info.ts | 46 ++++ src/gateway/server-methods/config.ts | 16 +- src/gateway/server-reload-handlers.ts | 89 +++++++- .../server-reload.config-during-reply.test.ts | 151 +++++++++++++ src/gateway/server-reload.integration.test.ts | 199 ++++++++++++++++++ .../server-reload.real-scenario.test.ts | 121 +++++++++++ src/gateway/server-restart-sentinel.ts | 28 +-- src/gateway/server.impl.ts | 8 +- src/imessage/monitor/monitor-provider.ts | 1 + src/infra/infra-runtime.test.ts | 111 +++++++++- src/infra/restart-sentinel.test.ts | 35 +++ src/infra/restart-sentinel.ts | 7 +- src/infra/restart.ts | 89 ++++++-- 21 files changed, 976 insertions(+), 76 deletions(-) create mode 100644 src/auto-reply/reply/dispatcher-registry.ts create mode 100644 src/config/sessions/delivery-info.ts create mode 100644 src/gateway/server-reload.config-during-reply.test.ts create mode 100644 src/gateway/server-reload.integration.test.ts create mode 100644 src/gateway/server-reload.real-scenario.test.ts diff --git a/src/agents/pi-embedded-runner/runs.ts b/src/agents/pi-embedded-runner/runs.ts index f5ca9721083..e0155874028 100644 --- a/src/agents/pi-embedded-runner/runs.ts +++ b/src/agents/pi-embedded-runner/runs.ts @@ -64,6 +64,10 @@ export function isEmbeddedPiRunStreaming(sessionId: string): boolean { return handle.isStreaming(); } +export function getActiveEmbeddedRunCount(): number { + return ACTIVE_EMBEDDED_RUNS.size; +} + export function waitForEmbeddedPiRunEnd(sessionId: string, timeoutMs = 15_000): Promise { if (!sessionId || !ACTIVE_EMBEDDED_RUNS.has(sessionId)) { return Promise.resolve(true); diff --git a/src/agents/tools/gateway-tool.ts b/src/agents/tools/gateway-tool.ts index 9560b323c4a..127fe1ff184 100644 --- a/src/agents/tools/gateway-tool.ts +++ b/src/agents/tools/gateway-tool.ts @@ -1,7 +1,7 @@ import { Type } from "@sinclair/typebox"; import type { OpenClawConfig } from "../../config/config.js"; -import { loadConfig, resolveConfigSnapshotHash } from "../../config/io.js"; -import { loadSessionStore, resolveStorePath } from "../../config/sessions.js"; +import { resolveConfigSnapshotHash } from "../../config/io.js"; +import { extractDeliveryInfo } from "../../config/sessions.js"; import { formatDoctorNonInteractiveHint, type RestartSentinelPayload, @@ -69,7 +69,7 @@ export function createGatewayTool(opts?: { label: "Gateway", name: "gateway", description: - "Restart, apply config, or update the gateway in-place (SIGUSR1). Use config.patch for safe partial config updates (merges with existing). Use config.apply only when replacing entire config. Both trigger restart after writing.", + "Restart, apply config, or update the gateway in-place (SIGUSR1). Use config.patch for safe partial config updates (merges with existing). Use config.apply only when replacing entire config. Both trigger restart after writing. Always pass a human-readable completion message via the `note` parameter so the system can deliver it to the user after restart.", parameters: GatewayToolSchema, execute: async (_toolCallId, args) => { const params = args as Record; @@ -93,34 +93,8 @@ export function createGatewayTool(opts?: { const note = typeof params.note === "string" && params.note.trim() ? params.note.trim() : undefined; // Extract channel + threadId for routing after restart - let deliveryContext: { channel?: string; to?: string; accountId?: string } | undefined; - let threadId: string | undefined; - if (sessionKey) { - const threadMarker = ":thread:"; - const threadIndex = sessionKey.lastIndexOf(threadMarker); - const baseSessionKey = threadIndex === -1 ? sessionKey : sessionKey.slice(0, threadIndex); - const threadIdRaw = - threadIndex === -1 ? undefined : sessionKey.slice(threadIndex + threadMarker.length); - threadId = threadIdRaw?.trim() || undefined; - try { - const cfg = loadConfig(); - const storePath = resolveStorePath(cfg.session?.store); - const store = loadSessionStore(storePath); - let entry = store[sessionKey]; - if (!entry?.deliveryContext && threadIndex !== -1 && baseSessionKey) { - entry = store[baseSessionKey]; - } - if (entry?.deliveryContext) { - deliveryContext = { - channel: entry.deliveryContext.channel, - to: entry.deliveryContext.to, - accountId: entry.deliveryContext.accountId, - }; - } - } catch { - // ignore: best-effort - } - } + // Supports both :thread: (most channels) and :topic: (Telegram) + const { deliveryContext, threadId } = extractDeliveryInfo(sessionKey); const payload: RestartSentinelPayload = { kind: "restart", status: "ok", diff --git a/src/auto-reply/reply/dispatch-from-config.test.ts b/src/auto-reply/reply/dispatch-from-config.test.ts index 01c96466965..4cc6657d2a2 100644 --- a/src/auto-reply/reply/dispatch-from-config.test.ts +++ b/src/auto-reply/reply/dispatch-from-config.test.ts @@ -64,6 +64,7 @@ function createDispatcher(): ReplyDispatcher { sendFinalReply: vi.fn(() => true), waitForIdle: vi.fn(async () => {}), getQueuedCounts: vi.fn(() => ({ tool: 0, block: 0, final: 0 })), + markComplete: vi.fn(), }; } diff --git a/src/auto-reply/reply/dispatch-from-config.ts b/src/auto-reply/reply/dispatch-from-config.ts index f04aff0a7b5..0f2cae6b4a2 100644 --- a/src/auto-reply/reply/dispatch-from-config.ts +++ b/src/auto-reply/reply/dispatch-from-config.ts @@ -454,5 +454,9 @@ export async function dispatchReplyFromConfig(params: { recordProcessed("error", { error: String(err) }); markIdle("message_error"); throw err; + } finally { + // Always clear the dispatcher reservation so a leaked pending count + // can never permanently block gateway restarts. + dispatcher.markComplete(); } } diff --git a/src/auto-reply/reply/dispatcher-registry.ts b/src/auto-reply/reply/dispatcher-registry.ts new file mode 100644 index 00000000000..0ef42fbf73f --- /dev/null +++ b/src/auto-reply/reply/dispatcher-registry.ts @@ -0,0 +1,58 @@ +/** + * Global registry for tracking active reply dispatchers. + * Used to ensure gateway restart waits for all replies to complete. + */ + +type TrackedDispatcher = { + readonly id: string; + readonly pending: () => number; + readonly waitForIdle: () => Promise; +}; + +const activeDispatchers = new Set(); +let nextId = 0; + +/** + * Register a reply dispatcher for global tracking. + * Returns an unregister function to call when the dispatcher is no longer needed. + */ +export function registerDispatcher(dispatcher: { + readonly pending: () => number; + readonly waitForIdle: () => Promise; +}): { id: string; unregister: () => void } { + const id = `dispatcher-${++nextId}`; + const tracked: TrackedDispatcher = { + id, + pending: dispatcher.pending, + waitForIdle: dispatcher.waitForIdle, + }; + activeDispatchers.add(tracked); + + const unregister = () => { + activeDispatchers.delete(tracked); + }; + + return { id, unregister }; +} + +/** + * Get the total number of pending replies across all dispatchers. + */ +export function getTotalPendingReplies(): number { + let total = 0; + for (const dispatcher of activeDispatchers) { + total += dispatcher.pending(); + } + return total; +} + +/** + * Clear all registered dispatchers (for testing). + * WARNING: Only use this in test cleanup! + */ +export function clearAllDispatchers(): void { + if (!process.env.VITEST && process.env.NODE_ENV !== "test") { + throw new Error("clearAllDispatchers() is only available in test environments"); + } + activeDispatchers.clear(); +} diff --git a/src/auto-reply/reply/reply-dispatcher.ts b/src/auto-reply/reply/reply-dispatcher.ts index 270efb001e5..9027af0693d 100644 --- a/src/auto-reply/reply/reply-dispatcher.ts +++ b/src/auto-reply/reply/reply-dispatcher.ts @@ -3,6 +3,7 @@ import type { GetReplyOptions, ReplyPayload } from "../types.js"; import type { ResponsePrefixContext } from "./response-prefix-template.js"; import type { TypingController } from "./typing.js"; import { sleep } from "../../utils.js"; +import { registerDispatcher } from "./dispatcher-registry.js"; import { normalizeReplyPayload, type NormalizeReplySkipReason } from "./normalize-reply.js"; export type ReplyDispatchKind = "tool" | "block" | "final"; @@ -74,6 +75,7 @@ export type ReplyDispatcher = { sendFinalReply: (payload: ReplyPayload) => boolean; waitForIdle: () => Promise; getQueuedCounts: () => Record; + markComplete: () => void; }; type NormalizeReplyPayloadInternalOptions = Pick< @@ -101,7 +103,10 @@ function normalizeReplyPayloadInternal( export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDispatcher { let sendChain: Promise = Promise.resolve(); // Track in-flight deliveries so we can emit a reliable "idle" signal. - let pending = 0; + // Start with pending=1 as a "reservation" to prevent premature gateway restart. + // This is decremented when markComplete() is called to signal no more replies will come. + let pending = 1; + let completeCalled = false; // Track whether we've sent a block reply (for human delay - skip delay on first block). let sentFirstBlock = false; // Serialize outbound replies to preserve tool/block/final order. @@ -111,6 +116,12 @@ export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDis final: 0, }; + // Register this dispatcher globally for gateway restart coordination. + const { unregister } = registerDispatcher({ + pending: () => pending, + waitForIdle: () => sendChain, + }); + const enqueue = (kind: ReplyDispatchKind, payload: ReplyPayload) => { const normalized = normalizeReplyPayloadInternal(payload, { responsePrefix: options.responsePrefix, @@ -140,6 +151,8 @@ export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDis await sleep(delayMs); } } + // Safe: deliver is called inside an async .then() callback, so even a synchronous + // throw becomes a rejection that flows through .catch()/.finally(), ensuring cleanup. await options.deliver(normalized, { kind }); }) .catch((err) => { @@ -147,19 +160,49 @@ export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDis }) .finally(() => { pending -= 1; + // Clear reservation if: + // 1. pending is now 1 (just the reservation left) + // 2. markComplete has been called + // 3. No more replies will be enqueued + if (pending === 1 && completeCalled) { + pending -= 1; // Clear the reservation + } if (pending === 0) { + // Unregister from global tracking when idle. + unregister(); options.onIdle?.(); } }); return true; }; + const markComplete = () => { + if (completeCalled) { + return; + } + completeCalled = true; + // If no replies were enqueued (pending is still 1 = just the reservation), + // schedule clearing the reservation after current microtasks complete. + // This gives any in-flight enqueue() calls a chance to increment pending. + void Promise.resolve().then(() => { + if (pending === 1 && completeCalled) { + // Still just the reservation, no replies were enqueued + pending -= 1; + if (pending === 0) { + unregister(); + options.onIdle?.(); + } + } + }); + }; + return { sendToolResult: (payload) => enqueue("tool", payload), sendBlockReply: (payload) => enqueue("block", payload), sendFinalReply: (payload) => enqueue("final", payload), waitForIdle: () => sendChain, getQueuedCounts: () => ({ ...queuedCounts }), + markComplete, }; } diff --git a/src/auto-reply/reply/reply-routing.test.ts b/src/auto-reply/reply/reply-routing.test.ts index 6637c6c1401..3d5179d6c0c 100644 --- a/src/auto-reply/reply/reply-routing.test.ts +++ b/src/auto-reply/reply/reply-routing.test.ts @@ -100,6 +100,8 @@ describe("createReplyDispatcher", () => { dispatcher.sendFinalReply({ text: "two" }); await dispatcher.waitForIdle(); + dispatcher.markComplete(); + await Promise.resolve(); expect(onIdle).toHaveBeenCalledTimes(1); }); diff --git a/src/config/sessions.ts b/src/config/sessions.ts index 20de39409b1..0ea031cf050 100644 --- a/src/config/sessions.ts +++ b/src/config/sessions.ts @@ -7,3 +7,4 @@ export * from "./sessions/session-key.js"; export * from "./sessions/store.js"; export * from "./sessions/types.js"; export * from "./sessions/transcript.js"; +export * from "./sessions/delivery-info.js"; diff --git a/src/config/sessions/delivery-info.ts b/src/config/sessions/delivery-info.ts new file mode 100644 index 00000000000..006f1db4490 --- /dev/null +++ b/src/config/sessions/delivery-info.ts @@ -0,0 +1,46 @@ +import { loadConfig } from "../io.js"; +import { resolveStorePath } from "./paths.js"; +import { loadSessionStore } from "./store.js"; + +/** + * Extract deliveryContext and threadId from a sessionKey. + * Supports both :thread: (most channels) and :topic: (Telegram). + */ +export function extractDeliveryInfo(sessionKey: string | undefined): { + deliveryContext: { channel?: string; to?: string; accountId?: string } | undefined; + threadId: string | undefined; +} { + if (!sessionKey) { + return { deliveryContext: undefined, threadId: undefined }; + } + const topicIndex = sessionKey.lastIndexOf(":topic:"); + const threadIndex = sessionKey.lastIndexOf(":thread:"); + const markerIndex = Math.max(topicIndex, threadIndex); + const marker = topicIndex > threadIndex ? ":topic:" : ":thread:"; + + const baseSessionKey = markerIndex === -1 ? sessionKey : sessionKey.slice(0, markerIndex); + const threadIdRaw = + markerIndex === -1 ? undefined : sessionKey.slice(markerIndex + marker.length); + const threadId = threadIdRaw?.trim() || undefined; + + let deliveryContext: { channel?: string; to?: string; accountId?: string } | undefined; + try { + const cfg = loadConfig(); + const storePath = resolveStorePath(cfg.session?.store); + const store = loadSessionStore(storePath); + let entry = store[sessionKey]; + if (!entry?.deliveryContext && markerIndex !== -1 && baseSessionKey) { + entry = store[baseSessionKey]; + } + if (entry?.deliveryContext) { + deliveryContext = { + channel: entry.deliveryContext.channel, + to: entry.deliveryContext.to, + accountId: entry.deliveryContext.accountId, + }; + } + } catch { + // ignore: best-effort + } + return { deliveryContext, threadId }; +} diff --git a/src/gateway/server-methods/config.ts b/src/gateway/server-methods/config.ts index d4be1a8667e..2e397728c64 100644 --- a/src/gateway/server-methods/config.ts +++ b/src/gateway/server-methods/config.ts @@ -18,6 +18,7 @@ import { restoreRedactedValues, } from "../../config/redact-snapshot.js"; import { buildConfigSchema, type ConfigSchemaResponse } from "../../config/schema.js"; +import { extractDeliveryInfo } from "../../config/sessions.js"; import { formatDoctorNonInteractiveHint, type RestartSentinelPayload, @@ -315,11 +316,17 @@ export const configHandlers: GatewayRequestHandlers = { ? Math.max(0, Math.floor(restartDelayMsRaw)) : undefined; + // Extract deliveryContext + threadId for routing after restart + // Supports both :thread: (most channels) and :topic: (Telegram) + const { deliveryContext, threadId } = extractDeliveryInfo(sessionKey); + const payload: RestartSentinelPayload = { - kind: "config-apply", + kind: "config-patch", status: "ok", ts: Date.now(), sessionKey, + deliveryContext, + threadId, message: note ?? null, doctorHint: formatDoctorNonInteractiveHint(), stats: { @@ -422,11 +429,18 @@ export const configHandlers: GatewayRequestHandlers = { ? Math.max(0, Math.floor(restartDelayMsRaw)) : undefined; + // Extract deliveryContext + threadId for routing after restart + // Supports both :thread: (most channels) and :topic: (Telegram) + const { deliveryContext: deliveryContextApply, threadId: threadIdApply } = + extractDeliveryInfo(sessionKey); + const payload: RestartSentinelPayload = { kind: "config-apply", status: "ok", ts: Date.now(), sessionKey, + deliveryContext: deliveryContextApply, + threadId: threadIdApply, message: note ?? null, doctorHint: formatDoctorNonInteractiveHint(), stats: { diff --git a/src/gateway/server-reload-handlers.ts b/src/gateway/server-reload-handlers.ts index 393a38cf778..02ec35bc306 100644 --- a/src/gateway/server-reload-handlers.ts +++ b/src/gateway/server-reload-handlers.ts @@ -2,15 +2,14 @@ import type { CliDeps } from "../cli/deps.js"; import type { loadConfig } from "../config/config.js"; import type { HeartbeatRunner } from "../infra/heartbeat-runner.js"; import type { ChannelKind, GatewayReloadPlan } from "./config-reload.js"; +import { getActiveEmbeddedRunCount } from "../agents/pi-embedded-runner/runs.js"; +import { getTotalPendingReplies } from "../auto-reply/reply/dispatcher-registry.js"; import { resolveAgentMaxConcurrent, resolveSubagentMaxConcurrent } from "../config/agent-limits.js"; import { startGmailWatcher, stopGmailWatcher } from "../hooks/gmail-watcher.js"; import { isTruthyEnvValue } from "../infra/env.js"; import { resetDirectoryCache } from "../infra/outbound/target-resolver.js"; -import { - authorizeGatewaySigusr1Restart, - setGatewaySigusr1RestartPolicy, -} from "../infra/restart.js"; -import { setCommandLaneConcurrency } from "../process/command-queue.js"; +import { emitGatewayRestart, setGatewaySigusr1RestartPolicy } from "../infra/restart.js"; +import { setCommandLaneConcurrency, getTotalQueueSize } from "../process/command-queue.js"; import { CommandLane } from "../process/lanes.js"; import { resolveHooksConfig } from "./hooks.js"; import { startBrowserControlServerIfEnabled } from "./server-browser.js"; @@ -140,6 +139,8 @@ export function createGatewayReloadHandlers(params: { params.setState(nextState); }; + let restartPending = false; + const requestGatewayRestart = ( plan: GatewayReloadPlan, nextConfig: ReturnType, @@ -148,13 +149,85 @@ export function createGatewayReloadHandlers(params: { const reasons = plan.restartReasons.length ? plan.restartReasons.join(", ") : plan.changedPaths.join(", "); - params.logReload.warn(`config change requires gateway restart (${reasons})`); + if (process.listenerCount("SIGUSR1") === 0) { params.logReload.warn("no SIGUSR1 listener found; restart skipped"); return; } - authorizeGatewaySigusr1Restart(); - process.emit("SIGUSR1"); + + // Check if there are active operations (commands in queue, pending replies, or embedded runs) + const queueSize = getTotalQueueSize(); + const pendingReplies = getTotalPendingReplies(); + const embeddedRuns = getActiveEmbeddedRunCount(); + const totalActive = queueSize + pendingReplies + embeddedRuns; + + if (totalActive > 0) { + // Avoid spinning up duplicate polling loops from repeated config changes. + if (restartPending) { + params.logReload.info( + `config change requires gateway restart (${reasons}) — already waiting for operations to complete`, + ); + return; + } + restartPending = true; + const details = []; + if (queueSize > 0) { + details.push(`${queueSize} queued operation(s)`); + } + if (pendingReplies > 0) { + details.push(`${pendingReplies} pending reply(ies)`); + } + if (embeddedRuns > 0) { + details.push(`${embeddedRuns} embedded run(s)`); + } + params.logReload.warn( + `config change requires gateway restart (${reasons}) — deferring until ${details.join(", ")} complete`, + ); + + // Wait for all operations and replies to complete before restarting (max 30 seconds) + const maxWaitMs = 30_000; + const checkIntervalMs = 500; + const startTime = Date.now(); + + const checkAndRestart = () => { + const currentQueueSize = getTotalQueueSize(); + const currentPendingReplies = getTotalPendingReplies(); + const currentEmbeddedRuns = getActiveEmbeddedRunCount(); + const currentTotalActive = currentQueueSize + currentPendingReplies + currentEmbeddedRuns; + const elapsed = Date.now() - startTime; + + if (currentTotalActive === 0) { + restartPending = false; + params.logReload.info("all operations and replies completed; restarting gateway now"); + emitGatewayRestart(); + } else if (elapsed >= maxWaitMs) { + const remainingDetails = []; + if (currentQueueSize > 0) { + remainingDetails.push(`${currentQueueSize} operation(s)`); + } + if (currentPendingReplies > 0) { + remainingDetails.push(`${currentPendingReplies} reply(ies)`); + } + if (currentEmbeddedRuns > 0) { + remainingDetails.push(`${currentEmbeddedRuns} embedded run(s)`); + } + restartPending = false; + params.logReload.warn( + `restart timeout after ${elapsed}ms with ${remainingDetails.join(", ")} still active; restarting anyway`, + ); + emitGatewayRestart(); + } else { + // Check again soon + setTimeout(checkAndRestart, checkIntervalMs); + } + }; + + setTimeout(checkAndRestart, checkIntervalMs); + } else { + // No active operations or pending replies, restart immediately + params.logReload.warn(`config change requires gateway restart (${reasons})`); + emitGatewayRestart(); + } }; return { applyHotReload, requestGatewayRestart }; diff --git a/src/gateway/server-reload.config-during-reply.test.ts b/src/gateway/server-reload.config-during-reply.test.ts new file mode 100644 index 00000000000..2ae95be5557 --- /dev/null +++ b/src/gateway/server-reload.config-during-reply.test.ts @@ -0,0 +1,151 @@ +/** + * E2E test for config reload during active reply sending. + * Tests that gateway restart is properly deferred until replies are sent. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + clearAllDispatchers, + getTotalPendingReplies, +} from "../auto-reply/reply/dispatcher-registry.js"; + +// Helper to flush all pending microtasks +async function flushMicrotasks() { + for (let i = 0; i < 10; i++) { + await Promise.resolve(); + } +} + +describe("gateway config reload during reply", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + // Wait for any pending microtasks (from markComplete()) to complete + await flushMicrotasks(); + clearAllDispatchers(); + }); + + it("should defer restart until reply dispatcher completes", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + const { getTotalQueueSize } = await import("../process/command-queue.js"); + + // Create a dispatcher (simulating message handling) + let deliveredReplies: string[] = []; + const dispatcher = createReplyDispatcher({ + deliver: async (payload) => { + // Simulate async reply delivery + await new Promise((resolve) => setTimeout(resolve, 100)); + deliveredReplies.push(payload.text ?? ""); + }, + onError: (err) => { + throw err; + }, + }); + + // Initially: pending=1 (reservation) + expect(getTotalPendingReplies()).toBe(1); + + // Simulate command finishing and enqueuing reply + dispatcher.sendFinalReply({ text: "Configuration updated successfully!" }); + + // Now: pending=2 (reservation + 1 enqueued reply) + expect(getTotalPendingReplies()).toBe(2); + + // Mark dispatcher complete (flags reservation for cleanup on last delivery) + dispatcher.markComplete(); + + // Reservation is still counted until the delivery .finally() clears it, + // but the important invariant is pending > 0 while delivery is in flight. + expect(getTotalPendingReplies()).toBeGreaterThan(0); + + // At this point, if gateway restart was requested, it should defer + // because getTotalPendingReplies() > 0 + + // Wait for reply to be delivered + await dispatcher.waitForIdle(); + + // Now: pending=0 (reply sent) + expect(getTotalPendingReplies()).toBe(0); + expect(deliveredReplies).toEqual(["Configuration updated successfully!"]); + + // Now restart can proceed safely + expect(getTotalQueueSize()).toBe(0); + expect(getTotalPendingReplies()).toBe(0); + }); + + it("should handle dispatcher reservation correctly when no replies sent", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + + let deliverCalled = false; + const dispatcher = createReplyDispatcher({ + deliver: async () => { + deliverCalled = true; + }, + }); + + // Initially: pending=1 (reservation) + expect(getTotalPendingReplies()).toBe(1); + + // Mark complete without sending any replies + dispatcher.markComplete(); + + // Reservation is cleared via microtask — flush it + await flushMicrotasks(); + + // Now: pending=0 (reservation cleared, no replies were enqueued) + expect(getTotalPendingReplies()).toBe(0); + + // Wait for idle (should resolve immediately since no replies) + await dispatcher.waitForIdle(); + + expect(deliverCalled).toBe(false); + expect(getTotalPendingReplies()).toBe(0); + }); + + it("should integrate dispatcher reservation with concurrent dispatchers", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + const { getTotalQueueSize } = await import("../process/command-queue.js"); + + const deliveredReplies: string[] = []; + const dispatcher = createReplyDispatcher({ + deliver: async (payload) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + deliveredReplies.push(payload.text ?? ""); + }, + }); + + // Dispatcher has reservation (pending=1) + expect(getTotalPendingReplies()).toBe(1); + + // Total active = queue + pending + const totalActive = getTotalQueueSize() + getTotalPendingReplies(); + expect(totalActive).toBe(1); // 0 queue + 1 pending + + // Command finishes, replies enqueued + dispatcher.sendFinalReply({ text: "Reply 1" }); + dispatcher.sendFinalReply({ text: "Reply 2" }); + + // Now: pending=3 (reservation + 2 replies) + expect(getTotalPendingReplies()).toBe(3); + + // Mark complete (flags reservation for cleanup on last delivery) + dispatcher.markComplete(); + + // Reservation still counted until delivery .finally() clears it, + // but the important invariant is pending > 0 while deliveries are in flight. + expect(getTotalPendingReplies()).toBeGreaterThan(0); + + // Wait for replies + await dispatcher.waitForIdle(); + + // Replies sent, pending=0 + expect(getTotalPendingReplies()).toBe(0); + expect(deliveredReplies).toEqual(["Reply 1", "Reply 2"]); + + // Now everything is idle + expect(getTotalPendingReplies()).toBe(0); + expect(getTotalQueueSize()).toBe(0); + }); +}); diff --git a/src/gateway/server-reload.integration.test.ts b/src/gateway/server-reload.integration.test.ts new file mode 100644 index 00000000000..d2ab045fac3 --- /dev/null +++ b/src/gateway/server-reload.integration.test.ts @@ -0,0 +1,199 @@ +/** + * Integration test simulating full message handling + config change + reply flow. + * This tests the complete scenario where a user configures an adapter via chat + * and ensures they get a reply before the gateway restarts. + */ +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; + +describe("gateway restart deferral integration", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + // Wait for any pending microtasks (from markComplete()) to complete + await Promise.resolve(); + const { clearAllDispatchers } = await import("../auto-reply/reply/dispatcher-registry.js"); + clearAllDispatchers(); + }); + + it("should defer restart until dispatcher completes with reply", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); + const { getTotalQueueSize } = await import("../process/command-queue.js"); + + const events: string[] = []; + + // T=0: Message received — dispatcher created (pending=1 reservation) + events.push("message-received"); + const deliveredReplies: Array<{ text: string; timestamp: number }> = []; + const dispatcher = createReplyDispatcher({ + deliver: async (payload) => { + // Simulate network delay + await new Promise((resolve) => setTimeout(resolve, 100)); + deliveredReplies.push({ + text: payload.text ?? "", + timestamp: Date.now(), + }); + events.push(`reply-delivered: ${payload.text}`); + }, + }); + events.push("dispatcher-created"); + + // T=1: Config change detected + events.push("config-change-detected"); + + // Check if restart should be deferred + const queueSize = getTotalQueueSize(); + const pendingReplies = getTotalPendingReplies(); + const totalActive = queueSize + pendingReplies; + + events.push(`defer-check: queue=${queueSize} pending=${pendingReplies} total=${totalActive}`); + + // Should defer because dispatcher has reservation + expect(totalActive).toBeGreaterThan(0); + expect(pendingReplies).toBe(1); // reservation + + if (totalActive > 0) { + events.push("restart-deferred"); + } + + // T=2: Command finishes, enqueue replies + dispatcher.sendFinalReply({ text: "Adapter configured successfully!" }); + dispatcher.sendFinalReply({ text: "Gateway will restart to apply changes." }); + events.push("replies-enqueued"); + + // Now pending should be 3 (reservation + 2 replies) + expect(getTotalPendingReplies()).toBe(3); + + // Mark command complete (flags reservation for cleanup on last delivery) + dispatcher.markComplete(); + events.push("command-complete"); + + // Reservation still counted until delivery .finally() clears it, + // but the important invariant is pending > 0 while deliveries are in flight. + expect(getTotalPendingReplies()).toBeGreaterThan(0); + + // T=3: Wait for replies to be delivered + await dispatcher.waitForIdle(); + events.push("dispatcher-idle"); + + // Replies should be delivered + expect(deliveredReplies).toHaveLength(2); + expect(deliveredReplies[0].text).toBe("Adapter configured successfully!"); + expect(deliveredReplies[1].text).toBe("Gateway will restart to apply changes."); + + // Pending should be 0 + expect(getTotalPendingReplies()).toBe(0); + + // T=4: Check if restart can proceed + const finalQueueSize = getTotalQueueSize(); + const finalPendingReplies = getTotalPendingReplies(); + const finalTotalActive = finalQueueSize + finalPendingReplies; + + events.push( + `restart-check: queue=${finalQueueSize} pending=${finalPendingReplies} total=${finalTotalActive}`, + ); + + // Everything should be idle now + expect(finalTotalActive).toBe(0); + events.push("restart-can-proceed"); + + // Verify event sequence + expect(events).toEqual([ + "message-received", + "dispatcher-created", + "config-change-detected", + "defer-check: queue=0 pending=1 total=1", + "restart-deferred", + "replies-enqueued", + "command-complete", + "reply-delivered: Adapter configured successfully!", + "reply-delivered: Gateway will restart to apply changes.", + "dispatcher-idle", + "restart-check: queue=0 pending=0 total=0", + "restart-can-proceed", + ]); + }); + + it("should handle concurrent dispatchers with config changes", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); + + // Simulate two messages being processed concurrently + const deliveredReplies: string[] = []; + + // Message 1 — dispatcher created + const dispatcher1 = createReplyDispatcher({ + deliver: async (payload) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + deliveredReplies.push(`msg1: ${payload.text}`); + }, + }); + + // Message 2 — dispatcher created + const dispatcher2 = createReplyDispatcher({ + deliver: async (payload) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + deliveredReplies.push(`msg2: ${payload.text}`); + }, + }); + + // Both dispatchers have reservations + expect(getTotalPendingReplies()).toBe(2); + + // Config change detected - should defer + const totalActive = getTotalPendingReplies(); + expect(totalActive).toBe(2); // 2 dispatcher reservations + + // Messages process and send replies + dispatcher1.sendFinalReply({ text: "Reply from message 1" }); + dispatcher1.markComplete(); + + dispatcher2.sendFinalReply({ text: "Reply from message 2" }); + dispatcher2.markComplete(); + + // Wait for both + await Promise.all([dispatcher1.waitForIdle(), dispatcher2.waitForIdle()]); + + // All idle + expect(getTotalPendingReplies()).toBe(0); + + // Replies delivered + expect(deliveredReplies).toHaveLength(2); + }); + + it("should handle rapid config changes without losing replies", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); + + const deliveredReplies: string[] = []; + + // Message received — dispatcher created + const dispatcher = createReplyDispatcher({ + deliver: async (payload) => { + await new Promise((resolve) => setTimeout(resolve, 200)); // Slow network + deliveredReplies.push(payload.text ?? ""); + }, + }); + + // Config change 1, 2, 3 (rapid changes) + // All should be deferred because dispatcher has pending replies + + // Send replies + dispatcher.sendFinalReply({ text: "Processing..." }); + dispatcher.sendFinalReply({ text: "Almost done..." }); + dispatcher.sendFinalReply({ text: "Complete!" }); + dispatcher.markComplete(); + + // Wait for all replies + await dispatcher.waitForIdle(); + + // All replies should be delivered + expect(deliveredReplies).toEqual(["Processing...", "Almost done...", "Complete!"]); + + // Now restart can proceed + expect(getTotalPendingReplies()).toBe(0); + }); +}); diff --git a/src/gateway/server-reload.real-scenario.test.ts b/src/gateway/server-reload.real-scenario.test.ts new file mode 100644 index 00000000000..c3da2723f4e --- /dev/null +++ b/src/gateway/server-reload.real-scenario.test.ts @@ -0,0 +1,121 @@ +/** + * REAL scenario test - simulates actual message handling with config changes. + * This test MUST fail if "imsg rpc not running" would occur in production. + */ +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; + +describe("real scenario: config change during message processing", () => { + let replyErrors: string[] = []; + + beforeEach(() => { + vi.clearAllMocks(); + replyErrors = []; + }); + + afterEach(async () => { + vi.restoreAllMocks(); + // Wait for any pending microtasks (from markComplete()) to complete + await Promise.resolve(); + const { clearAllDispatchers } = await import("../auto-reply/reply/dispatcher-registry.js"); + clearAllDispatchers(); + }); + + it("should NOT restart gateway while reply delivery is in flight", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); + + let rpcConnected = true; + const deliveredReplies: string[] = []; + + // Create dispatcher with slow delivery (simulates real network delay) + const dispatcher = createReplyDispatcher({ + deliver: async (payload) => { + if (!rpcConnected) { + const error = "Error: imsg rpc not running"; + replyErrors.push(error); + throw new Error(error); + } + // Slow delivery — restart checks will run during this window + await new Promise((resolve) => setTimeout(resolve, 500)); + deliveredReplies.push(payload.text ?? ""); + }, + onError: () => { + // Swallow delivery errors so the test can assert on replyErrors + }, + }); + + // Enqueue reply and immediately clear the reservation. + // This is the critical sequence: after markComplete(), the ONLY thing + // keeping pending > 0 is the in-flight delivery itself. + dispatcher.sendFinalReply({ text: "Configuration updated!" }); + dispatcher.markComplete(); + + // At this point: markComplete flagged, delivery is in flight. + // pending > 0 because the in-flight delivery keeps it alive. + const pendingDuringDelivery = getTotalPendingReplies(); + expect(pendingDuringDelivery).toBeGreaterThan(0); + + // Simulate restart checks while delivery is in progress. + // If the tracking is broken, pending would be 0 and we'd restart. + let restartTriggered = false; + for (let i = 0; i < 3; i++) { + await new Promise((resolve) => setTimeout(resolve, 100)); + const pending = getTotalPendingReplies(); + if (pending === 0) { + restartTriggered = true; + rpcConnected = false; + break; + } + } + + // Wait for delivery to complete + await dispatcher.waitForIdle(); + + // Now pending should be 0 — restart can proceed + expect(getTotalPendingReplies()).toBe(0); + + // CRITICAL: delivery must have succeeded without RPC being killed + expect(restartTriggered).toBe(false); + expect(replyErrors).toEqual([]); + expect(deliveredReplies).toEqual(["Configuration updated!"]); + }); + + it("should keep pending > 0 until reply is actually enqueued", async () => { + const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); + const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); + + const dispatcher = createReplyDispatcher({ + deliver: async (_payload) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + }, + }); + + // Initially: pending=1 (reservation) + expect(getTotalPendingReplies()).toBe(1); + + // Simulate command processing delay BEFORE reply is enqueued + await new Promise((resolve) => setTimeout(resolve, 100)); + + // During this delay, pending should STILL be 1 (reservation active) + expect(getTotalPendingReplies()).toBe(1); + + // Now enqueue reply + dispatcher.sendFinalReply({ text: "Reply" }); + + // Now pending should be 2 (reservation + reply) + expect(getTotalPendingReplies()).toBe(2); + + // Mark complete + dispatcher.markComplete(); + + // After markComplete, pending should still be > 0 if reply hasn't sent yet + const pendingAfterMarkComplete = getTotalPendingReplies(); + expect(pendingAfterMarkComplete).toBeGreaterThan(0); + + // Wait for reply to send + await dispatcher.waitForIdle(); + + // Now pending should be 0 + expect(getTotalPendingReplies()).toBe(0); + }); +}); diff --git a/src/gateway/server-restart-sentinel.ts b/src/gateway/server-restart-sentinel.ts index 2600a0b6380..901465b5684 100644 --- a/src/gateway/server-restart-sentinel.ts +++ b/src/gateway/server-restart-sentinel.ts @@ -1,8 +1,8 @@ import type { CliDeps } from "../cli/deps.js"; import { resolveAnnounceTargetFromKey } from "../agents/tools/sessions-send-helpers.js"; import { normalizeChannelId } from "../channels/plugins/index.js"; -import { agentCommand } from "../commands/agent.js"; import { resolveMainSessionKeyFromConfig } from "../config/sessions.js"; +import { deliverOutboundPayloads } from "../infra/outbound/deliver.js"; import { resolveOutboundTarget } from "../infra/outbound/targets.js"; import { consumeRestartSentinel, @@ -10,11 +10,10 @@ import { summarizeRestartSentinel, } from "../infra/restart-sentinel.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; -import { defaultRuntime } from "../runtime.js"; import { deliveryContextFromSession, mergeDeliveryContext } from "../utils/delivery-context.js"; import { loadSessionEntry } from "./session-utils.js"; -export async function scheduleRestartSentinelWake(params: { deps: CliDeps }) { +export async function scheduleRestartSentinelWake(_params: { deps: CliDeps }) { const sentinel = await consumeRestartSentinel(); if (!sentinel) { return; @@ -86,20 +85,15 @@ export async function scheduleRestartSentinelWake(params: { deps: CliDeps }) { (origin?.threadId != null ? String(origin.threadId) : undefined); try { - await agentCommand( - { - message, - sessionKey, - to: resolved.to, - channel, - deliver: true, - bestEffortDeliver: true, - messageChannel: channel, - threadId, - }, - defaultRuntime, - params.deps, - ); + await deliverOutboundPayloads({ + cfg, + channel, + to: resolved.to, + accountId: origin?.accountId, + threadId, + payloads: [{ text: message }], + bestEffort: true, + }); } catch (err) { enqueueSystemEvent(`${summary}\n${String(err)}`, { sessionKey }); } diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 3146c0c6deb..7cc895df499 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -5,8 +5,10 @@ import type { RuntimeEnv } from "../runtime.js"; import type { ControlUiRootState } from "./control-ui.js"; import type { startBrowserControlServerIfEnabled } from "./server-browser.js"; import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../agents/agent-scope.js"; +import { getActiveEmbeddedRunCount } from "../agents/pi-embedded-runner/runs.js"; import { registerSkillsChangeListener } from "../agents/skills/refresh.js"; import { initSubagentRegistry } from "../agents/subagent-registry.js"; +import { getTotalPendingReplies } from "../auto-reply/reply/dispatcher-registry.js"; import { type ChannelId, listChannelPlugins } from "../channels/plugins/index.js"; import { formatCliCommand } from "../cli/command-format.js"; import { createDefaultDeps } from "../cli/deps.js"; @@ -32,7 +34,7 @@ import { onHeartbeatEvent } from "../infra/heartbeat-events.js"; import { startHeartbeatRunner } from "../infra/heartbeat-runner.js"; import { getMachineDisplayName } from "../infra/machine-name.js"; import { ensureOpenClawCliOnPath } from "../infra/path-env.js"; -import { setGatewaySigusr1RestartPolicy } from "../infra/restart.js"; +import { setGatewaySigusr1RestartPolicy, setPreRestartDeferralCheck } from "../infra/restart.js"; import { primeRemoteSkillsCache, refreshRemoteBinsForConnectedNodes, @@ -42,6 +44,7 @@ import { scheduleGatewayUpdateCheck } from "../infra/update-startup.js"; import { startDiagnosticHeartbeat, stopDiagnosticHeartbeat } from "../logging/diagnostic.js"; import { createSubsystemLogger, runtimeForLogger } from "../logging/subsystem.js"; import { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; +import { getTotalQueueSize } from "../process/command-queue.js"; import { runOnboardingWizard } from "../wizard/onboarding.js"; import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.js"; import { startGatewayConfigReloader } from "./config-reload.js"; @@ -225,6 +228,9 @@ export async function startGatewayServer( startDiagnosticHeartbeat(); } setGatewaySigusr1RestartPolicy({ allowExternal: cfgAtStart.commands?.restart === true }); + setPreRestartDeferralCheck( + () => getTotalQueueSize() + getTotalPendingReplies() + getActiveEmbeddedRunCount(), + ); initSubagentRegistry(); const defaultAgentId = resolveDefaultAgentId(cfgAtStart); const defaultWorkspaceDir = resolveAgentWorkspaceDir(cfgAtStart, defaultAgentId); diff --git a/src/imessage/monitor/monitor-provider.ts b/src/imessage/monitor/monitor-provider.ts index a9e0d93f7cc..445fe73aeae 100644 --- a/src/imessage/monitor/monitor-provider.ts +++ b/src/imessage/monitor/monitor-provider.ts @@ -659,6 +659,7 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P onModelSelected, }, }); + if (!queuedFinal) { if (isGroup && historyKey) { clearHistoryEntriesIfEnabled({ diff --git a/src/infra/infra-runtime.test.ts b/src/infra/infra-runtime.test.ts index 926c1f224c6..61e7dff4393 100644 --- a/src/infra/infra-runtime.test.ts +++ b/src/infra/infra-runtime.test.ts @@ -9,6 +9,7 @@ import { isGatewaySigusr1RestartExternallyAllowed, scheduleGatewaySigusr1Restart, setGatewaySigusr1RestartPolicy, + setPreRestartDeferralCheck, } from "./restart.js"; import { createTelegramRetryRunner } from "./retry-policy.js"; import { getShellPathFromLoginShell, resetShellPathCacheForTests } from "./shell-env.js"; @@ -79,11 +80,15 @@ describe("infra runtime", () => { __testing.resetSigusr1State(); }); - it("consumes a scheduled authorization once", async () => { + it("authorizes exactly once when scheduled restart emits", async () => { expect(consumeGatewaySigusr1RestartAuthorization()).toBe(false); scheduleGatewaySigusr1Restart({ delayMs: 0 }); + // No pre-authorization before the scheduled emission fires. + expect(consumeGatewaySigusr1RestartAuthorization()).toBe(false); + await vi.advanceTimersByTimeAsync(0); + expect(consumeGatewaySigusr1RestartAuthorization()).toBe(true); expect(consumeGatewaySigusr1RestartAuthorization()).toBe(false); @@ -97,6 +102,110 @@ describe("infra runtime", () => { }); }); + describe("pre-restart deferral check", () => { + beforeEach(() => { + __testing.resetSigusr1State(); + vi.useFakeTimers(); + vi.spyOn(process, "kill").mockImplementation(() => true); + }); + + afterEach(async () => { + await vi.runOnlyPendingTimersAsync(); + vi.useRealTimers(); + vi.restoreAllMocks(); + __testing.resetSigusr1State(); + }); + + it("emits SIGUSR1 immediately when no deferral check is registered", async () => { + const emitSpy = vi.spyOn(process, "emit"); + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + scheduleGatewaySigusr1Restart({ delayMs: 0 }); + await vi.advanceTimersByTimeAsync(0); + expect(emitSpy).toHaveBeenCalledWith("SIGUSR1"); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); + + it("emits SIGUSR1 immediately when deferral check returns 0", async () => { + const emitSpy = vi.spyOn(process, "emit"); + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + setPreRestartDeferralCheck(() => 0); + scheduleGatewaySigusr1Restart({ delayMs: 0 }); + await vi.advanceTimersByTimeAsync(0); + expect(emitSpy).toHaveBeenCalledWith("SIGUSR1"); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); + + it("defers SIGUSR1 until deferral check returns 0", async () => { + const emitSpy = vi.spyOn(process, "emit"); + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + let pending = 2; + setPreRestartDeferralCheck(() => pending); + scheduleGatewaySigusr1Restart({ delayMs: 0 }); + + // After initial delay fires, deferral check returns 2 — should NOT emit yet + await vi.advanceTimersByTimeAsync(0); + expect(emitSpy).not.toHaveBeenCalledWith("SIGUSR1"); + + // After one poll (500ms), still pending + await vi.advanceTimersByTimeAsync(500); + expect(emitSpy).not.toHaveBeenCalledWith("SIGUSR1"); + + // Drain pending work + pending = 0; + await vi.advanceTimersByTimeAsync(500); + expect(emitSpy).toHaveBeenCalledWith("SIGUSR1"); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); + + it("emits SIGUSR1 after deferral timeout even if still pending", async () => { + const emitSpy = vi.spyOn(process, "emit"); + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + setPreRestartDeferralCheck(() => 5); // always pending + scheduleGatewaySigusr1Restart({ delayMs: 0 }); + + // Fire initial timeout + await vi.advanceTimersByTimeAsync(0); + expect(emitSpy).not.toHaveBeenCalledWith("SIGUSR1"); + + // Advance past the 30s max deferral wait + await vi.advanceTimersByTimeAsync(30_000); + expect(emitSpy).toHaveBeenCalledWith("SIGUSR1"); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); + + it("emits SIGUSR1 if deferral check throws", async () => { + const emitSpy = vi.spyOn(process, "emit"); + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + setPreRestartDeferralCheck(() => { + throw new Error("boom"); + }); + scheduleGatewaySigusr1Restart({ delayMs: 0 }); + await vi.advanceTimersByTimeAsync(0); + expect(emitSpy).toHaveBeenCalledWith("SIGUSR1"); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); + }); + describe("getShellPathFromLoginShell", () => { afterEach(() => resetShellPathCacheForTests()); diff --git a/src/infra/restart-sentinel.test.ts b/src/infra/restart-sentinel.test.ts index 638d389f561..5c1fa60632b 100644 --- a/src/infra/restart-sentinel.test.ts +++ b/src/infra/restart-sentinel.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { consumeRestartSentinel, + formatRestartSentinelMessage, readRestartSentinel, resolveRestartSentinelPath, trimLogTail, @@ -61,6 +62,40 @@ describe("restart sentinel", () => { await expect(fs.stat(filePath)).rejects.toThrow(); }); + it("formatRestartSentinelMessage uses custom message when present", () => { + const payload = { + kind: "config-apply" as const, + status: "ok" as const, + ts: Date.now(), + message: "Config updated successfully", + }; + expect(formatRestartSentinelMessage(payload)).toBe("Config updated successfully"); + }); + + it("formatRestartSentinelMessage falls back to summary when no message", () => { + const payload = { + kind: "update" as const, + status: "ok" as const, + ts: Date.now(), + stats: { mode: "git" }, + }; + const result = formatRestartSentinelMessage(payload); + expect(result).toContain("Gateway restart"); + expect(result).toContain("update"); + expect(result).toContain("ok"); + }); + + it("formatRestartSentinelMessage falls back to summary for blank message", () => { + const payload = { + kind: "restart" as const, + status: "ok" as const, + ts: Date.now(), + message: " ", + }; + const result = formatRestartSentinelMessage(payload); + expect(result).toContain("Gateway restart"); + }); + it("trims log tails", () => { const text = "a".repeat(9000); const trimmed = trimLogTail(text, 8000); diff --git a/src/infra/restart-sentinel.ts b/src/infra/restart-sentinel.ts index 1f3b13094f9..8405426cbd6 100644 --- a/src/infra/restart-sentinel.ts +++ b/src/infra/restart-sentinel.ts @@ -28,7 +28,7 @@ export type RestartSentinelStats = { }; export type RestartSentinelPayload = { - kind: "config-apply" | "update" | "restart"; + kind: "config-apply" | "config-patch" | "update" | "restart"; status: "ok" | "error" | "skipped"; ts: number; sessionKey?: string; @@ -109,7 +109,10 @@ export async function consumeRestartSentinel( } export function formatRestartSentinelMessage(payload: RestartSentinelPayload): string { - return `GatewayRestart:\n${JSON.stringify(payload, null, 2)}`; + if (payload.message?.trim()) { + return payload.message.trim(); + } + return summarizeRestartSentinel(payload); } export function summarizeRestartSentinel(payload: RestartSentinelPayload): string { diff --git a/src/infra/restart.ts b/src/infra/restart.ts index d671c112b53..830d0731049 100644 --- a/src/infra/restart.ts +++ b/src/infra/restart.ts @@ -17,6 +17,40 @@ const SIGUSR1_AUTH_GRACE_MS = 5000; let sigusr1AuthorizedCount = 0; let sigusr1AuthorizedUntil = 0; let sigusr1ExternalAllowed = false; +let preRestartCheck: (() => number) | null = null; +let sigusr1Emitted = false; + +/** + * Register a callback that scheduleGatewaySigusr1Restart checks before emitting SIGUSR1. + * The callback should return the number of pending items (0 = safe to restart). + */ +export function setPreRestartDeferralCheck(fn: () => number): void { + preRestartCheck = fn; +} + +/** + * Emit an authorized SIGUSR1 gateway restart, guarded against duplicate emissions. + * Returns true if SIGUSR1 was emitted, false if a restart was already emitted. + * Both scheduleGatewaySigusr1Restart and the config watcher should use this + * to ensure only one restart fires. + */ +export function emitGatewayRestart(): boolean { + if (sigusr1Emitted) { + return false; + } + sigusr1Emitted = true; + authorizeGatewaySigusr1Restart(); + try { + if (process.listenerCount("SIGUSR1") > 0) { + process.emit("SIGUSR1"); + } else { + process.kill(process.pid, "SIGUSR1"); + } + } catch { + /* ignore */ + } + return true; +} function resetSigusr1AuthorizationIfExpired(now = Date.now()) { if (sigusr1AuthorizedCount <= 0) { @@ -37,7 +71,7 @@ export function isGatewaySigusr1RestartExternallyAllowed() { return sigusr1ExternalAllowed; } -export function authorizeGatewaySigusr1Restart(delayMs = 0) { +function authorizeGatewaySigusr1Restart(delayMs = 0) { const delay = Math.max(0, Math.floor(delayMs)); const expiresAt = Date.now() + delay + SIGUSR1_AUTH_GRACE_MS; sigusr1AuthorizedCount += 1; @@ -51,6 +85,10 @@ export function consumeGatewaySigusr1RestartAuthorization(): boolean { if (sigusr1AuthorizedCount <= 0) { return false; } + // Reset the emission guard so the next restart cycle can fire. + // The run loop re-enters startGatewayServer() after close(), which + // re-registers setPreRestartDeferralCheck and can schedule new restarts. + sigusr1Emitted = false; sigusr1AuthorizedCount -= 1; if (sigusr1AuthorizedCount <= 0) { sigusr1AuthorizedUntil = 0; @@ -189,27 +227,48 @@ export function scheduleGatewaySigusr1Restart(opts?: { typeof opts?.reason === "string" && opts.reason.trim() ? opts.reason.trim().slice(0, 200) : undefined; - authorizeGatewaySigusr1Restart(delayMs); - const pid = process.pid; - const hasListener = process.listenerCount("SIGUSR1") > 0; + const DEFERRAL_POLL_MS = 500; + const DEFERRAL_MAX_WAIT_MS = 30_000; + setTimeout(() => { - try { - if (hasListener) { - process.emit("SIGUSR1"); - } else { - process.kill(pid, "SIGUSR1"); - } - } catch { - /* ignore */ + if (!preRestartCheck) { + emitGatewayRestart(); + return; } + let pending: number; + try { + pending = preRestartCheck(); + } catch { + emitGatewayRestart(); + return; + } + if (pending <= 0) { + emitGatewayRestart(); + return; + } + // Poll until pending work drains or timeout + let waited = 0; + const poll = setInterval(() => { + waited += DEFERRAL_POLL_MS; + let current: number; + try { + current = preRestartCheck!(); + } catch { + current = 0; + } + if (current <= 0 || waited >= DEFERRAL_MAX_WAIT_MS) { + clearInterval(poll); + emitGatewayRestart(); + } + }, DEFERRAL_POLL_MS); }, delayMs); return { ok: true, - pid, + pid: process.pid, signal: "SIGUSR1", delayMs, reason, - mode: hasListener ? "emit" : "signal", + mode: process.listenerCount("SIGUSR1") > 0 ? "emit" : "signal", }; } @@ -218,5 +277,7 @@ export const __testing = { sigusr1AuthorizedCount = 0; sigusr1AuthorizedUntil = 0; sigusr1ExternalAllowed = false; + preRestartCheck = null; + sigusr1Emitted = false; }, }; From e794ef047832e548da7a01b7c1c348abfd5bb972 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 23:30:35 +0000 Subject: [PATCH 0016/1038] perf(test): reduce hot-suite setup and duplicate test work --- src/auto-reply/reply.block-streaming.test.ts | 68 +----------- src/auto-reply/reply.raw-body.test.ts | 33 +----- src/infra/transport-ready.test.ts | 20 ++-- src/memory/index.test.ts | 78 ++++++-------- src/memory/manager.batch.test.ts | 105 +++---------------- 5 files changed, 65 insertions(+), 239 deletions(-) diff --git a/src/auto-reply/reply.block-streaming.test.ts b/src/auto-reply/reply.block-streaming.test.ts index 4d4fd8d1c8e..e051944dc9e 100644 --- a/src/auto-reply/reply.block-streaming.test.ts +++ b/src/auto-reply/reply.block-streaming.test.ts @@ -98,69 +98,7 @@ describe("block streaming", () => { ]); }); - it("waits for block replies before returning final payloads", async () => { - await withTempHome(async (home) => { - let releaseTyping: (() => void) | undefined; - const typingGate = new Promise((resolve) => { - releaseTyping = resolve; - }); - let resolveOnReplyStart: (() => void) | undefined; - const onReplyStartCalled = new Promise((resolve) => { - resolveOnReplyStart = resolve; - }); - const onReplyStart = vi.fn(() => { - resolveOnReplyStart?.(); - return typingGate; - }); - const onBlockReply = vi.fn().mockResolvedValue(undefined); - - const impl = async (params: RunEmbeddedPiAgentParams) => { - void params.onBlockReply?.({ text: "hello" }); - return { - payloads: [{ text: "hello" }], - meta: { - durationMs: 5, - agentMeta: { sessionId: "s", provider: "p", model: "m" }, - }, - }; - }; - piEmbeddedMock.runEmbeddedPiAgent.mockImplementation(impl); - - const replyPromise = getReplyFromConfig( - { - Body: "ping", - From: "+1004", - To: "+2000", - MessageSid: "msg-123", - Provider: "discord", - }, - { - onReplyStart, - onBlockReply, - disableBlockStreaming: false, - }, - { - agents: { - defaults: { - model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw"), - }, - }, - channels: { whatsapp: { allowFrom: ["*"] } }, - session: { store: path.join(home, "sessions.json") }, - }, - ); - - await onReplyStartCalled; - releaseTyping?.(); - - const res = await replyPromise; - expect(res).toBeUndefined(); - expect(onBlockReply).toHaveBeenCalledTimes(1); - }); - }); - - it("preserves block reply ordering when typing start is slow", async () => { + it("waits for block replies and preserves ordering when typing start is slow", async () => { await withTempHome(async (home) => { let releaseTyping: (() => void) | undefined; const typingGate = new Promise((resolve) => { @@ -197,7 +135,7 @@ describe("block streaming", () => { Body: "ping", From: "+1004", To: "+2000", - MessageSid: "msg-125", + MessageSid: "msg-123", Provider: "telegram", }, { @@ -309,7 +247,7 @@ describe("block streaming", () => { }, { onBlockReply, - blockReplyTimeoutMs: 10, + blockReplyTimeoutMs: 1, disableBlockStreaming: false, }, { diff --git a/src/auto-reply/reply.raw-body.test.ts b/src/auto-reply/reply.raw-body.test.ts index 75d586bffee..e66b174e05a 100644 --- a/src/auto-reply/reply.raw-body.test.ts +++ b/src/auto-reply/reply.raw-body.test.ts @@ -140,31 +140,6 @@ describe("RawBody directive parsing", () => { expectedIncludes: ["Thinking level set to high."], }); - await assertCommandReply({ - message: { - Body: "[Context]\nJake: /model status\n[from: Jake]", - RawBody: "/model status", - From: "+1222", - To: "+1222", - ChatType: "group", - CommandAuthorized: true, - }, - config: { - agents: { - defaults: { - model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw-2"), - models: { - "anthropic/claude-opus-4-5": {}, - }, - }, - }, - channels: { whatsapp: { allowFrom: ["*"] } }, - session: { store: path.join(home, "sessions-2.json") }, - }, - expectedIncludes: ["anthropic/claude-opus-4-5"], - }); - await assertCommandReply({ message: { Body: "[Context]\nJake: /verbose on\n[from: Jake]", @@ -178,11 +153,11 @@ describe("RawBody directive parsing", () => { agents: { defaults: { model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw-3"), + workspace: path.join(home, "openclaw-2"), }, }, channels: { whatsapp: { allowFrom: ["*"] } }, - session: { store: path.join(home, "sessions-3.json") }, + session: { store: path.join(home, "sessions-2.json") }, }, expectedIncludes: ["Verbose logging enabled."], }); @@ -204,11 +179,11 @@ describe("RawBody directive parsing", () => { agents: { defaults: { model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw-4"), + workspace: path.join(home, "openclaw-3"), }, }, channels: { whatsapp: { allowFrom: ["+1222"] } }, - session: { store: path.join(home, "sessions-4.json") }, + session: { store: path.join(home, "sessions-3.json") }, }, expectedIncludes: ["Session: agent:main:whatsapp:group:g1", "anthropic/claude-opus-4-5"], }); diff --git a/src/infra/transport-ready.test.ts b/src/infra/transport-ready.test.ts index adb2560ce16..2df90a6420e 100644 --- a/src/infra/transport-ready.test.ts +++ b/src/infra/transport-ready.test.ts @@ -15,22 +15,22 @@ describe("waitForTransportReady", () => { let attempts = 0; const readyPromise = waitForTransportReady({ label: "test transport", - timeoutMs: 500, - logAfterMs: 120, - logIntervalMs: 100, - pollIntervalMs: 80, + timeoutMs: 220, + logAfterMs: 60, + logIntervalMs: 1_000, + pollIntervalMs: 50, runtime, check: async () => { attempts += 1; - if (attempts > 4) { + if (attempts > 2) { return { ok: true }; } return { ok: false, error: "not ready" }; }, }); - for (let i = 0; i < 5; i += 1) { - await vi.advanceTimersByTimeAsync(80); + for (let i = 0; i < 3; i += 1) { + await vi.advanceTimersByTimeAsync(50); } await readyPromise; @@ -41,14 +41,14 @@ describe("waitForTransportReady", () => { const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; const waitPromise = waitForTransportReady({ label: "test transport", - timeoutMs: 200, + timeoutMs: 110, logAfterMs: 0, - logIntervalMs: 100, + logIntervalMs: 1_000, pollIntervalMs: 50, runtime, check: async () => ({ ok: false, error: "still down" }), }); - await vi.advanceTimersByTimeAsync(250); + await vi.advanceTimersByTimeAsync(200); await expect(waitPromise).rejects.toThrow("test transport not ready"); expect(runtime.error).toHaveBeenCalled(); }); diff --git a/src/memory/index.test.ts b/src/memory/index.test.ts index 3030c45dbb4..9f5d708a2b4 100644 --- a/src/memory/index.test.ts +++ b/src/memory/index.test.ts @@ -280,7 +280,7 @@ describe("memory index", () => { expect(results[0]?.path).toContain("memory/2026-01-12.md"); }); - it("hybrid weights can favor vector-only matches over keyword-only matches", async () => { + it("hybrid weights shift ranking between vector and keyword matches", async () => { const manyAlpha = Array.from({ length: 50 }, () => "Alpha").join(" "); await fs.writeFile( path.join(workspaceDir, "memory", "vector-only.md"), @@ -291,7 +291,7 @@ describe("memory index", () => { `${manyAlpha} beta id123.`, ); - const cfg = { + const vectorWeightedCfg = { agents: { defaults: { workspace: workspaceDir, @@ -315,12 +315,15 @@ describe("memory index", () => { list: [{ id: "main", default: true }], }, }; - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - expect(result.manager).not.toBeNull(); - if (!result.manager) { + const vectorWeighted = await getMemorySearchManager({ + cfg: vectorWeightedCfg, + agentId: "main", + }); + expect(vectorWeighted.manager).not.toBeNull(); + if (!vectorWeighted.manager) { throw new Error("manager missing"); } - manager = result.manager; + manager = vectorWeighted.manager; const status = manager.status(); if (!status.fts?.available) { @@ -328,28 +331,19 @@ describe("memory index", () => { } await manager.sync({ force: true }); - const results = await manager.search("alpha beta id123"); - expect(results.length).toBeGreaterThan(0); - const paths = results.map((r) => r.path); - expect(paths).toContain("memory/vector-only.md"); - expect(paths).toContain("memory/keyword-only.md"); - const vectorOnly = results.find((r) => r.path === "memory/vector-only.md"); - const keywordOnly = results.find((r) => r.path === "memory/keyword-only.md"); + const vectorResults = await manager.search("alpha beta id123"); + expect(vectorResults.length).toBeGreaterThan(0); + const vectorPaths = vectorResults.map((r) => r.path); + expect(vectorPaths).toContain("memory/vector-only.md"); + expect(vectorPaths).toContain("memory/keyword-only.md"); + const vectorOnly = vectorResults.find((r) => r.path === "memory/vector-only.md"); + const keywordOnly = vectorResults.find((r) => r.path === "memory/keyword-only.md"); expect((vectorOnly?.score ?? 0) > (keywordOnly?.score ?? 0)).toBe(true); - }); - it("hybrid weights can favor keyword matches when text weight dominates", async () => { - const manyAlpha = Array.from({ length: 50 }, () => "Alpha").join(" "); - await fs.writeFile( - path.join(workspaceDir, "memory", "vector-only.md"), - "Alpha beta. Alpha beta. Alpha beta. Alpha beta.", - ); - await fs.writeFile( - path.join(workspaceDir, "memory", "keyword-only.md"), - `${manyAlpha} beta id123.`, - ); + await manager.close(); + manager = null; - const cfg = { + const textWeightedCfg = { agents: { defaults: { workspace: workspaceDir, @@ -357,7 +351,7 @@ describe("memory index", () => { provider: "openai", model: "mock-embed", store: { path: indexPath, vector: { enabled: false } }, - sync: { watch: false, onSessionStart: false, onSearch: true }, + sync: { watch: false, onSessionStart: false, onSearch: false }, query: { minScore: 0, maxResults: 200, @@ -373,27 +367,21 @@ describe("memory index", () => { list: [{ id: "main", default: true }], }, }; - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - expect(result.manager).not.toBeNull(); - if (!result.manager) { + + const textWeighted = await getMemorySearchManager({ cfg: textWeightedCfg, agentId: "main" }); + expect(textWeighted.manager).not.toBeNull(); + if (!textWeighted.manager) { throw new Error("manager missing"); } - manager = result.manager; - - const status = manager.status(); - if (!status.fts?.available) { - return; - } - - await manager.sync({ force: true }); - const results = await manager.search("alpha beta id123"); - expect(results.length).toBeGreaterThan(0); - const paths = results.map((r) => r.path); - expect(paths).toContain("memory/vector-only.md"); - expect(paths).toContain("memory/keyword-only.md"); - const vectorOnly = results.find((r) => r.path === "memory/vector-only.md"); - const keywordOnly = results.find((r) => r.path === "memory/keyword-only.md"); - expect((keywordOnly?.score ?? 0) > (vectorOnly?.score ?? 0)).toBe(true); + manager = textWeighted.manager; + const keywordResults = await manager.search("alpha beta id123"); + expect(keywordResults.length).toBeGreaterThan(0); + const keywordPaths = keywordResults.map((r) => r.path); + expect(keywordPaths).toContain("memory/vector-only.md"); + expect(keywordPaths).toContain("memory/keyword-only.md"); + const vectorOnlyAfter = keywordResults.find((r) => r.path === "memory/vector-only.md"); + const keywordOnlyAfter = keywordResults.find((r) => r.path === "memory/keyword-only.md"); + expect((keywordOnlyAfter?.score ?? 0) > (vectorOnlyAfter?.score ?? 0)).toBe(true); }); it("reports vector availability after probe", async () => { diff --git a/src/memory/manager.batch.test.ts b/src/memory/manager.batch.test.ts index 2ac5eeb5be5..2cf1b30c056 100644 --- a/src/memory/manager.batch.test.ts +++ b/src/memory/manager.batch.test.ts @@ -281,7 +281,7 @@ describe("memory indexing with OpenAI batches", () => { expect(batchCreates).toBe(2); }); - it("falls back to non-batch on failure and resets failures after success", async () => { + it("tracks batch failures, resets on success, and disables after repeated failures", async () => { const content = ["flaky", "batch"].join("\n\n"); await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-09.md"), content); @@ -376,12 +376,14 @@ describe("memory indexing with OpenAI batches", () => { } manager = result.manager; + // First failure: fallback to regular embeddings and increment failure count. await manager.sync({ force: true }); expect(embedBatch).toHaveBeenCalled(); let status = manager.status(); expect(status.batch?.enabled).toBe(true); expect(status.batch?.failures).toBe(1); + // Success should reset failure count. embedBatch.mockClear(); mode = "ok"; await fs.writeFile( @@ -393,110 +395,33 @@ describe("memory indexing with OpenAI batches", () => { expect(status.batch?.enabled).toBe(true); expect(status.batch?.failures).toBe(0); expect(embedBatch).not.toHaveBeenCalled(); - }); - - it("disables batch after repeated failures and skips batch thereafter", async () => { - const content = ["repeat", "failures"].join("\n\n"); - await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-10.md"), content); - - let uploadedRequests: Array<{ custom_id?: string }> = []; - const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { - const url = - typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; - if (url.endsWith("/files")) { - const body = init?.body; - if (!(body instanceof FormData)) { - throw new Error("expected FormData upload"); - } - for (const [key, value] of body.entries()) { - if (key !== "file") { - continue; - } - if (typeof value === "string") { - uploadedRequests = value - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line) as { custom_id?: string }); - } else { - const text = await value.text(); - uploadedRequests = text - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line) as { custom_id?: string }); - } - } - return new Response(JSON.stringify({ id: "file_1" }), { - status: 200, - headers: { "Content-Type": "application/json" }, - }); - } - if (url.endsWith("/batches")) { - return new Response("batch failed", { status: 500 }); - } - if (url.endsWith("/files/file_out/content")) { - const lines = uploadedRequests.map((request, index) => - JSON.stringify({ - custom_id: request.custom_id, - response: { - status_code: 200, - body: { data: [{ embedding: [index + 1, 0, 0], index: 0 }] }, - }, - }), - ); - return new Response(lines.join("\n"), { - status: 200, - headers: { "Content-Type": "application/jsonl" }, - }); - } - throw new Error(`unexpected fetch ${url}`); - }); - - vi.stubGlobal("fetch", fetchMock); - - const cfg = { - agents: { - defaults: { - workspace: workspaceDir, - memorySearch: { - provider: "openai", - model: "text-embedding-3-small", - store: { path: indexPath }, - sync: { watch: false, onSessionStart: false, onSearch: false }, - query: { minScore: 0 }, - remote: { batch: { enabled: true, wait: true, pollIntervalMs: 1 } }, - }, - }, - list: [{ id: "main", default: true }], - }, - }; - - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - expect(result.manager).not.toBeNull(); - if (!result.manager) { - throw new Error("manager missing"); - } - manager = result.manager; + // Two more failures after reset should disable remote batching. + mode = "fail"; + await fs.writeFile( + path.join(workspaceDir, "memory", "2026-01-09.md"), + ["flaky", "batch", "fail-a"].join("\n\n"), + ); await manager.sync({ force: true }); - let status = manager.status(); + status = manager.status(); expect(status.batch?.enabled).toBe(true); expect(status.batch?.failures).toBe(1); - embedBatch.mockClear(); await fs.writeFile( - path.join(workspaceDir, "memory", "2026-01-10.md"), - ["repeat", "failures", "again"].join("\n\n"), + path.join(workspaceDir, "memory", "2026-01-09.md"), + ["flaky", "batch", "fail-b"].join("\n\n"), ); await manager.sync({ force: true }); status = manager.status(); expect(status.batch?.enabled).toBe(false); expect(status.batch?.failures).toBeGreaterThanOrEqual(2); + // Once disabled, batch endpoints are skipped and fallback embeddings run directly. const fetchCalls = fetchMock.mock.calls.length; embedBatch.mockClear(); await fs.writeFile( - path.join(workspaceDir, "memory", "2026-01-10.md"), - ["repeat", "failures", "fallback"].join("\n\n"), + path.join(workspaceDir, "memory", "2026-01-09.md"), + ["flaky", "batch", "fallback"].join("\n\n"), ); await manager.sync({ force: true }); expect(fetchMock.mock.calls.length).toBe(fetchCalls); From 2378d770d1810de0c5888210598e52f8ed136c58 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 23:33:08 +0000 Subject: [PATCH 0017/1038] perf(test): speed gateway suite resets with unique config roots --- src/gateway/test-helpers.server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gateway/test-helpers.server.ts b/src/gateway/test-helpers.server.ts index c58d2bb75c1..849e4243555 100644 --- a/src/gateway/test-helpers.server.ts +++ b/src/gateway/test-helpers.server.ts @@ -221,10 +221,10 @@ export function installGatewayTestHooks(options?: { scope?: "test" | "suite" }) if (scope === "suite") { beforeAll(async () => { await setupGatewayTestHome(); - await resetGatewayTestState({ uniqueConfigRoot: false }); + await resetGatewayTestState({ uniqueConfigRoot: true }); }); beforeEach(async () => { - await resetGatewayTestState({ uniqueConfigRoot: false }); + await resetGatewayTestState({ uniqueConfigRoot: true }); }, 60_000); afterEach(async () => { await cleanupGatewayTestHome({ restoreEnv: false }); From 874ff7089cc116682baed7aaca55b95ae5ff59e8 Mon Sep 17 00:00:00 2001 From: Taylor Asplund <62564740+DrCrinkle@users.noreply.github.com> Date: Fri, 13 Feb 2026 15:34:33 -0800 Subject: [PATCH 0018/1038] fix: ensure CLI exits after command completion (#12906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: ensure CLI exits after command completion The CLI process would hang indefinitely after commands like `openclaw gateway restart` completed successfully. Two root causes: 1. `runCli()` returned without calling `process.exit()` after `program.parseAsync()` resolved, and Commander.js does not force-exit the process. 2. `daemon-cli/register.ts` eagerly called `createDefaultDeps()` which imported all messaging-provider modules, creating persistent event-loop handles that prevented natural Node exit. Changes: - Add `flushAndExit()` helper that drains stdout/stderr before calling `process.exit()`, preventing truncated piped output in CI/scripts. - Call `flushAndExit()` after both `tryRouteCli()` and `program.parseAsync()` resolve. - Remove unnecessary `void createDefaultDeps()` from daemon-cli registration — daemon lifecycle commands never use messaging deps. - Make `serveAcpGateway()` return a promise that resolves on intentional shutdown (SIGINT/SIGTERM), so `openclaw acp` blocks `parseAsync` for the bridge lifetime and exits cleanly on signal. - Handle the returned promise in the standalone main-module entry point to avoid unhandled rejections. Fixes #12904 Co-Authored-By: Claude Opus 4.6 * fix: refactor CLI lifecycle and lazy outbound deps (#12906) (thanks @DrCrinkle) --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Peter Steinberger --- CHANGELOG.md | 1 + src/acp/server.ts | 34 ++++++++++++- src/cli/acp-cli.ts | 4 +- src/cli/daemon-cli/register.ts | 4 -- src/cli/deps.test.ts | 93 ++++++++++++++++++++++++++++++++++ src/cli/deps.ts | 44 +++++++++++----- src/cli/run-main.exit.test.ts | 49 ++++++++++++++++++ 7 files changed, 208 insertions(+), 21 deletions(-) create mode 100644 src/cli/deps.test.ts create mode 100644 src/cli/run-main.exit.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f4c55aa8f8d..56a5d758c41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- CLI: lazily load outbound provider dependencies and remove forced success-path exits so commands terminate naturally without killing intentional long-running foreground actions. (#12906) Thanks @DrCrinkle. - Clawdock: avoid Zsh readonly variable collisions in helper scripts. (#15501) Thanks @nkelner. - Discord: route autoThread replies to existing threads instead of the root channel. (#8302) Thanks @gavinbmoore, @thewilloftheshadow. - Agents/Image tool: cap image-analysis completion `maxTokens` by model capability (`min(4096, model.maxTokens)`) to avoid over-limit provider failures while still preventing truncation. (#11770) Thanks @detecti1. diff --git a/src/acp/server.ts b/src/acp/server.ts index 4a2c835b549..93acc4a523c 100644 --- a/src/acp/server.ts +++ b/src/acp/server.ts @@ -11,7 +11,7 @@ import { isMainModule } from "../infra/is-main.js"; import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js"; import { AcpGatewayAgent } from "./translator.js"; -export function serveAcpGateway(opts: AcpServerOptions = {}): void { +export function serveAcpGateway(opts: AcpServerOptions = {}): Promise { const cfg = loadConfig(); const connection = buildGatewayConnectionDetails({ config: cfg, @@ -34,6 +34,12 @@ export function serveAcpGateway(opts: AcpServerOptions = {}): void { auth.password; let agent: AcpGatewayAgent | null = null; + let onClosed!: () => void; + const closed = new Promise((resolve) => { + onClosed = resolve; + }); + let stopped = false; + const gateway = new GatewayClient({ url: connection.url, token: token || undefined, @@ -50,9 +56,29 @@ export function serveAcpGateway(opts: AcpServerOptions = {}): void { }, onClose: (code, reason) => { agent?.handleGatewayDisconnect(`${code}: ${reason}`); + // Resolve only on intentional shutdown (gateway.stop() sets closed + // which skips scheduleReconnect, then fires onClose). Transient + // disconnects are followed by automatic reconnect attempts. + if (stopped) { + onClosed(); + } }, }); + const shutdown = () => { + if (stopped) { + return; + } + stopped = true; + gateway.stop(); + // If no WebSocket is active (e.g. between reconnect attempts), + // gateway.stop() won't trigger onClose, so resolve directly. + onClosed(); + }; + + process.once("SIGINT", shutdown); + process.once("SIGTERM", shutdown); + const input = Writable.toWeb(process.stdout); const output = Readable.toWeb(process.stdin) as unknown as ReadableStream; const stream = ndJsonStream(input, output); @@ -64,6 +90,7 @@ export function serveAcpGateway(opts: AcpServerOptions = {}): void { }, stream); gateway.start(); + return closed; } function parseArgs(args: string[]): AcpServerOptions { @@ -140,5 +167,8 @@ Options: if (isMainModule({ currentFile: fileURLToPath(import.meta.url) })) { const opts = parseArgs(process.argv.slice(2)); - serveAcpGateway(opts); + serveAcpGateway(opts).catch((err) => { + console.error(String(err)); + process.exit(1); + }); } diff --git a/src/cli/acp-cli.ts b/src/cli/acp-cli.ts index 1be77e71fcd..c86deb48f28 100644 --- a/src/cli/acp-cli.ts +++ b/src/cli/acp-cli.ts @@ -22,9 +22,9 @@ export function registerAcpCli(program: Command) { "after", () => `\n${theme.muted("Docs:")} ${formatDocsLink("/cli/acp", "docs.openclaw.ai/cli/acp")}\n`, ) - .action((opts) => { + .action(async (opts) => { try { - serveAcpGateway({ + await serveAcpGateway({ gatewayUrl: opts.url as string | undefined, gatewayToken: opts.token as string | undefined, gatewayPassword: opts.password as string | undefined, diff --git a/src/cli/daemon-cli/register.ts b/src/cli/daemon-cli/register.ts index d1599a206aa..47e3dd09bdf 100644 --- a/src/cli/daemon-cli/register.ts +++ b/src/cli/daemon-cli/register.ts @@ -1,7 +1,6 @@ import type { Command } from "commander"; import { formatDocsLink } from "../../terminal/links.js"; import { theme } from "../../terminal/theme.js"; -import { createDefaultDeps } from "../deps.js"; import { runDaemonInstall, runDaemonRestart, @@ -83,7 +82,4 @@ export function registerDaemonCli(program: Command) { .action(async (opts) => { await runDaemonRestart(opts); }); - - // Build default deps (parity with other commands). - void createDefaultDeps(); } diff --git a/src/cli/deps.test.ts b/src/cli/deps.test.ts new file mode 100644 index 00000000000..34c28cece57 --- /dev/null +++ b/src/cli/deps.test.ts @@ -0,0 +1,93 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createDefaultDeps } from "./deps.js"; + +const moduleLoads = vi.hoisted(() => ({ + whatsapp: vi.fn(), + telegram: vi.fn(), + discord: vi.fn(), + slack: vi.fn(), + signal: vi.fn(), + imessage: vi.fn(), +})); + +const sendFns = vi.hoisted(() => ({ + whatsapp: vi.fn(async () => ({ messageId: "w1", toJid: "whatsapp:1" })), + telegram: vi.fn(async () => ({ messageId: "t1", chatId: "telegram:1" })), + discord: vi.fn(async () => ({ messageId: "d1", channelId: "discord:1" })), + slack: vi.fn(async () => ({ messageId: "s1", channelId: "slack:1" })), + signal: vi.fn(async () => ({ messageId: "sg1", conversationId: "signal:1" })), + imessage: vi.fn(async () => ({ messageId: "i1", chatId: "imessage:1" })), +})); + +vi.mock("../channels/web/index.js", () => { + moduleLoads.whatsapp(); + return { sendMessageWhatsApp: sendFns.whatsapp }; +}); + +vi.mock("../telegram/send.js", () => { + moduleLoads.telegram(); + return { sendMessageTelegram: sendFns.telegram }; +}); + +vi.mock("../discord/send.js", () => { + moduleLoads.discord(); + return { sendMessageDiscord: sendFns.discord }; +}); + +vi.mock("../slack/send.js", () => { + moduleLoads.slack(); + return { sendMessageSlack: sendFns.slack }; +}); + +vi.mock("../signal/send.js", () => { + moduleLoads.signal(); + return { sendMessageSignal: sendFns.signal }; +}); + +vi.mock("../imessage/send.js", () => { + moduleLoads.imessage(); + return { sendMessageIMessage: sendFns.imessage }; +}); + +describe("createDefaultDeps", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("does not load provider modules until a dependency is used", async () => { + const deps = createDefaultDeps(); + + expect(moduleLoads.whatsapp).not.toHaveBeenCalled(); + expect(moduleLoads.telegram).not.toHaveBeenCalled(); + expect(moduleLoads.discord).not.toHaveBeenCalled(); + expect(moduleLoads.slack).not.toHaveBeenCalled(); + expect(moduleLoads.signal).not.toHaveBeenCalled(); + expect(moduleLoads.imessage).not.toHaveBeenCalled(); + + const sendTelegram = deps.sendMessageTelegram as unknown as ( + ...args: unknown[] + ) => Promise; + await sendTelegram("chat", "hello", { verbose: false }); + + expect(moduleLoads.telegram).toHaveBeenCalledTimes(1); + expect(sendFns.telegram).toHaveBeenCalledTimes(1); + expect(moduleLoads.whatsapp).not.toHaveBeenCalled(); + expect(moduleLoads.discord).not.toHaveBeenCalled(); + expect(moduleLoads.slack).not.toHaveBeenCalled(); + expect(moduleLoads.signal).not.toHaveBeenCalled(); + expect(moduleLoads.imessage).not.toHaveBeenCalled(); + }); + + it("reuses module cache after first dynamic import", async () => { + const deps = createDefaultDeps(); + const sendDiscord = deps.sendMessageDiscord as unknown as ( + ...args: unknown[] + ) => Promise; + + await sendDiscord("channel", "first", { verbose: false }); + await sendDiscord("channel", "second", { verbose: false }); + + expect(moduleLoads.discord).toHaveBeenCalledTimes(1); + expect(sendFns.discord).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/cli/deps.ts b/src/cli/deps.ts index 1f0e8e587f0..a3c3c72ac49 100644 --- a/src/cli/deps.ts +++ b/src/cli/deps.ts @@ -1,10 +1,10 @@ +import type { sendMessageWhatsApp } from "../channels/web/index.js"; +import type { sendMessageDiscord } from "../discord/send.js"; +import type { sendMessageIMessage } from "../imessage/send.js"; import type { OutboundSendDeps } from "../infra/outbound/deliver.js"; -import { logWebSelfId, sendMessageWhatsApp } from "../channels/web/index.js"; -import { sendMessageDiscord } from "../discord/send.js"; -import { sendMessageIMessage } from "../imessage/send.js"; -import { sendMessageSignal } from "../signal/send.js"; -import { sendMessageSlack } from "../slack/send.js"; -import { sendMessageTelegram } from "../telegram/send.js"; +import type { sendMessageSignal } from "../signal/send.js"; +import type { sendMessageSlack } from "../slack/send.js"; +import type { sendMessageTelegram } from "../telegram/send.js"; export type CliDeps = { sendMessageWhatsApp: typeof sendMessageWhatsApp; @@ -17,12 +17,30 @@ export type CliDeps = { export function createDefaultDeps(): CliDeps { return { - sendMessageWhatsApp, - sendMessageTelegram, - sendMessageDiscord, - sendMessageSlack, - sendMessageSignal, - sendMessageIMessage, + sendMessageWhatsApp: async (...args) => { + const { sendMessageWhatsApp } = await import("../channels/web/index.js"); + return await sendMessageWhatsApp(...args); + }, + sendMessageTelegram: async (...args) => { + const { sendMessageTelegram } = await import("../telegram/send.js"); + return await sendMessageTelegram(...args); + }, + sendMessageDiscord: async (...args) => { + const { sendMessageDiscord } = await import("../discord/send.js"); + return await sendMessageDiscord(...args); + }, + sendMessageSlack: async (...args) => { + const { sendMessageSlack } = await import("../slack/send.js"); + return await sendMessageSlack(...args); + }, + sendMessageSignal: async (...args) => { + const { sendMessageSignal } = await import("../signal/send.js"); + return await sendMessageSignal(...args); + }, + sendMessageIMessage: async (...args) => { + const { sendMessageIMessage } = await import("../imessage/send.js"); + return await sendMessageIMessage(...args); + }, }; } @@ -38,4 +56,4 @@ export function createOutboundSendDeps(deps: CliDeps): OutboundSendDeps { }; } -export { logWebSelfId }; +export { logWebSelfId } from "../web/auth-store.js"; diff --git a/src/cli/run-main.exit.test.ts b/src/cli/run-main.exit.test.ts new file mode 100644 index 00000000000..86d74f09640 --- /dev/null +++ b/src/cli/run-main.exit.test.ts @@ -0,0 +1,49 @@ +import process from "node:process"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const tryRouteCliMock = vi.hoisted(() => vi.fn()); +const loadDotEnvMock = vi.hoisted(() => vi.fn()); +const normalizeEnvMock = vi.hoisted(() => vi.fn()); +const ensurePathMock = vi.hoisted(() => vi.fn()); +const assertRuntimeMock = vi.hoisted(() => vi.fn()); + +vi.mock("./route.js", () => ({ + tryRouteCli: tryRouteCliMock, +})); + +vi.mock("../infra/dotenv.js", () => ({ + loadDotEnv: loadDotEnvMock, +})); + +vi.mock("../infra/env.js", () => ({ + normalizeEnv: normalizeEnvMock, +})); + +vi.mock("../infra/path-env.js", () => ({ + ensureOpenClawCliOnPath: ensurePathMock, +})); + +vi.mock("../infra/runtime-guard.js", () => ({ + assertSupportedRuntime: assertRuntimeMock, +})); + +const { runCli } = await import("./run-main.js"); + +describe("runCli exit behavior", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("does not force process.exit after successful routed command", async () => { + tryRouteCliMock.mockResolvedValueOnce(true); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`unexpected process.exit(${String(code)})`); + }) as typeof process.exit); + + await runCli(["node", "openclaw", "status"]); + + expect(tryRouteCliMock).toHaveBeenCalledWith(["node", "openclaw", "status"]); + expect(exitSpy).not.toHaveBeenCalled(); + exitSpy.mockRestore(); + }); +}); From 51296e770c70c69a39cf11b234776c41798212a5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 23:37:05 +0000 Subject: [PATCH 0019/1038] feat(slack): land thread-ownership from @DarlingtonDeveloper (#15775) Land PR #15775 by @DarlingtonDeveloper: - add thread-ownership plugin and Slack message_sending hook wiring - include regression tests and changelog update Co-authored-by: Mike <108890394+DarlingtonDeveloper@users.noreply.github.com> --- CHANGELOG.md | 1 + extensions/thread-ownership/index.test.ts | 180 ++++++++++++++++++ extensions/thread-ownership/index.ts | 133 +++++++++++++ .../thread-ownership/openclaw.plugin.json | 28 +++ src/channels/plugins/outbound/slack.test.ts | 124 ++++++++++++ src/channels/plugins/outbound/slack.ts | 49 ++++- 6 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 extensions/thread-ownership/index.test.ts create mode 100644 extensions/thread-ownership/index.ts create mode 100644 extensions/thread-ownership/openclaw.plugin.json create mode 100644 src/channels/plugins/outbound/slack.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a5d758c41..98e88317aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Docs: https://docs.openclaw.ai - Skills: remove duplicate `local-places` Google Places skill/proxy and keep `goplaces` as the single supported Google Places path. - Discord: send voice messages with waveform previews from local audio files (including silent delivery). (#7253) Thanks @nyanjou. - Discord: add configurable presence status/activity/type/url (custom status defaults to activity text). (#10855) Thanks @h0tp-ftw. +- Slack/Plugins: add thread-ownership outbound gating via `message_sending` hooks, including @-mention bypass tracking and Slack outbound hook wiring for cancel/modify behavior. (#15775) Thanks @DarlingtonDeveloper. ### Fixes diff --git a/extensions/thread-ownership/index.test.ts b/extensions/thread-ownership/index.test.ts new file mode 100644 index 00000000000..3690938a1b0 --- /dev/null +++ b/extensions/thread-ownership/index.test.ts @@ -0,0 +1,180 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import register from "./index.js"; + +describe("thread-ownership plugin", () => { + const hooks: Record = {}; + const api = { + pluginConfig: {}, + config: { + agents: { + list: [{ id: "test-agent", default: true, identity: { name: "TestBot" } }], + }, + }, + id: "thread-ownership", + name: "Thread Ownership", + logger: { info: vi.fn(), warn: vi.fn(), debug: vi.fn() }, + on: vi.fn((hookName: string, handler: Function) => { + hooks[hookName] = handler; + }), + }; + + let originalFetch: typeof globalThis.fetch; + + beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(hooks)) delete hooks[key]; + + process.env.SLACK_FORWARDER_URL = "http://localhost:8750"; + process.env.SLACK_BOT_USER_ID = "U999"; + + originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.SLACK_FORWARDER_URL; + delete process.env.SLACK_BOT_USER_ID; + vi.restoreAllMocks(); + }); + + it("registers message_received and message_sending hooks", () => { + register(api as any); + + expect(api.on).toHaveBeenCalledTimes(2); + expect(api.on).toHaveBeenCalledWith("message_received", expect.any(Function)); + expect(api.on).toHaveBeenCalledWith("message_sending", expect.any(Function)); + }); + + describe("message_sending", () => { + beforeEach(() => { + register(api as any); + }); + + it("allows non-slack channels", async () => { + const result = await hooks.message_sending( + { content: "hello", metadata: { threadTs: "1234.5678", channelId: "C123" }, to: "C123" }, + { channelId: "discord", conversationId: "C123" }, + ); + + expect(result).toBeUndefined(); + expect(globalThis.fetch).not.toHaveBeenCalled(); + }); + + it("allows top-level messages (no threadTs)", async () => { + const result = await hooks.message_sending( + { content: "hello", metadata: {}, to: "C123" }, + { channelId: "slack", conversationId: "C123" }, + ); + + expect(result).toBeUndefined(); + expect(globalThis.fetch).not.toHaveBeenCalled(); + }); + + it("claims ownership successfully", async () => { + vi.mocked(globalThis.fetch).mockResolvedValue( + new Response(JSON.stringify({ owner: "test-agent" }), { status: 200 }), + ); + + const result = await hooks.message_sending( + { content: "hello", metadata: { threadTs: "1234.5678", channelId: "C123" }, to: "C123" }, + { channelId: "slack", conversationId: "C123" }, + ); + + expect(result).toBeUndefined(); + expect(globalThis.fetch).toHaveBeenCalledWith( + "http://localhost:8750/api/v1/ownership/C123/1234.5678", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ agent_id: "test-agent" }), + }), + ); + }); + + it("cancels when thread owned by another agent", async () => { + vi.mocked(globalThis.fetch).mockResolvedValue( + new Response(JSON.stringify({ owner: "other-agent" }), { status: 409 }), + ); + + const result = await hooks.message_sending( + { content: "hello", metadata: { threadTs: "1234.5678", channelId: "C123" }, to: "C123" }, + { channelId: "slack", conversationId: "C123" }, + ); + + expect(result).toEqual({ cancel: true }); + expect(api.logger.info).toHaveBeenCalledWith(expect.stringContaining("cancelled send")); + }); + + it("fails open on network error", async () => { + vi.mocked(globalThis.fetch).mockRejectedValue(new Error("ECONNREFUSED")); + + const result = await hooks.message_sending( + { content: "hello", metadata: { threadTs: "1234.5678", channelId: "C123" }, to: "C123" }, + { channelId: "slack", conversationId: "C123" }, + ); + + expect(result).toBeUndefined(); + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("ownership check failed"), + ); + }); + }); + + describe("message_received @-mention tracking", () => { + beforeEach(() => { + register(api as any); + }); + + it("tracks @-mentions and skips ownership check for mentioned threads", async () => { + // Simulate receiving a message that @-mentions the agent. + await hooks.message_received( + { content: "Hey @TestBot help me", metadata: { threadTs: "9999.0001", channelId: "C456" } }, + { channelId: "slack", conversationId: "C456" }, + ); + + // Now send in the same thread -- should skip the ownership HTTP call. + const result = await hooks.message_sending( + { content: "Sure!", metadata: { threadTs: "9999.0001", channelId: "C456" }, to: "C456" }, + { channelId: "slack", conversationId: "C456" }, + ); + + expect(result).toBeUndefined(); + expect(globalThis.fetch).not.toHaveBeenCalled(); + }); + + it("ignores @-mentions on non-slack channels", async () => { + // Use a unique thread key so module-level state from other tests doesn't interfere. + await hooks.message_received( + { content: "Hey @TestBot", metadata: { threadTs: "7777.0001", channelId: "C999" } }, + { channelId: "discord", conversationId: "C999" }, + ); + + // The mention should not have been tracked, so sending should still call fetch. + vi.mocked(globalThis.fetch).mockResolvedValue( + new Response(JSON.stringify({ owner: "test-agent" }), { status: 200 }), + ); + + await hooks.message_sending( + { content: "Sure!", metadata: { threadTs: "7777.0001", channelId: "C999" }, to: "C999" }, + { channelId: "slack", conversationId: "C999" }, + ); + + expect(globalThis.fetch).toHaveBeenCalled(); + }); + + it("tracks bot user ID mentions via <@U999> syntax", async () => { + await hooks.message_received( + { content: "Hey <@U999> help", metadata: { threadTs: "8888.0001", channelId: "C789" } }, + { channelId: "slack", conversationId: "C789" }, + ); + + const result = await hooks.message_sending( + { content: "On it!", metadata: { threadTs: "8888.0001", channelId: "C789" }, to: "C789" }, + { channelId: "slack", conversationId: "C789" }, + ); + + expect(result).toBeUndefined(); + expect(globalThis.fetch).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/extensions/thread-ownership/index.ts b/extensions/thread-ownership/index.ts new file mode 100644 index 00000000000..3db1ea94ff4 --- /dev/null +++ b/extensions/thread-ownership/index.ts @@ -0,0 +1,133 @@ +import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk"; + +type ThreadOwnershipConfig = { + forwarderUrl?: string; + abTestChannels?: string[]; +}; + +type AgentEntry = NonNullable["list"]>[number]; + +// In-memory set of {channel}:{thread} keys where this agent was @-mentioned. +// Entries expire after 5 minutes. +const mentionedThreads = new Map(); +const MENTION_TTL_MS = 5 * 60 * 1000; + +function cleanExpiredMentions(): void { + const now = Date.now(); + for (const [key, ts] of mentionedThreads) { + if (now - ts > MENTION_TTL_MS) { + mentionedThreads.delete(key); + } + } +} + +function resolveOwnershipAgent(config: OpenClawConfig): { id: string; name: string } { + const list = Array.isArray(config.agents?.list) + ? config.agents.list.filter((entry): entry is AgentEntry => + Boolean(entry && typeof entry === "object"), + ) + : []; + const selected = list.find((entry) => entry.default === true) ?? list[0]; + + const id = + typeof selected?.id === "string" && selected.id.trim() ? selected.id.trim() : "unknown"; + const identityName = + typeof selected?.identity?.name === "string" ? selected.identity.name.trim() : ""; + const fallbackName = typeof selected?.name === "string" ? selected.name.trim() : ""; + const name = identityName || fallbackName; + + return { id, name }; +} + +export default function register(api: OpenClawPluginApi) { + const pluginCfg = (api.pluginConfig ?? {}) as ThreadOwnershipConfig; + const forwarderUrl = ( + pluginCfg.forwarderUrl ?? + process.env.SLACK_FORWARDER_URL ?? + "http://slack-forwarder:8750" + ).replace(/\/$/, ""); + + const abTestChannels = new Set( + pluginCfg.abTestChannels ?? + process.env.THREAD_OWNERSHIP_CHANNELS?.split(",").filter(Boolean) ?? + [], + ); + + const { id: agentId, name: agentName } = resolveOwnershipAgent(api.config); + const botUserId = process.env.SLACK_BOT_USER_ID ?? ""; + + // --------------------------------------------------------------------------- + // message_received: track @-mentions so the agent can reply even if it + // doesn't own the thread. + // --------------------------------------------------------------------------- + api.on("message_received", async (event, ctx) => { + if (ctx.channelId !== "slack") return; + + const text = event.content ?? ""; + const threadTs = (event.metadata?.threadTs as string) ?? ""; + const channelId = (event.metadata?.channelId as string) ?? ctx.conversationId ?? ""; + + if (!threadTs || !channelId) return; + + // Check if this agent was @-mentioned. + const mentioned = + (agentName && text.includes(`@${agentName}`)) || + (botUserId && text.includes(`<@${botUserId}>`)); + + if (mentioned) { + cleanExpiredMentions(); + mentionedThreads.set(`${channelId}:${threadTs}`, Date.now()); + } + }); + + // --------------------------------------------------------------------------- + // message_sending: check thread ownership before sending to Slack. + // Returns { cancel: true } if another agent owns the thread. + // --------------------------------------------------------------------------- + api.on("message_sending", async (event, ctx) => { + if (ctx.channelId !== "slack") return; + + const threadTs = (event.metadata?.threadTs as string) ?? ""; + const channelId = (event.metadata?.channelId as string) ?? event.to; + + // Top-level messages (no thread) are always allowed. + if (!threadTs) return; + + // Only enforce in A/B test channels (if set is empty, skip entirely). + if (abTestChannels.size > 0 && !abTestChannels.has(channelId)) return; + + // If this agent was @-mentioned in this thread recently, skip ownership check. + cleanExpiredMentions(); + if (mentionedThreads.has(`${channelId}:${threadTs}`)) return; + + // Try to claim ownership via the forwarder HTTP API. + try { + const resp = await fetch(`${forwarderUrl}/api/v1/ownership/${channelId}/${threadTs}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ agent_id: agentId }), + signal: AbortSignal.timeout(3000), + }); + + if (resp.ok) { + // We own it (or just claimed it), proceed. + return; + } + + if (resp.status === 409) { + // Another agent owns this thread — cancel the send. + const body = (await resp.json()) as { owner?: string }; + api.logger.info?.( + `thread-ownership: cancelled send to ${channelId}:${threadTs} — owned by ${body.owner}`, + ); + return { cancel: true }; + } + + // Unexpected status — fail open. + api.logger.warn?.(`thread-ownership: unexpected status ${resp.status}, allowing send`); + } catch (err) { + // Network error — fail open. + api.logger.warn?.(`thread-ownership: ownership check failed (${String(err)}), allowing send`); + } + }); +} diff --git a/extensions/thread-ownership/openclaw.plugin.json b/extensions/thread-ownership/openclaw.plugin.json new file mode 100644 index 00000000000..2e020bdadec --- /dev/null +++ b/extensions/thread-ownership/openclaw.plugin.json @@ -0,0 +1,28 @@ +{ + "id": "thread-ownership", + "name": "Thread Ownership", + "description": "Prevents multiple agents from responding in the same Slack thread. Uses HTTP calls to the slack-forwarder ownership API.", + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": { + "forwarderUrl": { + "type": "string" + }, + "abTestChannels": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "uiHints": { + "forwarderUrl": { + "label": "Forwarder URL", + "help": "Base URL of the slack-forwarder ownership API (default: http://slack-forwarder:8750)" + }, + "abTestChannels": { + "label": "A/B Test Channels", + "help": "Slack channel IDs where thread ownership is enforced" + } + } +} diff --git a/src/channels/plugins/outbound/slack.test.ts b/src/channels/plugins/outbound/slack.test.ts new file mode 100644 index 00000000000..08863d24b7f --- /dev/null +++ b/src/channels/plugins/outbound/slack.test.ts @@ -0,0 +1,124 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../../../slack/send.js", () => ({ + sendMessageSlack: vi.fn().mockResolvedValue({ ts: "1234.5678", channel: "C123" }), +})); + +vi.mock("../../../plugins/hook-runner-global.js", () => ({ + getGlobalHookRunner: vi.fn(), +})); + +import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; +import { sendMessageSlack } from "../../../slack/send.js"; +import { slackOutbound } from "./slack.js"; + +describe("slack outbound hook wiring", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("calls send without hooks when no hooks registered", async () => { + vi.mocked(getGlobalHookRunner).mockReturnValue(null); + + await slackOutbound.sendText({ + to: "C123", + text: "hello", + accountId: "default", + replyToId: "1111.2222", + }); + + expect(sendMessageSlack).toHaveBeenCalledWith("C123", "hello", { + threadTs: "1111.2222", + accountId: "default", + }); + }); + + it("calls message_sending hook before sending", async () => { + const mockRunner = { + hasHooks: vi.fn().mockReturnValue(true), + runMessageSending: vi.fn().mockResolvedValue(undefined), + }; + // oxlint-disable-next-line typescript/no-explicit-any + vi.mocked(getGlobalHookRunner).mockReturnValue(mockRunner as any); + + await slackOutbound.sendText({ + to: "C123", + text: "hello", + accountId: "default", + replyToId: "1111.2222", + }); + + expect(mockRunner.hasHooks).toHaveBeenCalledWith("message_sending"); + expect(mockRunner.runMessageSending).toHaveBeenCalledWith( + { to: "C123", content: "hello", metadata: { threadTs: "1111.2222", channelId: "C123" } }, + { channelId: "slack", accountId: "default" }, + ); + expect(sendMessageSlack).toHaveBeenCalledWith("C123", "hello", { + threadTs: "1111.2222", + accountId: "default", + }); + }); + + it("cancels send when hook returns cancel:true", async () => { + const mockRunner = { + hasHooks: vi.fn().mockReturnValue(true), + runMessageSending: vi.fn().mockResolvedValue({ cancel: true }), + }; + // oxlint-disable-next-line typescript/no-explicit-any + vi.mocked(getGlobalHookRunner).mockReturnValue(mockRunner as any); + + const result = await slackOutbound.sendText({ + to: "C123", + text: "hello", + accountId: "default", + replyToId: "1111.2222", + }); + + expect(sendMessageSlack).not.toHaveBeenCalled(); + expect(result.channel).toBe("slack"); + }); + + it("modifies text when hook returns content", async () => { + const mockRunner = { + hasHooks: vi.fn().mockReturnValue(true), + runMessageSending: vi.fn().mockResolvedValue({ content: "modified" }), + }; + // oxlint-disable-next-line typescript/no-explicit-any + vi.mocked(getGlobalHookRunner).mockReturnValue(mockRunner as any); + + await slackOutbound.sendText({ + to: "C123", + text: "original", + accountId: "default", + replyToId: "1111.2222", + }); + + expect(sendMessageSlack).toHaveBeenCalledWith("C123", "modified", { + threadTs: "1111.2222", + accountId: "default", + }); + }); + + it("skips hooks when runner has no message_sending hooks", async () => { + const mockRunner = { + hasHooks: vi.fn().mockReturnValue(false), + runMessageSending: vi.fn(), + }; + // oxlint-disable-next-line typescript/no-explicit-any + vi.mocked(getGlobalHookRunner).mockReturnValue(mockRunner as any); + + await slackOutbound.sendText({ + to: "C123", + text: "hello", + accountId: "default", + replyToId: "1111.2222", + }); + + expect(mockRunner.runMessageSending).not.toHaveBeenCalled(); + expect(sendMessageSlack).toHaveBeenCalled(); + }); +}); diff --git a/src/channels/plugins/outbound/slack.ts b/src/channels/plugins/outbound/slack.ts index 08d27bd7073..dde96245538 100644 --- a/src/channels/plugins/outbound/slack.ts +++ b/src/channels/plugins/outbound/slack.ts @@ -1,4 +1,5 @@ import type { ChannelOutboundAdapter } from "../types.js"; +import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; import { sendMessageSlack } from "../../../slack/send.js"; export const slackOutbound: ChannelOutboundAdapter = { @@ -9,7 +10,29 @@ export const slackOutbound: ChannelOutboundAdapter = { const send = deps?.sendSlack ?? sendMessageSlack; // Use threadId fallback so routed tool notifications stay in the Slack thread. const threadTs = replyToId ?? (threadId != null ? String(threadId) : undefined); - const result = await send(to, text, { + let finalText = text; + + // Run message_sending hooks (e.g. thread-ownership can cancel the send). + const hookRunner = getGlobalHookRunner(); + if (hookRunner?.hasHooks("message_sending")) { + const hookResult = await hookRunner.runMessageSending( + { to, content: text, metadata: { threadTs, channelId: to } }, + { channelId: "slack", accountId: accountId ?? undefined }, + ); + if (hookResult?.cancel) { + return { + channel: "slack", + messageId: "cancelled-by-hook", + channelId: to, + meta: { cancelled: true }, + }; + } + if (hookResult?.content) { + finalText = hookResult.content; + } + } + + const result = await send(to, finalText, { threadTs, accountId: accountId ?? undefined, }); @@ -19,7 +42,29 @@ export const slackOutbound: ChannelOutboundAdapter = { const send = deps?.sendSlack ?? sendMessageSlack; // Use threadId fallback so routed tool notifications stay in the Slack thread. const threadTs = replyToId ?? (threadId != null ? String(threadId) : undefined); - const result = await send(to, text, { + let finalText = text; + + // Run message_sending hooks (e.g. thread-ownership can cancel the send). + const hookRunner = getGlobalHookRunner(); + if (hookRunner?.hasHooks("message_sending")) { + const hookResult = await hookRunner.runMessageSending( + { to, content: text, metadata: { threadTs, channelId: to, mediaUrl } }, + { channelId: "slack", accountId: accountId ?? undefined }, + ); + if (hookResult?.cancel) { + return { + channel: "slack", + messageId: "cancelled-by-hook", + channelId: to, + meta: { cancelled: true }, + }; + } + if (hookResult?.content) { + finalText = hookResult.content; + } + } + + const result = await send(to, finalText, { mediaUrl, threadTs, accountId: accountId ?? undefined, From ad57e561c6a82cbd13ee14d58fb92e927023a47c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 14 Feb 2026 00:38:10 +0100 Subject: [PATCH 0020/1038] refactor: unify gateway restart deferral and dispatcher cleanup --- src/auto-reply/dispatch.test.ts | 61 +++++++++++ src/auto-reply/dispatch.ts | 59 +++++++--- src/cli/gateway-cli/run-loop.test.ts | 4 + src/cli/gateway-cli/run-loop.ts | 2 + src/gateway/server-methods/chat.ts | 62 ++++++----- src/gateway/server-reload-handlers.ts | 117 ++++++++++---------- src/imessage/monitor/monitor-provider.ts | 26 +++-- src/infra/infra-runtime.test.ts | 21 ++++ src/infra/restart.ts | 133 ++++++++++++++++------- src/macos/gateway-daemon.ts | 7 +- 10 files changed, 337 insertions(+), 155 deletions(-) create mode 100644 src/auto-reply/dispatch.test.ts diff --git a/src/auto-reply/dispatch.test.ts b/src/auto-reply/dispatch.test.ts new file mode 100644 index 00000000000..b07f720ab8b --- /dev/null +++ b/src/auto-reply/dispatch.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import type { ReplyDispatcher } from "./reply/reply-dispatcher.js"; +import { withReplyDispatcher } from "./dispatch.js"; + +function createDispatcher(record: string[]): ReplyDispatcher { + return { + sendToolResult: () => true, + sendBlockReply: () => true, + sendFinalReply: () => true, + getQueuedCounts: () => ({ tool: 0, block: 0, final: 0 }), + markComplete: () => { + record.push("markComplete"); + }, + waitForIdle: async () => { + record.push("waitForIdle"); + }, + }; +} + +describe("withReplyDispatcher", () => { + it("always marks complete and waits for idle after success", async () => { + const order: string[] = []; + const dispatcher = createDispatcher(order); + + const result = await withReplyDispatcher({ + dispatcher, + run: async () => { + order.push("run"); + return "ok"; + }, + onSettled: () => { + order.push("onSettled"); + }, + }); + + expect(result).toBe("ok"); + expect(order).toEqual(["run", "markComplete", "waitForIdle", "onSettled"]); + }); + + it("still drains dispatcher after run throws", async () => { + const order: string[] = []; + const dispatcher = createDispatcher(order); + const onSettled = vi.fn(() => { + order.push("onSettled"); + }); + + await expect( + withReplyDispatcher({ + dispatcher, + run: async () => { + order.push("run"); + throw new Error("boom"); + }, + onSettled, + }), + ).rejects.toThrow("boom"); + + expect(onSettled).toHaveBeenCalledTimes(1); + expect(order).toEqual(["run", "markComplete", "waitForIdle", "onSettled"]); + }); +}); diff --git a/src/auto-reply/dispatch.ts b/src/auto-reply/dispatch.ts index d018623c7e0..32f89beb173 100644 --- a/src/auto-reply/dispatch.ts +++ b/src/auto-reply/dispatch.ts @@ -14,6 +14,24 @@ import { export type DispatchInboundResult = DispatchFromConfigResult; +export async function withReplyDispatcher(params: { + dispatcher: ReplyDispatcher; + run: () => Promise; + onSettled?: () => void | Promise; +}): Promise { + try { + return await params.run(); + } finally { + // Ensure dispatcher reservations are always released on every exit path. + params.dispatcher.markComplete(); + try { + await params.dispatcher.waitForIdle(); + } finally { + await params.onSettled?.(); + } + } +} + export async function dispatchInboundMessage(params: { ctx: MsgContext | FinalizedMsgContext; cfg: OpenClawConfig; @@ -41,20 +59,23 @@ export async function dispatchInboundMessageWithBufferedDispatcher(params: { const { dispatcher, replyOptions, markDispatchIdle } = createReplyDispatcherWithTyping( params.dispatcherOptions, ); - - const result = await dispatchInboundMessage({ - ctx: params.ctx, - cfg: params.cfg, + return await withReplyDispatcher({ dispatcher, - replyResolver: params.replyResolver, - replyOptions: { - ...params.replyOptions, - ...replyOptions, + run: async () => + dispatchInboundMessage({ + ctx: params.ctx, + cfg: params.cfg, + dispatcher, + replyResolver: params.replyResolver, + replyOptions: { + ...params.replyOptions, + ...replyOptions, + }, + }), + onSettled: () => { + markDispatchIdle(); }, }); - - markDispatchIdle(); - return result; } export async function dispatchInboundMessageWithDispatcher(params: { @@ -65,13 +86,15 @@ export async function dispatchInboundMessageWithDispatcher(params: { replyResolver?: typeof import("./reply.js").getReplyFromConfig; }): Promise { const dispatcher = createReplyDispatcher(params.dispatcherOptions); - const result = await dispatchInboundMessage({ - ctx: params.ctx, - cfg: params.cfg, + return await withReplyDispatcher({ dispatcher, - replyResolver: params.replyResolver, - replyOptions: params.replyOptions, + run: async () => + dispatchInboundMessage({ + ctx: params.ctx, + cfg: params.cfg, + dispatcher, + replyResolver: params.replyResolver, + replyOptions: params.replyOptions, + }), }); - await dispatcher.waitForIdle(); - return result; } diff --git a/src/cli/gateway-cli/run-loop.test.ts b/src/cli/gateway-cli/run-loop.test.ts index 928e02cc5e9..f2de12bcb57 100644 --- a/src/cli/gateway-cli/run-loop.test.ts +++ b/src/cli/gateway-cli/run-loop.test.ts @@ -5,6 +5,7 @@ const acquireGatewayLock = vi.fn(async () => ({ })); const consumeGatewaySigusr1RestartAuthorization = vi.fn(() => true); const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false); +const markGatewaySigusr1RestartHandled = vi.fn(); const getActiveTaskCount = vi.fn(() => 0); const waitForActiveTasks = vi.fn(async () => ({ drained: true })); const resetAllLanes = vi.fn(); @@ -22,6 +23,7 @@ vi.mock("../../infra/gateway-lock.js", () => ({ vi.mock("../../infra/restart.js", () => ({ consumeGatewaySigusr1RestartAuthorization: () => consumeGatewaySigusr1RestartAuthorization(), isGatewaySigusr1RestartExternallyAllowed: () => isGatewaySigusr1RestartExternallyAllowed(), + markGatewaySigusr1RestartHandled: () => markGatewaySigusr1RestartHandled(), })); vi.mock("../../process/command-queue.js", () => ({ @@ -100,6 +102,7 @@ describe("runGatewayLoop", () => { reason: "gateway restarting", restartExpectedMs: 1500, }); + expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledTimes(1); expect(resetAllLanes).toHaveBeenCalledTimes(1); process.emit("SIGUSR1"); @@ -109,6 +112,7 @@ describe("runGatewayLoop", () => { reason: "gateway restarting", restartExpectedMs: 1500, }); + expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledTimes(2); expect(resetAllLanes).toHaveBeenCalledTimes(2); } finally { removeNewSignalListeners("SIGTERM", beforeSigterm); diff --git a/src/cli/gateway-cli/run-loop.ts b/src/cli/gateway-cli/run-loop.ts index ec582fdcb8d..7cd1902f57f 100644 --- a/src/cli/gateway-cli/run-loop.ts +++ b/src/cli/gateway-cli/run-loop.ts @@ -4,6 +4,7 @@ import { acquireGatewayLock } from "../../infra/gateway-lock.js"; import { consumeGatewaySigusr1RestartAuthorization, isGatewaySigusr1RestartExternallyAllowed, + markGatewaySigusr1RestartHandled, } from "../../infra/restart.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { @@ -108,6 +109,7 @@ export async function runGatewayLoop(params: { ); return; } + markGatewaySigusr1RestartHandled(); request("restart", "SIGUSR1"); }; diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index 28ea99b60b2..b099364cb2a 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -6,7 +6,7 @@ import type { GatewayRequestContext, GatewayRequestHandlers } from "./types.js"; import { resolveSessionAgentId } from "../../agents/agent-scope.js"; import { resolveThinkingDefault } from "../../agents/model-selection.js"; import { resolveAgentTimeoutMs } from "../../agents/timeout.js"; -import { dispatchInboundMessage } from "../../auto-reply/dispatch.js"; +import { dispatchInboundMessage, withReplyDispatcher } from "../../auto-reply/dispatch.js"; import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.js"; import { createReplyPrefixOptions } from "../../channels/reply-prefix.js"; import { resolveSessionFilePath } from "../../config/sessions.js"; @@ -524,36 +524,40 @@ export const chatHandlers: GatewayRequestHandlers = { }); let agentRunStarted = false; - void dispatchInboundMessage({ - ctx, - cfg, + void withReplyDispatcher({ dispatcher, - replyOptions: { - runId: clientRunId, - abortSignal: abortController.signal, - images: parsedImages.length > 0 ? parsedImages : undefined, - disableBlockStreaming: true, - onAgentRunStart: (runId) => { - agentRunStarted = true; - const connId = typeof client?.connId === "string" ? client.connId : undefined; - const wantsToolEvents = hasGatewayClientCap( - client?.connect?.caps, - GATEWAY_CLIENT_CAPS.TOOL_EVENTS, - ); - if (connId && wantsToolEvents) { - context.registerToolEventRecipient(runId, connId); - // Register for any other active runs *in the same session* so - // late-joining clients (e.g. page refresh mid-response) receive - // in-progress tool events without leaking cross-session data. - for (const [activeRunId, active] of context.chatAbortControllers) { - if (activeRunId !== runId && active.sessionKey === p.sessionKey) { - context.registerToolEventRecipient(activeRunId, connId); + run: () => + dispatchInboundMessage({ + ctx, + cfg, + dispatcher, + replyOptions: { + runId: clientRunId, + abortSignal: abortController.signal, + images: parsedImages.length > 0 ? parsedImages : undefined, + disableBlockStreaming: true, + onAgentRunStart: (runId) => { + agentRunStarted = true; + const connId = typeof client?.connId === "string" ? client.connId : undefined; + const wantsToolEvents = hasGatewayClientCap( + client?.connect?.caps, + GATEWAY_CLIENT_CAPS.TOOL_EVENTS, + ); + if (connId && wantsToolEvents) { + context.registerToolEventRecipient(runId, connId); + // Register for any other active runs *in the same session* so + // late-joining clients (e.g. page refresh mid-response) receive + // in-progress tool events without leaking cross-session data. + for (const [activeRunId, active] of context.chatAbortControllers) { + if (activeRunId !== runId && active.sessionKey === p.sessionKey) { + context.registerToolEventRecipient(activeRunId, connId); + } + } } - } - } - }, - onModelSelected, - }, + }, + onModelSelected, + }, + }), }) .then(() => { if (!agentRunStarted) { diff --git a/src/gateway/server-reload-handlers.ts b/src/gateway/server-reload-handlers.ts index 02ec35bc306..6a2dfd2cb27 100644 --- a/src/gateway/server-reload-handlers.ts +++ b/src/gateway/server-reload-handlers.ts @@ -8,7 +8,11 @@ import { resolveAgentMaxConcurrent, resolveSubagentMaxConcurrent } from "../conf import { startGmailWatcher, stopGmailWatcher } from "../hooks/gmail-watcher.js"; import { isTruthyEnvValue } from "../infra/env.js"; import { resetDirectoryCache } from "../infra/outbound/target-resolver.js"; -import { emitGatewayRestart, setGatewaySigusr1RestartPolicy } from "../infra/restart.js"; +import { + deferGatewayRestartUntilIdle, + emitGatewayRestart, + setGatewaySigusr1RestartPolicy, +} from "../infra/restart.js"; import { setCommandLaneConcurrency, getTotalQueueSize } from "../process/command-queue.js"; import { CommandLane } from "../process/lanes.js"; import { resolveHooksConfig } from "./hooks.js"; @@ -155,13 +159,33 @@ export function createGatewayReloadHandlers(params: { return; } - // Check if there are active operations (commands in queue, pending replies, or embedded runs) - const queueSize = getTotalQueueSize(); - const pendingReplies = getTotalPendingReplies(); - const embeddedRuns = getActiveEmbeddedRunCount(); - const totalActive = queueSize + pendingReplies + embeddedRuns; + const getActiveCounts = () => { + const queueSize = getTotalQueueSize(); + const pendingReplies = getTotalPendingReplies(); + const embeddedRuns = getActiveEmbeddedRunCount(); + return { + queueSize, + pendingReplies, + embeddedRuns, + totalActive: queueSize + pendingReplies + embeddedRuns, + }; + }; + const formatActiveDetails = (counts: ReturnType) => { + const details = []; + if (counts.queueSize > 0) { + details.push(`${counts.queueSize} operation(s)`); + } + if (counts.pendingReplies > 0) { + details.push(`${counts.pendingReplies} reply(ies)`); + } + if (counts.embeddedRuns > 0) { + details.push(`${counts.embeddedRuns} embedded run(s)`); + } + return details; + }; + const active = getActiveCounts(); - if (totalActive > 0) { + if (active.totalActive > 0) { // Avoid spinning up duplicate polling loops from repeated config changes. if (restartPending) { params.logReload.info( @@ -170,63 +194,40 @@ export function createGatewayReloadHandlers(params: { return; } restartPending = true; - const details = []; - if (queueSize > 0) { - details.push(`${queueSize} queued operation(s)`); - } - if (pendingReplies > 0) { - details.push(`${pendingReplies} pending reply(ies)`); - } - if (embeddedRuns > 0) { - details.push(`${embeddedRuns} embedded run(s)`); - } + const initialDetails = formatActiveDetails(active); params.logReload.warn( - `config change requires gateway restart (${reasons}) — deferring until ${details.join(", ")} complete`, + `config change requires gateway restart (${reasons}) — deferring until ${initialDetails.join(", ")} complete`, ); - // Wait for all operations and replies to complete before restarting (max 30 seconds) - const maxWaitMs = 30_000; - const checkIntervalMs = 500; - const startTime = Date.now(); - - const checkAndRestart = () => { - const currentQueueSize = getTotalQueueSize(); - const currentPendingReplies = getTotalPendingReplies(); - const currentEmbeddedRuns = getActiveEmbeddedRunCount(); - const currentTotalActive = currentQueueSize + currentPendingReplies + currentEmbeddedRuns; - const elapsed = Date.now() - startTime; - - if (currentTotalActive === 0) { - restartPending = false; - params.logReload.info("all operations and replies completed; restarting gateway now"); - emitGatewayRestart(); - } else if (elapsed >= maxWaitMs) { - const remainingDetails = []; - if (currentQueueSize > 0) { - remainingDetails.push(`${currentQueueSize} operation(s)`); - } - if (currentPendingReplies > 0) { - remainingDetails.push(`${currentPendingReplies} reply(ies)`); - } - if (currentEmbeddedRuns > 0) { - remainingDetails.push(`${currentEmbeddedRuns} embedded run(s)`); - } - restartPending = false; - params.logReload.warn( - `restart timeout after ${elapsed}ms with ${remainingDetails.join(", ")} still active; restarting anyway`, - ); - emitGatewayRestart(); - } else { - // Check again soon - setTimeout(checkAndRestart, checkIntervalMs); - } - }; - - setTimeout(checkAndRestart, checkIntervalMs); + deferGatewayRestartUntilIdle({ + getPendingCount: () => getActiveCounts().totalActive, + hooks: { + onReady: () => { + restartPending = false; + params.logReload.info("all operations and replies completed; restarting gateway now"); + }, + onTimeout: (_pending, elapsedMs) => { + const remaining = formatActiveDetails(getActiveCounts()); + restartPending = false; + params.logReload.warn( + `restart timeout after ${elapsedMs}ms with ${remaining.join(", ")} still active; restarting anyway`, + ); + }, + onCheckError: (err) => { + restartPending = false; + params.logReload.warn( + `restart deferral check failed (${String(err)}); restarting gateway now`, + ); + }, + }, + }); } else { // No active operations or pending replies, restart immediately params.logReload.warn(`config change requires gateway restart (${reasons})`); - emitGatewayRestart(); + const emitted = emitGatewayRestart(); + if (!emitted) { + params.logReload.info("gateway restart already scheduled; skipping duplicate signal"); + } } }; diff --git a/src/imessage/monitor/monitor-provider.ts b/src/imessage/monitor/monitor-provider.ts index 445fe73aeae..771003f2fa9 100644 --- a/src/imessage/monitor/monitor-provider.ts +++ b/src/imessage/monitor/monitor-provider.ts @@ -3,7 +3,7 @@ import type { IMessagePayload, MonitorIMessageOpts } from "./types.js"; import { resolveHumanDelayConfig } from "../../agents/identity.js"; import { resolveTextChunkLimit } from "../../auto-reply/chunk.js"; import { hasControlCommand } from "../../auto-reply/command-detection.js"; -import { dispatchInboundMessage } from "../../auto-reply/dispatch.js"; +import { dispatchInboundMessage, withReplyDispatcher } from "../../auto-reply/dispatch.js"; import { formatInboundEnvelope, formatInboundFromLabel, @@ -647,17 +647,21 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P }, }); - const { queuedFinal } = await dispatchInboundMessage({ - ctx: ctxPayload, - cfg, + const { queuedFinal } = await withReplyDispatcher({ dispatcher, - replyOptions: { - disableBlockStreaming: - typeof accountInfo.config.blockStreaming === "boolean" - ? !accountInfo.config.blockStreaming - : undefined, - onModelSelected, - }, + run: () => + dispatchInboundMessage({ + ctx: ctxPayload, + cfg, + dispatcher, + replyOptions: { + disableBlockStreaming: + typeof accountInfo.config.blockStreaming === "boolean" + ? !accountInfo.config.blockStreaming + : undefined, + onModelSelected, + }, + }), }); if (!queuedFinal) { diff --git a/src/infra/infra-runtime.test.ts b/src/infra/infra-runtime.test.ts index 61e7dff4393..78e6d15f9a3 100644 --- a/src/infra/infra-runtime.test.ts +++ b/src/infra/infra-runtime.test.ts @@ -6,7 +6,9 @@ import { ensureBinary } from "./binaries.js"; import { __testing, consumeGatewaySigusr1RestartAuthorization, + emitGatewayRestart, isGatewaySigusr1RestartExternallyAllowed, + markGatewaySigusr1RestartHandled, scheduleGatewaySigusr1Restart, setGatewaySigusr1RestartPolicy, setPreRestartDeferralCheck, @@ -100,6 +102,25 @@ describe("infra runtime", () => { setGatewaySigusr1RestartPolicy({ allowExternal: true }); expect(isGatewaySigusr1RestartExternallyAllowed()).toBe(true); }); + + it("suppresses duplicate emit until the restart cycle is marked handled", () => { + const emitSpy = vi.spyOn(process, "emit"); + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + expect(emitGatewayRestart()).toBe(true); + expect(emitGatewayRestart()).toBe(false); + expect(consumeGatewaySigusr1RestartAuthorization()).toBe(true); + + markGatewaySigusr1RestartHandled(); + + expect(emitGatewayRestart()).toBe(true); + const sigusr1Emits = emitSpy.mock.calls.filter((args) => args[0] === "SIGUSR1"); + expect(sigusr1Emits.length).toBe(2); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); }); describe("pre-restart deferral check", () => { diff --git a/src/infra/restart.ts b/src/infra/restart.ts index 830d0731049..60540884b90 100644 --- a/src/infra/restart.ts +++ b/src/infra/restart.ts @@ -13,12 +13,20 @@ export type RestartAttempt = { const SPAWN_TIMEOUT_MS = 2000; const SIGUSR1_AUTH_GRACE_MS = 5000; +const DEFAULT_DEFERRAL_POLL_MS = 500; +const DEFAULT_DEFERRAL_MAX_WAIT_MS = 30_000; let sigusr1AuthorizedCount = 0; let sigusr1AuthorizedUntil = 0; let sigusr1ExternalAllowed = false; let preRestartCheck: (() => number) | null = null; -let sigusr1Emitted = false; +let restartCycleToken = 0; +let emittedRestartToken = 0; +let consumedRestartToken = 0; + +function hasUnconsumedRestartSignal(): boolean { + return emittedRestartToken > consumedRestartToken; +} /** * Register a callback that scheduleGatewaySigusr1Restart checks before emitting SIGUSR1. @@ -35,10 +43,11 @@ export function setPreRestartDeferralCheck(fn: () => number): void { * to ensure only one restart fires. */ export function emitGatewayRestart(): boolean { - if (sigusr1Emitted) { + if (hasUnconsumedRestartSignal()) { return false; } - sigusr1Emitted = true; + const cycleToken = ++restartCycleToken; + emittedRestartToken = cycleToken; authorizeGatewaySigusr1Restart(); try { if (process.listenerCount("SIGUSR1") > 0) { @@ -47,7 +56,9 @@ export function emitGatewayRestart(): boolean { process.kill(process.pid, "SIGUSR1"); } } catch { - /* ignore */ + // Roll back the cycle marker so future restart requests can still proceed. + emittedRestartToken = consumedRestartToken; + return false; } return true; } @@ -85,10 +96,6 @@ export function consumeGatewaySigusr1RestartAuthorization(): boolean { if (sigusr1AuthorizedCount <= 0) { return false; } - // Reset the emission guard so the next restart cycle can fire. - // The run loop re-enters startGatewayServer() after close(), which - // re-registers setPreRestartDeferralCheck and can schedule new restarts. - sigusr1Emitted = false; sigusr1AuthorizedCount -= 1; if (sigusr1AuthorizedCount <= 0) { sigusr1AuthorizedUntil = 0; @@ -96,6 +103,80 @@ export function consumeGatewaySigusr1RestartAuthorization(): boolean { return true; } +/** + * Mark the currently emitted SIGUSR1 restart cycle as consumed by the run loop. + * This explicitly advances the cycle state instead of resetting emit guards inside + * consumeGatewaySigusr1RestartAuthorization(). + */ +export function markGatewaySigusr1RestartHandled(): void { + if (hasUnconsumedRestartSignal()) { + consumedRestartToken = emittedRestartToken; + } +} + +export type RestartDeferralHooks = { + onDeferring?: (pending: number) => void; + onReady?: () => void; + onTimeout?: (pending: number, elapsedMs: number) => void; + onCheckError?: (err: unknown) => void; +}; + +/** + * Poll pending work until it drains (or times out), then emit one restart signal. + * Shared by both the direct RPC restart path and the config watcher path. + */ +export function deferGatewayRestartUntilIdle(opts: { + getPendingCount: () => number; + hooks?: RestartDeferralHooks; + pollMs?: number; + maxWaitMs?: number; +}): void { + const pollMsRaw = opts.pollMs ?? DEFAULT_DEFERRAL_POLL_MS; + const pollMs = Math.max(10, Math.floor(pollMsRaw)); + const maxWaitMsRaw = opts.maxWaitMs ?? DEFAULT_DEFERRAL_MAX_WAIT_MS; + const maxWaitMs = Math.max(pollMs, Math.floor(maxWaitMsRaw)); + + let pending: number; + try { + pending = opts.getPendingCount(); + } catch (err) { + opts.hooks?.onCheckError?.(err); + emitGatewayRestart(); + return; + } + if (pending <= 0) { + opts.hooks?.onReady?.(); + emitGatewayRestart(); + return; + } + + opts.hooks?.onDeferring?.(pending); + const startedAt = Date.now(); + const poll = setInterval(() => { + let current: number; + try { + current = opts.getPendingCount(); + } catch (err) { + clearInterval(poll); + opts.hooks?.onCheckError?.(err); + emitGatewayRestart(); + return; + } + if (current <= 0) { + clearInterval(poll); + opts.hooks?.onReady?.(); + emitGatewayRestart(); + return; + } + const elapsedMs = Date.now() - startedAt; + if (elapsedMs >= maxWaitMs) { + clearInterval(poll); + opts.hooks?.onTimeout?.(current, elapsedMs); + emitGatewayRestart(); + } + }, pollMs); +} + function formatSpawnDetail(result: { error?: unknown; status?: number | null; @@ -227,40 +308,14 @@ export function scheduleGatewaySigusr1Restart(opts?: { typeof opts?.reason === "string" && opts.reason.trim() ? opts.reason.trim().slice(0, 200) : undefined; - const DEFERRAL_POLL_MS = 500; - const DEFERRAL_MAX_WAIT_MS = 30_000; setTimeout(() => { - if (!preRestartCheck) { + const pendingCheck = preRestartCheck; + if (!pendingCheck) { emitGatewayRestart(); return; } - let pending: number; - try { - pending = preRestartCheck(); - } catch { - emitGatewayRestart(); - return; - } - if (pending <= 0) { - emitGatewayRestart(); - return; - } - // Poll until pending work drains or timeout - let waited = 0; - const poll = setInterval(() => { - waited += DEFERRAL_POLL_MS; - let current: number; - try { - current = preRestartCheck!(); - } catch { - current = 0; - } - if (current <= 0 || waited >= DEFERRAL_MAX_WAIT_MS) { - clearInterval(poll); - emitGatewayRestart(); - } - }, DEFERRAL_POLL_MS); + deferGatewayRestartUntilIdle({ getPendingCount: pendingCheck }); }, delayMs); return { ok: true, @@ -278,6 +333,8 @@ export const __testing = { sigusr1AuthorizedUntil = 0; sigusr1ExternalAllowed = false; preRestartCheck = null; - sigusr1Emitted = false; + restartCycleToken = 0; + emittedRestartToken = 0; + consumedRestartToken = 0; }, }; diff --git a/src/macos/gateway-daemon.ts b/src/macos/gateway-daemon.ts index 38fd5485ff0..a33ca94e81c 100644 --- a/src/macos/gateway-daemon.ts +++ b/src/macos/gateway-daemon.ts @@ -49,7 +49,11 @@ async function main() { { setGatewayWsLogStyle }, { setVerbose }, { acquireGatewayLock, GatewayLockError }, - { consumeGatewaySigusr1RestartAuthorization, isGatewaySigusr1RestartExternallyAllowed }, + { + consumeGatewaySigusr1RestartAuthorization, + isGatewaySigusr1RestartExternallyAllowed, + markGatewaySigusr1RestartHandled, + }, { defaultRuntime }, { enableConsoleCapture, setConsoleTimestampPrefix }, commandQueueMod, @@ -201,6 +205,7 @@ async function main() { ); return; } + markGatewaySigusr1RestartHandled(); request("restart", "SIGUSR1"); }; From 5caf829d28a0f69fa7c49e3aa9205ae9d16641b8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 23:40:25 +0000 Subject: [PATCH 0021/1038] perf(test): trim duplicate gateway and auto-reply test overhead --- src/auto-reply/reply.block-streaming.test.ts | 45 ---------- src/auto-reply/reply.raw-body.test.ts | 40 ++------- .../server-reload.config-during-reply.test.ts | 47 +---------- src/gateway/server-reload.integration.test.ts | 82 +------------------ .../server-reload.real-scenario.test.ts | 8 +- src/process/command-queue.test.ts | 46 +++++------ 6 files changed, 34 insertions(+), 234 deletions(-) diff --git a/src/auto-reply/reply.block-streaming.test.ts b/src/auto-reply/reply.block-streaming.test.ts index e051944dc9e..d982280ab47 100644 --- a/src/auto-reply/reply.block-streaming.test.ts +++ b/src/auto-reply/reply.block-streaming.test.ts @@ -164,51 +164,6 @@ describe("block streaming", () => { }); }); - it("drops final payloads when block replies streamed", async () => { - await withTempHome(async (home) => { - const onBlockReply = vi.fn().mockResolvedValue(undefined); - - const impl = async (params: RunEmbeddedPiAgentParams) => { - void params.onBlockReply?.({ text: "chunk-1" }); - return { - payloads: [{ text: "chunk-1\nchunk-2" }], - meta: { - durationMs: 5, - agentMeta: { sessionId: "s", provider: "p", model: "m" }, - }, - }; - }; - piEmbeddedMock.runEmbeddedPiAgent.mockImplementation(impl); - - const res = await getReplyFromConfig( - { - Body: "ping", - From: "+1004", - To: "+2000", - MessageSid: "msg-124", - Provider: "discord", - }, - { - onBlockReply, - disableBlockStreaming: false, - }, - { - agents: { - defaults: { - model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw"), - }, - }, - channels: { whatsapp: { allowFrom: ["*"] } }, - session: { store: path.join(home, "sessions.json") }, - }, - ); - - expect(res).toBeUndefined(); - expect(onBlockReply).toHaveBeenCalledTimes(1); - }); - }); - it("falls back to final payloads when block reply send times out", async () => { await withTempHome(async (home) => { let sawAbort = false; diff --git a/src/auto-reply/reply.raw-body.test.ts b/src/auto-reply/reply.raw-body.test.ts index e66b174e05a..0b19df8a124 100644 --- a/src/auto-reply/reply.raw-body.test.ts +++ b/src/auto-reply/reply.raw-body.test.ts @@ -161,36 +161,10 @@ describe("RawBody directive parsing", () => { }, expectedIncludes: ["Verbose logging enabled."], }); - - await assertCommandReply({ - message: { - Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /status\\n[from: Jake McInteer (+6421807830)]`, - RawBody: "/status", - ChatType: "group", - From: "+1222", - To: "+1222", - SessionKey: "agent:main:whatsapp:group:g1", - Provider: "whatsapp", - Surface: "whatsapp", - SenderE164: "+1222", - CommandAuthorized: true, - }, - config: { - agents: { - defaults: { - model: "anthropic/claude-opus-4-5", - workspace: path.join(home, "openclaw-3"), - }, - }, - channels: { whatsapp: { allowFrom: ["+1222"] } }, - session: { store: path.join(home, "sessions-3.json") }, - }, - expectedIncludes: ["Session: agent:main:whatsapp:group:g1", "anthropic/claude-opus-4-5"], - }); }); }); - it("preserves history when RawBody is provided for command parsing", async () => { + it("preserves history and reuses non-default agent session files", async () => { await withTempHome(async (home) => { vi.mocked(runEmbeddedPiAgent).mockResolvedValue({ payloads: [{ text: "ok" }], @@ -238,11 +212,6 @@ describe("RawBody directive parsing", () => { expect(prompt).toContain('"body": "hello"'); expect(prompt).toContain("status please"); expect(prompt).not.toContain("/think:high"); - }); - }); - - it("reuses non-default agent session files without throwing path validation errors", async () => { - await withTempHome(async (home) => { const agentId = "worker1"; const sessionId = "sess-worker-1"; const sessionKey = `agent:${agentId}:telegram:12345`; @@ -259,6 +228,7 @@ describe("RawBody directive parsing", () => { }, }); + vi.mocked(runEmbeddedPiAgent).mockReset(); vi.mocked(runEmbeddedPiAgent).mockResolvedValue({ payloads: [{ text: "ok" }], meta: { @@ -267,7 +237,7 @@ describe("RawBody directive parsing", () => { }, }); - const res = await getReplyFromConfig( + const resWorker = await getReplyFromConfig( { Body: "hello", From: "telegram:12345", @@ -288,8 +258,8 @@ describe("RawBody directive parsing", () => { }, ); - const text = Array.isArray(res) ? res[0]?.text : res?.text; - expect(text).toBe("ok"); + const textWorker = Array.isArray(resWorker) ? resWorker[0]?.text : resWorker?.text; + expect(textWorker).toBe("ok"); expect(runEmbeddedPiAgent).toHaveBeenCalledOnce(); expect(vi.mocked(runEmbeddedPiAgent).mock.calls[0]?.[0]?.sessionFile).toBe(sessionFile); }); diff --git a/src/gateway/server-reload.config-during-reply.test.ts b/src/gateway/server-reload.config-during-reply.test.ts index 2ae95be5557..326e9de759b 100644 --- a/src/gateway/server-reload.config-during-reply.test.ts +++ b/src/gateway/server-reload.config-during-reply.test.ts @@ -36,7 +36,7 @@ describe("gateway config reload during reply", () => { const dispatcher = createReplyDispatcher({ deliver: async (payload) => { // Simulate async reply delivery - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => setTimeout(resolve, 20)); deliveredReplies.push(payload.text ?? ""); }, onError: (err) => { @@ -103,49 +103,4 @@ describe("gateway config reload during reply", () => { expect(deliverCalled).toBe(false); expect(getTotalPendingReplies()).toBe(0); }); - - it("should integrate dispatcher reservation with concurrent dispatchers", async () => { - const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); - const { getTotalQueueSize } = await import("../process/command-queue.js"); - - const deliveredReplies: string[] = []; - const dispatcher = createReplyDispatcher({ - deliver: async (payload) => { - await new Promise((resolve) => setTimeout(resolve, 50)); - deliveredReplies.push(payload.text ?? ""); - }, - }); - - // Dispatcher has reservation (pending=1) - expect(getTotalPendingReplies()).toBe(1); - - // Total active = queue + pending - const totalActive = getTotalQueueSize() + getTotalPendingReplies(); - expect(totalActive).toBe(1); // 0 queue + 1 pending - - // Command finishes, replies enqueued - dispatcher.sendFinalReply({ text: "Reply 1" }); - dispatcher.sendFinalReply({ text: "Reply 2" }); - - // Now: pending=3 (reservation + 2 replies) - expect(getTotalPendingReplies()).toBe(3); - - // Mark complete (flags reservation for cleanup on last delivery) - dispatcher.markComplete(); - - // Reservation still counted until delivery .finally() clears it, - // but the important invariant is pending > 0 while deliveries are in flight. - expect(getTotalPendingReplies()).toBeGreaterThan(0); - - // Wait for replies - await dispatcher.waitForIdle(); - - // Replies sent, pending=0 - expect(getTotalPendingReplies()).toBe(0); - expect(deliveredReplies).toEqual(["Reply 1", "Reply 2"]); - - // Now everything is idle - expect(getTotalPendingReplies()).toBe(0); - expect(getTotalQueueSize()).toBe(0); - }); }); diff --git a/src/gateway/server-reload.integration.test.ts b/src/gateway/server-reload.integration.test.ts index d2ab045fac3..3bd1bc80e3d 100644 --- a/src/gateway/server-reload.integration.test.ts +++ b/src/gateway/server-reload.integration.test.ts @@ -31,7 +31,7 @@ describe("gateway restart deferral integration", () => { const dispatcher = createReplyDispatcher({ deliver: async (payload) => { // Simulate network delay - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => setTimeout(resolve, 20)); deliveredReplies.push({ text: payload.text ?? "", timestamp: Date.now(), @@ -116,84 +116,4 @@ describe("gateway restart deferral integration", () => { "restart-can-proceed", ]); }); - - it("should handle concurrent dispatchers with config changes", async () => { - const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); - const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); - - // Simulate two messages being processed concurrently - const deliveredReplies: string[] = []; - - // Message 1 — dispatcher created - const dispatcher1 = createReplyDispatcher({ - deliver: async (payload) => { - await new Promise((resolve) => setTimeout(resolve, 50)); - deliveredReplies.push(`msg1: ${payload.text}`); - }, - }); - - // Message 2 — dispatcher created - const dispatcher2 = createReplyDispatcher({ - deliver: async (payload) => { - await new Promise((resolve) => setTimeout(resolve, 50)); - deliveredReplies.push(`msg2: ${payload.text}`); - }, - }); - - // Both dispatchers have reservations - expect(getTotalPendingReplies()).toBe(2); - - // Config change detected - should defer - const totalActive = getTotalPendingReplies(); - expect(totalActive).toBe(2); // 2 dispatcher reservations - - // Messages process and send replies - dispatcher1.sendFinalReply({ text: "Reply from message 1" }); - dispatcher1.markComplete(); - - dispatcher2.sendFinalReply({ text: "Reply from message 2" }); - dispatcher2.markComplete(); - - // Wait for both - await Promise.all([dispatcher1.waitForIdle(), dispatcher2.waitForIdle()]); - - // All idle - expect(getTotalPendingReplies()).toBe(0); - - // Replies delivered - expect(deliveredReplies).toHaveLength(2); - }); - - it("should handle rapid config changes without losing replies", async () => { - const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); - const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); - - const deliveredReplies: string[] = []; - - // Message received — dispatcher created - const dispatcher = createReplyDispatcher({ - deliver: async (payload) => { - await new Promise((resolve) => setTimeout(resolve, 200)); // Slow network - deliveredReplies.push(payload.text ?? ""); - }, - }); - - // Config change 1, 2, 3 (rapid changes) - // All should be deferred because dispatcher has pending replies - - // Send replies - dispatcher.sendFinalReply({ text: "Processing..." }); - dispatcher.sendFinalReply({ text: "Almost done..." }); - dispatcher.sendFinalReply({ text: "Complete!" }); - dispatcher.markComplete(); - - // Wait for all replies - await dispatcher.waitForIdle(); - - // All replies should be delivered - expect(deliveredReplies).toEqual(["Processing...", "Almost done...", "Complete!"]); - - // Now restart can proceed - expect(getTotalPendingReplies()).toBe(0); - }); }); diff --git a/src/gateway/server-reload.real-scenario.test.ts b/src/gateway/server-reload.real-scenario.test.ts index c3da2723f4e..19ece2234ae 100644 --- a/src/gateway/server-reload.real-scenario.test.ts +++ b/src/gateway/server-reload.real-scenario.test.ts @@ -36,7 +36,7 @@ describe("real scenario: config change during message processing", () => { throw new Error(error); } // Slow delivery — restart checks will run during this window - await new Promise((resolve) => setTimeout(resolve, 500)); + await new Promise((resolve) => setTimeout(resolve, 150)); deliveredReplies.push(payload.text ?? ""); }, onError: () => { @@ -59,7 +59,7 @@ describe("real scenario: config change during message processing", () => { // If the tracking is broken, pending would be 0 and we'd restart. let restartTriggered = false; for (let i = 0; i < 3; i++) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => setTimeout(resolve, 25)); const pending = getTotalPendingReplies(); if (pending === 0) { restartTriggered = true; @@ -86,7 +86,7 @@ describe("real scenario: config change during message processing", () => { const dispatcher = createReplyDispatcher({ deliver: async (_payload) => { - await new Promise((resolve) => setTimeout(resolve, 50)); + await new Promise((resolve) => setTimeout(resolve, 10)); }, }); @@ -94,7 +94,7 @@ describe("real scenario: config change during message processing", () => { expect(getTotalPendingReplies()).toBe(1); // Simulate command processing delay BEFORE reply is enqueued - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => setTimeout(resolve, 20)); // During this delay, pending should STILL be 1 (reservation active) expect(getTotalPendingReplies()).toBe(1); diff --git a/src/process/command-queue.test.ts b/src/process/command-queue.test.ts index 5c0b20930af..79b8389a8b5 100644 --- a/src/process/command-queue.test.ts +++ b/src/process/command-queue.test.ts @@ -112,8 +112,6 @@ describe("command queue", () => { await blocker; }); - // Give the event loop a tick for the task to start. - await new Promise((r) => setTimeout(r, 5)); expect(getActiveTaskCount()).toBe(1); resolve1(); @@ -136,18 +134,21 @@ describe("command queue", () => { await blocker; }); - // Give the task a tick to start. - await new Promise((r) => setTimeout(r, 5)); + vi.useFakeTimers(); + try { + const drainPromise = waitForActiveTasks(5000); - const drainPromise = waitForActiveTasks(5000); + // Resolve the blocker after a short delay. + setTimeout(() => resolve1(), 10); + await vi.advanceTimersByTimeAsync(100); - // Resolve the blocker after a short delay. - setTimeout(() => resolve1(), 50); + const { drained } = await drainPromise; + expect(drained).toBe(true); - const { drained } = await drainPromise; - expect(drained).toBe(true); - - await task; + await task; + } finally { + vi.useRealTimers(); + } }); it("waitForActiveTasks returns drained=false on timeout", async () => { @@ -160,13 +161,18 @@ describe("command queue", () => { await blocker; }); - await new Promise((r) => setTimeout(r, 5)); + vi.useFakeTimers(); + try { + const waitPromise = waitForActiveTasks(50); + await vi.advanceTimersByTimeAsync(100); + const { drained } = await waitPromise; + expect(drained).toBe(false); - const { drained } = await waitForActiveTasks(50); - expect(drained).toBe(false); - - resolve1(); - await task; + resolve1(); + await task; + } finally { + vi.useRealTimers(); + } }); it("resetAllLanes drains queued work immediately after reset", async () => { @@ -228,15 +234,12 @@ describe("command queue", () => { const first = enqueueCommandInLane(lane, async () => { await blocker1; }); - await new Promise((r) => setTimeout(r, 5)); - const drainPromise = waitForActiveTasks(2000); // Starts after waitForActiveTasks snapshot and should not block drain completion. const second = enqueueCommandInLane(lane, async () => { await blocker2; }); - await new Promise((r) => setTimeout(r, 5)); expect(getActiveTaskCount()).toBeGreaterThanOrEqual(2); resolve1(); @@ -262,9 +265,6 @@ describe("command queue", () => { // Second task is queued behind the first. const second = enqueueCommand(async () => "second"); - // Give the first task a tick to start. - await new Promise((r) => setTimeout(r, 5)); - const removed = clearCommandLane(); expect(removed).toBe(1); // only the queued (not active) entry From d5e25e0ad885b47ffb949e8cc78a8aeec7df6bc5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 14 Feb 2026 00:41:27 +0100 Subject: [PATCH 0022/1038] refactor: centralize dispatcher lifecycle ownership --- src/auto-reply/dispatch.test.ts | 32 +++++++++- src/auto-reply/dispatch.ts | 59 +++++++++--------- src/auto-reply/reply/dispatch-from-config.ts | 7 --- .../monitor/message-handler.process.test.ts | 9 ++- src/gateway/server-methods/chat.ts | 62 +++++++++---------- src/imessage/monitor/monitor-provider.ts | 26 ++++---- 6 files changed, 107 insertions(+), 88 deletions(-) diff --git a/src/auto-reply/dispatch.test.ts b/src/auto-reply/dispatch.test.ts index b07f720ab8b..9e9630c406c 100644 --- a/src/auto-reply/dispatch.test.ts +++ b/src/auto-reply/dispatch.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; import type { ReplyDispatcher } from "./reply/reply-dispatcher.js"; -import { withReplyDispatcher } from "./dispatch.js"; +import { dispatchInboundMessage, withReplyDispatcher } from "./dispatch.js"; +import { buildTestCtx } from "./reply/test-ctx.js"; function createDispatcher(record: string[]): ReplyDispatcher { return { @@ -58,4 +60,32 @@ describe("withReplyDispatcher", () => { expect(onSettled).toHaveBeenCalledTimes(1); expect(order).toEqual(["run", "markComplete", "waitForIdle", "onSettled"]); }); + + it("dispatchInboundMessage owns dispatcher lifecycle", async () => { + const order: string[] = []; + const dispatcher = { + sendToolResult: () => true, + sendBlockReply: () => true, + sendFinalReply: () => { + order.push("sendFinalReply"); + return true; + }, + getQueuedCounts: () => ({ tool: 0, block: 0, final: 0 }), + markComplete: () => { + order.push("markComplete"); + }, + waitForIdle: async () => { + order.push("waitForIdle"); + }, + } satisfies ReplyDispatcher; + + await dispatchInboundMessage({ + ctx: buildTestCtx(), + cfg: {} as OpenClawConfig, + dispatcher, + replyResolver: async () => ({ text: "ok" }), + }); + + expect(order).toEqual(["sendFinalReply", "markComplete", "waitForIdle"]); + }); }); diff --git a/src/auto-reply/dispatch.ts b/src/auto-reply/dispatch.ts index 32f89beb173..54bf79a7bae 100644 --- a/src/auto-reply/dispatch.ts +++ b/src/auto-reply/dispatch.ts @@ -40,12 +40,16 @@ export async function dispatchInboundMessage(params: { replyResolver?: typeof import("./reply.js").getReplyFromConfig; }): Promise { const finalized = finalizeInboundContext(params.ctx); - return await dispatchReplyFromConfig({ - ctx: finalized, - cfg: params.cfg, + return await withReplyDispatcher({ dispatcher: params.dispatcher, - replyOptions: params.replyOptions, - replyResolver: params.replyResolver, + run: () => + dispatchReplyFromConfig({ + ctx: finalized, + cfg: params.cfg, + dispatcher: params.dispatcher, + replyOptions: params.replyOptions, + replyResolver: params.replyResolver, + }), }); } @@ -59,23 +63,20 @@ export async function dispatchInboundMessageWithBufferedDispatcher(params: { const { dispatcher, replyOptions, markDispatchIdle } = createReplyDispatcherWithTyping( params.dispatcherOptions, ); - return await withReplyDispatcher({ - dispatcher, - run: async () => - dispatchInboundMessage({ - ctx: params.ctx, - cfg: params.cfg, - dispatcher, - replyResolver: params.replyResolver, - replyOptions: { - ...params.replyOptions, - ...replyOptions, - }, - }), - onSettled: () => { - markDispatchIdle(); - }, - }); + try { + return await dispatchInboundMessage({ + ctx: params.ctx, + cfg: params.cfg, + dispatcher, + replyResolver: params.replyResolver, + replyOptions: { + ...params.replyOptions, + ...replyOptions, + }, + }); + } finally { + markDispatchIdle(); + } } export async function dispatchInboundMessageWithDispatcher(params: { @@ -86,15 +87,11 @@ export async function dispatchInboundMessageWithDispatcher(params: { replyResolver?: typeof import("./reply.js").getReplyFromConfig; }): Promise { const dispatcher = createReplyDispatcher(params.dispatcherOptions); - return await withReplyDispatcher({ + return await dispatchInboundMessage({ + ctx: params.ctx, + cfg: params.cfg, dispatcher, - run: async () => - dispatchInboundMessage({ - ctx: params.ctx, - cfg: params.cfg, - dispatcher, - replyResolver: params.replyResolver, - replyOptions: params.replyOptions, - }), + replyResolver: params.replyResolver, + replyOptions: params.replyOptions, }); } diff --git a/src/auto-reply/reply/dispatch-from-config.ts b/src/auto-reply/reply/dispatch-from-config.ts index 0f2cae6b4a2..45bd75040aa 100644 --- a/src/auto-reply/reply/dispatch-from-config.ts +++ b/src/auto-reply/reply/dispatch-from-config.ts @@ -278,7 +278,6 @@ export async function dispatchReplyFromConfig(params: { } else { queuedFinal = dispatcher.sendFinalReply(payload); } - await dispatcher.waitForIdle(); const counts = dispatcher.getQueuedCounts(); counts.final += routedFinalCount; recordProcessed("completed", { reason: "fast_abort" }); @@ -443,8 +442,6 @@ export async function dispatchReplyFromConfig(params: { } } - await dispatcher.waitForIdle(); - const counts = dispatcher.getQueuedCounts(); counts.final += routedFinalCount; recordProcessed("completed"); @@ -454,9 +451,5 @@ export async function dispatchReplyFromConfig(params: { recordProcessed("error", { error: String(err) }); markIdle("message_error"); throw err; - } finally { - // Always clear the dispatcher reservation so a leaked pending count - // can never permanently block gateway restarts. - dispatcher.markComplete(); } } diff --git a/src/discord/monitor/message-handler.process.test.ts b/src/discord/monitor/message-handler.process.test.ts index 619d120ca37..5e26257f317 100644 --- a/src/discord/monitor/message-handler.process.test.ts +++ b/src/discord/monitor/message-handler.process.test.ts @@ -20,7 +20,14 @@ vi.mock("../../auto-reply/reply/dispatch-from-config.js", () => ({ vi.mock("../../auto-reply/reply/reply-dispatcher.js", () => ({ createReplyDispatcherWithTyping: vi.fn(() => ({ - dispatcher: {}, + dispatcher: { + sendToolResult: vi.fn(() => true), + sendBlockReply: vi.fn(() => true), + sendFinalReply: vi.fn(() => true), + waitForIdle: vi.fn(async () => {}), + getQueuedCounts: vi.fn(() => ({ tool: 0, block: 0, final: 0 })), + markComplete: vi.fn(), + }, replyOptions: {}, markDispatchIdle: vi.fn(), })), diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index b099364cb2a..28ea99b60b2 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -6,7 +6,7 @@ import type { GatewayRequestContext, GatewayRequestHandlers } from "./types.js"; import { resolveSessionAgentId } from "../../agents/agent-scope.js"; import { resolveThinkingDefault } from "../../agents/model-selection.js"; import { resolveAgentTimeoutMs } from "../../agents/timeout.js"; -import { dispatchInboundMessage, withReplyDispatcher } from "../../auto-reply/dispatch.js"; +import { dispatchInboundMessage } from "../../auto-reply/dispatch.js"; import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.js"; import { createReplyPrefixOptions } from "../../channels/reply-prefix.js"; import { resolveSessionFilePath } from "../../config/sessions.js"; @@ -524,40 +524,36 @@ export const chatHandlers: GatewayRequestHandlers = { }); let agentRunStarted = false; - void withReplyDispatcher({ + void dispatchInboundMessage({ + ctx, + cfg, dispatcher, - run: () => - dispatchInboundMessage({ - ctx, - cfg, - dispatcher, - replyOptions: { - runId: clientRunId, - abortSignal: abortController.signal, - images: parsedImages.length > 0 ? parsedImages : undefined, - disableBlockStreaming: true, - onAgentRunStart: (runId) => { - agentRunStarted = true; - const connId = typeof client?.connId === "string" ? client.connId : undefined; - const wantsToolEvents = hasGatewayClientCap( - client?.connect?.caps, - GATEWAY_CLIENT_CAPS.TOOL_EVENTS, - ); - if (connId && wantsToolEvents) { - context.registerToolEventRecipient(runId, connId); - // Register for any other active runs *in the same session* so - // late-joining clients (e.g. page refresh mid-response) receive - // in-progress tool events without leaking cross-session data. - for (const [activeRunId, active] of context.chatAbortControllers) { - if (activeRunId !== runId && active.sessionKey === p.sessionKey) { - context.registerToolEventRecipient(activeRunId, connId); - } - } + replyOptions: { + runId: clientRunId, + abortSignal: abortController.signal, + images: parsedImages.length > 0 ? parsedImages : undefined, + disableBlockStreaming: true, + onAgentRunStart: (runId) => { + agentRunStarted = true; + const connId = typeof client?.connId === "string" ? client.connId : undefined; + const wantsToolEvents = hasGatewayClientCap( + client?.connect?.caps, + GATEWAY_CLIENT_CAPS.TOOL_EVENTS, + ); + if (connId && wantsToolEvents) { + context.registerToolEventRecipient(runId, connId); + // Register for any other active runs *in the same session* so + // late-joining clients (e.g. page refresh mid-response) receive + // in-progress tool events without leaking cross-session data. + for (const [activeRunId, active] of context.chatAbortControllers) { + if (activeRunId !== runId && active.sessionKey === p.sessionKey) { + context.registerToolEventRecipient(activeRunId, connId); } - }, - onModelSelected, - }, - }), + } + } + }, + onModelSelected, + }, }) .then(() => { if (!agentRunStarted) { diff --git a/src/imessage/monitor/monitor-provider.ts b/src/imessage/monitor/monitor-provider.ts index 771003f2fa9..445fe73aeae 100644 --- a/src/imessage/monitor/monitor-provider.ts +++ b/src/imessage/monitor/monitor-provider.ts @@ -3,7 +3,7 @@ import type { IMessagePayload, MonitorIMessageOpts } from "./types.js"; import { resolveHumanDelayConfig } from "../../agents/identity.js"; import { resolveTextChunkLimit } from "../../auto-reply/chunk.js"; import { hasControlCommand } from "../../auto-reply/command-detection.js"; -import { dispatchInboundMessage, withReplyDispatcher } from "../../auto-reply/dispatch.js"; +import { dispatchInboundMessage } from "../../auto-reply/dispatch.js"; import { formatInboundEnvelope, formatInboundFromLabel, @@ -647,21 +647,17 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P }, }); - const { queuedFinal } = await withReplyDispatcher({ + const { queuedFinal } = await dispatchInboundMessage({ + ctx: ctxPayload, + cfg, dispatcher, - run: () => - dispatchInboundMessage({ - ctx: ctxPayload, - cfg, - dispatcher, - replyOptions: { - disableBlockStreaming: - typeof accountInfo.config.blockStreaming === "boolean" - ? !accountInfo.config.blockStreaming - : undefined, - onModelSelected, - }, - }), + replyOptions: { + disableBlockStreaming: + typeof accountInfo.config.blockStreaming === "boolean" + ? !accountInfo.config.blockStreaming + : undefined, + onModelSelected, + }, }); if (!queuedFinal) { From 3bda3df7299049096ddb1ebd1d9cd689f5f74cb0 Mon Sep 17 00:00:00 2001 From: Jessy LANGE <89694096+jessy2027@users.noreply.github.com> Date: Sat, 14 Feb 2026 00:44:04 +0100 Subject: [PATCH 0023/1038] fix(browser): hot-reload profiles added after gateway start (#4841) (#8816) * fix(browser): hot-reload profiles added after gateway start (#4841) * style: format files with oxfmt * Fix hot-reload stale config fields bug in forProfile * Fix test order-dependency in hot-reload profiles test * Fix mock reset order to prevent stale cfgProfiles * Fix config cache blocking hot-reload by clearing cache before loadConfig * test: improve hot-reload test to properly exercise config cache - Add simulated cache behavior in mock - Prime cache before mutating config - Verify stale value without clearConfigCache - Verify fresh value after hot-reload Addresses review comment about test not exercising cache * test: add hot-reload tests for browser profiles in server context. * fix(browser): optimize profile hot-reload to avoid global cache clear * fix(browser): remove unused loadConfig import * fix(test): execute resetModules before test setup * feat: implement browser server context with profile hot-reloading and tab management. * fix(browser): harden profile hot-reload and shutdown cleanup * test(browser): use toSorted in known-profile names test --------- Co-authored-by: Peter Steinberger --- src/browser/control-service.ts | 10 +- ...server-context.hot-reload-profiles.test.ts | 214 ++++++++++++++++++ ...r-context.list-known-profile-names.test.ts | 40 ++++ src/browser/server-context.ts | 60 ++++- src/browser/server-context.types.ts | 1 + src/browser/server.ts | 10 +- src/config/config.ts | 1 + src/config/io.ts | 2 +- 8 files changed, 331 insertions(+), 7 deletions(-) create mode 100644 src/browser/server-context.hot-reload-profiles.test.ts create mode 100644 src/browser/server-context.list-known-profile-names.test.ts diff --git a/src/browser/control-service.ts b/src/browser/control-service.ts index 93bb89e93dd..55445fce603 100644 --- a/src/browser/control-service.ts +++ b/src/browser/control-service.ts @@ -3,7 +3,11 @@ import { createSubsystemLogger } from "../logging/subsystem.js"; import { resolveBrowserConfig, resolveProfile } from "./config.js"; import { ensureBrowserControlAuth } from "./control-auth.js"; import { ensureChromeExtensionRelayServer } from "./extension-relay.js"; -import { type BrowserServerState, createBrowserRouteContext } from "./server-context.js"; +import { + type BrowserServerState, + createBrowserRouteContext, + listKnownProfileNames, +} from "./server-context.js"; let state: BrowserServerState | null = null; const log = createSubsystemLogger("browser"); @@ -16,6 +20,7 @@ export function getBrowserControlState(): BrowserServerState | null { export function createBrowserControlContext() { return createBrowserRouteContext({ getState: () => state, + refreshConfigFromDisk: true, }); } @@ -71,10 +76,11 @@ export async function stopBrowserControlService(): Promise { const ctx = createBrowserRouteContext({ getState: () => state, + refreshConfigFromDisk: true, }); try { - for (const name of Object.keys(current.resolved.profiles)) { + for (const name of listKnownProfileNames(current)) { try { await ctx.forProfile(name).stopRunningBrowser(); } catch { diff --git a/src/browser/server-context.hot-reload-profiles.test.ts b/src/browser/server-context.hot-reload-profiles.test.ts new file mode 100644 index 00000000000..0ff64c23449 --- /dev/null +++ b/src/browser/server-context.hot-reload-profiles.test.ts @@ -0,0 +1,214 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +let cfgProfiles: Record = {}; + +// Simulate module-level cache behavior +let cachedConfig: ReturnType | null = null; + +function buildConfig() { + return { + browser: { + enabled: true, + color: "#FF4500", + headless: true, + defaultProfile: "openclaw", + profiles: { ...cfgProfiles }, + }, + }; +} + +vi.mock("../config/config.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createConfigIO: () => ({ + loadConfig: () => { + // Always return fresh config for createConfigIO to simulate fresh disk read + return buildConfig(); + }, + }), + loadConfig: () => { + // simulate stale loadConfig that doesn't see updates unless cache cleared + if (!cachedConfig) { + cachedConfig = buildConfig(); + } + return cachedConfig; + }, + clearConfigCache: vi.fn(() => { + // Clear the simulated cache + cachedConfig = null; + }), + writeConfigFile: vi.fn(async () => {}), + }; +}); + +vi.mock("./chrome.js", () => ({ + isChromeCdpReady: vi.fn(async () => false), + isChromeReachable: vi.fn(async () => false), + launchOpenClawChrome: vi.fn(async () => { + throw new Error("launch disabled"); + }), + resolveOpenClawUserDataDir: vi.fn(() => "/tmp/openclaw"), + stopOpenClawChrome: vi.fn(async () => {}), +})); + +vi.mock("./cdp.js", () => ({ + createTargetViaCdp: vi.fn(async () => { + throw new Error("cdp disabled"); + }), + normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), + snapshotAria: vi.fn(async () => ({ nodes: [] })), + getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => `${cdpUrl}${path}`), +})); + +vi.mock("./pw-ai.js", () => ({ + closePlaywrightBrowserConnection: vi.fn(async () => {}), +})); + +vi.mock("../media/store.js", () => ({ + ensureMediaDir: vi.fn(async () => {}), + saveMediaBuffer: vi.fn(async () => ({ path: "/tmp/fake.png" })), +})); + +describe("server-context hot-reload profiles", () => { + beforeEach(() => { + vi.resetModules(); + cfgProfiles = { + openclaw: { cdpPort: 18800, color: "#FF4500" }, + }; + cachedConfig = null; // Clear simulated cache + }); + + it("forProfile hot-reloads newly added profiles from config", async () => { + // Start with only openclaw profile + const { createBrowserRouteContext } = await import("./server-context.js"); + const { resolveBrowserConfig } = await import("./config.js"); + const { loadConfig } = await import("../config/config.js"); + + // 1. Prime the cache by calling loadConfig() first + const cfg = loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + + // Verify cache is primed (without desktop) + expect(cfg.browser.profiles.desktop).toBeUndefined(); + const state = { + server: null, + port: 18791, + resolved, + profiles: new Map(), + }; + + const ctx = createBrowserRouteContext({ + getState: () => state, + refreshConfigFromDisk: true, + }); + + // Initially, "desktop" profile should not exist + expect(() => ctx.forProfile("desktop")).toThrow(/not found/); + + // 2. Simulate adding a new profile to config (like user editing openclaw.json) + cfgProfiles.desktop = { cdpUrl: "http://127.0.0.1:9222", color: "#0066CC" }; + + // 3. Verify without clearConfigCache, loadConfig() still returns stale cached value + const staleCfg = loadConfig(); + expect(staleCfg.browser.profiles.desktop).toBeUndefined(); // Cache is stale! + + // 4. Now forProfile should hot-reload (calls createConfigIO().loadConfig() internally) + // It should NOT clear the global cache + const profileCtx = ctx.forProfile("desktop"); + expect(profileCtx.profile.name).toBe("desktop"); + expect(profileCtx.profile.cdpUrl).toBe("http://127.0.0.1:9222"); + + // 5. Verify the new profile was merged into the cached state + expect(state.resolved.profiles.desktop).toBeDefined(); + + // 6. Verify GLOBAL cache was NOT cleared - subsequent simple loadConfig() still sees STALE value + // This confirms the fix: we read fresh config for the specific profile lookup without flushing the global cache + const stillStaleCfg = loadConfig(); + expect(stillStaleCfg.browser.profiles.desktop).toBeUndefined(); + + // Verify clearConfigCache was not called + const { clearConfigCache } = await import("../config/config.js"); + expect(clearConfigCache).not.toHaveBeenCalled(); + }); + + it("forProfile still throws for profiles that don't exist in fresh config", async () => { + const { createBrowserRouteContext } = await import("./server-context.js"); + const { resolveBrowserConfig } = await import("./config.js"); + const { loadConfig } = await import("../config/config.js"); + + const cfg = loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + const state = { + server: null, + port: 18791, + resolved, + profiles: new Map(), + }; + + const ctx = createBrowserRouteContext({ + getState: () => state, + refreshConfigFromDisk: true, + }); + + // Profile that doesn't exist anywhere should still throw + expect(() => ctx.forProfile("nonexistent")).toThrow(/not found/); + }); + + it("forProfile refreshes existing profile config after loadConfig cache updates", async () => { + const { createBrowserRouteContext } = await import("./server-context.js"); + const { resolveBrowserConfig } = await import("./config.js"); + const { loadConfig } = await import("../config/config.js"); + + const cfg = loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + const state = { + server: null, + port: 18791, + resolved, + profiles: new Map(), + }; + + const ctx = createBrowserRouteContext({ + getState: () => state, + refreshConfigFromDisk: true, + }); + + const before = ctx.forProfile("openclaw"); + expect(before.profile.cdpPort).toBe(18800); + + cfgProfiles.openclaw = { cdpPort: 19999, color: "#FF4500" }; + cachedConfig = null; + + const after = ctx.forProfile("openclaw"); + expect(after.profile.cdpPort).toBe(19999); + expect(state.resolved.profiles.openclaw?.cdpPort).toBe(19999); + }); + + it("listProfiles refreshes config before enumerating profiles", async () => { + const { createBrowserRouteContext } = await import("./server-context.js"); + const { resolveBrowserConfig } = await import("./config.js"); + const { loadConfig } = await import("../config/config.js"); + + const cfg = loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + const state = { + server: null, + port: 18791, + resolved, + profiles: new Map(), + }; + + const ctx = createBrowserRouteContext({ + getState: () => state, + refreshConfigFromDisk: true, + }); + + cfgProfiles.desktop = { cdpPort: 19999, color: "#0066CC" }; + cachedConfig = null; + + const profiles = await ctx.listProfiles(); + expect(profiles.some((p) => p.name === "desktop")).toBe(true); + }); +}); diff --git a/src/browser/server-context.list-known-profile-names.test.ts b/src/browser/server-context.list-known-profile-names.test.ts new file mode 100644 index 00000000000..04c897563e9 --- /dev/null +++ b/src/browser/server-context.list-known-profile-names.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import type { BrowserServerState } from "./server-context.js"; +import { resolveBrowserConfig, resolveProfile } from "./config.js"; +import { listKnownProfileNames } from "./server-context.js"; + +describe("browser server-context listKnownProfileNames", () => { + it("includes configured and runtime-only profile names", () => { + const resolved = resolveBrowserConfig({ + defaultProfile: "openclaw", + profiles: { + openclaw: { cdpPort: 18800, color: "#FF4500" }, + }, + }); + const openclaw = resolveProfile(resolved, "openclaw"); + if (!openclaw) { + throw new Error("expected openclaw profile"); + } + + const state: BrowserServerState = { + server: null as unknown as BrowserServerState["server"], + port: 18791, + resolved, + profiles: new Map([ + [ + "stale-removed", + { + profile: { ...openclaw, name: "stale-removed" }, + running: null, + }, + ], + ]), + }; + + expect(listKnownProfileNames(state).toSorted()).toEqual([ + "chrome", + "openclaw", + "stale-removed", + ]); + }); +}); diff --git a/src/browser/server-context.ts b/src/browser/server-context.ts index d6e0e8f0474..658e75b3db1 100644 --- a/src/browser/server-context.ts +++ b/src/browser/server-context.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import type { ResolvedBrowserProfile } from "./config.js"; import type { PwAiModule } from "./pw-ai-module.js"; import type { + BrowserServerState, BrowserRouteContext, BrowserTab, ContextOptions, @@ -9,6 +10,7 @@ import type { ProfileRuntimeState, ProfileStatus, } from "./server-context.types.js"; +import { createConfigIO, loadConfig } from "../config/config.js"; import { appendCdpPath, createTargetViaCdp, getHeadersWithAuth, normalizeCdpWsUrl } from "./cdp.js"; import { isChromeCdpReady, @@ -17,7 +19,7 @@ import { resolveOpenClawUserDataDir, stopOpenClawChrome, } from "./chrome.js"; -import { resolveProfile } from "./config.js"; +import { resolveBrowserConfig, resolveProfile } from "./config.js"; import { ensureChromeExtensionRelayServer, stopChromeExtensionRelayServer, @@ -35,6 +37,14 @@ export type { ProfileStatus, } from "./server-context.types.js"; +export function listKnownProfileNames(state: BrowserServerState): string[] { + const names = new Set(Object.keys(state.resolved.profiles)); + for (const name of state.profiles.keys()) { + names.add(name); + } + return [...names]; +} + /** * Normalize a CDP WebSocket URL to use the correct base URL. */ @@ -559,6 +569,8 @@ function createProfileContext( } export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteContext { + const refreshConfigFromDisk = opts.refreshConfigFromDisk === true; + const state = () => { const current = opts.getState(); if (!current) { @@ -567,10 +579,53 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon return current; }; + const applyResolvedConfig = ( + current: BrowserServerState, + freshResolved: BrowserServerState["resolved"], + ) => { + current.resolved = freshResolved; + for (const [name, runtime] of current.profiles) { + const nextProfile = resolveProfile(freshResolved, name); + if (nextProfile) { + runtime.profile = nextProfile; + continue; + } + if (!runtime.running) { + current.profiles.delete(name); + } + } + }; + + const refreshResolvedConfig = (current: BrowserServerState) => { + if (!refreshConfigFromDisk) { + return; + } + const cfg = loadConfig(); + const freshResolved = resolveBrowserConfig(cfg.browser, cfg); + applyResolvedConfig(current, freshResolved); + }; + + const refreshResolvedConfigFresh = (current: BrowserServerState) => { + if (!refreshConfigFromDisk) { + return; + } + const freshCfg = createConfigIO().loadConfig(); + const freshResolved = resolveBrowserConfig(freshCfg.browser, freshCfg); + applyResolvedConfig(current, freshResolved); + }; + const forProfile = (profileName?: string): ProfileContext => { const current = state(); + refreshResolvedConfig(current); const name = profileName ?? current.resolved.defaultProfile; - const profile = resolveProfile(current.resolved, name); + let profile = resolveProfile(current.resolved, name); + + // Hot-reload: try fresh config if profile not found + if (!profile) { + refreshResolvedConfigFresh(current); + profile = resolveProfile(current.resolved, name); + } + if (!profile) { const available = Object.keys(current.resolved.profiles).join(", "); throw new Error(`Profile "${name}" not found. Available profiles: ${available || "(none)"}`); @@ -580,6 +635,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon const listProfiles = async (): Promise => { const current = state(); + refreshResolvedConfig(current); const result: ProfileStatus[] = []; for (const name of Object.keys(current.resolved.profiles)) { diff --git a/src/browser/server-context.types.ts b/src/browser/server-context.types.ts index 62a8ae02862..d9360b84916 100644 --- a/src/browser/server-context.types.ts +++ b/src/browser/server-context.types.ts @@ -72,4 +72,5 @@ export type ProfileStatus = { export type ContextOptions = { getState: () => BrowserServerState | null; onEnsureAttachTarget?: (profile: ResolvedBrowserProfile) => Promise; + refreshConfigFromDisk?: boolean; }; diff --git a/src/browser/server.ts b/src/browser/server.ts index 419bdbfdfa5..03f084f168d 100644 --- a/src/browser/server.ts +++ b/src/browser/server.ts @@ -9,7 +9,11 @@ import { ensureBrowserControlAuth, resolveBrowserControlAuth } from "./control-a import { ensureChromeExtensionRelayServer } from "./extension-relay.js"; import { isPwAiLoaded } from "./pw-ai-state.js"; import { registerBrowserRoutes } from "./routes/index.js"; -import { type BrowserServerState, createBrowserRouteContext } from "./server-context.js"; +import { + type BrowserServerState, + createBrowserRouteContext, + listKnownProfileNames, +} from "./server-context.js"; let state: BrowserServerState | null = null; const log = createSubsystemLogger("browser"); @@ -125,6 +129,7 @@ export async function startBrowserControlServerFromConfig(): Promise state, + refreshConfigFromDisk: true, }); registerBrowserRoutes(app as unknown as BrowserRouteRegistrar, ctx); @@ -173,12 +178,13 @@ export async function stopBrowserControlServer(): Promise { const ctx = createBrowserRouteContext({ getState: () => state, + refreshConfigFromDisk: true, }); try { const current = state; if (current) { - for (const name of Object.keys(current.resolved.profiles)) { + for (const name of listKnownProfileNames(current)) { try { await ctx.forProfile(name).stopRunningBrowser(); } catch { diff --git a/src/config/config.ts b/src/config/config.ts index 4761b7b215d..db3091c5f0e 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -1,4 +1,5 @@ export { + clearConfigCache, createConfigIO, loadConfig, parseConfigJson5, diff --git a/src/config/io.ts b/src/config/io.ts index 26d812d1469..64434a5a116 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -820,7 +820,7 @@ function shouldUseConfigCache(env: NodeJS.ProcessEnv): boolean { return resolveConfigCacheMs(env) > 0; } -function clearConfigCache(): void { +export function clearConfigCache(): void { configCache = null; } From ab71fdf821b2e10ed22f1ab554254b832b097f13 Mon Sep 17 00:00:00 2001 From: solstead <168413654+solstead@users.noreply.github.com> Date: Sat, 14 Feb 2026 06:45:45 +0700 Subject: [PATCH 0024/1038] Plugin API: compaction/reset hooks, bootstrap file globs, memory plugin status (#13287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add before_compaction and before_reset plugin hooks with session context - Pass session messages to before_compaction hook - Add before_reset plugin hook for /new and /reset commands - Add sessionId to plugin hook agent context * feat: extraBootstrapFiles config with glob pattern support Add extraBootstrapFiles to agent defaults config, allowing glob patterns (e.g. "projects/*/TOOLS.md") to auto-load project-level bootstrap files into agent context every turn. Missing files silently skipped. Co-Authored-By: Claude Opus 4.6 * fix(status): show custom memory plugins as enabled, not unavailable The status command probes memory availability using the built-in memory-core manager. Custom memory plugins (e.g. via plugin slot) can't be probed this way, so they incorrectly showed "unavailable". Now they show "enabled (plugin X)" without the misleading label. Co-Authored-By: Claude Opus 4.6 * fix: use async fs.glob and capture pre-compaction messages - Replace globSync (node:fs) with fs.glob (node:fs/promises) to match codebase conventions for async file operations - Capture session.messages BEFORE replaceMessages(limited) so before_compaction hook receives the full conversation history, not the already-truncated list * fix: resolve lint errors from CI (oxlint strict mode) - Add void to fire-and-forget IIFE (no-floating-promises) - Use String() for unknown catch params in template literals - Add curly braces to single-statement if (curly rule) * fix: resolve remaining CI lint errors in workspace.ts - Remove `| string` from WorkspaceBootstrapFileName union (made all typeof members redundant per no-redundant-type-constituents) - Use type assertion for extra bootstrap file names - Drop redundant await on fs.glob() AsyncIterable (await-thenable) * fix: address Greptile review — path traversal guard + fs/promises import - workspace.ts: use path.resolve() + traversal check in loadExtraBootstrapFiles() - commands-core.ts: import fs from node:fs/promises, drop fs.promises prefix Co-Authored-By: Claude Opus 4.6 * fix: resolve symlinks before workspace boundary check Greptile correctly identified that symlinks inside the workspace could point to files outside it, bypassing the path prefix check. Now uses fs.realpath() to resolve symlinks before verifying the real path stays within the workspace boundary. Co-Authored-By: Claude Opus 4.6 * fix: address Greptile review — hook reliability and type safety 1. before_compaction: add compactingCount field so plugins know both the full pre-compaction message count and the truncated count being fed to the compaction LLM. Clarify semantics in comment. 2. loadExtraBootstrapFiles: use path.basename() for the name field so "projects/quaid/TOOLS.md" maps to the known "TOOLS.md" type instead of an invalid WorkspaceBootstrapFileName cast. 3. before_reset: fire the hook even when no session file exists. Previously, short sessions without a persisted file would silently skip the hook. Now fires with empty messages array so plugins always know a reset occurred. Co-Authored-By: Claude Opus 4.6 * fix: validate bootstrap filenames and add compaction hook timeout - Only load extra bootstrap files whose basename matches a recognized workspace filename (AGENTS.md, TOOLS.md, etc.), preventing arbitrary files from being injected into agent context. - Wrap before_compaction hook in a 30-second Promise.race timeout so misbehaving plugins cannot stall the compaction pipeline. - Clarify hook comments: before_compaction is intentionally awaited (plugins need messages before they're discarded) but bounded. Co-Authored-By: Claude Opus 4.6 * fix: make before_compaction non-blocking, add sessionFile to after_compaction - before_compaction is now true fire-and-forget — no await, no timeout. Plugins that need full conversation data should persist it themselves and return quickly, or use after_compaction for async processing. - after_compaction now includes sessionFile path so plugins can read the full JSONL transcript asynchronously. All pre-compaction messages are preserved on disk, eliminating the need to block compaction. - Removes Promise.race timeout pattern that didn't actually cancel slow hooks (just raced past them while they continued running). Co-Authored-By: Claude Opus 4.6 * feat: add sessionFile to before_compaction for parallel processing The session JSONL already has all messages on disk before compaction starts. By providing sessionFile in before_compaction, plugins can read and extract data in parallel with the compaction LLM call rather than waiting for after_compaction. This is the optimal path for memory plugins that need the full conversation history. sessionFile is also kept on after_compaction for plugins that only need to act after compaction completes (analytics, cleanup, etc.). Co-Authored-By: Claude Opus 4.6 * refactor: move bootstrap extras into bundled hook --------- Co-authored-by: Solomon Steadman Co-authored-by: Claude Opus 4.6 Co-authored-by: Clawdbot Co-authored-by: Peter Steinberger --- docs/automation/hooks.md | 45 +++++++- docs/cli/hooks.md | 15 ++- src/agents/bootstrap-files.ts | 1 + src/agents/pi-embedded-runner/compact.ts | 50 +++++++++ src/agents/pi-embedded-runner/run/attempt.ts | 2 + ...rkspace.load-extra-bootstrap-files.test.ts | 53 +++++++++ src/agents/workspace.ts | 81 +++++++++++++ src/auto-reply/reply/commands-core.ts | 44 ++++++++ src/commands/status.command.ts | 4 + src/hooks/bundled/README.md | 14 +++ .../bundled/bootstrap-extra-files/HOOK.md | 53 +++++++++ .../bootstrap-extra-files/handler.test.ts | 106 ++++++++++++++++++ .../bundled/bootstrap-extra-files/handler.ts | 59 ++++++++++ src/plugins/hooks.ts | 15 +++ src/plugins/types.ts | 25 +++++ 15 files changed, 565 insertions(+), 2 deletions(-) create mode 100644 src/agents/workspace.load-extra-bootstrap-files.test.ts create mode 100644 src/hooks/bundled/bootstrap-extra-files/HOOK.md create mode 100644 src/hooks/bundled/bootstrap-extra-files/handler.test.ts create mode 100644 src/hooks/bundled/bootstrap-extra-files/handler.ts diff --git a/docs/automation/hooks.md b/docs/automation/hooks.md index 2030e9aeaf6..68c583a7a84 100644 --- a/docs/automation/hooks.md +++ b/docs/automation/hooks.md @@ -41,9 +41,10 @@ The hooks system allows you to: ### Bundled Hooks -OpenClaw ships with three bundled hooks that are automatically discovered: +OpenClaw ships with four bundled hooks that are automatically discovered: - **💾 session-memory**: Saves session context to your agent workspace (default `~/.openclaw/workspace/memory/`) when you issue `/new` +- **📎 bootstrap-extra-files**: Injects additional workspace bootstrap files from configured glob/path patterns during `agent:bootstrap` - **📝 command-logger**: Logs all command events to `~/.openclaw/logs/commands.log` - **🚀 boot-md**: Runs `BOOT.md` when the gateway starts (requires internal hooks enabled) @@ -484,6 +485,47 @@ Saves session context to memory when you issue `/new`. openclaw hooks enable session-memory ``` +### bootstrap-extra-files + +Injects additional bootstrap files (for example monorepo-local `AGENTS.md` / `TOOLS.md`) during `agent:bootstrap`. + +**Events**: `agent:bootstrap` + +**Requirements**: `workspace.dir` must be configured + +**Output**: No files written; bootstrap context is modified in-memory only. + +**Config**: + +```json +{ + "hooks": { + "internal": { + "enabled": true, + "entries": { + "bootstrap-extra-files": { + "enabled": true, + "paths": ["packages/*/AGENTS.md", "packages/*/TOOLS.md"] + } + } + } + } +} +``` + +**Notes**: + +- Paths are resolved relative to workspace. +- Files must stay inside workspace (realpath-checked). +- Only recognized bootstrap basenames are loaded. +- Subagent allowlist is preserved (`AGENTS.md` and `TOOLS.md` only). + +**Enable**: + +```bash +openclaw hooks enable bootstrap-extra-files +``` + ### command-logger Logs all command events to a centralized audit file. @@ -618,6 +660,7 @@ The gateway logs hook loading at startup: ``` Registered hook: session-memory -> command:new +Registered hook: bootstrap-extra-files -> agent:bootstrap Registered hook: command-logger -> command Registered hook: boot-md -> gateway:startup ``` diff --git a/docs/cli/hooks.md b/docs/cli/hooks.md index 6b4f42143e9..fdf72f83434 100644 --- a/docs/cli/hooks.md +++ b/docs/cli/hooks.md @@ -32,10 +32,11 @@ List all discovered hooks from workspace, managed, and bundled directories. **Example output:** ``` -Hooks (3/3 ready) +Hooks (4/4 ready) Ready: 🚀 boot-md ✓ - Run BOOT.md on gateway startup + 📎 bootstrap-extra-files ✓ - Inject extra workspace bootstrap files during agent bootstrap 📝 command-logger ✓ - Log all command events to a centralized audit file 💾 session-memory ✓ - Save session context to memory when /new command is issued ``` @@ -249,6 +250,18 @@ openclaw hooks enable session-memory **See:** [session-memory documentation](/automation/hooks#session-memory) +### bootstrap-extra-files + +Injects additional bootstrap files (for example monorepo-local `AGENTS.md` / `TOOLS.md`) during `agent:bootstrap`. + +**Enable:** + +```bash +openclaw hooks enable bootstrap-extra-files +``` + +**See:** [bootstrap-extra-files documentation](/automation/hooks#bootstrap-extra-files) + ### command-logger Logs all command events to a centralized audit file. diff --git a/src/agents/bootstrap-files.ts b/src/agents/bootstrap-files.ts index 30e825171e9..0954cd40e15 100644 --- a/src/agents/bootstrap-files.ts +++ b/src/agents/bootstrap-files.ts @@ -30,6 +30,7 @@ export async function resolveBootstrapFilesForRun(params: { await loadWorkspaceBootstrapFiles(params.workspaceDir), sessionKey, ); + return applyBootstrapHookOverrides({ files: bootstrapFiles, workspaceDir: params.workspaceDir, diff --git a/src/agents/pi-embedded-runner/compact.ts b/src/agents/pi-embedded-runner/compact.ts index 0eec28249ce..f50dfd7bcf1 100644 --- a/src/agents/pi-embedded-runner/compact.ts +++ b/src/agents/pi-embedded-runner/compact.ts @@ -13,6 +13,7 @@ import type { EmbeddedPiCompactResult } from "./types.js"; import { resolveHeartbeatPrompt } from "../../auto-reply/heartbeat.js"; import { resolveChannelCapabilities } from "../../config/channel-capabilities.js"; import { getMachineDisplayName } from "../../infra/machine-name.js"; +import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; import { type enqueueCommand, enqueueCommandInLane } from "../../process/command-queue.js"; import { isSubagentSessionKey } from "../../routing/session-key.js"; import { resolveSignalReactionLevel } from "../../signal/reaction-level.js"; @@ -431,6 +432,8 @@ export async function compactEmbeddedPiSessionDirect( const validated = transcriptPolicy.validateAnthropicTurns ? validateAnthropicTurns(validatedGemini) : validatedGemini; + // Capture full message history BEFORE limiting — plugins need the complete conversation + const preCompactionMessages = [...session.messages]; const truncated = limitHistoryTurns( validated, getDmHistoryLimitFromSessionKey(params.sessionKey, params.config), @@ -444,6 +447,34 @@ export async function compactEmbeddedPiSessionDirect( if (limited.length > 0) { session.agent.replaceMessages(limited); } + // Run before_compaction hooks (fire-and-forget). + // The session JSONL already contains all messages on disk, so plugins + // can read sessionFile asynchronously and process in parallel with + // the compaction LLM call — no need to block or wait for after_compaction. + const hookRunner = getGlobalHookRunner(); + const hookCtx = { + agentId: params.sessionKey?.split(":")[0] ?? "main", + sessionKey: params.sessionKey, + sessionId: params.sessionId, + workspaceDir: params.workspaceDir, + messageProvider: params.messageChannel ?? params.messageProvider, + }; + if (hookRunner?.hasHooks("before_compaction")) { + hookRunner + .runBeforeCompaction( + { + messageCount: preCompactionMessages.length, + compactingCount: limited.length, + messages: preCompactionMessages, + sessionFile: params.sessionFile, + }, + hookCtx, + ) + .catch((hookErr: unknown) => { + log.warn(`before_compaction hook failed: ${String(hookErr)}`); + }); + } + const result = await session.compact(params.customInstructions); // Estimate tokens after compaction by summing token estimates for remaining messages let tokensAfter: number | undefined; @@ -460,6 +491,25 @@ export async function compactEmbeddedPiSessionDirect( // If estimation fails, leave tokensAfter undefined tokensAfter = undefined; } + // Run after_compaction hooks (fire-and-forget). + // Also includes sessionFile for plugins that only need to act after + // compaction completes (e.g. analytics, cleanup). + if (hookRunner?.hasHooks("after_compaction")) { + hookRunner + .runAfterCompaction( + { + messageCount: session.messages.length, + tokenCount: tokensAfter, + compactedCount: limited.length - session.messages.length, + sessionFile: params.sessionFile, + }, + hookCtx, + ) + .catch((hookErr) => { + log.warn(`after_compaction hook failed: ${hookErr}`); + }); + } + return { ok: true, compacted: true, diff --git a/src/agents/pi-embedded-runner/run/attempt.ts b/src/agents/pi-embedded-runner/run/attempt.ts index 425a30a506d..dbb69e73e74 100644 --- a/src/agents/pi-embedded-runner/run/attempt.ts +++ b/src/agents/pi-embedded-runner/run/attempt.ts @@ -749,6 +749,7 @@ export async function runEmbeddedAttempt( { agentId: hookAgentId, sessionKey: params.sessionKey, + sessionId: params.sessionId, workspaceDir: params.workspaceDir, messageProvider: params.messageProvider ?? undefined, }, @@ -890,6 +891,7 @@ export async function runEmbeddedAttempt( { agentId: hookAgentId, sessionKey: params.sessionKey, + sessionId: params.sessionId, workspaceDir: params.workspaceDir, messageProvider: params.messageProvider ?? undefined, }, diff --git a/src/agents/workspace.load-extra-bootstrap-files.test.ts b/src/agents/workspace.load-extra-bootstrap-files.test.ts new file mode 100644 index 00000000000..32586029c02 --- /dev/null +++ b/src/agents/workspace.load-extra-bootstrap-files.test.ts @@ -0,0 +1,53 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { makeTempWorkspace } from "../test-helpers/workspace.js"; +import { loadExtraBootstrapFiles } from "./workspace.js"; + +describe("loadExtraBootstrapFiles", () => { + it("loads recognized bootstrap files from glob patterns", async () => { + const workspaceDir = await makeTempWorkspace("openclaw-extra-bootstrap-glob-"); + const packageDir = path.join(workspaceDir, "packages", "core"); + await fs.mkdir(packageDir, { recursive: true }); + await fs.writeFile(path.join(packageDir, "TOOLS.md"), "tools", "utf-8"); + await fs.writeFile(path.join(packageDir, "README.md"), "not bootstrap", "utf-8"); + + const files = await loadExtraBootstrapFiles(workspaceDir, ["packages/*/*"]); + + expect(files).toHaveLength(1); + expect(files[0]?.name).toBe("TOOLS.md"); + expect(files[0]?.content).toBe("tools"); + }); + + it("keeps path-traversal attempts outside workspace excluded", async () => { + const rootDir = await makeTempWorkspace("openclaw-extra-bootstrap-root-"); + const workspaceDir = path.join(rootDir, "workspace"); + const outsideDir = path.join(rootDir, "outside"); + await fs.mkdir(workspaceDir, { recursive: true }); + await fs.mkdir(outsideDir, { recursive: true }); + await fs.writeFile(path.join(outsideDir, "AGENTS.md"), "outside", "utf-8"); + + const files = await loadExtraBootstrapFiles(workspaceDir, ["../outside/AGENTS.md"]); + + expect(files).toHaveLength(0); + }); + + it("supports symlinked workspace roots with realpath checks", async () => { + if (process.platform === "win32") { + return; + } + + const rootDir = await makeTempWorkspace("openclaw-extra-bootstrap-symlink-"); + const realWorkspace = path.join(rootDir, "real-workspace"); + const linkedWorkspace = path.join(rootDir, "linked-workspace"); + await fs.mkdir(realWorkspace, { recursive: true }); + await fs.writeFile(path.join(realWorkspace, "AGENTS.md"), "linked agents", "utf-8"); + await fs.symlink(realWorkspace, linkedWorkspace, "dir"); + + const files = await loadExtraBootstrapFiles(linkedWorkspace, ["AGENTS.md"]); + + expect(files).toHaveLength(1); + expect(files[0]?.name).toBe("AGENTS.md"); + expect(files[0]?.content).toBe("linked agents"); + }); +}); diff --git a/src/agents/workspace.ts b/src/agents/workspace.ts index 486dff87cc0..c13fe29f72a 100644 --- a/src/agents/workspace.ts +++ b/src/agents/workspace.ts @@ -93,6 +93,19 @@ export type WorkspaceBootstrapFile = { missing: boolean; }; +/** Set of recognized bootstrap filenames for runtime validation */ +const VALID_BOOTSTRAP_NAMES: ReadonlySet = new Set([ + DEFAULT_AGENTS_FILENAME, + DEFAULT_SOUL_FILENAME, + DEFAULT_TOOLS_FILENAME, + DEFAULT_IDENTITY_FILENAME, + DEFAULT_USER_FILENAME, + DEFAULT_HEARTBEAT_FILENAME, + DEFAULT_BOOTSTRAP_FILENAME, + DEFAULT_MEMORY_FILENAME, + DEFAULT_MEMORY_ALT_FILENAME, +]); + async function writeFileIfMissing(filePath: string, content: string) { try { await fs.writeFile(filePath, content, { @@ -329,3 +342,71 @@ export function filterBootstrapFilesForSession( } return files.filter((file) => SUBAGENT_BOOTSTRAP_ALLOWLIST.has(file.name)); } + +export async function loadExtraBootstrapFiles( + dir: string, + extraPatterns: string[], +): Promise { + if (!extraPatterns.length) { + return []; + } + const resolvedDir = resolveUserPath(dir); + let realResolvedDir = resolvedDir; + try { + realResolvedDir = await fs.realpath(resolvedDir); + } catch { + // Keep lexical root if realpath fails. + } + + // Resolve glob patterns into concrete file paths + const resolvedPaths = new Set(); + for (const pattern of extraPatterns) { + if (pattern.includes("*") || pattern.includes("?") || pattern.includes("{")) { + try { + const matches = fs.glob(pattern, { cwd: resolvedDir }); + for await (const m of matches) { + resolvedPaths.add(m); + } + } catch { + // glob not available or pattern error — fall back to literal + resolvedPaths.add(pattern); + } + } else { + resolvedPaths.add(pattern); + } + } + + const result: WorkspaceBootstrapFile[] = []; + for (const relPath of resolvedPaths) { + const filePath = path.resolve(resolvedDir, relPath); + // Guard against path traversal — resolved path must stay within workspace + if (!filePath.startsWith(resolvedDir + path.sep) && filePath !== resolvedDir) { + continue; + } + try { + // Resolve symlinks and verify the real path is still within workspace + const realFilePath = await fs.realpath(filePath); + if ( + !realFilePath.startsWith(realResolvedDir + path.sep) && + realFilePath !== realResolvedDir + ) { + continue; + } + // Only load files whose basename is a recognized bootstrap filename + const baseName = path.basename(relPath); + if (!VALID_BOOTSTRAP_NAMES.has(baseName)) { + continue; + } + const content = await fs.readFile(realFilePath, "utf-8"); + result.push({ + name: baseName as WorkspaceBootstrapFileName, + path: filePath, + content, + missing: false, + }); + } catch { + // Silently skip missing extra files + } + } + return result; +} diff --git a/src/auto-reply/reply/commands-core.ts b/src/auto-reply/reply/commands-core.ts index c139fd6f646..e3586708488 100644 --- a/src/auto-reply/reply/commands-core.ts +++ b/src/auto-reply/reply/commands-core.ts @@ -1,3 +1,4 @@ +import fs from "node:fs/promises"; import type { CommandHandler, CommandHandlerResult, @@ -5,6 +6,7 @@ import type { } from "./commands-types.js"; import { logVerbose } from "../../globals.js"; import { createInternalHookEvent, triggerInternalHook } from "../../hooks/internal-hooks.js"; +import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; import { resolveSendPolicy } from "../../sessions/send-policy.js"; import { shouldHandleTextCommands } from "../commands-registry.js"; import { handleAllowlistCommand } from "./commands-allowlist.js"; @@ -104,6 +106,48 @@ export async function handleCommands(params: HandleCommandsParams): Promise { + try { + const messages: unknown[] = []; + if (sessionFile) { + const content = await fs.readFile(sessionFile, "utf-8"); + for (const line of content.split("\n")) { + if (!line.trim()) { + continue; + } + try { + const entry = JSON.parse(line); + if (entry.type === "message" && entry.message) { + messages.push(entry.message); + } + } catch { + // skip malformed lines + } + } + } else { + logVerbose("before_reset: no session file available, firing hook with empty messages"); + } + await hookRunner.runBeforeReset( + { sessionFile, messages, reason: commandAction }, + { + agentId: params.sessionKey?.split(":")[0] ?? "main", + sessionKey: params.sessionKey, + sessionId: prevEntry?.sessionId, + workspaceDir: params.workspaceDir, + }, + ); + } catch (err: unknown) { + logVerbose(`before_reset hook failed: ${String(err)}`); + } + })(); + } } const allowTextCommands = shouldHandleTextCommands({ diff --git a/src/commands/status.command.ts b/src/commands/status.command.ts index cbe5d6d78a7..04d1c505c25 100644 --- a/src/commands/status.command.ts +++ b/src/commands/status.command.ts @@ -312,6 +312,10 @@ export async function statusCommand( } if (!memory) { const slot = memoryPlugin.slot ? `plugin ${memoryPlugin.slot}` : "plugin"; + // Custom (non-built-in) memory plugins can't be probed — show enabled, not unavailable + if (memoryPlugin.slot && memoryPlugin.slot !== "memory-core") { + return `enabled (${slot})`; + } return muted(`enabled (${slot}) · unavailable`); } const parts: string[] = []; diff --git a/src/hooks/bundled/README.md b/src/hooks/bundled/README.md index 4587d20a256..b3fb4e131a1 100644 --- a/src/hooks/bundled/README.md +++ b/src/hooks/bundled/README.md @@ -18,6 +18,20 @@ Automatically saves session context to memory when you issue `/new`. openclaw hooks enable session-memory ``` +### 📎 bootstrap-extra-files + +Injects extra bootstrap files (for example monorepo `AGENTS.md`/`TOOLS.md`) during prompt assembly. + +**Events**: `agent:bootstrap` +**What it does**: Expands configured workspace glob/path patterns and appends matching bootstrap files to injected context. +**Output**: No files written; context is modified in-memory only. + +**Enable**: + +```bash +openclaw hooks enable bootstrap-extra-files +``` + ### 📝 command-logger Logs all command events to a centralized audit file. diff --git a/src/hooks/bundled/bootstrap-extra-files/HOOK.md b/src/hooks/bundled/bootstrap-extra-files/HOOK.md new file mode 100644 index 00000000000..a46a07efd68 --- /dev/null +++ b/src/hooks/bundled/bootstrap-extra-files/HOOK.md @@ -0,0 +1,53 @@ +--- +name: bootstrap-extra-files +description: "Inject additional workspace bootstrap files via glob/path patterns" +homepage: https://docs.openclaw.ai/automation/hooks#bootstrap-extra-files +metadata: + { + "openclaw": + { + "emoji": "📎", + "events": ["agent:bootstrap"], + "requires": { "config": ["workspace.dir"] }, + "install": [{ "id": "bundled", "kind": "bundled", "label": "Bundled with OpenClaw" }], + }, + } +--- + +# Bootstrap Extra Files Hook + +Loads additional bootstrap files into `Project Context` during `agent:bootstrap`. + +## Why + +Use this when your workspace has multiple context roots (for example monorepos) and +you want to include extra `AGENTS.md`/`TOOLS.md`-class files without changing the +workspace root. + +## Configuration + +```json +{ + "hooks": { + "internal": { + "enabled": true, + "entries": { + "bootstrap-extra-files": { + "enabled": true, + "paths": ["packages/*/AGENTS.md", "packages/*/TOOLS.md"] + } + } + } + } +} +``` + +## Options + +- `paths` (string[]): preferred list of glob/path patterns. +- `patterns` (string[]): alias of `paths`. +- `files` (string[]): alias of `paths`. + +All paths are resolved from the workspace and must stay inside it (including realpath checks). +Only recognized bootstrap basenames are loaded (`AGENTS.md`, `SOUL.md`, `TOOLS.md`, +`IDENTITY.md`, `USER.md`, `HEARTBEAT.md`, `BOOTSTRAP.md`, `MEMORY.md`, `memory.md`). diff --git a/src/hooks/bundled/bootstrap-extra-files/handler.test.ts b/src/hooks/bundled/bootstrap-extra-files/handler.test.ts new file mode 100644 index 00000000000..2b945ad07a5 --- /dev/null +++ b/src/hooks/bundled/bootstrap-extra-files/handler.test.ts @@ -0,0 +1,106 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../../../config/config.js"; +import type { AgentBootstrapHookContext } from "../../hooks.js"; +import { makeTempWorkspace, writeWorkspaceFile } from "../../../test-helpers/workspace.js"; +import { createHookEvent } from "../../hooks.js"; +import handler from "./handler.js"; + +describe("bootstrap-extra-files hook", () => { + it("appends extra bootstrap files from configured patterns", async () => { + const tempDir = await makeTempWorkspace("openclaw-bootstrap-extra-"); + const extraDir = path.join(tempDir, "packages", "core"); + await fs.mkdir(extraDir, { recursive: true }); + await fs.writeFile(path.join(extraDir, "AGENTS.md"), "extra agents", "utf-8"); + + const cfg: OpenClawConfig = { + hooks: { + internal: { + entries: { + "bootstrap-extra-files": { + enabled: true, + paths: ["packages/*/AGENTS.md"], + }, + }, + }, + }, + }; + + const context: AgentBootstrapHookContext = { + workspaceDir: tempDir, + bootstrapFiles: [ + { + name: "AGENTS.md", + path: await writeWorkspaceFile({ + dir: tempDir, + name: "AGENTS.md", + content: "root agents", + }), + content: "root agents", + missing: false, + }, + ], + cfg, + sessionKey: "agent:main:main", + }; + + const event = createHookEvent("agent", "bootstrap", "agent:main:main", context); + await handler(event); + + const injected = context.bootstrapFiles.filter((f) => f.name === "AGENTS.md"); + expect(injected).toHaveLength(2); + expect(injected.some((f) => f.path.endsWith(path.join("packages", "core", "AGENTS.md")))).toBe( + true, + ); + }); + + it("re-applies subagent bootstrap allowlist after extras are added", async () => { + const tempDir = await makeTempWorkspace("openclaw-bootstrap-extra-subagent-"); + const extraDir = path.join(tempDir, "packages", "persona"); + await fs.mkdir(extraDir, { recursive: true }); + await fs.writeFile(path.join(extraDir, "SOUL.md"), "evil", "utf-8"); + + const cfg: OpenClawConfig = { + hooks: { + internal: { + entries: { + "bootstrap-extra-files": { + enabled: true, + paths: ["packages/*/SOUL.md"], + }, + }, + }, + }, + }; + + const context: AgentBootstrapHookContext = { + workspaceDir: tempDir, + bootstrapFiles: [ + { + name: "AGENTS.md", + path: await writeWorkspaceFile({ + dir: tempDir, + name: "AGENTS.md", + content: "root agents", + }), + content: "root agents", + missing: false, + }, + { + name: "TOOLS.md", + path: await writeWorkspaceFile({ dir: tempDir, name: "TOOLS.md", content: "root tools" }), + content: "root tools", + missing: false, + }, + ], + cfg, + sessionKey: "agent:main:subagent:abc", + }; + + const event = createHookEvent("agent", "bootstrap", "agent:main:subagent:abc", context); + await handler(event); + + expect(context.bootstrapFiles.map((f) => f.name).toSorted()).toEqual(["AGENTS.md", "TOOLS.md"]); + }); +}); diff --git a/src/hooks/bundled/bootstrap-extra-files/handler.ts b/src/hooks/bundled/bootstrap-extra-files/handler.ts new file mode 100644 index 00000000000..ada7286909d --- /dev/null +++ b/src/hooks/bundled/bootstrap-extra-files/handler.ts @@ -0,0 +1,59 @@ +import { + filterBootstrapFilesForSession, + loadExtraBootstrapFiles, +} from "../../../agents/workspace.js"; +import { resolveHookConfig } from "../../config.js"; +import { isAgentBootstrapEvent, type HookHandler } from "../../hooks.js"; + +const HOOK_KEY = "bootstrap-extra-files"; + +function normalizeStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.map((v) => (typeof v === "string" ? v.trim() : "")).filter(Boolean); +} + +function resolveExtraBootstrapPatterns(hookConfig: Record): string[] { + const fromPaths = normalizeStringArray(hookConfig.paths); + if (fromPaths.length > 0) { + return fromPaths; + } + const fromPatterns = normalizeStringArray(hookConfig.patterns); + if (fromPatterns.length > 0) { + return fromPatterns; + } + return normalizeStringArray(hookConfig.files); +} + +const bootstrapExtraFilesHook: HookHandler = async (event) => { + if (!isAgentBootstrapEvent(event)) { + return; + } + + const context = event.context; + const hookConfig = resolveHookConfig(context.cfg, HOOK_KEY); + if (!hookConfig || hookConfig.enabled === false) { + return; + } + + const patterns = resolveExtraBootstrapPatterns(hookConfig as Record); + if (patterns.length === 0) { + return; + } + + try { + const extras = await loadExtraBootstrapFiles(context.workspaceDir, patterns); + if (extras.length === 0) { + return; + } + context.bootstrapFiles = filterBootstrapFilesForSession( + [...context.bootstrapFiles, ...extras], + context.sessionKey, + ); + } catch (err) { + console.warn(`[bootstrap-extra-files] failed: ${String(err)}`); + } +}; + +export default bootstrapExtraFilesHook; diff --git a/src/plugins/hooks.ts b/src/plugins/hooks.ts index d74c23c5b21..040ce1d35c8 100644 --- a/src/plugins/hooks.ts +++ b/src/plugins/hooks.ts @@ -14,6 +14,7 @@ import type { PluginHookBeforeAgentStartEvent, PluginHookBeforeAgentStartResult, PluginHookBeforeCompactionEvent, + PluginHookBeforeResetEvent, PluginHookBeforeToolCallEvent, PluginHookBeforeToolCallResult, PluginHookGatewayContext, @@ -42,6 +43,7 @@ export type { PluginHookBeforeAgentStartResult, PluginHookAgentEndEvent, PluginHookBeforeCompactionEvent, + PluginHookBeforeResetEvent, PluginHookAfterCompactionEvent, PluginHookMessageContext, PluginHookMessageReceivedEvent, @@ -230,6 +232,18 @@ export function createHookRunner(registry: PluginRegistry, options: HookRunnerOp return runVoidHook("after_compaction", event, ctx); } + /** + * Run before_reset hook. + * Fired when /new or /reset clears a session, before messages are lost. + * Runs in parallel (fire-and-forget). + */ + async function runBeforeReset( + event: PluginHookBeforeResetEvent, + ctx: PluginHookAgentContext, + ): Promise { + return runVoidHook("before_reset", event, ctx); + } + // ========================================================================= // Message Hooks // ========================================================================= @@ -447,6 +461,7 @@ export function createHookRunner(registry: PluginRegistry, options: HookRunnerOp runAgentEnd, runBeforeCompaction, runAfterCompaction, + runBeforeReset, // Message hooks runMessageReceived, runMessageSending, diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 27c6fff2425..32a961df6e6 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -300,6 +300,7 @@ export type PluginHookName = | "agent_end" | "before_compaction" | "after_compaction" + | "before_reset" | "message_received" | "message_sending" | "message_sent" @@ -315,6 +316,7 @@ export type PluginHookName = export type PluginHookAgentContext = { agentId?: string; sessionKey?: string; + sessionId?: string; workspaceDir?: string; messageProvider?: string; }; @@ -340,14 +342,33 @@ export type PluginHookAgentEndEvent = { // Compaction hooks export type PluginHookBeforeCompactionEvent = { + /** Total messages in the session before any truncation or compaction */ messageCount: number; + /** Messages being fed to the compaction LLM (after history-limit truncation) */ + compactingCount?: number; tokenCount?: number; + messages?: unknown[]; + /** Path to the session JSONL transcript. All messages are already on disk + * before compaction starts, so plugins can read this file asynchronously + * and process in parallel with the compaction LLM call. */ + sessionFile?: string; +}; + +// before_reset hook — fired when /new or /reset clears a session +export type PluginHookBeforeResetEvent = { + sessionFile?: string; + messages?: unknown[]; + reason?: string; }; export type PluginHookAfterCompactionEvent = { messageCount: number; tokenCount?: number; compactedCount: number; + /** Path to the session JSONL transcript. All pre-compaction messages are + * preserved on disk, so plugins can read and process them asynchronously + * without blocking the compaction pipeline. */ + sessionFile?: string; }; // Message context @@ -486,6 +507,10 @@ export type PluginHookHandlerMap = { event: PluginHookAfterCompactionEvent, ctx: PluginHookAgentContext, ) => Promise | void; + before_reset: ( + event: PluginHookBeforeResetEvent, + ctx: PluginHookAgentContext, + ) => Promise | void; message_received: ( event: PluginHookMessageReceivedEvent, ctx: PluginHookMessageContext, From 4bef423d833244fc7fc4fe2680c3da91489afbb6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Feb 2026 23:50:04 +0000 Subject: [PATCH 0025/1038] perf(test): reduce gateway reload waits and trim duplicate invoke coverage --- src/auto-reply/reply.block-streaming.test.ts | 24 +++++------- src/auto-reply/reply.raw-body.test.ts | 6 +-- .../server-reload.config-during-reply.test.ts | 4 +- src/gateway/server-reload.integration.test.ts | 4 +- .../server-reload.real-scenario.test.ts | 28 ++++++++++--- src/gateway/server.nodes.late-invoke.test.ts | 18 ++++----- src/gateway/tools-invoke-http.test.ts | 39 +------------------ 7 files changed, 46 insertions(+), 77 deletions(-) diff --git a/src/auto-reply/reply.block-streaming.test.ts b/src/auto-reply/reply.block-streaming.test.ts index d982280ab47..18c037789c1 100644 --- a/src/auto-reply/reply.block-streaming.test.ts +++ b/src/auto-reply/reply.block-streaming.test.ts @@ -164,7 +164,7 @@ describe("block streaming", () => { }); }); - it("falls back to final payloads when block reply send times out", async () => { + it("falls back to final payloads and respects telegram streamMode block", async () => { await withTempHome(async (home) => { let sawAbort = false; const onBlockReply = vi.fn((_, context) => { @@ -220,32 +220,26 @@ describe("block streaming", () => { const res = await replyPromise; expect(res).toMatchObject({ text: "final" }); expect(sawAbort).toBe(true); - }); - }); - it("does not enable block streaming for telegram streamMode block", async () => { - await withTempHome(async (home) => { - const onBlockReply = vi.fn().mockResolvedValue(undefined); - - const impl = async () => ({ + const onBlockReplyStreamMode = vi.fn().mockResolvedValue(undefined); + piEmbeddedMock.runEmbeddedPiAgent.mockImplementation(async () => ({ payloads: [{ text: "final" }], meta: { durationMs: 5, agentMeta: { sessionId: "s", provider: "p", model: "m" }, }, - }); - piEmbeddedMock.runEmbeddedPiAgent.mockImplementation(impl); + })); - const res = await getReplyFromConfig( + const resStreamMode = await getReplyFromConfig( { Body: "ping", From: "+1004", To: "+2000", - MessageSid: "msg-126", + MessageSid: "msg-127", Provider: "telegram", }, { - onBlockReply, + onBlockReply: onBlockReplyStreamMode, }, { agents: { @@ -259,8 +253,8 @@ describe("block streaming", () => { }, ); - expect(res?.text).toBe("final"); - expect(onBlockReply).not.toHaveBeenCalled(); + expect(resStreamMode?.text).toBe("final"); + expect(onBlockReplyStreamMode).not.toHaveBeenCalled(); }); }); }); diff --git a/src/auto-reply/reply.raw-body.test.ts b/src/auto-reply/reply.raw-body.test.ts index 0b19df8a124..8ec67b88af4 100644 --- a/src/auto-reply/reply.raw-body.test.ts +++ b/src/auto-reply/reply.raw-body.test.ts @@ -102,7 +102,7 @@ describe("RawBody directive parsing", () => { vi.clearAllMocks(); }); - it("detects command directives from RawBody/CommandBody in wrapped group messages", async () => { + it("handles directives, history, and non-default agent session files", async () => { await withTempHome(async (home) => { const assertCommandReply = async (input: { message: ReplyMessage; @@ -161,11 +161,7 @@ describe("RawBody directive parsing", () => { }, expectedIncludes: ["Verbose logging enabled."], }); - }); - }); - it("preserves history and reuses non-default agent session files", async () => { - await withTempHome(async (home) => { vi.mocked(runEmbeddedPiAgent).mockResolvedValue({ payloads: [{ text: "ok" }], meta: { diff --git a/src/gateway/server-reload.config-during-reply.test.ts b/src/gateway/server-reload.config-during-reply.test.ts index 326e9de759b..c0a72650904 100644 --- a/src/gateway/server-reload.config-during-reply.test.ts +++ b/src/gateway/server-reload.config-during-reply.test.ts @@ -35,8 +35,8 @@ describe("gateway config reload during reply", () => { let deliveredReplies: string[] = []; const dispatcher = createReplyDispatcher({ deliver: async (payload) => { - // Simulate async reply delivery - await new Promise((resolve) => setTimeout(resolve, 20)); + // Keep delivery asynchronous without real wall-clock delay. + await Promise.resolve(); deliveredReplies.push(payload.text ?? ""); }, onError: (err) => { diff --git a/src/gateway/server-reload.integration.test.ts b/src/gateway/server-reload.integration.test.ts index 3bd1bc80e3d..698b1041fd6 100644 --- a/src/gateway/server-reload.integration.test.ts +++ b/src/gateway/server-reload.integration.test.ts @@ -30,8 +30,8 @@ describe("gateway restart deferral integration", () => { const deliveredReplies: Array<{ text: string; timestamp: number }> = []; const dispatcher = createReplyDispatcher({ deliver: async (payload) => { - // Simulate network delay - await new Promise((resolve) => setTimeout(resolve, 20)); + // Keep delivery asynchronous without real wall-clock delay. + await Promise.resolve(); deliveredReplies.push({ text: payload.text ?? "", timestamp: Date.now(), diff --git a/src/gateway/server-reload.real-scenario.test.ts b/src/gateway/server-reload.real-scenario.test.ts index 19ece2234ae..dc10891ff7e 100644 --- a/src/gateway/server-reload.real-scenario.test.ts +++ b/src/gateway/server-reload.real-scenario.test.ts @@ -4,6 +4,16 @@ */ import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + describe("real scenario: config change during message processing", () => { let replyErrors: string[] = []; @@ -26,8 +36,10 @@ describe("real scenario: config change during message processing", () => { let rpcConnected = true; const deliveredReplies: string[] = []; + const deliveryStarted = createDeferred(); + const allowDelivery = createDeferred(); - // Create dispatcher with slow delivery (simulates real network delay) + // Hold delivery open so restart checks run while reply is in-flight. const dispatcher = createReplyDispatcher({ deliver: async (payload) => { if (!rpcConnected) { @@ -35,8 +47,8 @@ describe("real scenario: config change during message processing", () => { replyErrors.push(error); throw new Error(error); } - // Slow delivery — restart checks will run during this window - await new Promise((resolve) => setTimeout(resolve, 150)); + deliveryStarted.resolve(); + await allowDelivery.promise; deliveredReplies.push(payload.text ?? ""); }, onError: () => { @@ -49,6 +61,7 @@ describe("real scenario: config change during message processing", () => { // keeping pending > 0 is the in-flight delivery itself. dispatcher.sendFinalReply({ text: "Configuration updated!" }); dispatcher.markComplete(); + await deliveryStarted.promise; // At this point: markComplete flagged, delivery is in flight. // pending > 0 because the in-flight delivery keeps it alive. @@ -59,7 +72,7 @@ describe("real scenario: config change during message processing", () => { // If the tracking is broken, pending would be 0 and we'd restart. let restartTriggered = false; for (let i = 0; i < 3; i++) { - await new Promise((resolve) => setTimeout(resolve, 25)); + await Promise.resolve(); const pending = getTotalPendingReplies(); if (pending === 0) { restartTriggered = true; @@ -68,6 +81,7 @@ describe("real scenario: config change during message processing", () => { } } + allowDelivery.resolve(); // Wait for delivery to complete await dispatcher.waitForIdle(); @@ -83,10 +97,11 @@ describe("real scenario: config change during message processing", () => { it("should keep pending > 0 until reply is actually enqueued", async () => { const { createReplyDispatcher } = await import("../auto-reply/reply/reply-dispatcher.js"); const { getTotalPendingReplies } = await import("../auto-reply/reply/dispatcher-registry.js"); + const allowDelivery = createDeferred(); const dispatcher = createReplyDispatcher({ deliver: async (_payload) => { - await new Promise((resolve) => setTimeout(resolve, 10)); + await allowDelivery.promise; }, }); @@ -94,7 +109,7 @@ describe("real scenario: config change during message processing", () => { expect(getTotalPendingReplies()).toBe(1); // Simulate command processing delay BEFORE reply is enqueued - await new Promise((resolve) => setTimeout(resolve, 20)); + await Promise.resolve(); // During this delay, pending should STILL be 1 (reservation active) expect(getTotalPendingReplies()).toBe(1); @@ -112,6 +127,7 @@ describe("real scenario: config change during message processing", () => { const pendingAfterMarkComplete = getTotalPendingReplies(); expect(pendingAfterMarkComplete).toBeGreaterThan(0); + allowDelivery.resolve(); // Wait for reply to send await dispatcher.waitForIdle(); diff --git a/src/gateway/server.nodes.late-invoke.test.ts b/src/gateway/server.nodes.late-invoke.test.ts index b965e773464..8219b87842e 100644 --- a/src/gateway/server.nodes.late-invoke.test.ts +++ b/src/gateway/server.nodes.late-invoke.test.ts @@ -15,26 +15,25 @@ vi.mock("../infra/update-runner.js", () => ({ import { connectOk, + getFreePort, installGatewayTestHooks, rpcReq, - startServerWithClient, + startGatewayServer, } from "./test-helpers.js"; +import { testState } from "./test-helpers.mocks.js"; installGatewayTestHooks({ scope: "suite" }); -let server: Awaited>["server"]; -let ws: WebSocket; +let server: Awaited>; let port: number; let nodeWs: WebSocket; let nodeId: string; beforeAll(async () => { const token = "test-gateway-token-1234567890"; - const started = await startServerWithClient(token); - server = started.server; - ws = started.ws; - port = started.port; - await connectOk(ws, { token }); + testState.gatewayAuth = { mode: "token", token }; + port = await getFreePort(); + server = await startGatewayServer(port, { bind: "loopback" }); nodeWs = new WebSocket(`ws://127.0.0.1:${port}`); await new Promise((resolve) => nodeWs.once("open", resolve)); @@ -55,8 +54,7 @@ beforeAll(async () => { }); afterAll(async () => { - nodeWs.close(); - ws.close(); + nodeWs.terminate(); await server.close(); }); diff --git a/src/gateway/tools-invoke-http.test.ts b/src/gateway/tools-invoke-http.test.ts index 0db60b71885..d373c274100 100644 --- a/src/gateway/tools-invoke-http.test.ts +++ b/src/gateway/tools-invoke-http.test.ts @@ -46,7 +46,7 @@ const invokeAgentsList = async (params: { } return await fetch(`http://127.0.0.1:${params.port}/tools/invoke`, { method: "POST", - headers: { "content-type": "application/json", ...params.headers }, + headers: { "content-type": "application/json", connection: "close", ...params.headers }, body: JSON.stringify(body), }); }; @@ -71,7 +71,7 @@ const invokeTool = async (params: { } return await fetch(`http://127.0.0.1:${params.port}/tools/invoke`, { method: "POST", - headers: { "content-type": "application/json", ...params.headers }, + headers: { "content-type": "application/json", connection: "close", ...params.headers }, body: JSON.stringify(body), }); }; @@ -144,41 +144,6 @@ describe("POST /tools/invoke", () => { expect(implicitBody.ok).toBe(true); }); - it("handles dedicated auth modes for password accept and token reject", async () => { - allowAgentsListForMain(); - - const passwordPort = await getFreePort(); - const passwordServer = await startGatewayServer(passwordPort, { - bind: "loopback", - auth: { mode: "password", password: "secret" }, - }); - try { - const passwordRes = await invokeAgentsList({ - port: passwordPort, - headers: { authorization: "Bearer secret" }, - sessionKey: "main", - }); - expect(passwordRes.status).toBe(200); - } finally { - await passwordServer.close(); - } - - const tokenPort = await getFreePort(); - const tokenServer = await startGatewayServer(tokenPort, { - bind: "loopback", - auth: { mode: "token", token: "t" }, - }); - try { - const tokenRes = await invokeAgentsList({ - port: tokenPort, - sessionKey: "main", - }); - expect(tokenRes.status).toBe(401); - } finally { - await tokenServer.close(); - } - }); - it("routes tools invoke before plugin HTTP handlers", async () => { const pluginHandler = vi.fn(async (_req: IncomingMessage, res: ServerResponse) => { res.statusCode = 418; From 1055e71c4b3bc0fda10f1f8ccda25eba2d8f6917 Mon Sep 17 00:00:00 2001 From: Divanoli Mydeen Pitchai <12023205+divanoli@users.noreply.github.com> Date: Sat, 14 Feb 2026 02:51:47 +0300 Subject: [PATCH 0026/1038] fix(telegram): auto-wrap .md file references in backticks to prevent URL previews (#8649) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(telegram): auto-wrap file references with TLD extensions to prevent URL previews Telegram's auto-linker aggressively treats filenames like HEARTBEAT.md, README.md, main.go, script.py as URLs and generates domain registrar previews. This fix adds comprehensive protection for file extensions that share TLDs: - High priority: .md, .go, .py, .pl, .ai, .sh - Medium priority: .io, .tv, .fm, .am, .at, .be, .cc, .co Implementation: - Added wrapFileReferencesInHtml() in format.ts - Runs AFTER markdown→HTML conversion - Tokenizes HTML to respect tag boundaries - Skips content inside , 
,  tags (no nesting issues)
- Applied to all rendering paths: renderTelegramHtmlText, markdownToTelegramHtml,
  markdownToTelegramChunks, and delivery.ts fallback

Addresses review comments:
- P1: Now handles chunked rendering paths correctly
- P2: No longer wraps inside existing code blocks (token-based parsing)
- No lookbehinds used (broad Node compatibility)

Includes comprehensive test suite in format.wrap-md.test.ts

AI-assisted: true

* fix(telegram): prevent URL previews for file refs with TLD extensions

Two layers were causing spurious link previews for file references like
`README.md`, `backup.sh`, `main.go`:

1. **markdown-it linkify** converts `README.md` to
   `README.md` (.md = Moldova TLD)
2. **Telegram auto-linker** treats remaining bare text as URLs

## Changes

### Primary fix: suppress auto-linkified file refs in buildTelegramLink
- Added `isAutoLinkedFileRef()` helper that detects when linkify auto-
  generated a link from a bare filename (href = "http://" + label)
- Rejects paths with domain-like segments (dots in non-final path parts)
- Modified `buildTelegramLink()` to return null for these, so file refs
  stay as plain text and get wrapped in `` by the wrapper

### Safety-net: de-linkify in wrapFileReferencesInHtml
- Added pre-pass that catches auto-linkified anchors in pre-rendered HTML
- Handles edge cases where HTML is passed directly (textMode: "html")
- Reuses `isAutoLinkedFileRef()` logic — no duplication

### Bug fixes discovered during review
- **Fixed `isClosing` bug (line 169)**: the check `match[1] === "/"`
  was wrong — the regex `(<\/?)}` captures `<` or `...
- Prevents wrapping inside any level of protected tags Add 4 tests for edge cases: - Nested code tags (depth tracking) - Multiple anchor tags in sequence - Auto-linked anchor with backreference match - Anchor with different href/label (no match) * fix(telegram): add escapeHtml and escapeRegex for defense in depth Code review fixes: 1. Escape filename with escapeHtml() before inserting into tags - Prevents HTML injection if regex ever matches unsafe chars - Defense in depth (current regex already limits to safe chars) 2. Escape extensions with escapeRegex() before joining into pattern - Prevents regex breakage if extensions contain metacharacters - Future-proofs against extensions like 'c++' or 'd.ts' Add tests documenting regex safety boundaries: - Filenames with special chars (&, <, >) don't match - Only [a-zA-Z0-9_.\-./] chars are captured * fix(telegram): catch orphaned single-letter TLD patterns When text like 'R&D.md' doesn't match the main file pattern (because & breaks the character class), the 'D.md' part can still be auto-linked by Telegram as a domain (https://d.md/). Add second pass to catch orphaned TLD patterns like 'D.md', 'R.io', 'X.ai' that follow non-alphanumeric characters and wrap them in tags. Pattern: ([^a-zA-Z0-9]|^)([A-Za-z]\.(?:extensions))(?=[^a-zA-Z0-9/]|$) Tests added: - 'wraps orphaned TLD pattern after special character' (R&D.md → R&D.md) - 'wraps orphaned single-letter TLD patterns' (X.ai, R.io) * refactor(telegram): remove popular domain TLDs from file extension list Remove .ai, .io, .tv, .fm from FILE_EXTENSIONS_WITH_TLD because: - These are commonly used as real domains (x.ai, vercel.io, github.io) - Rarely used as actual file extensions - Users are more likely referring to websites than files Keep: md, sh, py, go, pl (common file extensions, rarely intentional domains) Keep: am, at, be, cc, co (less common as intentional domain references) Update tests to reflect the change: - Add test for supported extensions (.am, .at, .be, .cc, .co) - Add test verifying popular TLDs stay as links * fix(telegram): prevent orphaned TLD wrapping inside HTML tags Code review fixes: 1. Orphaned TLD pass now checks if match is inside HTML tag - Uses lastIndexOf('<') vs lastIndexOf('>') to detect tag context - Skips wrapping when between < and > (inside attributes) - Prevents invalid HTML like 2. textMode: 'html' now trusts caller markup - Returns text unchanged instead of wrapping - Caller owns HTML structure in this mode Tests added: - 'does not wrap orphaned TLD inside href attributes' - 'does not wrap orphaned TLD inside any HTML attribute' - 'does not wrap in HTML mode (trusts caller markup)' * refactor(telegram): use snapshot for orphaned TLD offset clarity Use explicit snapshot variable when checking tag positions in orphaned TLD pass. While JavaScript's replace() doesn't mutate during iteration, this makes intent explicit and adds test coverage for multi-TLD HTML. Co-Authored-By: Claude Opus 4.5 * fix(telegram): prevent orphaned TLD wrapping inside code/pre tags - Add depth tracking for code/pre tags in orphaned TLD pass - Fix test to expect valid HTML output - 55 tests now covering nested tag scenarios Co-Authored-By: Claude Opus 4.5 * fix(telegram): clamp depth counters and add anchor tracking to orphaned pass - Clamp depth counters at 0 for malformed HTML with stray closing tags - Add anchor depth tracking to orphaned TLD pass to prevent wrapping inside link text (e.g., R&D.md) - 57 tests covering all edge cases Co-Authored-By: Claude Opus 4.5 * fix(telegram): keep .co domains linked and wrap punctuated file refs --------- Co-authored-by: Claude Opus 4.5 Co-authored-by: Peter Steinberger --- src/telegram/bot/delivery.ts | 5 +- src/telegram/format.ts | 211 ++++++++++++++- src/telegram/format.wrap-md.test.ts | 404 ++++++++++++++++++++++++++++ 3 files changed, 615 insertions(+), 5 deletions(-) create mode 100644 src/telegram/format.wrap-md.test.ts diff --git a/src/telegram/bot/delivery.ts b/src/telegram/bot/delivery.ts index bd97d570889..732227ed023 100644 --- a/src/telegram/bot/delivery.ts +++ b/src/telegram/bot/delivery.ts @@ -18,6 +18,7 @@ import { markdownToTelegramChunks, markdownToTelegramHtml, renderTelegramHtmlText, + wrapFileReferencesInHtml, } from "../format.js"; import { buildInlineKeyboard } from "../send.js"; import { cacheSticker, getCachedSticker } from "../sticker-cache.js"; @@ -76,7 +77,9 @@ export async function deliverReplies(params: { const nested = markdownToTelegramChunks(chunk, textLimit, { tableMode: params.tableMode }); if (!nested.length && chunk) { chunks.push({ - html: markdownToTelegramHtml(chunk, { tableMode: params.tableMode }), + html: wrapFileReferencesInHtml( + markdownToTelegramHtml(chunk, { tableMode: params.tableMode, wrapFileRefs: false }), + ), text: chunk, }); continue; diff --git a/src/telegram/format.ts b/src/telegram/format.ts index eb457edff0c..dae60ff1d96 100644 --- a/src/telegram/format.ts +++ b/src/telegram/format.ts @@ -20,7 +20,56 @@ function escapeHtmlAttr(text: string): string { return escapeHtml(text).replace(/"/g, """); } -function buildTelegramLink(link: MarkdownLinkSpan, _text: string) { +/** + * File extensions that share TLDs and commonly appear in code/documentation. + * These are wrapped in tags to prevent Telegram from generating + * spurious domain registrar previews. + * + * Only includes extensions that are: + * 1. Commonly used as file extensions in code/docs + * 2. Rarely used as intentional domain references + * + * Excluded: .ai, .io, .tv, .fm (popular domain TLDs like x.ai, vercel.io, github.io) + */ +const FILE_EXTENSIONS_WITH_TLD = new Set([ + "md", // Markdown (Moldova) - very common in repos + "go", // Go language - common in Go projects + "py", // Python (Paraguay) - common in Python projects + "pl", // Perl (Poland) - common in Perl projects + "sh", // Shell (Saint Helena) - common for scripts + "am", // Automake files (Armenia) + "at", // Assembly (Austria) + "be", // Backend files (Belgium) + "cc", // C++ source (Cocos Islands) +]); + +/** Detects when markdown-it linkify auto-generated a link from a bare filename (e.g. README.md → http://README.md) */ +function isAutoLinkedFileRef(href: string, label: string): boolean { + const stripped = href.replace(/^https?:\/\//i, ""); + if (stripped !== label) { + return false; + } + const dotIndex = label.lastIndexOf("."); + if (dotIndex < 1) { + return false; + } + const ext = label.slice(dotIndex + 1).toLowerCase(); + if (!FILE_EXTENSIONS_WITH_TLD.has(ext)) { + return false; + } + // Reject if any path segment before the filename contains a dot (looks like a domain) + const segments = label.split("/"); + if (segments.length > 1) { + for (let i = 0; i < segments.length - 1; i++) { + if (segments[i].includes(".")) { + return false; + } + } + } + return true; +} + +function buildTelegramLink(link: MarkdownLinkSpan, text: string) { const href = link.href.trim(); if (!href) { return null; @@ -28,6 +77,11 @@ function buildTelegramLink(link: MarkdownLinkSpan, _text: string) { if (link.start === link.end) { return null; } + // Suppress auto-linkified file references (e.g. README.md → http://README.md) + const label = text.slice(link.start, link.end); + if (isAutoLinkedFileRef(href, label)) { + return null; + } const safeHref = escapeHtmlAttr(href); return { start: link.start, @@ -55,7 +109,7 @@ function renderTelegramHtml(ir: MarkdownIR): string { export function markdownToTelegramHtml( markdown: string, - options: { tableMode?: MarkdownTableMode } = {}, + options: { tableMode?: MarkdownTableMode; wrapFileRefs?: boolean } = {}, ): string { const ir = markdownToIR(markdown ?? "", { linkify: true, @@ -64,7 +118,154 @@ export function markdownToTelegramHtml( blockquotePrefix: "", tableMode: options.tableMode, }); - return renderTelegramHtml(ir); + const html = renderTelegramHtml(ir); + // Apply file reference wrapping if requested (for chunked rendering) + if (options.wrapFileRefs !== false) { + return wrapFileReferencesInHtml(html); + } + return html; +} + +/** + * Wraps standalone file references (with TLD extensions) in tags. + * This prevents Telegram from treating them as URLs and generating + * irrelevant domain registrar previews. + * + * Runs AFTER markdown→HTML conversion to avoid modifying HTML attributes. + * Skips content inside , 
, and  tags to avoid nesting issues.
+ */
+/** Escape regex metacharacters in a string */
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+export function wrapFileReferencesInHtml(html: string): string {
+  // Build regex pattern for all tracked extensions (escape metacharacters for safety)
+  const extensionsPattern = Array.from(FILE_EXTENSIONS_WITH_TLD).map(escapeRegex).join("|");
+
+  // Safety-net: de-linkify auto-generated anchors where href="http://Link';
+    const result = wrapFileReferencesInHtml(input);
+    expect(result).toBe(input);
+  });
+
+  it("does not wrap file refs inside real URL anchor tags", () => {
+    const input = 'Visit example.com/README.md';
+    const result = wrapFileReferencesInHtml(input);
+    expect(result).toBe(input);
+  });
+
+  it("handles mixed content correctly", () => {
+    const result = wrapFileReferencesInHtml("Check README.md and CONTRIBUTING.md");
+    expect(result).toContain("README.md");
+    expect(result).toContain("CONTRIBUTING.md");
+  });
+
+  it("handles edge cases", () => {
+    expect(wrapFileReferencesInHtml("No markdown files here")).not.toContain("");
+    expect(wrapFileReferencesInHtml("File.md at start")).toContain("File.md");
+    expect(wrapFileReferencesInHtml("Ends with file.md")).toContain("file.md");
+  });
+
+  it("wraps file refs with punctuation boundaries", () => {
+    expect(wrapFileReferencesInHtml("See README.md.")).toContain("README.md.");
+    expect(wrapFileReferencesInHtml("See README.md,")).toContain("README.md,");
+    expect(wrapFileReferencesInHtml("(README.md)")).toContain("(README.md)");
+    expect(wrapFileReferencesInHtml("README.md:")).toContain("README.md:");
+  });
+
+  it("de-linkifies auto-linkified file ref anchors", () => {
+    const input = 'README.md';
+    expect(wrapFileReferencesInHtml(input)).toBe("README.md");
+  });
+
+  it("de-linkifies auto-linkified path anchors", () => {
+    const input = 'squad/friday/HEARTBEAT.md';
+    expect(wrapFileReferencesInHtml(input)).toBe("squad/friday/HEARTBEAT.md");
+  });
+
+  it("preserves explicit links where label differs from href", () => {
+    const input = 'click here';
+    expect(wrapFileReferencesInHtml(input)).toBe(input);
+  });
+
+  it("wraps file ref after closing anchor tag", () => {
+    const input = 'link then README.md';
+    const result = wrapFileReferencesInHtml(input);
+    expect(result).toContain(" then README.md");
+  });
+});
+
+describe("renderTelegramHtmlText - file reference wrapping", () => {
+  it("wraps file references in markdown mode", () => {
+    const result = renderTelegramHtmlText("Check README.md");
+    expect(result).toContain("README.md");
+  });
+
+  it("does not wrap in HTML mode (trusts caller markup)", () => {
+    // textMode: "html" should pass through unchanged - caller owns the markup
+    const result = renderTelegramHtmlText("Check README.md", { textMode: "html" });
+    expect(result).toBe("Check README.md");
+    expect(result).not.toContain("");
+  });
+
+  it("does not double-wrap already code-formatted content", () => {
+    const result = renderTelegramHtmlText("Already `wrapped.md` here");
+    // Should have code tags but not nested
+    expect(result).toContain("");
+    expect(result).not.toContain("");
+  });
+});
+
+describe("markdownToTelegramHtml - file reference wrapping", () => {
+  it("wraps file references by default", () => {
+    const result = markdownToTelegramHtml("Check README.md");
+    expect(result).toContain("README.md");
+  });
+
+  it("can skip wrapping when requested", () => {
+    const result = markdownToTelegramHtml("Check README.md", { wrapFileRefs: false });
+    expect(result).not.toContain("README.md");
+  });
+
+  it("wraps multiple file types in a single message", () => {
+    const result = markdownToTelegramHtml("Edit main.go and script.py");
+    expect(result).toContain("main.go");
+    expect(result).toContain("script.py");
+  });
+
+  it("preserves real URLs as anchor tags", () => {
+    const result = markdownToTelegramHtml("Visit https://example.com");
+    expect(result).toContain('');
+  });
+
+  it("preserves explicit markdown links even when href looks like a file ref", () => {
+    const result = markdownToTelegramHtml("[docs](http://README.md)");
+    expect(result).toContain('docs');
+  });
+
+  it("wraps file ref after real URL in same message", () => {
+    const result = markdownToTelegramHtml("Visit https://example.com and README.md");
+    expect(result).toContain('');
+    expect(result).toContain("README.md");
+  });
+});
+
+describe("markdownToTelegramChunks - file reference wrapping", () => {
+  it("wraps file references in chunked output", () => {
+    const chunks = markdownToTelegramChunks("Check README.md and backup.sh", 4096);
+    expect(chunks.length).toBeGreaterThan(0);
+    expect(chunks[0].html).toContain("README.md");
+    expect(chunks[0].html).toContain("backup.sh");
+  });
+});
+
+describe("edge cases", () => {
+  it("wraps file ref inside bold tags", () => {
+    const result = markdownToTelegramHtml("**README.md**");
+    expect(result).toBe("README.md");
+  });
+
+  it("wraps file ref inside italic tags", () => {
+    const result = markdownToTelegramHtml("*script.py*");
+    expect(result).toBe("script.py");
+  });
+
+  it("does not wrap inside fenced code blocks", () => {
+    const result = markdownToTelegramHtml("```\nREADME.md\n```");
+    expect(result).toBe("
README.md\n
");
+    expect(result).not.toContain("");
+  });
+
+  it("preserves domain-like paths as anchor tags", () => {
+    const result = markdownToTelegramHtml("example.com/README.md");
+    expect(result).toContain('');
+    expect(result).not.toContain("");
+  });
+
+  it("preserves github URLs with file paths", () => {
+    const result = markdownToTelegramHtml("https://github.com/foo/README.md");
+    expect(result).toContain('');
+  });
+
+  it("handles wrapFileRefs: false (plain text output)", () => {
+    const result = markdownToTelegramHtml("README.md", { wrapFileRefs: false });
+    // buildTelegramLink returns null, so no  tag; wrapFileRefs: false skips 
+    expect(result).toBe("README.md");
+  });
+
+  it("wraps supported TLD extensions (.am, .at, .be, .cc)", () => {
+    const result = markdownToTelegramHtml("Makefile.am and code.at and app.be and main.cc");
+    expect(result).toContain("Makefile.am");
+    expect(result).toContain("code.at");
+    expect(result).toContain("app.be");
+    expect(result).toContain("main.cc");
+  });
+
+  it("does not wrap popular domain TLDs (.ai, .io, .tv, .fm)", () => {
+    // These are commonly used as real domains (x.ai, vercel.io, github.io)
+    const result = markdownToTelegramHtml("Check x.ai and vercel.io and app.tv and radio.fm");
+    // Should be links, not code
+    expect(result).toContain('');
+    expect(result).toContain('');
+    expect(result).toContain('');
+    expect(result).toContain('');
+  });
+
+  it("keeps .co domains as links", () => {
+    const result = markdownToTelegramHtml("Visit t.co and openclaw.co");
+    expect(result).toContain('');
+    expect(result).toContain('');
+    expect(result).not.toContain("t.co");
+    expect(result).not.toContain("openclaw.co");
+  });
+
+  it("does not wrap non-TLD extensions", () => {
+    const result = markdownToTelegramHtml("image.png and style.css and script.js");
+    expect(result).not.toContain("image.png");
+    expect(result).not.toContain("style.css");
+    expect(result).not.toContain("script.js");
+  });
+
+  it("handles file ref at start of message", () => {
+    const result = markdownToTelegramHtml("README.md is important");
+    expect(result).toBe("README.md is important");
+  });
+
+  it("handles file ref at end of message", () => {
+    const result = markdownToTelegramHtml("Check the README.md");
+    expect(result).toBe("Check the README.md");
+  });
+
+  it("handles multiple file refs in sequence", () => {
+    const result = markdownToTelegramHtml("README.md CHANGELOG.md LICENSE.md");
+    expect(result).toContain("README.md");
+    expect(result).toContain("CHANGELOG.md");
+    expect(result).toContain("LICENSE.md");
+  });
+
+  it("handles nested path without domain-like segments", () => {
+    const result = markdownToTelegramHtml("src/utils/helpers/format.go");
+    expect(result).toContain("src/utils/helpers/format.go");
+  });
+
+  it("wraps path with version-like segment (not a domain)", () => {
+    // v1.0/README.md is not linkified by markdown-it (no TLD), so it's wrapped
+    const result = markdownToTelegramHtml("v1.0/README.md");
+    expect(result).toContain("v1.0/README.md");
+  });
+
+  it("preserves domain path with version segment", () => {
+    // example.com/v1.0/README.md IS linkified (has domain), preserved as link
+    const result = markdownToTelegramHtml("example.com/v1.0/README.md");
+    expect(result).toContain('');
+  });
+
+  it("handles file ref with hyphen and underscore in name", () => {
+    const result = markdownToTelegramHtml("my-file_name.md");
+    expect(result).toContain("my-file_name.md");
+  });
+
+  it("handles uppercase extensions", () => {
+    const result = markdownToTelegramHtml("README.MD and SCRIPT.PY");
+    expect(result).toContain("README.MD");
+    expect(result).toContain("SCRIPT.PY");
+  });
+
+  it("handles nested code tags (depth tracking)", () => {
+    // Nested  inside 
 - should not wrap inner content
+    const input = "
README.md
 then script.py";
+    const result = wrapFileReferencesInHtml(input);
+    expect(result).toBe("
README.md
 then script.py");
+  });
+
+  it("handles multiple anchor tags in sequence", () => {
+    const input =
+      'link1 README.md link2 script.py';
+    const result = wrapFileReferencesInHtml(input);
+    expect(result).toContain(" README.md  script.py");
+  });
+
+  it("handles auto-linked anchor with backreference match", () => {
+    // The regex uses \1 backreference - href must equal label
+    const input = 'README.md';
+    expect(wrapFileReferencesInHtml(input)).toBe("README.md");
+  });
+
+  it("preserves anchor when href and label differ (no backreference match)", () => {
+    // Different href and label - should NOT de-linkify
+    const input = 'README.md';
+    expect(wrapFileReferencesInHtml(input)).toBe(input);
+  });
+
+  it("wraps orphaned TLD pattern after special character", () => {
+    // R&D.md - the & breaks the main pattern, but D.md could be auto-linked
+    // So we wrap the orphaned D.md part to prevent Telegram linking it
+    const input = "R&D.md";
+    const result = wrapFileReferencesInHtml(input);
+    expect(result).toBe("R&D.md");
+  });
+
+  it("wraps orphaned single-letter TLD patterns", () => {
+    // Use extensions still in the set (md, sh, py, go)
+    const result1 = wrapFileReferencesInHtml("X.md is cool");
+    expect(result1).toContain("X.md");
+
+    const result2 = wrapFileReferencesInHtml("Check R.sh");
+    expect(result2).toContain("R.sh");
+  });
+
+  it("does not match filenames containing angle brackets", () => {
+    // The regex character class [a-zA-Z0-9_.\\-./] doesn't include < >
+    // so these won't be matched and wrapped (which is correct/safe)
+    const input = "file