diff --git a/CHANGELOG.md b/CHANGELOG.md index 534922abe57..ce8a07061ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,10 @@ Docs: https://docs.openclaw.ai - Agents/fallback observability: add structured, sanitized model-fallback decision and auth-profile failure-state events with correlated run IDs so cooldown probes and failover paths are easier to trace in logs. (#41337) thanks @altaywtf. - Protocol/Swift model sync: regenerate pending node work Swift bindings after the landed `node.pending.*` schema additions so generated protocol artifacts are consistent again. (#41477) Thanks @mbelinky. - Discord/reply chunking: resolve the effective `maxLinesPerMessage` config across live reply paths and preserve `chunkMode` in the fast send path so long Discord replies no longer split unexpectedly at the default 17-line limit. (#40133) thanks @rbutera. +- Logging/probe observations: suppress structured embedded and model-fallback probe warnings on the console without hiding error or fatal output. (#41338) thanks @altaywtf. +- Agents/fallback: treat HTTP 499 responses as transient in both raw-text and structured failover paths so Anthropic-style client-closed overload responses trigger model fallback reliably. (#41468) thanks @zeroasterisk. +- Plugins/context-engine model auth: expose `runtime.modelAuth` and plugin-sdk auth helpers so plugins can resolve provider/model API keys through the normal auth pipeline. (#41090) thanks @xinhuagu. +- CLI/memory teardown: close cached memory search/index managers in the one-shot CLI shutdown path so watcher-backed memory caches no longer keep completed CLI runs alive after output finishes. (#40389) thanks @Julbarth. ## 2026.3.8 diff --git a/extensions/test-utils/plugin-runtime-mock.ts b/extensions/test-utils/plugin-runtime-mock.ts index 8c599599a31..81e3fdedeec 100644 --- a/extensions/test-utils/plugin-runtime-mock.ts +++ b/extensions/test-utils/plugin-runtime-mock.ts @@ -253,6 +253,11 @@ export function createPluginRuntimeMock(overrides: DeepPartial = state: { resolveStateDir: vi.fn(() => "/tmp/openclaw"), }, + modelAuth: { + getApiKeyForModel: vi.fn() as unknown as PluginRuntime["modelAuth"]["getApiKeyForModel"], + resolveApiKeyForProvider: + vi.fn() as unknown as PluginRuntime["modelAuth"]["resolveApiKeyForProvider"], + }, subagent: { run: vi.fn(), waitForRun: vi.fn(), diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts index a99cfb5c4b2..db01c03d8c4 100644 --- a/src/agents/failover-error.test.ts +++ b/src/agents/failover-error.test.ts @@ -67,6 +67,7 @@ describe("failover-error", () => { expect(resolveFailoverReasonFromError({ statusCode: "429" })).toBe("rate_limit"); expect(resolveFailoverReasonFromError({ status: 403 })).toBe("auth"); expect(resolveFailoverReasonFromError({ status: 408 })).toBe("timeout"); + expect(resolveFailoverReasonFromError({ status: 499 })).toBe("timeout"); expect(resolveFailoverReasonFromError({ status: 400 })).toBe("format"); // Keep the status-only path behavior-preserving and conservative. expect(resolveFailoverReasonFromError({ status: 500 })).toBeNull(); @@ -93,6 +94,12 @@ describe("failover-error", () => { message: ANTHROPIC_OVERLOADED_PAYLOAD, }), ).toBe("overloaded"); + expect( + resolveFailoverReasonFromError({ + status: 499, + message: ANTHROPIC_OVERLOADED_PAYLOAD, + }), + ).toBe("overloaded"); expect( resolveFailoverReasonFromError({ status: 429, diff --git a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts index 86fd90e7161..f60a127a0ab 100644 --- a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts +++ b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts @@ -443,6 +443,7 @@ describe("isLikelyContextOverflowError", () => { describe("isTransientHttpError", () => { it("returns true for retryable 5xx status codes", () => { + expect(isTransientHttpError("499 Client Closed Request")).toBe(true); expect(isTransientHttpError("500 Internal Server Error")).toBe(true); expect(isTransientHttpError("502 Bad Gateway")).toBe(true); expect(isTransientHttpError("503 Service Unavailable")).toBe(true); @@ -457,6 +458,19 @@ describe("isTransientHttpError", () => { }); }); +describe("classifyFailoverReasonFromHttpStatus", () => { + it("treats HTTP 499 as transient for structured errors", () => { + expect(classifyFailoverReasonFromHttpStatus(499)).toBe("timeout"); + expect(classifyFailoverReasonFromHttpStatus(499, "499 Client Closed Request")).toBe("timeout"); + expect( + classifyFailoverReasonFromHttpStatus( + 499, + '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', + ), + ).toBe("overloaded"); + }); +}); + describe("isFailoverErrorMessage", () => { it("matches auth/rate/billing/timeout", () => { const samples = [ diff --git a/src/agents/pi-embedded-helpers/errors.ts b/src/agents/pi-embedded-helpers/errors.ts index 4cf347150bf..9ab52c04355 100644 --- a/src/agents/pi-embedded-helpers/errors.ts +++ b/src/agents/pi-embedded-helpers/errors.ts @@ -189,7 +189,7 @@ const HTTP_STATUS_PREFIX_RE = /^(?:http\s*)?(\d{3})\s+(.+)$/i; const HTTP_STATUS_CODE_PREFIX_RE = /^(?:http\s*)?(\d{3})(?:\s+([\s\S]+))?$/i; const HTML_ERROR_PREFIX_RE = /^\s*(?: { "1.5", ]), ); - expect(args).toEqual(expect.arrayContaining(["--env", "LANG=C.UTF-8"])); + expect(args).toEqual( + expect.arrayContaining([ + "--env", + "LANG=C.UTF-8", + "--env", + `OPENCLAW_CLI=${OPENCLAW_CLI_ENV_VALUE}`, + ]), + ); const ulimitValues: string[] = []; for (let i = 0; i < args.length; i += 1) { diff --git a/src/agents/sandbox/docker.ts b/src/agents/sandbox/docker.ts index 2bd9dad12b5..68c95e343ea 100644 --- a/src/agents/sandbox/docker.ts +++ b/src/agents/sandbox/docker.ts @@ -162,6 +162,7 @@ export function execDockerRaw( } import { formatCliCommand } from "../../cli/command-format.js"; +import { markOpenClawExecEnv } from "../../infra/openclaw-exec-env.js"; import { defaultRuntime } from "../../runtime.js"; import { computeSandboxConfigHash } from "./config-hash.js"; import { DEFAULT_SANDBOX_IMAGE } from "./constants.js"; @@ -365,7 +366,7 @@ export function buildSandboxCreateArgs(params: { if (params.cfg.user) { args.push("--user", params.cfg.user); } - const envSanitization = sanitizeEnvVars(params.cfg.env ?? {}); + const envSanitization = sanitizeEnvVars(markOpenClawExecEnv(params.cfg.env ?? {})); if (envSanitization.blocked.length > 0) { log.warn(`Blocked sensitive environment variables: ${envSanitization.blocked.join(", ")}`); } diff --git a/src/cli/run-main.exit.test.ts b/src/cli/run-main.exit.test.ts index 86d74f09640..3e56c1ce794 100644 --- a/src/cli/run-main.exit.test.ts +++ b/src/cli/run-main.exit.test.ts @@ -6,6 +6,7 @@ const loadDotEnvMock = vi.hoisted(() => vi.fn()); const normalizeEnvMock = vi.hoisted(() => vi.fn()); const ensurePathMock = vi.hoisted(() => vi.fn()); const assertRuntimeMock = vi.hoisted(() => vi.fn()); +const closeAllMemorySearchManagersMock = vi.hoisted(() => vi.fn(async () => {})); vi.mock("./route.js", () => ({ tryRouteCli: tryRouteCliMock, @@ -27,6 +28,10 @@ vi.mock("../infra/runtime-guard.js", () => ({ assertSupportedRuntime: assertRuntimeMock, })); +vi.mock("../memory/search-manager.js", () => ({ + closeAllMemorySearchManagers: closeAllMemorySearchManagersMock, +})); + const { runCli } = await import("./run-main.js"); describe("runCli exit behavior", () => { @@ -43,6 +48,7 @@ describe("runCli exit behavior", () => { await runCli(["node", "openclaw", "status"]); expect(tryRouteCliMock).toHaveBeenCalledWith(["node", "openclaw", "status"]); + expect(closeAllMemorySearchManagersMock).toHaveBeenCalledTimes(1); expect(exitSpy).not.toHaveBeenCalled(); exitSpy.mockRestore(); }); diff --git a/src/cli/run-main.ts b/src/cli/run-main.ts index e80ce97b845..c0673ddf2af 100644 --- a/src/cli/run-main.ts +++ b/src/cli/run-main.ts @@ -13,6 +13,15 @@ import { applyCliProfileEnv, parseCliProfileArgs } from "./profile.js"; import { tryRouteCli } from "./route.js"; import { normalizeWindowsArgv } from "./windows-argv.js"; +async function closeCliMemoryManagers(): Promise { + try { + const { closeAllMemorySearchManagers } = await import("../memory/search-manager.js"); + await closeAllMemorySearchManagers(); + } catch { + // Best-effort teardown for short-lived CLI processes. + } +} + export function rewriteUpdateFlagArgv(argv: string[]): string[] { const index = argv.indexOf("--update"); if (index === -1) { @@ -82,59 +91,63 @@ export async function runCli(argv: string[] = process.argv) { // Enforce the minimum supported runtime before doing any work. assertSupportedRuntime(); - if (await tryRouteCli(normalizedArgv)) { - return; - } - - // Capture all console output into structured logs while keeping stdout/stderr behavior. - enableConsoleCapture(); - - const { buildProgram } = await import("./program.js"); - const program = buildProgram(); - - // Global error handlers to prevent silent crashes from unhandled rejections/exceptions. - // These log the error and exit gracefully instead of crashing without trace. - installUnhandledRejectionHandler(); - - process.on("uncaughtException", (error) => { - console.error("[openclaw] Uncaught exception:", formatUncaughtError(error)); - process.exit(1); - }); - - const parseArgv = rewriteUpdateFlagArgv(normalizedArgv); - // Register the primary command (builtin or subcli) so help and command parsing - // are correct even with lazy command registration. - const primary = getPrimaryCommand(parseArgv); - if (primary) { - const { getProgramContext } = await import("./program/program-context.js"); - const ctx = getProgramContext(program); - if (ctx) { - const { registerCoreCliByName } = await import("./program/command-registry.js"); - await registerCoreCliByName(program, ctx, primary, parseArgv); + try { + if (await tryRouteCli(normalizedArgv)) { + return; } - const { registerSubCliByName } = await import("./program/register.subclis.js"); - await registerSubCliByName(program, primary); - } - const hasBuiltinPrimary = - primary !== null && program.commands.some((command) => command.name() === primary); - const shouldSkipPluginRegistration = shouldSkipPluginCommandRegistration({ - argv: parseArgv, - primary, - hasBuiltinPrimary, - }); - if (!shouldSkipPluginRegistration) { - // Register plugin CLI commands before parsing - const { registerPluginCliCommands } = await import("../plugins/cli.js"); - const { loadValidatedConfigForPluginRegistration } = - await import("./program/register.subclis.js"); - const config = await loadValidatedConfigForPluginRegistration(); - if (config) { - registerPluginCliCommands(program, config); + // Capture all console output into structured logs while keeping stdout/stderr behavior. + enableConsoleCapture(); + + const { buildProgram } = await import("./program.js"); + const program = buildProgram(); + + // Global error handlers to prevent silent crashes from unhandled rejections/exceptions. + // These log the error and exit gracefully instead of crashing without trace. + installUnhandledRejectionHandler(); + + process.on("uncaughtException", (error) => { + console.error("[openclaw] Uncaught exception:", formatUncaughtError(error)); + process.exit(1); + }); + + const parseArgv = rewriteUpdateFlagArgv(normalizedArgv); + // Register the primary command (builtin or subcli) so help and command parsing + // are correct even with lazy command registration. + const primary = getPrimaryCommand(parseArgv); + if (primary) { + const { getProgramContext } = await import("./program/program-context.js"); + const ctx = getProgramContext(program); + if (ctx) { + const { registerCoreCliByName } = await import("./program/command-registry.js"); + await registerCoreCliByName(program, ctx, primary, parseArgv); + } + const { registerSubCliByName } = await import("./program/register.subclis.js"); + await registerSubCliByName(program, primary); } - } - await program.parseAsync(parseArgv); + const hasBuiltinPrimary = + primary !== null && program.commands.some((command) => command.name() === primary); + const shouldSkipPluginRegistration = shouldSkipPluginCommandRegistration({ + argv: parseArgv, + primary, + hasBuiltinPrimary, + }); + if (!shouldSkipPluginRegistration) { + // Register plugin CLI commands before parsing + const { registerPluginCliCommands } = await import("../plugins/cli.js"); + const { loadValidatedConfigForPluginRegistration } = + await import("./program/register.subclis.js"); + const config = await loadValidatedConfigForPluginRegistration(); + if (config) { + registerPluginCliCommands(program, config); + } + } + + await program.parseAsync(parseArgv); + } finally { + await closeCliMemoryManagers(); + } } export function isCliMainModule(): boolean { diff --git a/src/entry.ts b/src/entry.ts index 50b08029d05..14a839f38b9 100644 --- a/src/entry.ts +++ b/src/entry.ts @@ -9,6 +9,7 @@ import { shouldSkipRespawnForArgv } from "./cli/respawn-policy.js"; import { normalizeWindowsArgv } from "./cli/windows-argv.js"; import { isTruthyEnvValue, normalizeEnv } from "./infra/env.js"; import { isMainModule } from "./infra/is-main.js"; +import { ensureOpenClawExecMarkerOnProcess } from "./infra/openclaw-exec-env.js"; import { installProcessWarningFilter } from "./infra/warning-filter.js"; import { attachChildProcessBridge } from "./process/child-process-bridge.js"; @@ -41,6 +42,7 @@ if ( // Imported as a dependency — skip all entry-point side effects. } else { process.title = "openclaw"; + ensureOpenClawExecMarkerOnProcess(); installProcessWarningFilter(); normalizeEnv(); if (!isTruthyEnvValue(process.env.NODE_DISABLE_COMPILE_CACHE)) { diff --git a/src/infra/host-env-security.test.ts b/src/infra/host-env-security.test.ts index 116006dbbcf..4e7bcdb9ed9 100644 --- a/src/infra/host-env-security.test.ts +++ b/src/infra/host-env-security.test.ts @@ -10,6 +10,7 @@ import { sanitizeHostExecEnv, sanitizeSystemRunEnvOverrides, } from "./host-env-security.js"; +import { OPENCLAW_CLI_ENV_VALUE } from "./openclaw-exec-env.js"; describe("isDangerousHostEnvVarName", () => { it("matches dangerous keys and prefixes case-insensitively", () => { @@ -40,6 +41,7 @@ describe("sanitizeHostExecEnv", () => { }); expect(env).toEqual({ + OPENCLAW_CLI: OPENCLAW_CLI_ENV_VALUE, PATH: "/usr/bin:/bin", OK: "1", }); @@ -68,6 +70,7 @@ describe("sanitizeHostExecEnv", () => { }); expect(env.PATH).toBe("/usr/bin:/bin"); + expect(env.OPENCLAW_CLI).toBe(OPENCLAW_CLI_ENV_VALUE); expect(env.BASH_ENV).toBeUndefined(); expect(env.GIT_SSH_COMMAND).toBeUndefined(); expect(env.EDITOR).toBeUndefined(); @@ -91,6 +94,7 @@ describe("sanitizeHostExecEnv", () => { }); expect(env.PATH).toBe("/usr/bin:/bin"); + expect(env.OPENCLAW_CLI).toBe(OPENCLAW_CLI_ENV_VALUE); expect(env.OK).toBe("1"); expect(env.SHELLOPTS).toBeUndefined(); expect(env.PS4).toBeUndefined(); @@ -109,6 +113,7 @@ describe("sanitizeHostExecEnv", () => { }); expect(env.GOOD_KEY).toBe("ok"); + expect(env.OPENCLAW_CLI).toBe(OPENCLAW_CLI_ENV_VALUE); expect(env[" BAD KEY"]).toBeUndefined(); expect(env["NOT-PORTABLE"]).toBeUndefined(); }); diff --git a/src/infra/host-env-security.ts b/src/infra/host-env-security.ts index 56b30bd0818..8c5d0989fdd 100644 --- a/src/infra/host-env-security.ts +++ b/src/infra/host-env-security.ts @@ -1,4 +1,5 @@ import HOST_ENV_SECURITY_POLICY_JSON from "./host-env-security-policy.json" with { type: "json" }; +import { markOpenClawExecEnv } from "./openclaw-exec-env.js"; const PORTABLE_ENV_VAR_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/; @@ -101,7 +102,7 @@ export function sanitizeHostExecEnv(params?: { } if (!overrides) { - return merged; + return markOpenClawExecEnv(merged); } for (const [rawKey, value] of Object.entries(overrides)) { @@ -124,7 +125,7 @@ export function sanitizeHostExecEnv(params?: { merged[key] = value; } - return merged; + return markOpenClawExecEnv(merged); } export function sanitizeSystemRunEnvOverrides(params?: { diff --git a/src/infra/openclaw-exec-env.ts b/src/infra/openclaw-exec-env.ts new file mode 100644 index 00000000000..b4e8a876584 --- /dev/null +++ b/src/infra/openclaw-exec-env.ts @@ -0,0 +1,16 @@ +export const OPENCLAW_CLI_ENV_VAR = "OPENCLAW_CLI"; +export const OPENCLAW_CLI_ENV_VALUE = "1"; + +export function markOpenClawExecEnv>(env: T): T { + return { + ...env, + [OPENCLAW_CLI_ENV_VAR]: OPENCLAW_CLI_ENV_VALUE, + }; +} + +export function ensureOpenClawExecMarkerOnProcess( + env: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + env[OPENCLAW_CLI_ENV_VAR] = OPENCLAW_CLI_ENV_VALUE; + return env; +} diff --git a/src/logging/subsystem.test.ts b/src/logging/subsystem.test.ts index e389d78ba8a..06f504f47de 100644 --- a/src/logging/subsystem.test.ts +++ b/src/logging/subsystem.test.ts @@ -1,11 +1,13 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { setConsoleSubsystemFilter } from "./console.js"; import { resetLogger, setLoggerOverride } from "./logger.js"; +import { loggingState } from "./state.js"; import { createSubsystemLogger } from "./subsystem.js"; afterEach(() => { setConsoleSubsystemFilter(null); setLoggerOverride(null); + loggingState.rawConsole = null; resetLogger(); }); @@ -53,4 +55,118 @@ describe("createSubsystemLogger().isEnabled", () => { expect(log.isEnabled("info", "file")).toBe(true); expect(log.isEnabled("info")).toBe(true); }); + + it("suppresses probe warnings for embedded subsystems based on structured run metadata", () => { + setLoggerOverride({ level: "silent", consoleLevel: "warn" }); + const warn = vi.fn(); + loggingState.rawConsole = { + log: vi.fn(), + info: vi.fn(), + warn, + error: vi.fn(), + }; + const log = createSubsystemLogger("agent/embedded").child("failover"); + + log.warn("embedded run failover decision", { + runId: "probe-test-run", + consoleMessage: "embedded run failover decision", + }); + + expect(warn).not.toHaveBeenCalled(); + }); + + it("does not suppress probe errors for embedded subsystems", () => { + setLoggerOverride({ level: "silent", consoleLevel: "error" }); + const error = vi.fn(); + loggingState.rawConsole = { + log: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error, + }; + const log = createSubsystemLogger("agent/embedded").child("failover"); + + log.error("embedded run failover decision", { + runId: "probe-test-run", + consoleMessage: "embedded run failover decision", + }); + + expect(error).toHaveBeenCalledTimes(1); + }); + + it("suppresses probe warnings for model-fallback child subsystems based on structured run metadata", () => { + setLoggerOverride({ level: "silent", consoleLevel: "warn" }); + const warn = vi.fn(); + loggingState.rawConsole = { + log: vi.fn(), + info: vi.fn(), + warn, + error: vi.fn(), + }; + const log = createSubsystemLogger("model-fallback").child("decision"); + + log.warn("model fallback decision", { + runId: "probe-test-run", + consoleMessage: "model fallback decision", + }); + + expect(warn).not.toHaveBeenCalled(); + }); + + it("does not suppress probe errors for model-fallback child subsystems", () => { + setLoggerOverride({ level: "silent", consoleLevel: "error" }); + const error = vi.fn(); + loggingState.rawConsole = { + log: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error, + }; + const log = createSubsystemLogger("model-fallback").child("decision"); + + log.error("model fallback decision", { + runId: "probe-test-run", + consoleMessage: "model fallback decision", + }); + + expect(error).toHaveBeenCalledTimes(1); + }); + + it("still emits non-probe warnings for embedded subsystems", () => { + setLoggerOverride({ level: "silent", consoleLevel: "warn" }); + const warn = vi.fn(); + loggingState.rawConsole = { + log: vi.fn(), + info: vi.fn(), + warn, + error: vi.fn(), + }; + const log = createSubsystemLogger("agent/embedded").child("auth-profiles"); + + log.warn("auth profile failure state updated", { + runId: "run-123", + consoleMessage: "auth profile failure state updated", + }); + + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("still emits non-probe model-fallback child warnings", () => { + setLoggerOverride({ level: "silent", consoleLevel: "warn" }); + const warn = vi.fn(); + loggingState.rawConsole = { + log: vi.fn(), + info: vi.fn(), + warn, + error: vi.fn(), + }; + const log = createSubsystemLogger("model-fallback").child("decision"); + + log.warn("model fallback decision", { + runId: "run-123", + consoleMessage: "model fallback decision", + }); + + expect(warn).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/logging/subsystem.ts b/src/logging/subsystem.ts index 18be000e9ba..5c6ce58a43d 100644 --- a/src/logging/subsystem.ts +++ b/src/logging/subsystem.ts @@ -250,6 +250,38 @@ function writeConsoleLine(level: LogLevel, line: string) { } } +function shouldSuppressProbeConsoleLine(params: { + level: LogLevel; + subsystem: string; + message: string; + meta?: Record; +}): boolean { + if (isVerbose()) { + return false; + } + if (params.level === "error" || params.level === "fatal") { + return false; + } + const isProbeSuppressedSubsystem = + params.subsystem === "agent/embedded" || + params.subsystem.startsWith("agent/embedded/") || + params.subsystem === "model-fallback" || + params.subsystem.startsWith("model-fallback/"); + if (!isProbeSuppressedSubsystem) { + return false; + } + const runLikeId = + typeof params.meta?.runId === "string" + ? params.meta.runId + : typeof params.meta?.sessionId === "string" + ? params.meta.sessionId + : undefined; + if (runLikeId?.startsWith("probe-")) { + return true; + } + return /(sessionId|runId)=probe-/.test(params.message); +} + function logToFile( fileLogger: TsLogger, level: LogLevel, @@ -309,9 +341,12 @@ export function createSubsystemLogger(subsystem: string): SubsystemLogger { } const consoleMessage = consoleMessageOverride ?? message; if ( - !isVerbose() && - subsystem === "agent/embedded" && - /(sessionId|runId)=probe-/.test(consoleMessage) + shouldSuppressProbeConsoleLine({ + level, + subsystem, + message: consoleMessage, + meta: fileMeta, + }) ) { return; } @@ -355,11 +390,7 @@ export function createSubsystemLogger(subsystem: string): SubsystemLogger { logToFile(getFileLogger(), "info", message, { raw: true }); } if (isConsoleEnabled("info")) { - if ( - !isVerbose() && - subsystem === "agent/embedded" && - /(sessionId|runId)=probe-/.test(message) - ) { + if (shouldSuppressProbeConsoleLine({ level: "info", subsystem, message })) { return; } writeConsoleLine("info", message); diff --git a/src/memory/index.ts b/src/memory/index.ts index 4d2df05a399..86ca52e1d27 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -4,4 +4,8 @@ export type { MemorySearchManager, MemorySearchResult, } from "./types.js"; -export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js"; +export { + closeAllMemorySearchManagers, + getMemorySearchManager, + type MemorySearchManagerResult, +} from "./search-manager.js"; diff --git a/src/memory/manager-runtime.ts b/src/memory/manager-runtime.ts index b46b3708a6e..3e910b5676a 100644 --- a/src/memory/manager-runtime.ts +++ b/src/memory/manager-runtime.ts @@ -1 +1 @@ -export { MemoryIndexManager } from "./manager.js"; +export { closeAllMemoryIndexManagers, MemoryIndexManager } from "./manager.js"; diff --git a/src/memory/manager.get-concurrency.test.ts b/src/memory/manager.get-concurrency.test.ts index e7d040217a8..67b10768fc3 100644 --- a/src/memory/manager.get-concurrency.test.ts +++ b/src/memory/manager.get-concurrency.test.ts @@ -4,6 +4,10 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { getMemorySearchManager, type MemoryIndexManager } from "./index.js"; +import { + closeAllMemoryIndexManagers, + MemoryIndexManager as RawMemoryIndexManager, +} from "./manager.js"; import "./test-runtime-mocks.js"; const hoisted = vi.hoisted(() => ({ @@ -78,4 +82,37 @@ describe("memory manager cache hydration", () => { await managers[0].close(); }); + + it("drains in-flight manager creation during global teardown", async () => { + const indexPath = path.join(workspaceDir, "index.sqlite"); + const cfg = { + agents: { + defaults: { + workspace: workspaceDir, + memorySearch: { + provider: "openai", + model: "mock-embed", + store: { path: indexPath, vector: { enabled: false } }, + sync: { watch: false, onSessionStart: false, onSearch: false }, + }, + }, + list: [{ id: "main", default: true }], + }, + } as OpenClawConfig; + + hoisted.providerDelayMs = 100; + + const pendingResult = RawMemoryIndexManager.get({ cfg, agentId: "main" }); + await closeAllMemoryIndexManagers(); + const firstManager = await pendingResult; + + const secondManager = await RawMemoryIndexManager.get({ cfg, agentId: "main" }); + + expect(firstManager).toBeTruthy(); + expect(secondManager).toBeTruthy(); + expect(Object.is(secondManager, firstManager)).toBe(false); + expect(hoisted.providerCreateCalls).toBe(2); + + await secondManager?.close?.(); + }); }); diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 1d2fb49e88b..9b1ff74e54c 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -42,6 +42,22 @@ const log = createSubsystemLogger("memory"); const INDEX_CACHE = new Map(); const INDEX_CACHE_PENDING = new Map>(); +export async function closeAllMemoryIndexManagers(): Promise { + const pending = Array.from(INDEX_CACHE_PENDING.values()); + if (pending.length > 0) { + await Promise.allSettled(pending); + } + const managers = Array.from(INDEX_CACHE.values()); + INDEX_CACHE.clear(); + for (const manager of managers) { + try { + await manager.close(); + } catch (err) { + log.warn(`failed to close memory index manager: ${String(err)}`); + } + } +} + export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements MemorySearchManager { private readonly cacheKey: string; protected readonly cfg: OpenClawConfig; diff --git a/src/memory/search-manager.test.ts b/src/memory/search-manager.test.ts index d853f5af1fa..1f705aeddcf 100644 --- a/src/memory/search-manager.test.ts +++ b/src/memory/search-manager.test.ts @@ -29,53 +29,53 @@ function createManagerStatus(params: { }; } -const qmdManagerStatus = createManagerStatus({ - backend: "qmd", - provider: "qmd", - model: "qmd", - requestedProvider: "qmd", - withMemorySourceCounts: true, -}); - -const fallbackManagerStatus = createManagerStatus({ - backend: "builtin", - provider: "openai", - model: "text-embedding-3-small", - requestedProvider: "openai", -}); - -const mockPrimary = { +const mockPrimary = vi.hoisted(() => ({ search: vi.fn(async () => []), readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })), - status: vi.fn(() => qmdManagerStatus), + status: vi.fn(() => + createManagerStatus({ + backend: "qmd", + provider: "qmd", + model: "qmd", + requestedProvider: "qmd", + withMemorySourceCounts: true, + }), + ), sync: vi.fn(async () => {}), probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })), probeVectorAvailability: vi.fn(async () => true), close: vi.fn(async () => {}), -}; +})); -const fallbackSearch = vi.fn(async () => [ - { - path: "MEMORY.md", - startLine: 1, - endLine: 1, - score: 1, - snippet: "fallback", - source: "memory" as const, - }, -]); - -const fallbackManager = { - search: fallbackSearch, +const fallbackManager = vi.hoisted(() => ({ + search: vi.fn(async () => [ + { + path: "MEMORY.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "fallback", + source: "memory" as const, + }, + ]), readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })), - status: vi.fn(() => fallbackManagerStatus), + status: vi.fn(() => + createManagerStatus({ + backend: "builtin", + provider: "openai", + model: "text-embedding-3-small", + requestedProvider: "openai", + }), + ), sync: vi.fn(async () => {}), probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })), probeVectorAvailability: vi.fn(async () => true), close: vi.fn(async () => {}), -}; +})); -const mockMemoryIndexGet = vi.fn(async () => fallbackManager); +const fallbackSearch = fallbackManager.search; +const mockMemoryIndexGet = vi.hoisted(() => vi.fn(async () => fallbackManager)); +const mockCloseAllMemoryIndexManagers = vi.hoisted(() => vi.fn(async () => {})); vi.mock("./qmd-manager.js", () => ({ QmdMemoryManager: { @@ -83,14 +83,15 @@ vi.mock("./qmd-manager.js", () => ({ }, })); -vi.mock("./manager.js", () => ({ +vi.mock("./manager-runtime.js", () => ({ MemoryIndexManager: { get: mockMemoryIndexGet, }, + closeAllMemoryIndexManagers: mockCloseAllMemoryIndexManagers, })); import { QmdMemoryManager } from "./qmd-manager.js"; -import { getMemorySearchManager } from "./search-manager.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./search-manager.js"; // eslint-disable-next-line @typescript-eslint/unbound-method -- mocked static function const createQmdManagerMock = vi.mocked(QmdMemoryManager.create); @@ -119,7 +120,8 @@ async function createFailedQmdSearchHarness(params: { agentId: string; errorMess return { cfg, manager: requireManager(first), firstResult: first }; } -beforeEach(() => { +beforeEach(async () => { + await closeAllMemorySearchManagers(); mockPrimary.search.mockClear(); mockPrimary.readFile.mockClear(); mockPrimary.status.mockClear(); @@ -134,6 +136,7 @@ beforeEach(() => { fallbackManager.probeEmbeddingAvailability.mockClear(); fallbackManager.probeVectorAvailability.mockClear(); fallbackManager.close.mockClear(); + mockCloseAllMemoryIndexManagers.mockClear(); mockMemoryIndexGet.mockClear(); mockMemoryIndexGet.mockResolvedValue(fallbackManager); createQmdManagerMock.mockClear(); @@ -243,4 +246,34 @@ describe("getMemorySearchManager caching", () => { await expect(firstManager.search("hello")).rejects.toThrow("qmd query failed"); }); + + it("closes cached managers on global teardown", async () => { + const cfg = createQmdCfg("teardown-agent"); + const first = await getMemorySearchManager({ cfg, agentId: "teardown-agent" }); + const firstManager = requireManager(first); + + await closeAllMemorySearchManagers(); + + expect(mockPrimary.close).toHaveBeenCalledTimes(1); + expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1); + + const second = await getMemorySearchManager({ cfg, agentId: "teardown-agent" }); + expect(second.manager).toBeTruthy(); + expect(second.manager).not.toBe(firstManager); + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(createQmdManagerMock).toHaveBeenCalledTimes(2); + }); + + it("closes builtin index managers on teardown after runtime is loaded", async () => { + const retryAgentId = "teardown-with-fallback"; + const { manager } = await createFailedQmdSearchHarness({ + agentId: retryAgentId, + errorMessage: "qmd query failed", + }); + await manager.search("hello"); + + await closeAllMemorySearchManagers(); + + expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index f4e351fdc1a..ea581b5d6da 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -85,6 +85,22 @@ export async function getMemorySearchManager(params: { } } +export async function closeAllMemorySearchManagers(): Promise { + const managers = Array.from(QMD_MANAGER_CACHE.values()); + QMD_MANAGER_CACHE.clear(); + for (const manager of managers) { + try { + await manager.close?.(); + } catch (err) { + log.warn(`failed to close qmd memory manager: ${String(err)}`); + } + } + if (managerRuntimePromise !== null) { + const { closeAllMemoryIndexManagers } = await loadManagerRuntime(); + await closeAllMemoryIndexManagers(); + } +} + class FallbackMemoryManager implements MemorySearchManager { private fallback: MemorySearchManager | null = null; private primaryFailed = false; diff --git a/src/plugin-sdk/index.ts b/src/plugin-sdk/index.ts index 3e1ba0f03ab..35709dc4fec 100644 --- a/src/plugin-sdk/index.ts +++ b/src/plugin-sdk/index.ts @@ -801,5 +801,11 @@ export type { export { registerContextEngine } from "../context-engine/registry.js"; export type { ContextEngineFactory } from "../context-engine/registry.js"; +// Model authentication types for plugins. +// Plugins should use runtime.modelAuth (which strips unsafe overrides like +// agentDir/store) rather than importing raw helpers directly. +export { requireApiKey } from "../agents/model-auth.js"; +export type { ResolvedProviderAuth } from "../agents/model-auth.js"; + // Security utilities export { redactSensitiveText } from "../logging/redact.js"; diff --git a/src/plugins/runtime/index.test.ts b/src/plugins/runtime/index.test.ts index 77b3de66062..5ec2df28199 100644 --- a/src/plugins/runtime/index.test.ts +++ b/src/plugins/runtime/index.test.ts @@ -53,4 +53,21 @@ describe("plugin runtime command execution", () => { const runtime = createPluginRuntime(); expect(runtime.system.requestHeartbeatNow).toBe(requestHeartbeatNow); }); + + it("exposes runtime.modelAuth with getApiKeyForModel and resolveApiKeyForProvider", () => { + const runtime = createPluginRuntime(); + expect(runtime.modelAuth).toBeDefined(); + expect(typeof runtime.modelAuth.getApiKeyForModel).toBe("function"); + expect(typeof runtime.modelAuth.resolveApiKeyForProvider).toBe("function"); + }); + + it("modelAuth wrappers strip agentDir and store to prevent credential steering", async () => { + // The wrappers should not forward agentDir or store from plugin callers. + // We verify this by checking the wrapper functions exist and are not the + // raw implementations (they are wrapped, not direct references). + const { getApiKeyForModel: rawGetApiKey } = await import("../../agents/model-auth.js"); + const runtime = createPluginRuntime(); + // Wrappers should NOT be the same reference as the raw functions + expect(runtime.modelAuth.getApiKeyForModel).not.toBe(rawGetApiKey); + }); }); diff --git a/src/plugins/runtime/index.ts b/src/plugins/runtime/index.ts index 68b672db1b4..12d33168cd3 100644 --- a/src/plugins/runtime/index.ts +++ b/src/plugins/runtime/index.ts @@ -1,4 +1,8 @@ import { createRequire } from "node:module"; +import { + getApiKeyForModel as getApiKeyForModelRaw, + resolveApiKeyForProvider as resolveApiKeyForProviderRaw, +} from "../../agents/model-auth.js"; import { resolveStateDir } from "../../config/paths.js"; import { transcribeAudioFile } from "../../media-understanding/transcribe-audio.js"; import { textToSpeechTelephony } from "../../tts/tts.js"; @@ -59,6 +63,24 @@ export function createPluginRuntime(_options: CreatePluginRuntimeOptions = {}): events: createRuntimeEvents(), logging: createRuntimeLogging(), state: { resolveStateDir }, + modelAuth: { + // Wrap model-auth helpers so plugins cannot steer credential lookups: + // - agentDir / store: stripped (prevents reading other agents' stores) + // - profileId / preferredProfile: stripped (prevents cross-provider + // credential access via profile steering) + // Plugins only specify provider/model; the core auth pipeline picks + // the appropriate credential automatically. + getApiKeyForModel: (params) => + getApiKeyForModelRaw({ + model: params.model, + cfg: params.cfg, + }), + resolveApiKeyForProvider: (params) => + resolveApiKeyForProviderRaw({ + provider: params.provider, + cfg: params.cfg, + }), + }, } satisfies PluginRuntime; return runtime; diff --git a/src/plugins/runtime/types-core.ts b/src/plugins/runtime/types-core.ts index 524b3a5f6a2..bfbb747c9c4 100644 --- a/src/plugins/runtime/types-core.ts +++ b/src/plugins/runtime/types-core.ts @@ -52,4 +52,16 @@ export type PluginRuntimeCore = { state: { resolveStateDir: typeof import("../../config/paths.js").resolveStateDir; }; + modelAuth: { + /** Resolve auth for a model. Only provider/model and optional cfg are used. */ + getApiKeyForModel: (params: { + model: import("@mariozechner/pi-ai").Model; + cfg?: import("../../config/config.js").OpenClawConfig; + }) => Promise; + /** Resolve auth for a provider by name. Only provider and optional cfg are used. */ + resolveApiKeyForProvider: (params: { + provider: string; + cfg?: import("../../config/config.js").OpenClawConfig; + }) => Promise; + }; }; diff --git a/src/process/exec.test.ts b/src/process/exec.test.ts index 19937d6cb32..88d9cfdd71e 100644 --- a/src/process/exec.test.ts +++ b/src/process/exec.test.ts @@ -3,6 +3,7 @@ import { EventEmitter } from "node:events"; import fs from "node:fs"; import process from "node:process"; import { describe, expect, it, vi } from "vitest"; +import { OPENCLAW_CLI_ENV_VALUE } from "../infra/openclaw-exec-env.js"; import { attachChildProcessBridge } from "./child-process-bridge.js"; import { resolveCommandEnv, runCommandWithTimeout, shouldSpawnWithShell } from "./exec.js"; @@ -31,6 +32,7 @@ describe("runCommandWithTimeout", () => { expect(resolved.OPENCLAW_BASE_ENV).toBe("base"); expect(resolved.OPENCLAW_TEST_ENV).toBe("ok"); expect(resolved.OPENCLAW_TO_REMOVE).toBeUndefined(); + expect(resolved.OPENCLAW_CLI).toBe(OPENCLAW_CLI_ENV_VALUE); }); it("suppresses npm fund prompts for npm argv", async () => { diff --git a/src/process/exec.ts b/src/process/exec.ts index ddc572092d8..3464a083894 100644 --- a/src/process/exec.ts +++ b/src/process/exec.ts @@ -4,6 +4,7 @@ import path from "node:path"; import process from "node:process"; import { promisify } from "node:util"; import { danger, shouldLogVerbose } from "../globals.js"; +import { markOpenClawExecEnv } from "../infra/openclaw-exec-env.js"; import { logDebug, logError } from "../logger.js"; import { resolveCommandStdio } from "./spawn-utils.js"; @@ -213,7 +214,7 @@ export function resolveCommandEnv(params: { resolvedEnv.npm_config_fund = "false"; } } - return resolvedEnv; + return markOpenClawExecEnv(resolvedEnv); } export async function runCommandWithTimeout(