diff --git a/src/agents/sandbox/browser.create.test.ts b/src/agents/sandbox/browser.create.test.ts index 7962b0909ce..077db23c53b 100644 --- a/src/agents/sandbox/browser.create.test.ts +++ b/src/agents/sandbox/browser.create.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { BROWSER_BRIDGES } from "./browser-bridges.js"; import { ensureSandboxBrowser } from "./browser.js"; import { resetNoVncObserverTokensForTests } from "./novnc-auth.js"; +import { collectDockerFlagValues, findDockerArgsCall } from "./test-args.js"; import type { SandboxConfig } from "./types.js"; const dockerMocks = vi.hoisted(() => ({ @@ -85,16 +86,6 @@ function buildConfig(enableNoVnc: boolean): SandboxConfig { }; } -function envEntriesFromDockerArgs(args: string[]): string[] { - const values: string[] = []; - for (let i = 0; i < args.length; i += 1) { - if (args[i] === "-e" && typeof args[i + 1] === "string") { - values.push(args[i + 1]); - } - } - return values; -} - describe("ensureSandboxBrowser create args", () => { beforeEach(() => { BROWSER_BRIDGES.clear(); @@ -151,13 +142,11 @@ describe("ensureSandboxBrowser create args", () => { cfg: buildConfig(true), }); - const createArgs = dockerMocks.execDocker.mock.calls.find( - (call: unknown[]) => Array.isArray(call[0]) && call[0][0] === "create", - )?.[0] as string[] | undefined; + const createArgs = findDockerArgsCall(dockerMocks.execDocker.mock.calls, "create"); expect(createArgs).toBeDefined(); expect(createArgs).toContain("127.0.0.1::6080"); - const envEntries = envEntriesFromDockerArgs(createArgs ?? []); + const envEntries = collectDockerFlagValues(createArgs ?? [], "-e"); expect(envEntries).toContain("OPENCLAW_BROWSER_NO_SANDBOX=1"); const passwordEntry = envEntries.find((entry) => entry.startsWith("OPENCLAW_BROWSER_NOVNC_PASSWORD="), @@ -175,10 +164,8 @@ describe("ensureSandboxBrowser create args", () => { cfg: buildConfig(false), }); - const createArgs = dockerMocks.execDocker.mock.calls.find( - (call: unknown[]) => Array.isArray(call[0]) && call[0][0] === "create", - )?.[0] as string[] | undefined; - const envEntries = envEntriesFromDockerArgs(createArgs ?? []); + const createArgs = findDockerArgsCall(dockerMocks.execDocker.mock.calls, "create"); + const envEntries = collectDockerFlagValues(createArgs ?? [], "-e"); expect(envEntries.some((entry) => entry.startsWith("OPENCLAW_BROWSER_NOVNC_PASSWORD="))).toBe( false, ); @@ -196,9 +183,7 @@ describe("ensureSandboxBrowser create args", () => { cfg, }); - const createArgs = dockerMocks.execDocker.mock.calls.find( - (call: unknown[]) => Array.isArray(call[0]) && call[0][0] === "create", - )?.[0] as string[] | undefined; + const createArgs = findDockerArgsCall(dockerMocks.execDocker.mock.calls, "create"); expect(createArgs).toBeDefined(); expect(createArgs).toContain("/tmp/workspace:/workspace:ro"); @@ -215,9 +200,7 @@ describe("ensureSandboxBrowser create args", () => { cfg, }); - const createArgs = dockerMocks.execDocker.mock.calls.find( - (call: unknown[]) => Array.isArray(call[0]) && call[0][0] === "create", - )?.[0] as string[] | undefined; + const createArgs = findDockerArgsCall(dockerMocks.execDocker.mock.calls, "create"); expect(createArgs).toBeDefined(); expect(createArgs).toContain("/tmp/workspace:/workspace"); diff --git a/src/agents/sandbox/browser.ts b/src/agents/sandbox/browser.ts index a26c7a685b5..a0fdae3babe 100644 --- a/src/agents/sandbox/browser.ts +++ b/src/agents/sandbox/browser.ts @@ -11,11 +11,7 @@ import { defaultRuntime } from "../../runtime.js"; import { BROWSER_BRIDGES } from "./browser-bridges.js"; import { computeSandboxBrowserConfigHash } from "./config-hash.js"; import { resolveSandboxBrowserDockerCreateConfig } from "./config.js"; -import { - DEFAULT_SANDBOX_BROWSER_IMAGE, - SANDBOX_AGENT_WORKSPACE_MOUNT, - SANDBOX_BROWSER_SECURITY_HASH_EPOCH, -} from "./constants.js"; +import { DEFAULT_SANDBOX_BROWSER_IMAGE, SANDBOX_BROWSER_SECURITY_HASH_EPOCH } from "./constants.js"; import { buildSandboxCreateArgs, dockerContainerState, @@ -37,6 +33,7 @@ import { resolveSandboxAgentId, slugifySessionKey } from "./shared.js"; import { isToolAllowed } from "./tool-policy.js"; import type { SandboxBrowserContext, SandboxConfig } from "./types.js"; import { validateNetworkMode } from "./validate-sandbox-security.js"; +import { appendWorkspaceMountArgs } from "./workspace-mounts.js"; const HOT_BROWSER_WINDOW_MS = 5 * 60 * 1000; const CDP_SOURCE_RANGE_ENV_KEY = "OPENCLAW_BROWSER_CDP_SOURCE_RANGE"; @@ -237,15 +234,13 @@ export async function ensureSandboxBrowser(params: { includeBinds: false, bindSourceRoots: [params.workspaceDir, params.agentWorkspaceDir], }); - const mainMountSuffix = params.cfg.workspaceAccess === "rw" ? "" : ":ro"; - args.push("-v", `${params.workspaceDir}:${params.cfg.docker.workdir}${mainMountSuffix}`); - if (params.cfg.workspaceAccess !== "none" && params.workspaceDir !== params.agentWorkspaceDir) { - const agentMountSuffix = params.cfg.workspaceAccess === "ro" ? ":ro" : ""; - args.push( - "-v", - `${params.agentWorkspaceDir}:${SANDBOX_AGENT_WORKSPACE_MOUNT}${agentMountSuffix}`, - ); - } + appendWorkspaceMountArgs({ + args, + workspaceDir: params.workspaceDir, + agentWorkspaceDir: params.agentWorkspaceDir, + workdir: params.cfg.docker.workdir, + workspaceAccess: params.cfg.workspaceAccess, + }); if (browserDockerCfg.binds?.length) { for (const bind of browserDockerCfg.binds) { args.push("-v", bind); diff --git a/src/agents/sandbox/docker.config-hash-recreate.test.ts b/src/agents/sandbox/docker.config-hash-recreate.test.ts index 289bbef49e1..b2cd24c6630 100644 --- a/src/agents/sandbox/docker.config-hash-recreate.test.ts +++ b/src/agents/sandbox/docker.config-hash-recreate.test.ts @@ -3,6 +3,7 @@ import { Readable } from "node:stream"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { computeSandboxConfigHash } from "./config-hash.js"; import { ensureSandboxContainer } from "./docker.js"; +import { collectDockerFlagValues } from "./test-args.js"; import type { SandboxConfig } from "./types.js"; type SpawnCall = { @@ -237,13 +238,7 @@ describe("ensureSandboxContainer config-hash recreation", () => { expect(createCall).toBeDefined(); expect(createCall?.args).toContain(`openclaw.configHash=${expectedHash}`); - const bindArgs: string[] = []; - const args = createCall?.args ?? []; - for (let i = 0; i < args.length; i += 1) { - if (args[i] === "-v" && typeof args[i + 1] === "string") { - bindArgs.push(args[i + 1]); - } - } + const bindArgs = collectDockerFlagValues(createCall?.args ?? [], "-v"); const workspaceMountIdx = bindArgs.indexOf("/tmp/workspace:/workspace"); const customMountIdx = bindArgs.indexOf("/tmp/workspace-shared/USER.md:/workspace/USER.md:ro"); expect(workspaceMountIdx).toBeGreaterThanOrEqual(0); @@ -277,13 +272,7 @@ describe("ensureSandboxContainer config-hash recreation", () => { ); expect(createCall).toBeDefined(); - const bindArgs: string[] = []; - const args = createCall?.args ?? []; - for (let i = 0; i < args.length; i += 1) { - if (args[i] === "-v" && typeof args[i + 1] === "string") { - bindArgs.push(args[i + 1]); - } - } + const bindArgs = collectDockerFlagValues(createCall?.args ?? [], "-v"); expect(bindArgs).toContain(expectedMainMount); }, ); diff --git a/src/agents/sandbox/docker.ts b/src/agents/sandbox/docker.ts index a3550ac76ef..2bd9dad12b5 100644 --- a/src/agents/sandbox/docker.ts +++ b/src/agents/sandbox/docker.ts @@ -164,11 +164,12 @@ export function execDockerRaw( import { formatCliCommand } from "../../cli/command-format.js"; import { defaultRuntime } from "../../runtime.js"; import { computeSandboxConfigHash } from "./config-hash.js"; -import { DEFAULT_SANDBOX_IMAGE, SANDBOX_AGENT_WORKSPACE_MOUNT } from "./constants.js"; +import { DEFAULT_SANDBOX_IMAGE } from "./constants.js"; import { readRegistry, updateRegistry } from "./registry.js"; import { resolveSandboxAgentId, resolveSandboxScopeKey, slugifySessionKey } from "./shared.js"; import type { SandboxConfig, SandboxDockerConfig, SandboxWorkspaceAccess } from "./types.js"; import { validateSandboxSecurity } from "./validate-sandbox-security.js"; +import { appendWorkspaceMountArgs } from "./workspace-mounts.js"; const log = createSubsystemLogger("docker"); @@ -452,15 +453,13 @@ async function createSandboxContainer(params: { bindSourceRoots: [workspaceDir, params.agentWorkspaceDir], }); args.push("--workdir", cfg.workdir); - const mainMountSuffix = params.workspaceAccess === "rw" ? "" : ":ro"; - args.push("-v", `${workspaceDir}:${cfg.workdir}${mainMountSuffix}`); - if (params.workspaceAccess !== "none" && workspaceDir !== params.agentWorkspaceDir) { - const agentMountSuffix = params.workspaceAccess === "ro" ? ":ro" : ""; - args.push( - "-v", - `${params.agentWorkspaceDir}:${SANDBOX_AGENT_WORKSPACE_MOUNT}${agentMountSuffix}`, - ); - } + appendWorkspaceMountArgs({ + args, + workspaceDir, + agentWorkspaceDir: params.agentWorkspaceDir, + workdir: cfg.workdir, + workspaceAccess: params.workspaceAccess, + }); appendCustomBinds(args, cfg); args.push(cfg.image, "sleep", "infinity"); diff --git a/src/agents/sandbox/test-args.ts b/src/agents/sandbox/test-args.ts new file mode 100644 index 00000000000..342b22616a1 --- /dev/null +++ b/src/agents/sandbox/test-args.ts @@ -0,0 +1,15 @@ +export function findDockerArgsCall(calls: unknown[][], command: string): string[] | undefined { + return calls.find((call) => Array.isArray(call[0]) && call[0][0] === command)?.[0] as + | string[] + | undefined; +} + +export function collectDockerFlagValues(args: string[], flag: string): string[] { + const values: string[] = []; + for (let i = 0; i < args.length; i += 1) { + if (args[i] === flag && typeof args[i + 1] === "string") { + values.push(args[i + 1]); + } + } + return values; +} diff --git a/src/agents/sandbox/workspace-mounts.test.ts b/src/agents/sandbox/workspace-mounts.test.ts new file mode 100644 index 00000000000..0fe8c3897b3 --- /dev/null +++ b/src/agents/sandbox/workspace-mounts.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { appendWorkspaceMountArgs } from "./workspace-mounts.js"; + +describe("appendWorkspaceMountArgs", () => { + it.each([ + { access: "rw" as const, expected: "/tmp/workspace:/workspace" }, + { access: "ro" as const, expected: "/tmp/workspace:/workspace:ro" }, + { access: "none" as const, expected: "/tmp/workspace:/workspace:ro" }, + ])("sets main mount permissions for workspaceAccess=$access", ({ access, expected }) => { + const args: string[] = []; + appendWorkspaceMountArgs({ + args, + workspaceDir: "/tmp/workspace", + agentWorkspaceDir: "/tmp/agent-workspace", + workdir: "/workspace", + workspaceAccess: access, + }); + + expect(args).toContain(expected); + }); + + it("omits agent workspace mount when workspaceAccess is none", () => { + const args: string[] = []; + appendWorkspaceMountArgs({ + args, + workspaceDir: "/tmp/workspace", + agentWorkspaceDir: "/tmp/agent-workspace", + workdir: "/workspace", + workspaceAccess: "none", + }); + + const mounts = args.filter((arg) => arg.startsWith("/tmp/")); + expect(mounts).toEqual(["/tmp/workspace:/workspace:ro"]); + }); + + it("omits agent workspace mount when paths are identical", () => { + const args: string[] = []; + appendWorkspaceMountArgs({ + args, + workspaceDir: "/tmp/workspace", + agentWorkspaceDir: "/tmp/workspace", + workdir: "/workspace", + workspaceAccess: "rw", + }); + + const mounts = args.filter((arg) => arg.startsWith("/tmp/")); + expect(mounts).toEqual(["/tmp/workspace:/workspace"]); + }); +}); diff --git a/src/agents/sandbox/workspace-mounts.ts b/src/agents/sandbox/workspace-mounts.ts new file mode 100644 index 00000000000..ee7627eb1ad --- /dev/null +++ b/src/agents/sandbox/workspace-mounts.ts @@ -0,0 +1,28 @@ +import { SANDBOX_AGENT_WORKSPACE_MOUNT } from "./constants.js"; +import type { SandboxWorkspaceAccess } from "./types.js"; + +function mainWorkspaceMountSuffix(access: SandboxWorkspaceAccess): "" | ":ro" { + return access === "rw" ? "" : ":ro"; +} + +function agentWorkspaceMountSuffix(access: SandboxWorkspaceAccess): "" | ":ro" { + return access === "ro" ? ":ro" : ""; +} + +export function appendWorkspaceMountArgs(params: { + args: string[]; + workspaceDir: string; + agentWorkspaceDir: string; + workdir: string; + workspaceAccess: SandboxWorkspaceAccess; +}) { + const { args, workspaceDir, agentWorkspaceDir, workdir, workspaceAccess } = params; + + args.push("-v", `${workspaceDir}:${workdir}${mainWorkspaceMountSuffix(workspaceAccess)}`); + if (workspaceAccess !== "none" && workspaceDir !== agentWorkspaceDir) { + args.push( + "-v", + `${agentWorkspaceDir}:${SANDBOX_AGENT_WORKSPACE_MOUNT}${agentWorkspaceMountSuffix(workspaceAccess)}`, + ); + } +} diff --git a/src/auto-reply/reply/reply-inline-whitespace.test.ts b/src/auto-reply/reply/reply-inline-whitespace.test.ts new file mode 100644 index 00000000000..c9d2858b684 --- /dev/null +++ b/src/auto-reply/reply/reply-inline-whitespace.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from "vitest"; +import { collapseInlineHorizontalWhitespace } from "./reply-inline-whitespace.js"; + +describe("collapseInlineHorizontalWhitespace", () => { + it("collapses spaces and tabs but preserves newlines", () => { + const value = "hello\t\tworld\n next\tline"; + expect(collapseInlineHorizontalWhitespace(value)).toBe("hello world\n next line"); + }); +}); diff --git a/src/auto-reply/reply/reply-inline-whitespace.ts b/src/auto-reply/reply/reply-inline-whitespace.ts new file mode 100644 index 00000000000..c8b05c67272 --- /dev/null +++ b/src/auto-reply/reply/reply-inline-whitespace.ts @@ -0,0 +1,5 @@ +const INLINE_HORIZONTAL_WHITESPACE_RE = /[^\S\n]+/g; + +export function collapseInlineHorizontalWhitespace(value: string): string { + return value.replace(INLINE_HORIZONTAL_WHITESPACE_RE, " "); +} diff --git a/src/auto-reply/reply/reply-inline.ts b/src/auto-reply/reply/reply-inline.ts index 6fe84df394e..367c946eae4 100644 --- a/src/auto-reply/reply/reply-inline.ts +++ b/src/auto-reply/reply/reply-inline.ts @@ -1,3 +1,5 @@ +import { collapseInlineHorizontalWhitespace } from "./reply-inline-whitespace.js"; + const INLINE_SIMPLE_COMMAND_ALIASES = new Map([ ["/help", "/help"], ["/commands", "/commands"], @@ -24,10 +26,7 @@ export function extractInlineSimpleCommand(body?: string): { if (!command) { return null; } - const cleaned = body - .replace(match[0], " ") - .replace(/[^\S\n]+/g, " ") - .trim(); + const cleaned = collapseInlineHorizontalWhitespace(body.replace(match[0], " ")).trim(); return { command, cleaned }; } @@ -41,9 +40,6 @@ export function stripInlineStatus(body: string): { } // Use [^\S\n]+ instead of \s+ to only collapse horizontal whitespace, // preserving newlines so multi-line messages keep their paragraph structure. - const cleaned = trimmed - .replace(INLINE_STATUS_RE, " ") - .replace(/[^\S\n]+/g, " ") - .trim(); + const cleaned = collapseInlineHorizontalWhitespace(trimmed.replace(INLINE_STATUS_RE, " ")).trim(); return { cleaned, didStrip: cleaned !== trimmed }; } diff --git a/src/channels/plugins/actions/reaction-message-id.test.ts b/src/channels/plugins/actions/reaction-message-id.test.ts new file mode 100644 index 00000000000..290243ee988 --- /dev/null +++ b/src/channels/plugins/actions/reaction-message-id.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { resolveReactionMessageId } from "./reaction-message-id.js"; + +describe("resolveReactionMessageId", () => { + it("uses explicit messageId when present", () => { + const result = resolveReactionMessageId({ + args: { messageId: "456" }, + toolContext: { currentMessageId: "123" }, + }); + expect(result).toBe("456"); + }); + + it("accepts snake_case message_id alias", () => { + const result = resolveReactionMessageId({ args: { message_id: "789" } }); + expect(result).toBe("789"); + }); + + it("falls back to toolContext.currentMessageId", () => { + const result = resolveReactionMessageId({ + args: {}, + toolContext: { currentMessageId: "9001" }, + }); + expect(result).toBe("9001"); + }); +}); diff --git a/src/channels/plugins/actions/reaction-message-id.ts b/src/channels/plugins/actions/reaction-message-id.ts new file mode 100644 index 00000000000..d5c00578549 --- /dev/null +++ b/src/channels/plugins/actions/reaction-message-id.ts @@ -0,0 +1,12 @@ +import { readStringOrNumberParam } from "../../../agents/tools/common.js"; + +type ReactionToolContext = { + currentMessageId?: string | number; +}; + +export function resolveReactionMessageId(params: { + args: Record; + toolContext?: ReactionToolContext; +}): string | number | undefined { + return readStringOrNumberParam(params.args, "messageId") ?? params.toolContext?.currentMessageId; +} diff --git a/src/channels/plugins/actions/signal.ts b/src/channels/plugins/actions/signal.ts index ff5433b3895..c934a039f99 100644 --- a/src/channels/plugins/actions/signal.ts +++ b/src/channels/plugins/actions/signal.ts @@ -3,6 +3,7 @@ import { listEnabledSignalAccounts, resolveSignalAccount } from "../../../signal import { resolveSignalReactionLevel } from "../../../signal/reaction-level.js"; import { sendReactionSignal, removeReactionSignal } from "../../../signal/send-reactions.js"; import type { ChannelMessageActionAdapter, ChannelMessageActionName } from "../types.js"; +import { resolveReactionMessageId } from "./reaction-message-id.js"; const providerId = "signal"; const GROUP_PREFIX = "group:"; @@ -126,9 +127,8 @@ export const signalMessageActions: ChannelMessageActionAdapter = { throw new Error("recipient or group required"); } - const messageId = - readStringParam(params, "messageId") ?? - (toolContext?.currentMessageId != null ? String(toolContext.currentMessageId) : undefined); + const messageIdRaw = resolveReactionMessageId({ args: params, toolContext }); + const messageId = messageIdRaw != null ? String(messageIdRaw) : undefined; if (!messageId) { throw new Error( "messageId (timestamp) required. Provide messageId explicitly or react to the current inbound message.", diff --git a/src/channels/plugins/actions/telegram.ts b/src/channels/plugins/actions/telegram.ts index 537ea2fee3c..4f0f1a85c2d 100644 --- a/src/channels/plugins/actions/telegram.ts +++ b/src/channels/plugins/actions/telegram.ts @@ -13,6 +13,7 @@ import { } from "../../../telegram/accounts.js"; import { isTelegramInlineButtonsEnabled } from "../../../telegram/inline-buttons.js"; import type { ChannelMessageActionAdapter, ChannelMessageActionName } from "../types.js"; +import { resolveReactionMessageId } from "./reaction-message-id.js"; import { createUnionActionGate, listTokenSourcedAccounts } from "./shared.js"; const providerId = "telegram"; @@ -122,8 +123,7 @@ export const telegramMessageActions: ChannelMessageActionAdapter = { } if (action === "react") { - const messageId = - readStringOrNumberParam(params, "messageId") ?? toolContext?.currentMessageId; + const messageId = resolveReactionMessageId({ args: params, toolContext }); const emoji = readStringParam(params, "emoji", { allowEmpty: true }); const remove = typeof params.remove === "boolean" ? params.remove : undefined; return await handleTelegramAction( diff --git a/src/gateway/http-utils.request-context.test.ts b/src/gateway/http-utils.request-context.test.ts new file mode 100644 index 00000000000..21c7aeb6efc --- /dev/null +++ b/src/gateway/http-utils.request-context.test.ts @@ -0,0 +1,45 @@ +import type { IncomingMessage } from "node:http"; +import { describe, expect, it } from "vitest"; +import { resolveGatewayRequestContext } from "./http-utils.js"; + +function createReq(headers: Record = {}): IncomingMessage { + return { headers } as IncomingMessage; +} + +describe("resolveGatewayRequestContext", () => { + it("uses normalized x-openclaw-message-channel when enabled", () => { + const result = resolveGatewayRequestContext({ + req: createReq({ "x-openclaw-message-channel": " Custom-Channel " }), + model: "openclaw", + sessionPrefix: "openai", + defaultMessageChannel: "webchat", + useMessageChannelHeader: true, + }); + + expect(result.messageChannel).toBe("custom-channel"); + }); + + it("uses default messageChannel when header support is disabled", () => { + const result = resolveGatewayRequestContext({ + req: createReq({ "x-openclaw-message-channel": "custom-channel" }), + model: "openclaw", + sessionPrefix: "openresponses", + defaultMessageChannel: "webchat", + useMessageChannelHeader: false, + }); + + expect(result.messageChannel).toBe("webchat"); + }); + + it("includes session prefix and user in generated session key", () => { + const result = resolveGatewayRequestContext({ + req: createReq(), + model: "openclaw", + user: "alice", + sessionPrefix: "openresponses", + defaultMessageChannel: "webchat", + }); + + expect(result.sessionKey).toContain("openresponses-user:alice"); + }); +}); diff --git a/src/gateway/http-utils.ts b/src/gateway/http-utils.ts index fe183265f54..f3ffa8af7da 100644 --- a/src/gateway/http-utils.ts +++ b/src/gateway/http-utils.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import type { IncomingMessage } from "node:http"; import { buildAgentMainSessionKey, normalizeAgentId } from "../routing/session-key.js"; +import { normalizeMessageChannel } from "../utils/message-channel.js"; export function getHeader(req: IncomingMessage, name: string): string | undefined { const raw = req.headers[name.toLowerCase()]; @@ -77,3 +78,27 @@ export function resolveSessionKey(params: { const mainKey = user ? `${params.prefix}-user:${user}` : `${params.prefix}:${randomUUID()}`; return buildAgentMainSessionKey({ agentId: params.agentId, mainKey }); } + +export function resolveGatewayRequestContext(params: { + req: IncomingMessage; + model: string | undefined; + user?: string | undefined; + sessionPrefix: string; + defaultMessageChannel: string; + useMessageChannelHeader?: boolean; +}): { agentId: string; sessionKey: string; messageChannel: string } { + const agentId = resolveAgentIdForRequest({ req: params.req, model: params.model }); + const sessionKey = resolveSessionKey({ + req: params.req, + agentId, + user: params.user, + prefix: params.sessionPrefix, + }); + + const messageChannel = params.useMessageChannelHeader + ? (normalizeMessageChannel(getHeader(params.req, "x-openclaw-message-channel")) ?? + params.defaultMessageChannel) + : params.defaultMessageChannel; + + return { agentId, sessionKey, messageChannel }; +} diff --git a/src/gateway/openai-http.ts b/src/gateway/openai-http.ts index b78b9e0176f..7048b3d6d68 100644 --- a/src/gateway/openai-http.ts +++ b/src/gateway/openai-http.ts @@ -5,7 +5,6 @@ import { agentCommand } from "../commands/agent.js"; import { emitAgentEvent, onAgentEvent } from "../infra/agent-events.js"; import { logWarn } from "../logger.js"; import { defaultRuntime } from "../runtime.js"; -import { normalizeMessageChannel } from "../utils/message-channel.js"; import { resolveAssistantStreamDeltaText } from "./agent-event-assistant-text.js"; import { buildAgentMessageFromConversationEntries, @@ -15,7 +14,7 @@ import type { AuthRateLimiter } from "./auth-rate-limit.js"; import type { ResolvedGatewayAuth } from "./auth.js"; import { sendJson, setSseHeaders, writeDone } from "./http-common.js"; import { handleGatewayPostJsonEndpoint } from "./http-endpoint-helpers.js"; -import { getHeader, resolveAgentIdForRequest, resolveSessionKey } from "./http-utils.js"; +import { resolveGatewayRequestContext } from "./http-utils.js"; type OpenAiHttpOptions = { auth: ResolvedGatewayAuth; @@ -174,14 +173,6 @@ function buildAgentPrompt(messagesUnknown: unknown): { }; } -function resolveOpenAiSessionKey(params: { - req: IncomingMessage; - agentId: string; - user?: string | undefined; -}): string { - return resolveSessionKey({ ...params, prefix: "openai" }); -} - function coerceRequest(val: unknown): OpenAiChatCompletionRequest { if (!val || typeof val !== "object") { return {}; @@ -226,10 +217,14 @@ export async function handleOpenAiHttpRequest( const model = typeof payload.model === "string" ? payload.model : "openclaw"; const user = typeof payload.user === "string" ? payload.user : undefined; - const agentId = resolveAgentIdForRequest({ req, model }); - const sessionKey = resolveOpenAiSessionKey({ req, agentId, user }); - const messageChannel = - normalizeMessageChannel(getHeader(req, "x-openclaw-message-channel")) ?? "webchat"; + const { sessionKey, messageChannel } = resolveGatewayRequestContext({ + req, + model, + user, + sessionPrefix: "openai", + defaultMessageChannel: "webchat", + useMessageChannelHeader: true, + }); const prompt = buildAgentPrompt(payload.messages); if (!prompt.message) { sendJson(res, 400, { diff --git a/src/gateway/openresponses-http.test.ts b/src/gateway/openresponses-http.test.ts index ba2af49e954..ac8bf0efb31 100644 --- a/src/gateway/openresponses-http.test.ts +++ b/src/gateway/openresponses-http.test.ts @@ -163,6 +163,9 @@ describe("OpenResponses HTTP API (e2e)", () => { expect((optsHeader as { sessionKey?: string } | undefined)?.sessionKey ?? "").toMatch( /^agent:beta:/, ); + expect((optsHeader as { messageChannel?: string } | undefined)?.messageChannel).toBe( + "webchat", + ); await ensureResponseConsumed(resHeader); mockAgentOnce([{ text: "hello" }]); @@ -174,6 +177,19 @@ describe("OpenResponses HTTP API (e2e)", () => { ); await ensureResponseConsumed(resModel); + mockAgentOnce([{ text: "hello" }]); + const resChannelHeader = await postResponses( + port, + { model: "openclaw", input: "hi" }, + { "x-openclaw-message-channel": "custom-client-channel" }, + ); + expect(resChannelHeader.status).toBe(200); + const optsChannelHeader = (agentCommand.mock.calls[0] as unknown[] | undefined)?.[0]; + expect((optsChannelHeader as { messageChannel?: string } | undefined)?.messageChannel).toBe( + "webchat", + ); + await ensureResponseConsumed(resChannelHeader); + mockAgentOnce([{ text: "hello" }]); const resUser = await postResponses(port, { user: "alice", diff --git a/src/gateway/openresponses-http.ts b/src/gateway/openresponses-http.ts index ab1a4a5e0d0..70dc7a719a9 100644 --- a/src/gateway/openresponses-http.ts +++ b/src/gateway/openresponses-http.ts @@ -34,7 +34,7 @@ import type { AuthRateLimiter } from "./auth-rate-limit.js"; import type { ResolvedGatewayAuth } from "./auth.js"; import { sendJson, setSseHeaders, writeDone } from "./http-common.js"; import { handleGatewayPostJsonEndpoint } from "./http-endpoint-helpers.js"; -import { resolveAgentIdForRequest, resolveSessionKey } from "./http-utils.js"; +import { resolveGatewayRequestContext } from "./http-utils.js"; import { CreateResponseBodySchema, type CreateResponseBody, @@ -151,14 +151,6 @@ function applyToolChoice(params: { export { buildAgentPrompt } from "./openresponses-prompt.js"; -function resolveOpenResponsesSessionKey(params: { - req: IncomingMessage; - agentId: string; - user?: string | undefined; -}): string { - return resolveSessionKey({ ...params, prefix: "openresponses" }); -} - function createEmptyUsage(): Usage { return { input_tokens: 0, output_tokens: 0, total_tokens: 0 }; } @@ -241,6 +233,7 @@ async function runResponsesAgentCommand(params: { streamParams: { maxTokens: number } | undefined; sessionKey: string; runId: string; + messageChannel: string; deps: ReturnType; }) { return agentCommand( @@ -253,7 +246,7 @@ async function runResponsesAgentCommand(params: { sessionKey: params.sessionKey, runId: params.runId, deliver: false, - messageChannel: "webchat", + messageChannel: params.messageChannel, bestEffortDeliver: false, }, defaultRuntime, @@ -412,8 +405,14 @@ export async function handleOpenResponsesHttpRequest( }); return true; } - const agentId = resolveAgentIdForRequest({ req, model }); - const sessionKey = resolveOpenResponsesSessionKey({ req, agentId, user }); + const { sessionKey, messageChannel } = resolveGatewayRequestContext({ + req, + model, + user, + sessionPrefix: "openresponses", + defaultMessageChannel: "webchat", + useMessageChannelHeader: false, + }); // Build prompt from input const prompt = buildAgentPrompt(payload.input); @@ -459,6 +458,7 @@ export async function handleOpenResponsesHttpRequest( streamParams, sessionKey, runId: responseId, + messageChannel, deps, }); @@ -691,6 +691,7 @@ export async function handleOpenResponsesHttpRequest( streamParams, sessionKey, runId: responseId, + messageChannel, deps, }); diff --git a/src/secrets/provider-resolvers.ts b/src/secrets/provider-resolvers.ts new file mode 100644 index 00000000000..0c4bb835c15 --- /dev/null +++ b/src/secrets/provider-resolvers.ts @@ -0,0 +1,569 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { + ExecSecretProviderConfig, + FileSecretProviderConfig, + SecretProviderConfig, + SecretRef, +} from "../config/types.secrets.js"; +import { inspectPathPermissions, safeStat } from "../security/audit-fs.js"; +import { isPathInside } from "../security/scan-paths.js"; +import { resolveUserPath } from "../utils.js"; +import { readJsonPointer } from "./json-pointer.js"; +import { SINGLE_VALUE_FILE_REF_ID } from "./ref-contract.js"; +import { isNonEmptyString, isRecord, normalizePositiveInt } from "./shared.js"; + +const DEFAULT_FILE_MAX_BYTES = 1024 * 1024; +const DEFAULT_FILE_TIMEOUT_MS = 5_000; +const DEFAULT_EXEC_TIMEOUT_MS = 5_000; +const DEFAULT_EXEC_MAX_OUTPUT_BYTES = 1024 * 1024; +const WINDOWS_ABS_PATH_PATTERN = /^[A-Za-z]:[\\/]/; +const WINDOWS_UNC_PATH_PATTERN = /^\\\\[^\\]+\\[^\\]+/; + +export type SecretRefResolveCache = { + resolvedByRefKey?: Map>; + filePayloadByProvider?: Map>; +}; + +export type ResolutionLimits = { + maxProviderConcurrency: number; + maxRefsPerProvider: number; + maxBatchBytes: number; +}; + +export type ProviderResolutionOutput = Map; + +function isAbsolutePathname(value: string): boolean { + return ( + path.isAbsolute(value) || + WINDOWS_ABS_PATH_PATTERN.test(value) || + WINDOWS_UNC_PATH_PATTERN.test(value) + ); +} + +async function assertSecurePath(params: { + targetPath: string; + label: string; + trustedDirs?: string[]; + allowInsecurePath?: boolean; + allowReadableByOthers?: boolean; + allowSymlinkPath?: boolean; +}): Promise { + if (!isAbsolutePathname(params.targetPath)) { + throw new Error(`${params.label} must be an absolute path.`); + } + + let effectivePath = params.targetPath; + let stat = await safeStat(effectivePath); + if (!stat.ok) { + throw new Error(`${params.label} is not readable: ${effectivePath}`); + } + if (stat.isDir) { + throw new Error(`${params.label} must be a file: ${effectivePath}`); + } + if (stat.isSymlink) { + if (!params.allowSymlinkPath) { + throw new Error(`${params.label} must not be a symlink: ${effectivePath}`); + } + try { + effectivePath = await fs.realpath(effectivePath); + } catch { + throw new Error(`${params.label} symlink target is not readable: ${params.targetPath}`); + } + if (!isAbsolutePathname(effectivePath)) { + throw new Error(`${params.label} resolved symlink target must be an absolute path.`); + } + stat = await safeStat(effectivePath); + if (!stat.ok) { + throw new Error(`${params.label} is not readable: ${effectivePath}`); + } + if (stat.isDir) { + throw new Error(`${params.label} must be a file: ${effectivePath}`); + } + if (stat.isSymlink) { + throw new Error(`${params.label} symlink target must not be a symlink: ${effectivePath}`); + } + } + + if (params.trustedDirs && params.trustedDirs.length > 0) { + const trusted = params.trustedDirs.map((entry) => resolveUserPath(entry)); + const inTrustedDir = trusted.some((dir) => isPathInside(dir, effectivePath)); + if (!inTrustedDir) { + throw new Error(`${params.label} is outside trustedDirs: ${effectivePath}`); + } + } + if (params.allowInsecurePath) { + return effectivePath; + } + + const perms = await inspectPathPermissions(effectivePath); + if (!perms.ok) { + throw new Error(`${params.label} permissions could not be verified: ${effectivePath}`); + } + const writableByOthers = perms.worldWritable || perms.groupWritable; + const readableByOthers = perms.worldReadable || perms.groupReadable; + if (writableByOthers || (!params.allowReadableByOthers && readableByOthers)) { + throw new Error(`${params.label} permissions are too open: ${effectivePath}`); + } + + if (process.platform === "win32" && perms.source === "unknown") { + throw new Error( + `${params.label} ACL verification unavailable on Windows for ${effectivePath}.`, + ); + } + + if (process.platform !== "win32" && typeof process.getuid === "function" && stat.uid != null) { + const uid = process.getuid(); + if (stat.uid !== uid) { + throw new Error( + `${params.label} must be owned by the current user (uid=${uid}): ${effectivePath}`, + ); + } + } + return effectivePath; +} + +async function readFileProviderPayload(params: { + providerName: string; + providerConfig: FileSecretProviderConfig; + cache?: SecretRefResolveCache; +}): Promise { + const cacheKey = params.providerName; + const cache = params.cache; + if (cache?.filePayloadByProvider?.has(cacheKey)) { + return await (cache.filePayloadByProvider.get(cacheKey) as Promise); + } + + const filePath = resolveUserPath(params.providerConfig.path); + const readPromise = (async () => { + const secureFilePath = await assertSecurePath({ + targetPath: filePath, + label: `secrets.providers.${params.providerName}.path`, + }); + const timeoutMs = normalizePositiveInt( + params.providerConfig.timeoutMs, + DEFAULT_FILE_TIMEOUT_MS, + ); + const maxBytes = normalizePositiveInt(params.providerConfig.maxBytes, DEFAULT_FILE_MAX_BYTES); + const abortController = new AbortController(); + const timeoutErrorMessage = `File provider "${params.providerName}" timed out after ${timeoutMs}ms.`; + let timeoutHandle: NodeJS.Timeout | null = null; + const timeoutPromise = new Promise((_resolve, reject) => { + timeoutHandle = setTimeout(() => { + abortController.abort(); + reject(new Error(timeoutErrorMessage)); + }, timeoutMs); + }); + try { + const payload = await Promise.race([ + fs.readFile(secureFilePath, { signal: abortController.signal }), + timeoutPromise, + ]); + if (payload.byteLength > maxBytes) { + throw new Error(`File provider "${params.providerName}" exceeded maxBytes (${maxBytes}).`); + } + const text = payload.toString("utf8"); + if (params.providerConfig.mode === "singleValue") { + return text.replace(/\r?\n$/, ""); + } + const parsed = JSON.parse(text) as unknown; + if (!isRecord(parsed)) { + throw new Error(`File provider "${params.providerName}" payload is not a JSON object.`); + } + return parsed; + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + throw new Error(timeoutErrorMessage, { cause: error }); + } + throw error; + } finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + } + })(); + + if (cache) { + cache.filePayloadByProvider ??= new Map(); + cache.filePayloadByProvider.set(cacheKey, readPromise); + } + return await readPromise; +} + +async function resolveEnvRefs(params: { + refs: SecretRef[]; + providerName: string; + providerConfig: Extract; + env: NodeJS.ProcessEnv; +}): Promise { + const resolved = new Map(); + const allowlist = params.providerConfig.allowlist + ? new Set(params.providerConfig.allowlist) + : null; + for (const ref of params.refs) { + if (allowlist && !allowlist.has(ref.id)) { + throw new Error( + `Environment variable "${ref.id}" is not allowlisted in secrets.providers.${params.providerName}.allowlist.`, + ); + } + const envValue = params.env[ref.id] ?? process.env[ref.id]; + if (!isNonEmptyString(envValue)) { + throw new Error(`Environment variable "${ref.id}" is missing or empty.`); + } + resolved.set(ref.id, envValue); + } + return resolved; +} + +async function resolveFileRefs(params: { + refs: SecretRef[]; + providerName: string; + providerConfig: FileSecretProviderConfig; + cache?: SecretRefResolveCache; +}): Promise { + const payload = await readFileProviderPayload({ + providerName: params.providerName, + providerConfig: params.providerConfig, + cache: params.cache, + }); + const mode = params.providerConfig.mode ?? "json"; + const resolved = new Map(); + if (mode === "singleValue") { + for (const ref of params.refs) { + if (ref.id !== SINGLE_VALUE_FILE_REF_ID) { + throw new Error( + `singleValue file provider "${params.providerName}" expects ref id "${SINGLE_VALUE_FILE_REF_ID}".`, + ); + } + resolved.set(ref.id, payload); + } + return resolved; + } + for (const ref of params.refs) { + resolved.set(ref.id, readJsonPointer(payload, ref.id, { onMissing: "throw" })); + } + return resolved; +} + +type ExecRunResult = { + stdout: string; + stderr: string; + code: number | null; + signal: NodeJS.Signals | null; + termination: "exit" | "timeout" | "no-output-timeout"; +}; + +function isIgnorableStdinWriteError(error: unknown): boolean { + if (typeof error !== "object" || error === null || !("code" in error)) { + return false; + } + const code = String(error.code); + return code === "EPIPE" || code === "ERR_STREAM_DESTROYED"; +} + +async function runExecResolver(params: { + command: string; + args: string[]; + cwd: string; + env: NodeJS.ProcessEnv; + input: string; + timeoutMs: number; + noOutputTimeoutMs: number; + maxOutputBytes: number; +}): Promise { + return await new Promise((resolve, reject) => { + const child = spawn(params.command, params.args, { + cwd: params.cwd, + env: params.env, + stdio: ["pipe", "pipe", "pipe"], + shell: false, + windowsHide: true, + }); + + let settled = false; + let stdout = ""; + let stderr = ""; + let timedOut = false; + let noOutputTimedOut = false; + let outputBytes = 0; + let noOutputTimer: NodeJS.Timeout | null = null; + const timeoutTimer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, params.timeoutMs); + + const clearTimers = () => { + clearTimeout(timeoutTimer); + if (noOutputTimer) { + clearTimeout(noOutputTimer); + noOutputTimer = null; + } + }; + + const armNoOutputTimer = () => { + if (noOutputTimer) { + clearTimeout(noOutputTimer); + } + noOutputTimer = setTimeout(() => { + noOutputTimedOut = true; + child.kill("SIGKILL"); + }, params.noOutputTimeoutMs); + }; + + const append = (chunk: Buffer | string, target: "stdout" | "stderr") => { + const text = typeof chunk === "string" ? chunk : chunk.toString("utf8"); + outputBytes += Buffer.byteLength(text, "utf8"); + if (outputBytes > params.maxOutputBytes) { + child.kill("SIGKILL"); + if (!settled) { + settled = true; + clearTimers(); + reject( + new Error(`Exec provider output exceeded maxOutputBytes (${params.maxOutputBytes}).`), + ); + } + return; + } + if (target === "stdout") { + stdout += text; + } else { + stderr += text; + } + armNoOutputTimer(); + }; + + armNoOutputTimer(); + child.on("error", (error) => { + if (settled) { + return; + } + settled = true; + clearTimers(); + reject(error); + }); + child.stdout?.on("data", (chunk) => append(chunk, "stdout")); + child.stderr?.on("data", (chunk) => append(chunk, "stderr")); + child.on("close", (code, signal) => { + if (settled) { + return; + } + settled = true; + clearTimers(); + resolve({ + stdout, + stderr, + code, + signal, + termination: noOutputTimedOut ? "no-output-timeout" : timedOut ? "timeout" : "exit", + }); + }); + + const handleStdinError = (error: unknown) => { + if (isIgnorableStdinWriteError(error) || settled) { + return; + } + settled = true; + clearTimers(); + reject(error instanceof Error ? error : new Error(String(error))); + }; + child.stdin?.on("error", handleStdinError); + try { + child.stdin?.end(params.input); + } catch (error) { + handleStdinError(error); + } + }); +} + +function parseExecValues(params: { + providerName: string; + ids: string[]; + stdout: string; + jsonOnly: boolean; +}): Record { + const trimmed = params.stdout.trim(); + if (!trimmed) { + throw new Error(`Exec provider "${params.providerName}" returned empty stdout.`); + } + + let parsed: unknown; + if (!params.jsonOnly && params.ids.length === 1) { + try { + parsed = JSON.parse(trimmed) as unknown; + } catch { + return { [params.ids[0]]: trimmed }; + } + } else { + try { + parsed = JSON.parse(trimmed) as unknown; + } catch { + throw new Error(`Exec provider "${params.providerName}" returned invalid JSON.`); + } + } + + if (!isRecord(parsed)) { + if (!params.jsonOnly && params.ids.length === 1 && typeof parsed === "string") { + return { [params.ids[0]]: parsed }; + } + throw new Error(`Exec provider "${params.providerName}" response must be an object.`); + } + if (parsed.protocolVersion !== 1) { + throw new Error(`Exec provider "${params.providerName}" protocolVersion must be 1.`); + } + const responseValues = parsed.values; + if (!isRecord(responseValues)) { + throw new Error(`Exec provider "${params.providerName}" response missing "values".`); + } + const responseErrors = isRecord(parsed.errors) ? parsed.errors : null; + const out: Record = {}; + for (const id of params.ids) { + if (responseErrors && id in responseErrors) { + const entry = responseErrors[id]; + if (isRecord(entry) && typeof entry.message === "string" && entry.message.trim()) { + throw new Error( + `Exec provider "${params.providerName}" failed for id "${id}" (${entry.message.trim()}).`, + ); + } + throw new Error(`Exec provider "${params.providerName}" failed for id "${id}".`); + } + if (!(id in responseValues)) { + throw new Error(`Exec provider "${params.providerName}" response missing id "${id}".`); + } + out[id] = responseValues[id]; + } + return out; +} + +async function resolveExecRefs(params: { + refs: SecretRef[]; + providerName: string; + providerConfig: ExecSecretProviderConfig; + env: NodeJS.ProcessEnv; + limits: ResolutionLimits; +}): Promise { + const ids = [...new Set(params.refs.map((ref) => ref.id))]; + if (ids.length > params.limits.maxRefsPerProvider) { + throw new Error( + `Exec provider "${params.providerName}" exceeded maxRefsPerProvider (${params.limits.maxRefsPerProvider}).`, + ); + } + + const commandPath = resolveUserPath(params.providerConfig.command); + const secureCommandPath = await assertSecurePath({ + targetPath: commandPath, + label: `secrets.providers.${params.providerName}.command`, + trustedDirs: params.providerConfig.trustedDirs, + allowInsecurePath: params.providerConfig.allowInsecurePath, + allowReadableByOthers: true, + allowSymlinkPath: params.providerConfig.allowSymlinkCommand, + }); + + const requestPayload = { + protocolVersion: 1, + provider: params.providerName, + ids, + }; + const input = JSON.stringify(requestPayload); + if (Buffer.byteLength(input, "utf8") > params.limits.maxBatchBytes) { + throw new Error( + `Exec provider "${params.providerName}" request exceeded maxBatchBytes (${params.limits.maxBatchBytes}).`, + ); + } + + const childEnv: NodeJS.ProcessEnv = {}; + for (const key of params.providerConfig.passEnv ?? []) { + const value = params.env[key] ?? process.env[key]; + if (value !== undefined) { + childEnv[key] = value; + } + } + for (const [key, value] of Object.entries(params.providerConfig.env ?? {})) { + childEnv[key] = value; + } + + const timeoutMs = normalizePositiveInt(params.providerConfig.timeoutMs, DEFAULT_EXEC_TIMEOUT_MS); + const noOutputTimeoutMs = normalizePositiveInt( + params.providerConfig.noOutputTimeoutMs, + timeoutMs, + ); + const maxOutputBytes = normalizePositiveInt( + params.providerConfig.maxOutputBytes, + DEFAULT_EXEC_MAX_OUTPUT_BYTES, + ); + const jsonOnly = params.providerConfig.jsonOnly ?? true; + + const result = await runExecResolver({ + command: secureCommandPath, + args: params.providerConfig.args ?? [], + cwd: path.dirname(secureCommandPath), + env: childEnv, + input, + timeoutMs, + noOutputTimeoutMs, + maxOutputBytes, + }); + if (result.termination === "timeout") { + throw new Error(`Exec provider "${params.providerName}" timed out after ${timeoutMs}ms.`); + } + if (result.termination === "no-output-timeout") { + throw new Error( + `Exec provider "${params.providerName}" produced no output for ${noOutputTimeoutMs}ms.`, + ); + } + if (result.code !== 0) { + throw new Error( + `Exec provider "${params.providerName}" exited with code ${String(result.code)}.`, + ); + } + + const values = parseExecValues({ + providerName: params.providerName, + ids, + stdout: result.stdout, + jsonOnly, + }); + const resolved = new Map(); + for (const id of ids) { + resolved.set(id, values[id]); + } + return resolved; +} + +export async function resolveProviderRefs(params: { + refs: SecretRef[]; + providerName: string; + providerConfig: SecretProviderConfig; + env: NodeJS.ProcessEnv; + cache?: SecretRefResolveCache; + limits: ResolutionLimits; +}): Promise { + if (params.providerConfig.source === "env") { + return await resolveEnvRefs({ + refs: params.refs, + providerName: params.providerName, + providerConfig: params.providerConfig, + env: params.env, + }); + } + if (params.providerConfig.source === "file") { + return await resolveFileRefs({ + refs: params.refs, + providerName: params.providerName, + providerConfig: params.providerConfig, + cache: params.cache, + }); + } + if (params.providerConfig.source === "exec") { + return await resolveExecRefs({ + refs: params.refs, + providerName: params.providerName, + providerConfig: params.providerConfig, + env: params.env, + limits: params.limits, + }); + } + throw new Error( + `Unsupported secret provider source "${String((params.providerConfig as { source?: unknown }).source)}".`, + ); +} diff --git a/src/secrets/resolve.ts b/src/secrets/resolve.ts index fc34d48c9a0..eb5311cde2b 100644 --- a/src/secrets/resolve.ts +++ b/src/secrets/resolve.ts @@ -1,40 +1,18 @@ -import { spawn } from "node:child_process"; -import fs from "node:fs/promises"; -import path from "node:path"; import type { OpenClawConfig } from "../config/config.js"; -import type { - ExecSecretProviderConfig, - FileSecretProviderConfig, - SecretProviderConfig, - SecretRef, - SecretRefSource, -} from "../config/types.secrets.js"; -import { inspectPathPermissions, safeStat } from "../security/audit-fs.js"; -import { isPathInside } from "../security/scan-paths.js"; -import { resolveUserPath } from "../utils.js"; +import type { SecretProviderConfig, SecretRef, SecretRefSource } from "../config/types.secrets.js"; import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js"; -import { readJsonPointer } from "./json-pointer.js"; import { - SINGLE_VALUE_FILE_REF_ID, - resolveDefaultSecretProviderAlias, - secretRefKey, -} from "./ref-contract.js"; -import { isNonEmptyString, isRecord, normalizePositiveInt } from "./shared.js"; + type ProviderResolutionOutput, + type ResolutionLimits, + resolveProviderRefs, + type SecretRefResolveCache, +} from "./provider-resolvers.js"; +import { resolveDefaultSecretProviderAlias, secretRefKey } from "./ref-contract.js"; +import { isNonEmptyString, normalizePositiveInt } from "./shared.js"; const DEFAULT_PROVIDER_CONCURRENCY = 4; const DEFAULT_MAX_REFS_PER_PROVIDER = 512; const DEFAULT_MAX_BATCH_BYTES = 256 * 1024; -const DEFAULT_FILE_MAX_BYTES = 1024 * 1024; -const DEFAULT_FILE_TIMEOUT_MS = 5_000; -const DEFAULT_EXEC_TIMEOUT_MS = 5_000; -const DEFAULT_EXEC_MAX_OUTPUT_BYTES = 1024 * 1024; -const WINDOWS_ABS_PATH_PATTERN = /^[A-Za-z]:[\\/]/; -const WINDOWS_UNC_PATH_PATTERN = /^\\\\[^\\]+\\[^\\]+/; - -export type SecretRefResolveCache = { - resolvedByRefKey?: Map>; - filePayloadByProvider?: Map>; -}; type ResolveSecretRefOptions = { config: OpenClawConfig; @@ -42,22 +20,6 @@ type ResolveSecretRefOptions = { cache?: SecretRefResolveCache; }; -type ResolutionLimits = { - maxProviderConcurrency: number; - maxRefsPerProvider: number; - maxBatchBytes: number; -}; - -type ProviderResolutionOutput = Map; - -function isAbsolutePathname(value: string): boolean { - return ( - path.isAbsolute(value) || - WINDOWS_ABS_PATH_PATTERN.test(value) || - WINDOWS_UNC_PATH_PATTERN.test(value) - ); -} - function resolveResolutionLimits(config: OpenClawConfig): ResolutionLimits { const resolution = config.secrets?.resolution; return { @@ -95,532 +57,6 @@ function resolveConfiguredProvider(ref: SecretRef, config: OpenClawConfig): Secr return providerConfig; } -async function assertSecurePath(params: { - targetPath: string; - label: string; - trustedDirs?: string[]; - allowInsecurePath?: boolean; - allowReadableByOthers?: boolean; - allowSymlinkPath?: boolean; -}): Promise { - if (!isAbsolutePathname(params.targetPath)) { - throw new Error(`${params.label} must be an absolute path.`); - } - - let effectivePath = params.targetPath; - let stat = await safeStat(effectivePath); - if (!stat.ok) { - throw new Error(`${params.label} is not readable: ${effectivePath}`); - } - if (stat.isDir) { - throw new Error(`${params.label} must be a file: ${effectivePath}`); - } - if (stat.isSymlink) { - if (!params.allowSymlinkPath) { - throw new Error(`${params.label} must not be a symlink: ${effectivePath}`); - } - try { - effectivePath = await fs.realpath(effectivePath); - } catch { - throw new Error(`${params.label} symlink target is not readable: ${params.targetPath}`); - } - if (!isAbsolutePathname(effectivePath)) { - throw new Error(`${params.label} resolved symlink target must be an absolute path.`); - } - stat = await safeStat(effectivePath); - if (!stat.ok) { - throw new Error(`${params.label} is not readable: ${effectivePath}`); - } - if (stat.isDir) { - throw new Error(`${params.label} must be a file: ${effectivePath}`); - } - if (stat.isSymlink) { - throw new Error(`${params.label} symlink target must not be a symlink: ${effectivePath}`); - } - } - - if (params.trustedDirs && params.trustedDirs.length > 0) { - const trusted = params.trustedDirs.map((entry) => resolveUserPath(entry)); - const inTrustedDir = trusted.some((dir) => isPathInside(dir, effectivePath)); - if (!inTrustedDir) { - throw new Error(`${params.label} is outside trustedDirs: ${effectivePath}`); - } - } - if (params.allowInsecurePath) { - return effectivePath; - } - - const perms = await inspectPathPermissions(effectivePath); - if (!perms.ok) { - throw new Error(`${params.label} permissions could not be verified: ${effectivePath}`); - } - const writableByOthers = perms.worldWritable || perms.groupWritable; - const readableByOthers = perms.worldReadable || perms.groupReadable; - if (writableByOthers || (!params.allowReadableByOthers && readableByOthers)) { - throw new Error(`${params.label} permissions are too open: ${effectivePath}`); - } - - if (process.platform === "win32" && perms.source === "unknown") { - throw new Error( - `${params.label} ACL verification unavailable on Windows for ${effectivePath}.`, - ); - } - - if (process.platform !== "win32" && typeof process.getuid === "function" && stat.uid != null) { - const uid = process.getuid(); - if (stat.uid !== uid) { - throw new Error( - `${params.label} must be owned by the current user (uid=${uid}): ${effectivePath}`, - ); - } - } - return effectivePath; -} - -async function readFileProviderPayload(params: { - providerName: string; - providerConfig: FileSecretProviderConfig; - cache?: SecretRefResolveCache; -}): Promise { - const cacheKey = params.providerName; - const cache = params.cache; - if (cache?.filePayloadByProvider?.has(cacheKey)) { - return await (cache.filePayloadByProvider.get(cacheKey) as Promise); - } - - const filePath = resolveUserPath(params.providerConfig.path); - const readPromise = (async () => { - const secureFilePath = await assertSecurePath({ - targetPath: filePath, - label: `secrets.providers.${params.providerName}.path`, - }); - const timeoutMs = normalizePositiveInt( - params.providerConfig.timeoutMs, - DEFAULT_FILE_TIMEOUT_MS, - ); - const maxBytes = normalizePositiveInt(params.providerConfig.maxBytes, DEFAULT_FILE_MAX_BYTES); - const abortController = new AbortController(); - const timeoutErrorMessage = `File provider "${params.providerName}" timed out after ${timeoutMs}ms.`; - let timeoutHandle: NodeJS.Timeout | null = null; - const timeoutPromise = new Promise((_resolve, reject) => { - timeoutHandle = setTimeout(() => { - abortController.abort(); - reject(new Error(timeoutErrorMessage)); - }, timeoutMs); - }); - try { - const payload = await Promise.race([ - fs.readFile(secureFilePath, { signal: abortController.signal }), - timeoutPromise, - ]); - if (payload.byteLength > maxBytes) { - throw new Error(`File provider "${params.providerName}" exceeded maxBytes (${maxBytes}).`); - } - const text = payload.toString("utf8"); - if (params.providerConfig.mode === "singleValue") { - return text.replace(/\r?\n$/, ""); - } - const parsed = JSON.parse(text) as unknown; - if (!isRecord(parsed)) { - throw new Error(`File provider "${params.providerName}" payload is not a JSON object.`); - } - return parsed; - } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - throw new Error(timeoutErrorMessage, { cause: error }); - } - throw error; - } finally { - if (timeoutHandle) { - clearTimeout(timeoutHandle); - } - } - })(); - - if (cache) { - cache.filePayloadByProvider ??= new Map(); - cache.filePayloadByProvider.set(cacheKey, readPromise); - } - return await readPromise; -} - -async function resolveEnvRefs(params: { - refs: SecretRef[]; - providerName: string; - providerConfig: Extract; - env: NodeJS.ProcessEnv; -}): Promise { - const resolved = new Map(); - const allowlist = params.providerConfig.allowlist - ? new Set(params.providerConfig.allowlist) - : null; - for (const ref of params.refs) { - if (allowlist && !allowlist.has(ref.id)) { - throw new Error( - `Environment variable "${ref.id}" is not allowlisted in secrets.providers.${params.providerName}.allowlist.`, - ); - } - const envValue = params.env[ref.id] ?? process.env[ref.id]; - if (!isNonEmptyString(envValue)) { - throw new Error(`Environment variable "${ref.id}" is missing or empty.`); - } - resolved.set(ref.id, envValue); - } - return resolved; -} - -async function resolveFileRefs(params: { - refs: SecretRef[]; - providerName: string; - providerConfig: FileSecretProviderConfig; - cache?: SecretRefResolveCache; -}): Promise { - const payload = await readFileProviderPayload({ - providerName: params.providerName, - providerConfig: params.providerConfig, - cache: params.cache, - }); - const mode = params.providerConfig.mode ?? "json"; - const resolved = new Map(); - if (mode === "singleValue") { - for (const ref of params.refs) { - if (ref.id !== SINGLE_VALUE_FILE_REF_ID) { - throw new Error( - `singleValue file provider "${params.providerName}" expects ref id "${SINGLE_VALUE_FILE_REF_ID}".`, - ); - } - resolved.set(ref.id, payload); - } - return resolved; - } - for (const ref of params.refs) { - resolved.set(ref.id, readJsonPointer(payload, ref.id, { onMissing: "throw" })); - } - return resolved; -} - -type ExecRunResult = { - stdout: string; - stderr: string; - code: number | null; - signal: NodeJS.Signals | null; - termination: "exit" | "timeout" | "no-output-timeout"; -}; - -function isIgnorableStdinWriteError(error: unknown): boolean { - if (typeof error !== "object" || error === null || !("code" in error)) { - return false; - } - const code = String(error.code); - return code === "EPIPE" || code === "ERR_STREAM_DESTROYED"; -} - -async function runExecResolver(params: { - command: string; - args: string[]; - cwd: string; - env: NodeJS.ProcessEnv; - input: string; - timeoutMs: number; - noOutputTimeoutMs: number; - maxOutputBytes: number; -}): Promise { - return await new Promise((resolve, reject) => { - const child = spawn(params.command, params.args, { - cwd: params.cwd, - env: params.env, - stdio: ["pipe", "pipe", "pipe"], - shell: false, - windowsHide: true, - }); - - let settled = false; - let stdout = ""; - let stderr = ""; - let timedOut = false; - let noOutputTimedOut = false; - let outputBytes = 0; - let noOutputTimer: NodeJS.Timeout | null = null; - const timeoutTimer = setTimeout(() => { - timedOut = true; - child.kill("SIGKILL"); - }, params.timeoutMs); - - const clearTimers = () => { - clearTimeout(timeoutTimer); - if (noOutputTimer) { - clearTimeout(noOutputTimer); - noOutputTimer = null; - } - }; - - const armNoOutputTimer = () => { - if (noOutputTimer) { - clearTimeout(noOutputTimer); - } - noOutputTimer = setTimeout(() => { - noOutputTimedOut = true; - child.kill("SIGKILL"); - }, params.noOutputTimeoutMs); - }; - - const append = (chunk: Buffer | string, target: "stdout" | "stderr") => { - const text = typeof chunk === "string" ? chunk : chunk.toString("utf8"); - outputBytes += Buffer.byteLength(text, "utf8"); - if (outputBytes > params.maxOutputBytes) { - child.kill("SIGKILL"); - if (!settled) { - settled = true; - clearTimers(); - reject( - new Error(`Exec provider output exceeded maxOutputBytes (${params.maxOutputBytes}).`), - ); - } - return; - } - if (target === "stdout") { - stdout += text; - } else { - stderr += text; - } - armNoOutputTimer(); - }; - - armNoOutputTimer(); - child.on("error", (error) => { - if (settled) { - return; - } - settled = true; - clearTimers(); - reject(error); - }); - child.stdout?.on("data", (chunk) => append(chunk, "stdout")); - child.stderr?.on("data", (chunk) => append(chunk, "stderr")); - child.on("close", (code, signal) => { - if (settled) { - return; - } - settled = true; - clearTimers(); - resolve({ - stdout, - stderr, - code, - signal, - termination: noOutputTimedOut ? "no-output-timeout" : timedOut ? "timeout" : "exit", - }); - }); - - const handleStdinError = (error: unknown) => { - if (isIgnorableStdinWriteError(error) || settled) { - return; - } - settled = true; - clearTimers(); - reject(error instanceof Error ? error : new Error(String(error))); - }; - child.stdin?.on("error", handleStdinError); - try { - child.stdin?.end(params.input); - } catch (error) { - handleStdinError(error); - } - }); -} - -function parseExecValues(params: { - providerName: string; - ids: string[]; - stdout: string; - jsonOnly: boolean; -}): Record { - const trimmed = params.stdout.trim(); - if (!trimmed) { - throw new Error(`Exec provider "${params.providerName}" returned empty stdout.`); - } - - let parsed: unknown; - if (!params.jsonOnly && params.ids.length === 1) { - try { - parsed = JSON.parse(trimmed) as unknown; - } catch { - return { [params.ids[0]]: trimmed }; - } - } else { - try { - parsed = JSON.parse(trimmed) as unknown; - } catch { - throw new Error(`Exec provider "${params.providerName}" returned invalid JSON.`); - } - } - - if (!isRecord(parsed)) { - if (!params.jsonOnly && params.ids.length === 1 && typeof parsed === "string") { - return { [params.ids[0]]: parsed }; - } - throw new Error(`Exec provider "${params.providerName}" response must be an object.`); - } - if (parsed.protocolVersion !== 1) { - throw new Error(`Exec provider "${params.providerName}" protocolVersion must be 1.`); - } - const responseValues = parsed.values; - if (!isRecord(responseValues)) { - throw new Error(`Exec provider "${params.providerName}" response missing "values".`); - } - const responseErrors = isRecord(parsed.errors) ? parsed.errors : null; - const out: Record = {}; - for (const id of params.ids) { - if (responseErrors && id in responseErrors) { - const entry = responseErrors[id]; - if (isRecord(entry) && typeof entry.message === "string" && entry.message.trim()) { - throw new Error( - `Exec provider "${params.providerName}" failed for id "${id}" (${entry.message.trim()}).`, - ); - } - throw new Error(`Exec provider "${params.providerName}" failed for id "${id}".`); - } - if (!(id in responseValues)) { - throw new Error(`Exec provider "${params.providerName}" response missing id "${id}".`); - } - out[id] = responseValues[id]; - } - return out; -} - -async function resolveExecRefs(params: { - refs: SecretRef[]; - providerName: string; - providerConfig: ExecSecretProviderConfig; - env: NodeJS.ProcessEnv; - limits: ResolutionLimits; -}): Promise { - const ids = [...new Set(params.refs.map((ref) => ref.id))]; - if (ids.length > params.limits.maxRefsPerProvider) { - throw new Error( - `Exec provider "${params.providerName}" exceeded maxRefsPerProvider (${params.limits.maxRefsPerProvider}).`, - ); - } - - const commandPath = resolveUserPath(params.providerConfig.command); - const secureCommandPath = await assertSecurePath({ - targetPath: commandPath, - label: `secrets.providers.${params.providerName}.command`, - trustedDirs: params.providerConfig.trustedDirs, - allowInsecurePath: params.providerConfig.allowInsecurePath, - allowReadableByOthers: true, - allowSymlinkPath: params.providerConfig.allowSymlinkCommand, - }); - - const requestPayload = { - protocolVersion: 1, - provider: params.providerName, - ids, - }; - const input = JSON.stringify(requestPayload); - if (Buffer.byteLength(input, "utf8") > params.limits.maxBatchBytes) { - throw new Error( - `Exec provider "${params.providerName}" request exceeded maxBatchBytes (${params.limits.maxBatchBytes}).`, - ); - } - - const childEnv: NodeJS.ProcessEnv = {}; - for (const key of params.providerConfig.passEnv ?? []) { - const value = params.env[key] ?? process.env[key]; - if (value !== undefined) { - childEnv[key] = value; - } - } - for (const [key, value] of Object.entries(params.providerConfig.env ?? {})) { - childEnv[key] = value; - } - - const timeoutMs = normalizePositiveInt(params.providerConfig.timeoutMs, DEFAULT_EXEC_TIMEOUT_MS); - const noOutputTimeoutMs = normalizePositiveInt( - params.providerConfig.noOutputTimeoutMs, - timeoutMs, - ); - const maxOutputBytes = normalizePositiveInt( - params.providerConfig.maxOutputBytes, - DEFAULT_EXEC_MAX_OUTPUT_BYTES, - ); - const jsonOnly = params.providerConfig.jsonOnly ?? true; - - const result = await runExecResolver({ - command: secureCommandPath, - args: params.providerConfig.args ?? [], - cwd: path.dirname(secureCommandPath), - env: childEnv, - input, - timeoutMs, - noOutputTimeoutMs, - maxOutputBytes, - }); - if (result.termination === "timeout") { - throw new Error(`Exec provider "${params.providerName}" timed out after ${timeoutMs}ms.`); - } - if (result.termination === "no-output-timeout") { - throw new Error( - `Exec provider "${params.providerName}" produced no output for ${noOutputTimeoutMs}ms.`, - ); - } - if (result.code !== 0) { - throw new Error( - `Exec provider "${params.providerName}" exited with code ${String(result.code)}.`, - ); - } - - const values = parseExecValues({ - providerName: params.providerName, - ids, - stdout: result.stdout, - jsonOnly, - }); - const resolved = new Map(); - for (const id of ids) { - resolved.set(id, values[id]); - } - return resolved; -} - -async function resolveProviderRefs(params: { - refs: SecretRef[]; - source: SecretRefSource; - providerName: string; - providerConfig: SecretProviderConfig; - options: ResolveSecretRefOptions; - limits: ResolutionLimits; -}): Promise { - if (params.providerConfig.source === "env") { - return await resolveEnvRefs({ - refs: params.refs, - providerName: params.providerName, - providerConfig: params.providerConfig, - env: params.options.env ?? process.env, - }); - } - if (params.providerConfig.source === "file") { - return await resolveFileRefs({ - refs: params.refs, - providerName: params.providerName, - providerConfig: params.providerConfig, - cache: params.options.cache, - }); - } - if (params.providerConfig.source === "exec") { - return await resolveExecRefs({ - refs: params.refs, - providerName: params.providerName, - providerConfig: params.providerConfig, - env: params.options.env ?? process.env, - limits: params.limits, - }); - } - throw new Error( - `Unsupported secret provider source "${String((params.providerConfig as { source?: unknown }).source)}".`, - ); -} - export async function resolveSecretRefValues( refs: SecretRef[], options: ResolveSecretRefOptions, @@ -652,6 +88,7 @@ export async function resolveSecretRefValues( grouped.set(key, { source: ref.source, providerName: ref.provider, refs: [ref] }); } + const taskEnv = options.env ?? process.env; const tasks = [...grouped.values()].map( (group) => async (): Promise<{ group: typeof group; values: ProviderResolutionOutput }> => { if (group.refs.length > limits.maxRefsPerProvider) { @@ -662,10 +99,10 @@ export async function resolveSecretRefValues( const providerConfig = resolveConfiguredProvider(group.refs[0], options.config); const values = await resolveProviderRefs({ refs: group.refs, - source: group.source, providerName: group.providerName, providerConfig, - options, + env: taskEnv, + cache: options.cache, limits, }); return { group, values }; @@ -732,3 +169,5 @@ export async function resolveSecretRefString( } return resolved; } + +export type { SecretRefResolveCache };