diff --git a/docs/tools/web.md b/docs/tools/web.md index 8d5b6bff5f1..1bf8cc1e867 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -514,3 +514,31 @@ Notes: - Responses are cached (default 15 minutes) to reduce repeated fetches. - If you use tool profiles/allowlists, add `web_search`/`web_fetch` or `group:web`. - If the API key is missing, `web_search` returns a short setup hint with a docs link. + +### SSRF policy for web_fetch + +`tools.web.fetch.ssrfPolicy` lets you tighten or relax the SSRF guard for `web_fetch` requests without affecting other tools. The optional fields mirror the browser-level settings: + +- `allowPrivateNetwork`: legacy alias for `dangerouslyAllowPrivateNetwork`. Set to `true` to permit private/internal IP addresses. +- `dangerouslyAllowPrivateNetwork`: high-risk toggle that removes private/internal/special-use blocking. Enable only in trusted, isolated environments. +- `allowedHostnames`: explicitly allowed hostnames or IPs that bypass private network checks even when private access is blocked globally. +- `hostnameAllowlist`: pattern-based allowlist (e.g. `*.internal`) that shortlists which hostnames `web_fetch` is allowed to reach. + +Example: + +```json5 +{ + tools: { + web: { + fetch: { + ssrfPolicy: { + hostnameAllowlist: ["example.com", "*.example.internal"], + allowedHostnames: ["192.168.1.42"], + }, + }, + }, + }, +} +``` + +**Risk note:** `dangerouslyAllowPrivateNetwork` (and its alias `allowPrivateNetwork`) undermines the default SSRF blocking, so avoid enabling it unless you fully trust the target network. Prefer allowlists/whitelists to expand access only for specific hosts. If you do need to relax the guard, keep a narrow `hostnameAllowlist` or `allowedHostnames` and monitor the usage closely. diff --git a/docs/zh-CN/tools/web.md b/docs/zh-CN/tools/web.md index 44026c67e29..75916dffcaa 100644 --- a/docs/zh-CN/tools/web.md +++ b/docs/zh-CN/tools/web.md @@ -287,3 +287,30 @@ await web_search({ - 响应会被缓存（默认 15 分钟）以减少重复获取。 - 如果你使用工具配置文件/允许列表，添加 `web_search`/`web_fetch` 或 `group:web`。 - 如果缺少 Brave 密钥，`web_search` 返回一个简短的设置提示和文档链接。 + +* +### web_fetch 的 SSRF 策略 + `tools.web.fetch.ssrfPolicy` 允许你在不影响其他工具的前提下，放宽或收紧 `web_fetch` 的 SSRF 保护。可选字段沿用了浏览器层的语义： + +- `allowPrivateNetwork`：`dangerouslyAllowPrivateNetwork` 的兼容别名。设置为 `true` 可允许访问私有/内部 IP。 +- `dangerouslyAllowPrivateNetwork`：高风险开关，禁用私网/特殊用途地址拦截。仅在完全受控的环境中启用。 +- `allowedHostnames`：显式允许的主机名或 IP，哪怕私网检查仍在，也能绕过拦截。 +- `hostnameAllowlist`：支持模式（例如 `*.internal`）的 hostname 白名单，用于限定 `web_fetch` 的目标范围。 + +示例： + +```json5 +{ + tools: { + web: { + fetch: { + ssrfPolicy: { + hostnameAllowlist: ["example.com", "*.example.internal"], + allowedHostnames: ["192.168.1.42"], + }, + }, + }, + }, +} +``` + +**风险提示：** `dangerouslyAllowPrivateNetwork`（及其别名 `allowPrivateNetwork`）会弱化默认 SSRF 拦截，非必要不要启用。优先使用 allowlist/hostname 精准放行特定主机；若必须放宽，务必限制 `hostnameAllowlist` 或 `allowedHostnames`，并密切监控访问。 diff --git a/src/agents/tools/web-fetch.ssrf.test.ts b/src/agents/tools/web-fetch.ssrf.test.ts index c0489c9b5ba..919504dfaac 100644 --- a/src/agents/tools/web-fetch.ssrf.test.ts +++ b/src/agents/tools/web-fetch.ssrf.test.ts @@ -5,6 +5,7 @@ import { makeFetchHeaders } from "./web-fetch.test-harness.js"; const lookupMock = vi.fn(); const resolvePinnedHostname = ssrf.resolvePinnedHostname; +const resolvePinnedHostnameWithPolicy = ssrf.resolvePinnedHostnameWithPolicy; function redirectResponse(location: string): Response { return { @@ -34,16 +35,26 @@ function setMockFetch( async function createWebFetchToolForTest(params?: { firecrawl?: { enabled?: boolean; apiKey?: string }; + ssrfPolicy?: ssrf.SsrFPolicy; }) { const { createWebFetchTool } = await import("./web-tools.js"); + + // Build config with ssrfPolicy injected via tools.web.fetch.ssrfPolicy + const fetchConfig: Record = { + cacheTtlMinutes: 0, + firecrawl: params?.firecrawl ?? { enabled: false }, + }; + + // Inject ssrfPolicy via config key if provided + if (params?.ssrfPolicy) { + fetchConfig.ssrfPolicy = params.ssrfPolicy; + } + return createWebFetchTool({ config: { tools: { web: { - fetch: { - cacheTtlMinutes: 0, - firecrawl: params?.firecrawl ?? { enabled: false }, - }, + fetch: fetchConfig, }, }, }, @@ -65,6 +76,12 @@ describe("web_fetch SSRF protection", () => { vi.spyOn(ssrf, "resolvePinnedHostname").mockImplementation((hostname) => resolvePinnedHostname(hostname, lookupMock), ); + vi.spyOn(ssrf, "resolvePinnedHostnameWithPolicy").mockImplementation((hostname, params) => + resolvePinnedHostnameWithPolicy(hostname, { + ...params, + lookupFn: lookupMock, + }), + ); }); afterEach(() => { @@ -137,4 +154,73 @@ describe("web_fetch SSRF protection", () => { extractor: "raw", }); }); + + it("allows private IP when allowPrivateNetwork is true (via config)", async () => { + setMockFetch().mockResolvedValue(textResponse("ok")); + lookupMock.mockResolvedValue([{ address: "192.168.1.1", family: 4 }]); + const tool = await createWebFetchToolForTest({ + ssrfPolicy: { allowPrivateNetwork: true }, + }); + + await expect(tool?.execute?.("call", { url: "http://192.168.1.1" })).resolves.toBeDefined(); + }); + + it("allows whitelisted hostnames (via config)", async () => { + setMockFetch().mockResolvedValue(textResponse("ok")); + lookupMock.mockResolvedValue([{ address: "192.168.1.1", family: 4 }]); + const tool = await createWebFetchToolForTest({ + ssrfPolicy: { allowedHostnames: ["192.168.1.1"] }, + }); + + await expect(tool?.execute?.("call", { url: "http://192.168.1.1" })).resolves.toBeDefined(); + }); + + it("cache key differentiates between different SSRF policies", async () => { + const { createWebFetchTool } = await import("./web-tools.js"); + + lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]); + + const testUrl = "https://example.com/page"; + let callCount = 0; + setMockFetch().mockImplementation(async () => { + callCount++; + return textResponse(`response-${callCount}`); + }); + + // Create tool with caching ENABLED (non-zero cacheTtlMinutes) + const createTool = (ssrfPolicy?: ssrf.SsrFPolicy) => { + const fetchConfig: Record = { + cacheTtlMinutes: 15, // Enable caching for this test + firecrawl: { enabled: false }, + }; + if (ssrfPolicy) { + fetchConfig.ssrfPolicy = ssrfPolicy; + } + return createWebFetchTool({ + config: { + tools: { + web: { + fetch: fetchConfig, + }, + }, + }, + }); + }; + + // First, fetch with no SSRF policy + const toolNoPolicy = createTool(); + const result1 = await toolNoPolicy?.execute?.("call", { url: testUrl }); + expect(callCount).toBe(1); + expect(result1?.details?.text).toContain("response-1"); + + // Fetch the same URL with a different SSRF policy + // This should NOT hit the cache from the first call, creating a new fetch + const toolWithPolicy = createTool({ allowPrivateNetwork: true }); + const result2 = await toolWithPolicy?.execute?.("call", { url: testUrl }); + expect(callCount).toBe(2); // Should have called fetch again, not used cache + expect(result2?.details?.text).toContain("response-2"); + + // Verify different policies produce different results due to separate cache entries + expect(result1?.details?.text).not.toEqual(result2?.details?.text); + }); }); diff --git a/src/agents/tools/web-fetch.ts b/src/agents/tools/web-fetch.ts index 92f94bf3a28..cc85d6d934e 100644 --- a/src/agents/tools/web-fetch.ts +++ b/src/agents/tools/web-fetch.ts @@ -1,7 +1,7 @@ import { Type } from "@sinclair/typebox"; import type { OpenClawConfig } from "../../config/config.js"; import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js"; -import { SsrFBlockedError } from "../../infra/net/ssrf.js"; +import { SsrFBlockedError, type SsrFPolicy } from "../../infra/net/ssrf.js"; import { logDebug } from "../../logger.js"; import type { RuntimeWebFetchFirecrawlMetadata } from "../../secrets/runtime-web-tools.js"; import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js"; @@ -458,6 +458,7 @@ type WebFetchRuntimeParams = FirecrawlRuntimeParams & { cacheTtlMs: number; userAgent: string; readabilityEnabled: boolean; + ssrfPolicy?: SsrFPolicy; }; function toFirecrawlContentParams( @@ -513,7 +514,7 @@ async function maybeFetchFirecrawlWebFetchPayload( async function runWebFetch(params: WebFetchRuntimeParams): Promise> { const cacheKey = normalizeCacheKey( - `fetch:${params.url}:${params.extractMode}:${params.maxChars}`, + `fetch:${params.url}:${params.extractMode}:${params.maxChars}:${JSON.stringify(params.ssrfPolicy ?? null)}`, ); const cached = readCache(FETCH_CACHE, cacheKey); if (cached) { @@ -539,6 +540,7 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise & { timeoutSeconds?: number; useEnvProxy?: boolean; + ssrfPolicy?: SsrFPolicy; }; -type WebToolEndpointFetchOptions = Omit; +type WebToolEndpointFetchOptions = Omit< + WebToolGuardedFetchOptions, + "policy" | "useEnvProxy" | "ssrfPolicy" +>; function resolveTimeoutMs(params: { timeoutMs?: number; @@ -37,10 +41,11 @@ function resolveTimeoutMs(params: { export async function fetchWithWebToolsNetworkGuard( params: WebToolGuardedFetchOptions, ): Promise { - const { timeoutSeconds, useEnvProxy, ...rest } = params; + const { timeoutSeconds, useEnvProxy, ssrfPolicy, ...rest } = params; const resolved = { ...rest, timeoutMs: resolveTimeoutMs({ timeoutMs: rest.timeoutMs, timeoutSeconds }), + policy: ssrfPolicy ?? rest.policy, }; return fetchWithSsrFGuard( useEnvProxy diff --git a/src/config/schema.help.quality.test.ts b/src/config/schema.help.quality.test.ts index 18e1947d88f..2d5dba885cc 100644 --- a/src/config/schema.help.quality.test.ts +++ b/src/config/schema.help.quality.test.ts @@ -554,6 +554,11 @@ const FINAL_BACKLOG_TARGET_KEYS = [ "browser.ssrfPolicy.dangerouslyAllowPrivateNetwork", "browser.ssrfPolicy.allowedHostnames", "browser.ssrfPolicy.hostnameAllowlist", + "tools.web.fetch.ssrfPolicy", + "tools.web.fetch.ssrfPolicy.allowPrivateNetwork", + "tools.web.fetch.ssrfPolicy.dangerouslyAllowPrivateNetwork", + "tools.web.fetch.ssrfPolicy.allowedHostnames", + "tools.web.fetch.ssrfPolicy.hostnameAllowlist", "diagnostics.enabled", "diagnostics.otel.enabled", "diagnostics.otel.endpoint", diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 947726bd7e8..641774dc922 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -690,6 +690,16 @@ export const FIELD_HELP: Record = { "tools.web.fetch.firecrawl.maxAgeMs": "Firecrawl maxAge (ms) for cached results when supported by the API.", "tools.web.fetch.firecrawl.timeoutSeconds": "Timeout in seconds for Firecrawl requests.", + "tools.web.fetch.ssrfPolicy": + "SSRF policy for web_fetch requests. Use to allow or restrict access to private/internal network addresses.", + "tools.web.fetch.ssrfPolicy.allowPrivateNetwork": + "Legacy alias for dangerouslyAllowPrivateNetwork. Allow web_fetch to access private/internal network addresses.", + "tools.web.fetch.ssrfPolicy.dangerouslyAllowPrivateNetwork": + "Allow web_fetch to access private/internal/special-use IP addresses. Only enable in trusted network environments.", + "tools.web.fetch.ssrfPolicy.allowedHostnames": + "Explicitly allowed hostnames or IPs for web_fetch, even when private network access is otherwise blocked.", + "tools.web.fetch.ssrfPolicy.hostnameAllowlist": + "Pattern-based hostname allowlist for web_fetch (e.g. *.internal). Matched hosts bypass SSRF blocking.", models: "Model catalog root for provider definitions, merge/replace behavior, and optional Bedrock discovery integration. Keep provider definitions explicit and validated before relying on production failover paths.", "models.mode": diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts index 53317e2fcd2..e432ea6ce5f 100644 --- a/src/config/schema.labels.ts +++ b/src/config/schema.labels.ts @@ -233,6 +233,12 @@ export const FIELD_LABELS: Record = { "tools.web.fetch.firecrawl.onlyMainContent": "Firecrawl Main Content Only", "tools.web.fetch.firecrawl.maxAgeMs": "Firecrawl Cache Max Age (ms)", "tools.web.fetch.firecrawl.timeoutSeconds": "Firecrawl Timeout (sec)", + "tools.web.fetch.ssrfPolicy": "Web Fetch SSRF Policy", + "tools.web.fetch.ssrfPolicy.allowPrivateNetwork": "Web Fetch Allow Private Network", + "tools.web.fetch.ssrfPolicy.dangerouslyAllowPrivateNetwork": + "Web Fetch Dangerously Allow Private Network", + "tools.web.fetch.ssrfPolicy.allowedHostnames": "Web Fetch Allowed Hostnames", + "tools.web.fetch.ssrfPolicy.hostnameAllowlist": "Web Fetch Hostname Allowlist", "gateway.controlUi.basePath": "Control UI Base Path", "gateway.controlUi.root": "Control UI Assets Root", "gateway.controlUi.allowedOrigins": "Control UI Allowed Origins", diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index f42fa365f6f..7cf383f87aa 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -1,5 +1,6 @@ import type { ChatType } from "../channels/chat-type.js"; import type { SafeBinProfileFixture } from "../infra/exec-safe-bin-policy.js"; +import type { SsrFPolicy } from "../infra/net/ssrf.js"; import type { AgentElevatedAllowFromConfig, SessionSendPolicyAction } from "./types.base.js"; import type { SecretInput } from "./types.secrets.js"; @@ -519,6 +520,12 @@ export type ToolsConfig = { /** Timeout in seconds for Firecrawl requests. */ timeoutSeconds?: number; }; + /** + * SSRF policy overrides for web_fetch requests. + * Use allowedHostnames to permit specific internal hosts, or dangerouslyAllowPrivateNetwork + * to disable all private-network blocking (not recommended for production). + */ + ssrfPolicy?: SsrFPolicy; }; }; media?: MediaToolsConfig; diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 10f0f8637e9..7f7bc6fca5a 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -343,6 +343,15 @@ export const ToolsWebFetchSchema = z }) .strict() .optional(), + ssrfPolicy: z + .object({ + allowPrivateNetwork: z.boolean().optional(), + dangerouslyAllowPrivateNetwork: z.boolean().optional(), + allowedHostnames: z.array(z.string()).optional(), + hostnameAllowlist: z.array(z.string()).optional(), + }) + .strict() + .optional(), }) .strict() .optional();