Merge ca311868278d340c0edf683fa041a1173f719364 into 598f1826d8b2bc969aace2c6459824737667218c

docs/tools/web.md

@@ -514,3 +514,31 @@ Notes:
 - Responses are cached (default 15 minutes) to reduce repeated fetches.
 - If you use tool profiles/allowlists, add `web_search`/`web_fetch` or `group:web`.
 - If the API key is missing, `web_search` returns a short setup hint with a docs link.
+
+### SSRF policy for web_fetch
+
+`tools.web.fetch.ssrfPolicy` lets you tighten or relax the SSRF guard for `web_fetch` requests without affecting other tools. The optional fields mirror the browser-level settings:
+
+- `allowPrivateNetwork`: legacy alias for `dangerouslyAllowPrivateNetwork`. Set to `true` to permit private/internal IP addresses.
+- `dangerouslyAllowPrivateNetwork`: high-risk toggle that removes private/internal/special-use blocking. Enable only in trusted, isolated environments.
+- `allowedHostnames`: explicitly allowed hostnames or IPs that bypass private network checks even when private access is blocked globally.
+- `hostnameAllowlist`: pattern-based allowlist (e.g. `*.internal`) that shortlists which hostnames `web_fetch` is allowed to reach.
+
+Example:
+
+```json5
+{
+  tools: {
+    web: {
+      fetch: {
+        ssrfPolicy: {
+          hostnameAllowlist: ["example.com", "*.example.internal"],
+          allowedHostnames: ["192.168.1.42"],
+        },
+      },
+    },
+  },
+}
+```
+
+**Risk note:** `dangerouslyAllowPrivateNetwork` (and its alias `allowPrivateNetwork`) undermines the default SSRF blocking, so avoid enabling it unless you fully trust the target network. Prefer allowlists/whitelists to expand access only for specific hosts. If you do need to relax the guard, keep a narrow `hostnameAllowlist` or `allowedHostnames` and monitor the usage closely.

docs/zh-CN/tools/web.md

@@ -287,3 +287,30 @@ await web_search({
 - 响应会被缓存（默认 15 分钟）以减少重复获取。
 - 如果你使用工具配置文件/允许列表，添加 `web_search`/`web_fetch` 或 `group:web`。
 - 如果缺少 Brave 密钥，`web_search` 返回一个简短的设置提示和文档链接。
+
+* +### web_fetch 的 SSRF 策略
+  `tools.web.fetch.ssrfPolicy` 允许你在不影响其他工具的前提下，放宽或收紧 `web_fetch` 的 SSRF 保护。可选字段沿用了浏览器层的语义：
+
+- `allowPrivateNetwork`：`dangerouslyAllowPrivateNetwork` 的兼容别名。设置为 `true` 可允许访问私有/内部 IP。
+- `dangerouslyAllowPrivateNetwork`：高风险开关，禁用私网/特殊用途地址拦截。仅在完全受控的环境中启用。
+- `allowedHostnames`：显式允许的主机名或 IP，哪怕私网检查仍在，也能绕过拦截。
+- `hostnameAllowlist`：支持模式（例如 `*.internal`）的 hostname 白名单，用于限定 `web_fetch` 的目标范围。
+
+示例：
+
+```json5
+{
+  tools: {
+    web: {
+      fetch: {
+        ssrfPolicy: {
+          hostnameAllowlist: ["example.com", "*.example.internal"],
+          allowedHostnames: ["192.168.1.42"],
+        },
+      },
+    },
+  },
+}
+```
+
+**风险提示：** `dangerouslyAllowPrivateNetwork`（及其别名 `allowPrivateNetwork`）会弱化默认 SSRF 拦截，非必要不要启用。优先使用 allowlist/hostname 精准放行特定主机；若必须放宽，务必限制 `hostnameAllowlist` 或 `allowedHostnames`，并密切监控访问。

src/agents/tools/web-fetch.ssrf.test.ts

@@ -5,6 +5,7 @@ import { makeFetchHeaders } from "./web-fetch.test-harness.js";
 
 const lookupMock = vi.fn();
 const resolvePinnedHostname = ssrf.resolvePinnedHostname;
+const resolvePinnedHostnameWithPolicy = ssrf.resolvePinnedHostnameWithPolicy;
 
 function redirectResponse(location: string): Response {
   return {
@@ -34,16 +35,26 @@ function setMockFetch(
 
 async function createWebFetchToolForTest(params?: {
   firecrawl?: { enabled?: boolean; apiKey?: string };
+  ssrfPolicy?: ssrf.SsrFPolicy;
 }) {
   const { createWebFetchTool } = await import("./web-tools.js");
+
+  // Build config with ssrfPolicy injected via tools.web.fetch.ssrfPolicy
+  const fetchConfig: Record<string, unknown> = {
+    cacheTtlMinutes: 0,
+    firecrawl: params?.firecrawl ?? { enabled: false },
+  };
+
+  // Inject ssrfPolicy via config key if provided
+  if (params?.ssrfPolicy) {
+    fetchConfig.ssrfPolicy = params.ssrfPolicy;
+  }
+
   return createWebFetchTool({
     config: {
       tools: {
         web: {
-          fetch: {
-            cacheTtlMinutes: 0,
-            firecrawl: params?.firecrawl ?? { enabled: false },
-          },
+          fetch: fetchConfig,
         },
       },
     },
@@ -65,6 +76,12 @@ describe("web_fetch SSRF protection", () => {
     vi.spyOn(ssrf, "resolvePinnedHostname").mockImplementation((hostname) =>
       resolvePinnedHostname(hostname, lookupMock),
     );
+    vi.spyOn(ssrf, "resolvePinnedHostnameWithPolicy").mockImplementation((hostname, params) =>
+      resolvePinnedHostnameWithPolicy(hostname, {
+        ...params,
+        lookupFn: lookupMock,
+      }),
+    );
   });
 
   afterEach(() => {
@@ -137,4 +154,73 @@ describe("web_fetch SSRF protection", () => {
       extractor: "raw",
     });
   });
+
+  it("allows private IP when allowPrivateNetwork is true (via config)", async () => {
+    setMockFetch().mockResolvedValue(textResponse("ok"));
+    lookupMock.mockResolvedValue([{ address: "192.168.1.1", family: 4 }]);
+    const tool = await createWebFetchToolForTest({
+      ssrfPolicy: { allowPrivateNetwork: true },
+    });
+
+    await expect(tool?.execute?.("call", { url: "http://192.168.1.1" })).resolves.toBeDefined();
+  });
+
+  it("allows whitelisted hostnames (via config)", async () => {
+    setMockFetch().mockResolvedValue(textResponse("ok"));
+    lookupMock.mockResolvedValue([{ address: "192.168.1.1", family: 4 }]);
+    const tool = await createWebFetchToolForTest({
+      ssrfPolicy: { allowedHostnames: ["192.168.1.1"] },
+    });
+
+    await expect(tool?.execute?.("call", { url: "http://192.168.1.1" })).resolves.toBeDefined();
+  });
+
+  it("cache key differentiates between different SSRF policies", async () => {
+    const { createWebFetchTool } = await import("./web-tools.js");
+
+    lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
+
+    const testUrl = "https://example.com/page";
+    let callCount = 0;
+    setMockFetch().mockImplementation(async () => {
+      callCount++;
+      return textResponse(`response-${callCount}`);
+    });
+
+    // Create tool with caching ENABLED (non-zero cacheTtlMinutes)
+    const createTool = (ssrfPolicy?: ssrf.SsrFPolicy) => {
+      const fetchConfig: Record<string, unknown> = {
+        cacheTtlMinutes: 15, // Enable caching for this test
+        firecrawl: { enabled: false },
+      };
+      if (ssrfPolicy) {
+        fetchConfig.ssrfPolicy = ssrfPolicy;
+      }
+      return createWebFetchTool({
+        config: {
+          tools: {
+            web: {
+              fetch: fetchConfig,
+            },
+          },
+        },
+      });
+    };
+
+    // First, fetch with no SSRF policy
+    const toolNoPolicy = createTool();
+    const result1 = await toolNoPolicy?.execute?.("call", { url: testUrl });
+    expect(callCount).toBe(1);
+    expect(result1?.details?.text).toContain("response-1");
+
+    // Fetch the same URL with a different SSRF policy
+    // This should NOT hit the cache from the first call, creating a new fetch
+    const toolWithPolicy = createTool({ allowPrivateNetwork: true });
+    const result2 = await toolWithPolicy?.execute?.("call", { url: testUrl });
+    expect(callCount).toBe(2); // Should have called fetch again, not used cache
+    expect(result2?.details?.text).toContain("response-2");
+
+    // Verify different policies produce different results due to separate cache entries
+    expect(result1?.details?.text).not.toEqual(result2?.details?.text);
+  });
 });

src/agents/tools/web-fetch.ts

@@ -1,7 +1,7 @@
 import { Type } from "@sinclair/typebox";
 import type { OpenClawConfig } from "../../config/config.js";
 import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js";
-import { SsrFBlockedError } from "../../infra/net/ssrf.js";
+import { SsrFBlockedError, type SsrFPolicy } from "../../infra/net/ssrf.js";
 import { logDebug } from "../../logger.js";
 import type { RuntimeWebFetchFirecrawlMetadata } from "../../secrets/runtime-web-tools.js";
 import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
@@ -458,6 +458,7 @@ type WebFetchRuntimeParams = FirecrawlRuntimeParams & {
   cacheTtlMs: number;
   userAgent: string;
   readabilityEnabled: boolean;
+  ssrfPolicy?: SsrFPolicy;
 };
 
 function toFirecrawlContentParams(
@@ -513,7 +514,7 @@ async function maybeFetchFirecrawlWebFetchPayload(
 
 async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string, unknown>> {
   const cacheKey = normalizeCacheKey(
-    `fetch:${params.url}:${params.extractMode}:${params.maxChars}`,
+    `fetch:${params.url}:${params.extractMode}:${params.maxChars}:${JSON.stringify(params.ssrfPolicy ?? null)}`,
   );
   const cached = readCache(FETCH_CACHE, cacheKey);
   if (cached) {
@@ -539,6 +540,7 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
       url: params.url,
       maxRedirects: params.maxRedirects,
       timeoutSeconds: params.timeoutSeconds,
+      ssrfPolicy: params.ssrfPolicy,
       init: {
         headers: {
           Accept: "text/markdown, text/html;q=0.9, */*;q=0.1",
@@ -551,7 +553,7 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
     finalUrl = result.finalUrl;
     release = result.release;
 
-    // Cloudflare Markdown for Agents — log token budget hint when present
+    // Cloudflare Markdown for Agents - log token budget hint when present
     const markdownTokens = res.headers.get("x-markdown-tokens");
     if (markdownTokens) {
       logDebug(
@@ -761,6 +763,7 @@ export function createWebFetchTool(options?: {
     (fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) ||
     DEFAULT_FETCH_USER_AGENT;
   const maxResponseBytes = resolveFetchMaxResponseBytes(fetch);
+  const ssrfPolicy = fetch?.ssrfPolicy;
   return {
     label: "Web Fetch",
     name: "web_fetch",
@@ -787,6 +790,7 @@ export function createWebFetchTool(options?: {
         cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
         userAgent,
         readabilityEnabled,
+        ssrfPolicy,
         firecrawlEnabled,
         firecrawlApiKey,
         firecrawlBaseUrl,

src/agents/tools/web-guarded-fetch.ts

@@ -18,8 +18,12 @@ type WebToolGuardedFetchOptions = Omit<
 > & {
   timeoutSeconds?: number;
   useEnvProxy?: boolean;
+  ssrfPolicy?: SsrFPolicy;
 };
-type WebToolEndpointFetchOptions = Omit<WebToolGuardedFetchOptions, "policy" | "useEnvProxy">;
+type WebToolEndpointFetchOptions = Omit<
+  WebToolGuardedFetchOptions,
+  "policy" | "useEnvProxy" | "ssrfPolicy"
+>;
 
 function resolveTimeoutMs(params: {
   timeoutMs?: number;
@@ -37,10 +41,11 @@ function resolveTimeoutMs(params: {
 export async function fetchWithWebToolsNetworkGuard(
   params: WebToolGuardedFetchOptions,
 ): Promise<GuardedFetchResult> {
-  const { timeoutSeconds, useEnvProxy, ...rest } = params;
+  const { timeoutSeconds, useEnvProxy, ssrfPolicy, ...rest } = params;
   const resolved = {
     ...rest,
     timeoutMs: resolveTimeoutMs({ timeoutMs: rest.timeoutMs, timeoutSeconds }),
+    policy: ssrfPolicy ?? rest.policy,
   };
   return fetchWithSsrFGuard(
     useEnvProxy

src/config/schema.help.quality.test.ts

@@ -554,6 +554,11 @@ const FINAL_BACKLOG_TARGET_KEYS = [
   "browser.ssrfPolicy.dangerouslyAllowPrivateNetwork",
   "browser.ssrfPolicy.allowedHostnames",
   "browser.ssrfPolicy.hostnameAllowlist",
+  "tools.web.fetch.ssrfPolicy",
+  "tools.web.fetch.ssrfPolicy.allowPrivateNetwork",
+  "tools.web.fetch.ssrfPolicy.dangerouslyAllowPrivateNetwork",
+  "tools.web.fetch.ssrfPolicy.allowedHostnames",
+  "tools.web.fetch.ssrfPolicy.hostnameAllowlist",
   "diagnostics.enabled",
   "diagnostics.otel.enabled",
   "diagnostics.otel.endpoint",

src/config/schema.help.ts

@@ -690,6 +690,16 @@ export const FIELD_HELP: Record<string, string> = {
   "tools.web.fetch.firecrawl.maxAgeMs":
     "Firecrawl maxAge (ms) for cached results when supported by the API.",
   "tools.web.fetch.firecrawl.timeoutSeconds": "Timeout in seconds for Firecrawl requests.",
+  "tools.web.fetch.ssrfPolicy":
+    "SSRF policy for web_fetch requests. Use to allow or restrict access to private/internal network addresses.",
+  "tools.web.fetch.ssrfPolicy.allowPrivateNetwork":
+    "Legacy alias for dangerouslyAllowPrivateNetwork. Allow web_fetch to access private/internal network addresses.",
+  "tools.web.fetch.ssrfPolicy.dangerouslyAllowPrivateNetwork":
+    "Allow web_fetch to access private/internal/special-use IP addresses. Only enable in trusted network environments.",
+  "tools.web.fetch.ssrfPolicy.allowedHostnames":
+    "Explicitly allowed hostnames or IPs for web_fetch, even when private network access is otherwise blocked.",
+  "tools.web.fetch.ssrfPolicy.hostnameAllowlist":
+    "Pattern-based hostname allowlist for web_fetch (e.g. *.internal). Matched hosts bypass SSRF blocking.",
   models:
     "Model catalog root for provider definitions, merge/replace behavior, and optional Bedrock discovery integration. Keep provider definitions explicit and validated before relying on production failover paths.",
   "models.mode":

src/config/schema.labels.ts

@@ -233,6 +233,12 @@ export const FIELD_LABELS: Record<string, string> = {
   "tools.web.fetch.firecrawl.onlyMainContent": "Firecrawl Main Content Only",
   "tools.web.fetch.firecrawl.maxAgeMs": "Firecrawl Cache Max Age (ms)",
   "tools.web.fetch.firecrawl.timeoutSeconds": "Firecrawl Timeout (sec)",
+  "tools.web.fetch.ssrfPolicy": "Web Fetch SSRF Policy",
+  "tools.web.fetch.ssrfPolicy.allowPrivateNetwork": "Web Fetch Allow Private Network",
+  "tools.web.fetch.ssrfPolicy.dangerouslyAllowPrivateNetwork":
+    "Web Fetch Dangerously Allow Private Network",
+  "tools.web.fetch.ssrfPolicy.allowedHostnames": "Web Fetch Allowed Hostnames",
+  "tools.web.fetch.ssrfPolicy.hostnameAllowlist": "Web Fetch Hostname Allowlist",
   "gateway.controlUi.basePath": "Control UI Base Path",
   "gateway.controlUi.root": "Control UI Assets Root",
   "gateway.controlUi.allowedOrigins": "Control UI Allowed Origins",

src/config/types.tools.ts

@@ -1,5 +1,6 @@
 import type { ChatType } from "../channels/chat-type.js";
 import type { SafeBinProfileFixture } from "../infra/exec-safe-bin-policy.js";
+import type { SsrFPolicy } from "../infra/net/ssrf.js";
 import type { AgentElevatedAllowFromConfig, SessionSendPolicyAction } from "./types.base.js";
 import type { SecretInput } from "./types.secrets.js";
 
@@ -519,6 +520,12 @@ export type ToolsConfig = {
         /** Timeout in seconds for Firecrawl requests. */
         timeoutSeconds?: number;
       };
+      /**
+       * SSRF policy overrides for web_fetch requests.
+       * Use allowedHostnames to permit specific internal hosts, or dangerouslyAllowPrivateNetwork
+       * to disable all private-network blocking (not recommended for production).
+       */
+      ssrfPolicy?: SsrFPolicy;
     };
   };
   media?: MediaToolsConfig;

src/config/zod-schema.agent-runtime.ts

@@ -343,6 +343,15 @@ export const ToolsWebFetchSchema = z
       })
       .strict()
       .optional(),
+    ssrfPolicy: z
+      .object({
+        allowPrivateNetwork: z.boolean().optional(),
+        dangerouslyAllowPrivateNetwork: z.boolean().optional(),
+        allowedHostnames: z.array(z.string()).optional(),
+        hostnameAllowlist: z.array(z.string()).optional(),
+      })
+      .strict()
+      .optional(),
   })
   .strict()
   .optional();