feat(web-fetch): add ssrfPolicy.allowRfc2544BenchmarkRange config option
This PR adds a scoped ssrfPolicy config block to web_fetch, allowing users behind fake-IP proxy tools (Clash TUN, Surge, etc.) to enable RFC 2544 benchmark range access. ## Changes - src/config/zod-schema.agent-runtime.ts: add ssrfPolicy to ToolsWebFetchSchema - src/config/types.tools.ts: add TypeScript type and JSDoc for ssrfPolicy - src/agents/tools/web-fetch.ts: - Add ssrfPolicy to WebFetchRuntimeParams - Pass policy to fetchWithWebToolsNetworkGuard - Include ssrfPolicy in cache key to prevent cross-policy cache bypass ## Problem OpenClaw's SSRF guard blocks RFC 2544 benchmark range (198.18.0.0/15), which is used by fake-IP proxy tools like Clash, Surge, and Mihomo. This causes web_fetch to fail for users behind these proxies. ## Solution Add a configuration option: tools.web.fetch.ssrfPolicy.allowRfc2544BenchmarkRange: true This allows users to explicitly opt-in to allowing the RFC 2544 range, restoring web_fetch functionality for fake-IP proxy environments. Closes #25322, #27597, #48080, #48961, #49377, #49444
This commit is contained in:
parent
598f1826d8
commit
8bc6ffd6a2
@ -458,6 +458,9 @@ type WebFetchRuntimeParams = FirecrawlRuntimeParams & {
|
||||
cacheTtlMs: number;
|
||||
userAgent: string;
|
||||
readabilityEnabled: boolean;
|
||||
ssrfPolicy?: {
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
function toFirecrawlContentParams(
|
||||
@ -512,8 +515,10 @@ async function maybeFetchFirecrawlWebFetchPayload(
|
||||
}
|
||||
|
||||
async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string, unknown>> {
|
||||
// Include ssrfPolicy in cache key to prevent cross-policy cache bypass
|
||||
const ssrfPolicySuffix = params.ssrfPolicy?.allowRfc2544BenchmarkRange ? ":rfc2544" : "";
|
||||
const cacheKey = normalizeCacheKey(
|
||||
`fetch:${params.url}:${params.extractMode}:${params.maxChars}`,
|
||||
`fetch:${params.url}:${params.extractMode}:${params.maxChars}${ssrfPolicySuffix}`,
|
||||
);
|
||||
const cached = readCache(FETCH_CACHE, cacheKey);
|
||||
if (cached) {
|
||||
@ -534,11 +539,18 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
|
||||
let res: Response;
|
||||
let release: (() => Promise<void>) | null = null;
|
||||
let finalUrl = params.url;
|
||||
|
||||
// Build SSRF policy from config
|
||||
const policy = params.ssrfPolicy?.allowRfc2544BenchmarkRange
|
||||
? { allowRfc2544BenchmarkRange: true }
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
const result = await fetchWithWebToolsNetworkGuard({
|
||||
url: params.url,
|
||||
maxRedirects: params.maxRedirects,
|
||||
timeoutSeconds: params.timeoutSeconds,
|
||||
policy,
|
||||
init: {
|
||||
headers: {
|
||||
Accept: "text/markdown, text/html;q=0.9, */*;q=0.1",
|
||||
@ -741,6 +753,7 @@ export function createWebFetchTool(options?: {
|
||||
return null;
|
||||
}
|
||||
const readabilityEnabled = resolveFetchReadabilityEnabled(fetch);
|
||||
const ssrfPolicy = fetch?.ssrfPolicy;
|
||||
const firecrawl = resolveFirecrawlConfig(fetch);
|
||||
const runtimeFirecrawlActive = options?.runtimeFirecrawl?.active;
|
||||
const shouldResolveFirecrawlApiKey =
|
||||
@ -787,6 +800,7 @@ export function createWebFetchTool(options?: {
|
||||
cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
|
||||
userAgent,
|
||||
readabilityEnabled,
|
||||
ssrfPolicy,
|
||||
firecrawlEnabled,
|
||||
firecrawlApiKey,
|
||||
firecrawlBaseUrl,
|
||||
|
||||
@ -505,6 +505,11 @@ export type ToolsConfig = {
|
||||
userAgent?: string;
|
||||
/** Use Readability to extract main content (default: true). */
|
||||
readability?: boolean;
|
||||
/** SSRF policy configuration for web_fetch. */
|
||||
ssrfPolicy?: {
|
||||
/** Allow RFC 2544 benchmark range IPs (198.18.0.0/15) for fake-IP proxy compatibility (e.g., Clash TUN mode, Surge). */
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
};
|
||||
firecrawl?: {
|
||||
/** Enable Firecrawl fallback (default: true when apiKey is set). */
|
||||
enabled?: boolean;
|
||||
|
||||
@ -332,6 +332,12 @@ export const ToolsWebFetchSchema = z
|
||||
maxRedirects: z.number().int().nonnegative().optional(),
|
||||
userAgent: z.string().optional(),
|
||||
readability: z.boolean().optional(),
|
||||
ssrfPolicy: z
|
||||
.object({
|
||||
allowRfc2544BenchmarkRange: z.boolean().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
firecrawl: z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user