Merge 09e845f862f1407bb51e9e96efd8b437f931b716 into d78e13f545136fcbba1feceecc5e0485a06c33a6

This commit is contained in:
xingxing 2026-03-21 12:50:15 +08:00 committed by GitHub
commit 7f24227642
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 27 additions and 1 deletions

View File

@ -458,6 +458,9 @@ type WebFetchRuntimeParams = FirecrawlRuntimeParams & {
cacheTtlMs: number;
userAgent: string;
readabilityEnabled: boolean;
ssrfPolicy?: {
allowRfc2544BenchmarkRange?: boolean;
};
};
function toFirecrawlContentParams(
@ -512,8 +515,11 @@ async function maybeFetchFirecrawlWebFetchPayload(
}
async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string, unknown>> {
// Include ssrfPolicy in cache key to prevent cross-policy cache bypass
// Use JSON.stringify to encode the full policy object, ensuring future fields are automatically included
const ssrfPolicySuffix = params.ssrfPolicy ? `:${JSON.stringify(params.ssrfPolicy)}` : "";
const cacheKey = normalizeCacheKey(
`fetch:${params.url}:${params.extractMode}:${params.maxChars}`,
`fetch:${params.url}:${params.extractMode}:${params.maxChars}${ssrfPolicySuffix}`,
);
const cached = readCache(FETCH_CACHE, cacheKey);
if (cached) {
@ -534,11 +540,18 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
let res: Response;
let release: (() => Promise<void>) | null = null;
let finalUrl = params.url;
// Build SSRF policy from config
const policy = params.ssrfPolicy?.allowRfc2544BenchmarkRange
? { allowRfc2544BenchmarkRange: true }
: undefined;
try {
const result = await fetchWithWebToolsNetworkGuard({
url: params.url,
maxRedirects: params.maxRedirects,
timeoutSeconds: params.timeoutSeconds,
policy,
init: {
headers: {
Accept: "text/markdown, text/html;q=0.9, */*;q=0.1",
@ -741,6 +754,7 @@ export function createWebFetchTool(options?: {
return null;
}
const readabilityEnabled = resolveFetchReadabilityEnabled(fetch);
const ssrfPolicy = fetch?.ssrfPolicy;
const firecrawl = resolveFirecrawlConfig(fetch);
const runtimeFirecrawlActive = options?.runtimeFirecrawl?.active;
const shouldResolveFirecrawlApiKey =
@ -787,6 +801,7 @@ export function createWebFetchTool(options?: {
cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
userAgent,
readabilityEnabled,
ssrfPolicy,
firecrawlEnabled,
firecrawlApiKey,
firecrawlBaseUrl,

View File

@ -505,6 +505,11 @@ export type ToolsConfig = {
userAgent?: string;
/** Use Readability to extract main content (default: true). */
readability?: boolean;
/** SSRF policy configuration for web_fetch. */
ssrfPolicy?: {
/** Allow RFC 2544 benchmark range IPs (198.18.0.0/15) for fake-IP proxy compatibility (e.g., Clash TUN mode, Surge). */
allowRfc2544BenchmarkRange?: boolean;
};
firecrawl?: {
/** Enable Firecrawl fallback (default: true when apiKey is set). */
enabled?: boolean;

View File

@ -332,6 +332,12 @@ export const ToolsWebFetchSchema = z
maxRedirects: z.number().int().nonnegative().optional(),
userAgent: z.string().optional(),
readability: z.boolean().optional(),
ssrfPolicy: z
.object({
allowRfc2544BenchmarkRange: z.boolean().optional(),
})
.strict()
.optional(),
firecrawl: z
.object({
enabled: z.boolean().optional(),