Merge 09e845f862f1407bb51e9e96efd8b437f931b716 into d78e13f545136fcbba1feceecc5e0485a06c33a6
This commit is contained in:
commit
7f24227642
@ -458,6 +458,9 @@ type WebFetchRuntimeParams = FirecrawlRuntimeParams & {
|
||||
cacheTtlMs: number;
|
||||
userAgent: string;
|
||||
readabilityEnabled: boolean;
|
||||
ssrfPolicy?: {
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
function toFirecrawlContentParams(
|
||||
@ -512,8 +515,11 @@ async function maybeFetchFirecrawlWebFetchPayload(
|
||||
}
|
||||
|
||||
async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string, unknown>> {
|
||||
// Include ssrfPolicy in cache key to prevent cross-policy cache bypass
|
||||
// Use JSON.stringify to encode the full policy object, ensuring future fields are automatically included
|
||||
const ssrfPolicySuffix = params.ssrfPolicy ? `:${JSON.stringify(params.ssrfPolicy)}` : "";
|
||||
const cacheKey = normalizeCacheKey(
|
||||
`fetch:${params.url}:${params.extractMode}:${params.maxChars}`,
|
||||
`fetch:${params.url}:${params.extractMode}:${params.maxChars}${ssrfPolicySuffix}`,
|
||||
);
|
||||
const cached = readCache(FETCH_CACHE, cacheKey);
|
||||
if (cached) {
|
||||
@ -534,11 +540,18 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
|
||||
let res: Response;
|
||||
let release: (() => Promise<void>) | null = null;
|
||||
let finalUrl = params.url;
|
||||
|
||||
// Build SSRF policy from config
|
||||
const policy = params.ssrfPolicy?.allowRfc2544BenchmarkRange
|
||||
? { allowRfc2544BenchmarkRange: true }
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
const result = await fetchWithWebToolsNetworkGuard({
|
||||
url: params.url,
|
||||
maxRedirects: params.maxRedirects,
|
||||
timeoutSeconds: params.timeoutSeconds,
|
||||
policy,
|
||||
init: {
|
||||
headers: {
|
||||
Accept: "text/markdown, text/html;q=0.9, */*;q=0.1",
|
||||
@ -741,6 +754,7 @@ export function createWebFetchTool(options?: {
|
||||
return null;
|
||||
}
|
||||
const readabilityEnabled = resolveFetchReadabilityEnabled(fetch);
|
||||
const ssrfPolicy = fetch?.ssrfPolicy;
|
||||
const firecrawl = resolveFirecrawlConfig(fetch);
|
||||
const runtimeFirecrawlActive = options?.runtimeFirecrawl?.active;
|
||||
const shouldResolveFirecrawlApiKey =
|
||||
@ -787,6 +801,7 @@ export function createWebFetchTool(options?: {
|
||||
cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
|
||||
userAgent,
|
||||
readabilityEnabled,
|
||||
ssrfPolicy,
|
||||
firecrawlEnabled,
|
||||
firecrawlApiKey,
|
||||
firecrawlBaseUrl,
|
||||
|
||||
@ -505,6 +505,11 @@ export type ToolsConfig = {
|
||||
userAgent?: string;
|
||||
/** Use Readability to extract main content (default: true). */
|
||||
readability?: boolean;
|
||||
/** SSRF policy configuration for web_fetch. */
|
||||
ssrfPolicy?: {
|
||||
/** Allow RFC 2544 benchmark range IPs (198.18.0.0/15) for fake-IP proxy compatibility (e.g., Clash TUN mode, Surge). */
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
};
|
||||
firecrawl?: {
|
||||
/** Enable Firecrawl fallback (default: true when apiKey is set). */
|
||||
enabled?: boolean;
|
||||
|
||||
@ -332,6 +332,12 @@ export const ToolsWebFetchSchema = z
|
||||
maxRedirects: z.number().int().nonnegative().optional(),
|
||||
userAgent: z.string().optional(),
|
||||
readability: z.boolean().optional(),
|
||||
ssrfPolicy: z
|
||||
.object({
|
||||
allowRfc2544BenchmarkRange: z.boolean().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
firecrawl: z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user