diff --git a/src/agents/tools/web-fetch.ts b/src/agents/tools/web-fetch.ts index 9206386a796..fc0aaead98e 100644 --- a/src/agents/tools/web-fetch.ts +++ b/src/agents/tools/web-fetch.ts @@ -1,16 +1,12 @@ import { Type } from "@sinclair/typebox"; import type { OpenClawConfig } from "../../config/config.js"; +import type { AnyAgentTool } from "./common.js"; import { fetchWithSsrFGuard } from "../../infra/net/fetch-guard.js"; -import { - matchesHostnameAllowlist, - normalizeHostnameAllowlist, - SsrFBlockedError, -} from "../../infra/net/ssrf.js"; +import { SsrFBlockedError } from "../../infra/net/ssrf.js"; import { logDebug } from "../../logger.js"; import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js"; import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; import { stringEnum } from "../schema/typebox.js"; -import type { AnyAgentTool } from "./common.js"; import { jsonResult, readNumberParam, readStringParam } from "./common.js"; import { extractReadableContent, @@ -26,7 +22,6 @@ import { normalizeCacheKey, readCache, readResponseText, - resolveWebUrlAllowlist, resolveCacheTtlMs, resolveTimeoutSeconds, withTimeout, @@ -73,22 +68,6 @@ type WebFetchConfig = NonNullable["web"] extends infer : undefined : undefined; -type WebConfig = NonNullable["web"]; - -export function resolveFetchUrlAllowlist(web?: WebConfig): string[] | undefined { - return resolveWebUrlAllowlist(web); -} - -export function isUrlAllowedByAllowlist(url: string, allowlist: string[]): boolean { - try { - const hostname = new URL(url).hostname; - const normalizedAllowlist = normalizeHostnameAllowlist(allowlist); - return matchesHostnameAllowlist(hostname, normalizedAllowlist); - } catch { - return false; - } -} - type FirecrawlFetchConfig = | { enabled?: boolean; @@ -753,7 +732,6 @@ export function createWebFetchTool(options?: { (fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) || DEFAULT_FETCH_USER_AGENT; const maxResponseBytes = resolveFetchMaxResponseBytes(fetch); - const urlAllowlist = resolveFetchUrlAllowlist(options?.config?.tools?.web); return { label: "Web Fetch", name: "web_fetch", @@ -763,25 +741,6 @@ export function createWebFetchTool(options?: { execute: async (_toolCallId, args) => { const params = args as Record; const url = readStringParam(params, "url", { required: true }); - - // Check URL against allowlist if configured - if (urlAllowlist && urlAllowlist.length > 0) { - if (!isUrlAllowedByAllowlist(url, urlAllowlist)) { - let hostname: string; - try { - hostname = new URL(url).hostname; - } catch { - hostname = url; - } - return jsonResult({ - error: "url_not_allowed", - message: `URL not in allowlist. Allowed domains: ${urlAllowlist.join(", ")}`, - blockedUrl: url, - blockedHostname: hostname, - }); - } - } - const extractMode = readStringParam(params, "extractMode") === "text" ? "text" : "markdown"; const maxChars = readNumberParam(params, "maxChars", { integer: true }); const maxCharsCap = resolveFetchMaxCharsCap(fetch); diff --git a/src/agents/tools/web-search.ts b/src/agents/tools/web-search.ts index a47737594df..be174b951d3 100644 --- a/src/agents/tools/web-search.ts +++ b/src/agents/tools/web-search.ts @@ -1,10 +1,9 @@ import { Type } from "@sinclair/typebox"; -import { formatCliCommand } from "../../cli/command-format.js"; import type { OpenClawConfig } from "../../config/config.js"; -import { matchesHostnameAllowlist, normalizeHostnameAllowlist } from "../../infra/net/ssrf.js"; +import type { AnyAgentTool } from "./common.js"; +import { formatCliCommand } from "../../cli/command-format.js"; import { wrapWebContent } from "../../security/external-content.js"; import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; -import type { AnyAgentTool } from "./common.js"; import { jsonResult, readNumberParam, readStringParam } from "./common.js"; import { CacheEntry, @@ -13,7 +12,6 @@ import { normalizeCacheKey, readCache, readResponseText, - resolveWebUrlAllowlist, resolveCacheTtlMs, resolveTimeoutSeconds, withTimeout, @@ -77,33 +75,6 @@ type WebSearchConfig = NonNullable["web"] extends infer : undefined : undefined; -type WebConfig = NonNullable["web"]; - -export function resolveUrlAllowlist(web?: WebConfig): string[] | undefined { - return resolveWebUrlAllowlist(web); -} - -export function filterResultsByAllowlist( - results: Array<{ url?: string; siteName?: string }>, - allowlist: string[], -): Array<{ url?: string; siteName?: string }> { - if (allowlist.length === 0) { - return results; - } - const normalizedAllowlist = normalizeHostnameAllowlist(allowlist); - return results.filter((result) => { - if (!result.url) { - return true; // Keep entries without URL - } - try { - const hostname = new URL(result.url).hostname; - return matchesHostnameAllowlist(hostname, normalizedAllowlist); - } catch { - return true; // Keep entries with invalid URLs (let them pass through) - } - }); -} - type BraveSearchResult = { title?: string; url?: string; @@ -595,7 +566,6 @@ async function runWebSearch(params: { perplexityModel?: string; grokModel?: string; grokInlineCitations?: boolean; - urlAllowlist?: string[]; }): Promise> { const cacheKey = normalizeCacheKey( params.provider === "brave" @@ -718,15 +688,10 @@ async function runWebSearch(params: { }; }); - // Filter results by urlAllowlist if configured - const filteredResults = params.urlAllowlist - ? filterResultsByAllowlist(mapped, params.urlAllowlist) - : mapped; - const payload = { query: params.query, provider: params.provider, - count: filteredResults.length, + count: mapped.length, tookMs: Date.now() - start, externalContent: { untrusted: true, @@ -734,7 +699,7 @@ async function runWebSearch(params: { provider: params.provider, wrapped: true, }, - results: filteredResults, + results: mapped, }; writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); return payload; @@ -752,7 +717,6 @@ export function createWebSearchTool(options?: { const provider = resolveSearchProvider(search); const perplexityConfig = resolvePerplexityConfig(search); const grokConfig = resolveGrokConfig(search); - const urlAllowlist = resolveUrlAllowlist(options?.config?.tools?.web); const description = provider === "perplexity" @@ -822,7 +786,6 @@ export function createWebSearchTool(options?: { perplexityModel: resolvePerplexityModel(perplexityConfig), grokModel: resolveGrokModel(grokConfig), grokInlineCitations: resolveGrokInlineCitations(grokConfig), - urlAllowlist, }); return jsonResult(result); }, @@ -840,6 +803,4 @@ export const __testing = { resolveGrokModel, resolveGrokInlineCitations, extractGrokContent, - resolveUrlAllowlist, - filterResultsByAllowlist, } as const; diff --git a/src/agents/tools/web-shared.ts b/src/agents/tools/web-shared.ts index 419d8a5cdb1..da0fbb38beb 100644 --- a/src/agents/tools/web-shared.ts +++ b/src/agents/tools/web-shared.ts @@ -8,20 +8,6 @@ export const DEFAULT_TIMEOUT_SECONDS = 30; export const DEFAULT_CACHE_TTL_MINUTES = 15; const DEFAULT_CACHE_MAX_ENTRIES = 100; -export function resolveWebUrlAllowlist(web: unknown): string[] | undefined { - if (!web || typeof web !== "object") { - return undefined; - } - if (!("urlAllowlist" in web)) { - return undefined; - } - const allowlist = (web as { urlAllowlist?: unknown }).urlAllowlist; - if (!Array.isArray(allowlist)) { - return undefined; - } - return allowlist.length > 0 ? allowlist : undefined; -} - export function resolveTimeoutSeconds(value: unknown, fallback: number): number { const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback; return Math.max(1, Math.floor(parsed)); diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 8f21e4aca84..5f9445d47f6 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -116,8 +116,6 @@ export const FIELD_HELP: Record = { "Perplexity base URL override (default: https://openrouter.ai/api/v1 or https://api.perplexity.ai).", "tools.web.search.perplexity.model": 'Perplexity model override (default: "perplexity/sonar-pro").', - "tools.web.urlAllowlist": - "Optional URL/domain allowlist shared by web_search and web_fetch. Accepts domain patterns like 'example.com', '*.github.com'. When configured, only matching URLs are allowed.", "tools.web.fetch.enabled": "Enable the web_fetch tool (lightweight HTTP fetch).", "tools.web.fetch.maxChars": "Max characters returned by web_fetch (truncated).", "tools.web.fetch.maxCharsCap": diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index cd4186e72dc..516baf5a0a6 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -395,8 +395,6 @@ export type ToolsConfig = { /** Optional tool policy overrides keyed by provider id or "provider/model". */ byProvider?: Record; web?: { - /** Optional URL/domain allowlist for web tools. When configured, only URLs matching these patterns are allowed. */ - urlAllowlist?: string[]; search?: { /** Enable web search tool (default: true when API key is present). */ enabled?: boolean; diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 6d65655e233..78b61b9a078 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -267,7 +267,6 @@ export const ToolsWebFetchSchema = z export const ToolsWebSchema = z .object({ - urlAllowlist: z.array(z.string()).optional(), search: ToolsWebSearchSchema, fetch: ToolsWebFetchSchema, }) diff --git a/src/infra/net/ssrf.ts b/src/infra/net/ssrf.ts index 3a7b1946195..fce4204f4ff 100644 --- a/src/infra/net/ssrf.ts +++ b/src/infra/net/ssrf.ts @@ -33,7 +33,7 @@ function normalizeHostnameSet(values?: string[]): Set { return new Set(values.map((value) => normalizeHostname(value)).filter(Boolean)); } -export function normalizeHostnameAllowlist(values?: string[]): string[] { +function normalizeHostnameAllowlist(values?: string[]): string[] { if (!values || values.length === 0) { return []; } @@ -57,7 +57,7 @@ function isHostnameAllowedByPattern(hostname: string, pattern: string): boolean return hostname === pattern; } -export function matchesHostnameAllowlist(hostname: string, allowlist: string[]): boolean { +function matchesHostnameAllowlist(hostname: string, allowlist: string[]): boolean { if (allowlist.length === 0) { return true; }