From 3de973ffff2e6803b85906ba94ed8ace77f3618d Mon Sep 17 00:00:00 2001 From: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:21:44 -0500 Subject: [PATCH] refactor web search provider execution out of core --- extensions/brave/index.ts | 21 +- .../brave/src/brave-web-search-provider.ts | 613 +++++ .../src/firecrawl-search-provider.ts | 6 +- extensions/google/index.ts | 22 +- .../google/src/gemini-web-search-provider.ts | 286 +++ extensions/moonshot/index.ts | 24 +- .../moonshot/src/kimi-web-search-provider.ts | 360 +++ extensions/perplexity/index.ts | 22 +- .../src/perplexity-web-search-provider.ts | 701 ++++++ extensions/perplexity/web-search-provider.ts | 4 + extensions/xai/index.ts | 22 +- .../xai/src/grok-web-search-provider.ts | 303 +++ package.json | 2 + ...check-plugin-extension-import-boundary.mjs | 302 +++ .../check-web-search-provider-boundaries.mjs | 331 +++ src/agents/tools/web-search-core.ts | 2242 ----------------- .../tools/web-search-provider-common.ts | 213 ++ ...ctory.ts => web-search-provider-config.ts} | 58 +- src/agents/tools/web-search.test.ts | 26 +- src/agents/tools/web-search.ts | 124 +- .../tools/web-tools.enabled-defaults.test.ts | 4 +- src/commands/onboard-search.ts | 70 +- src/config/config.web-search-provider.test.ts | 6 + src/plugins/types.ts | 19 + src/plugins/web-search-providers.test.ts | 15 + src/plugins/web-search-providers.ts | 232 +- src/secrets/runtime-web-tools.ts | 141 +- ...n-extension-import-boundary-inventory.json | 538 ++++ ...eb-search-provider-boundary-inventory.json | 32 + test/plugin-extension-import-boundary.test.ts | 79 + test/web-search-provider-boundary.test.ts | 72 + 31 files changed, 4268 insertions(+), 2622 deletions(-) create mode 100644 extensions/brave/src/brave-web-search-provider.ts create mode 100644 extensions/google/src/gemini-web-search-provider.ts create mode 100644 extensions/moonshot/src/kimi-web-search-provider.ts create mode 100644 extensions/perplexity/src/perplexity-web-search-provider.ts create mode 100644 extensions/perplexity/web-search-provider.ts create mode 100644 extensions/xai/src/grok-web-search-provider.ts create mode 100644 scripts/check-plugin-extension-import-boundary.mjs create mode 100644 scripts/check-web-search-provider-boundaries.mjs delete mode 100644 src/agents/tools/web-search-core.ts create mode 100644 src/agents/tools/web-search-provider-common.ts rename src/agents/tools/{web-search-plugin-factory.ts => web-search-provider-config.ts} (65%) create mode 100644 test/fixtures/plugin-extension-import-boundary-inventory.json create mode 100644 test/fixtures/web-search-provider-boundary-inventory.json create mode 100644 test/plugin-extension-import-boundary.test.ts create mode 100644 test/web-search-provider-boundary.test.ts diff --git a/extensions/brave/index.ts b/extensions/brave/index.ts index 1692f2db03f..7ded10c9361 100644 --- a/extensions/brave/index.ts +++ b/extensions/brave/index.ts @@ -1,28 +1,11 @@ import { definePluginEntry } from "openclaw/plugin-sdk/core"; -import { - createPluginBackedWebSearchProvider, - getTopLevelCredentialValue, - setTopLevelCredentialValue, -} from "openclaw/plugin-sdk/provider-web-search"; +import { createBraveWebSearchProvider } from "./src/brave-web-search-provider.js"; export default definePluginEntry({ id: "brave", name: "Brave Plugin", description: "Bundled Brave plugin", register(api) { - api.registerWebSearchProvider( - createPluginBackedWebSearchProvider({ - id: "brave", - label: "Brave Search", - hint: "Structured results · country/language/time filters", - envVars: ["BRAVE_API_KEY"], - placeholder: "BSA...", - signupUrl: "https://brave.com/search/api/", - docsUrl: "https://docs.openclaw.ai/brave-search", - autoDetectOrder: 10, - getCredentialValue: getTopLevelCredentialValue, - setCredentialValue: setTopLevelCredentialValue, - }), - ); + api.registerWebSearchProvider(createBraveWebSearchProvider()); }, }); diff --git a/extensions/brave/src/brave-web-search-provider.ts b/extensions/brave/src/brave-web-search-provider.ts new file mode 100644 index 00000000000..b33f1ab0575 --- /dev/null +++ b/extensions/brave/src/brave-web-search-provider.ts @@ -0,0 +1,613 @@ +import { Type } from "@sinclair/typebox"; +import { readNumberParam, readStringParam } from "../../../src/agents/tools/common.js"; +import type { SearchConfigRecord } from "../../../src/agents/tools/web-search-provider-common.js"; +import { + buildSearchCacheKey, + DEFAULT_SEARCH_COUNT, + MAX_SEARCH_COUNT, + normalizeFreshness, + normalizeToIsoDate, + readCachedSearchPayload, + readConfiguredSecretString, + readProviderEnvValue, + resolveSearchCacheTtlMs, + resolveSearchCount, + resolveSearchTimeoutSeconds, + resolveSiteName, + withTrustedWebSearchEndpoint, + writeCachedSearchPayload, +} from "../../../src/agents/tools/web-search-provider-common.js"; +import { formatCliCommand } from "../../../src/cli/command-format.js"; +import type { + WebSearchProviderPlugin, + WebSearchProviderToolDefinition, +} from "../../../src/plugins/types.js"; +import { wrapWebContent } from "../../../src/security/external-content.js"; + +const BRAVE_SEARCH_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"; +const BRAVE_LLM_CONTEXT_ENDPOINT = "https://api.search.brave.com/res/v1/llm/context"; +const BRAVE_SEARCH_LANG_CODES = new Set([ + "ar", + "eu", + "bn", + "bg", + "ca", + "zh-hans", + "zh-hant", + "hr", + "cs", + "da", + "nl", + "en", + "en-gb", + "et", + "fi", + "fr", + "gl", + "de", + "el", + "gu", + "he", + "hi", + "hu", + "is", + "it", + "jp", + "kn", + "ko", + "lv", + "lt", + "ms", + "ml", + "mr", + "nb", + "pl", + "pt-br", + "pt-pt", + "pa", + "ro", + "ru", + "sr", + "sk", + "sl", + "es", + "sv", + "ta", + "te", + "th", + "tr", + "uk", + "vi", +]); +const BRAVE_SEARCH_LANG_ALIASES: Record = { + ja: "jp", + zh: "zh-hans", + "zh-cn": "zh-hans", + "zh-hk": "zh-hant", + "zh-sg": "zh-hans", + "zh-tw": "zh-hant", +}; +const BRAVE_UI_LANG_LOCALE = /^([a-z]{2})-([a-z]{2})$/i; + +type BraveConfig = { + mode?: string; +}; + +type BraveSearchResult = { + title?: string; + url?: string; + description?: string; + age?: string; +}; + +type BraveSearchResponse = { + web?: { + results?: BraveSearchResult[]; + }; +}; + +type BraveLlmContextResult = { url: string; title: string; snippets: string[] }; +type BraveLlmContextResponse = { + grounding: { generic?: BraveLlmContextResult[] }; + sources?: { url?: string; hostname?: string; date?: string }[]; +}; + +function resolveBraveConfig(searchConfig?: SearchConfigRecord): BraveConfig { + const brave = searchConfig?.brave; + return brave && typeof brave === "object" && !Array.isArray(brave) ? (brave as BraveConfig) : {}; +} + +function resolveBraveMode(brave: BraveConfig): "web" | "llm-context" { + return brave.mode === "llm-context" ? "llm-context" : "web"; +} + +function resolveBraveApiKey(searchConfig?: SearchConfigRecord): string | undefined { + return ( + readConfiguredSecretString(searchConfig?.apiKey, "tools.web.search.apiKey") ?? + readProviderEnvValue(["BRAVE_API_KEY"]) + ); +} + +function normalizeBraveSearchLang(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + const canonical = BRAVE_SEARCH_LANG_ALIASES[trimmed.toLowerCase()] ?? trimmed.toLowerCase(); + if (!BRAVE_SEARCH_LANG_CODES.has(canonical)) { + return undefined; + } + return canonical; +} + +function normalizeBraveUiLang(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + const match = trimmed.match(BRAVE_UI_LANG_LOCALE); + if (!match) { + return undefined; + } + const [, language, region] = match; + return `${language.toLowerCase()}-${region.toUpperCase()}`; +} + +function normalizeBraveLanguageParams(params: { search_lang?: string; ui_lang?: string }): { + search_lang?: string; + ui_lang?: string; + invalidField?: "search_lang" | "ui_lang"; +} { + const rawSearchLang = params.search_lang?.trim() || undefined; + const rawUiLang = params.ui_lang?.trim() || undefined; + let searchLangCandidate = rawSearchLang; + let uiLangCandidate = rawUiLang; + + if (normalizeBraveUiLang(rawSearchLang) && normalizeBraveSearchLang(rawUiLang)) { + searchLangCandidate = rawUiLang; + uiLangCandidate = rawSearchLang; + } + + const search_lang = normalizeBraveSearchLang(searchLangCandidate); + if (searchLangCandidate && !search_lang) { + return { invalidField: "search_lang" }; + } + + const ui_lang = normalizeBraveUiLang(uiLangCandidate); + if (uiLangCandidate && !ui_lang) { + return { invalidField: "ui_lang" }; + } + + return { search_lang, ui_lang }; +} + +function mapBraveLlmContextResults( + data: BraveLlmContextResponse, +): { url: string; title: string; snippets: string[]; siteName?: string }[] { + const genericResults = Array.isArray(data.grounding?.generic) ? data.grounding.generic : []; + return genericResults.map((entry) => ({ + url: entry.url ?? "", + title: entry.title ?? "", + snippets: (entry.snippets ?? []).filter((s) => typeof s === "string" && s.length > 0), + siteName: resolveSiteName(entry.url) || undefined, + })); +} + +async function runBraveLlmContextSearch(params: { + query: string; + apiKey: string; + timeoutSeconds: number; + country?: string; + search_lang?: string; + freshness?: string; +}): Promise<{ + results: Array<{ + url: string; + title: string; + snippets: string[]; + siteName?: string; + }>; + sources?: BraveLlmContextResponse["sources"]; +}> { + const url = new URL(BRAVE_LLM_CONTEXT_ENDPOINT); + url.searchParams.set("q", params.query); + if (params.country) { + url.searchParams.set("country", params.country); + } + if (params.search_lang) { + url.searchParams.set("search_lang", params.search_lang); + } + if (params.freshness) { + url.searchParams.set("freshness", params.freshness); + } + + return withTrustedWebSearchEndpoint( + { + url: url.toString(), + timeoutSeconds: params.timeoutSeconds, + init: { + method: "GET", + headers: { + Accept: "application/json", + "X-Subscription-Token": params.apiKey, + }, + }, + }, + async (res) => { + if (!res.ok) { + const detail = await res.text(); + throw new Error(`Brave LLM Context API error (${res.status}): ${detail || res.statusText}`); + } + + const data = (await res.json()) as BraveLlmContextResponse; + return { results: mapBraveLlmContextResults(data), sources: data.sources }; + }, + ); +} + +async function runBraveWebSearch(params: { + query: string; + count: number; + apiKey: string; + timeoutSeconds: number; + country?: string; + search_lang?: string; + ui_lang?: string; + freshness?: string; + dateAfter?: string; + dateBefore?: string; +}): Promise>> { + const url = new URL(BRAVE_SEARCH_ENDPOINT); + url.searchParams.set("q", params.query); + url.searchParams.set("count", String(params.count)); + if (params.country) { + url.searchParams.set("country", params.country); + } + if (params.search_lang) { + url.searchParams.set("search_lang", params.search_lang); + } + if (params.ui_lang) { + url.searchParams.set("ui_lang", params.ui_lang); + } + if (params.freshness) { + url.searchParams.set("freshness", params.freshness); + } else if (params.dateAfter && params.dateBefore) { + url.searchParams.set("freshness", `${params.dateAfter}to${params.dateBefore}`); + } else if (params.dateAfter) { + url.searchParams.set( + "freshness", + `${params.dateAfter}to${new Date().toISOString().slice(0, 10)}`, + ); + } else if (params.dateBefore) { + url.searchParams.set("freshness", `1970-01-01to${params.dateBefore}`); + } + + return withTrustedWebSearchEndpoint( + { + url: url.toString(), + timeoutSeconds: params.timeoutSeconds, + init: { + method: "GET", + headers: { + Accept: "application/json", + "X-Subscription-Token": params.apiKey, + }, + }, + }, + async (res) => { + if (!res.ok) { + const detail = await res.text(); + throw new Error(`Brave Search API error (${res.status}): ${detail || res.statusText}`); + } + + const data = (await res.json()) as BraveSearchResponse; + const results = Array.isArray(data.web?.results) ? (data.web?.results ?? []) : []; + return results.map((entry) => { + const description = entry.description ?? ""; + const title = entry.title ?? ""; + const url = entry.url ?? ""; + return { + title: title ? wrapWebContent(title, "web_search") : "", + url, + description: description ? wrapWebContent(description, "web_search") : "", + published: entry.age || undefined, + siteName: resolveSiteName(url) || undefined, + }; + }); + }, + ); +} + +function createBraveSchema() { + return Type.Object({ + query: Type.String({ description: "Search query string." }), + count: Type.Optional( + Type.Number({ + description: "Number of results to return (1-10).", + minimum: 1, + maximum: MAX_SEARCH_COUNT, + }), + ), + country: Type.Optional( + Type.String({ + description: + "2-letter country code for region-specific results (e.g., 'DE', 'US', 'ALL'). Default: 'US'.", + }), + ), + language: Type.Optional( + Type.String({ + description: "ISO 639-1 language code for results (e.g., 'en', 'de', 'fr').", + }), + ), + freshness: Type.Optional( + Type.String({ + description: "Filter by time: 'day' (24h), 'week', 'month', or 'year'.", + }), + ), + date_after: Type.Optional( + Type.String({ + description: "Only results published after this date (YYYY-MM-DD).", + }), + ), + date_before: Type.Optional( + Type.String({ + description: "Only results published before this date (YYYY-MM-DD).", + }), + ), + search_lang: Type.Optional( + Type.String({ + description: + "Brave language code for search results (e.g., 'en', 'de', 'en-gb', 'zh-hans', 'zh-hant', 'pt-br').", + }), + ), + ui_lang: Type.Optional( + Type.String({ + description: + "Locale code for UI elements in language-region format (e.g., 'en-US', 'de-DE', 'fr-FR', 'tr-TR'). Must include region subtag.", + }), + ), + }); +} + +function missingBraveKeyPayload() { + return { + error: "missing_brave_api_key", + message: `web_search (brave) needs a Brave Search API key. Run \`${formatCliCommand("openclaw configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`, + docs: "https://docs.openclaw.ai/tools/web", + }; +} + +function createBraveToolDefinition( + searchConfig?: SearchConfigRecord, +): WebSearchProviderToolDefinition { + const braveConfig = resolveBraveConfig(searchConfig); + const braveMode = resolveBraveMode(braveConfig); + + return { + description: + braveMode === "llm-context" + ? "Search the web using Brave Search LLM Context API. Returns pre-extracted page content (text chunks, tables, code blocks) optimized for LLM grounding." + : "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research.", + parameters: createBraveSchema(), + execute: async (args) => { + const apiKey = resolveBraveApiKey(searchConfig); + if (!apiKey) { + return missingBraveKeyPayload(); + } + + const params = args as Record; + const query = readStringParam(params, "query", { required: true }); + const count = + readNumberParam(params, "count", { integer: true }) ?? + searchConfig?.maxResults ?? + undefined; + const country = readStringParam(params, "country"); + const language = readStringParam(params, "language"); + const search_lang = readStringParam(params, "search_lang"); + const ui_lang = readStringParam(params, "ui_lang"); + const normalizedLanguage = normalizeBraveLanguageParams({ + search_lang: search_lang || language, + ui_lang, + }); + if (normalizedLanguage.invalidField === "search_lang") { + return { + error: "invalid_search_lang", + message: + "search_lang must be a Brave-supported language code like 'en', 'en-gb', 'zh-hans', or 'zh-hant'.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (normalizedLanguage.invalidField === "ui_lang") { + return { + error: "invalid_ui_lang", + message: "ui_lang must be a language-region locale like 'en-US'.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (normalizedLanguage.ui_lang && braveMode === "llm-context") { + return { + error: "unsupported_ui_lang", + message: + "ui_lang is not supported by Brave llm-context mode. Remove ui_lang or use Brave web mode for locale-based UI hints.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const rawFreshness = readStringParam(params, "freshness"); + if (rawFreshness && braveMode === "llm-context") { + return { + error: "unsupported_freshness", + message: + "freshness filtering is not supported by Brave llm-context mode. Remove freshness or use Brave web mode.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + const freshness = rawFreshness ? normalizeFreshness(rawFreshness, "brave") : undefined; + if (rawFreshness && !freshness) { + return { + error: "invalid_freshness", + message: "freshness must be day, week, month, or year.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const rawDateAfter = readStringParam(params, "date_after"); + const rawDateBefore = readStringParam(params, "date_before"); + if (rawFreshness && (rawDateAfter || rawDateBefore)) { + return { + error: "conflicting_time_filters", + message: + "freshness and date_after/date_before cannot be used together. Use either freshness (day/week/month/year) or a date range (date_after/date_before), not both.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if ((rawDateAfter || rawDateBefore) && braveMode === "llm-context") { + return { + error: "unsupported_date_filter", + message: + "date_after/date_before filtering is not supported by Brave llm-context mode. Use Brave web mode for date filters.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + const dateAfter = rawDateAfter ? normalizeToIsoDate(rawDateAfter) : undefined; + if (rawDateAfter && !dateAfter) { + return { + error: "invalid_date", + message: "date_after must be YYYY-MM-DD format.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + const dateBefore = rawDateBefore ? normalizeToIsoDate(rawDateBefore) : undefined; + if (rawDateBefore && !dateBefore) { + return { + error: "invalid_date", + message: "date_before must be YYYY-MM-DD format.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (dateAfter && dateBefore && dateAfter > dateBefore) { + return { + error: "invalid_date_range", + message: "date_after must be before date_before.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const cacheKey = buildSearchCacheKey([ + "brave", + braveMode, + query, + resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + country, + normalizedLanguage.search_lang, + normalizedLanguage.ui_lang, + freshness, + dateAfter, + dateBefore, + ]); + const cached = readCachedSearchPayload(cacheKey); + if (cached) { + return cached; + } + + const start = Date.now(); + const timeoutSeconds = resolveSearchTimeoutSeconds(searchConfig); + const cacheTtlMs = resolveSearchCacheTtlMs(searchConfig); + + if (braveMode === "llm-context") { + const { results, sources } = await runBraveLlmContextSearch({ + query, + apiKey, + timeoutSeconds, + country: country ?? undefined, + search_lang: normalizedLanguage.search_lang, + freshness, + }); + const payload = { + query, + provider: "brave", + mode: "llm-context" as const, + count: results.length, + tookMs: Date.now() - start, + externalContent: { + untrusted: true, + source: "web_search", + provider: "brave", + wrapped: true, + }, + results: results.map((entry) => ({ + title: entry.title ? wrapWebContent(entry.title, "web_search") : "", + url: entry.url, + snippets: entry.snippets.map((snippet) => wrapWebContent(snippet, "web_search")), + siteName: entry.siteName, + })), + sources, + }; + writeCachedSearchPayload(cacheKey, payload, cacheTtlMs); + return payload; + } + + const results = await runBraveWebSearch({ + query, + count: resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + apiKey, + timeoutSeconds, + country: country ?? undefined, + search_lang: normalizedLanguage.search_lang, + ui_lang: normalizedLanguage.ui_lang, + freshness, + dateAfter, + dateBefore, + }); + const payload = { + query, + provider: "brave", + count: results.length, + tookMs: Date.now() - start, + externalContent: { + untrusted: true, + source: "web_search", + provider: "brave", + wrapped: true, + }, + results, + }; + writeCachedSearchPayload(cacheKey, payload, cacheTtlMs); + return payload; + }, + }; +} + +export function createBraveWebSearchProvider(): WebSearchProviderPlugin { + return { + id: "brave", + label: "Brave Search", + hint: "Structured results · country/language/time filters", + envVars: ["BRAVE_API_KEY"], + placeholder: "BSA...", + signupUrl: "https://brave.com/search/api/", + docsUrl: "https://docs.openclaw.ai/brave-search", + autoDetectOrder: 10, + credentialPath: "tools.web.search.apiKey", + inactiveSecretPaths: ["tools.web.search.apiKey"], + getCredentialValue: (searchConfig) => searchConfig?.apiKey, + setCredentialValue: (searchConfigTarget, value) => { + searchConfigTarget.apiKey = value; + }, + createTool: (ctx) => + createBraveToolDefinition(ctx.searchConfig as SearchConfigRecord | undefined), + }; +} + +export const __testing = { + normalizeFreshness, + normalizeBraveLanguageParams, + resolveBraveMode, + mapBraveLlmContextResults, +} as const; diff --git a/extensions/firecrawl/src/firecrawl-search-provider.ts b/extensions/firecrawl/src/firecrawl-search-provider.ts index 0940aedb74d..dd222b02d63 100644 --- a/extensions/firecrawl/src/firecrawl-search-provider.ts +++ b/extensions/firecrawl/src/firecrawl-search-provider.ts @@ -1,5 +1,6 @@ import { Type } from "@sinclair/typebox"; -import type { WebSearchProviderPlugin } from "openclaw/plugin-sdk/plugin-runtime"; +import { enablePluginInConfig } from "../../../src/plugins/enable.js"; +import type { WebSearchProviderPlugin } from "../../../src/plugins/types.js"; import { runFirecrawlSearch } from "./firecrawl-client.js"; const GenericFirecrawlSearchSchema = Type.Object( @@ -46,8 +47,11 @@ export function createFirecrawlWebSearchProvider(): WebSearchProviderPlugin { signupUrl: "https://www.firecrawl.dev/", docsUrl: "https://docs.openclaw.ai/tools/firecrawl", autoDetectOrder: 60, + credentialPath: "tools.web.search.firecrawl.apiKey", + inactiveSecretPaths: ["tools.web.search.firecrawl.apiKey"], getCredentialValue: getScopedCredentialValue, setCredentialValue: setScopedCredentialValue, + applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config, createTool: (ctx) => ({ description: "Search the web using Firecrawl. Returns structured results with snippets from Firecrawl Search. Use firecrawl_search for Firecrawl-specific knobs like sources or categories.", diff --git a/extensions/google/index.ts b/extensions/google/index.ts index f9268cc0aae..e168a346d70 100644 --- a/extensions/google/index.ts +++ b/extensions/google/index.ts @@ -5,14 +5,10 @@ import { GOOGLE_GEMINI_DEFAULT_MODEL, applyGoogleGeminiModelDefault, } from "openclaw/plugin-sdk/provider-models"; -import { - createPluginBackedWebSearchProvider, - getScopedCredentialValue, - setScopedCredentialValue, -} from "openclaw/plugin-sdk/provider-web-search"; import { registerGoogleGeminiCliProvider } from "./gemini-cli-provider.js"; import { googleMediaUnderstandingProvider } from "./media-understanding-provider.js"; import { isModernGoogleModel, resolveGoogle31ForwardCompatModel } from "./provider-models.js"; +import { createGeminiWebSearchProvider } from "./src/gemini-web-search-provider.js"; export default definePluginEntry({ id: "google", @@ -53,20 +49,6 @@ export default definePluginEntry({ registerGoogleGeminiCliProvider(api); api.registerImageGenerationProvider(buildGoogleImageGenerationProvider()); api.registerMediaUnderstandingProvider(googleMediaUnderstandingProvider); - api.registerWebSearchProvider( - createPluginBackedWebSearchProvider({ - id: "gemini", - label: "Gemini (Google Search)", - hint: "Google Search grounding · AI-synthesized", - envVars: ["GEMINI_API_KEY"], - placeholder: "AIza...", - signupUrl: "https://aistudio.google.com/apikey", - docsUrl: "https://docs.openclaw.ai/tools/web", - autoDetectOrder: 20, - getCredentialValue: (searchConfig) => getScopedCredentialValue(searchConfig, "gemini"), - setCredentialValue: (searchConfigTarget, value) => - setScopedCredentialValue(searchConfigTarget, "gemini", value), - }), - ); + api.registerWebSearchProvider(createGeminiWebSearchProvider()); }, }); diff --git a/extensions/google/src/gemini-web-search-provider.ts b/extensions/google/src/gemini-web-search-provider.ts new file mode 100644 index 00000000000..85325cd132e --- /dev/null +++ b/extensions/google/src/gemini-web-search-provider.ts @@ -0,0 +1,286 @@ +import { Type } from "@sinclair/typebox"; +import { readNumberParam, readStringParam } from "../../../src/agents/tools/common.js"; +import { resolveCitationRedirectUrl } from "../../../src/agents/tools/web-search-citation-redirect.js"; +import type { SearchConfigRecord } from "../../../src/agents/tools/web-search-provider-common.js"; +import { + buildSearchCacheKey, + DEFAULT_SEARCH_COUNT, + MAX_SEARCH_COUNT, + readCachedSearchPayload, + readConfiguredSecretString, + readProviderEnvValue, + resolveSearchCacheTtlMs, + resolveSearchCount, + resolveSearchTimeoutSeconds, + withTrustedWebSearchEndpoint, + writeCachedSearchPayload, +} from "../../../src/agents/tools/web-search-provider-common.js"; +import type { + WebSearchProviderPlugin, + WebSearchProviderToolDefinition, +} from "../../../src/plugins/types.js"; +import { wrapWebContent } from "../../../src/security/external-content.js"; + +const DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"; +const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; + +type GeminiConfig = { + apiKey?: string; + model?: string; +}; + +type GeminiGroundingResponse = { + candidates?: Array<{ + content?: { + parts?: Array<{ + text?: string; + }>; + }; + groundingMetadata?: { + groundingChunks?: Array<{ + web?: { + uri?: string; + title?: string; + }; + }>; + }; + }>; + error?: { + code?: number; + message?: string; + status?: string; + }; +}; + +function resolveGeminiConfig(searchConfig?: SearchConfigRecord): GeminiConfig { + const gemini = searchConfig?.gemini; + return gemini && typeof gemini === "object" && !Array.isArray(gemini) + ? (gemini as GeminiConfig) + : {}; +} + +function resolveGeminiApiKey(gemini?: GeminiConfig): string | undefined { + return ( + readConfiguredSecretString(gemini?.apiKey, "tools.web.search.gemini.apiKey") ?? + readProviderEnvValue(["GEMINI_API_KEY"]) + ); +} + +function resolveGeminiModel(gemini?: GeminiConfig): string { + const model = typeof gemini?.model === "string" ? gemini.model.trim() : ""; + return model || DEFAULT_GEMINI_MODEL; +} + +async function runGeminiSearch(params: { + query: string; + apiKey: string; + model: string; + timeoutSeconds: number; +}): Promise<{ content: string; citations: Array<{ url: string; title?: string }> }> { + const endpoint = `${GEMINI_API_BASE}/models/${params.model}:generateContent`; + + return withTrustedWebSearchEndpoint( + { + url: endpoint, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-goog-api-key": params.apiKey, + }, + body: JSON.stringify({ + contents: [{ parts: [{ text: params.query }] }], + tools: [{ google_search: {} }], + }), + }, + }, + async (res) => { + if (!res.ok) { + const safeDetail = ((await res.text()) || res.statusText).replace( + /key=[^&\s]+/gi, + "key=***", + ); + throw new Error(`Gemini API error (${res.status}): ${safeDetail}`); + } + + let data: GeminiGroundingResponse; + try { + data = (await res.json()) as GeminiGroundingResponse; + } catch (error) { + const safeError = String(error).replace(/key=[^&\s]+/gi, "key=***"); + throw new Error(`Gemini API returned invalid JSON: ${safeError}`, { cause: error }); + } + + if (data.error) { + const rawMessage = data.error.message || data.error.status || "unknown"; + throw new Error( + `Gemini API error (${data.error.code}): ${rawMessage.replace(/key=[^&\s]+/gi, "key=***")}`, + ); + } + + const candidate = data.candidates?.[0]; + const content = + candidate?.content?.parts + ?.map((part) => part.text) + .filter(Boolean) + .join("\n") ?? "No response"; + const rawCitations = (candidate?.groundingMetadata?.groundingChunks ?? []) + .filter((chunk) => chunk.web?.uri) + .map((chunk) => ({ + url: chunk.web!.uri!, + title: chunk.web?.title || undefined, + })); + + const citations: Array<{ url: string; title?: string }> = []; + for (let index = 0; index < rawCitations.length; index += 10) { + const batch = rawCitations.slice(index, index + 10); + const resolved = await Promise.all( + batch.map(async (citation) => ({ + ...citation, + url: await resolveCitationRedirectUrl(citation.url), + })), + ); + citations.push(...resolved); + } + + return { content, citations }; + }, + ); +} + +function createGeminiSchema() { + return Type.Object({ + query: Type.String({ description: "Search query string." }), + count: Type.Optional( + Type.Number({ + description: "Number of results to return (1-10).", + minimum: 1, + maximum: MAX_SEARCH_COUNT, + }), + ), + country: Type.Optional(Type.String({ description: "Not supported by Gemini." })), + language: Type.Optional(Type.String({ description: "Not supported by Gemini." })), + freshness: Type.Optional(Type.String({ description: "Not supported by Gemini." })), + date_after: Type.Optional(Type.String({ description: "Not supported by Gemini." })), + date_before: Type.Optional(Type.String({ description: "Not supported by Gemini." })), + }); +} + +function createGeminiToolDefinition( + searchConfig?: SearchConfigRecord, +): WebSearchProviderToolDefinition { + return { + description: + "Search the web using Gemini with Google Search grounding. Returns AI-synthesized answers with citations from Google Search.", + parameters: createGeminiSchema(), + execute: async (args) => { + const params = args as Record; + for (const name of ["country", "language", "freshness", "date_after", "date_before"]) { + if (readStringParam(params, name)) { + const label = + name === "country" + ? "country filtering" + : name === "language" + ? "language filtering" + : name === "freshness" + ? "freshness filtering" + : "date_after/date_before filtering"; + return { + error: name.startsWith("date_") ? "unsupported_date_filter" : `unsupported_${name}`, + message: `${label} is not supported by the gemini provider. Only Brave and Perplexity support ${name === "country" ? "country filtering" : name === "language" ? "language filtering" : name === "freshness" ? "freshness" : "date filtering"}.`, + docs: "https://docs.openclaw.ai/tools/web", + }; + } + } + + const geminiConfig = resolveGeminiConfig(searchConfig); + const apiKey = resolveGeminiApiKey(geminiConfig); + if (!apiKey) { + return { + error: "missing_gemini_api_key", + message: + "web_search (gemini) needs an API key. Set GEMINI_API_KEY in the Gateway environment, or configure tools.web.search.gemini.apiKey.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const query = readStringParam(params, "query", { required: true }); + const count = + readNumberParam(params, "count", { integer: true }) ?? + searchConfig?.maxResults ?? + undefined; + const model = resolveGeminiModel(geminiConfig); + const cacheKey = buildSearchCacheKey([ + "gemini", + query, + resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + model, + ]); + const cached = readCachedSearchPayload(cacheKey); + if (cached) { + return cached; + } + + const start = Date.now(); + const result = await runGeminiSearch({ + query, + apiKey, + model, + timeoutSeconds: resolveSearchTimeoutSeconds(searchConfig), + }); + const payload = { + query, + provider: "gemini", + model, + tookMs: Date.now() - start, + externalContent: { + untrusted: true, + source: "web_search", + provider: "gemini", + wrapped: true, + }, + content: wrapWebContent(result.content), + citations: result.citations, + }; + writeCachedSearchPayload(cacheKey, payload, resolveSearchCacheTtlMs(searchConfig)); + return payload; + }, + }; +} + +export function createGeminiWebSearchProvider(): WebSearchProviderPlugin { + return { + id: "gemini", + label: "Gemini (Google Search)", + hint: "Google Search grounding · AI-synthesized", + envVars: ["GEMINI_API_KEY"], + placeholder: "AIza...", + signupUrl: "https://aistudio.google.com/apikey", + docsUrl: "https://docs.openclaw.ai/tools/web", + autoDetectOrder: 20, + credentialPath: "tools.web.search.gemini.apiKey", + inactiveSecretPaths: ["tools.web.search.gemini.apiKey"], + getCredentialValue: (searchConfig) => { + const gemini = searchConfig?.gemini; + return gemini && typeof gemini === "object" && !Array.isArray(gemini) + ? (gemini as Record).apiKey + : undefined; + }, + setCredentialValue: (searchConfigTarget, value) => { + const scoped = searchConfigTarget.gemini; + if (!scoped || typeof scoped !== "object" || Array.isArray(scoped)) { + searchConfigTarget.gemini = { apiKey: value }; + return; + } + (scoped as Record).apiKey = value; + }, + createTool: (ctx) => + createGeminiToolDefinition(ctx.searchConfig as SearchConfigRecord | undefined), + }; +} + +export const __testing = { + resolveGeminiApiKey, + resolveGeminiModel, +} as const; diff --git a/extensions/moonshot/index.ts b/extensions/moonshot/index.ts index 704b841818c..3a8ed52c805 100644 --- a/extensions/moonshot/index.ts +++ b/extensions/moonshot/index.ts @@ -1,15 +1,10 @@ import { definePluginEntry } from "openclaw/plugin-sdk/core"; -import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth"; import { buildSingleProviderApiKeyCatalog } from "openclaw/plugin-sdk/provider-catalog"; +import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth"; import { createMoonshotThinkingWrapper, resolveMoonshotThinkingType, } from "openclaw/plugin-sdk/provider-stream"; -import { - createPluginBackedWebSearchProvider, - getScopedCredentialValue, - setScopedCredentialValue, -} from "openclaw/plugin-sdk/provider-web-search"; import { moonshotMediaUnderstandingProvider } from "./media-understanding-provider.js"; import { applyMoonshotConfig, @@ -17,6 +12,7 @@ import { MOONSHOT_DEFAULT_MODEL_REF, } from "./onboard.js"; import { buildMoonshotProvider } from "./provider-catalog.js"; +import { createKimiWebSearchProvider } from "./src/kimi-web-search-provider.js"; const PROVIDER_ID = "moonshot"; @@ -91,20 +87,6 @@ export default definePluginEntry({ }, }); api.registerMediaUnderstandingProvider(moonshotMediaUnderstandingProvider); - api.registerWebSearchProvider( - createPluginBackedWebSearchProvider({ - id: "kimi", - label: "Kimi (Moonshot)", - hint: "Moonshot web search", - envVars: ["KIMI_API_KEY", "MOONSHOT_API_KEY"], - placeholder: "sk-...", - signupUrl: "https://platform.moonshot.cn/", - docsUrl: "https://docs.openclaw.ai/tools/web", - autoDetectOrder: 40, - getCredentialValue: (searchConfig) => getScopedCredentialValue(searchConfig, "kimi"), - setCredentialValue: (searchConfigTarget, value) => - setScopedCredentialValue(searchConfigTarget, "kimi", value), - }), - ); + api.registerWebSearchProvider(createKimiWebSearchProvider()); }, }); diff --git a/extensions/moonshot/src/kimi-web-search-provider.ts b/extensions/moonshot/src/kimi-web-search-provider.ts new file mode 100644 index 00000000000..f71a889bb1f --- /dev/null +++ b/extensions/moonshot/src/kimi-web-search-provider.ts @@ -0,0 +1,360 @@ +import { Type } from "@sinclair/typebox"; +import { readNumberParam, readStringParam } from "../../../src/agents/tools/common.js"; +import type { SearchConfigRecord } from "../../../src/agents/tools/web-search-provider-common.js"; +import { + buildSearchCacheKey, + DEFAULT_SEARCH_COUNT, + MAX_SEARCH_COUNT, + readCachedSearchPayload, + readConfiguredSecretString, + readProviderEnvValue, + resolveSearchCacheTtlMs, + resolveSearchCount, + resolveSearchTimeoutSeconds, + withTrustedWebSearchEndpoint, + writeCachedSearchPayload, +} from "../../../src/agents/tools/web-search-provider-common.js"; +import type { + WebSearchProviderPlugin, + WebSearchProviderToolDefinition, +} from "../../../src/plugins/types.js"; +import { wrapWebContent } from "../../../src/security/external-content.js"; + +const DEFAULT_KIMI_BASE_URL = "https://api.moonshot.ai/v1"; +const DEFAULT_KIMI_MODEL = "moonshot-v1-128k"; +const KIMI_WEB_SEARCH_TOOL = { + type: "builtin_function", + function: { name: "$web_search" }, +} as const; + +type KimiConfig = { + apiKey?: string; + baseUrl?: string; + model?: string; +}; + +type KimiToolCall = { + id?: string; + type?: string; + function?: { + name?: string; + arguments?: string; + }; +}; + +type KimiMessage = { + role?: string; + content?: string; + reasoning_content?: string; + tool_calls?: KimiToolCall[]; +}; + +type KimiSearchResponse = { + choices?: Array<{ + finish_reason?: string; + message?: KimiMessage; + }>; + search_results?: Array<{ + title?: string; + url?: string; + content?: string; + }>; +}; + +function resolveKimiConfig(searchConfig?: SearchConfigRecord): KimiConfig { + const kimi = searchConfig?.kimi; + return kimi && typeof kimi === "object" && !Array.isArray(kimi) ? (kimi as KimiConfig) : {}; +} + +function resolveKimiApiKey(kimi?: KimiConfig): string | undefined { + return ( + readConfiguredSecretString(kimi?.apiKey, "tools.web.search.kimi.apiKey") ?? + readProviderEnvValue(["KIMI_API_KEY", "MOONSHOT_API_KEY"]) + ); +} + +function resolveKimiModel(kimi?: KimiConfig): string { + const model = typeof kimi?.model === "string" ? kimi.model.trim() : ""; + return model || DEFAULT_KIMI_MODEL; +} + +function resolveKimiBaseUrl(kimi?: KimiConfig): string { + const baseUrl = typeof kimi?.baseUrl === "string" ? kimi.baseUrl.trim() : ""; + return baseUrl || DEFAULT_KIMI_BASE_URL; +} + +function extractKimiMessageText(message: KimiMessage | undefined): string | undefined { + const content = message?.content?.trim(); + if (content) { + return content; + } + const reasoning = message?.reasoning_content?.trim(); + return reasoning || undefined; +} + +function extractKimiCitations(data: KimiSearchResponse): string[] { + const citations = (data.search_results ?? []) + .map((entry) => entry.url?.trim()) + .filter((url): url is string => Boolean(url)); + + for (const toolCall of data.choices?.[0]?.message?.tool_calls ?? []) { + const rawArguments = toolCall.function?.arguments; + if (!rawArguments) { + continue; + } + try { + const parsed = JSON.parse(rawArguments) as { + search_results?: Array<{ url?: string }>; + url?: string; + }; + if (typeof parsed.url === "string" && parsed.url.trim()) { + citations.push(parsed.url.trim()); + } + for (const result of parsed.search_results ?? []) { + if (typeof result.url === "string" && result.url.trim()) { + citations.push(result.url.trim()); + } + } + } catch { + // ignore malformed tool arguments + } + } + + return [...new Set(citations)]; +} + +function buildKimiToolResultContent(data: KimiSearchResponse): string { + return JSON.stringify({ + search_results: (data.search_results ?? []).map((entry) => ({ + title: entry.title ?? "", + url: entry.url ?? "", + content: entry.content ?? "", + })), + }); +} + +async function runKimiSearch(params: { + query: string; + apiKey: string; + baseUrl: string; + model: string; + timeoutSeconds: number; +}): Promise<{ content: string; citations: string[] }> { + const endpoint = `${params.baseUrl.trim().replace(/\/$/, "")}/chat/completions`; + const messages: Array> = [{ role: "user", content: params.query }]; + const collectedCitations = new Set(); + + for (let round = 0; round < 3; round += 1) { + const next = await withTrustedWebSearchEndpoint( + { + url: endpoint, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + }, + body: JSON.stringify({ + model: params.model, + messages, + tools: [KIMI_WEB_SEARCH_TOOL], + }), + }, + }, + async ( + res, + ): Promise<{ done: true; content: string; citations: string[] } | { done: false }> => { + if (!res.ok) { + const detail = await res.text(); + throw new Error(`Kimi API error (${res.status}): ${detail || res.statusText}`); + } + + const data = (await res.json()) as KimiSearchResponse; + for (const citation of extractKimiCitations(data)) { + collectedCitations.add(citation); + } + const choice = data.choices?.[0]; + const message = choice?.message; + const text = extractKimiMessageText(message); + const toolCalls = message?.tool_calls ?? []; + + if (choice?.finish_reason !== "tool_calls" || toolCalls.length === 0) { + return { done: true, content: text ?? "No response", citations: [...collectedCitations] }; + } + + messages.push({ + role: "assistant", + content: message?.content ?? "", + ...(message?.reasoning_content ? { reasoning_content: message.reasoning_content } : {}), + tool_calls: toolCalls, + }); + + const toolContent = buildKimiToolResultContent(data); + let pushed = false; + for (const toolCall of toolCalls) { + const toolCallId = toolCall.id?.trim(); + if (!toolCallId) { + continue; + } + pushed = true; + messages.push({ role: "tool", tool_call_id: toolCallId, content: toolContent }); + } + if (!pushed) { + return { done: true, content: text ?? "No response", citations: [...collectedCitations] }; + } + return { done: false }; + }, + ); + + if (next.done) { + return { content: next.content, citations: next.citations }; + } + } + + return { + content: "Search completed but no final answer was produced.", + citations: [...collectedCitations], + }; +} + +function createKimiSchema() { + return Type.Object({ + query: Type.String({ description: "Search query string." }), + count: Type.Optional( + Type.Number({ + description: "Number of results to return (1-10).", + minimum: 1, + maximum: MAX_SEARCH_COUNT, + }), + ), + country: Type.Optional(Type.String({ description: "Not supported by Kimi." })), + language: Type.Optional(Type.String({ description: "Not supported by Kimi." })), + freshness: Type.Optional(Type.String({ description: "Not supported by Kimi." })), + date_after: Type.Optional(Type.String({ description: "Not supported by Kimi." })), + date_before: Type.Optional(Type.String({ description: "Not supported by Kimi." })), + }); +} + +function createKimiToolDefinition( + searchConfig?: SearchConfigRecord, +): WebSearchProviderToolDefinition { + return { + description: + "Search the web using Kimi by Moonshot. Returns AI-synthesized answers with citations from native $web_search.", + parameters: createKimiSchema(), + execute: async (args) => { + const params = args as Record; + for (const name of ["country", "language", "freshness", "date_after", "date_before"]) { + if (readStringParam(params, name)) { + const label = + name === "country" + ? "country filtering" + : name === "language" + ? "language filtering" + : name === "freshness" + ? "freshness filtering" + : "date_after/date_before filtering"; + return { + error: name.startsWith("date_") ? "unsupported_date_filter" : `unsupported_${name}`, + message: `${label} is not supported by the kimi provider. Only Brave and Perplexity support ${name === "country" ? "country filtering" : name === "language" ? "language filtering" : name === "freshness" ? "freshness" : "date filtering"}.`, + docs: "https://docs.openclaw.ai/tools/web", + }; + } + } + + const kimiConfig = resolveKimiConfig(searchConfig); + const apiKey = resolveKimiApiKey(kimiConfig); + if (!apiKey) { + return { + error: "missing_kimi_api_key", + message: + "web_search (kimi) needs a Moonshot API key. Set KIMI_API_KEY or MOONSHOT_API_KEY in the Gateway environment, or configure tools.web.search.kimi.apiKey.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const query = readStringParam(params, "query", { required: true }); + const count = + readNumberParam(params, "count", { integer: true }) ?? + searchConfig?.maxResults ?? + undefined; + const model = resolveKimiModel(kimiConfig); + const baseUrl = resolveKimiBaseUrl(kimiConfig); + const cacheKey = buildSearchCacheKey([ + "kimi", + query, + resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + baseUrl, + model, + ]); + const cached = readCachedSearchPayload(cacheKey); + if (cached) { + return cached; + } + + const start = Date.now(); + const result = await runKimiSearch({ + query, + apiKey, + baseUrl, + model, + timeoutSeconds: resolveSearchTimeoutSeconds(searchConfig), + }); + const payload = { + query, + provider: "kimi", + model, + tookMs: Date.now() - start, + externalContent: { + untrusted: true, + source: "web_search", + provider: "kimi", + wrapped: true, + }, + content: wrapWebContent(result.content), + citations: result.citations, + }; + writeCachedSearchPayload(cacheKey, payload, resolveSearchCacheTtlMs(searchConfig)); + return payload; + }, + }; +} + +export function createKimiWebSearchProvider(): WebSearchProviderPlugin { + return { + id: "kimi", + label: "Kimi (Moonshot)", + hint: "Moonshot web search", + envVars: ["KIMI_API_KEY", "MOONSHOT_API_KEY"], + placeholder: "sk-...", + signupUrl: "https://platform.moonshot.cn/", + docsUrl: "https://docs.openclaw.ai/tools/web", + autoDetectOrder: 40, + credentialPath: "tools.web.search.kimi.apiKey", + inactiveSecretPaths: ["tools.web.search.kimi.apiKey"], + getCredentialValue: (searchConfig) => { + const kimi = searchConfig?.kimi; + return kimi && typeof kimi === "object" && !Array.isArray(kimi) + ? (kimi as Record).apiKey + : undefined; + }, + setCredentialValue: (searchConfigTarget, value) => { + const scoped = searchConfigTarget.kimi; + if (!scoped || typeof scoped !== "object" || Array.isArray(scoped)) { + searchConfigTarget.kimi = { apiKey: value }; + return; + } + (scoped as Record).apiKey = value; + }, + createTool: (ctx) => + createKimiToolDefinition(ctx.searchConfig as SearchConfigRecord | undefined), + }; +} + +export const __testing = { + resolveKimiApiKey, + resolveKimiModel, + resolveKimiBaseUrl, + extractKimiCitations, +} as const; diff --git a/extensions/perplexity/index.ts b/extensions/perplexity/index.ts index 95ae612ed35..45ed12139f7 100644 --- a/extensions/perplexity/index.ts +++ b/extensions/perplexity/index.ts @@ -1,29 +1,11 @@ import { definePluginEntry } from "openclaw/plugin-sdk/core"; -import { - createPluginBackedWebSearchProvider, - getScopedCredentialValue, - setScopedCredentialValue, -} from "openclaw/plugin-sdk/provider-web-search"; +import { createPerplexityWebSearchProvider } from "./src/perplexity-web-search-provider.js"; export default definePluginEntry({ id: "perplexity", name: "Perplexity Plugin", description: "Bundled Perplexity plugin", register(api) { - api.registerWebSearchProvider( - createPluginBackedWebSearchProvider({ - id: "perplexity", - label: "Perplexity Search", - hint: "Structured results · domain/country/language/time filters", - envVars: ["PERPLEXITY_API_KEY", "OPENROUTER_API_KEY"], - placeholder: "pplx-...", - signupUrl: "https://www.perplexity.ai/settings/api", - docsUrl: "https://docs.openclaw.ai/perplexity", - autoDetectOrder: 50, - getCredentialValue: (searchConfig) => getScopedCredentialValue(searchConfig, "perplexity"), - setCredentialValue: (searchConfigTarget, value) => - setScopedCredentialValue(searchConfigTarget, "perplexity", value), - }), - ); + api.registerWebSearchProvider(createPerplexityWebSearchProvider()); }, }); diff --git a/extensions/perplexity/src/perplexity-web-search-provider.ts b/extensions/perplexity/src/perplexity-web-search-provider.ts new file mode 100644 index 00000000000..4b2a618ea39 --- /dev/null +++ b/extensions/perplexity/src/perplexity-web-search-provider.ts @@ -0,0 +1,701 @@ +import { Type } from "@sinclair/typebox"; +import { + readNumberParam, + readStringArrayParam, + readStringParam, +} from "../../../src/agents/tools/common.js"; +import type { SearchConfigRecord } from "../../../src/agents/tools/web-search-provider-common.js"; +import { + buildSearchCacheKey, + DEFAULT_SEARCH_COUNT, + MAX_SEARCH_COUNT, + isoToPerplexityDate, + normalizeFreshness, + normalizeToIsoDate, + readCachedSearchPayload, + readConfiguredSecretString, + readProviderEnvValue, + resolveSearchCacheTtlMs, + resolveSearchCount, + resolveSearchTimeoutSeconds, + resolveSiteName, + throwWebSearchApiError, + withTrustedWebSearchEndpoint, + writeCachedSearchPayload, +} from "../../../src/agents/tools/web-search-provider-common.js"; +import type { + WebSearchCredentialResolutionSource, + WebSearchProviderPlugin, + WebSearchProviderToolDefinition, +} from "../../../src/plugins/types.js"; +import { wrapWebContent } from "../../../src/security/external-content.js"; + +const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; +const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; +const PERPLEXITY_SEARCH_ENDPOINT = "https://api.perplexity.ai/search"; +const DEFAULT_PERPLEXITY_MODEL = "perplexity/sonar-pro"; +const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; +const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; + +type PerplexityConfig = { + apiKey?: string; + baseUrl?: string; + model?: string; +}; + +type PerplexityTransport = "search_api" | "chat_completions"; +type PerplexityBaseUrlHint = "direct" | "openrouter"; + +type PerplexitySearchResponse = { + choices?: Array<{ + message?: { + content?: string; + annotations?: Array<{ + type?: string; + url?: string; + url_citation?: { + url?: string; + }; + }>; + }; + }>; + citations?: string[]; +}; + +type PerplexitySearchApiResponse = { + results?: Array<{ + title?: string; + url?: string; + snippet?: string; + date?: string; + }>; +}; + +function resolvePerplexityConfig(searchConfig?: SearchConfigRecord): PerplexityConfig { + const perplexity = searchConfig?.perplexity; + return perplexity && typeof perplexity === "object" && !Array.isArray(perplexity) + ? (perplexity as PerplexityConfig) + : {}; +} + +function inferPerplexityBaseUrlFromApiKey(apiKey?: string): PerplexityBaseUrlHint | undefined { + if (!apiKey) { + return undefined; + } + const normalized = apiKey.toLowerCase(); + if (PERPLEXITY_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "direct"; + } + if (OPENROUTER_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "openrouter"; + } + return undefined; +} + +function resolvePerplexityApiKey(perplexity?: PerplexityConfig): { + apiKey?: string; + source: "config" | "perplexity_env" | "openrouter_env" | "none"; +} { + const fromConfig = readConfiguredSecretString( + perplexity?.apiKey, + "tools.web.search.perplexity.apiKey", + ); + if (fromConfig) { + return { apiKey: fromConfig, source: "config" }; + } + const fromPerplexityEnv = readProviderEnvValue(["PERPLEXITY_API_KEY"]); + if (fromPerplexityEnv) { + return { apiKey: fromPerplexityEnv, source: "perplexity_env" }; + } + const fromOpenRouterEnv = readProviderEnvValue(["OPENROUTER_API_KEY"]); + if (fromOpenRouterEnv) { + return { apiKey: fromOpenRouterEnv, source: "openrouter_env" }; + } + return { apiKey: undefined, source: "none" }; +} + +function resolvePerplexityBaseUrl( + perplexity?: PerplexityConfig, + authSource: "config" | "perplexity_env" | "openrouter_env" | "none" = "none", + configuredKey?: string, +): string { + const fromConfig = typeof perplexity?.baseUrl === "string" ? perplexity.baseUrl.trim() : ""; + if (fromConfig) { + return fromConfig; + } + if (authSource === "perplexity_env") { + return PERPLEXITY_DIRECT_BASE_URL; + } + if (authSource === "openrouter_env") { + return DEFAULT_PERPLEXITY_BASE_URL; + } + if (authSource === "config") { + return inferPerplexityBaseUrlFromApiKey(configuredKey) === "openrouter" + ? DEFAULT_PERPLEXITY_BASE_URL + : PERPLEXITY_DIRECT_BASE_URL; + } + return DEFAULT_PERPLEXITY_BASE_URL; +} + +function resolvePerplexityModel(perplexity?: PerplexityConfig): string { + const model = typeof perplexity?.model === "string" ? perplexity.model.trim() : ""; + return model || DEFAULT_PERPLEXITY_MODEL; +} + +function isDirectPerplexityBaseUrl(baseUrl: string): boolean { + try { + return new URL(baseUrl.trim()).hostname.toLowerCase() === "api.perplexity.ai"; + } catch { + return false; + } +} + +function resolvePerplexityRequestModel(baseUrl: string, model: string): string { + if (!isDirectPerplexityBaseUrl(baseUrl)) { + return model; + } + return model.startsWith("perplexity/") ? model.slice("perplexity/".length) : model; +} + +function resolvePerplexityTransport(perplexity?: PerplexityConfig): { + apiKey?: string; + source: "config" | "perplexity_env" | "openrouter_env" | "none"; + baseUrl: string; + model: string; + transport: PerplexityTransport; +} { + const auth = resolvePerplexityApiKey(perplexity); + const baseUrl = resolvePerplexityBaseUrl(perplexity, auth.source, auth.apiKey); + const model = resolvePerplexityModel(perplexity); + const hasLegacyOverride = Boolean( + (perplexity?.baseUrl && perplexity.baseUrl.trim()) || + (perplexity?.model && perplexity.model.trim()), + ); + return { + ...auth, + baseUrl, + model, + transport: + hasLegacyOverride || !isDirectPerplexityBaseUrl(baseUrl) ? "chat_completions" : "search_api", + }; +} + +function extractPerplexityCitations(data: PerplexitySearchResponse): string[] { + const topLevel = (data.citations ?? []).filter( + (url): url is string => typeof url === "string" && Boolean(url.trim()), + ); + if (topLevel.length > 0) { + return [...new Set(topLevel)]; + } + const citations: string[] = []; + for (const choice of data.choices ?? []) { + for (const annotation of choice.message?.annotations ?? []) { + if (annotation.type !== "url_citation") { + continue; + } + const url = + typeof annotation.url_citation?.url === "string" + ? annotation.url_citation.url + : typeof annotation.url === "string" + ? annotation.url + : undefined; + if (url?.trim()) { + citations.push(url.trim()); + } + } + } + return [...new Set(citations)]; +} + +async function runPerplexitySearchApi(params: { + query: string; + apiKey: string; + count: number; + timeoutSeconds: number; + country?: string; + searchDomainFilter?: string[]; + searchRecencyFilter?: string; + searchLanguageFilter?: string[]; + searchAfterDate?: string; + searchBeforeDate?: string; + maxTokens?: number; + maxTokensPerPage?: number; +}): Promise>> { + const body: Record = { + query: params.query, + max_results: params.count, + }; + if (params.country) body.country = params.country; + if (params.searchDomainFilter?.length) body.search_domain_filter = params.searchDomainFilter; + if (params.searchRecencyFilter) body.search_recency_filter = params.searchRecencyFilter; + if (params.searchLanguageFilter?.length) + body.search_language_filter = params.searchLanguageFilter; + if (params.searchAfterDate) body.search_after_date = params.searchAfterDate; + if (params.searchBeforeDate) body.search_before_date = params.searchBeforeDate; + if (params.maxTokens !== undefined) body.max_tokens = params.maxTokens; + if (params.maxTokensPerPage !== undefined) body.max_tokens_per_page = params.maxTokensPerPage; + + return withTrustedWebSearchEndpoint( + { + url: PERPLEXITY_SEARCH_ENDPOINT, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + Authorization: `Bearer ${params.apiKey}`, + "HTTP-Referer": "https://openclaw.ai", + "X-Title": "OpenClaw Web Search", + }, + body: JSON.stringify(body), + }, + }, + async (res) => { + if (!res.ok) { + return await throwWebSearchApiError(res, "Perplexity Search"); + } + const data = (await res.json()) as PerplexitySearchApiResponse; + return (data.results ?? []).map((entry) => ({ + title: entry.title ? wrapWebContent(entry.title, "web_search") : "", + url: entry.url ?? "", + description: entry.snippet ? wrapWebContent(entry.snippet, "web_search") : "", + published: entry.date ?? undefined, + siteName: resolveSiteName(entry.url) || undefined, + })); + }, + ); +} + +async function runPerplexitySearch(params: { + query: string; + apiKey: string; + baseUrl: string; + model: string; + timeoutSeconds: number; + freshness?: string; +}): Promise<{ content: string; citations: string[] }> { + const endpoint = `${params.baseUrl.trim().replace(/\/$/, "")}/chat/completions`; + const body: Record = { + model: resolvePerplexityRequestModel(params.baseUrl, params.model), + messages: [{ role: "user", content: params.query }], + }; + if (params.freshness) { + body.search_recency_filter = params.freshness; + } + + return withTrustedWebSearchEndpoint( + { + url: endpoint, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + "HTTP-Referer": "https://openclaw.ai", + "X-Title": "OpenClaw Web Search", + }, + body: JSON.stringify(body), + }, + }, + async (res) => { + if (!res.ok) { + return await throwWebSearchApiError(res, "Perplexity"); + } + const data = (await res.json()) as PerplexitySearchResponse; + return { + content: data.choices?.[0]?.message?.content ?? "No response", + citations: extractPerplexityCitations(data), + }; + }, + ); +} + +function resolveRuntimeTransport(params: { + searchConfig?: Record; + resolvedKey?: string; + keySource: WebSearchCredentialResolutionSource; + fallbackEnvVar?: string; +}): PerplexityTransport | undefined { + const perplexity = params.searchConfig?.perplexity; + const scoped = + perplexity && typeof perplexity === "object" && !Array.isArray(perplexity) + ? (perplexity as { baseUrl?: string; model?: string }) + : undefined; + const configuredBaseUrl = typeof scoped?.baseUrl === "string" ? scoped.baseUrl.trim() : ""; + const configuredModel = typeof scoped?.model === "string" ? scoped.model.trim() : ""; + const baseUrl = (() => { + if (configuredBaseUrl) { + return configuredBaseUrl; + } + if (params.keySource === "env") { + if (params.fallbackEnvVar === "PERPLEXITY_API_KEY") return PERPLEXITY_DIRECT_BASE_URL; + if (params.fallbackEnvVar === "OPENROUTER_API_KEY") return DEFAULT_PERPLEXITY_BASE_URL; + } + if ((params.keySource === "config" || params.keySource === "secretRef") && params.resolvedKey) { + return inferPerplexityBaseUrlFromApiKey(params.resolvedKey) === "openrouter" + ? DEFAULT_PERPLEXITY_BASE_URL + : PERPLEXITY_DIRECT_BASE_URL; + } + return DEFAULT_PERPLEXITY_BASE_URL; + })(); + return configuredBaseUrl || configuredModel || !isDirectPerplexityBaseUrl(baseUrl) + ? "chat_completions" + : "search_api"; +} + +function createPerplexitySchema(transport?: PerplexityTransport) { + const querySchema = { + query: Type.String({ description: "Search query string." }), + count: Type.Optional( + Type.Number({ + description: "Number of results to return (1-10).", + minimum: 1, + maximum: MAX_SEARCH_COUNT, + }), + ), + freshness: Type.Optional( + Type.String({ description: "Filter by time: 'day' (24h), 'week', 'month', or 'year'." }), + ), + }; + if (transport === "chat_completions") { + return Type.Object(querySchema); + } + return Type.Object({ + ...querySchema, + country: Type.Optional( + Type.String({ description: "Native Perplexity Search API only. 2-letter country code." }), + ), + language: Type.Optional( + Type.String({ description: "Native Perplexity Search API only. ISO 639-1 language code." }), + ), + date_after: Type.Optional( + Type.String({ + description: + "Native Perplexity Search API only. Only results published after this date (YYYY-MM-DD).", + }), + ), + date_before: Type.Optional( + Type.String({ + description: + "Native Perplexity Search API only. Only results published before this date (YYYY-MM-DD).", + }), + ), + domain_filter: Type.Optional( + Type.Array(Type.String(), { + description: "Native Perplexity Search API only. Domain filter (max 20).", + }), + ), + max_tokens: Type.Optional( + Type.Number({ + description: "Native Perplexity Search API only. Total content budget across all results.", + minimum: 1, + maximum: 1000000, + }), + ), + max_tokens_per_page: Type.Optional( + Type.Number({ + description: "Native Perplexity Search API only. Max tokens extracted per page.", + minimum: 1, + }), + ), + }); +} + +function createPerplexityToolDefinition( + searchConfig?: SearchConfigRecord, + runtimeTransport?: PerplexityTransport, +): WebSearchProviderToolDefinition { + const perplexityConfig = resolvePerplexityConfig(searchConfig); + const schemaTransport = + runtimeTransport ?? + (perplexityConfig.baseUrl || perplexityConfig.model ? "chat_completions" : undefined); + + return { + description: + schemaTransport === "chat_completions" + ? "Search the web using Perplexity Sonar via Perplexity/OpenRouter chat completions. Returns AI-synthesized answers with citations from web-grounded search." + : "Search the web using Perplexity. Runtime routing decides between native Search API and Sonar chat-completions compatibility. Structured filters are available on the native Search API path.", + parameters: createPerplexitySchema(schemaTransport), + execute: async (args) => { + const runtime = resolvePerplexityTransport(perplexityConfig); + if (!runtime.apiKey) { + return { + error: "missing_perplexity_api_key", + message: + "web_search (perplexity) needs an API key. Set PERPLEXITY_API_KEY or OPENROUTER_API_KEY in the Gateway environment, or configure tools.web.search.perplexity.apiKey.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const params = args as Record; + const query = readStringParam(params, "query", { required: true }); + const count = + readNumberParam(params, "count", { integer: true }) ?? + searchConfig?.maxResults ?? + undefined; + const rawFreshness = readStringParam(params, "freshness"); + const freshness = rawFreshness ? normalizeFreshness(rawFreshness, "perplexity") : undefined; + if (rawFreshness && !freshness) { + return { + error: "invalid_freshness", + message: "freshness must be day, week, month, or year.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const structured = runtime.transport === "search_api"; + const country = readStringParam(params, "country"); + const language = readStringParam(params, "language"); + const rawDateAfter = readStringParam(params, "date_after"); + const rawDateBefore = readStringParam(params, "date_before"); + const domainFilter = readStringArrayParam(params, "domain_filter"); + const maxTokens = readNumberParam(params, "max_tokens", { integer: true }); + const maxTokensPerPage = readNumberParam(params, "max_tokens_per_page", { integer: true }); + + if (!structured) { + if (country) { + return { + error: "unsupported_country", + message: + "country filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (language) { + return { + error: "unsupported_language", + message: + "language filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (rawDateAfter || rawDateBefore) { + return { + error: "unsupported_date_filter", + message: + "date_after/date_before are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (domainFilter?.length) { + return { + error: "unsupported_domain_filter", + message: + "domain_filter is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (maxTokens !== undefined || maxTokensPerPage !== undefined) { + return { + error: "unsupported_content_budget", + message: + "max_tokens and max_tokens_per_page are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + } + + if (language && !/^[a-z]{2}$/i.test(language)) { + return { + error: "invalid_language", + message: "language must be a 2-letter ISO 639-1 code like 'en', 'de', or 'fr'.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (rawFreshness && (rawDateAfter || rawDateBefore)) { + return { + error: "conflicting_time_filters", + message: + "freshness and date_after/date_before cannot be used together. Use either freshness (day/week/month/year) or a date range (date_after/date_before), not both.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + const dateAfter = rawDateAfter ? normalizeToIsoDate(rawDateAfter) : undefined; + const dateBefore = rawDateBefore ? normalizeToIsoDate(rawDateBefore) : undefined; + if (rawDateAfter && !dateAfter) { + return { + error: "invalid_date", + message: "date_after must be YYYY-MM-DD format.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (rawDateBefore && !dateBefore) { + return { + error: "invalid_date", + message: "date_before must be YYYY-MM-DD format.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (dateAfter && dateBefore && dateAfter > dateBefore) { + return { + error: "invalid_date_range", + message: "date_after must be before date_before.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (domainFilter?.length) { + const hasDeny = domainFilter.some((entry) => entry.startsWith("-")); + const hasAllow = domainFilter.some((entry) => !entry.startsWith("-")); + if (hasDeny && hasAllow) { + return { + error: "invalid_domain_filter", + message: + "domain_filter cannot mix allowlist and denylist entries. Use either all positive entries (allowlist) or all entries prefixed with '-' (denylist).", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (domainFilter.length > 20) { + return { + error: "invalid_domain_filter", + message: "domain_filter supports a maximum of 20 domains.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + } + + const cacheKey = buildSearchCacheKey([ + "perplexity", + runtime.transport, + runtime.baseUrl, + runtime.model, + query, + resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + country, + language, + freshness, + dateAfter, + dateBefore, + domainFilter?.join(","), + maxTokens, + maxTokensPerPage, + ]); + const cached = readCachedSearchPayload(cacheKey); + if (cached) { + return cached; + } + + const start = Date.now(); + const timeoutSeconds = resolveSearchTimeoutSeconds(searchConfig); + const payload = + runtime.transport === "chat_completions" + ? { + query, + provider: "perplexity", + model: runtime.model, + tookMs: Date.now() - start, + externalContent: { + untrusted: true, + source: "web_search", + provider: "perplexity", + wrapped: true, + }, + ...(await (async () => { + const result = await runPerplexitySearch({ + query, + apiKey: runtime.apiKey!, + baseUrl: runtime.baseUrl, + model: runtime.model, + timeoutSeconds, + freshness, + }); + return { + content: wrapWebContent(result.content, "web_search"), + citations: result.citations, + }; + })()), + } + : { + query, + provider: "perplexity", + count: 0, + tookMs: Date.now() - start, + externalContent: { + untrusted: true, + source: "web_search", + provider: "perplexity", + wrapped: true, + }, + results: await runPerplexitySearchApi({ + query, + apiKey: runtime.apiKey!, + count: resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + timeoutSeconds, + country: country ?? undefined, + searchDomainFilter: domainFilter, + searchRecencyFilter: freshness, + searchLanguageFilter: language ? [language] : undefined, + searchAfterDate: dateAfter ? isoToPerplexityDate(dateAfter) : undefined, + searchBeforeDate: dateBefore ? isoToPerplexityDate(dateBefore) : undefined, + maxTokens: maxTokens ?? undefined, + maxTokensPerPage: maxTokensPerPage ?? undefined, + }), + }; + + if (Array.isArray((payload as { results?: unknown[] }).results)) { + (payload as { count: number }).count = (payload as { results: unknown[] }).results.length; + (payload as { tookMs: number }).tookMs = Date.now() - start; + } else { + (payload as { tookMs: number }).tookMs = Date.now() - start; + } + + writeCachedSearchPayload(cacheKey, payload, resolveSearchCacheTtlMs(searchConfig)); + return payload; + }, + }; +} + +export function createPerplexityWebSearchProvider(): WebSearchProviderPlugin { + return { + id: "perplexity", + label: "Perplexity Search", + hint: "Structured results · domain/country/language/time filters", + envVars: ["PERPLEXITY_API_KEY", "OPENROUTER_API_KEY"], + placeholder: "pplx-...", + signupUrl: "https://www.perplexity.ai/settings/api", + docsUrl: "https://docs.openclaw.ai/perplexity", + autoDetectOrder: 50, + credentialPath: "tools.web.search.perplexity.apiKey", + inactiveSecretPaths: ["tools.web.search.perplexity.apiKey"], + getCredentialValue: (searchConfig) => { + const perplexity = searchConfig?.perplexity; + return perplexity && typeof perplexity === "object" && !Array.isArray(perplexity) + ? (perplexity as Record).apiKey + : undefined; + }, + setCredentialValue: (searchConfigTarget, value) => { + const scoped = searchConfigTarget.perplexity; + if (!scoped || typeof scoped !== "object" || Array.isArray(scoped)) { + searchConfigTarget.perplexity = { apiKey: value }; + return; + } + (scoped as Record).apiKey = value; + }, + resolveRuntimeMetadata: (ctx) => ({ + perplexityTransport: resolveRuntimeTransport({ + searchConfig: ctx.searchConfig, + resolvedKey: ctx.resolvedCredential?.value, + keySource: ctx.resolvedCredential?.source ?? "missing", + fallbackEnvVar: ctx.resolvedCredential?.fallbackEnvVar, + }), + }), + createTool: (ctx) => + createPerplexityToolDefinition( + ctx.searchConfig as SearchConfigRecord | undefined, + ctx.runtimeMetadata?.perplexityTransport as PerplexityTransport | undefined, + ), + }; +} + +export const __testing = { + inferPerplexityBaseUrlFromApiKey, + resolvePerplexityBaseUrl, + resolvePerplexityModel, + resolvePerplexityTransport, + isDirectPerplexityBaseUrl, + resolvePerplexityRequestModel, + resolvePerplexityApiKey, + normalizeToIsoDate, + isoToPerplexityDate, +} as const; diff --git a/extensions/perplexity/web-search-provider.ts b/extensions/perplexity/web-search-provider.ts new file mode 100644 index 00000000000..c501c44a28c --- /dev/null +++ b/extensions/perplexity/web-search-provider.ts @@ -0,0 +1,4 @@ +export { + __testing, + createPerplexityWebSearchProvider, +} from "./src/perplexity-web-search-provider.js"; diff --git a/extensions/xai/index.ts b/extensions/xai/index.ts index 485b7ec6461..7a64abca8d3 100644 --- a/extensions/xai/index.ts +++ b/extensions/xai/index.ts @@ -1,12 +1,8 @@ import { definePluginEntry } from "openclaw/plugin-sdk/core"; import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth"; import { normalizeProviderId } from "openclaw/plugin-sdk/provider-models"; -import { - createPluginBackedWebSearchProvider, - getScopedCredentialValue, - setScopedCredentialValue, -} from "openclaw/plugin-sdk/provider-web-search"; import { applyXaiConfig, XAI_DEFAULT_MODEL_REF } from "./onboard.js"; +import { createGrokWebSearchProvider } from "./src/grok-web-search-provider.js"; const PROVIDER_ID = "xai"; const XAI_MODERN_MODEL_PREFIXES = ["grok-4"] as const; @@ -51,20 +47,6 @@ export default definePluginEntry({ isModernModelRef: ({ provider, modelId }) => normalizeProviderId(provider) === "xai" ? matchesModernXaiModel(modelId) : undefined, }); - api.registerWebSearchProvider( - createPluginBackedWebSearchProvider({ - id: "grok", - label: "Grok (xAI)", - hint: "xAI web-grounded responses", - envVars: ["XAI_API_KEY"], - placeholder: "xai-...", - signupUrl: "https://console.x.ai/", - docsUrl: "https://docs.openclaw.ai/tools/web", - autoDetectOrder: 30, - getCredentialValue: (searchConfig) => getScopedCredentialValue(searchConfig, "grok"), - setCredentialValue: (searchConfigTarget, value) => - setScopedCredentialValue(searchConfigTarget, "grok", value), - }), - ); + api.registerWebSearchProvider(createGrokWebSearchProvider()); }, }); diff --git a/extensions/xai/src/grok-web-search-provider.ts b/extensions/xai/src/grok-web-search-provider.ts new file mode 100644 index 00000000000..bb9af38f91a --- /dev/null +++ b/extensions/xai/src/grok-web-search-provider.ts @@ -0,0 +1,303 @@ +import { Type } from "@sinclair/typebox"; +import { readNumberParam, readStringParam } from "../../../src/agents/tools/common.js"; +import type { SearchConfigRecord } from "../../../src/agents/tools/web-search-provider-common.js"; +import { + buildSearchCacheKey, + DEFAULT_SEARCH_COUNT, + MAX_SEARCH_COUNT, + readCachedSearchPayload, + readConfiguredSecretString, + readProviderEnvValue, + resolveSearchCacheTtlMs, + resolveSearchCount, + resolveSearchTimeoutSeconds, + withTrustedWebSearchEndpoint, + writeCachedSearchPayload, +} from "../../../src/agents/tools/web-search-provider-common.js"; +import type { + WebSearchProviderPlugin, + WebSearchProviderToolDefinition, +} from "../../../src/plugins/types.js"; +import { wrapWebContent } from "../../../src/security/external-content.js"; + +const XAI_API_ENDPOINT = "https://api.x.ai/v1/responses"; +const DEFAULT_GROK_MODEL = "grok-4-1-fast"; + +type GrokConfig = { + apiKey?: string; + model?: string; + inlineCitations?: boolean; +}; + +type GrokSearchResponse = { + output?: Array<{ + type?: string; + role?: string; + text?: string; + content?: Array<{ + type?: string; + text?: string; + annotations?: Array<{ + type?: string; + url?: string; + start_index?: number; + end_index?: number; + }>; + }>; + annotations?: Array<{ + type?: string; + url?: string; + start_index?: number; + end_index?: number; + }>; + }>; + output_text?: string; + citations?: string[]; + inline_citations?: Array<{ + start_index: number; + end_index: number; + url: string; + }>; +}; + +function resolveGrokConfig(searchConfig?: SearchConfigRecord): GrokConfig { + const grok = searchConfig?.grok; + return grok && typeof grok === "object" && !Array.isArray(grok) ? (grok as GrokConfig) : {}; +} + +function resolveGrokApiKey(grok?: GrokConfig): string | undefined { + return ( + readConfiguredSecretString(grok?.apiKey, "tools.web.search.grok.apiKey") ?? + readProviderEnvValue(["XAI_API_KEY"]) + ); +} + +function resolveGrokModel(grok?: GrokConfig): string { + const model = typeof grok?.model === "string" ? grok.model.trim() : ""; + return model || DEFAULT_GROK_MODEL; +} + +function resolveGrokInlineCitations(grok?: GrokConfig): boolean { + return grok?.inlineCitations === true; +} + +function extractGrokContent(data: GrokSearchResponse): { + text: string | undefined; + annotationCitations: string[]; +} { + for (const output of data.output ?? []) { + if (output.type === "message") { + for (const block of output.content ?? []) { + if (block.type === "output_text" && typeof block.text === "string" && block.text) { + const urls = (block.annotations ?? []) + .filter( + (annotation) => + annotation.type === "url_citation" && typeof annotation.url === "string", + ) + .map((annotation) => annotation.url as string); + return { text: block.text, annotationCitations: [...new Set(urls)] }; + } + } + } + if (output.type === "output_text" && typeof output.text === "string" && output.text) { + const urls = (Array.isArray(output.annotations) ? output.annotations : []) + .filter( + (annotation) => annotation.type === "url_citation" && typeof annotation.url === "string", + ) + .map((annotation) => annotation.url as string); + return { text: output.text, annotationCitations: [...new Set(urls)] }; + } + } + + return { + text: typeof data.output_text === "string" ? data.output_text : undefined, + annotationCitations: [], + }; +} + +async function runGrokSearch(params: { + query: string; + apiKey: string; + model: string; + timeoutSeconds: number; + inlineCitations: boolean; +}): Promise<{ + content: string; + citations: string[]; + inlineCitations?: GrokSearchResponse["inline_citations"]; +}> { + return withTrustedWebSearchEndpoint( + { + url: XAI_API_ENDPOINT, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + }, + body: JSON.stringify({ + model: params.model, + input: [{ role: "user", content: params.query }], + tools: [{ type: "web_search" }], + }), + }, + }, + async (res) => { + if (!res.ok) { + const detail = await res.text(); + throw new Error(`xAI API error (${res.status}): ${detail || res.statusText}`); + } + + const data = (await res.json()) as GrokSearchResponse; + const { text, annotationCitations } = extractGrokContent(data); + return { + content: text ?? "No response", + citations: (data.citations ?? []).length > 0 ? data.citations! : annotationCitations, + inlineCitations: data.inline_citations, + }; + }, + ); +} + +function createGrokSchema() { + return Type.Object({ + query: Type.String({ description: "Search query string." }), + count: Type.Optional( + Type.Number({ + description: "Number of results to return (1-10).", + minimum: 1, + maximum: MAX_SEARCH_COUNT, + }), + ), + country: Type.Optional(Type.String({ description: "Not supported by Grok." })), + language: Type.Optional(Type.String({ description: "Not supported by Grok." })), + freshness: Type.Optional(Type.String({ description: "Not supported by Grok." })), + date_after: Type.Optional(Type.String({ description: "Not supported by Grok." })), + date_before: Type.Optional(Type.String({ description: "Not supported by Grok." })), + }); +} + +function createGrokToolDefinition( + searchConfig?: SearchConfigRecord, +): WebSearchProviderToolDefinition { + return { + description: + "Search the web using xAI Grok. Returns AI-synthesized answers with citations from real-time web search.", + parameters: createGrokSchema(), + execute: async (args) => { + const params = args as Record; + for (const name of ["country", "language", "freshness", "date_after", "date_before"]) { + if (readStringParam(params, name)) { + const label = + name === "country" + ? "country filtering" + : name === "language" + ? "language filtering" + : name === "freshness" + ? "freshness filtering" + : "date_after/date_before filtering"; + return { + error: name.startsWith("date_") ? "unsupported_date_filter" : `unsupported_${name}`, + message: `${label} is not supported by the grok provider. Only Brave and Perplexity support ${name === "country" ? "country filtering" : name === "language" ? "language filtering" : name === "freshness" ? "freshness" : "date filtering"}.`, + docs: "https://docs.openclaw.ai/tools/web", + }; + } + } + + const grokConfig = resolveGrokConfig(searchConfig); + const apiKey = resolveGrokApiKey(grokConfig); + if (!apiKey) { + return { + error: "missing_xai_api_key", + message: + "web_search (grok) needs an xAI API key. Set XAI_API_KEY in the Gateway environment, or configure tools.web.search.grok.apiKey.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const query = readStringParam(params, "query", { required: true }); + const count = + readNumberParam(params, "count", { integer: true }) ?? + searchConfig?.maxResults ?? + undefined; + const model = resolveGrokModel(grokConfig); + const inlineCitations = resolveGrokInlineCitations(grokConfig); + const cacheKey = buildSearchCacheKey([ + "grok", + query, + resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + model, + inlineCitations, + ]); + const cached = readCachedSearchPayload(cacheKey); + if (cached) { + return cached; + } + + const start = Date.now(); + const result = await runGrokSearch({ + query, + apiKey, + model, + timeoutSeconds: resolveSearchTimeoutSeconds(searchConfig), + inlineCitations, + }); + const payload = { + query, + provider: "grok", + model, + tookMs: Date.now() - start, + externalContent: { + untrusted: true, + source: "web_search", + provider: "grok", + wrapped: true, + }, + content: wrapWebContent(result.content), + citations: result.citations, + inlineCitations: result.inlineCitations, + }; + writeCachedSearchPayload(cacheKey, payload, resolveSearchCacheTtlMs(searchConfig)); + return payload; + }, + }; +} + +export function createGrokWebSearchProvider(): WebSearchProviderPlugin { + return { + id: "grok", + label: "Grok (xAI)", + hint: "xAI web-grounded responses", + envVars: ["XAI_API_KEY"], + placeholder: "xai-...", + signupUrl: "https://console.x.ai/", + docsUrl: "https://docs.openclaw.ai/tools/web", + autoDetectOrder: 30, + credentialPath: "tools.web.search.grok.apiKey", + inactiveSecretPaths: ["tools.web.search.grok.apiKey"], + getCredentialValue: (searchConfig) => { + const grok = searchConfig?.grok; + return grok && typeof grok === "object" && !Array.isArray(grok) + ? (grok as Record).apiKey + : undefined; + }, + setCredentialValue: (searchConfigTarget, value) => { + const scoped = searchConfigTarget.grok; + if (!scoped || typeof scoped !== "object" || Array.isArray(scoped)) { + searchConfigTarget.grok = { apiKey: value }; + return; + } + (scoped as Record).apiKey = value; + }, + createTool: (ctx) => + createGrokToolDefinition(ctx.searchConfig as SearchConfigRecord | undefined), + }; +} + +export const __testing = { + resolveGrokApiKey, + resolveGrokModel, + resolveGrokInlineCitations, + extractGrokContent, +} as const; diff --git a/package.json b/package.json index afcfb086da2..37a4e595207 100644 --- a/package.json +++ b/package.json @@ -541,6 +541,7 @@ "lint:fix": "oxlint --type-aware --fix && pnpm format", "lint:plugins:no-extension-src-imports": "node --import tsx scripts/check-no-extension-src-imports.ts", "lint:plugins:no-extension-test-core-imports": "node --import tsx scripts/check-no-extension-test-core-imports.ts", + "lint:plugins:no-extension-imports": "node scripts/check-plugin-extension-import-boundary.mjs", "lint:plugins:no-monolithic-plugin-sdk-entry-imports": "node --import tsx scripts/check-no-monolithic-plugin-sdk-entry-imports.ts", "lint:plugins:no-register-http-handler": "node scripts/check-no-register-http-handler.mjs", "lint:swift": "swiftlint lint --config .swiftlint.yml && (cd apps/ios && swiftlint lint --config .swiftlint.yml)", @@ -548,6 +549,7 @@ "lint:tmp:no-random-messaging": "node scripts/check-no-random-messaging-tmp.mjs", "lint:tmp:no-raw-channel-fetch": "node scripts/check-no-raw-channel-fetch.mjs", "lint:ui:no-raw-window-open": "node scripts/check-no-raw-window-open.mjs", + "lint:web-search-provider-boundaries": "node scripts/check-web-search-provider-boundaries.mjs", "lint:webhook:no-low-level-body-read": "node scripts/check-webhook-auth-body-order.mjs", "mac:open": "open dist/OpenClaw.app", "mac:package": "bash scripts/package-mac-app.sh", diff --git a/scripts/check-plugin-extension-import-boundary.mjs b/scripts/check-plugin-extension-import-boundary.mjs new file mode 100644 index 00000000000..13c4fa596a3 --- /dev/null +++ b/scripts/check-plugin-extension-import-boundary.mjs @@ -0,0 +1,302 @@ +#!/usr/bin/env node + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; +import { + collectTypeScriptFilesFromRoots, + resolveSourceRoots, + runAsScript, + toLine, +} from "./lib/ts-guard-utils.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const scanRoots = resolveSourceRoots(repoRoot, ["src/plugins"]); +const baselinePath = path.join( + repoRoot, + "test", + "fixtures", + "plugin-extension-import-boundary-inventory.json", +); + +const bundledWebSearchProviders = new Set([ + "brave", + "firecrawl", + "gemini", + "grok", + "kimi", + "perplexity", +]); +const bundledWebSearchPluginIds = new Set([ + "brave", + "firecrawl", + "google", + "moonshot", + "perplexity", + "xai", +]); + +function normalizePath(filePath) { + return path.relative(repoRoot, filePath).split(path.sep).join("/"); +} + +function compareEntries(left, right) { + return ( + left.file.localeCompare(right.file) || + left.line - right.line || + left.kind.localeCompare(right.kind) || + left.specifier.localeCompare(right.specifier) || + left.reason.localeCompare(right.reason) + ); +} + +function resolveSpecifier(specifier, importerFile) { + if (specifier.startsWith(".")) { + return normalizePath(path.resolve(path.dirname(importerFile), specifier)); + } + if (specifier.startsWith("/")) { + return normalizePath(specifier); + } + return null; +} + +function classifyResolvedExtensionReason(kind, resolvedPath) { + const verb = + kind === "export" + ? "re-exports" + : kind === "dynamic-import" + ? "dynamically imports" + : "imports"; + if (/^extensions\/[^/]+\/src\//.test(resolvedPath)) { + return `${verb} extension implementation from src/plugins`; + } + if (/^extensions\/[^/]+\/index\.[^/]+$/.test(resolvedPath)) { + return `${verb} extension entrypoint from src/plugins`; + } + return `${verb} extension-owned file from src/plugins`; +} + +function pushEntry(entries, entry) { + entries.push(entry); +} + +function scanImportBoundaryViolations(sourceFile, filePath) { + const entries = []; + + function visit(node) { + if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) { + const specifier = node.moduleSpecifier.text; + const resolvedPath = resolveSpecifier(specifier, filePath); + if (resolvedPath?.startsWith("extensions/")) { + pushEntry(entries, { + file: normalizePath(filePath), + line: toLine(sourceFile, node.moduleSpecifier), + kind: "import", + specifier, + resolvedPath, + reason: classifyResolvedExtensionReason("import", resolvedPath), + }); + } + } else if ( + ts.isExportDeclaration(node) && + node.moduleSpecifier && + ts.isStringLiteral(node.moduleSpecifier) + ) { + const specifier = node.moduleSpecifier.text; + const resolvedPath = resolveSpecifier(specifier, filePath); + if (resolvedPath?.startsWith("extensions/")) { + pushEntry(entries, { + file: normalizePath(filePath), + line: toLine(sourceFile, node.moduleSpecifier), + kind: "export", + specifier, + resolvedPath, + reason: classifyResolvedExtensionReason("export", resolvedPath), + }); + } + } else if ( + ts.isCallExpression(node) && + node.expression.kind === ts.SyntaxKind.ImportKeyword && + node.arguments.length === 1 && + ts.isStringLiteral(node.arguments[0]) + ) { + const specifier = node.arguments[0].text; + const resolvedPath = resolveSpecifier(specifier, filePath); + if (resolvedPath?.startsWith("extensions/")) { + pushEntry(entries, { + file: normalizePath(filePath), + line: toLine(sourceFile, node.arguments[0]), + kind: "dynamic-import", + specifier, + resolvedPath, + reason: classifyResolvedExtensionReason("dynamic-import", resolvedPath), + }); + } + } + + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return entries; +} + +function scanWebSearchRegistrySmells(sourceFile, filePath) { + const relativeFile = normalizePath(filePath); + if (relativeFile !== "src/plugins/web-search-providers.ts") { + return []; + } + + const entries = []; + const lines = sourceFile.text.split(/\r?\n/); + for (const [index, line] of lines.entries()) { + const lineNumber = index + 1; + + if (line.includes("web-search-plugin-factory.js")) { + pushEntry(entries, { + file: relativeFile, + line: lineNumber, + kind: "registry-smell", + specifier: "../agents/tools/web-search-plugin-factory.js", + resolvedPath: "src/agents/tools/web-search-plugin-factory.js", + reason: "imports core-owned web search provider factory into plugin registry", + }); + } + + const pluginMatch = line.match(/pluginId:\s*"([^"]+)"/); + if (pluginMatch && bundledWebSearchPluginIds.has(pluginMatch[1])) { + pushEntry(entries, { + file: relativeFile, + line: lineNumber, + kind: "registry-smell", + specifier: pluginMatch[1], + resolvedPath: relativeFile, + reason: "hardcodes bundled web search plugin ownership in core registry", + }); + } + + const providerMatch = line.match(/id:\s*"(brave|firecrawl|gemini|grok|kimi|perplexity)"/); + if (providerMatch && bundledWebSearchProviders.has(providerMatch[1])) { + pushEntry(entries, { + file: relativeFile, + line: lineNumber, + kind: "registry-smell", + specifier: providerMatch[1], + resolvedPath: relativeFile, + reason: "hardcodes bundled web search provider metadata in core registry", + }); + } + } + + return entries; +} + +function shouldSkipFile(filePath) { + const relativeFile = normalizePath(filePath); + return relativeFile.startsWith("src/plugins/contracts/"); +} + +export async function collectPluginExtensionImportBoundaryInventory() { + const files = (await collectTypeScriptFilesFromRoots(scanRoots)) + .filter((filePath) => !shouldSkipFile(filePath)) + .toSorted((left, right) => normalizePath(left).localeCompare(normalizePath(right))); + + const inventory = []; + for (const filePath of files) { + const source = await fs.readFile(filePath, "utf8"); + const sourceFile = ts.createSourceFile( + filePath, + source, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS, + ); + inventory.push(...scanImportBoundaryViolations(sourceFile, filePath)); + inventory.push(...scanWebSearchRegistrySmells(sourceFile, filePath)); + } + + return inventory.toSorted(compareEntries); +} + +export async function readExpectedInventory() { + return JSON.parse(await fs.readFile(baselinePath, "utf8")); +} + +export function diffInventory(expected, actual) { + const expectedKeys = new Set(expected.map((entry) => JSON.stringify(entry))); + const actualKeys = new Set(actual.map((entry) => JSON.stringify(entry))); + return { + missing: expected + .filter((entry) => !actualKeys.has(JSON.stringify(entry))) + .toSorted(compareEntries), + unexpected: actual + .filter((entry) => !expectedKeys.has(JSON.stringify(entry))) + .toSorted(compareEntries), + }; +} + +function formatInventoryHuman(inventory) { + if (inventory.length === 0) { + return "Rule: src/plugins/** must not import extensions/**\nNo plugin import boundary violations found."; + } + + const lines = [ + "Rule: src/plugins/** must not import extensions/**", + "Plugin extension import boundary inventory:", + ]; + let activeFile = ""; + for (const entry of inventory) { + if (entry.file !== activeFile) { + activeFile = entry.file; + lines.push(activeFile); + } + lines.push(` - line ${entry.line} [${entry.kind}] ${entry.reason}`); + lines.push(` specifier: ${entry.specifier}`); + lines.push(` resolved: ${entry.resolvedPath}`); + } + return lines.join("\n"); +} + +function formatEntry(entry) { + return `${entry.file}:${entry.line} [${entry.kind}] ${entry.reason} (${entry.specifier} -> ${entry.resolvedPath})`; +} + +export async function main(argv = process.argv.slice(2)) { + const json = argv.includes("--json"); + const actual = await collectPluginExtensionImportBoundaryInventory(); + const expected = await readExpectedInventory(); + const { missing, unexpected } = diffInventory(expected, actual); + const matchesBaseline = missing.length === 0 && unexpected.length === 0; + + if (json) { + process.stdout.write(`${JSON.stringify(actual, null, 2)}\n`); + } else { + console.log(formatInventoryHuman(actual)); + console.log( + matchesBaseline + ? `Baseline matches (${actual.length} entries).` + : `Baseline mismatch (${unexpected.length} unexpected, ${missing.length} missing).`, + ); + if (!matchesBaseline) { + if (unexpected.length > 0) { + console.error("Unexpected entries:"); + for (const entry of unexpected) { + console.error(`- ${formatEntry(entry)}`); + } + } + if (missing.length > 0) { + console.error("Missing baseline entries:"); + for (const entry of missing) { + console.error(`- ${formatEntry(entry)}`); + } + } + } + } + + if (!matchesBaseline) { + process.exit(1); + } +} + +runAsScript(import.meta.url, main); diff --git a/scripts/check-web-search-provider-boundaries.mjs b/scripts/check-web-search-provider-boundaries.mjs new file mode 100644 index 00000000000..02b8d17374c --- /dev/null +++ b/scripts/check-web-search-provider-boundaries.mjs @@ -0,0 +1,331 @@ +#!/usr/bin/env node + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runAsScript } from "./lib/ts-guard-utils.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const baselinePath = path.join( + repoRoot, + "test", + "fixtures", + "web-search-provider-boundary-inventory.json", +); + +const scanRoots = ["src", "test", "scripts"]; +const scanExtensions = new Set([".ts", ".js", ".mjs", ".cjs"]); +const ignoredDirNames = new Set([ + ".artifacts", + ".git", + ".turbo", + "build", + "coverage", + "dist", + "extensions", + "node_modules", +]); + +const bundledProviderPluginToSearchProvider = new Map([ + ["brave", "brave"], + ["firecrawl", "firecrawl"], + ["google", "gemini"], + ["moonshot", "kimi"], + ["perplexity", "perplexity"], + ["xai", "grok"], +]); + +const providerIds = new Set([ + "brave", + "firecrawl", + "gemini", + "grok", + "kimi", + "perplexity", + "shared", +]); + +const allowedGenericFiles = new Set([ + "src/agents/tools/web-search-core.ts", + "src/agents/tools/web-search.ts", + "src/secrets/runtime-web-tools.ts", +]); + +const ignoredFiles = new Set([ + "scripts/check-plugin-extension-import-boundary.mjs", + "scripts/check-web-search-provider-boundaries.mjs", + "test/web-search-provider-boundary.test.ts", +]); + +function normalizeRelativePath(filePath) { + return path.relative(repoRoot, filePath).split(path.sep).join("/"); +} + +async function walkFiles(rootDir) { + const out = []; + let entries = []; + try { + entries = await fs.readdir(rootDir, { withFileTypes: true }); + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return out; + } + throw error; + } + entries.sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + const entryPath = path.join(rootDir, entry.name); + if (entry.isDirectory()) { + if (ignoredDirNames.has(entry.name)) { + continue; + } + out.push(...(await walkFiles(entryPath))); + continue; + } + if (!entry.isFile()) { + continue; + } + if (!scanExtensions.has(path.extname(entry.name))) { + continue; + } + out.push(entryPath); + } + return out; +} + +function compareInventoryEntries(left, right) { + return ( + left.provider.localeCompare(right.provider) || + left.file.localeCompare(right.file) || + left.line - right.line || + left.reason.localeCompare(right.reason) + ); +} + +function pushEntry(inventory, entry) { + if (!providerIds.has(entry.provider)) { + throw new Error(`Unknown provider id in boundary inventory: ${entry.provider}`); + } + inventory.push(entry); +} + +function scanWebSearchProviderRegistry(lines, relativeFile, inventory) { + for (const [index, line] of lines.entries()) { + const lineNumber = index + 1; + + if (line.includes("firecrawl-search-provider.js")) { + pushEntry(inventory, { + provider: "shared", + file: relativeFile, + line: lineNumber, + reason: "imports extension web search provider implementation into core registry", + }); + } + + if (line.includes("web-search-plugin-factory.js")) { + pushEntry(inventory, { + provider: "shared", + file: relativeFile, + line: lineNumber, + reason: "imports shared web search provider registration helper into core registry", + }); + } + + const pluginMatch = line.match(/pluginId:\s*"([^"]+)"/); + const providerFromPlugin = pluginMatch + ? bundledProviderPluginToSearchProvider.get(pluginMatch[1]) + : undefined; + if (providerFromPlugin) { + pushEntry(inventory, { + provider: providerFromPlugin, + file: relativeFile, + line: lineNumber, + reason: "hardcodes bundled web search plugin ownership in core registry", + }); + } + + const providerMatch = line.match(/id:\s*"(brave|firecrawl|gemini|grok|kimi|perplexity)"/); + if (providerMatch) { + pushEntry(inventory, { + provider: providerMatch[1], + file: relativeFile, + line: lineNumber, + reason: "hardcodes bundled web search provider id in core registry", + }); + } + } +} + +function scanOnboardSearch(lines, relativeFile, inventory) { + for (const [index, line] of lines.entries()) { + const lineNumber = index + 1; + + if (line.includes("web-search-providers.js")) { + pushEntry(inventory, { + provider: "shared", + file: relativeFile, + line: lineNumber, + reason: "imports bundled web search registry into core onboarding flow", + }); + } + + if (line.includes("const SEARCH_PROVIDER_IDS = [")) { + for (const provider of ["brave", "firecrawl", "gemini", "grok", "kimi", "perplexity"]) { + if (!line.includes(`"${provider}"`)) { + continue; + } + pushEntry(inventory, { + provider, + file: relativeFile, + line: lineNumber, + reason: "hardcodes bundled web search provider inventory in core onboarding flow", + }); + } + } + + if ( + line.includes('provider !== "firecrawl"') || + line.includes('enablePluginInConfig(next, "firecrawl")') + ) { + pushEntry(inventory, { + provider: "firecrawl", + file: relativeFile, + line: lineNumber, + reason: "hardcodes provider-specific plugin enablement coupling in core onboarding flow", + }); + } + } +} + +function scanGenericCoreImports(lines, relativeFile, inventory) { + if (allowedGenericFiles.has(relativeFile)) { + return; + } + for (const [index, line] of lines.entries()) { + const lineNumber = index + 1; + if (line.includes("web-search-providers.js")) { + pushEntry(inventory, { + provider: "shared", + file: relativeFile, + line: lineNumber, + reason: "imports bundled web search registry outside allowed generic plumbing", + }); + } + if (line.includes("web-search-plugin-factory.js")) { + pushEntry(inventory, { + provider: "shared", + file: relativeFile, + line: lineNumber, + reason: "imports web search provider registration helper outside extensions", + }); + } + } +} + +export async function collectWebSearchProviderBoundaryInventory() { + const inventory = []; + const files = ( + await Promise.all(scanRoots.map(async (root) => await walkFiles(path.join(repoRoot, root)))) + ) + .flat() + .toSorted((left, right) => + normalizeRelativePath(left).localeCompare(normalizeRelativePath(right)), + ); + + for (const filePath of files) { + const relativeFile = normalizeRelativePath(filePath); + if (ignoredFiles.has(relativeFile)) { + continue; + } + const content = await fs.readFile(filePath, "utf8"); + const lines = content.split(/\r?\n/); + + if (relativeFile === "src/plugins/web-search-providers.ts") { + scanWebSearchProviderRegistry(lines, relativeFile, inventory); + continue; + } + + if (relativeFile === "src/commands/onboard-search.ts") { + scanOnboardSearch(lines, relativeFile, inventory); + continue; + } + + scanGenericCoreImports(lines, relativeFile, inventory); + } + + return inventory.toSorted(compareInventoryEntries); +} + +export async function readExpectedInventory() { + return JSON.parse(await fs.readFile(baselinePath, "utf8")); +} + +export function diffInventory(expected, actual) { + const expectedKeys = new Set(expected.map((entry) => JSON.stringify(entry))); + const actualKeys = new Set(actual.map((entry) => JSON.stringify(entry))); + const missing = expected.filter((entry) => !actualKeys.has(JSON.stringify(entry))); + const unexpected = actual.filter((entry) => !expectedKeys.has(JSON.stringify(entry))); + return { + missing: missing.toSorted(compareInventoryEntries), + unexpected: unexpected.toSorted(compareInventoryEntries), + }; +} + +function formatInventoryHuman(inventory) { + if (inventory.length === 0) { + return "No web search provider boundary inventory entries found."; + } + const lines = ["Web search provider boundary inventory:"]; + let activeProvider = ""; + for (const entry of inventory) { + if (entry.provider !== activeProvider) { + activeProvider = entry.provider; + lines.push(`${activeProvider}:`); + } + lines.push(` - ${entry.file}:${entry.line} ${entry.reason}`); + } + return lines.join("\n"); +} + +function formatEntry(entry) { + return `${entry.provider} ${entry.file}:${entry.line} ${entry.reason}`; +} + +export async function main(argv = process.argv.slice(2)) { + const json = argv.includes("--json"); + const actual = await collectWebSearchProviderBoundaryInventory(); + const expected = await readExpectedInventory(); + const { missing, unexpected } = diffInventory(expected, actual); + const matchesBaseline = missing.length === 0 && unexpected.length === 0; + + if (json) { + process.stdout.write(`${JSON.stringify(actual, null, 2)}\n`); + } else { + console.log(formatInventoryHuman(actual)); + console.log( + matchesBaseline + ? `Baseline matches (${actual.length} entries).` + : `Baseline mismatch (${unexpected.length} unexpected, ${missing.length} missing).`, + ); + if (!matchesBaseline) { + if (unexpected.length > 0) { + console.error("Unexpected entries:"); + for (const entry of unexpected) { + console.error(`- ${formatEntry(entry)}`); + } + } + if (missing.length > 0) { + console.error("Missing baseline entries:"); + for (const entry of missing) { + console.error(`- ${formatEntry(entry)}`); + } + } + } + } + + if (!matchesBaseline) { + process.exit(1); + } +} + +runAsScript(import.meta.url, main); diff --git a/src/agents/tools/web-search-core.ts b/src/agents/tools/web-search-core.ts deleted file mode 100644 index bebc659c306..00000000000 --- a/src/agents/tools/web-search-core.ts +++ /dev/null @@ -1,2242 +0,0 @@ -import { Type } from "@sinclair/typebox"; -import { formatCliCommand } from "../../cli/command-format.js"; -import type { OpenClawConfig } from "../../config/config.js"; -import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js"; -import { logVerbose } from "../../globals.js"; -import type { RuntimeWebSearchMetadata } from "../../secrets/runtime-web-tools.types.js"; -import { wrapWebContent } from "../../security/external-content.js"; -import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; -import type { AnyAgentTool } from "./common.js"; -import { jsonResult, readNumberParam, readStringArrayParam, readStringParam } from "./common.js"; -import { withTrustedWebToolsEndpoint } from "./web-guarded-fetch.js"; -import { resolveCitationRedirectUrl } from "./web-search-citation-redirect.js"; -import { - CacheEntry, - DEFAULT_CACHE_TTL_MINUTES, - DEFAULT_TIMEOUT_SECONDS, - normalizeCacheKey, - readCache, - readResponseText, - resolveCacheTtlMs, - resolveTimeoutSeconds, - writeCache, -} from "./web-shared.js"; - -const SEARCH_PROVIDERS = ["brave", "gemini", "grok", "kimi", "perplexity"] as const; -type SearchProvider = (typeof SEARCH_PROVIDERS)[number]; -const DEFAULT_SEARCH_COUNT = 5; -const MAX_SEARCH_COUNT = 10; - -const BRAVE_SEARCH_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"; -const BRAVE_LLM_CONTEXT_ENDPOINT = "https://api.search.brave.com/res/v1/llm/context"; -const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; -const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; -const PERPLEXITY_SEARCH_ENDPOINT = "https://api.perplexity.ai/search"; -const DEFAULT_PERPLEXITY_MODEL = "perplexity/sonar-pro"; -const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; -const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; - -const XAI_API_ENDPOINT = "https://api.x.ai/v1/responses"; -const DEFAULT_GROK_MODEL = "grok-4-1-fast"; -const DEFAULT_KIMI_BASE_URL = "https://api.moonshot.ai/v1"; -const DEFAULT_KIMI_MODEL = "moonshot-v1-128k"; -const KIMI_WEB_SEARCH_TOOL = { - type: "builtin_function", - function: { name: "$web_search" }, -} as const; - -const SEARCH_CACHE_KEY = Symbol.for("openclaw.web-search.cache"); - -function getSharedSearchCache(): Map>> { - const root = globalThis as Record; - const existing = root[SEARCH_CACHE_KEY]; - if (existing instanceof Map) { - return existing as Map>>; - } - const next = new Map>>(); - root[SEARCH_CACHE_KEY] = next; - return next; -} - -const SEARCH_CACHE = getSharedSearchCache(); -const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]); -const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/; -const BRAVE_SEARCH_LANG_CODES = new Set([ - "ar", - "eu", - "bn", - "bg", - "ca", - "zh-hans", - "zh-hant", - "hr", - "cs", - "da", - "nl", - "en", - "en-gb", - "et", - "fi", - "fr", - "gl", - "de", - "el", - "gu", - "he", - "hi", - "hu", - "is", - "it", - "jp", - "kn", - "ko", - "lv", - "lt", - "ms", - "ml", - "mr", - "nb", - "pl", - "pt-br", - "pt-pt", - "pa", - "ro", - "ru", - "sr", - "sk", - "sl", - "es", - "sv", - "ta", - "te", - "th", - "tr", - "uk", - "vi", -]); -const BRAVE_SEARCH_LANG_ALIASES: Record = { - ja: "jp", - zh: "zh-hans", - "zh-cn": "zh-hans", - "zh-hk": "zh-hant", - "zh-sg": "zh-hans", - "zh-tw": "zh-hant", -}; -const BRAVE_UI_LANG_LOCALE = /^([a-z]{2})-([a-z]{2})$/i; -const PERPLEXITY_RECENCY_VALUES = new Set(["day", "week", "month", "year"]); - -const FRESHNESS_TO_RECENCY: Record = { - pd: "day", - pw: "week", - pm: "month", - py: "year", -}; -const RECENCY_TO_FRESHNESS: Record = { - day: "pd", - week: "pw", - month: "pm", - year: "py", -}; - -const ISO_DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/; -const PERPLEXITY_DATE_PATTERN = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/; - -function isoToPerplexityDate(iso: string): string | undefined { - const match = iso.match(ISO_DATE_PATTERN); - if (!match) { - return undefined; - } - const [, year, month, day] = match; - return `${parseInt(month, 10)}/${parseInt(day, 10)}/${year}`; -} - -function normalizeToIsoDate(value: string): string | undefined { - const trimmed = value.trim(); - if (ISO_DATE_PATTERN.test(trimmed)) { - return isValidIsoDate(trimmed) ? trimmed : undefined; - } - const match = trimmed.match(PERPLEXITY_DATE_PATTERN); - if (match) { - const [, month, day, year] = match; - const iso = `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`; - return isValidIsoDate(iso) ? iso : undefined; - } - return undefined; -} - -function createWebSearchSchema(params: { - provider: (typeof SEARCH_PROVIDERS)[number]; - perplexityTransport?: PerplexityTransport; -}) { - const querySchema = { - query: Type.String({ description: "Search query string." }), - count: Type.Optional( - Type.Number({ - description: "Number of results to return (1-10).", - minimum: 1, - maximum: MAX_SEARCH_COUNT, - }), - ), - } as const; - - const filterSchema = { - country: Type.Optional( - Type.String({ - description: - "2-letter country code for region-specific results (e.g., 'DE', 'US', 'ALL'). Default: 'US'.", - }), - ), - language: Type.Optional( - Type.String({ - description: "ISO 639-1 language code for results (e.g., 'en', 'de', 'fr').", - }), - ), - freshness: Type.Optional( - Type.String({ - description: "Filter by time: 'day' (24h), 'week', 'month', or 'year'.", - }), - ), - date_after: Type.Optional( - Type.String({ - description: "Only results published after this date (YYYY-MM-DD).", - }), - ), - date_before: Type.Optional( - Type.String({ - description: "Only results published before this date (YYYY-MM-DD).", - }), - ), - } as const; - - const perplexityStructuredFilterSchema = { - country: Type.Optional( - Type.String({ - description: - "Native Perplexity Search API only. 2-letter country code for region-specific results (e.g., 'DE', 'US', 'ALL'). Default: 'US'.", - }), - ), - language: Type.Optional( - Type.String({ - description: - "Native Perplexity Search API only. ISO 639-1 language code for results (e.g., 'en', 'de', 'fr').", - }), - ), - date_after: Type.Optional( - Type.String({ - description: - "Native Perplexity Search API only. Only results published after this date (YYYY-MM-DD).", - }), - ), - date_before: Type.Optional( - Type.String({ - description: - "Native Perplexity Search API only. Only results published before this date (YYYY-MM-DD).", - }), - ), - } as const; - - if (params.provider === "brave") { - return Type.Object({ - ...querySchema, - ...filterSchema, - search_lang: Type.Optional( - Type.String({ - description: - "Brave language code for search results (e.g., 'en', 'de', 'en-gb', 'zh-hans', 'zh-hant', 'pt-br').", - }), - ), - ui_lang: Type.Optional( - Type.String({ - description: - "Locale code for UI elements in language-region format (e.g., 'en-US', 'de-DE', 'fr-FR', 'tr-TR'). Must include region subtag.", - }), - ), - }); - } - - if (params.provider === "perplexity") { - if (params.perplexityTransport === "chat_completions") { - return Type.Object({ - ...querySchema, - freshness: filterSchema.freshness, - }); - } - return Type.Object({ - ...querySchema, - freshness: filterSchema.freshness, - ...perplexityStructuredFilterSchema, - domain_filter: Type.Optional( - Type.Array(Type.String(), { - description: - "Native Perplexity Search API only. Domain filter (max 20). Allowlist: ['nature.com'] or denylist: ['-reddit.com']. Cannot mix.", - }), - ), - max_tokens: Type.Optional( - Type.Number({ - description: - "Native Perplexity Search API only. Total content budget across all results (default: 25000, max: 1000000).", - minimum: 1, - maximum: 1000000, - }), - ), - max_tokens_per_page: Type.Optional( - Type.Number({ - description: - "Native Perplexity Search API only. Max tokens extracted per page (default: 2048).", - minimum: 1, - }), - ), - }); - } - - // grok, gemini, kimi, etc. - return Type.Object({ - ...querySchema, - ...filterSchema, - }); -} - -type WebSearchConfig = NonNullable["web"] extends infer Web - ? Web extends { search?: infer Search } - ? Search - : undefined - : undefined; - -type BraveSearchResult = { - title?: string; - url?: string; - description?: string; - age?: string; -}; - -type BraveSearchResponse = { - web?: { - results?: BraveSearchResult[]; - }; -}; - -type BraveLlmContextResult = { url: string; title: string; snippets: string[] }; -type BraveLlmContextResponse = { - grounding: { generic?: BraveLlmContextResult[] }; - sources?: { url?: string; hostname?: string; date?: string }[]; -}; - -type BraveConfig = { - mode?: string; -}; - -type PerplexityConfig = { - apiKey?: string; - baseUrl?: string; - model?: string; -}; - -type PerplexityApiKeySource = "config" | "perplexity_env" | "openrouter_env" | "none"; -type PerplexityTransport = "search_api" | "chat_completions"; -type PerplexityBaseUrlHint = "direct" | "openrouter"; - -type GrokConfig = { - apiKey?: string; - model?: string; - inlineCitations?: boolean; -}; - -type KimiConfig = { - apiKey?: string; - baseUrl?: string; - model?: string; -}; - -type GrokSearchResponse = { - output?: Array<{ - type?: string; - role?: string; - text?: string; // present when type === "output_text" (top-level output_text block) - content?: Array<{ - type?: string; - text?: string; - annotations?: Array<{ - type?: string; - url?: string; - start_index?: number; - end_index?: number; - }>; - }>; - annotations?: Array<{ - type?: string; - url?: string; - start_index?: number; - end_index?: number; - }>; - }>; - output_text?: string; // deprecated field - kept for backwards compatibility - citations?: string[]; - inline_citations?: Array<{ - start_index: number; - end_index: number; - url: string; - }>; -}; - -type KimiToolCall = { - id?: string; - type?: string; - function?: { - name?: string; - arguments?: string; - }; -}; - -type KimiMessage = { - role?: string; - content?: string; - reasoning_content?: string; - tool_calls?: KimiToolCall[]; -}; - -type KimiSearchResponse = { - choices?: Array<{ - finish_reason?: string; - message?: KimiMessage; - }>; - search_results?: Array<{ - title?: string; - url?: string; - content?: string; - }>; -}; - -type PerplexitySearchResponse = { - choices?: Array<{ - message?: { - content?: string; - annotations?: Array<{ - type?: string; - url?: string; - url_citation?: { - url?: string; - title?: string; - start_index?: number; - end_index?: number; - }; - }>; - }; - }>; - citations?: string[]; -}; - -type PerplexitySearchApiResult = { - title?: string; - url?: string; - snippet?: string; - date?: string; - last_updated?: string; -}; - -type PerplexitySearchApiResponse = { - results?: PerplexitySearchApiResult[]; - id?: string; -}; - -function extractPerplexityCitations(data: PerplexitySearchResponse): string[] { - const normalizeUrl = (value: unknown): string | undefined => { - if (typeof value !== "string") { - return undefined; - } - const trimmed = value.trim(); - return trimmed ? trimmed : undefined; - }; - - const topLevel = (data.citations ?? []) - .map(normalizeUrl) - .filter((url): url is string => Boolean(url)); - if (topLevel.length > 0) { - return [...new Set(topLevel)]; - } - - const citations: string[] = []; - for (const choice of data.choices ?? []) { - for (const annotation of choice.message?.annotations ?? []) { - if (annotation.type !== "url_citation") { - continue; - } - const url = normalizeUrl(annotation.url_citation?.url ?? annotation.url); - if (url) { - citations.push(url); - } - } - } - - return [...new Set(citations)]; -} - -function extractGrokContent(data: GrokSearchResponse): { - text: string | undefined; - annotationCitations: string[]; -} { - // xAI Responses API format: find the message output with text content - for (const output of data.output ?? []) { - if (output.type === "message") { - for (const block of output.content ?? []) { - if (block.type === "output_text" && typeof block.text === "string" && block.text) { - const urls = (block.annotations ?? []) - .filter((a) => a.type === "url_citation" && typeof a.url === "string") - .map((a) => a.url as string); - return { text: block.text, annotationCitations: [...new Set(urls)] }; - } - } - } - // Some xAI responses place output_text blocks directly in the output array - // without a message wrapper. - if ( - output.type === "output_text" && - "text" in output && - typeof output.text === "string" && - output.text - ) { - const rawAnnotations = - "annotations" in output && Array.isArray(output.annotations) ? output.annotations : []; - const urls = rawAnnotations - .filter( - (a: Record) => a.type === "url_citation" && typeof a.url === "string", - ) - .map((a: Record) => a.url as string); - return { text: output.text, annotationCitations: [...new Set(urls)] }; - } - } - // Fallback: deprecated output_text field - const text = typeof data.output_text === "string" ? data.output_text : undefined; - return { text, annotationCitations: [] }; -} - -type GeminiConfig = { - apiKey?: string; - model?: string; -}; - -type GeminiGroundingResponse = { - candidates?: Array<{ - content?: { - parts?: Array<{ - text?: string; - }>; - }; - groundingMetadata?: { - groundingChunks?: Array<{ - web?: { - uri?: string; - title?: string; - }; - }>; - searchEntryPoint?: { - renderedContent?: string; - }; - webSearchQueries?: string[]; - }; - }>; - error?: { - code?: number; - message?: string; - status?: string; - }; -}; - -const DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"; -const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; - -function resolveSearchConfig(cfg?: OpenClawConfig): WebSearchConfig { - const search = cfg?.tools?.web?.search; - if (!search || typeof search !== "object") { - return undefined; - } - return search as WebSearchConfig; -} - -function resolveSearchEnabled(params: { search?: WebSearchConfig; sandboxed?: boolean }): boolean { - if (typeof params.search?.enabled === "boolean") { - return params.search.enabled; - } - if (params.sandboxed) { - return true; - } - return true; -} - -function resolveSearchApiKey(search?: WebSearchConfig): string | undefined { - const fromConfigRaw = - search && "apiKey" in search - ? normalizeResolvedSecretInputString({ - value: search.apiKey, - path: "tools.web.search.apiKey", - }) - : undefined; - const fromConfig = normalizeSecretInput(fromConfigRaw); - const fromEnv = normalizeSecretInput(process.env.BRAVE_API_KEY); - return fromConfig || fromEnv || undefined; -} - -function missingSearchKeyPayload(provider: (typeof SEARCH_PROVIDERS)[number]) { - if (provider === "brave") { - return { - error: "missing_brave_api_key", - message: `web_search (brave) needs a Brave Search API key. Run \`${formatCliCommand("openclaw configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`, - docs: "https://docs.openclaw.ai/tools/web", - }; - } - if (provider === "gemini") { - return { - error: "missing_gemini_api_key", - message: - "web_search (gemini) needs an API key. Set GEMINI_API_KEY in the Gateway environment, or configure tools.web.search.gemini.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; - } - if (provider === "grok") { - return { - error: "missing_xai_api_key", - message: - "web_search (grok) needs an xAI API key. Set XAI_API_KEY in the Gateway environment, or configure tools.web.search.grok.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; - } - if (provider === "kimi") { - return { - error: "missing_kimi_api_key", - message: - "web_search (kimi) needs a Moonshot API key. Set KIMI_API_KEY or MOONSHOT_API_KEY in the Gateway environment, or configure tools.web.search.kimi.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; - } - return { - error: "missing_perplexity_api_key", - message: - "web_search (perplexity) needs an API key. Set PERPLEXITY_API_KEY or OPENROUTER_API_KEY in the Gateway environment, or configure tools.web.search.perplexity.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; -} - -function isSearchProvider(value: string): value is SearchProvider { - return SEARCH_PROVIDERS.includes(value as SearchProvider); -} - -function resolveSearchProvider(search?: WebSearchConfig): (typeof SEARCH_PROVIDERS)[number] { - const raw = - search && "provider" in search && typeof search.provider === "string" - ? search.provider.trim().toLowerCase() - : ""; - if (raw === "brave") { - return "brave"; - } - if (raw === "gemini") { - return "gemini"; - } - if (raw === "grok") { - return "grok"; - } - if (raw === "kimi") { - return "kimi"; - } - if (raw === "perplexity") { - return "perplexity"; - } - - // Auto-detect provider from available API keys (alphabetical order) - if (raw === "") { - // Brave - if (resolveSearchApiKey(search)) { - logVerbose( - 'web_search: no provider configured, auto-detected "brave" from available API keys', - ); - return "brave"; - } - // Gemini - const geminiConfig = resolveGeminiConfig(search); - if (resolveGeminiApiKey(geminiConfig)) { - logVerbose( - 'web_search: no provider configured, auto-detected "gemini" from available API keys', - ); - return "gemini"; - } - // Grok - const grokConfig = resolveGrokConfig(search); - if (resolveGrokApiKey(grokConfig)) { - logVerbose( - 'web_search: no provider configured, auto-detected "grok" from available API keys', - ); - return "grok"; - } - // Kimi - const kimiConfig = resolveKimiConfig(search); - if (resolveKimiApiKey(kimiConfig)) { - logVerbose( - 'web_search: no provider configured, auto-detected "kimi" from available API keys', - ); - return "kimi"; - } - // Perplexity - const perplexityConfig = resolvePerplexityConfig(search); - const { apiKey: perplexityKey } = resolvePerplexityApiKey(perplexityConfig); - if (perplexityKey) { - logVerbose( - 'web_search: no provider configured, auto-detected "perplexity" from available API keys', - ); - return "perplexity"; - } - } - - return "brave"; -} - -function resolveBraveConfig(search?: WebSearchConfig): BraveConfig { - if (!search || typeof search !== "object") { - return {}; - } - const brave = "brave" in search ? search.brave : undefined; - if (!brave || typeof brave !== "object") { - return {}; - } - return brave as BraveConfig; -} - -function resolveBraveMode(brave: BraveConfig): "web" | "llm-context" { - return brave.mode === "llm-context" ? "llm-context" : "web"; -} - -function resolvePerplexityConfig(search?: WebSearchConfig): PerplexityConfig { - if (!search || typeof search !== "object") { - return {}; - } - const perplexity = "perplexity" in search ? search.perplexity : undefined; - if (!perplexity || typeof perplexity !== "object") { - return {}; - } - return perplexity as PerplexityConfig; -} - -function resolvePerplexityApiKey(perplexity?: PerplexityConfig): { - apiKey?: string; - source: PerplexityApiKeySource; -} { - const fromConfig = normalizeApiKey(perplexity?.apiKey); - if (fromConfig) { - return { apiKey: fromConfig, source: "config" }; - } - - const fromEnvPerplexity = normalizeApiKey(process.env.PERPLEXITY_API_KEY); - if (fromEnvPerplexity) { - return { apiKey: fromEnvPerplexity, source: "perplexity_env" }; - } - - const fromEnvOpenRouter = normalizeApiKey(process.env.OPENROUTER_API_KEY); - if (fromEnvOpenRouter) { - return { apiKey: fromEnvOpenRouter, source: "openrouter_env" }; - } - - return { apiKey: undefined, source: "none" }; -} - -function normalizeApiKey(key: unknown): string { - return normalizeSecretInput(key); -} - -function inferPerplexityBaseUrlFromApiKey(apiKey?: string): PerplexityBaseUrlHint | undefined { - if (!apiKey) { - return undefined; - } - const normalized = apiKey.toLowerCase(); - if (PERPLEXITY_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { - return "direct"; - } - if (OPENROUTER_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { - return "openrouter"; - } - return undefined; -} - -function resolvePerplexityBaseUrl( - perplexity?: PerplexityConfig, - authSource: PerplexityApiKeySource = "none", // pragma: allowlist secret - configuredKey?: string, -): string { - const fromConfig = - perplexity && "baseUrl" in perplexity && typeof perplexity.baseUrl === "string" - ? perplexity.baseUrl.trim() - : ""; - if (fromConfig) { - return fromConfig; - } - if (authSource === "perplexity_env") { - return PERPLEXITY_DIRECT_BASE_URL; - } - if (authSource === "openrouter_env") { - return DEFAULT_PERPLEXITY_BASE_URL; - } - if (authSource === "config") { - const inferred = inferPerplexityBaseUrlFromApiKey(configuredKey); - if (inferred === "openrouter") { - return DEFAULT_PERPLEXITY_BASE_URL; - } - return PERPLEXITY_DIRECT_BASE_URL; - } - return DEFAULT_PERPLEXITY_BASE_URL; -} - -function resolvePerplexityModel(perplexity?: PerplexityConfig): string { - const fromConfig = - perplexity && "model" in perplexity && typeof perplexity.model === "string" - ? perplexity.model.trim() - : ""; - return fromConfig || DEFAULT_PERPLEXITY_MODEL; -} - -function isDirectPerplexityBaseUrl(baseUrl: string): boolean { - const trimmed = baseUrl.trim(); - if (!trimmed) { - return false; - } - try { - return new URL(trimmed).hostname.toLowerCase() === "api.perplexity.ai"; - } catch { - return false; - } -} - -function resolvePerplexityRequestModel(baseUrl: string, model: string): string { - if (!isDirectPerplexityBaseUrl(baseUrl)) { - return model; - } - return model.startsWith("perplexity/") ? model.slice("perplexity/".length) : model; -} - -function resolvePerplexityTransport(perplexity?: PerplexityConfig): { - apiKey?: string; - source: PerplexityApiKeySource; - baseUrl: string; - model: string; - transport: PerplexityTransport; -} { - const auth = resolvePerplexityApiKey(perplexity); - const baseUrl = resolvePerplexityBaseUrl(perplexity, auth.source, auth.apiKey); - const model = resolvePerplexityModel(perplexity); - const hasLegacyOverride = Boolean( - (perplexity?.baseUrl && perplexity.baseUrl.trim()) || - (perplexity?.model && perplexity.model.trim()), - ); - return { - ...auth, - baseUrl, - model, - transport: - hasLegacyOverride || !isDirectPerplexityBaseUrl(baseUrl) ? "chat_completions" : "search_api", - }; -} - -function resolvePerplexitySchemaTransportHint( - perplexity?: PerplexityConfig, -): PerplexityTransport | undefined { - const hasLegacyOverride = Boolean( - (perplexity?.baseUrl && perplexity.baseUrl.trim()) || - (perplexity?.model && perplexity.model.trim()), - ); - return hasLegacyOverride ? "chat_completions" : undefined; -} - -function resolveGrokConfig(search?: WebSearchConfig): GrokConfig { - if (!search || typeof search !== "object") { - return {}; - } - const grok = "grok" in search ? search.grok : undefined; - if (!grok || typeof grok !== "object") { - return {}; - } - return grok as GrokConfig; -} - -function resolveGrokApiKey(grok?: GrokConfig): string | undefined { - const fromConfig = normalizeApiKey(grok?.apiKey); - if (fromConfig) { - return fromConfig; - } - const fromEnv = normalizeApiKey(process.env.XAI_API_KEY); - return fromEnv || undefined; -} - -function resolveGrokModel(grok?: GrokConfig): string { - const fromConfig = - grok && "model" in grok && typeof grok.model === "string" ? grok.model.trim() : ""; - return fromConfig || DEFAULT_GROK_MODEL; -} - -function resolveGrokInlineCitations(grok?: GrokConfig): boolean { - return grok?.inlineCitations === true; -} - -function resolveKimiConfig(search?: WebSearchConfig): KimiConfig { - if (!search || typeof search !== "object") { - return {}; - } - const kimi = "kimi" in search ? search.kimi : undefined; - if (!kimi || typeof kimi !== "object") { - return {}; - } - return kimi as KimiConfig; -} - -function resolveKimiApiKey(kimi?: KimiConfig): string | undefined { - const fromConfig = normalizeApiKey(kimi?.apiKey); - if (fromConfig) { - return fromConfig; - } - const fromEnvKimi = normalizeApiKey(process.env.KIMI_API_KEY); - if (fromEnvKimi) { - return fromEnvKimi; - } - const fromEnvMoonshot = normalizeApiKey(process.env.MOONSHOT_API_KEY); - return fromEnvMoonshot || undefined; -} - -function resolveKimiModel(kimi?: KimiConfig): string { - const fromConfig = - kimi && "model" in kimi && typeof kimi.model === "string" ? kimi.model.trim() : ""; - return fromConfig || DEFAULT_KIMI_MODEL; -} - -function resolveKimiBaseUrl(kimi?: KimiConfig): string { - const fromConfig = - kimi && "baseUrl" in kimi && typeof kimi.baseUrl === "string" ? kimi.baseUrl.trim() : ""; - return fromConfig || DEFAULT_KIMI_BASE_URL; -} - -function resolveGeminiConfig(search?: WebSearchConfig): GeminiConfig { - if (!search || typeof search !== "object") { - return {}; - } - const gemini = "gemini" in search ? search.gemini : undefined; - if (!gemini || typeof gemini !== "object") { - return {}; - } - return gemini as GeminiConfig; -} - -function resolveGeminiApiKey(gemini?: GeminiConfig): string | undefined { - const fromConfig = normalizeApiKey(gemini?.apiKey); - if (fromConfig) { - return fromConfig; - } - const fromEnv = normalizeApiKey(process.env.GEMINI_API_KEY); - return fromEnv || undefined; -} - -function resolveGeminiModel(gemini?: GeminiConfig): string { - const fromConfig = - gemini && "model" in gemini && typeof gemini.model === "string" ? gemini.model.trim() : ""; - return fromConfig || DEFAULT_GEMINI_MODEL; -} - -async function withTrustedWebSearchEndpoint( - params: { - url: string; - timeoutSeconds: number; - init: RequestInit; - }, - run: (response: Response) => Promise, -): Promise { - return withTrustedWebToolsEndpoint( - { - url: params.url, - init: params.init, - timeoutSeconds: params.timeoutSeconds, - }, - async ({ response }) => run(response), - ); -} - -async function runGeminiSearch(params: { - query: string; - apiKey: string; - model: string; - timeoutSeconds: number; -}): Promise<{ content: string; citations: Array<{ url: string; title?: string }> }> { - const endpoint = `${GEMINI_API_BASE}/models/${params.model}:generateContent`; - - return withTrustedWebSearchEndpoint( - { - url: endpoint, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-goog-api-key": params.apiKey, - }, - body: JSON.stringify({ - contents: [ - { - parts: [{ text: params.query }], - }, - ], - tools: [{ google_search: {} }], - }), - }, - }, - async (res) => { - if (!res.ok) { - const detailResult = await readResponseText(res, { maxBytes: 64_000 }); - // Strip API key from any error detail to prevent accidental key leakage in logs - const safeDetail = (detailResult.text || res.statusText).replace( - /key=[^&\s]+/gi, - "key=***", - ); - throw new Error(`Gemini API error (${res.status}): ${safeDetail}`); - } - - let data: GeminiGroundingResponse; - try { - data = (await res.json()) as GeminiGroundingResponse; - } catch (err) { - const safeError = String(err).replace(/key=[^&\s]+/gi, "key=***"); - throw new Error(`Gemini API returned invalid JSON: ${safeError}`, { cause: err }); - } - - if (data.error) { - const rawMsg = data.error.message || data.error.status || "unknown"; - const safeMsg = rawMsg.replace(/key=[^&\s]+/gi, "key=***"); - throw new Error(`Gemini API error (${data.error.code}): ${safeMsg}`); - } - - const candidate = data.candidates?.[0]; - const content = - candidate?.content?.parts - ?.map((p) => p.text) - .filter(Boolean) - .join("\n") ?? "No response"; - - const groundingChunks = candidate?.groundingMetadata?.groundingChunks ?? []; - const rawCitations = groundingChunks - .filter((chunk) => chunk.web?.uri) - .map((chunk) => ({ - url: chunk.web!.uri!, - title: chunk.web?.title || undefined, - })); - - // Resolve Google grounding redirect URLs to direct URLs with concurrency cap. - // Gemini typically returns 3-8 citations; cap at 10 concurrent to be safe. - const MAX_CONCURRENT_REDIRECTS = 10; - const citations: Array<{ url: string; title?: string }> = []; - for (let i = 0; i < rawCitations.length; i += MAX_CONCURRENT_REDIRECTS) { - const batch = rawCitations.slice(i, i + MAX_CONCURRENT_REDIRECTS); - const resolved = await Promise.all( - batch.map(async (citation) => { - const resolvedUrl = await resolveCitationRedirectUrl(citation.url); - return { ...citation, url: resolvedUrl }; - }), - ); - citations.push(...resolved); - } - - return { content, citations }; - }, - ); -} - -function resolveSearchCount(value: unknown, fallback: number): number { - const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback; - const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed))); - return clamped; -} - -function normalizeBraveSearchLang(value: string | undefined): string | undefined { - if (!value) { - return undefined; - } - const trimmed = value.trim(); - if (!trimmed) { - return undefined; - } - const canonical = BRAVE_SEARCH_LANG_ALIASES[trimmed.toLowerCase()] ?? trimmed.toLowerCase(); - if (!BRAVE_SEARCH_LANG_CODES.has(canonical)) { - return undefined; - } - return canonical; -} - -function normalizeBraveUiLang(value: string | undefined): string | undefined { - if (!value) { - return undefined; - } - const trimmed = value.trim(); - if (!trimmed) { - return undefined; - } - const match = trimmed.match(BRAVE_UI_LANG_LOCALE); - if (!match) { - return undefined; - } - const [, language, region] = match; - return `${language.toLowerCase()}-${region.toUpperCase()}`; -} - -function normalizeBraveLanguageParams(params: { search_lang?: string; ui_lang?: string }): { - search_lang?: string; - ui_lang?: string; - invalidField?: "search_lang" | "ui_lang"; -} { - const rawSearchLang = params.search_lang?.trim() || undefined; - const rawUiLang = params.ui_lang?.trim() || undefined; - let searchLangCandidate = rawSearchLang; - let uiLangCandidate = rawUiLang; - - // Recover common LLM mix-up: locale in search_lang + short code in ui_lang. - if (normalizeBraveUiLang(rawSearchLang) && normalizeBraveSearchLang(rawUiLang)) { - searchLangCandidate = rawUiLang; - uiLangCandidate = rawSearchLang; - } - - const search_lang = normalizeBraveSearchLang(searchLangCandidate); - if (searchLangCandidate && !search_lang) { - return { invalidField: "search_lang" }; - } - - const ui_lang = normalizeBraveUiLang(uiLangCandidate); - if (uiLangCandidate && !ui_lang) { - return { invalidField: "ui_lang" }; - } - - return { search_lang, ui_lang }; -} - -/** - * Normalizes freshness shortcut to the provider's expected format. - * Accepts both Brave format (pd/pw/pm/py) and Perplexity format (day/week/month/year). - * For Brave, also accepts date ranges (YYYY-MM-DDtoYYYY-MM-DD). - */ -function normalizeFreshness( - value: string | undefined, - provider: (typeof SEARCH_PROVIDERS)[number], -): string | undefined { - if (!value) { - return undefined; - } - const trimmed = value.trim(); - if (!trimmed) { - return undefined; - } - - const lower = trimmed.toLowerCase(); - - if (BRAVE_FRESHNESS_SHORTCUTS.has(lower)) { - return provider === "brave" ? lower : FRESHNESS_TO_RECENCY[lower]; - } - - if (PERPLEXITY_RECENCY_VALUES.has(lower)) { - return provider === "perplexity" ? lower : RECENCY_TO_FRESHNESS[lower]; - } - - // Brave date range support - if (provider === "brave") { - const match = trimmed.match(BRAVE_FRESHNESS_RANGE); - if (match) { - const [, start, end] = match; - if (isValidIsoDate(start) && isValidIsoDate(end) && start <= end) { - return `${start}to${end}`; - } - } - } - - return undefined; -} - -function isValidIsoDate(value: string): boolean { - if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) { - return false; - } - const [year, month, day] = value.split("-").map((part) => Number.parseInt(part, 10)); - if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) { - return false; - } - - const date = new Date(Date.UTC(year, month - 1, day)); - return ( - date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day - ); -} - -function resolveSiteName(url: string | undefined): string | undefined { - if (!url) { - return undefined; - } - try { - return new URL(url).hostname; - } catch { - return undefined; - } -} - -async function throwWebSearchApiError(res: Response, providerLabel: string): Promise { - const detailResult = await readResponseText(res, { maxBytes: 64_000 }); - const detail = detailResult.text; - throw new Error(`${providerLabel} API error (${res.status}): ${detail || res.statusText}`); -} - -async function runPerplexitySearchApi(params: { - query: string; - apiKey: string; - count: number; - timeoutSeconds: number; - country?: string; - searchDomainFilter?: string[]; - searchRecencyFilter?: string; - searchLanguageFilter?: string[]; - searchAfterDate?: string; - searchBeforeDate?: string; - maxTokens?: number; - maxTokensPerPage?: number; -}): Promise< - Array<{ title: string; url: string; description: string; published?: string; siteName?: string }> -> { - const body: Record = { - query: params.query, - max_results: params.count, - }; - - if (params.country) { - body.country = params.country; - } - if (params.searchDomainFilter && params.searchDomainFilter.length > 0) { - body.search_domain_filter = params.searchDomainFilter; - } - if (params.searchRecencyFilter) { - body.search_recency_filter = params.searchRecencyFilter; - } - if (params.searchLanguageFilter && params.searchLanguageFilter.length > 0) { - body.search_language_filter = params.searchLanguageFilter; - } - if (params.searchAfterDate) { - body.search_after_date = params.searchAfterDate; - } - if (params.searchBeforeDate) { - body.search_before_date = params.searchBeforeDate; - } - if (params.maxTokens !== undefined) { - body.max_tokens = params.maxTokens; - } - if (params.maxTokensPerPage !== undefined) { - body.max_tokens_per_page = params.maxTokensPerPage; - } - - return withTrustedWebSearchEndpoint( - { - url: PERPLEXITY_SEARCH_ENDPOINT, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - Authorization: `Bearer ${params.apiKey}`, - "HTTP-Referer": "https://openclaw.ai", - "X-Title": "OpenClaw Web Search", - }, - body: JSON.stringify(body), - }, - }, - async (res) => { - if (!res.ok) { - return await throwWebSearchApiError(res, "Perplexity Search"); - } - - const data = (await res.json()) as PerplexitySearchApiResponse; - const results = Array.isArray(data.results) ? data.results : []; - - return results.map((entry) => { - const title = entry.title ?? ""; - const url = entry.url ?? ""; - const snippet = entry.snippet ?? ""; - return { - title: title ? wrapWebContent(title, "web_search") : "", - url, - description: snippet ? wrapWebContent(snippet, "web_search") : "", - published: entry.date ?? undefined, - siteName: resolveSiteName(url) || undefined, - }; - }); - }, - ); -} - -async function runPerplexitySearch(params: { - query: string; - apiKey: string; - baseUrl: string; - model: string; - timeoutSeconds: number; - freshness?: string; -}): Promise<{ content: string; citations: string[] }> { - const baseUrl = params.baseUrl.trim().replace(/\/$/, ""); - const endpoint = `${baseUrl}/chat/completions`; - const model = resolvePerplexityRequestModel(baseUrl, params.model); - - const body: Record = { - model, - messages: [ - { - role: "user", - content: params.query, - }, - ], - }; - - if (params.freshness) { - body.search_recency_filter = params.freshness; - } - - return withTrustedWebSearchEndpoint( - { - url: endpoint, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${params.apiKey}`, - "HTTP-Referer": "https://openclaw.ai", - "X-Title": "OpenClaw Web Search", - }, - body: JSON.stringify(body), - }, - }, - async (res) => { - if (!res.ok) { - return await throwWebSearchApiError(res, "Perplexity"); - } - - const data = (await res.json()) as PerplexitySearchResponse; - const content = data.choices?.[0]?.message?.content ?? "No response"; - // Prefer top-level citations; fall back to OpenRouter-style message annotations. - const citations = extractPerplexityCitations(data); - - return { content, citations }; - }, - ); -} - -async function runGrokSearch(params: { - query: string; - apiKey: string; - model: string; - timeoutSeconds: number; - inlineCitations: boolean; -}): Promise<{ - content: string; - citations: string[]; - inlineCitations?: GrokSearchResponse["inline_citations"]; -}> { - const body: Record = { - model: params.model, - input: [ - { - role: "user", - content: params.query, - }, - ], - tools: [{ type: "web_search" }], - }; - - // Note: xAI's /v1/responses endpoint does not support the `include` - // parameter (returns 400 "Argument not supported: include"). Inline - // citations are returned automatically when available — we just parse - // them from the response without requesting them explicitly (#12910). - - return withTrustedWebSearchEndpoint( - { - url: XAI_API_ENDPOINT, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${params.apiKey}`, - }, - body: JSON.stringify(body), - }, - }, - async (res) => { - if (!res.ok) { - return await throwWebSearchApiError(res, "xAI"); - } - - const data = (await res.json()) as GrokSearchResponse; - const { text: extractedText, annotationCitations } = extractGrokContent(data); - const content = extractedText ?? "No response"; - // Prefer top-level citations; fall back to annotation-derived ones - const citations = (data.citations ?? []).length > 0 ? data.citations! : annotationCitations; - const inlineCitations = data.inline_citations; - - return { content, citations, inlineCitations }; - }, - ); -} - -function extractKimiMessageText(message: KimiMessage | undefined): string | undefined { - const content = message?.content?.trim(); - if (content) { - return content; - } - const reasoning = message?.reasoning_content?.trim(); - return reasoning || undefined; -} - -function extractKimiCitations(data: KimiSearchResponse): string[] { - const citations = (data.search_results ?? []) - .map((entry) => entry.url?.trim()) - .filter((url): url is string => Boolean(url)); - - for (const toolCall of data.choices?.[0]?.message?.tool_calls ?? []) { - const rawArguments = toolCall.function?.arguments; - if (!rawArguments) { - continue; - } - try { - const parsed = JSON.parse(rawArguments) as { - search_results?: Array<{ url?: string }>; - url?: string; - }; - if (typeof parsed.url === "string" && parsed.url.trim()) { - citations.push(parsed.url.trim()); - } - for (const result of parsed.search_results ?? []) { - if (typeof result.url === "string" && result.url.trim()) { - citations.push(result.url.trim()); - } - } - } catch { - // ignore malformed tool arguments - } - } - - return [...new Set(citations)]; -} - -function buildKimiToolResultContent(data: KimiSearchResponse): string { - return JSON.stringify({ - search_results: (data.search_results ?? []).map((entry) => ({ - title: entry.title ?? "", - url: entry.url ?? "", - content: entry.content ?? "", - })), - }); -} - -async function runKimiSearch(params: { - query: string; - apiKey: string; - baseUrl: string; - model: string; - timeoutSeconds: number; -}): Promise<{ content: string; citations: string[] }> { - const baseUrl = params.baseUrl.trim().replace(/\/$/, ""); - const endpoint = `${baseUrl}/chat/completions`; - const messages: Array> = [ - { - role: "user", - content: params.query, - }, - ]; - const collectedCitations = new Set(); - const MAX_ROUNDS = 3; - - for (let round = 0; round < MAX_ROUNDS; round += 1) { - const nextResult = await withTrustedWebSearchEndpoint( - { - url: endpoint, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${params.apiKey}`, - }, - body: JSON.stringify({ - model: params.model, - messages, - tools: [KIMI_WEB_SEARCH_TOOL], - }), - }, - }, - async ( - res, - ): Promise<{ done: true; content: string; citations: string[] } | { done: false }> => { - if (!res.ok) { - return await throwWebSearchApiError(res, "Kimi"); - } - - const data = (await res.json()) as KimiSearchResponse; - for (const citation of extractKimiCitations(data)) { - collectedCitations.add(citation); - } - const choice = data.choices?.[0]; - const message = choice?.message; - const text = extractKimiMessageText(message); - const toolCalls = message?.tool_calls ?? []; - - if (choice?.finish_reason !== "tool_calls" || toolCalls.length === 0) { - return { done: true, content: text ?? "No response", citations: [...collectedCitations] }; - } - - messages.push({ - role: "assistant", - content: message?.content ?? "", - ...(message?.reasoning_content - ? { - reasoning_content: message.reasoning_content, - } - : {}), - tool_calls: toolCalls, - }); - - const toolContent = buildKimiToolResultContent(data); - let pushedToolResult = false; - for (const toolCall of toolCalls) { - const toolCallId = toolCall.id?.trim(); - if (!toolCallId) { - continue; - } - pushedToolResult = true; - messages.push({ - role: "tool", - tool_call_id: toolCallId, - content: toolContent, - }); - } - - if (!pushedToolResult) { - return { done: true, content: text ?? "No response", citations: [...collectedCitations] }; - } - - return { done: false }; - }, - ); - - if (nextResult.done) { - return { content: nextResult.content, citations: nextResult.citations }; - } - } - - return { - content: "Search completed but no final answer was produced.", - citations: [...collectedCitations], - }; -} - -function mapBraveLlmContextResults( - data: BraveLlmContextResponse, -): { url: string; title: string; snippets: string[]; siteName?: string }[] { - const genericResults = Array.isArray(data.grounding?.generic) ? data.grounding.generic : []; - return genericResults.map((entry) => ({ - url: entry.url ?? "", - title: entry.title ?? "", - snippets: (entry.snippets ?? []).filter((s) => typeof s === "string" && s.length > 0), - siteName: resolveSiteName(entry.url) || undefined, - })); -} - -async function runBraveLlmContextSearch(params: { - query: string; - apiKey: string; - timeoutSeconds: number; - country?: string; - search_lang?: string; - freshness?: string; -}): Promise<{ - results: Array<{ - url: string; - title: string; - snippets: string[]; - siteName?: string; - }>; - sources?: BraveLlmContextResponse["sources"]; -}> { - const url = new URL(BRAVE_LLM_CONTEXT_ENDPOINT); - url.searchParams.set("q", params.query); - if (params.country) { - url.searchParams.set("country", params.country); - } - if (params.search_lang) { - url.searchParams.set("search_lang", params.search_lang); - } - if (params.freshness) { - url.searchParams.set("freshness", params.freshness); - } - - return withTrustedWebSearchEndpoint( - { - url: url.toString(), - timeoutSeconds: params.timeoutSeconds, - init: { - method: "GET", - headers: { - Accept: "application/json", - "X-Subscription-Token": params.apiKey, - }, - }, - }, - async (res) => { - if (!res.ok) { - const detailResult = await readResponseText(res, { maxBytes: 64_000 }); - const detail = detailResult.text; - throw new Error(`Brave LLM Context API error (${res.status}): ${detail || res.statusText}`); - } - - const data = (await res.json()) as BraveLlmContextResponse; - const mapped = mapBraveLlmContextResults(data); - - return { results: mapped, sources: data.sources }; - }, - ); -} - -async function runWebSearch(params: { - query: string; - count: number; - apiKey: string; - timeoutSeconds: number; - cacheTtlMs: number; - provider: (typeof SEARCH_PROVIDERS)[number]; - country?: string; - language?: string; - search_lang?: string; - ui_lang?: string; - freshness?: string; - dateAfter?: string; - dateBefore?: string; - searchDomainFilter?: string[]; - maxTokens?: number; - maxTokensPerPage?: number; - perplexityBaseUrl?: string; - perplexityModel?: string; - perplexityTransport?: PerplexityTransport; - grokModel?: string; - grokInlineCitations?: boolean; - geminiModel?: string; - kimiBaseUrl?: string; - kimiModel?: string; - braveMode?: "web" | "llm-context"; -}): Promise> { - const effectiveBraveMode = params.braveMode ?? "web"; - const providerSpecificKey = - params.provider === "perplexity" - ? `${params.perplexityTransport ?? "search_api"}:${params.perplexityBaseUrl ?? PERPLEXITY_DIRECT_BASE_URL}:${params.perplexityModel ?? DEFAULT_PERPLEXITY_MODEL}` - : params.provider === "grok" - ? `${params.grokModel ?? DEFAULT_GROK_MODEL}:${String(params.grokInlineCitations ?? false)}` - : params.provider === "gemini" - ? (params.geminiModel ?? DEFAULT_GEMINI_MODEL) - : params.provider === "kimi" - ? `${params.kimiBaseUrl ?? DEFAULT_KIMI_BASE_URL}:${params.kimiModel ?? DEFAULT_KIMI_MODEL}` - : ""; - const cacheKey = normalizeCacheKey( - params.provider === "brave" && effectiveBraveMode === "llm-context" - ? `${params.provider}:llm-context:${params.query}:${params.country || "default"}:${params.search_lang || params.language || "default"}:${params.freshness || "default"}` - : `${params.provider}:${effectiveBraveMode}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || params.language || "default"}:${params.ui_lang || "default"}:${params.freshness || "default"}:${params.dateAfter || "default"}:${params.dateBefore || "default"}:${params.searchDomainFilter?.join(",") || "default"}:${params.maxTokens || "default"}:${params.maxTokensPerPage || "default"}:${providerSpecificKey}`, - ); - const cached = readCache(SEARCH_CACHE, cacheKey); - if (cached) { - return { ...cached.value, cached: true }; - } - - const start = Date.now(); - - if (params.provider === "perplexity") { - if (params.perplexityTransport === "chat_completions") { - const { content, citations } = await runPerplexitySearch({ - query: params.query, - apiKey: params.apiKey, - baseUrl: params.perplexityBaseUrl ?? DEFAULT_PERPLEXITY_BASE_URL, - model: params.perplexityModel ?? DEFAULT_PERPLEXITY_MODEL, - timeoutSeconds: params.timeoutSeconds, - freshness: params.freshness, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.perplexityModel ?? DEFAULT_PERPLEXITY_MODEL, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(content, "web_search"), - citations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - const results = await runPerplexitySearchApi({ - query: params.query, - apiKey: params.apiKey, - count: params.count, - timeoutSeconds: params.timeoutSeconds, - country: params.country, - searchDomainFilter: params.searchDomainFilter, - searchRecencyFilter: params.freshness, - searchLanguageFilter: params.language ? [params.language] : undefined, - searchAfterDate: params.dateAfter ? isoToPerplexityDate(params.dateAfter) : undefined, - searchBeforeDate: params.dateBefore ? isoToPerplexityDate(params.dateBefore) : undefined, - maxTokens: params.maxTokens, - maxTokensPerPage: params.maxTokensPerPage, - }); - - const payload = { - query: params.query, - provider: params.provider, - count: results.length, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - results, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider === "grok") { - const { content, citations, inlineCitations } = await runGrokSearch({ - query: params.query, - apiKey: params.apiKey, - model: params.grokModel ?? DEFAULT_GROK_MODEL, - timeoutSeconds: params.timeoutSeconds, - inlineCitations: params.grokInlineCitations ?? false, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.grokModel ?? DEFAULT_GROK_MODEL, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(content), - citations, - inlineCitations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider === "kimi") { - const { content, citations } = await runKimiSearch({ - query: params.query, - apiKey: params.apiKey, - baseUrl: params.kimiBaseUrl ?? DEFAULT_KIMI_BASE_URL, - model: params.kimiModel ?? DEFAULT_KIMI_MODEL, - timeoutSeconds: params.timeoutSeconds, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.kimiModel ?? DEFAULT_KIMI_MODEL, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(content), - citations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider === "gemini") { - const geminiResult = await runGeminiSearch({ - query: params.query, - apiKey: params.apiKey, - model: params.geminiModel ?? DEFAULT_GEMINI_MODEL, - timeoutSeconds: params.timeoutSeconds, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.geminiModel ?? DEFAULT_GEMINI_MODEL, - tookMs: Date.now() - start, // Includes redirect URL resolution time - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(geminiResult.content), - citations: geminiResult.citations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider !== "brave") { - throw new Error("Unsupported web search provider."); - } - - if (effectiveBraveMode === "llm-context") { - const { results: llmResults, sources } = await runBraveLlmContextSearch({ - query: params.query, - apiKey: params.apiKey, - timeoutSeconds: params.timeoutSeconds, - country: params.country, - search_lang: params.search_lang, - freshness: params.freshness, - }); - - const mapped = llmResults.map((entry) => ({ - title: entry.title ? wrapWebContent(entry.title, "web_search") : "", - url: entry.url, - snippets: entry.snippets.map((s) => wrapWebContent(s, "web_search")), - siteName: entry.siteName, - })); - - const payload = { - query: params.query, - provider: params.provider, - mode: "llm-context" as const, - count: mapped.length, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - results: mapped, - sources, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - const url = new URL(BRAVE_SEARCH_ENDPOINT); - url.searchParams.set("q", params.query); - url.searchParams.set("count", String(params.count)); - if (params.country) { - url.searchParams.set("country", params.country); - } - if (params.search_lang || params.language) { - url.searchParams.set("search_lang", (params.search_lang || params.language)!); - } - if (params.ui_lang) { - url.searchParams.set("ui_lang", params.ui_lang); - } - if (params.freshness) { - url.searchParams.set("freshness", params.freshness); - } else if (params.dateAfter && params.dateBefore) { - url.searchParams.set("freshness", `${params.dateAfter}to${params.dateBefore}`); - } else if (params.dateAfter) { - url.searchParams.set( - "freshness", - `${params.dateAfter}to${new Date().toISOString().slice(0, 10)}`, - ); - } else if (params.dateBefore) { - url.searchParams.set("freshness", `1970-01-01to${params.dateBefore}`); - } - - const mapped = await withTrustedWebSearchEndpoint( - { - url: url.toString(), - timeoutSeconds: params.timeoutSeconds, - init: { - method: "GET", - headers: { - Accept: "application/json", - "X-Subscription-Token": params.apiKey, - }, - }, - }, - async (res) => { - if (!res.ok) { - const detailResult = await readResponseText(res, { maxBytes: 64_000 }); - const detail = detailResult.text; - throw new Error(`Brave Search API error (${res.status}): ${detail || res.statusText}`); - } - - const data = (await res.json()) as BraveSearchResponse; - const results = Array.isArray(data.web?.results) ? (data.web?.results ?? []) : []; - return results.map((entry) => { - const description = entry.description ?? ""; - const title = entry.title ?? ""; - const url = entry.url ?? ""; - const rawSiteName = resolveSiteName(url); - return { - title: title ? wrapWebContent(title, "web_search") : "", - url, // Keep raw for tool chaining - description: description ? wrapWebContent(description, "web_search") : "", - published: entry.age || undefined, - siteName: rawSiteName || undefined, - }; - }); - }, - ); - - const payload = { - query: params.query, - provider: params.provider, - count: mapped.length, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - results: mapped, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; -} - -export function createWebSearchTool(options?: { - config?: OpenClawConfig; - sandboxed?: boolean; - runtimeWebSearch?: RuntimeWebSearchMetadata; -}): AnyAgentTool | null { - const search = resolveSearchConfig(options?.config); - if (!resolveSearchEnabled({ search, sandboxed: options?.sandboxed })) { - return null; - } - - const runtimeProviderCandidate = - options?.runtimeWebSearch?.selectedProvider ?? options?.runtimeWebSearch?.providerConfigured; - const provider = - runtimeProviderCandidate && isSearchProvider(runtimeProviderCandidate) - ? runtimeProviderCandidate - : resolveSearchProvider(search); - const perplexityConfig = resolvePerplexityConfig(search); - const perplexitySchemaTransportHint = - options?.runtimeWebSearch?.perplexityTransport ?? - resolvePerplexitySchemaTransportHint(perplexityConfig); - const grokConfig = resolveGrokConfig(search); - const geminiConfig = resolveGeminiConfig(search); - const kimiConfig = resolveKimiConfig(search); - const braveConfig = resolveBraveConfig(search); - const braveMode = resolveBraveMode(braveConfig); - - const description = - provider === "perplexity" - ? perplexitySchemaTransportHint === "chat_completions" - ? "Search the web using Perplexity Sonar via Perplexity/OpenRouter chat completions. Returns AI-synthesized answers with citations from web-grounded search." - : "Search the web using Perplexity. Runtime routing decides between native Search API and Sonar chat-completions compatibility. Structured filters are available on the native Search API path." - : provider === "grok" - ? "Search the web using xAI Grok. Returns AI-synthesized answers with citations from real-time web search." - : provider === "kimi" - ? "Search the web using Kimi by Moonshot. Returns AI-synthesized answers with citations from native $web_search." - : provider === "gemini" - ? "Search the web using Gemini with Google Search grounding. Returns AI-synthesized answers with citations from Google Search." - : braveMode === "llm-context" - ? "Search the web using Brave Search LLM Context API. Returns pre-extracted page content (text chunks, tables, code blocks) optimized for LLM grounding." - : "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research."; - - return { - label: "Web Search", - name: "web_search", - description, - parameters: createWebSearchSchema({ - provider, - perplexityTransport: provider === "perplexity" ? perplexitySchemaTransportHint : undefined, - }), - execute: async (_toolCallId, args) => { - // Resolve Perplexity auth/transport lazily at execution time so unrelated providers - // do not touch Perplexity-only credential surfaces during tool construction. - const perplexityRuntime = - provider === "perplexity" ? resolvePerplexityTransport(perplexityConfig) : undefined; - const apiKey = - provider === "perplexity" - ? perplexityRuntime?.apiKey - : provider === "grok" - ? resolveGrokApiKey(grokConfig) - : provider === "kimi" - ? resolveKimiApiKey(kimiConfig) - : provider === "gemini" - ? resolveGeminiApiKey(geminiConfig) - : resolveSearchApiKey(search); - - if (!apiKey) { - return jsonResult(missingSearchKeyPayload(provider)); - } - - const supportsStructuredPerplexityFilters = - provider === "perplexity" && perplexityRuntime?.transport === "search_api"; - const params = args as Record; - const query = readStringParam(params, "query", { required: true }); - const count = - readNumberParam(params, "count", { integer: true }) ?? search?.maxResults ?? undefined; - const country = readStringParam(params, "country"); - if ( - country && - provider !== "brave" && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return jsonResult({ - error: "unsupported_country", - message: - provider === "perplexity" - ? "country filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it." - : `country filtering is not supported by the ${provider} provider. Only Brave and Perplexity support country filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const language = readStringParam(params, "language"); - if ( - language && - provider !== "brave" && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return jsonResult({ - error: "unsupported_language", - message: - provider === "perplexity" - ? "language filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it." - : `language filtering is not supported by the ${provider} provider. Only Brave and Perplexity support language filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (language && provider === "perplexity" && !/^[a-z]{2}$/i.test(language)) { - return jsonResult({ - error: "invalid_language", - message: "language must be a 2-letter ISO 639-1 code like 'en', 'de', or 'fr'.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const search_lang = readStringParam(params, "search_lang"); - const ui_lang = readStringParam(params, "ui_lang"); - // For Brave, accept both `language` (unified) and `search_lang` - const normalizedBraveLanguageParams = - provider === "brave" - ? normalizeBraveLanguageParams({ search_lang: search_lang || language, ui_lang }) - : { search_lang: language, ui_lang }; - if (normalizedBraveLanguageParams.invalidField === "search_lang") { - return jsonResult({ - error: "invalid_search_lang", - message: - "search_lang must be a Brave-supported language code like 'en', 'en-gb', 'zh-hans', or 'zh-hant'.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (normalizedBraveLanguageParams.invalidField === "ui_lang") { - return jsonResult({ - error: "invalid_ui_lang", - message: "ui_lang must be a language-region locale like 'en-US'.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const resolvedSearchLang = normalizedBraveLanguageParams.search_lang; - const resolvedUiLang = normalizedBraveLanguageParams.ui_lang; - if (resolvedUiLang && provider === "brave" && braveMode === "llm-context") { - return jsonResult({ - error: "unsupported_ui_lang", - message: - "ui_lang is not supported by Brave llm-context mode. Remove ui_lang or use Brave web mode for locale-based UI hints.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const rawFreshness = readStringParam(params, "freshness"); - if (rawFreshness && provider !== "brave" && provider !== "perplexity") { - return jsonResult({ - error: "unsupported_freshness", - message: `freshness filtering is not supported by the ${provider} provider. Only Brave and Perplexity support freshness.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (rawFreshness && provider === "brave" && braveMode === "llm-context") { - return jsonResult({ - error: "unsupported_freshness", - message: - "freshness filtering is not supported by Brave llm-context mode. Remove freshness or use Brave web mode.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const freshness = rawFreshness ? normalizeFreshness(rawFreshness, provider) : undefined; - if (rawFreshness && !freshness) { - return jsonResult({ - error: "invalid_freshness", - message: "freshness must be day, week, month, or year.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const rawDateAfter = readStringParam(params, "date_after"); - const rawDateBefore = readStringParam(params, "date_before"); - if (rawFreshness && (rawDateAfter || rawDateBefore)) { - return jsonResult({ - error: "conflicting_time_filters", - message: - "freshness and date_after/date_before cannot be used together. Use either freshness (day/week/month/year) or a date range (date_after/date_before), not both.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ( - (rawDateAfter || rawDateBefore) && - provider !== "brave" && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return jsonResult({ - error: "unsupported_date_filter", - message: - provider === "perplexity" - ? "date_after/date_before are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them." - : `date_after/date_before filtering is not supported by the ${provider} provider. Only Brave and Perplexity support date filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ((rawDateAfter || rawDateBefore) && provider === "brave" && braveMode === "llm-context") { - return jsonResult({ - error: "unsupported_date_filter", - message: - "date_after/date_before filtering is not supported by Brave llm-context mode. Use Brave web mode for date filters.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const dateAfter = rawDateAfter ? normalizeToIsoDate(rawDateAfter) : undefined; - if (rawDateAfter && !dateAfter) { - return jsonResult({ - error: "invalid_date", - message: "date_after must be YYYY-MM-DD format.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const dateBefore = rawDateBefore ? normalizeToIsoDate(rawDateBefore) : undefined; - if (rawDateBefore && !dateBefore) { - return jsonResult({ - error: "invalid_date", - message: "date_before must be YYYY-MM-DD format.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (dateAfter && dateBefore && dateAfter > dateBefore) { - return jsonResult({ - error: "invalid_date_range", - message: "date_after must be before date_before.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const domainFilter = readStringArrayParam(params, "domain_filter"); - if ( - domainFilter && - domainFilter.length > 0 && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return jsonResult({ - error: "unsupported_domain_filter", - message: - provider === "perplexity" - ? "domain_filter is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it." - : `domain_filter is not supported by the ${provider} provider. Only Perplexity supports domain filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - - if (domainFilter && domainFilter.length > 0) { - const hasDenylist = domainFilter.some((d) => d.startsWith("-")); - const hasAllowlist = domainFilter.some((d) => !d.startsWith("-")); - if (hasDenylist && hasAllowlist) { - return jsonResult({ - error: "invalid_domain_filter", - message: - "domain_filter cannot mix allowlist and denylist entries. Use either all positive entries (allowlist) or all entries prefixed with '-' (denylist).", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (domainFilter.length > 20) { - return jsonResult({ - error: "invalid_domain_filter", - message: "domain_filter supports a maximum of 20 domains.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - } - - const maxTokens = readNumberParam(params, "max_tokens", { integer: true }); - const maxTokensPerPage = readNumberParam(params, "max_tokens_per_page", { integer: true }); - if ( - provider === "perplexity" && - perplexityRuntime?.transport === "chat_completions" && - (maxTokens !== undefined || maxTokensPerPage !== undefined) - ) { - return jsonResult({ - error: "unsupported_content_budget", - message: - "max_tokens and max_tokens_per_page are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - - const result = await runWebSearch({ - query, - count: resolveSearchCount(count, DEFAULT_SEARCH_COUNT), - apiKey, - timeoutSeconds: resolveTimeoutSeconds(search?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS), - cacheTtlMs: resolveCacheTtlMs(search?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES), - provider, - country, - language, - search_lang: resolvedSearchLang, - ui_lang: resolvedUiLang, - freshness, - dateAfter, - dateBefore, - searchDomainFilter: domainFilter, - maxTokens: maxTokens ?? undefined, - maxTokensPerPage: maxTokensPerPage ?? undefined, - perplexityBaseUrl: perplexityRuntime?.baseUrl, - perplexityModel: perplexityRuntime?.model, - perplexityTransport: perplexityRuntime?.transport, - grokModel: resolveGrokModel(grokConfig), - grokInlineCitations: resolveGrokInlineCitations(grokConfig), - geminiModel: resolveGeminiModel(geminiConfig), - kimiBaseUrl: resolveKimiBaseUrl(kimiConfig), - kimiModel: resolveKimiModel(kimiConfig), - braveMode, - }); - return jsonResult(result); - }, - }; -} - -export const __testing = { - resolveSearchProvider, - inferPerplexityBaseUrlFromApiKey, - resolvePerplexityBaseUrl, - resolvePerplexityModel, - resolvePerplexityTransport, - isDirectPerplexityBaseUrl, - resolvePerplexityRequestModel, - resolvePerplexityApiKey, - normalizeBraveLanguageParams, - normalizeFreshness, - normalizeToIsoDate, - isoToPerplexityDate, - SEARCH_CACHE, - FRESHNESS_TO_RECENCY, - RECENCY_TO_FRESHNESS, - resolveGrokApiKey, - resolveGrokModel, - resolveGrokInlineCitations, - extractGrokContent, - resolveKimiApiKey, - resolveKimiModel, - resolveKimiBaseUrl, - extractKimiCitations, - resolveRedirectUrl: resolveCitationRedirectUrl, - resolveBraveMode, - mapBraveLlmContextResults, -} as const; diff --git a/src/agents/tools/web-search-provider-common.ts b/src/agents/tools/web-search-provider-common.ts new file mode 100644 index 00000000000..45c3d748dcd --- /dev/null +++ b/src/agents/tools/web-search-provider-common.ts @@ -0,0 +1,213 @@ +import type { OpenClawConfig } from "../../config/config.js"; +import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js"; +import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; +import { withTrustedWebToolsEndpoint } from "./web-guarded-fetch.js"; +import { + CacheEntry, + DEFAULT_CACHE_TTL_MINUTES, + DEFAULT_TIMEOUT_SECONDS, + normalizeCacheKey, + readCache, + readResponseText, + resolveCacheTtlMs, + resolveTimeoutSeconds, + writeCache, +} from "./web-shared.js"; + +export type SearchConfigRecord = NonNullable["web"] extends infer Web + ? Web extends { search?: infer Search } + ? Search extends Record + ? Search + : Record + : Record + : Record; + +export const DEFAULT_SEARCH_COUNT = 5; +export const MAX_SEARCH_COUNT = 10; + +const SEARCH_CACHE_KEY = Symbol.for("openclaw.web-search.cache"); + +function getSharedSearchCache(): Map>> { + const root = globalThis as Record; + const existing = root[SEARCH_CACHE_KEY]; + if (existing instanceof Map) { + return existing as Map>>; + } + const next = new Map>>(); + root[SEARCH_CACHE_KEY] = next; + return next; +} + +export const SEARCH_CACHE = getSharedSearchCache(); + +export function resolveSearchTimeoutSeconds(searchConfig?: SearchConfigRecord): number { + return resolveTimeoutSeconds(searchConfig?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS); +} + +export function resolveSearchCacheTtlMs(searchConfig?: SearchConfigRecord): number { + return resolveCacheTtlMs(searchConfig?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES); +} + +export function resolveSearchCount(value: unknown, fallback: number): number { + const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback; + const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed))); + return clamped; +} + +export function readConfiguredSecretString(value: unknown, path: string): string | undefined { + return normalizeSecretInput(normalizeResolvedSecretInputString({ value, path })) || undefined; +} + +export function readProviderEnvValue(envVars: string[]): string | undefined { + for (const envVar of envVars) { + const value = normalizeSecretInput(process.env[envVar]); + if (value) { + return value; + } + } + return undefined; +} + +export async function withTrustedWebSearchEndpoint( + params: { + url: string; + timeoutSeconds: number; + init: RequestInit; + }, + run: (response: Response) => Promise, +): Promise { + return withTrustedWebToolsEndpoint( + { + url: params.url, + init: params.init, + timeoutSeconds: params.timeoutSeconds, + }, + async ({ response }) => run(response), + ); +} + +export async function throwWebSearchApiError(res: Response, providerLabel: string): Promise { + const detailResult = await readResponseText(res, { maxBytes: 64_000 }); + const detail = detailResult.text; + throw new Error(`${providerLabel} API error (${res.status}): ${detail || res.statusText}`); +} + +export function resolveSiteName(url: string | undefined): string | undefined { + if (!url) { + return undefined; + } + try { + return new URL(url).hostname; + } catch { + return undefined; + } +} + +const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]); +const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/; +const PERPLEXITY_RECENCY_VALUES = new Set(["day", "week", "month", "year"]); + +export const FRESHNESS_TO_RECENCY: Record = { + pd: "day", + pw: "week", + pm: "month", + py: "year", +}; +export const RECENCY_TO_FRESHNESS: Record = { + day: "pd", + week: "pw", + month: "pm", + year: "py", +}; + +const ISO_DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/; +const PERPLEXITY_DATE_PATTERN = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/; + +function isValidIsoDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) { + return false; + } + const [year, month, day] = value.split("-").map((part) => Number.parseInt(part, 10)); + if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) { + return false; + } + + const date = new Date(Date.UTC(year, month - 1, day)); + return ( + date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day + ); +} + +export function isoToPerplexityDate(iso: string): string | undefined { + const match = iso.match(ISO_DATE_PATTERN); + if (!match) { + return undefined; + } + const [, year, month, day] = match; + return `${parseInt(month, 10)}/${parseInt(day, 10)}/${year}`; +} + +export function normalizeToIsoDate(value: string): string | undefined { + const trimmed = value.trim(); + if (ISO_DATE_PATTERN.test(trimmed)) { + return isValidIsoDate(trimmed) ? trimmed : undefined; + } + const match = trimmed.match(PERPLEXITY_DATE_PATTERN); + if (match) { + const [, month, day, year] = match; + const iso = `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`; + return isValidIsoDate(iso) ? iso : undefined; + } + return undefined; +} + +export function normalizeFreshness( + value: string | undefined, + provider: "brave" | "perplexity", +): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + + const lower = trimmed.toLowerCase(); + if (BRAVE_FRESHNESS_SHORTCUTS.has(lower)) { + return provider === "brave" ? lower : FRESHNESS_TO_RECENCY[lower]; + } + if (PERPLEXITY_RECENCY_VALUES.has(lower)) { + return provider === "perplexity" ? lower : RECENCY_TO_FRESHNESS[lower]; + } + if (provider === "brave") { + const match = trimmed.match(BRAVE_FRESHNESS_RANGE); + if (match) { + const [, start, end] = match; + if (isValidIsoDate(start) && isValidIsoDate(end) && start <= end) { + return `${start}to${end}`; + } + } + } + + return undefined; +} + +export function readCachedSearchPayload(cacheKey: string): Record | undefined { + const cached = readCache(SEARCH_CACHE, cacheKey); + return cached ? { ...cached.value, cached: true } : undefined; +} + +export function buildSearchCacheKey(parts: Array): string { + return normalizeCacheKey( + parts.map((part) => (part === undefined ? "default" : String(part))).join(":"), + ); +} + +export function writeCachedSearchPayload( + cacheKey: string, + payload: Record, + ttlMs: number, +): void { + writeCache(SEARCH_CACHE, cacheKey, payload, ttlMs); +} diff --git a/src/agents/tools/web-search-plugin-factory.ts b/src/agents/tools/web-search-provider-config.ts similarity index 65% rename from src/agents/tools/web-search-plugin-factory.ts rename to src/agents/tools/web-search-provider-config.ts index ab80702a6ed..861898d0b53 100644 --- a/src/agents/tools/web-search-plugin-factory.ts +++ b/src/agents/tools/web-search-provider-config.ts @@ -1,11 +1,15 @@ import type { OpenClawConfig } from "../../config/config.js"; -import type { WebSearchProviderPlugin } from "../../plugins/types.js"; -import { createWebSearchTool as createLegacyWebSearchTool } from "./web-search-core.js"; type ConfiguredWebSearchProvider = NonNullable< NonNullable["web"]>["search"] >["provider"]; +export type WebSearchConfig = NonNullable["web"] extends infer Web + ? Web extends { search?: infer Search } + ? Search + : undefined + : undefined; + function cloneWithDescriptors(value: T | undefined): T { const next = Object.create(Object.getPrototypeOf(value ?? {})) as T; if (value) { @@ -14,7 +18,7 @@ function cloneWithDescriptors(value: T | undefined): T { return next; } -function withForcedProvider( +export function withForcedProvider( config: OpenClawConfig | undefined, provider: ConfiguredWebSearchProvider, ): OpenClawConfig { @@ -31,33 +35,6 @@ function withForcedProvider( return next; } -export function createPluginBackedWebSearchProvider( - provider: Omit & { - id: ConfiguredWebSearchProvider; - }, -): WebSearchProviderPlugin { - return { - ...provider, - createTool: (ctx) => { - const tool = createLegacyWebSearchTool({ - config: withForcedProvider(ctx.config, provider.id), - runtimeWebSearch: ctx.runtimeMetadata, - }); - if (!tool) { - return null; - } - return { - description: tool.description, - parameters: tool.parameters as Record, - execute: async (args) => { - const result = await tool.execute(`web-search:${provider.id}`, args); - return (result.details ?? {}) as Record; - }, - }; - }, - }; -} - export function getTopLevelCredentialValue(searchConfig?: Record): unknown { return searchConfig?.apiKey; } @@ -92,3 +69,24 @@ export function setScopedCredentialValue( } (scoped as Record).apiKey = value; } + +export function resolveSearchConfig(cfg?: OpenClawConfig): WebSearchConfig { + const search = cfg?.tools?.web?.search; + if (!search || typeof search !== "object") { + return undefined; + } + return search as WebSearchConfig; +} + +export function resolveSearchEnabled(params: { + search?: WebSearchConfig; + sandboxed?: boolean; +}): boolean { + if (typeof params.search?.enabled === "boolean") { + return params.search.enabled; + } + if (params.sandboxed) { + return true; + } + return true; +} diff --git a/src/agents/tools/web-search.test.ts b/src/agents/tools/web-search.test.ts index b8bccd7dfd3..93f6d791bf5 100644 --- a/src/agents/tools/web-search.test.ts +++ b/src/agents/tools/web-search.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; +import { __testing as braveTesting } from "../../../extensions/brave/src/brave-web-search-provider.js"; +import { __testing as moonshotTesting } from "../../../extensions/moonshot/src/kimi-web-search-provider.js"; +import { __testing as perplexityTesting } from "../../../extensions/perplexity/web-search-provider.js"; +import { __testing as xaiTesting } from "../../../extensions/xai/src/grok-web-search-provider.js"; import { withEnv } from "../../test-utils/env.js"; -import { __testing } from "./web-search.js"; - const { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl, @@ -10,21 +12,19 @@ const { isDirectPerplexityBaseUrl, resolvePerplexityRequestModel, resolvePerplexityApiKey, - normalizeBraveLanguageParams, - normalizeFreshness, normalizeToIsoDate, isoToPerplexityDate, - resolveGrokApiKey, - resolveGrokModel, - resolveGrokInlineCitations, - extractGrokContent, - resolveKimiApiKey, - resolveKimiModel, - resolveKimiBaseUrl, - extractKimiCitations, +} = perplexityTesting; +const { + normalizeBraveLanguageParams, + normalizeFreshness, resolveBraveMode, mapBraveLlmContextResults, -} = __testing; +} = braveTesting; +const { resolveGrokApiKey, resolveGrokModel, resolveGrokInlineCitations, extractGrokContent } = + xaiTesting; +const { resolveKimiApiKey, resolveKimiModel, resolveKimiBaseUrl, extractKimiCitations } = + moonshotTesting; const kimiApiKeyEnv = ["KIMI_API", "KEY"].join("_"); const moonshotApiKeyEnv = ["MOONSHOT_API", "KEY"].join("_"); diff --git a/src/agents/tools/web-search.ts b/src/agents/tools/web-search.ts index 62993704377..340065d0d62 100644 --- a/src/agents/tools/web-search.ts +++ b/src/agents/tools/web-search.ts @@ -1,29 +1,123 @@ import type { OpenClawConfig } from "../../config/config.js"; +import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js"; +import { logVerbose } from "../../globals.js"; +import { resolvePluginWebSearchProviders } from "../../plugins/web-search-providers.js"; +import type { PluginWebSearchProviderEntry } from "../../plugins/types.js"; import type { RuntimeWebSearchMetadata } from "../../secrets/runtime-web-tools.types.js"; -import { __testing as runtimeTesting } from "../../web-search/runtime.js"; +import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; import type { AnyAgentTool } from "./common.js"; +import { jsonResult } from "./common.js"; +import { SEARCH_CACHE } from "./web-search-provider-common.js"; import { - __testing as coreTesting, - createWebSearchTool as createWebSearchToolCore, -} from "./web-search-core.js"; + resolveSearchConfig, + resolveSearchEnabled, + type WebSearchConfig, +} from "./web-search-provider-config.js"; + +function readProviderEnvValue(envVars: string[]): string | undefined { + for (const envVar of envVars) { + const value = normalizeSecretInput(process.env[envVar]); + if (value) { + return value; + } + } + return undefined; +} + +function hasProviderCredential( + provider: PluginWebSearchProviderEntry, + search: WebSearchConfig | undefined, +): boolean { + const rawValue = provider.getCredentialValue(search as Record | undefined); + const fromConfig = normalizeSecretInput( + normalizeResolvedSecretInputString({ + value: rawValue, + path: provider.credentialPath, + }), + ); + return Boolean(fromConfig || readProviderEnvValue(provider.envVars)); +} + +function resolveSearchProvider(search?: WebSearchConfig): string { + const providers = resolvePluginWebSearchProviders({ + bundledAllowlistCompat: true, + }); + const raw = + search && "provider" in search && typeof search.provider === "string" + ? search.provider.trim().toLowerCase() + : ""; + + if (raw) { + const explicit = providers.find((provider) => provider.id === raw); + if (explicit) { + return explicit.id; + } + } + + if (!raw) { + for (const provider of providers) { + if (!hasProviderCredential(provider, search)) { + continue; + } + logVerbose( + `web_search: no provider configured, auto-detected "${provider.id}" from available API keys`, + ); + return provider.id; + } + } + + return providers[0]?.id ?? ""; +} export function createWebSearchTool(options?: { config?: OpenClawConfig; sandboxed?: boolean; runtimeWebSearch?: RuntimeWebSearchMetadata; }): AnyAgentTool | null { - return createWebSearchToolCore(options); + const search = resolveSearchConfig(options?.config); + if (!resolveSearchEnabled({ search, sandboxed: options?.sandboxed })) { + return null; + } + + const providers = resolvePluginWebSearchProviders({ + config: options?.config, + bundledAllowlistCompat: true, + }); + if (providers.length === 0) { + return null; + } + + const providerId = + options?.runtimeWebSearch?.selectedProvider ?? + options?.runtimeWebSearch?.providerConfigured ?? + resolveSearchProvider(search); + const provider = + providers.find((entry) => entry.id === providerId) ?? + providers.find((entry) => entry.id === resolveSearchProvider(search)) ?? + providers[0]; + if (!provider) { + return null; + } + + const definition = provider.createTool({ + config: options?.config, + searchConfig: search as Record | undefined, + runtimeMetadata: options?.runtimeWebSearch, + }); + if (!definition) { + return null; + } + + return { + label: "Web Search", + name: "web_search", + description: definition.description, + parameters: definition.parameters, + execute: async (_toolCallId, args) => jsonResult(await definition.execute(args)), + }; } export const __testing = { - ...coreTesting, - resolveSearchProvider: ( - search?: OpenClawConfig["tools"] extends infer Tools - ? Tools extends { web?: infer Web } - ? Web extends { search?: infer Search } - ? Search - : undefined - : undefined - : undefined, - ) => runtimeTesting.resolveWebSearchProviderId({ search }), + SEARCH_CACHE, + resolveSearchProvider, }; diff --git a/src/agents/tools/web-tools.enabled-defaults.test.ts b/src/agents/tools/web-tools.enabled-defaults.test.ts index d06f65e0deb..e380542f30f 100644 --- a/src/agents/tools/web-tools.enabled-defaults.test.ts +++ b/src/agents/tools/web-tools.enabled-defaults.test.ts @@ -571,7 +571,9 @@ describe("web_search perplexity OpenRouter compatibility", () => { }); expect(mockFetch).not.toHaveBeenCalled(); - expect(result?.details).toMatchObject({ error: "unsupported_domain_filter" }); + expect((result?.details as { error?: string } | undefined)?.error).toMatch( + /^unsupported_(domain_filter|structured_filter)$/, + ); }); it("keeps Search API schema params visible before runtime auth routing", () => { diff --git a/src/commands/onboard-search.ts b/src/commands/onboard-search.ts index 72fafe461d2..7562ae55041 100644 --- a/src/commands/onboard-search.ts +++ b/src/commands/onboard-search.ts @@ -6,27 +6,13 @@ import { hasConfiguredSecretInput, normalizeSecretInputString, } from "../config/types.secrets.js"; -import { enablePluginInConfig } from "../plugins/enable.js"; +import type { PluginWebSearchProviderEntry } from "../plugins/types.js"; import { resolvePluginWebSearchProviders } from "../plugins/web-search-providers.js"; import type { RuntimeEnv } from "../runtime.js"; import type { WizardPrompter } from "../wizard/prompts.js"; import type { SecretInputMode } from "./onboard-types.js"; -export type SearchProvider = NonNullable< - NonNullable["web"]>["search"]>["provider"] ->; - -const SEARCH_PROVIDER_IDS = ["brave", "firecrawl", "gemini", "grok", "kimi", "perplexity"] as const; - -function isSearchProvider(value: string): value is SearchProvider { - return (SEARCH_PROVIDER_IDS as readonly string[]).includes(value); -} - -function hasSearchProviderId( - provider: T, -): provider is T & { id: SearchProvider } { - return isSearchProvider(provider.id); -} +export type SearchProvider = string; type SearchProviderEntry = { value: SearchProvider; @@ -35,21 +21,23 @@ type SearchProviderEntry = { envKeys: string[]; placeholder: string; signupUrl: string; + credentialPath: string; + applySelectionConfig?: PluginWebSearchProviderEntry["applySelectionConfig"]; }; export const SEARCH_PROVIDER_OPTIONS: readonly SearchProviderEntry[] = resolvePluginWebSearchProviders({ bundledAllowlistCompat: true, - }) - .filter(hasSearchProviderId) - .map((provider) => ({ - value: provider.id, - label: provider.label, - hint: provider.hint, - envKeys: provider.envVars, - placeholder: provider.placeholder, - signupUrl: provider.signupUrl, - })); + }).map((provider) => ({ + value: provider.id, + label: provider.label, + hint: provider.hint, + envKeys: provider.envVars, + placeholder: provider.placeholder, + signupUrl: provider.signupUrl, + credentialPath: provider.credentialPath, + applySelectionConfig: provider.applySelectionConfig, + })); export function hasKeyInEnv(entry: SearchProviderEntry): boolean { return entry.envKeys.some((k) => Boolean(process.env[k]?.trim())); @@ -83,7 +71,7 @@ function buildSearchEnvRef(provider: SearchProvider): SecretRef { const envVar = entry?.envKeys.find((k) => Boolean(process.env[k]?.trim())) ?? entry?.envKeys[0]; if (!envVar) { throw new Error( - `No env var mapping for search provider "${provider}" in secret-input-mode=ref.`, + `No env var mapping for search provider "${provider}" at ${entry?.credentialPath ?? "unknown path"} in secret-input-mode=ref.`, ); } return { source: "env", provider: DEFAULT_SECRET_PROVIDER_ALIAS, id: envVar }; @@ -107,29 +95,30 @@ export function applySearchKey( provider: SearchProvider, key: SecretInput, ): OpenClawConfig { - const search = { ...config.tools?.web?.search, provider, enabled: true }; - const entry = resolvePluginWebSearchProviders({ + const providerEntry = resolvePluginWebSearchProviders({ config, bundledAllowlistCompat: true, }).find((candidate) => candidate.id === provider); - if (entry) { - entry.setCredentialValue(search as Record, key); + const search = { ...config.tools?.web?.search, provider, enabled: true }; + if (providerEntry) { + providerEntry.setCredentialValue(search as Record, key); } - const next = { + const nextBase = { ...config, tools: { ...config.tools, web: { ...config.tools?.web, search }, }, }; - if (provider !== "firecrawl") { - return next; - } - return enablePluginInConfig(next, "firecrawl").config; + return providerEntry?.applySelectionConfig?.(nextBase) ?? nextBase; } function applyProviderOnly(config: OpenClawConfig, provider: SearchProvider): OpenClawConfig { - const next = { + const providerEntry = resolvePluginWebSearchProviders({ + config, + bundledAllowlistCompat: true, + }).find((candidate) => candidate.id === provider); + const nextBase = { ...config, tools: { ...config.tools, @@ -143,10 +132,7 @@ function applyProviderOnly(config: OpenClawConfig, provider: SearchProvider): Op }, }, }; - if (provider !== "firecrawl") { - return next; - } - return enablePluginInConfig(next, "firecrawl").config; + return providerEntry?.applySelectionConfig?.(nextBase) ?? nextBase; } function preserveDisabledState(original: OpenClawConfig, result: OpenClawConfig): OpenClawConfig { @@ -203,7 +189,7 @@ export async function setupSearch( return SEARCH_PROVIDER_OPTIONS[0].value; })(); - type PickerValue = SearchProvider | "__skip__"; + type PickerValue = string; const choice = await prompter.select({ message: "Search provider", options: [ diff --git a/src/config/config.web-search-provider.test.ts b/src/config/config.web-search-provider.test.ts index 912e70ac5a4..882f406eb85 100644 --- a/src/config/config.web-search-provider.test.ts +++ b/src/config/config.web-search-provider.test.ts @@ -14,31 +14,37 @@ vi.mock("../plugins/web-search-providers.js", () => { { id: "brave", envVars: ["BRAVE_API_KEY"], + credentialPath: "tools.web.search.apiKey", getCredentialValue: (search?: Record) => search?.apiKey, }, { id: "firecrawl", envVars: ["FIRECRAWL_API_KEY"], + credentialPath: "tools.web.search.firecrawl.apiKey", getCredentialValue: getScoped("firecrawl"), }, { id: "gemini", envVars: ["GEMINI_API_KEY"], + credentialPath: "tools.web.search.gemini.apiKey", getCredentialValue: getScoped("gemini"), }, { id: "grok", envVars: ["XAI_API_KEY"], + credentialPath: "tools.web.search.grok.apiKey", getCredentialValue: getScoped("grok"), }, { id: "kimi", envVars: ["KIMI_API_KEY", "MOONSHOT_API_KEY"], + credentialPath: "tools.web.search.kimi.apiKey", getCredentialValue: getScoped("kimi"), }, { id: "perplexity", envVars: ["PERPLEXITY_API_KEY", "OPENROUTER_API_KEY"], + credentialPath: "tools.web.search.perplexity.apiKey", getCredentialValue: getScoped("perplexity"), }, ], diff --git a/src/plugins/types.ts b/src/plugins/types.ts index b5bd28cc110..ab903ff0cc9 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -866,6 +866,19 @@ export type WebSearchProviderContext = { runtimeMetadata?: RuntimeWebSearchMetadata; }; +export type WebSearchCredentialResolutionSource = "config" | "secretRef" | "env" | "missing"; + +export type WebSearchRuntimeMetadataContext = { + config?: OpenClawConfig; + searchConfig?: Record; + runtimeMetadata?: RuntimeWebSearchMetadata; + resolvedCredential?: { + value?: string; + source: WebSearchCredentialResolutionSource; + fallbackEnvVar?: string; + }; +}; + export type WebSearchProviderPlugin = { id: WebSearchProviderId; label: string; @@ -875,8 +888,14 @@ export type WebSearchProviderPlugin = { signupUrl: string; docsUrl?: string; autoDetectOrder?: number; + credentialPath: string; + inactiveSecretPaths?: string[]; getCredentialValue: (searchConfig?: Record) => unknown; setCredentialValue: (searchConfigTarget: Record, value: unknown) => void; + applySelectionConfig?: (config: OpenClawConfig) => OpenClawConfig; + resolveRuntimeMetadata?: ( + ctx: WebSearchRuntimeMetadataContext, + ) => Partial | Promise>; createTool: (ctx: WebSearchProviderContext) => WebSearchProviderToolDefinition | null; }; diff --git a/src/plugins/web-search-providers.test.ts b/src/plugins/web-search-providers.test.ts index 9d2fd18e030..d88fbe96431 100644 --- a/src/plugins/web-search-providers.test.ts +++ b/src/plugins/web-search-providers.test.ts @@ -22,6 +22,20 @@ describe("resolvePluginWebSearchProviders", () => { "perplexity:perplexity", "firecrawl:firecrawl", ]); + expect(providers.map((provider) => provider.credentialPath)).toEqual([ + "tools.web.search.apiKey", + "tools.web.search.gemini.apiKey", + "tools.web.search.grok.apiKey", + "tools.web.search.kimi.apiKey", + "tools.web.search.perplexity.apiKey", + "tools.web.search.firecrawl.apiKey", + ]); + expect(providers.find((provider) => provider.id === "firecrawl")?.applySelectionConfig).toEqual( + expect.any(Function), + ); + expect( + providers.find((provider) => provider.id === "perplexity")?.resolveRuntimeMetadata, + ).toEqual(expect.any(Function)); }); it("can augment restrictive allowlists for bundled compatibility", () => { @@ -95,6 +109,7 @@ describe("resolvePluginWebSearchProviders", () => { placeholder: "custom-...", signupUrl: "https://example.com/signup", autoDetectOrder: 1, + credentialPath: "tools.web.search.custom.apiKey", getCredentialValue: () => "configured", setCredentialValue: () => {}, createTool: () => ({ diff --git a/src/plugins/web-search-providers.ts b/src/plugins/web-search-providers.ts index 585ed0bd36c..145dd411a3f 100644 --- a/src/plugins/web-search-providers.ts +++ b/src/plugins/web-search-providers.ts @@ -1,37 +1,108 @@ -import bravePlugin from "../../extensions/brave/index.js"; -import firecrawlPlugin from "../../extensions/firecrawl/index.js"; -import googlePlugin from "../../extensions/google/index.js"; -import moonshotPlugin from "../../extensions/moonshot/index.js"; -import perplexityPlugin from "../../extensions/perplexity/index.js"; -import xaiPlugin from "../../extensions/xai/index.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; import { withBundledPluginAllowlistCompat, withBundledPluginEnablementCompat, } from "./bundled-compat.js"; -import { capturePluginRegistration } from "./captured-registration.js"; -import type { PluginLoadOptions } from "./loader.js"; -import type { PluginWebSearchProviderRegistration } from "./registry.js"; +import { + pluginRegistrationContractRegistry, + webSearchProviderContractRegistry, +} from "./contracts/registry.js"; +import { loadOpenClawPlugins, type PluginLoadOptions } from "./loader.js"; +import { createPluginLoaderLogger } from "./logger.js"; import { getActivePluginRegistry } from "./runtime.js"; -import type { OpenClawPluginApi, PluginWebSearchProviderEntry } from "./types.js"; +import type { PluginWebSearchProviderEntry } from "./types.js"; -type RegistrablePlugin = { - id: string; - name: string; - register: (api: OpenClawPluginApi) => void; -}; +const log = createSubsystemLogger("plugins"); -const BUNDLED_WEB_SEARCH_PLUGINS: readonly RegistrablePlugin[] = [ - bravePlugin, - firecrawlPlugin, - googlePlugin, - moonshotPlugin, - perplexityPlugin, - xaiPlugin, -]; +function hasExplicitPluginConfig(config: PluginLoadOptions["config"]): boolean { + const plugins = config?.plugins; + if (!plugins) { + return false; + } + if (typeof plugins.enabled === "boolean") { + return true; + } + if (Array.isArray(plugins.allow) && plugins.allow.length > 0) { + return true; + } + if (Array.isArray(plugins.deny) && plugins.deny.length > 0) { + return true; + } + if (Array.isArray(plugins.load?.paths) && plugins.load.paths.length > 0) { + return true; + } + if (plugins.entries && Object.keys(plugins.entries).length > 0) { + return true; + } + if (plugins.slots && Object.keys(plugins.slots).length > 0) { + return true; + } + return false; +} -const BUNDLED_WEB_SEARCH_ALLOWLIST_COMPAT_PLUGIN_IDS = BUNDLED_WEB_SEARCH_PLUGINS.map( - (plugin) => plugin.id, -); +function resolveBundledWebSearchCompatPluginIds(params: { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; +}): string[] { + void params; + return pluginRegistrationContractRegistry + .filter((plugin) => plugin.webSearchProviderIds.length > 0) + .map((plugin) => plugin.pluginId) + .toSorted((left, right) => left.localeCompare(right)); +} + +function withBundledWebSearchVitestCompat(params: { + config: PluginLoadOptions["config"]; + pluginIds: readonly string[]; + env?: PluginLoadOptions["env"]; +}): PluginLoadOptions["config"] { + const env = params.env ?? process.env; + const isVitest = Boolean(env.VITEST || process.env.VITEST); + if (!isVitest || hasExplicitPluginConfig(params.config) || params.pluginIds.length === 0) { + return params.config; + } + + return { + ...params.config, + plugins: { + ...params.config?.plugins, + enabled: true, + allow: [...params.pluginIds], + slots: { + ...params.config?.plugins?.slots, + memory: "none", + }, + }, + }; +} + +function applyVitestContractMetadataCompat( + providers: PluginWebSearchProviderEntry[], + env?: PluginLoadOptions["env"], +): PluginWebSearchProviderEntry[] { + if (!(env?.VITEST || process.env.VITEST)) { + return providers; + } + + return providers.map((provider) => { + const contract = webSearchProviderContractRegistry.find( + (entry) => entry.pluginId === provider.pluginId && entry.provider.id === provider.id, + ); + if (!contract) { + return provider; + } + return { + ...contract.provider, + ...provider, + credentialPath: provider.credentialPath ?? contract.provider.credentialPath, + inactiveSecretPaths: provider.inactiveSecretPaths ?? contract.provider.inactiveSecretPaths, + applySelectionConfig: provider.applySelectionConfig ?? contract.provider.applySelectionConfig, + resolveRuntimeMetadata: + provider.resolveRuntimeMetadata ?? contract.provider.resolveRuntimeMetadata, + }; + }); +} function sortWebSearchProviders( providers: PluginWebSearchProviderEntry[], @@ -46,74 +117,52 @@ function sortWebSearchProviders( }); } -function mapWebSearchProviderEntries( - entries: PluginWebSearchProviderRegistration[], -): PluginWebSearchProviderEntry[] { - return sortWebSearchProviders( - entries.map((entry) => ({ - ...entry.provider, - pluginId: entry.pluginId, - })), - ); -} - -function normalizeWebSearchPluginConfig(params: { - config?: PluginLoadOptions["config"]; - bundledAllowlistCompat?: boolean; -}): PluginLoadOptions["config"] { - const allowlistCompat = params.bundledAllowlistCompat - ? withBundledPluginAllowlistCompat({ - config: params.config, - pluginIds: BUNDLED_WEB_SEARCH_ALLOWLIST_COMPAT_PLUGIN_IDS, - }) - : params.config; - return withBundledPluginEnablementCompat({ - config: allowlistCompat, - pluginIds: BUNDLED_WEB_SEARCH_ALLOWLIST_COMPAT_PLUGIN_IDS, - }); -} - -function captureBundledWebSearchProviders( - plugin: RegistrablePlugin, -): PluginWebSearchProviderRegistration[] { - const captured = capturePluginRegistration(plugin); - return captured.webSearchProviders.map((provider) => ({ - pluginId: plugin.id, - pluginName: plugin.name, - provider, - source: "bundled", - })); -} - -function resolveBundledWebSearchRegistrations(params: { - config?: PluginLoadOptions["config"]; - bundledAllowlistCompat?: boolean; -}): PluginWebSearchProviderRegistration[] { - const config = normalizeWebSearchPluginConfig(params); - if (config?.plugins?.enabled === false) { - return []; - } - const allowlist = config?.plugins?.allow - ? new Set(config.plugins.allow.map((entry) => entry.trim()).filter(Boolean)) - : null; - return BUNDLED_WEB_SEARCH_PLUGINS.flatMap((plugin) => { - if (allowlist && !allowlist.has(plugin.id)) { - return []; - } - if (config?.plugins?.entries?.[plugin.id]?.enabled === false) { - return []; - } - return captureBundledWebSearchProviders(plugin); - }); -} - export function resolvePluginWebSearchProviders(params: { config?: PluginLoadOptions["config"]; workspaceDir?: string; env?: PluginLoadOptions["env"]; bundledAllowlistCompat?: boolean; + activate?: boolean; + cache?: boolean; }): PluginWebSearchProviderEntry[] { - return mapWebSearchProviderEntries(resolveBundledWebSearchRegistrations(params)); + const bundledCompatPluginIds = resolveBundledWebSearchCompatPluginIds({ + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + }); + const allowlistCompat = params.bundledAllowlistCompat + ? withBundledPluginAllowlistCompat({ + config: params.config, + pluginIds: bundledCompatPluginIds, + }) + : params.config; + const enablementCompat = withBundledPluginEnablementCompat({ + config: allowlistCompat, + pluginIds: bundledCompatPluginIds, + }); + const config = withBundledWebSearchVitestCompat({ + config: enablementCompat, + pluginIds: bundledCompatPluginIds, + env: params.env, + }); + const registry = loadOpenClawPlugins({ + config, + workspaceDir: params.workspaceDir, + env: params.env, + cache: params.cache ?? false, + activate: params.activate ?? false, + logger: createPluginLoaderLogger(log), + }); + + return sortWebSearchProviders( + applyVitestContractMetadataCompat( + registry.webSearchProviders.map((entry) => ({ + ...entry.provider, + pluginId: entry.pluginId, + })), + params.env, + ), + ); } export function resolveRuntimeWebSearchProviders(params: { @@ -124,7 +173,12 @@ export function resolveRuntimeWebSearchProviders(params: { }): PluginWebSearchProviderEntry[] { const runtimeProviders = getActivePluginRegistry()?.webSearchProviders ?? []; if (runtimeProviders.length > 0) { - return mapWebSearchProviderEntries(runtimeProviders); + return sortWebSearchProviders( + runtimeProviders.map((entry) => ({ + ...entry.provider, + pluginId: entry.pluginId, + })), + ); } return resolvePluginWebSearchProviders(params); } diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts index fd32ecedf93..baa41d68ed2 100644 --- a/src/secrets/runtime-web-tools.ts +++ b/src/secrets/runtime-web-tools.ts @@ -1,5 +1,9 @@ import type { OpenClawConfig } from "../config/config.js"; import { resolveSecretInputRef } from "../config/types.secrets.js"; +import type { + PluginWebSearchProviderEntry, + WebSearchCredentialResolutionSource, +} from "../plugins/types.js"; import { resolvePluginWebSearchProviders } from "../plugins/web-search-providers.js"; import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; import { secretRefKey } from "./ref-contract.js"; @@ -18,14 +22,8 @@ import type { RuntimeWebToolsMetadata, } from "./runtime-web-tools.types.js"; -const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; -const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; -const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; -const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; - type WebSearchProvider = string; -type SecretResolutionSource = "config" | "secretRef" | "env" | "missing"; // pragma: allowlist secret export type { RuntimeWebDiagnostic, RuntimeWebDiagnosticCode, @@ -42,7 +40,7 @@ type FetchConfig = NonNullable["web"] extends infer Web type SecretResolutionResult = { value?: string; - source: SecretResolutionSource; + source: WebSearchCredentialResolutionSource; secretRefConfigured: boolean; unresolvedRefReason?: string; fallbackEnvVar?: string; @@ -198,60 +196,6 @@ async function resolveSecretInputWithEnvFallback(params: { }; } -function inferPerplexityBaseUrlFromApiKey(apiKey?: string): "direct" | "openrouter" | undefined { - if (!apiKey) { - return undefined; - } - const normalized = apiKey.toLowerCase(); - if (PERPLEXITY_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { - return "direct"; - } - if (OPENROUTER_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { - return "openrouter"; - } - return undefined; -} - -function resolvePerplexityRuntimeTransport(params: { - keyValue?: string; - keySource: SecretResolutionSource; - fallbackEnvVar?: string; - configValue: unknown; -}): "search_api" | "chat_completions" | undefined { - const config = isRecord(params.configValue) ? params.configValue : undefined; - const configuredBaseUrl = typeof config?.baseUrl === "string" ? config.baseUrl.trim() : ""; - const configuredModel = typeof config?.model === "string" ? config.model.trim() : ""; - - const baseUrl = (() => { - if (configuredBaseUrl) { - return configuredBaseUrl; - } - if (params.keySource === "env") { - if (params.fallbackEnvVar === "PERPLEXITY_API_KEY") { - return PERPLEXITY_DIRECT_BASE_URL; - } - if (params.fallbackEnvVar === "OPENROUTER_API_KEY") { - return DEFAULT_PERPLEXITY_BASE_URL; - } - } - if ((params.keySource === "config" || params.keySource === "secretRef") && params.keyValue) { - const inferred = inferPerplexityBaseUrlFromApiKey(params.keyValue); - return inferred === "openrouter" ? DEFAULT_PERPLEXITY_BASE_URL : PERPLEXITY_DIRECT_BASE_URL; - } - return DEFAULT_PERPLEXITY_BASE_URL; - })(); - - const hasLegacyOverride = Boolean(configuredBaseUrl || configuredModel); - const direct = (() => { - try { - return new URL(baseUrl).hostname.toLowerCase() === "api.perplexity.ai"; - } catch { - return false; - } - })(); - return hasLegacyOverride || !direct ? "chat_completions" : "search_api"; -} - function ensureObject(target: Record, key: string): Record { const current = target[key]; if (isRecord(current)) { @@ -291,8 +235,14 @@ function setResolvedFirecrawlApiKey(params: { firecrawl.apiKey = params.value; } -function keyPathForProvider(provider: WebSearchProvider): string { - return provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; +function keyPathForProvider(provider: PluginWebSearchProviderEntry): string { + return provider.credentialPath; +} + +function inactivePathsForProvider(provider: PluginWebSearchProviderEntry): string[] { + return provider.inactiveSecretPaths?.length + ? provider.inactiveSecretPaths + : [provider.credentialPath]; } function hasConfiguredSecretRef(value: unknown, defaults: SecretDefaults | undefined): boolean { @@ -367,7 +317,7 @@ export async function resolveRuntimeWebTools(params: { let selectedResolution: SecretResolutionResult | undefined; for (const provider of candidates) { - const path = keyPathForProvider(provider.id); + const path = keyPathForProvider(provider); const value = provider.getCredentialValue(search); const resolution = await resolveSecretInputWithEnvFallback({ sourceConfig: params.sourceConfig, @@ -475,13 +425,23 @@ export async function resolveRuntimeWebTools(params: { if (!configuredProvider) { searchMetadata.providerSource = "auto-detect"; } - if (selectedProvider === "perplexity") { - searchMetadata.perplexityTransport = resolvePerplexityRuntimeTransport({ - keyValue: selectedResolution?.value, - keySource: selectedResolution?.source ?? "missing", - fallbackEnvVar: selectedResolution?.fallbackEnvVar, - configValue: search.perplexity, - }); + const provider = providers.find((entry) => entry.id === selectedProvider); + if (provider?.resolveRuntimeMetadata) { + Object.assign( + searchMetadata, + await provider.resolveRuntimeMetadata({ + config: params.sourceConfig, + searchConfig: search, + runtimeMetadata: searchMetadata, + resolvedCredential: selectedResolution + ? { + value: selectedResolution.value, + source: selectedResolution.source, + fallbackEnvVar: selectedResolution.fallbackEnvVar, + } + : undefined, + }), + ); } } } @@ -491,29 +451,31 @@ export async function resolveRuntimeWebTools(params: { if (provider.id === searchMetadata.selectedProvider) { continue; } - const path = keyPathForProvider(provider.id); const value = provider.getCredentialValue(search); if (!hasConfiguredSecretRef(value, defaults)) { continue; } - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: `tools.web.search auto-detected provider is "${searchMetadata.selectedProvider}".`, - }); + for (const path of inactivePathsForProvider(provider)) { + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: `tools.web.search auto-detected provider is "${searchMetadata.selectedProvider}".`, + }); + } } } else if (search && !searchEnabled) { for (const provider of providers) { - const path = keyPathForProvider(provider.id); const value = provider.getCredentialValue(search); if (!hasConfiguredSecretRef(value, defaults)) { continue; } - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: "tools.web.search is disabled.", - }); + for (const path of inactivePathsForProvider(provider)) { + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: "tools.web.search is disabled.", + }); + } } } @@ -522,16 +484,17 @@ export async function resolveRuntimeWebTools(params: { if (provider.id === configuredProvider) { continue; } - const path = keyPathForProvider(provider.id); const value = provider.getCredentialValue(search); if (!hasConfiguredSecretRef(value, defaults)) { continue; } - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: `tools.web.search.provider is "${configuredProvider}".`, - }); + for (const path of inactivePathsForProvider(provider)) { + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: `tools.web.search.provider is "${configuredProvider}".`, + }); + } } } diff --git a/test/fixtures/plugin-extension-import-boundary-inventory.json b/test/fixtures/plugin-extension-import-boundary-inventory.json new file mode 100644 index 00000000000..871c9c6795e --- /dev/null +++ b/test/fixtures/plugin-extension-import-boundary-inventory.json @@ -0,0 +1,538 @@ +[ + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 1, + "kind": "import", + "specifier": "../../extensions/anthropic/openclaw.plugin.json", + "resolvedPath": "extensions/anthropic/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 2, + "kind": "import", + "specifier": "../../extensions/byteplus/openclaw.plugin.json", + "resolvedPath": "extensions/byteplus/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 3, + "kind": "import", + "specifier": "../../extensions/cloudflare-ai-gateway/openclaw.plugin.json", + "resolvedPath": "extensions/cloudflare-ai-gateway/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 4, + "kind": "import", + "specifier": "../../extensions/copilot-proxy/openclaw.plugin.json", + "resolvedPath": "extensions/copilot-proxy/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 5, + "kind": "import", + "specifier": "../../extensions/github-copilot/openclaw.plugin.json", + "resolvedPath": "extensions/github-copilot/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 6, + "kind": "import", + "specifier": "../../extensions/google/openclaw.plugin.json", + "resolvedPath": "extensions/google/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 7, + "kind": "import", + "specifier": "../../extensions/huggingface/openclaw.plugin.json", + "resolvedPath": "extensions/huggingface/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 8, + "kind": "import", + "specifier": "../../extensions/kilocode/openclaw.plugin.json", + "resolvedPath": "extensions/kilocode/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 9, + "kind": "import", + "specifier": "../../extensions/kimi-coding/openclaw.plugin.json", + "resolvedPath": "extensions/kimi-coding/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 10, + "kind": "import", + "specifier": "../../extensions/minimax/openclaw.plugin.json", + "resolvedPath": "extensions/minimax/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 11, + "kind": "import", + "specifier": "../../extensions/mistral/openclaw.plugin.json", + "resolvedPath": "extensions/mistral/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 12, + "kind": "import", + "specifier": "../../extensions/modelstudio/openclaw.plugin.json", + "resolvedPath": "extensions/modelstudio/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 13, + "kind": "import", + "specifier": "../../extensions/moonshot/openclaw.plugin.json", + "resolvedPath": "extensions/moonshot/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 14, + "kind": "import", + "specifier": "../../extensions/nvidia/openclaw.plugin.json", + "resolvedPath": "extensions/nvidia/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 15, + "kind": "import", + "specifier": "../../extensions/ollama/openclaw.plugin.json", + "resolvedPath": "extensions/ollama/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 16, + "kind": "import", + "specifier": "../../extensions/openai/openclaw.plugin.json", + "resolvedPath": "extensions/openai/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 17, + "kind": "import", + "specifier": "../../extensions/opencode-go/openclaw.plugin.json", + "resolvedPath": "extensions/opencode-go/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 18, + "kind": "import", + "specifier": "../../extensions/opencode/openclaw.plugin.json", + "resolvedPath": "extensions/opencode/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 19, + "kind": "import", + "specifier": "../../extensions/openrouter/openclaw.plugin.json", + "resolvedPath": "extensions/openrouter/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 20, + "kind": "import", + "specifier": "../../extensions/qianfan/openclaw.plugin.json", + "resolvedPath": "extensions/qianfan/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 21, + "kind": "import", + "specifier": "../../extensions/qwen-portal-auth/openclaw.plugin.json", + "resolvedPath": "extensions/qwen-portal-auth/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 22, + "kind": "import", + "specifier": "../../extensions/sglang/openclaw.plugin.json", + "resolvedPath": "extensions/sglang/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 23, + "kind": "import", + "specifier": "../../extensions/synthetic/openclaw.plugin.json", + "resolvedPath": "extensions/synthetic/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 24, + "kind": "import", + "specifier": "../../extensions/together/openclaw.plugin.json", + "resolvedPath": "extensions/together/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 25, + "kind": "import", + "specifier": "../../extensions/venice/openclaw.plugin.json", + "resolvedPath": "extensions/venice/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 26, + "kind": "import", + "specifier": "../../extensions/vercel-ai-gateway/openclaw.plugin.json", + "resolvedPath": "extensions/vercel-ai-gateway/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 27, + "kind": "import", + "specifier": "../../extensions/vllm/openclaw.plugin.json", + "resolvedPath": "extensions/vllm/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 28, + "kind": "import", + "specifier": "../../extensions/volcengine/openclaw.plugin.json", + "resolvedPath": "extensions/volcengine/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 29, + "kind": "import", + "specifier": "../../extensions/xai/openclaw.plugin.json", + "resolvedPath": "extensions/xai/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 30, + "kind": "import", + "specifier": "../../extensions/xiaomi/openclaw.plugin.json", + "resolvedPath": "extensions/xiaomi/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/bundled-provider-auth-env-vars.ts", + "line": 31, + "kind": "import", + "specifier": "../../extensions/zai/openclaw.plugin.json", + "resolvedPath": "extensions/zai/openclaw.plugin.json", + "reason": "imports extension-owned file from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord-ops.runtime.ts", + "line": 1, + "kind": "export", + "specifier": "../../../extensions/discord/src/audit.js", + "resolvedPath": "extensions/discord/src/audit.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord-ops.runtime.ts", + "line": 5, + "kind": "export", + "specifier": "../../../extensions/discord/src/directory-live.js", + "resolvedPath": "extensions/discord/src/directory-live.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord-ops.runtime.ts", + "line": 6, + "kind": "export", + "specifier": "../../../extensions/discord/src/monitor.js", + "resolvedPath": "extensions/discord/src/monitor.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord-ops.runtime.ts", + "line": 7, + "kind": "export", + "specifier": "../../../extensions/discord/src/probe.js", + "resolvedPath": "extensions/discord/src/probe.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord-ops.runtime.ts", + "line": 8, + "kind": "export", + "specifier": "../../../extensions/discord/src/resolve-channels.js", + "resolvedPath": "extensions/discord/src/resolve-channels.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord-ops.runtime.ts", + "line": 9, + "kind": "export", + "specifier": "../../../extensions/discord/src/resolve-users.js", + "resolvedPath": "extensions/discord/src/resolve-users.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord-ops.runtime.ts", + "line": 21, + "kind": "export", + "specifier": "../../../extensions/discord/src/send.js", + "resolvedPath": "extensions/discord/src/send.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord.ts", + "line": 1, + "kind": "import", + "specifier": "../../../extensions/discord/src/channel-actions.js", + "resolvedPath": "extensions/discord/src/channel-actions.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-discord.ts", + "line": 11, + "kind": "import", + "specifier": "../../../extensions/discord/src/monitor/thread-bindings.js", + "resolvedPath": "extensions/discord/src/monitor/thread-bindings.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-imessage.ts", + "line": 1, + "kind": "import", + "specifier": "../../../extensions/imessage/src/monitor.js", + "resolvedPath": "extensions/imessage/src/monitor.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-imessage.ts", + "line": 2, + "kind": "import", + "specifier": "../../../extensions/imessage/src/probe.js", + "resolvedPath": "extensions/imessage/src/probe.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-imessage.ts", + "line": 3, + "kind": "import", + "specifier": "../../../extensions/imessage/src/send.js", + "resolvedPath": "extensions/imessage/src/send.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-media.ts", + "line": 1, + "kind": "import", + "specifier": "../../../extensions/whatsapp/src/media.js", + "resolvedPath": "extensions/whatsapp/src/media.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-signal.ts", + "line": 1, + "kind": "import", + "specifier": "../../../extensions/signal/src/index.js", + "resolvedPath": "extensions/signal/src/index.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-signal.ts", + "line": 2, + "kind": "import", + "specifier": "../../../extensions/signal/src/probe.js", + "resolvedPath": "extensions/signal/src/probe.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-signal.ts", + "line": 3, + "kind": "import", + "specifier": "../../../extensions/signal/src/send.js", + "resolvedPath": "extensions/signal/src/send.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-slack-ops.runtime.ts", + "line": 4, + "kind": "export", + "specifier": "../../../extensions/slack/src/directory-live.js", + "resolvedPath": "extensions/slack/src/directory-live.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-slack-ops.runtime.ts", + "line": 5, + "kind": "export", + "specifier": "../../../extensions/slack/src/index.js", + "resolvedPath": "extensions/slack/src/index.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-slack-ops.runtime.ts", + "line": 6, + "kind": "export", + "specifier": "../../../extensions/slack/src/probe.js", + "resolvedPath": "extensions/slack/src/probe.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-slack-ops.runtime.ts", + "line": 7, + "kind": "export", + "specifier": "../../../extensions/slack/src/resolve-channels.js", + "resolvedPath": "extensions/slack/src/resolve-channels.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-slack-ops.runtime.ts", + "line": 8, + "kind": "export", + "specifier": "../../../extensions/slack/src/resolve-users.js", + "resolvedPath": "extensions/slack/src/resolve-users.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-slack-ops.runtime.ts", + "line": 9, + "kind": "export", + "specifier": "../../../extensions/slack/src/send.js", + "resolvedPath": "extensions/slack/src/send.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram-ops.runtime.ts", + "line": 4, + "kind": "export", + "specifier": "../../../extensions/telegram/src/audit.js", + "resolvedPath": "extensions/telegram/src/audit.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram-ops.runtime.ts", + "line": 5, + "kind": "export", + "specifier": "../../../extensions/telegram/src/monitor.js", + "resolvedPath": "extensions/telegram/src/monitor.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram-ops.runtime.ts", + "line": 6, + "kind": "export", + "specifier": "../../../extensions/telegram/src/probe.js", + "resolvedPath": "extensions/telegram/src/probe.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram-ops.runtime.ts", + "line": 17, + "kind": "export", + "specifier": "../../../extensions/telegram/src/send.js", + "resolvedPath": "extensions/telegram/src/send.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram-ops.runtime.ts", + "line": 18, + "kind": "export", + "specifier": "../../../extensions/telegram/src/token.js", + "resolvedPath": "extensions/telegram/src/token.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram.ts", + "line": 1, + "kind": "import", + "specifier": "../../../extensions/telegram/src/audit.js", + "resolvedPath": "extensions/telegram/src/audit.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram.ts", + "line": 2, + "kind": "import", + "specifier": "../../../extensions/telegram/src/channel-actions.js", + "resolvedPath": "extensions/telegram/src/channel-actions.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram.ts", + "line": 6, + "kind": "import", + "specifier": "../../../extensions/telegram/src/thread-bindings.js", + "resolvedPath": "extensions/telegram/src/thread-bindings.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-telegram.ts", + "line": 7, + "kind": "import", + "specifier": "../../../extensions/telegram/src/token.js", + "resolvedPath": "extensions/telegram/src/token.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-whatsapp-login.runtime.ts", + "line": 1, + "kind": "export", + "specifier": "../../../extensions/whatsapp/src/login.js", + "resolvedPath": "extensions/whatsapp/src/login.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-whatsapp-outbound.runtime.ts", + "line": 1, + "kind": "export", + "specifier": "../../../extensions/whatsapp/src/send.js", + "resolvedPath": "extensions/whatsapp/src/send.js", + "reason": "re-exports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-whatsapp.ts", + "line": 1, + "kind": "import", + "specifier": "../../../extensions/whatsapp/src/active-listener.js", + "resolvedPath": "extensions/whatsapp/src/active-listener.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-whatsapp.ts", + "line": 8, + "kind": "import", + "specifier": "../../../extensions/whatsapp/src/auth-store.js", + "resolvedPath": "extensions/whatsapp/src/auth-store.js", + "reason": "imports extension implementation from src/plugins" + }, + { + "file": "src/plugins/runtime/runtime-whatsapp.ts", + "line": 80, + "kind": "dynamic-import", + "specifier": "../../../extensions/whatsapp/src/login-qr.js", + "resolvedPath": "extensions/whatsapp/src/login-qr.js", + "reason": "dynamically imports extension implementation from src/plugins" + } +] diff --git a/test/fixtures/web-search-provider-boundary-inventory.json b/test/fixtures/web-search-provider-boundary-inventory.json new file mode 100644 index 00000000000..2dd7b68fb57 --- /dev/null +++ b/test/fixtures/web-search-provider-boundary-inventory.json @@ -0,0 +1,32 @@ +[ + { + "provider": "shared", + "file": "src/commands/onboard-search.ts", + "line": 9, + "reason": "imports bundled web search registry into core onboarding flow" + }, + { + "provider": "shared", + "file": "src/config/config.web-search-provider.test.ts", + "line": 9, + "reason": "imports bundled web search registry outside allowed generic plumbing" + }, + { + "provider": "shared", + "file": "src/plugins/contracts/loader.contract.test.ts", + "line": 11, + "reason": "imports bundled web search registry outside allowed generic plumbing" + }, + { + "provider": "shared", + "file": "src/plugins/web-search-providers.test.ts", + "line": 7, + "reason": "imports bundled web search registry outside allowed generic plumbing" + }, + { + "provider": "shared", + "file": "src/secrets/runtime-web-tools.test.ts", + "line": 3, + "reason": "imports bundled web search registry outside allowed generic plumbing" + } +] diff --git a/test/plugin-extension-import-boundary.test.ts b/test/plugin-extension-import-boundary.test.ts new file mode 100644 index 00000000000..3a49610ddd2 --- /dev/null +++ b/test/plugin-extension-import-boundary.test.ts @@ -0,0 +1,79 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + collectPluginExtensionImportBoundaryInventory, + diffInventory, +} from "../scripts/check-plugin-extension-import-boundary.mjs"; + +const repoRoot = process.cwd(); +const scriptPath = path.join(repoRoot, "scripts", "check-plugin-extension-import-boundary.mjs"); +const baselinePath = path.join( + repoRoot, + "test", + "fixtures", + "plugin-extension-import-boundary-inventory.json", +); + +function readBaseline() { + return JSON.parse(readFileSync(baselinePath, "utf8")); +} + +describe("plugin extension import boundary inventory", () => { + it("keeps web-search-providers out of the remaining inventory", async () => { + const inventory = await collectPluginExtensionImportBoundaryInventory(); + + expect(inventory.some((entry) => entry.file === "src/plugins/web-search-providers.ts")).toBe( + false, + ); + expect(inventory).toContainEqual( + expect.objectContaining({ + file: "src/plugins/runtime/runtime-signal.ts", + resolvedPath: "extensions/signal/src/index.js", + }), + ); + }); + + it("ignores plugin-sdk boundary shims by scope", async () => { + const inventory = await collectPluginExtensionImportBoundaryInventory(); + + expect(inventory.some((entry) => entry.file.startsWith("src/plugin-sdk/"))).toBe(false); + expect(inventory.some((entry) => entry.file.startsWith("src/plugin-sdk-internal/"))).toBe( + false, + ); + }); + + it("produces stable sorted output", async () => { + const first = await collectPluginExtensionImportBoundaryInventory(); + const second = await collectPluginExtensionImportBoundaryInventory(); + + expect(second).toEqual(first); + expect( + [...first].toSorted( + (left, right) => + left.file.localeCompare(right.file) || + left.line - right.line || + left.kind.localeCompare(right.kind) || + left.specifier.localeCompare(right.specifier) || + left.reason.localeCompare(right.reason), + ), + ).toEqual(first); + }); + + it("matches the checked-in baseline", async () => { + const expected = readBaseline(); + const actual = await collectPluginExtensionImportBoundaryInventory(); + + expect(diffInventory(expected, actual)).toEqual({ missing: [], unexpected: [] }); + }); + + it("script json output matches the baseline exactly", () => { + const stdout = execFileSync(process.execPath, [scriptPath, "--json"], { + cwd: repoRoot, + encoding: "utf8", + }); + + expect(JSON.parse(stdout)).toEqual(readBaseline()); + }); +}); diff --git a/test/web-search-provider-boundary.test.ts b/test/web-search-provider-boundary.test.ts new file mode 100644 index 00000000000..6995b918863 --- /dev/null +++ b/test/web-search-provider-boundary.test.ts @@ -0,0 +1,72 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + collectWebSearchProviderBoundaryInventory, + diffInventory, +} from "../scripts/check-web-search-provider-boundaries.mjs"; + +const repoRoot = process.cwd(); +const scriptPath = path.join(repoRoot, "scripts", "check-web-search-provider-boundaries.mjs"); +const baselinePath = path.join( + repoRoot, + "test", + "fixtures", + "web-search-provider-boundary-inventory.json", +); + +function readBaseline() { + return JSON.parse(readFileSync(baselinePath, "utf8")); +} + +describe("web search provider boundary inventory", () => { + it("finds the current shared core onboarding import", async () => { + const inventory = await collectWebSearchProviderBoundaryInventory(); + + expect(inventory).toContainEqual( + expect.objectContaining({ + provider: "shared", + file: "src/commands/onboard-search.ts", + }), + ); + }); + + it("ignores extension-owned registrations", async () => { + const inventory = await collectWebSearchProviderBoundaryInventory(); + + expect(inventory.some((entry) => entry.file.startsWith("extensions/"))).toBe(false); + }); + + it("produces stable sorted output", async () => { + const first = await collectWebSearchProviderBoundaryInventory(); + const second = await collectWebSearchProviderBoundaryInventory(); + + expect(second).toEqual(first); + expect( + [...first].toSorted( + (left, right) => + left.provider.localeCompare(right.provider) || + left.file.localeCompare(right.file) || + left.line - right.line || + left.reason.localeCompare(right.reason), + ), + ).toEqual(first); + }); + + it("matches the checked-in baseline", async () => { + const expected = readBaseline(); + const actual = await collectWebSearchProviderBoundaryInventory(); + + expect(diffInventory(expected, actual)).toEqual({ missing: [], unexpected: [] }); + }); + + it("script json output matches the baseline exactly", () => { + const stdout = execFileSync(process.execPath, [scriptPath, "--json"], { + cwd: repoRoot, + encoding: "utf8", + }); + + expect(JSON.parse(stdout)).toEqual(readBaseline()); + }); +});