diff --git a/extensions/firecrawl/src/firecrawl-client.ts b/extensions/firecrawl/src/firecrawl-client.ts index 565e1d6aac3..fa38c5bdabe 100644 --- a/extensions/firecrawl/src/firecrawl-client.ts +++ b/extensions/firecrawl/src/firecrawl-client.ts @@ -1,11 +1,10 @@ import { markdownToText, truncateText } from "openclaw/plugin-sdk/agent-runtime"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; -import { withTrustedWebToolsEndpoint } from "openclaw/plugin-sdk/provider-web-search"; import { DEFAULT_CACHE_TTL_MINUTES, normalizeCacheKey, + postTrustedWebToolsJson, readCache, - readResponseText, resolveCacheTtlMs, writeCache, } from "openclaw/plugin-sdk/provider-web-search"; @@ -29,7 +28,6 @@ const SCRAPE_CACHE = new Map< >(); const DEFAULT_SEARCH_COUNT = 5; const DEFAULT_SCRAPE_MAX_CHARS = 50_000; -const DEFAULT_ERROR_MAX_BYTES = 64_000; type FirecrawlSearchItem = { title: string; @@ -88,51 +86,6 @@ function resolveSiteName(urlRaw: string): string | undefined { } } -async function postFirecrawlJson(params: { - baseUrl: string; - pathname: "/v2/search" | "/v2/scrape"; - apiKey: string; - body: Record; - timeoutSeconds: number; - errorLabel: string; -}): Promise> { - const endpoint = resolveEndpoint(params.baseUrl, params.pathname); - return await withTrustedWebToolsEndpoint( - { - url: endpoint, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - Accept: "application/json", - Authorization: `Bearer ${params.apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(params.body), - }, - }, - async ({ response }) => { - if (!response.ok) { - const detail = await readResponseText(response, { maxBytes: DEFAULT_ERROR_MAX_BYTES }); - throw new Error( - `${params.errorLabel} API error (${response.status}): ${detail.text || response.statusText}`, - ); - } - const payload = (await response.json()) as Record; - if (payload.success === false) { - const error = - typeof payload.error === "string" - ? payload.error - : typeof payload.message === "string" - ? payload.message - : "unknown error"; - throw new Error(`${params.errorLabel} API error: ${error}`); - } - return payload; - }, - ); -} - function resolveSearchItems(payload: Record): FirecrawlSearchItem[] { const candidates = [ payload.data, @@ -279,14 +232,28 @@ export async function runFirecrawlSearch( } const start = Date.now(); - const payload = await postFirecrawlJson({ - baseUrl, - pathname: "/v2/search", - apiKey, - body, - timeoutSeconds, - errorLabel: "Firecrawl Search", - }); + const payload = await postTrustedWebToolsJson( + { + url: resolveEndpoint(baseUrl, "/v2/search"), + timeoutSeconds, + apiKey, + body, + errorLabel: "Firecrawl Search", + }, + async (response) => { + const payload = (await response.json()) as Record; + if (payload.success === false) { + const error = + typeof payload.error === "string" + ? payload.error + : typeof payload.message === "string" + ? payload.message + : "unknown error"; + throw new Error(`Firecrawl Search API error: ${error}`); + } + return payload; + }, + ); const result = buildSearchPayload({ query: params.query, provider: "firecrawl", @@ -409,22 +376,24 @@ export async function runFirecrawlScrape( return { ...cached.value, cached: true }; } - const payload = await postFirecrawlJson({ - baseUrl, - pathname: "/v2/scrape", - apiKey, - timeoutSeconds, - errorLabel: "Firecrawl", - body: { - url: params.url, - formats: ["markdown"], - onlyMainContent, - timeout: timeoutSeconds * 1000, - maxAge: maxAgeMs, - proxy, - storeInCache, + const payload = await postTrustedWebToolsJson( + { + url: resolveEndpoint(baseUrl, "/v2/scrape"), + timeoutSeconds, + apiKey, + errorLabel: "Firecrawl", + body: { + url: params.url, + formats: ["markdown"], + onlyMainContent, + timeout: timeoutSeconds * 1000, + maxAge: maxAgeMs, + proxy, + storeInCache, + }, }, - }); + async (response) => (await response.json()) as Record, + ); const result = parseFirecrawlScrapePayload({ payload, url: params.url, diff --git a/extensions/tavily/src/tavily-client.ts b/extensions/tavily/src/tavily-client.ts index 8308f8b8772..c57f5850af3 100644 --- a/extensions/tavily/src/tavily-client.ts +++ b/extensions/tavily/src/tavily-client.ts @@ -1,10 +1,9 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; -import { withTrustedWebToolsEndpoint } from "openclaw/plugin-sdk/provider-web-search"; import { DEFAULT_CACHE_TTL_MINUTES, normalizeCacheKey, + postTrustedWebToolsJson, readCache, - readResponseText, resolveCacheTtlMs, writeCache, } from "openclaw/plugin-sdk/provider-web-search"; @@ -26,7 +25,6 @@ const EXTRACT_CACHE = new Map< { value: Record; expiresAt: number; insertedAt: number } >(); const DEFAULT_SEARCH_COUNT = 5; -const DEFAULT_ERROR_MAX_BYTES = 64_000; export type TavilySearchParams = { cfg?: OpenClawConfig; @@ -67,41 +65,6 @@ function resolveEndpoint(baseUrl: string, pathname: string): string { } } -async function postTavilyJson(params: { - baseUrl: string; - pathname: string; - apiKey: string; - body: Record; - timeoutSeconds: number; - errorLabel: string; -}): Promise> { - const endpoint = resolveEndpoint(params.baseUrl, params.pathname); - return await withTrustedWebToolsEndpoint( - { - url: endpoint, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - Accept: "application/json", - Authorization: `Bearer ${params.apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(params.body), - }, - }, - async ({ response }) => { - if (!response.ok) { - const detail = await readResponseText(response, { maxBytes: DEFAULT_ERROR_MAX_BYTES }); - throw new Error( - `${params.errorLabel} API error (${response.status}): ${detail.text || response.statusText}`, - ); - } - return (await response.json()) as Record; - }, - ); -} - export async function runTavilySearch( params: TavilySearchParams, ): Promise> { @@ -149,14 +112,16 @@ export async function runTavilySearch( if (params.excludeDomains?.length) body.exclude_domains = params.excludeDomains; const start = Date.now(); - const payload = await postTavilyJson({ - baseUrl, - pathname: "/search", - apiKey, - body, - timeoutSeconds, - errorLabel: "Tavily Search", - }); + const payload = await postTrustedWebToolsJson( + { + url: resolveEndpoint(baseUrl, "/search"), + timeoutSeconds, + apiKey, + body, + errorLabel: "Tavily Search", + }, + async (response) => (await response.json()) as Record, + ); const rawResults = Array.isArray(payload.results) ? payload.results : []; const results = rawResults.map((r: Record) => ({ @@ -228,14 +193,16 @@ export async function runTavilyExtract( if (params.includeImages) body.include_images = true; const start = Date.now(); - const payload = await postTavilyJson({ - baseUrl, - pathname: "/extract", - apiKey, - body, - timeoutSeconds, - errorLabel: "Tavily Extract", - }); + const payload = await postTrustedWebToolsJson( + { + url: resolveEndpoint(baseUrl, "/extract"), + timeoutSeconds, + apiKey, + body, + errorLabel: "Tavily Extract", + }, + async (response) => (await response.json()) as Record, + ); const rawResults = Array.isArray(payload.results) ? payload.results : []; const results = rawResults.map((r: Record) => ({ @@ -282,5 +249,5 @@ export async function runTavilyExtract( } export const __testing = { - postTavilyJson, + resolveEndpoint, }; diff --git a/extensions/xai/src/grok-web-search-provider.ts b/extensions/xai/src/grok-web-search-provider.ts index d9a6f0f8d46..705a8299917 100644 --- a/extensions/xai/src/grok-web-search-provider.ts +++ b/extensions/xai/src/grok-web-search-provider.ts @@ -5,12 +5,12 @@ import { DEFAULT_SEARCH_COUNT, getScopedCredentialValue, MAX_SEARCH_COUNT, - mergeScopedSearchConfig, readCachedSearchPayload, readConfiguredSecretString, readNumberParam, readProviderEnvValue, readStringParam, + mergeScopedSearchConfig, resolveProviderWebSearchPluginConfig, resolveSearchCacheTtlMs, resolveSearchCount, @@ -20,151 +20,24 @@ import { type SearchConfigRecord, type WebSearchProviderPlugin, type WebSearchProviderToolDefinition, - withTrustedWebSearchEndpoint, - wrapWebContent, writeCachedSearchPayload, } from "openclaw/plugin-sdk/provider-web-search"; +import { + buildXaiWebSearchPayload, + extractXaiWebSearchContent, + requestXaiWebSearch, + resolveXaiInlineCitations, + resolveXaiSearchConfig, + resolveXaiWebSearchModel, +} from "./web-search-shared.js"; -const XAI_API_ENDPOINT = "https://api.x.ai/v1/responses"; -const DEFAULT_GROK_MODEL = "grok-4-1-fast"; - -type GrokConfig = { - apiKey?: string; - model?: string; - inlineCitations?: boolean; -}; - -type GrokSearchResponse = { - output?: Array<{ - type?: string; - role?: string; - text?: string; - content?: Array<{ - type?: string; - text?: string; - annotations?: Array<{ - type?: string; - url?: string; - start_index?: number; - end_index?: number; - }>; - }>; - annotations?: Array<{ - type?: string; - url?: string; - start_index?: number; - end_index?: number; - }>; - }>; - output_text?: string; - citations?: string[]; - inline_citations?: Array<{ - start_index: number; - end_index: number; - url: string; - }>; -}; - -function resolveGrokConfig(searchConfig?: SearchConfigRecord): GrokConfig { - const grok = searchConfig?.grok; - return grok && typeof grok === "object" && !Array.isArray(grok) ? (grok as GrokConfig) : {}; -} - -function resolveGrokApiKey(grok?: GrokConfig): string | undefined { +function resolveGrokApiKey(grok?: Record): string | undefined { return ( readConfiguredSecretString(grok?.apiKey, "tools.web.search.grok.apiKey") ?? readProviderEnvValue(["XAI_API_KEY"]) ); } -function resolveGrokModel(grok?: GrokConfig): string { - const model = typeof grok?.model === "string" ? grok.model.trim() : ""; - return model || DEFAULT_GROK_MODEL; -} - -function resolveGrokInlineCitations(grok?: GrokConfig): boolean { - return grok?.inlineCitations === true; -} - -function extractGrokContent(data: GrokSearchResponse): { - text: string | undefined; - annotationCitations: string[]; -} { - for (const output of data.output ?? []) { - if (output.type === "message") { - for (const block of output.content ?? []) { - if (block.type === "output_text" && typeof block.text === "string" && block.text) { - const urls = (block.annotations ?? []) - .filter( - (annotation) => - annotation.type === "url_citation" && typeof annotation.url === "string", - ) - .map((annotation) => annotation.url as string); - return { text: block.text, annotationCitations: [...new Set(urls)] }; - } - } - } - if (output.type === "output_text" && typeof output.text === "string" && output.text) { - const urls = (Array.isArray(output.annotations) ? output.annotations : []) - .filter( - (annotation) => annotation.type === "url_citation" && typeof annotation.url === "string", - ) - .map((annotation) => annotation.url as string); - return { text: output.text, annotationCitations: [...new Set(urls)] }; - } - } - - return { - text: typeof data.output_text === "string" ? data.output_text : undefined, - annotationCitations: [], - }; -} - -async function runGrokSearch(params: { - query: string; - apiKey: string; - model: string; - timeoutSeconds: number; - inlineCitations: boolean; -}): Promise<{ - content: string; - citations: string[]; - inlineCitations?: GrokSearchResponse["inline_citations"]; -}> { - return withTrustedWebSearchEndpoint( - { - url: XAI_API_ENDPOINT, - timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${params.apiKey}`, - }, - body: JSON.stringify({ - model: params.model, - input: [{ role: "user", content: params.query }], - tools: [{ type: "web_search" }], - }), - }, - }, - async (res) => { - if (!res.ok) { - const detail = await res.text(); - throw new Error(`xAI API error (${res.status}): ${detail || res.statusText}`); - } - - const data = (await res.json()) as GrokSearchResponse; - const { text, annotationCitations } = extractGrokContent(data); - return { - content: text ?? "No response", - citations: (data.citations ?? []).length > 0 ? data.citations! : annotationCitations, - inlineCitations: data.inline_citations, - }; - }, - ); -} - function createGrokSchema() { return Type.Object({ query: Type.String({ description: "Search query string." }), @@ -197,7 +70,7 @@ function createGrokToolDefinition( return unsupportedResponse; } - const grokConfig = resolveGrokConfig(searchConfig); + const grokConfig = resolveXaiSearchConfig(searchConfig); const apiKey = resolveGrokApiKey(grokConfig); if (!apiKey) { return { @@ -213,8 +86,8 @@ function createGrokToolDefinition( readNumberParam(params, "count", { integer: true }) ?? searchConfig?.maxResults ?? undefined; - const model = resolveGrokModel(grokConfig); - const inlineCitations = resolveGrokInlineCitations(grokConfig); + const model = resolveXaiWebSearchModel(searchConfig); + const inlineCitations = resolveXaiInlineCitations(searchConfig); const cacheKey = buildSearchCacheKey([ "grok", query, @@ -228,28 +101,22 @@ function createGrokToolDefinition( } const start = Date.now(); - const result = await runGrokSearch({ + const result = await requestXaiWebSearch({ query, apiKey, model, timeoutSeconds: resolveSearchTimeoutSeconds(searchConfig), inlineCitations, }); - const payload = { + const payload = buildXaiWebSearchPayload({ query, provider: "grok", model, tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: "grok", - wrapped: true, - }, - content: wrapWebContent(result.content), + content: result.content, citations: result.citations, inlineCitations: result.inlineCitations, - }; + }); writeCachedSearchPayload(cacheKey, payload, resolveSearchCacheTtlMs(searchConfig)); return payload; }, @@ -289,7 +156,15 @@ export function createGrokWebSearchProvider(): WebSearchProviderPlugin { export const __testing = { resolveGrokApiKey, - resolveGrokModel, - resolveGrokInlineCitations, - extractGrokContent, + resolveGrokModel: (grok?: Record) => + resolveXaiWebSearchModel(grok ? { grok } : undefined), + resolveGrokInlineCitations: (grok?: Record) => + resolveXaiInlineCitations(grok ? { grok } : undefined), + extractGrokContent: extractXaiWebSearchContent, + extractXaiWebSearchContent, + resolveXaiInlineCitations, + resolveXaiSearchConfig, + resolveXaiWebSearchModel, + requestXaiWebSearch, + buildXaiWebSearchPayload, } as const; diff --git a/extensions/xai/src/web-search-shared.ts b/extensions/xai/src/web-search-shared.ts new file mode 100644 index 00000000000..47616bcf13c --- /dev/null +++ b/extensions/xai/src/web-search-shared.ts @@ -0,0 +1,171 @@ +import { postTrustedWebToolsJson, wrapWebContent } from "openclaw/plugin-sdk/provider-web-search"; + +export const XAI_WEB_SEARCH_ENDPOINT = "https://api.x.ai/v1/responses"; +export const XAI_DEFAULT_WEB_SEARCH_MODEL = "grok-4-1-fast"; + +export type XaiWebSearchResponse = { + output?: Array<{ + type?: string; + text?: string; + content?: Array<{ + type?: string; + text?: string; + annotations?: Array<{ + type?: string; + url?: string; + }>; + }>; + annotations?: Array<{ + type?: string; + url?: string; + }>; + }>; + output_text?: string; + citations?: string[]; + inline_citations?: Array<{ + start_index: number; + end_index: number; + url: string; + }>; +}; + +type XaiWebSearchConfig = Record & { + model?: unknown; + inlineCitations?: unknown; +}; + +export type XaiWebSearchResult = { + content: string; + citations: string[]; + inlineCitations?: XaiWebSearchResponse["inline_citations"]; +}; + +export function buildXaiWebSearchPayload(params: { + query: string; + provider: string; + model: string; + tookMs: number; + content: string; + citations: string[]; + inlineCitations?: XaiWebSearchResponse["inline_citations"]; +}): Record { + return { + query: params.query, + provider: params.provider, + model: params.model, + tookMs: params.tookMs, + externalContent: { + untrusted: true, + source: "web_search", + provider: params.provider, + wrapped: true, + }, + content: wrapWebContent(params.content, "web_search"), + citations: params.citations, + ...(params.inlineCitations ? { inlineCitations: params.inlineCitations } : {}), + }; +} + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +export function resolveXaiSearchConfig(searchConfig?: Record): XaiWebSearchConfig { + return (asRecord(searchConfig?.grok) as XaiWebSearchConfig | undefined) ?? {}; +} + +export function resolveXaiWebSearchModel(searchConfig?: Record): string { + const config = resolveXaiSearchConfig(searchConfig); + return typeof config.model === "string" && config.model.trim() + ? config.model.trim() + : XAI_DEFAULT_WEB_SEARCH_MODEL; +} + +export function resolveXaiInlineCitations(searchConfig?: Record): boolean { + return resolveXaiSearchConfig(searchConfig).inlineCitations === true; +} + +export function extractXaiWebSearchContent(data: XaiWebSearchResponse): { + text: string | undefined; + annotationCitations: string[]; +} { + for (const output of data.output ?? []) { + if (output.type === "message") { + for (const block of output.content ?? []) { + if (block.type === "output_text" && typeof block.text === "string" && block.text) { + const urls = (block.annotations ?? []) + .filter( + (annotation) => + annotation.type === "url_citation" && typeof annotation.url === "string", + ) + .map((annotation) => annotation.url as string); + return { text: block.text, annotationCitations: [...new Set(urls)] }; + } + } + } + + if (output.type === "output_text" && typeof output.text === "string" && output.text) { + const urls = (output.annotations ?? []) + .filter( + (annotation) => annotation.type === "url_citation" && typeof annotation.url === "string", + ) + .map((annotation) => annotation.url as string); + return { text: output.text, annotationCitations: [...new Set(urls)] }; + } + } + + return { + text: typeof data.output_text === "string" ? data.output_text : undefined, + annotationCitations: [], + }; +} + +export async function requestXaiWebSearch(params: { + query: string; + model: string; + apiKey: string; + timeoutSeconds: number; + inlineCitations: boolean; +}): Promise { + return await postTrustedWebToolsJson( + { + url: XAI_WEB_SEARCH_ENDPOINT, + timeoutSeconds: params.timeoutSeconds, + apiKey: params.apiKey, + body: { + model: params.model, + input: [{ role: "user", content: params.query }], + tools: [{ type: "web_search" }], + }, + errorLabel: "xAI", + }, + async (response) => { + const data = (await response.json()) as XaiWebSearchResponse; + const { text, annotationCitations } = extractXaiWebSearchContent(data); + const citations = + Array.isArray(data.citations) && data.citations.length > 0 + ? data.citations + : annotationCitations; + return { + content: text ?? "No response", + citations, + inlineCitations: + params.inlineCitations && Array.isArray(data.inline_citations) + ? data.inline_citations + : undefined, + }; + }, + ); +} + +export const __testing = { + buildXaiWebSearchPayload, + extractXaiWebSearchContent, + resolveXaiInlineCitations, + resolveXaiSearchConfig, + resolveXaiWebSearchModel, + requestXaiWebSearch, + XAI_DEFAULT_WEB_SEARCH_MODEL, +} as const; diff --git a/extensions/xai/web-search.ts b/extensions/xai/web-search.ts index c1d97652d54..d160892c0c5 100644 --- a/extensions/xai/web-search.ts +++ b/extensions/xai/web-search.ts @@ -5,133 +5,29 @@ import { getScopedCredentialValue, normalizeCacheKey, readCache, - readResponseText, + readNumberParam, + readStringParam, resolveCacheTtlMs, resolveTimeoutSeconds, resolveWebSearchProviderCredential, setScopedCredentialValue, type WebSearchProviderPlugin, - withTrustedWebToolsEndpoint, - wrapWebContent, writeCache, } from "openclaw/plugin-sdk/provider-web-search"; +import { + buildXaiWebSearchPayload, + extractXaiWebSearchContent, + requestXaiWebSearch, + resolveXaiInlineCitations, + resolveXaiWebSearchModel, +} from "./src/web-search-shared.js"; -const XAI_WEB_SEARCH_ENDPOINT = "https://api.x.ai/v1/responses"; -const XAI_DEFAULT_WEB_SEARCH_MODEL = "grok-4-1-fast"; const XAI_WEB_SEARCH_CACHE = new Map< string, { value: Record; insertedAt: number; expiresAt: number } >(); -type XaiWebSearchResponse = { - output?: Array<{ - type?: string; - text?: string; - content?: Array<{ - type?: string; - text?: string; - annotations?: Array<{ - type?: string; - url?: string; - }>; - }>; - annotations?: Array<{ - type?: string; - url?: string; - }>; - }>; - output_text?: string; - citations?: string[]; - inline_citations?: Array<{ - start_index: number; - end_index: number; - url: string; - }>; -}; - -function extractXaiWebSearchContent(data: XaiWebSearchResponse): { - text: string | undefined; - annotationCitations: string[]; -} { - for (const output of data.output ?? []) { - if (output.type === "message") { - for (const block of output.content ?? []) { - if (block.type === "output_text" && typeof block.text === "string" && block.text) { - const urls = (block.annotations ?? []) - .filter( - (annotation) => - annotation.type === "url_citation" && typeof annotation.url === "string", - ) - .map((annotation) => annotation.url as string); - return { text: block.text, annotationCitations: [...new Set(urls)] }; - } - } - } - - if (output.type === "output_text" && typeof output.text === "string" && output.text) { - const urls = (output.annotations ?? []) - .filter( - (annotation) => annotation.type === "url_citation" && typeof annotation.url === "string", - ) - .map((annotation) => annotation.url as string); - return { text: output.text, annotationCitations: [...new Set(urls)] }; - } - } - - return { - text: typeof data.output_text === "string" ? data.output_text : undefined, - annotationCitations: [], - }; -} - -function asRecord(value: unknown): Record | undefined { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : undefined; -} - -function resolveXaiWebSearchConfig( - searchConfig?: Record, -): Record { - return asRecord(searchConfig?.grok) ?? {}; -} - -function resolveXaiWebSearchModel(searchConfig?: Record): string { - const config = resolveXaiWebSearchConfig(searchConfig); - return typeof config.model === "string" && config.model.trim() - ? config.model.trim() - : XAI_DEFAULT_WEB_SEARCH_MODEL; -} - -function resolveXaiInlineCitations(searchConfig?: Record): boolean { - return resolveXaiWebSearchConfig(searchConfig).inlineCitations === true; -} - -function readQuery(args: Record): string { - const value = typeof args.query === "string" ? args.query.trim() : ""; - if (!value) { - throw new Error("query required"); - } - return value; -} - -function readCount(args: Record): number { - const raw = args.count; - const parsed = - typeof raw === "number" && Number.isFinite(raw) - ? raw - : typeof raw === "string" && raw.trim() - ? Number.parseFloat(raw) - : 5; - return Math.max(1, Math.min(10, Math.trunc(parsed))); -} - -async function throwXaiWebSearchApiError(res: Response): Promise { - const detailResult = await readResponseText(res, { maxBytes: 64_000 }); - throw new Error(`xAI API error (${res.status}): ${detailResult.text || res.statusText}`); -} - -async function runXaiWebSearch(params: { +function runXaiWebSearch(params: { query: string; model: string; apiKey: string; @@ -144,61 +40,31 @@ async function runXaiWebSearch(params: { ); const cached = readCache(XAI_WEB_SEARCH_CACHE, cacheKey); if (cached) { - return { ...cached.value, cached: true }; + return Promise.resolve({ ...cached.value, cached: true }); } - const startedAt = Date.now(); - const payload = await withTrustedWebToolsEndpoint( - { - url: XAI_WEB_SEARCH_ENDPOINT, + return (async () => { + const startedAt = Date.now(); + const result = await requestXaiWebSearch({ + query: params.query, + model: params.model, + apiKey: params.apiKey, timeoutSeconds: params.timeoutSeconds, - init: { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${params.apiKey}`, - }, - body: JSON.stringify({ - model: params.model, - input: [{ role: "user", content: params.query }], - tools: [{ type: "web_search" }], - }), - }, - }, - async ({ response }) => { - if (!response.ok) { - return await throwXaiWebSearchApiError(response); - } + inlineCitations: params.inlineCitations, + }); + const payload = buildXaiWebSearchPayload({ + query: params.query, + provider: "grok", + model: params.model, + tookMs: Date.now() - startedAt, + content: result.content, + citations: result.citations, + inlineCitations: result.inlineCitations, + }); - const data = (await response.json()) as XaiWebSearchResponse; - const { text, annotationCitations } = extractXaiWebSearchContent(data); - const citations = - Array.isArray(data.citations) && data.citations.length > 0 - ? data.citations - : annotationCitations; - - return { - query: params.query, - provider: "grok", - model: params.model, - tookMs: Date.now() - startedAt, - externalContent: { - untrusted: true, - source: "web_search", - provider: "grok", - wrapped: true, - }, - content: wrapWebContent(text ?? "No response", "web_search"), - citations, - ...(params.inlineCitations && Array.isArray(data.inline_citations) - ? { inlineCitations: data.inline_citations } - : {}), - }; - }, - ); - - writeCache(XAI_WEB_SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; + writeCache(XAI_WEB_SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); + return payload; + })(); } export function createXaiWebSearchProvider(): WebSearchProviderPlugin { @@ -246,8 +112,9 @@ export function createXaiWebSearchProvider(): WebSearchProviderPlugin { }; } - const query = readQuery(args); - const count = readCount(args); + const query = readStringParam(args, "query", { required: true }); + void readNumberParam(args, "count", { integer: true }); + return await runXaiWebSearch({ query, model: resolveXaiWebSearchModel(ctx.searchConfig), @@ -268,7 +135,9 @@ export function createXaiWebSearchProvider(): WebSearchProviderPlugin { } export const __testing = { + buildXaiWebSearchPayload, extractXaiWebSearchContent, - resolveXaiWebSearchModel, resolveXaiInlineCitations, + resolveXaiWebSearchModel, + requestXaiWebSearch, }; diff --git a/src/agents/tools/web-search-provider-common.ts b/src/agents/tools/web-search-provider-common.ts index f69876ed04a..79827ef7cb8 100644 --- a/src/agents/tools/web-search-provider-common.ts +++ b/src/agents/tools/web-search-provider-common.ts @@ -92,6 +92,45 @@ export async function withTrustedWebSearchEndpoint( ); } +export async function postTrustedWebToolsJson( + params: { + url: string; + timeoutSeconds: number; + apiKey: string; + body: Record; + errorLabel: string; + maxErrorBytes?: number; + }, + parseResponse: (response: Response) => Promise, +): Promise { + return withTrustedWebToolsEndpoint( + { + url: params.url, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + Accept: "application/json", + Authorization: `Bearer ${params.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(params.body), + }, + }, + async ({ response }) => { + if (!response.ok) { + const detail = await readResponseText(response, { + maxBytes: params.maxErrorBytes ?? 64_000, + }); + throw new Error( + `${params.errorLabel} API error (${response.status}): ${detail.text || response.statusText}`, + ); + } + return await parseResponse(response); + }, + ); +} + export async function throwWebSearchApiError(res: Response, providerLabel: string): Promise { const detailResult = await readResponseText(res, { maxBytes: 64_000 }); const detail = detailResult.text; diff --git a/src/plugin-sdk/provider-web-search.ts b/src/plugin-sdk/provider-web-search.ts index 258d26e7ee4..9ed067cbf23 100644 --- a/src/plugin-sdk/provider-web-search.ts +++ b/src/plugin-sdk/provider-web-search.ts @@ -23,6 +23,7 @@ export { resolveSearchCount, resolveSearchTimeoutSeconds, resolveSiteName, + postTrustedWebToolsJson, throwWebSearchApiError, withTrustedWebSearchEndpoint, writeCachedSearchPayload, diff --git a/src/plugins/config-state.ts b/src/plugins/config-state.ts index 26827e50aa3..986f038e4cd 100644 --- a/src/plugins/config-state.ts +++ b/src/plugins/config-state.ts @@ -193,7 +193,7 @@ const hasExplicitMemorySlot = (plugins?: OpenClawConfig["plugins"]) => const hasExplicitMemoryEntry = (plugins?: OpenClawConfig["plugins"]) => Boolean(plugins?.entries && Object.prototype.hasOwnProperty.call(plugins.entries, "memory-core")); -const hasExplicitPluginConfig = (plugins?: OpenClawConfig["plugins"]) => { +export const hasExplicitPluginConfig = (plugins?: OpenClawConfig["plugins"]) => { if (!plugins) { return false; } diff --git a/src/plugins/providers.ts b/src/plugins/providers.ts index e966e9d4128..7c69aa7ca41 100644 --- a/src/plugins/providers.ts +++ b/src/plugins/providers.ts @@ -4,6 +4,7 @@ import { withBundledPluginAllowlistCompat, withBundledPluginEnablementCompat, } from "./bundled-compat.js"; +import { hasExplicitPluginConfig } from "./config-state.js"; import { normalizePluginsConfig, resolveEffectiveEnableState } from "./config-state.js"; import { loadOpenClawPlugins, type PluginLoadOptions } from "./loader.js"; import { createPluginLoaderLogger } from "./logger.js"; @@ -12,39 +13,17 @@ import type { ProviderPlugin } from "./types.js"; const log = createSubsystemLogger("plugins"); -function hasExplicitPluginConfig(config: PluginLoadOptions["config"]): boolean { - const plugins = config?.plugins; - if (!plugins) { - return false; - } - if (typeof plugins.enabled === "boolean") { - return true; - } - if (Array.isArray(plugins.allow) && plugins.allow.length > 0) { - return true; - } - if (Array.isArray(plugins.deny) && plugins.deny.length > 0) { - return true; - } - if (Array.isArray(plugins.load?.paths) && plugins.load.paths.length > 0) { - return true; - } - if (plugins.entries && Object.keys(plugins.entries).length > 0) { - return true; - } - if (plugins.slots && Object.keys(plugins.slots).length > 0) { - return true; - } - return false; -} - function withBundledProviderVitestCompat(params: { config: PluginLoadOptions["config"]; pluginIds: readonly string[]; env?: PluginLoadOptions["env"]; }): PluginLoadOptions["config"] { const env = params.env ?? process.env; - if (!env.VITEST || hasExplicitPluginConfig(params.config) || params.pluginIds.length === 0) { + if ( + !env.VITEST || + hasExplicitPluginConfig(params.config?.plugins) || + params.pluginIds.length === 0 + ) { return params.config; } diff --git a/src/plugins/web-search-providers.shared.ts b/src/plugins/web-search-providers.shared.ts index 29ba9527590..31a90f50915 100644 --- a/src/plugins/web-search-providers.shared.ts +++ b/src/plugins/web-search-providers.shared.ts @@ -3,36 +3,14 @@ import { withBundledPluginEnablementCompat, } from "./bundled-compat.js"; import { resolveBundledWebSearchPluginIds } from "./bundled-web-search.js"; -import { normalizePluginsConfig, type NormalizedPluginsConfig } from "./config-state.js"; +import { + hasExplicitPluginConfig, + normalizePluginsConfig, + type NormalizedPluginsConfig, +} from "./config-state.js"; import type { PluginLoadOptions } from "./loader.js"; import type { PluginWebSearchProviderEntry } from "./types.js"; -export function hasExplicitPluginConfig(config: PluginLoadOptions["config"]): boolean { - const plugins = config?.plugins; - if (!plugins) { - return false; - } - if (typeof plugins.enabled === "boolean") { - return true; - } - if (Array.isArray(plugins.allow) && plugins.allow.length > 0) { - return true; - } - if (Array.isArray(plugins.deny) && plugins.deny.length > 0) { - return true; - } - if (Array.isArray(plugins.load?.paths) && plugins.load.paths.length > 0) { - return true; - } - if (plugins.entries && Object.keys(plugins.entries).length > 0) { - return true; - } - if (plugins.slots && Object.keys(plugins.slots).length > 0) { - return true; - } - return false; -} - function resolveBundledWebSearchCompatPluginIds(params: { config?: PluginLoadOptions["config"]; workspaceDir?: string; @@ -52,7 +30,11 @@ function withBundledWebSearchVitestCompat(params: { }): PluginLoadOptions["config"] { const env = params.env ?? process.env; const isVitest = Boolean(env.VITEST || process.env.VITEST); - if (!isVitest || hasExplicitPluginConfig(params.config) || params.pluginIds.length === 0) { + if ( + !isVitest || + hasExplicitPluginConfig(params.config?.plugins) || + params.pluginIds.length === 0 + ) { return params.config; }