diff --git a/src/plugins/bundled-web-search.test.ts b/src/plugins/bundled-web-search.test.ts index 7db116a426f..921bd66868e 100644 --- a/src/plugins/bundled-web-search.test.ts +++ b/src/plugins/bundled-web-search.test.ts @@ -1,13 +1,193 @@ -import { expect, it } from "vitest"; -import { resolveBundledWebSearchPluginIds } from "./bundled-web-search.js"; +import { describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import { + listBundledWebSearchProviders, + resolveBundledWebSearchPluginIds, +} from "./bundled-web-search.js"; +import { webSearchProviderContractRegistry } from "./contracts/registry.js"; -it("keeps bundled web search compat ids aligned with bundled manifests", () => { - expect(resolveBundledWebSearchPluginIds({})).toEqual([ - "brave", - "firecrawl", - "google", - "moonshot", - "perplexity", - "xai", - ]); +describe("bundled web search metadata", () => { + function toComparableEntry(params: { + pluginId: string; + provider: { + id: string; + label: string; + hint: string; + envVars: string[]; + placeholder: string; + signupUrl: string; + docsUrl?: string; + autoDetectOrder?: number; + credentialPath: string; + inactiveSecretPaths?: string[]; + getConfiguredCredentialValue?: unknown; + setConfiguredCredentialValue?: unknown; + applySelectionConfig?: unknown; + resolveRuntimeMetadata?: unknown; + }; + }) { + return { + pluginId: params.pluginId, + id: params.provider.id, + label: params.provider.label, + hint: params.provider.hint, + envVars: params.provider.envVars, + placeholder: params.provider.placeholder, + signupUrl: params.provider.signupUrl, + docsUrl: params.provider.docsUrl, + autoDetectOrder: params.provider.autoDetectOrder, + credentialPath: params.provider.credentialPath, + inactiveSecretPaths: params.provider.inactiveSecretPaths, + hasConfiguredCredentialAccessors: + typeof params.provider.getConfiguredCredentialValue === "function" && + typeof params.provider.setConfiguredCredentialValue === "function", + hasApplySelectionConfig: typeof params.provider.applySelectionConfig === "function", + hasResolveRuntimeMetadata: typeof params.provider.resolveRuntimeMetadata === "function", + }; + } + + function sortComparableEntries< + T extends { + autoDetectOrder?: number; + id: string; + pluginId: string; + }, + >(entries: T[]): T[] { + return [...entries].toSorted((left, right) => { + const leftOrder = left.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; + const rightOrder = right.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; + return ( + leftOrder - rightOrder || + left.id.localeCompare(right.id) || + left.pluginId.localeCompare(right.pluginId) + ); + }); + } + + it("keeps bundled web search compat ids aligned with bundled manifests", () => { + expect(resolveBundledWebSearchPluginIds({})).toEqual([ + "brave", + "firecrawl", + "google", + "moonshot", + "perplexity", + "xai", + ]); + }); + + it("keeps fast-path bundled provider metadata aligned with bundled plugin contracts", async () => { + const fastPathProviders = listBundledWebSearchProviders(); + + expect( + sortComparableEntries( + fastPathProviders.map((provider) => + toComparableEntry({ + pluginId: provider.pluginId, + provider, + }), + ), + ), + ).toEqual( + sortComparableEntries( + webSearchProviderContractRegistry.map(({ pluginId, provider }) => + toComparableEntry({ + pluginId, + provider, + }), + ), + ), + ); + + for (const fastPathProvider of fastPathProviders) { + const contractEntry = webSearchProviderContractRegistry.find( + (entry) => + entry.pluginId === fastPathProvider.pluginId && entry.provider.id === fastPathProvider.id, + ); + expect(contractEntry).toBeDefined(); + const contractProvider = contractEntry!.provider; + + const fastSearchConfig: Record = {}; + const contractSearchConfig: Record = {}; + fastPathProvider.setCredentialValue(fastSearchConfig, "test-key"); + contractProvider.setCredentialValue(contractSearchConfig, "test-key"); + expect(fastSearchConfig).toEqual(contractSearchConfig); + expect(fastPathProvider.getCredentialValue(fastSearchConfig)).toEqual( + contractProvider.getCredentialValue(contractSearchConfig), + ); + + const fastConfig = {} as OpenClawConfig; + const contractConfig = {} as OpenClawConfig; + fastPathProvider.setConfiguredCredentialValue?.(fastConfig, "test-key"); + contractProvider.setConfiguredCredentialValue?.(contractConfig, "test-key"); + expect(fastConfig).toEqual(contractConfig); + expect(fastPathProvider.getConfiguredCredentialValue?.(fastConfig)).toEqual( + contractProvider.getConfiguredCredentialValue?.(contractConfig), + ); + + if (fastPathProvider.applySelectionConfig || contractProvider.applySelectionConfig) { + expect(fastPathProvider.applySelectionConfig?.({} as OpenClawConfig)).toEqual( + contractProvider.applySelectionConfig?.({} as OpenClawConfig), + ); + } + + if (fastPathProvider.resolveRuntimeMetadata || contractProvider.resolveRuntimeMetadata) { + const metadataCases = [ + { + searchConfig: fastSearchConfig, + resolvedCredential: { + value: "pplx-test", + source: "secretRef" as const, + fallbackEnvVar: undefined, + }, + }, + { + searchConfig: fastSearchConfig, + resolvedCredential: { + value: undefined, + source: "env" as const, + fallbackEnvVar: "OPENROUTER_API_KEY", + }, + }, + { + searchConfig: { + ...fastSearchConfig, + perplexity: { + ...(fastSearchConfig.perplexity as Record | undefined), + model: "custom-model", + }, + }, + resolvedCredential: { + value: "pplx-test", + source: "secretRef" as const, + fallbackEnvVar: undefined, + }, + }, + ]; + + for (const testCase of metadataCases) { + expect( + await fastPathProvider.resolveRuntimeMetadata?.({ + config: fastConfig, + searchConfig: testCase.searchConfig, + runtimeMetadata: { + diagnostics: [], + providerSource: "configured", + }, + resolvedCredential: testCase.resolvedCredential, + }), + ).toEqual( + await contractProvider.resolveRuntimeMetadata?.({ + config: contractConfig, + searchConfig: testCase.searchConfig, + runtimeMetadata: { + diagnostics: [], + providerSource: "configured", + }, + resolvedCredential: testCase.resolvedCredential, + }), + ); + } + } + } + }); }); diff --git a/src/plugins/bundled-web-search.ts b/src/plugins/bundled-web-search.ts index 248928b093c..d1f2ce342f8 100644 --- a/src/plugins/bundled-web-search.ts +++ b/src/plugins/bundled-web-search.ts @@ -1,17 +1,251 @@ +import { + getScopedCredentialValue, + getTopLevelCredentialValue, + resolveProviderWebSearchPluginConfig, + setProviderWebSearchPluginConfigValue, + setScopedCredentialValue, + setTopLevelCredentialValue, +} from "../agents/tools/web-search-provider-config.js"; +import type { OpenClawConfig } from "../config/config.js"; +import type { RuntimeWebSearchMetadata } from "../secrets/runtime-web-tools.types.js"; +import { enablePluginInConfig } from "./enable.js"; import type { PluginLoadOptions } from "./loader.js"; import { loadPluginManifestRegistry } from "./manifest-registry.js"; +import type { PluginWebSearchProviderEntry, WebSearchRuntimeMetadataContext } from "./types.js"; + +const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; +const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; +const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; +const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; + +type BundledWebSearchProviderDescriptor = { + pluginId: string; + id: string; + label: string; + hint: string; + envVars: string[]; + placeholder: string; + signupUrl: string; + docsUrl?: string; + autoDetectOrder: number; + credentialPath: string; + inactiveSecretPaths: string[]; + credentialScope: + | { kind: "top-level" } + | { + kind: "scoped"; + key: string; + }; + supportsConfiguredCredentialValue?: boolean; + applySelectionConfig?: (config: OpenClawConfig) => OpenClawConfig; + resolveRuntimeMetadata?: ( + ctx: WebSearchRuntimeMetadataContext, + ) => Partial; +}; + +function inferPerplexityBaseUrlFromApiKey(apiKey?: string): "direct" | "openrouter" | undefined { + if (!apiKey) { + return undefined; + } + const normalized = apiKey.toLowerCase(); + if (PERPLEXITY_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "direct"; + } + if (OPENROUTER_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "openrouter"; + } + return undefined; +} + +function isDirectPerplexityBaseUrl(baseUrl: string): boolean { + try { + return new URL(baseUrl.trim()).hostname.toLowerCase() === "api.perplexity.ai"; + } catch { + return false; + } +} + +function resolvePerplexityRuntimeMetadata( + ctx: WebSearchRuntimeMetadataContext, +): Partial { + const perplexity = ctx.searchConfig?.perplexity; + const scoped = + perplexity && typeof perplexity === "object" && !Array.isArray(perplexity) + ? (perplexity as { baseUrl?: string; model?: string }) + : undefined; + const configuredBaseUrl = typeof scoped?.baseUrl === "string" ? scoped.baseUrl.trim() : ""; + const configuredModel = typeof scoped?.model === "string" ? scoped.model.trim() : ""; + const keySource = ctx.resolvedCredential?.source ?? "missing"; + const baseUrl = (() => { + if (configuredBaseUrl) { + return configuredBaseUrl; + } + if (keySource === "env") { + if (ctx.resolvedCredential?.fallbackEnvVar === "PERPLEXITY_API_KEY") { + return PERPLEXITY_DIRECT_BASE_URL; + } + if (ctx.resolvedCredential?.fallbackEnvVar === "OPENROUTER_API_KEY") { + return DEFAULT_PERPLEXITY_BASE_URL; + } + } + if ((keySource === "config" || keySource === "secretRef") && ctx.resolvedCredential?.value) { + return inferPerplexityBaseUrlFromApiKey(ctx.resolvedCredential.value) === "openrouter" + ? DEFAULT_PERPLEXITY_BASE_URL + : PERPLEXITY_DIRECT_BASE_URL; + } + return DEFAULT_PERPLEXITY_BASE_URL; + })(); + return { + perplexityTransport: + configuredBaseUrl || configuredModel || !isDirectPerplexityBaseUrl(baseUrl) + ? "chat_completions" + : "search_api", + }; +} + +const BUNDLED_WEB_SEARCH_PROVIDER_DESCRIPTORS = [ + { + pluginId: "brave", + id: "brave", + label: "Brave Search", + hint: "Structured results · country/language/time filters", + envVars: ["BRAVE_API_KEY"], + placeholder: "BSA...", + signupUrl: "https://brave.com/search/api/", + docsUrl: "https://docs.openclaw.ai/brave-search", + autoDetectOrder: 10, + credentialPath: "plugins.entries.brave.config.webSearch.apiKey", + inactiveSecretPaths: ["plugins.entries.brave.config.webSearch.apiKey"], + credentialScope: { kind: "top-level" }, + }, + { + pluginId: "google", + id: "gemini", + label: "Gemini (Google Search)", + hint: "Google Search grounding · AI-synthesized", + envVars: ["GEMINI_API_KEY"], + placeholder: "AIza...", + signupUrl: "https://aistudio.google.com/apikey", + docsUrl: "https://docs.openclaw.ai/tools/web", + autoDetectOrder: 20, + credentialPath: "plugins.entries.google.config.webSearch.apiKey", + inactiveSecretPaths: ["plugins.entries.google.config.webSearch.apiKey"], + credentialScope: { kind: "scoped", key: "gemini" }, + }, + { + pluginId: "xai", + id: "grok", + label: "Grok (xAI)", + hint: "xAI web-grounded responses", + envVars: ["XAI_API_KEY"], + placeholder: "xai-...", + signupUrl: "https://console.x.ai/", + docsUrl: "https://docs.openclaw.ai/tools/web", + autoDetectOrder: 30, + credentialPath: "plugins.entries.xai.config.webSearch.apiKey", + inactiveSecretPaths: ["plugins.entries.xai.config.webSearch.apiKey"], + credentialScope: { kind: "scoped", key: "grok" }, + supportsConfiguredCredentialValue: false, + }, + { + pluginId: "moonshot", + id: "kimi", + label: "Kimi (Moonshot)", + hint: "Moonshot web search", + envVars: ["KIMI_API_KEY", "MOONSHOT_API_KEY"], + placeholder: "sk-...", + signupUrl: "https://platform.moonshot.cn/", + docsUrl: "https://docs.openclaw.ai/tools/web", + autoDetectOrder: 40, + credentialPath: "plugins.entries.moonshot.config.webSearch.apiKey", + inactiveSecretPaths: ["plugins.entries.moonshot.config.webSearch.apiKey"], + credentialScope: { kind: "scoped", key: "kimi" }, + }, + { + pluginId: "perplexity", + id: "perplexity", + label: "Perplexity Search", + hint: "Structured results · domain/country/language/time filters", + envVars: ["PERPLEXITY_API_KEY", "OPENROUTER_API_KEY"], + placeholder: "pplx-...", + signupUrl: "https://www.perplexity.ai/settings/api", + docsUrl: "https://docs.openclaw.ai/perplexity", + autoDetectOrder: 50, + credentialPath: "plugins.entries.perplexity.config.webSearch.apiKey", + inactiveSecretPaths: ["plugins.entries.perplexity.config.webSearch.apiKey"], + credentialScope: { kind: "scoped", key: "perplexity" }, + resolveRuntimeMetadata: resolvePerplexityRuntimeMetadata, + }, + { + pluginId: "firecrawl", + id: "firecrawl", + label: "Firecrawl Search", + hint: "Structured results with optional result scraping", + envVars: ["FIRECRAWL_API_KEY"], + placeholder: "fc-...", + signupUrl: "https://www.firecrawl.dev/", + docsUrl: "https://docs.openclaw.ai/tools/firecrawl", + autoDetectOrder: 60, + credentialPath: "plugins.entries.firecrawl.config.webSearch.apiKey", + inactiveSecretPaths: ["plugins.entries.firecrawl.config.webSearch.apiKey"], + credentialScope: { kind: "scoped", key: "firecrawl" }, + applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config, + }, +] as const satisfies ReadonlyArray; export const BUNDLED_WEB_SEARCH_PLUGIN_IDS = [ - "brave", - "firecrawl", - "google", - "moonshot", - "perplexity", - "xai", -] as const; + ...new Set(BUNDLED_WEB_SEARCH_PROVIDER_DESCRIPTORS.map((descriptor) => descriptor.pluginId)), +] as ReadonlyArray; const bundledWebSearchPluginIdSet = new Set(BUNDLED_WEB_SEARCH_PLUGIN_IDS); +function buildBundledWebSearchProviderEntry( + descriptor: BundledWebSearchProviderDescriptor, +): PluginWebSearchProviderEntry { + const scopedKey = + descriptor.credentialScope.kind === "scoped" ? descriptor.credentialScope.key : undefined; + return { + pluginId: descriptor.pluginId, + id: descriptor.id, + label: descriptor.label, + hint: descriptor.hint, + envVars: [...descriptor.envVars], + placeholder: descriptor.placeholder, + signupUrl: descriptor.signupUrl, + docsUrl: descriptor.docsUrl, + autoDetectOrder: descriptor.autoDetectOrder, + credentialPath: descriptor.credentialPath, + inactiveSecretPaths: [...descriptor.inactiveSecretPaths], + getCredentialValue: + descriptor.credentialScope.kind === "top-level" + ? getTopLevelCredentialValue + : (searchConfig) => getScopedCredentialValue(searchConfig, scopedKey!), + setCredentialValue: + descriptor.credentialScope.kind === "top-level" + ? setTopLevelCredentialValue + : (searchConfigTarget, value) => + setScopedCredentialValue(searchConfigTarget, scopedKey!, value), + getConfiguredCredentialValue: + descriptor.supportsConfiguredCredentialValue === false + ? undefined + : (config) => resolveProviderWebSearchPluginConfig(config, descriptor.pluginId)?.apiKey, + setConfiguredCredentialValue: + descriptor.supportsConfiguredCredentialValue === false + ? undefined + : (configTarget, value) => { + setProviderWebSearchPluginConfigValue( + configTarget, + descriptor.pluginId, + "apiKey", + value, + ); + }, + applySelectionConfig: descriptor.applySelectionConfig, + resolveRuntimeMetadata: descriptor.resolveRuntimeMetadata, + createTool: () => null, + }; +} + export function resolveBundledWebSearchPluginIds(params: { config?: PluginLoadOptions["config"]; workspaceDir?: string; @@ -27,3 +261,19 @@ export function resolveBundledWebSearchPluginIds(params: { .map((plugin) => plugin.id) .toSorted((left, right) => left.localeCompare(right)); } + +export function listBundledWebSearchProviders(): PluginWebSearchProviderEntry[] { + return BUNDLED_WEB_SEARCH_PROVIDER_DESCRIPTORS.map((descriptor) => + buildBundledWebSearchProviderEntry(descriptor), + ); +} + +export function resolveBundledWebSearchPluginId( + providerId: string | undefined, +): string | undefined { + if (!providerId) { + return undefined; + } + return BUNDLED_WEB_SEARCH_PROVIDER_DESCRIPTORS.find((descriptor) => descriptor.id === providerId) + ?.pluginId; +} diff --git a/src/plugins/web-search-providers.test.ts b/src/plugins/web-search-providers.test.ts index 54a4f6ebdd3..77efae73237 100644 --- a/src/plugins/web-search-providers.test.ts +++ b/src/plugins/web-search-providers.test.ts @@ -3,6 +3,7 @@ import type { OpenClawConfig } from "../config/config.js"; import { createEmptyPluginRegistry } from "./registry.js"; import { setActivePluginRegistry } from "./runtime.js"; import { + resolveBundledPluginWebSearchProviders, resolvePluginWebSearchProviders, resolveRuntimeWebSearchProviders, } from "./web-search-providers.js"; @@ -170,6 +171,43 @@ describe("resolvePluginWebSearchProviders", () => { expect(providers).toEqual([]); }); + it("can resolve bundled providers without the plugin loader", () => { + const providers = resolveBundledPluginWebSearchProviders({ + bundledAllowlistCompat: true, + }); + + expect(providers.map((provider) => `${provider.pluginId}:${provider.id}`)).toEqual([ + "brave:brave", + "google:gemini", + "xai:grok", + "moonshot:kimi", + "perplexity:perplexity", + "firecrawl:firecrawl", + ]); + expect(loadOpenClawPluginsMock).not.toHaveBeenCalled(); + }); + + it("can scope bundled resolution to one plugin id", () => { + const providers = resolveBundledPluginWebSearchProviders({ + config: { + tools: { + web: { + search: { + provider: "gemini", + }, + }, + }, + }, + bundledAllowlistCompat: true, + onlyPluginIds: ["google"], + }); + + expect(providers.map((provider) => `${provider.pluginId}:${provider.id}`)).toEqual([ + "google:gemini", + ]); + expect(loadOpenClawPluginsMock).not.toHaveBeenCalled(); + }); + it("prefers the active plugin registry for runtime resolution", () => { const registry = createEmptyPluginRegistry(); registry.webSearchProviders.push({ diff --git a/src/plugins/web-search-providers.ts b/src/plugins/web-search-providers.ts index b415d7c7675..81acd38c827 100644 --- a/src/plugins/web-search-providers.ts +++ b/src/plugins/web-search-providers.ts @@ -3,7 +3,15 @@ import { withBundledPluginAllowlistCompat, withBundledPluginEnablementCompat, } from "./bundled-compat.js"; -import { resolveBundledWebSearchPluginIds } from "./bundled-web-search.js"; +import { + listBundledWebSearchProviders as listBundledWebSearchProviderEntries, + resolveBundledWebSearchPluginIds, +} from "./bundled-web-search.js"; +import { + normalizePluginsConfig, + resolveEffectiveEnableState, + type NormalizedPluginsConfig, +} from "./config-state.js"; import { loadOpenClawPlugins, type PluginLoadOptions } from "./loader.js"; import { createPluginLoaderLogger } from "./logger.js"; import { getActivePluginRegistry } from "./runtime.js"; @@ -87,14 +95,15 @@ function sortWebSearchProviders( }); } -export function resolvePluginWebSearchProviders(params: { +function resolveBundledWebSearchResolutionConfig(params: { config?: PluginLoadOptions["config"]; workspaceDir?: string; env?: PluginLoadOptions["env"]; bundledAllowlistCompat?: boolean; - activate?: boolean; - cache?: boolean; -}): PluginWebSearchProviderEntry[] { +}): { + config: PluginLoadOptions["config"]; + normalized: NormalizedPluginsConfig; +} { const bundledCompatPluginIds = resolveBundledWebSearchCompatPluginIds({ config: params.config, workspaceDir: params.workspaceDir, @@ -115,6 +124,50 @@ export function resolvePluginWebSearchProviders(params: { pluginIds: bundledCompatPluginIds, env: params.env, }); + + return { + config, + normalized: normalizePluginsConfig(config?.plugins), + }; +} + +function listBundledWebSearchProviders(): PluginWebSearchProviderEntry[] { + return sortWebSearchProviders(listBundledWebSearchProviderEntries()); +} + +export function resolveBundledPluginWebSearchProviders(params: { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + bundledAllowlistCompat?: boolean; + onlyPluginIds?: readonly string[]; +}): PluginWebSearchProviderEntry[] { + const { config, normalized } = resolveBundledWebSearchResolutionConfig(params); + const onlyPluginIdSet = + params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null; + + return listBundledWebSearchProviders().filter((provider) => { + if (onlyPluginIdSet && !onlyPluginIdSet.has(provider.pluginId)) { + return false; + } + return resolveEffectiveEnableState({ + id: provider.pluginId, + origin: "bundled", + config: normalized, + rootConfig: config, + }).enabled; + }); +} + +export function resolvePluginWebSearchProviders(params: { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + bundledAllowlistCompat?: boolean; + activate?: boolean; + cache?: boolean; +}): PluginWebSearchProviderEntry[] { + const { config } = resolveBundledWebSearchResolutionConfig(params); const registry = loadOpenClawPlugins({ config, workspaceDir: params.workspaceDir,