diff --git a/src/extension-host/runtime-backend-catalog.test.ts b/src/extension-host/runtime-backend-catalog.test.ts new file mode 100644 index 00000000000..eae0e2aa04f --- /dev/null +++ b/src/extension-host/runtime-backend-catalog.test.ts @@ -0,0 +1,121 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("./embedding-runtime-registry.js", () => ({ + EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS: ["openai", "gemini", "voyage", "mistral"], +})); + +vi.mock("./media-runtime-registry.js", () => ({ + buildExtensionHostMediaUnderstandingRegistry: vi.fn( + () => + new Map([ + [ + "openai", + { + id: "openai", + capabilities: ["image", "video"], + }, + ], + [ + "google", + { + id: "google", + capabilities: ["image"], + }, + ], + [ + "deepgram", + { + id: "deepgram", + capabilities: ["audio"], + }, + ], + ]), + ), + normalizeExtensionHostMediaProviderId: vi.fn((id: string) => + id.trim().toLowerCase() === "gemini" ? "google" : id.trim().toLowerCase(), + ), +})); + +vi.mock("./tts-runtime-registry.js", () => ({ + listExtensionHostTtsRuntimeProviders: vi.fn(() => [ + { id: "openai", supportsTelephony: true }, + { id: "elevenlabs", supportsTelephony: true }, + { id: "edge", supportsTelephony: false }, + ]), +})); + +describe("runtime-backend-catalog", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("publishes embedding backends as host-owned runtime-backend catalog entries", async () => { + const catalog = await import("./runtime-backend-catalog.js"); + const entries = catalog.listExtensionHostEmbeddingRuntimeBackendCatalogEntries(); + + expect(entries.map((entry) => entry.backendId)).toEqual([ + "local", + "openai", + "gemini", + "voyage", + "mistral", + "ollama", + ]); + expect( + entries.every((entry) => entry.family === catalog.EXTENSION_HOST_RUNTIME_BACKEND_FAMILY), + ).toBe(true); + expect(entries.every((entry) => entry.subsystemId === "embedding")).toBe(true); + expect(entries[0]?.capabilities).toContain("embed.query"); + expect(entries[0]?.metadata).toMatchObject({ autoSelectable: true }); + expect(entries.at(-1)?.metadata).toMatchObject({ autoSelectable: false }); + }); + + it("splits media providers into subsystem-specific runtime-backend catalog entries", async () => { + const catalog = await import("./runtime-backend-catalog.js"); + const entries = catalog.listExtensionHostMediaRuntimeBackendCatalogEntries(); + + expect(entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + subsystemId: "media.image", + backendId: "openai", + capabilities: ["image"], + }), + expect.objectContaining({ + subsystemId: "media.audio", + backendId: "deepgram", + capabilities: ["audio"], + }), + ]), + ); + expect(entries.find((entry) => entry.backendId === "google")?.selectorKeys).toContain("gemini"); + }); + + it("publishes TTS backends with telephony capability metadata", async () => { + const catalog = await import("./runtime-backend-catalog.js"); + const entries = catalog.listExtensionHostTtsRuntimeBackendCatalogEntries(); + + expect(entries.map((entry) => entry.backendId)).toEqual(["openai", "elevenlabs", "edge"]); + expect(entries.find((entry) => entry.backendId === "openai")?.capabilities).toContain( + "tts.telephony", + ); + expect(entries.find((entry) => entry.backendId === "edge")?.capabilities).toEqual([ + "tts.synthesis", + ]); + }); + + it("aggregates runtime-backend catalog entries across subsystem families", async () => { + const catalog = await import("./runtime-backend-catalog.js"); + const entries = catalog.listExtensionHostRuntimeBackendCatalogEntries(); + const ids = new Set(entries.map((entry) => entry.id)); + + expect(ids.size).toBe(entries.length); + expect( + catalog.getExtensionHostRuntimeBackendCatalogEntry({ subsystemId: "tts", backendId: "edge" }), + ).toMatchObject({ + id: `${catalog.EXTENSION_HOST_RUNTIME_BACKEND_FAMILY}:tts:edge`, + subsystemId: "tts", + backendId: "edge", + }); + }); +}); diff --git a/src/extension-host/runtime-backend-catalog.ts b/src/extension-host/runtime-backend-catalog.ts new file mode 100644 index 00000000000..c46188d12a9 --- /dev/null +++ b/src/extension-host/runtime-backend-catalog.ts @@ -0,0 +1,139 @@ +import type { MediaUnderstandingCapability } from "../media-understanding/types.js"; +import { EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS } from "./embedding-runtime-registry.js"; +import type { EmbeddingProviderId } from "./embedding-runtime-types.js"; +import { + buildExtensionHostMediaUnderstandingRegistry, + normalizeExtensionHostMediaProviderId, +} from "./media-runtime-registry.js"; +import { listExtensionHostTtsRuntimeProviders } from "./tts-runtime-registry.js"; + +export const EXTENSION_HOST_RUNTIME_BACKEND_FAMILY = "capability.runtime-backend"; + +export type ExtensionHostRuntimeBackendFamily = typeof EXTENSION_HOST_RUNTIME_BACKEND_FAMILY; + +export type ExtensionHostRuntimeBackendSubsystemId = + | "embedding" + | "media.audio" + | "media.image" + | "media.video" + | "tts"; + +export type ExtensionHostRuntimeBackendCatalogEntry = { + id: string; + family: ExtensionHostRuntimeBackendFamily; + subsystemId: ExtensionHostRuntimeBackendSubsystemId; + backendId: string; + source: "builtin"; + defaultRank: number; + selectorKeys: readonly string[]; + capabilities: readonly string[]; + metadata?: Record; +}; + +const EXTENSION_HOST_EMBEDDING_BACKEND_IDS = [ + "local", + ...EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS, + "ollama", +] as const satisfies readonly EmbeddingProviderId[]; + +function buildRuntimeBackendCatalogId( + subsystemId: ExtensionHostRuntimeBackendSubsystemId, + backendId: string, +): string { + return `${EXTENSION_HOST_RUNTIME_BACKEND_FAMILY}:${subsystemId}:${backendId}`; +} + +function mapMediaCapabilityToSubsystem( + capability: MediaUnderstandingCapability, +): ExtensionHostRuntimeBackendSubsystemId { + if (capability === "audio") { + return "media.audio"; + } + if (capability === "video") { + return "media.video"; + } + return "media.image"; +} + +function buildMediaSelectorKeys(providerId: string): readonly string[] { + const normalized = normalizeExtensionHostMediaProviderId(providerId); + if (normalized === "google") { + return [providerId, "gemini"]; + } + return normalized === providerId ? [providerId] : [providerId, normalized]; +} + +export function listExtensionHostEmbeddingRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] { + return EXTENSION_HOST_EMBEDDING_BACKEND_IDS.map((backendId, defaultRank) => ({ + id: buildRuntimeBackendCatalogId("embedding", backendId), + family: EXTENSION_HOST_RUNTIME_BACKEND_FAMILY, + subsystemId: "embedding", + backendId, + source: "builtin", + defaultRank, + selectorKeys: [backendId], + capabilities: ["embed.query", "embed.batch"], + metadata: { + autoSelectable: + backendId === "local" || EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS.includes(backendId), + }, + })); +} + +export function listExtensionHostMediaRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] { + const registry = buildExtensionHostMediaUnderstandingRegistry(); + const entries: ExtensionHostRuntimeBackendCatalogEntry[] = []; + let defaultRank = 0; + for (const provider of registry.values()) { + for (const capability of provider.capabilities ?? []) { + const subsystemId = mapMediaCapabilityToSubsystem(capability); + entries.push({ + id: buildRuntimeBackendCatalogId(subsystemId, provider.id), + family: EXTENSION_HOST_RUNTIME_BACKEND_FAMILY, + subsystemId, + backendId: provider.id, + source: "builtin", + defaultRank, + selectorKeys: buildMediaSelectorKeys(provider.id), + capabilities: [capability], + }); + } + defaultRank += 1; + } + return entries; +} + +export function listExtensionHostTtsRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] { + return listExtensionHostTtsRuntimeProviders().map((provider, defaultRank) => ({ + id: buildRuntimeBackendCatalogId("tts", provider.id), + family: EXTENSION_HOST_RUNTIME_BACKEND_FAMILY, + subsystemId: "tts", + backendId: provider.id, + source: "builtin", + defaultRank, + selectorKeys: [provider.id], + capabilities: provider.supportsTelephony + ? ["tts.synthesis", "tts.telephony"] + : ["tts.synthesis"], + metadata: { + supportsTelephony: provider.supportsTelephony, + }, + })); +} + +export function listExtensionHostRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] { + return [ + ...listExtensionHostEmbeddingRuntimeBackendCatalogEntries(), + ...listExtensionHostMediaRuntimeBackendCatalogEntries(), + ...listExtensionHostTtsRuntimeBackendCatalogEntries(), + ]; +} + +export function getExtensionHostRuntimeBackendCatalogEntry(params: { + subsystemId: ExtensionHostRuntimeBackendSubsystemId; + backendId: string; +}): ExtensionHostRuntimeBackendCatalogEntry | undefined { + return listExtensionHostRuntimeBackendCatalogEntries().find( + (entry) => entry.subsystemId === params.subsystemId && entry.backendId === params.backendId, + ); +} diff --git a/src/extension-host/schema.ts b/src/extension-host/schema.ts index 7472ed3f4da..e2d10a037ee 100644 --- a/src/extension-host/schema.ts +++ b/src/extension-host/schema.ts @@ -24,6 +24,7 @@ export type ResolvedContributionKind = | "capability.context-engine" | "capability.memory" | "capability.provider-integration" + | "capability.runtime-backend" | "surface.channel-catalog" | "surface.config" | "surface.install";