Extensions: add runtime backend catalog

src/extension-host/runtime-backend-catalog.test.ts

@@ -0,0 +1,121 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("./embedding-runtime-registry.js", () => ({
+  EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS: ["openai", "gemini", "voyage", "mistral"],
+}));
+
+vi.mock("./media-runtime-registry.js", () => ({
+  buildExtensionHostMediaUnderstandingRegistry: vi.fn(
+    () =>
+      new Map([
+        [
+          "openai",
+          {
+            id: "openai",
+            capabilities: ["image", "video"],
+          },
+        ],
+        [
+          "google",
+          {
+            id: "google",
+            capabilities: ["image"],
+          },
+        ],
+        [
+          "deepgram",
+          {
+            id: "deepgram",
+            capabilities: ["audio"],
+          },
+        ],
+      ]),
+  ),
+  normalizeExtensionHostMediaProviderId: vi.fn((id: string) =>
+    id.trim().toLowerCase() === "gemini" ? "google" : id.trim().toLowerCase(),
+  ),
+}));
+
+vi.mock("./tts-runtime-registry.js", () => ({
+  listExtensionHostTtsRuntimeProviders: vi.fn(() => [
+    { id: "openai", supportsTelephony: true },
+    { id: "elevenlabs", supportsTelephony: true },
+    { id: "edge", supportsTelephony: false },
+  ]),
+}));
+
+describe("runtime-backend-catalog", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("publishes embedding backends as host-owned runtime-backend catalog entries", async () => {
+    const catalog = await import("./runtime-backend-catalog.js");
+    const entries = catalog.listExtensionHostEmbeddingRuntimeBackendCatalogEntries();
+
+    expect(entries.map((entry) => entry.backendId)).toEqual([
+      "local",
+      "openai",
+      "gemini",
+      "voyage",
+      "mistral",
+      "ollama",
+    ]);
+    expect(
+      entries.every((entry) => entry.family === catalog.EXTENSION_HOST_RUNTIME_BACKEND_FAMILY),
+    ).toBe(true);
+    expect(entries.every((entry) => entry.subsystemId === "embedding")).toBe(true);
+    expect(entries[0]?.capabilities).toContain("embed.query");
+    expect(entries[0]?.metadata).toMatchObject({ autoSelectable: true });
+    expect(entries.at(-1)?.metadata).toMatchObject({ autoSelectable: false });
+  });
+
+  it("splits media providers into subsystem-specific runtime-backend catalog entries", async () => {
+    const catalog = await import("./runtime-backend-catalog.js");
+    const entries = catalog.listExtensionHostMediaRuntimeBackendCatalogEntries();
+
+    expect(entries).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          subsystemId: "media.image",
+          backendId: "openai",
+          capabilities: ["image"],
+        }),
+        expect.objectContaining({
+          subsystemId: "media.audio",
+          backendId: "deepgram",
+          capabilities: ["audio"],
+        }),
+      ]),
+    );
+    expect(entries.find((entry) => entry.backendId === "google")?.selectorKeys).toContain("gemini");
+  });
+
+  it("publishes TTS backends with telephony capability metadata", async () => {
+    const catalog = await import("./runtime-backend-catalog.js");
+    const entries = catalog.listExtensionHostTtsRuntimeBackendCatalogEntries();
+
+    expect(entries.map((entry) => entry.backendId)).toEqual(["openai", "elevenlabs", "edge"]);
+    expect(entries.find((entry) => entry.backendId === "openai")?.capabilities).toContain(
+      "tts.telephony",
+    );
+    expect(entries.find((entry) => entry.backendId === "edge")?.capabilities).toEqual([
+      "tts.synthesis",
+    ]);
+  });
+
+  it("aggregates runtime-backend catalog entries across subsystem families", async () => {
+    const catalog = await import("./runtime-backend-catalog.js");
+    const entries = catalog.listExtensionHostRuntimeBackendCatalogEntries();
+    const ids = new Set(entries.map((entry) => entry.id));
+
+    expect(ids.size).toBe(entries.length);
+    expect(
+      catalog.getExtensionHostRuntimeBackendCatalogEntry({ subsystemId: "tts", backendId: "edge" }),
+    ).toMatchObject({
+      id: `${catalog.EXTENSION_HOST_RUNTIME_BACKEND_FAMILY}:tts:edge`,
+      subsystemId: "tts",
+      backendId: "edge",
+    });
+  });
+});

src/extension-host/runtime-backend-catalog.ts

@@ -0,0 +1,139 @@
+import type { MediaUnderstandingCapability } from "../media-understanding/types.js";
+import { EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS } from "./embedding-runtime-registry.js";
+import type { EmbeddingProviderId } from "./embedding-runtime-types.js";
+import {
+  buildExtensionHostMediaUnderstandingRegistry,
+  normalizeExtensionHostMediaProviderId,
+} from "./media-runtime-registry.js";
+import { listExtensionHostTtsRuntimeProviders } from "./tts-runtime-registry.js";
+
+export const EXTENSION_HOST_RUNTIME_BACKEND_FAMILY = "capability.runtime-backend";
+
+export type ExtensionHostRuntimeBackendFamily = typeof EXTENSION_HOST_RUNTIME_BACKEND_FAMILY;
+
+export type ExtensionHostRuntimeBackendSubsystemId =
+  | "embedding"
+  | "media.audio"
+  | "media.image"
+  | "media.video"
+  | "tts";
+
+export type ExtensionHostRuntimeBackendCatalogEntry = {
+  id: string;
+  family: ExtensionHostRuntimeBackendFamily;
+  subsystemId: ExtensionHostRuntimeBackendSubsystemId;
+  backendId: string;
+  source: "builtin";
+  defaultRank: number;
+  selectorKeys: readonly string[];
+  capabilities: readonly string[];
+  metadata?: Record<string, unknown>;
+};
+
+const EXTENSION_HOST_EMBEDDING_BACKEND_IDS = [
+  "local",
+  ...EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS,
+  "ollama",
+] as const satisfies readonly EmbeddingProviderId[];
+
+function buildRuntimeBackendCatalogId(
+  subsystemId: ExtensionHostRuntimeBackendSubsystemId,
+  backendId: string,
+): string {
+  return `${EXTENSION_HOST_RUNTIME_BACKEND_FAMILY}:${subsystemId}:${backendId}`;
+}
+
+function mapMediaCapabilityToSubsystem(
+  capability: MediaUnderstandingCapability,
+): ExtensionHostRuntimeBackendSubsystemId {
+  if (capability === "audio") {
+    return "media.audio";
+  }
+  if (capability === "video") {
+    return "media.video";
+  }
+  return "media.image";
+}
+
+function buildMediaSelectorKeys(providerId: string): readonly string[] {
+  const normalized = normalizeExtensionHostMediaProviderId(providerId);
+  if (normalized === "google") {
+    return [providerId, "gemini"];
+  }
+  return normalized === providerId ? [providerId] : [providerId, normalized];
+}
+
+export function listExtensionHostEmbeddingRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] {
+  return EXTENSION_HOST_EMBEDDING_BACKEND_IDS.map((backendId, defaultRank) => ({
+    id: buildRuntimeBackendCatalogId("embedding", backendId),
+    family: EXTENSION_HOST_RUNTIME_BACKEND_FAMILY,
+    subsystemId: "embedding",
+    backendId,
+    source: "builtin",
+    defaultRank,
+    selectorKeys: [backendId],
+    capabilities: ["embed.query", "embed.batch"],
+    metadata: {
+      autoSelectable:
+        backendId === "local" || EXTENSION_HOST_REMOTE_EMBEDDING_PROVIDER_IDS.includes(backendId),
+    },
+  }));
+}
+
+export function listExtensionHostMediaRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] {
+  const registry = buildExtensionHostMediaUnderstandingRegistry();
+  const entries: ExtensionHostRuntimeBackendCatalogEntry[] = [];
+  let defaultRank = 0;
+  for (const provider of registry.values()) {
+    for (const capability of provider.capabilities ?? []) {
+      const subsystemId = mapMediaCapabilityToSubsystem(capability);
+      entries.push({
+        id: buildRuntimeBackendCatalogId(subsystemId, provider.id),
+        family: EXTENSION_HOST_RUNTIME_BACKEND_FAMILY,
+        subsystemId,
+        backendId: provider.id,
+        source: "builtin",
+        defaultRank,
+        selectorKeys: buildMediaSelectorKeys(provider.id),
+        capabilities: [capability],
+      });
+    }
+    defaultRank += 1;
+  }
+  return entries;
+}
+
+export function listExtensionHostTtsRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] {
+  return listExtensionHostTtsRuntimeProviders().map((provider, defaultRank) => ({
+    id: buildRuntimeBackendCatalogId("tts", provider.id),
+    family: EXTENSION_HOST_RUNTIME_BACKEND_FAMILY,
+    subsystemId: "tts",
+    backendId: provider.id,
+    source: "builtin",
+    defaultRank,
+    selectorKeys: [provider.id],
+    capabilities: provider.supportsTelephony
+      ? ["tts.synthesis", "tts.telephony"]
+      : ["tts.synthesis"],
+    metadata: {
+      supportsTelephony: provider.supportsTelephony,
+    },
+  }));
+}
+
+export function listExtensionHostRuntimeBackendCatalogEntries(): readonly ExtensionHostRuntimeBackendCatalogEntry[] {
+  return [
+    ...listExtensionHostEmbeddingRuntimeBackendCatalogEntries(),
+    ...listExtensionHostMediaRuntimeBackendCatalogEntries(),
+    ...listExtensionHostTtsRuntimeBackendCatalogEntries(),
+  ];
+}
+
+export function getExtensionHostRuntimeBackendCatalogEntry(params: {
+  subsystemId: ExtensionHostRuntimeBackendSubsystemId;
+  backendId: string;
+}): ExtensionHostRuntimeBackendCatalogEntry | undefined {
+  return listExtensionHostRuntimeBackendCatalogEntries().find(
+    (entry) => entry.subsystemId === params.subsystemId && entry.backendId === params.backendId,
+  );
+}

src/extension-host/schema.ts

@@ -24,6 +24,7 @@ export type ResolvedContributionKind =
   | "capability.context-engine"
   | "capability.memory"
   | "capability.provider-integration"
+  | "capability.runtime-backend"
   | "surface.channel-catalog"
   | "surface.config"
   | "surface.install";