diff --git a/src/agents/model-compat.test.ts b/src/agents/model-compat.test.ts index c1e79f4757a..385384ce3f7 100644 --- a/src/agents/model-compat.test.ts +++ b/src/agents/model-compat.test.ts @@ -338,6 +338,81 @@ describe("normalizeModelCompat", () => { }); }); +describe("normalizeModelCompat — vLLM and SGLang providers", () => { + function makeVllmModel(baseUrl: string): Model { + return { + id: "Qwen3.5-27B", + name: "Qwen3.5-27B", + api: "openai-completions", + provider: "vllm", + baseUrl, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32768, + maxTokens: 4096, + } as Model; + } + + function makeSglangModel(baseUrl: string): Model { + return { + ...makeVllmModel(baseUrl), + provider: "sglang", + } as Model; + } + + it("defaults supportsUsageInStreaming to true for vLLM self-hosted endpoints", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); + + it("defaults supportsUsageInStreaming to true for vLLM localhost endpoints", () => { + const model = makeVllmModel("http://127.0.0.1:8000/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); + + it("defaults supportsUsageInStreaming to true for SGLang self-hosted endpoints", () => { + const model = makeSglangModel("http://192.168.1.10:30000/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); + + it("still forces supportsDeveloperRole off for vLLM endpoints", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsDeveloperRole(normalized)).toBe(false); + }); + + it("still forces supportsStrictMode off for vLLM endpoints", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsStrictMode(normalized)).toBe(false); + }); + + it("respects explicit supportsUsageInStreaming false override for vLLM", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + (model as { compat?: unknown }).compat = { supportsUsageInStreaming: false }; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(false); + }); + + it("does not mutate caller model reference for vLLM", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(normalized).not.toBe(model); + expect(supportsUsageInStreaming(model)).toBeUndefined(); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); +}); + describe("isModernModelRef", () => { it("uses provider runtime hooks before fallback heuristics", () => { providerRuntimeMocks.resolveProviderModernModelRef.mockReturnValue(false); diff --git a/src/agents/model-compat.ts b/src/agents/model-compat.ts index efdad0e4958..cdd8696f703 100644 --- a/src/agents/model-compat.ts +++ b/src/agents/model-compat.ts @@ -82,6 +82,14 @@ function isOpenAINativeEndpoint(baseUrl: string): boolean { } } +/** + * Self-hosted providers known to correctly support `stream_options.include_usage` + * (i.e., they return a terminal usage-only chunk at the end of the stream). + * Unlike arbitrary OpenAI-compatible proxies, these are well-tested inference + * engines that faithfully implement the OpenAI streaming usage spec. + */ +const STREAMING_USAGE_ALLOWLIST = new Set(["vllm", "sglang"]); + function isAnthropicMessagesModel(model: Model): model is Model<"anthropic-messages"> { return model.api === "anthropic-messages"; } @@ -138,6 +146,13 @@ export function normalizeModelCompat(model: Model): Model { return model; } + // Self-hosted inference engines (vLLM, SGLang) correctly implement the + // OpenAI streaming usage spec and return a terminal usage-only chunk. + // Default supportsUsageInStreaming to true for these known providers so + // token counts are recorded even when no explicit compat override is set. + const providerLower = (model.provider ?? "").toLowerCase(); + const defaultStreamingUsage = STREAMING_USAGE_ALLOWLIST.has(providerLower); + // Return a new object — do not mutate the caller's model reference. return { ...model, @@ -145,12 +160,12 @@ export function normalizeModelCompat(model: Model): Model { ? { ...compat, supportsDeveloperRole: forcedDeveloperRole || false, - ...(hasStreamingUsageOverride ? {} : { supportsUsageInStreaming: false }), + ...(hasStreamingUsageOverride ? {} : { supportsUsageInStreaming: defaultStreamingUsage }), supportsStrictMode: targetStrictMode, } : { supportsDeveloperRole: false, - supportsUsageInStreaming: false, + supportsUsageInStreaming: defaultStreamingUsage, supportsStrictMode: false, }, } as typeof model;