From c33ea8e30a78eab31ef52279546bd5e647a7b68b Mon Sep 17 00:00:00 2001 From: Antonio Date: Tue, 17 Mar 2026 13:21:50 -0300 Subject: [PATCH 1/3] fix: enable token usage tracking for vLLM and SGLang providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vLLM (and SGLang) correctly implement the OpenAI streaming usage spec and return a terminal usage-only chunk when stream_options.include_usage is set. However, normalizeModelCompat was unconditionally disabling supportsUsageInStreaming for all non-native OpenAI endpoints, which prevented stream_options from being sent to these providers. This caused token counts to never be recorded in session files, making the dashboard show "n/a" in the Tokens column and /status report "0/66k (0%)" for all vLLM sessions. (#47639) The fix introduces STREAMING_USAGE_ALLOWLIST — a set of known self-hosted inference engines that faithfully implement the spec — and defaults supportsUsageInStreaming to true for providers in that list. Explicit user-configured compat overrides are still respected, so a user can opt out by setting compat.supportsUsageInStreaming: false. Fixes #47639 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/agents/model-compat.test.ts | 75 +++++++++++++++++++++++++++++++++ src/agents/model-compat.ts | 19 ++++++++- 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/agents/model-compat.test.ts b/src/agents/model-compat.test.ts index c1e79f4757a..385384ce3f7 100644 --- a/src/agents/model-compat.test.ts +++ b/src/agents/model-compat.test.ts @@ -338,6 +338,81 @@ describe("normalizeModelCompat", () => { }); }); +describe("normalizeModelCompat — vLLM and SGLang providers", () => { + function makeVllmModel(baseUrl: string): Model { + return { + id: "Qwen3.5-27B", + name: "Qwen3.5-27B", + api: "openai-completions", + provider: "vllm", + baseUrl, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32768, + maxTokens: 4096, + } as Model; + } + + function makeSglangModel(baseUrl: string): Model { + return { + ...makeVllmModel(baseUrl), + provider: "sglang", + } as Model; + } + + it("defaults supportsUsageInStreaming to true for vLLM self-hosted endpoints", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); + + it("defaults supportsUsageInStreaming to true for vLLM localhost endpoints", () => { + const model = makeVllmModel("http://127.0.0.1:8000/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); + + it("defaults supportsUsageInStreaming to true for SGLang self-hosted endpoints", () => { + const model = makeSglangModel("http://192.168.1.10:30000/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); + + it("still forces supportsDeveloperRole off for vLLM endpoints", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsDeveloperRole(normalized)).toBe(false); + }); + + it("still forces supportsStrictMode off for vLLM endpoints", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(supportsStrictMode(normalized)).toBe(false); + }); + + it("respects explicit supportsUsageInStreaming false override for vLLM", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + (model as { compat?: unknown }).compat = { supportsUsageInStreaming: false }; + const normalized = normalizeModelCompat(model); + expect(supportsUsageInStreaming(normalized)).toBe(false); + }); + + it("does not mutate caller model reference for vLLM", () => { + const model = makeVllmModel("http://192.168.88.35:9090/v1"); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(normalized).not.toBe(model); + expect(supportsUsageInStreaming(model)).toBeUndefined(); + expect(supportsUsageInStreaming(normalized)).toBe(true); + }); +}); + describe("isModernModelRef", () => { it("uses provider runtime hooks before fallback heuristics", () => { providerRuntimeMocks.resolveProviderModernModelRef.mockReturnValue(false); diff --git a/src/agents/model-compat.ts b/src/agents/model-compat.ts index efdad0e4958..cdd8696f703 100644 --- a/src/agents/model-compat.ts +++ b/src/agents/model-compat.ts @@ -82,6 +82,14 @@ function isOpenAINativeEndpoint(baseUrl: string): boolean { } } +/** + * Self-hosted providers known to correctly support `stream_options.include_usage` + * (i.e., they return a terminal usage-only chunk at the end of the stream). + * Unlike arbitrary OpenAI-compatible proxies, these are well-tested inference + * engines that faithfully implement the OpenAI streaming usage spec. + */ +const STREAMING_USAGE_ALLOWLIST = new Set(["vllm", "sglang"]); + function isAnthropicMessagesModel(model: Model): model is Model<"anthropic-messages"> { return model.api === "anthropic-messages"; } @@ -138,6 +146,13 @@ export function normalizeModelCompat(model: Model): Model { return model; } + // Self-hosted inference engines (vLLM, SGLang) correctly implement the + // OpenAI streaming usage spec and return a terminal usage-only chunk. + // Default supportsUsageInStreaming to true for these known providers so + // token counts are recorded even when no explicit compat override is set. + const providerLower = (model.provider ?? "").toLowerCase(); + const defaultStreamingUsage = STREAMING_USAGE_ALLOWLIST.has(providerLower); + // Return a new object — do not mutate the caller's model reference. return { ...model, @@ -145,12 +160,12 @@ export function normalizeModelCompat(model: Model): Model { ? { ...compat, supportsDeveloperRole: forcedDeveloperRole || false, - ...(hasStreamingUsageOverride ? {} : { supportsUsageInStreaming: false }), + ...(hasStreamingUsageOverride ? {} : { supportsUsageInStreaming: defaultStreamingUsage }), supportsStrictMode: targetStrictMode, } : { supportsDeveloperRole: false, - supportsUsageInStreaming: false, + supportsUsageInStreaming: defaultStreamingUsage, supportsStrictMode: false, }, } as typeof model; From 41a587571524895eeb0240d12ee33ed40c5d1345 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 20 Mar 2026 10:00:47 -0300 Subject: [PATCH 2/3] fix(test): use async jiti import path for Node 24.13+ compatibility Node 24.13+ sealed the require property on ESM module objects as a read-only getter, breaking Jiti's sync CJS path when tryNative is false. Switch to the async .import() helper which uses dynamic import() instead. --- src/plugins/loader.git-path-regression.test.ts | 8 ++++---- src/plugins/loader.test.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/plugins/loader.git-path-regression.test.ts b/src/plugins/loader.git-path-regression.test.ts index 23ab4f4243d..674f2fd53c0 100644 --- a/src/plugins/loader.git-path-regression.test.ts +++ b/src/plugins/loader.git-path-regression.test.ts @@ -77,9 +77,9 @@ export const copiedRuntimeMarker = { ...__testing.buildPluginLoaderJitiOptions({}), tryNative: false, }); - // The production loader uses sync Jiti evaluation, so this regression test - // should exercise the same seam instead of Jiti's async import helper. - expect(() => withoutAlias(copiedChannelRuntime)).toThrow(); + // Node 24.13+ sealed `require` on ESM module objects, breaking Jiti's sync + // CJS path when tryNative is false. Use the async `.import()` helper instead. + await expect(async () => await withoutAlias.import(copiedChannelRuntime)).rejects.toThrow(); const withAlias = createJiti(jitiBaseUrl, { ...__testing.buildPluginLoaderJitiOptions({ @@ -87,7 +87,7 @@ export const copiedRuntimeMarker = { }), tryNative: false, }); - expect(withAlias(copiedChannelRuntime)).toMatchObject({ + expect(await withAlias.import(copiedChannelRuntime)).toMatchObject({ copiedRuntimeMarker: { PAIRING_APPROVED_MESSAGE: "paired", resolveOutboundSendDep: expect.any(Function), diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index a4bf12fad15..a2f93d2ad80 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -3595,9 +3595,9 @@ export const syntheticRuntimeMarker = { ...__testing.buildPluginLoaderJitiOptions({}), tryNative: false, }); - // The production loader uses sync Jiti evaluation, so this boundary should - // follow the same path instead of the async import helper. - expect(() => withoutAlias(copiedChannelRuntime)).toThrow(); + // Node 24.13+ sealed `require` on ESM module objects, breaking Jiti's sync + // CJS path when tryNative is false. Use the async `.import()` helper instead. + await expect(async () => await withoutAlias.import(copiedChannelRuntime)).rejects.toThrow(); const withAlias = createJiti(jitiBaseUrl, { ...__testing.buildPluginLoaderJitiOptions({ @@ -3605,7 +3605,7 @@ export const syntheticRuntimeMarker = { }), tryNative: false, }); - expect(withAlias(copiedChannelRuntime)).toMatchObject({ + expect(await withAlias.import(copiedChannelRuntime)).toMatchObject({ syntheticRuntimeMarker: { resolveOutboundSendDep: expect.any(Function), }, From 429364f01cdac31a6df6f0b67c1cab43acbdf48b Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 20 Mar 2026 10:30:21 -0300 Subject: [PATCH 3/3] =?UTF-8?q?revert(test):=20restore=20upstream=20jiti?= =?UTF-8?q?=20test=20=E2=80=94=20Node=2024.13=20breakage=20is=20upstream?= =?UTF-8?q?=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loader.git-path-regression test fails on Node 24.13+ due to require being sealed as a read-only getter on ESM module objects. Both sync and async Jiti paths are affected. This is an upstream issue that needs to be fixed in the test infrastructure, not in our feature branches. --- src/plugins/loader.git-path-regression.test.ts | 8 ++++---- src/plugins/loader.test.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/plugins/loader.git-path-regression.test.ts b/src/plugins/loader.git-path-regression.test.ts index 674f2fd53c0..23ab4f4243d 100644 --- a/src/plugins/loader.git-path-regression.test.ts +++ b/src/plugins/loader.git-path-regression.test.ts @@ -77,9 +77,9 @@ export const copiedRuntimeMarker = { ...__testing.buildPluginLoaderJitiOptions({}), tryNative: false, }); - // Node 24.13+ sealed `require` on ESM module objects, breaking Jiti's sync - // CJS path when tryNative is false. Use the async `.import()` helper instead. - await expect(async () => await withoutAlias.import(copiedChannelRuntime)).rejects.toThrow(); + // The production loader uses sync Jiti evaluation, so this regression test + // should exercise the same seam instead of Jiti's async import helper. + expect(() => withoutAlias(copiedChannelRuntime)).toThrow(); const withAlias = createJiti(jitiBaseUrl, { ...__testing.buildPluginLoaderJitiOptions({ @@ -87,7 +87,7 @@ export const copiedRuntimeMarker = { }), tryNative: false, }); - expect(await withAlias.import(copiedChannelRuntime)).toMatchObject({ + expect(withAlias(copiedChannelRuntime)).toMatchObject({ copiedRuntimeMarker: { PAIRING_APPROVED_MESSAGE: "paired", resolveOutboundSendDep: expect.any(Function), diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index a2f93d2ad80..a4bf12fad15 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -3595,9 +3595,9 @@ export const syntheticRuntimeMarker = { ...__testing.buildPluginLoaderJitiOptions({}), tryNative: false, }); - // Node 24.13+ sealed `require` on ESM module objects, breaking Jiti's sync - // CJS path when tryNative is false. Use the async `.import()` helper instead. - await expect(async () => await withoutAlias.import(copiedChannelRuntime)).rejects.toThrow(); + // The production loader uses sync Jiti evaluation, so this boundary should + // follow the same path instead of the async import helper. + expect(() => withoutAlias(copiedChannelRuntime)).toThrow(); const withAlias = createJiti(jitiBaseUrl, { ...__testing.buildPluginLoaderJitiOptions({ @@ -3605,7 +3605,7 @@ export const syntheticRuntimeMarker = { }), tryNative: false, }); - expect(await withAlias.import(copiedChannelRuntime)).toMatchObject({ + expect(withAlias(copiedChannelRuntime)).toMatchObject({ syntheticRuntimeMarker: { resolveOutboundSendDep: expect.any(Function), },