Merge 4db9a6d8d04b4c1858371cad3be8d277d944cf3f into d78e13f545136fcbba1feceecc5e0485a06c33a6
This commit is contained in:
Joseph Krug
GitHub
commit
8cc5545ef4
@ -361,7 +361,7 @@ function syncPostCompactionSessionMemory(params: {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
async function runPostCompactionSideEffects(params: {
|
||||
export async function runPostCompactionSideEffects(params: {
|
||||
config?: OpenClawConfig;
|
||||
sessionKey?: string;
|
||||
sessionFile: string;
|
||||
|
||||
@ -62,6 +62,7 @@ export const mockedContextEngine = {
|
||||
|
||||
export const mockedContextEngineCompact = mockedContextEngine.compact;
|
||||
export const mockedCompactDirect = mockedContextEngine.compact;
|
||||
export const mockedRunPostCompactionSideEffects = vi.fn(async () => {});
|
||||
export const mockedEnsureRuntimePluginsLoaded = vi.fn<(params?: unknown) => void>();
|
||||
export const mockedPrepareProviderRuntimeAuth = vi.fn(async () => undefined);
|
||||
export const mockedRunEmbeddedAttempt =
|
||||
@ -137,6 +138,15 @@ export const mockedIsLikelyContextOverflowError = vi.fn((msg?: string) => {
|
||||
export const mockedPickFallbackThinkingLevel = vi.fn<(params?: unknown) => ThinkLevel | null>(
|
||||
() => null,
|
||||
);
|
||||
export const mockedGetApiKeyForModel = vi.fn(
|
||||
async ({ profileId }: { profileId?: string } = {}) => ({
|
||||
apiKey: "test-key",
|
||||
profileId: profileId ?? "test-profile",
|
||||
source: "test",
|
||||
mode: "api-key" as const,
|
||||
}),
|
||||
);
|
||||
export const mockedResolveAuthProfileOrder = vi.fn(() => [] as string[]);
|
||||
|
||||
export const overflowBaseRunParams = {
|
||||
sessionId: "test-session",
|
||||
@ -226,6 +236,19 @@ export function resetRunOverflowCompactionHarnessMocks(): void {
|
||||
});
|
||||
mockedPickFallbackThinkingLevel.mockReset();
|
||||
mockedPickFallbackThinkingLevel.mockReturnValue(null);
|
||||
mockedGetApiKeyForModel.mockReset();
|
||||
mockedGetApiKeyForModel.mockImplementation(
|
||||
async ({ profileId }: { profileId?: string } = {}) => ({
|
||||
apiKey: "test-key",
|
||||
profileId: profileId ?? "test-profile",
|
||||
source: "test",
|
||||
mode: "api-key",
|
||||
}),
|
||||
);
|
||||
mockedResolveAuthProfileOrder.mockReset();
|
||||
mockedResolveAuthProfileOrder.mockReturnValue([]);
|
||||
mockedRunPostCompactionSideEffects.mockReset();
|
||||
mockedRunPostCompactionSideEffects.mockResolvedValue(undefined);
|
||||
}
|
||||
|
||||
export async function loadRunOverflowCompactionHarness(): Promise<{
|
||||
@ -329,12 +352,8 @@ export async function loadRunOverflowCompactionHarness(): Promise<{
|
||||
vi.doMock("../model-auth.js", () => ({
|
||||
applyLocalNoAuthHeaderOverride: vi.fn((model: unknown) => model),
|
||||
ensureAuthProfileStore: vi.fn(() => ({})),
|
||||
getApiKeyForModel: vi.fn(async () => ({
|
||||
apiKey: "test-key",
|
||||
profileId: "test-profile",
|
||||
source: "test",
|
||||
})),
|
||||
resolveAuthProfileOrder: vi.fn(() => []),
|
||||
getApiKeyForModel: mockedGetApiKeyForModel,
|
||||
resolveAuthProfileOrder: mockedResolveAuthProfileOrder,
|
||||
}));
|
||||
|
||||
vi.doMock("../models-config.js", () => ({
|
||||
@ -399,6 +418,10 @@ export async function loadRunOverflowCompactionHarness(): Promise<{
|
||||
sessionLikelyHasOversizedToolResults: mockedSessionLikelyHasOversizedToolResults,
|
||||
}));
|
||||
|
||||
vi.doMock("./compact.js", () => ({
|
||||
runPostCompactionSideEffects: mockedRunPostCompactionSideEffects,
|
||||
}));
|
||||
|
||||
vi.doMock("./utils.js", () => ({
|
||||
describeUnknownError: vi.fn((err: unknown) => {
|
||||
if (err instanceof Error) {
|
||||
|
||||
@ -46,6 +46,7 @@ describe("runEmbeddedPiAgent overflow compaction trigger routing", () => {
|
||||
mockedGlobalHookRunner.runBeforeAgentStart.mockReset();
|
||||
mockedGlobalHookRunner.runBeforeCompaction.mockReset();
|
||||
mockedGlobalHookRunner.runAfterCompaction.mockReset();
|
||||
mockedPickFallbackThinkingLevel.mockReset();
|
||||
mockedContextEngine.info.ownsCompaction = false;
|
||||
mockedCompactDirect.mockResolvedValue({
|
||||
ok: false,
|
||||
@ -66,6 +67,7 @@ describe("runEmbeddedPiAgent overflow compaction trigger routing", () => {
|
||||
truncatedCount: 0,
|
||||
reason: "no oversized tool results",
|
||||
});
|
||||
mockedPickFallbackThinkingLevel.mockReturnValue(undefined);
|
||||
mockedGlobalHookRunner.hasHooks.mockImplementation(() => false);
|
||||
});
|
||||
|
||||
|
||||
@ -0,0 +1,520 @@
|
||||
import { beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { makeAttemptResult, makeCompactionSuccess } from "./run.overflow-compaction.fixture.js";
|
||||
import {
|
||||
loadRunOverflowCompactionHarness,
|
||||
mockedCoerceToFailoverError,
|
||||
mockedDescribeFailoverError,
|
||||
mockedGlobalHookRunner,
|
||||
mockedResolveFailoverStatus,
|
||||
mockedCompactDirect,
|
||||
mockedContextEngine,
|
||||
mockedRunEmbeddedAttempt,
|
||||
mockedGetApiKeyForModel,
|
||||
mockedPickFallbackThinkingLevel,
|
||||
mockedResolveAuthProfileOrder,
|
||||
resetRunOverflowCompactionHarnessMocks,
|
||||
mockedSessionLikelyHasOversizedToolResults,
|
||||
mockedTruncateOversizedToolResultsInSession,
|
||||
mockedRunPostCompactionSideEffects,
|
||||
overflowBaseRunParams,
|
||||
} from "./run.overflow-compaction.harness.js";
|
||||
|
||||
let runEmbeddedPiAgent: typeof import("./run.js").runEmbeddedPiAgent;
|
||||
|
||||
const useTwoAuthProfiles = () => {
|
||||
mockedResolveAuthProfileOrder.mockReturnValue(["profile-a", "profile-b"]);
|
||||
mockedGetApiKeyForModel.mockImplementation(async ({ profileId } = {}) => ({
|
||||
apiKey: `test-key-${profileId ?? "profile-a"}`,
|
||||
profileId: profileId ?? "profile-a",
|
||||
source: "test",
|
||||
mode: "api-key",
|
||||
}));
|
||||
};
|
||||
|
||||
describe("timeout-triggered compaction", () => {
|
||||
beforeAll(async () => {
|
||||
({ runEmbeddedPiAgent } = await loadRunOverflowCompactionHarness());
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
resetRunOverflowCompactionHarnessMocks();
|
||||
mockedRunEmbeddedAttempt.mockReset();
|
||||
mockedCompactDirect.mockReset();
|
||||
mockedCoerceToFailoverError.mockReset();
|
||||
mockedDescribeFailoverError.mockReset();
|
||||
mockedResolveFailoverStatus.mockReset();
|
||||
mockedSessionLikelyHasOversizedToolResults.mockReset();
|
||||
mockedTruncateOversizedToolResultsInSession.mockReset();
|
||||
mockedGlobalHookRunner.runBeforeAgentStart.mockReset();
|
||||
mockedGlobalHookRunner.runBeforeCompaction.mockReset();
|
||||
mockedGlobalHookRunner.runAfterCompaction.mockReset();
|
||||
mockedPickFallbackThinkingLevel.mockReset();
|
||||
mockedRunPostCompactionSideEffects.mockReset();
|
||||
mockedRunPostCompactionSideEffects.mockResolvedValue(undefined);
|
||||
mockedContextEngine.info.ownsCompaction = false;
|
||||
mockedCompactDirect.mockResolvedValue({
|
||||
ok: false,
|
||||
compacted: false,
|
||||
reason: "nothing to compact",
|
||||
});
|
||||
mockedCoerceToFailoverError.mockReturnValue(null);
|
||||
mockedDescribeFailoverError.mockImplementation((err: unknown) => ({
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
reason: undefined,
|
||||
status: undefined,
|
||||
code: undefined,
|
||||
}));
|
||||
mockedSessionLikelyHasOversizedToolResults.mockReturnValue(false);
|
||||
mockedTruncateOversizedToolResultsInSession.mockResolvedValue({
|
||||
truncated: false,
|
||||
truncatedCount: 0,
|
||||
reason: "no oversized tool results",
|
||||
});
|
||||
mockedPickFallbackThinkingLevel.mockReturnValue(undefined);
|
||||
mockedGlobalHookRunner.hasHooks.mockImplementation(() => false);
|
||||
mockedGetApiKeyForModel.mockImplementation(async ({ profileId }) => ({
|
||||
apiKey: "test-key",
|
||||
profileId: profileId ?? "test-profile",
|
||||
source: "test",
|
||||
mode: "api-key",
|
||||
}));
|
||||
mockedResolveAuthProfileOrder.mockReturnValue([]);
|
||||
});
|
||||
|
||||
it("attempts compaction when LLM times out with high prompt token usage (>65%)", async () => {
|
||||
// First attempt: timeout with high prompt usage (150k / 200k = 75%)
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
// Compaction succeeds
|
||||
mockedCompactDirect.mockResolvedValueOnce(
|
||||
makeCompactionSuccess({
|
||||
summary: "timeout recovery compaction",
|
||||
tokensBefore: 150000,
|
||||
tokensAfter: 80000,
|
||||
}),
|
||||
);
|
||||
// Retry after compaction succeeds
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(makeAttemptResult({ promptError: null }));
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
expect(mockedCompactDirect).toHaveBeenCalledTimes(1);
|
||||
expect(mockedCompactDirect).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
sessionId: "test-session",
|
||||
sessionFile: "/tmp/session.json",
|
||||
tokenBudget: 200000,
|
||||
force: true,
|
||||
compactionTarget: "budget",
|
||||
runtimeContext: expect.objectContaining({
|
||||
trigger: "timeout_recovery",
|
||||
attempt: 1,
|
||||
maxAttempts: 2,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.meta.error).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retries the prompt after successful timeout compaction", async () => {
|
||||
// First attempt: timeout with high prompt usage
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 160000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
// Compaction succeeds
|
||||
mockedCompactDirect.mockResolvedValueOnce(
|
||||
makeCompactionSuccess({
|
||||
summary: "compacted for timeout",
|
||||
tokensBefore: 160000,
|
||||
tokensAfter: 60000,
|
||||
}),
|
||||
);
|
||||
// Second attempt succeeds
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(makeAttemptResult({ promptError: null }));
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
// Verify the loop continued (retry happened)
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
// Post-compaction side effects (transcript update, memory sync) should fire
|
||||
expect(mockedRunPostCompactionSideEffects).toHaveBeenCalledTimes(1);
|
||||
expect(mockedRunPostCompactionSideEffects).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
sessionFile: "/tmp/session.json",
|
||||
}),
|
||||
);
|
||||
expect(result.meta.error).toBeUndefined();
|
||||
});
|
||||
|
||||
it("falls through to normal handling when timeout compaction fails", async () => {
|
||||
// Timeout with high prompt usage
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
// Compaction does not reduce context
|
||||
mockedCompactDirect.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
compacted: false,
|
||||
reason: "nothing to compact",
|
||||
});
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
// Compaction was attempted but failed → falls through to timeout error payload
|
||||
expect(mockedCompactDirect).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
|
||||
it("does not attempt compaction when prompt token usage is low", async () => {
|
||||
// Timeout with low prompt usage (20k / 200k = 10%)
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 20000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
// No compaction attempt for low usage
|
||||
expect(mockedCompactDirect).not.toHaveBeenCalled();
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
|
||||
it("does not attempt compaction for low-context timeouts on later retries", async () => {
|
||||
mockedPickFallbackThinkingLevel.mockReturnValueOnce("low");
|
||||
mockedRunEmbeddedAttempt
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
promptError: new Error("unsupported reasoning mode"),
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 20000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(mockedCompactDirect).not.toHaveBeenCalled();
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
|
||||
it("still attempts compaction for timed-out attempts that set aborted", async () => {
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
aborted: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 180000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
mockedCompactDirect.mockResolvedValueOnce(
|
||||
makeCompactionSuccess({
|
||||
summary: "timeout recovery compaction",
|
||||
tokensBefore: 180000,
|
||||
tokensAfter: 90000,
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(makeAttemptResult({ promptError: null }));
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
expect(mockedCompactDirect).toHaveBeenCalledTimes(1);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.meta.error).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not attempt compaction when timedOutDuringCompaction is true", async () => {
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
timedOutDuringCompaction: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 180000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
// timedOutDuringCompaction skips timeout-triggered compaction
|
||||
expect(mockedCompactDirect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls through to failover rotation after max timeout compaction attempts", async () => {
|
||||
// First attempt: timeout with high prompt usage (150k / 200k = 75%)
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
// First compaction succeeds
|
||||
mockedCompactDirect.mockResolvedValueOnce(
|
||||
makeCompactionSuccess({
|
||||
summary: "timeout recovery compaction 1",
|
||||
tokensBefore: 150000,
|
||||
tokensAfter: 80000,
|
||||
}),
|
||||
);
|
||||
// Second attempt after compaction: also times out with high usage
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 140000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
// Second compaction also succeeds
|
||||
mockedCompactDirect.mockResolvedValueOnce(
|
||||
makeCompactionSuccess({
|
||||
summary: "timeout recovery compaction 2",
|
||||
tokensBefore: 140000,
|
||||
tokensAfter: 70000,
|
||||
}),
|
||||
);
|
||||
// Third attempt after second compaction: still times out
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 130000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
// Both compaction attempts used; third timeout falls through.
|
||||
expect(mockedCompactDirect).toHaveBeenCalledTimes(2);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3);
|
||||
// Falls through to timeout error payload (failover rotation path)
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
|
||||
it("catches thrown errors from contextEngine.compact during timeout recovery", async () => {
|
||||
// Timeout with high prompt usage
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
// Compaction throws
|
||||
mockedCompactDirect.mockRejectedValueOnce(new Error("engine crashed"));
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
// Should not crash — falls through to normal timeout handling
|
||||
expect(mockedCompactDirect).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
|
||||
it("fires compaction hooks during timeout recovery for ownsCompaction engines", async () => {
|
||||
mockedContextEngine.info.ownsCompaction = true;
|
||||
mockedGlobalHookRunner.hasHooks.mockImplementation(
|
||||
(hookName) => hookName === "before_compaction" || hookName === "after_compaction",
|
||||
);
|
||||
mockedRunEmbeddedAttempt
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 160000 },
|
||||
} as never,
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(makeAttemptResult({ promptError: null }));
|
||||
mockedCompactDirect.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
compacted: true,
|
||||
result: {
|
||||
summary: "engine-owned timeout compaction",
|
||||
tokensAfter: 70,
|
||||
},
|
||||
});
|
||||
|
||||
await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
expect(mockedGlobalHookRunner.runBeforeCompaction).toHaveBeenCalledWith(
|
||||
{ messageCount: -1, sessionFile: "/tmp/session.json" },
|
||||
expect.objectContaining({
|
||||
sessionKey: "test-key",
|
||||
}),
|
||||
);
|
||||
expect(mockedGlobalHookRunner.runAfterCompaction).toHaveBeenCalledWith(
|
||||
{
|
||||
messageCount: -1,
|
||||
compactedCount: -1,
|
||||
tokenCount: 70,
|
||||
sessionFile: "/tmp/session.json",
|
||||
},
|
||||
expect.objectContaining({
|
||||
sessionKey: "test-key",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("counts compacted:false timeout compactions against the retry cap across profile rotation", async () => {
|
||||
useTwoAuthProfiles();
|
||||
// Attempt 1 (profile-a): timeout → compaction #1 fails → rotate to profile-b
|
||||
mockedRunEmbeddedAttempt
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
aborted: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
)
|
||||
// Attempt 2 (profile-b): timeout → compaction #2 fails → cap exhausted → rotation
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
aborted: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
mockedCompactDirect
|
||||
.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
compacted: false,
|
||||
reason: "nothing to compact",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
compacted: false,
|
||||
reason: "nothing to compact",
|
||||
});
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
expect(mockedCompactDirect).toHaveBeenCalledTimes(2);
|
||||
expect(mockedCompactDirect).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
runtimeContext: expect.objectContaining({
|
||||
authProfileId: "profile-a",
|
||||
attempt: 1,
|
||||
maxAttempts: 2,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockedCompactDirect).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
runtimeContext: expect.objectContaining({
|
||||
authProfileId: "profile-b",
|
||||
attempt: 2,
|
||||
maxAttempts: 2,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
|
||||
it("counts thrown timeout compactions against the retry cap across profile rotation", async () => {
|
||||
useTwoAuthProfiles();
|
||||
// Attempt 1 (profile-a): timeout → compaction #1 throws → rotate to profile-b
|
||||
mockedRunEmbeddedAttempt
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
aborted: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
)
|
||||
// Attempt 2 (profile-b): timeout → compaction #2 throws → cap exhausted → rotation
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
aborted: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
mockedCompactDirect
|
||||
.mockRejectedValueOnce(new Error("engine crashed"))
|
||||
.mockRejectedValueOnce(new Error("engine crashed again"));
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
expect(mockedCompactDirect).toHaveBeenCalledTimes(2);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ authProfileId: "profile-a" }),
|
||||
);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ authProfileId: "profile-b" }),
|
||||
);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
|
||||
it("uses prompt/input tokens for ratio, not total tokens", async () => {
|
||||
// Timeout where total tokens are high (150k) but input/prompt tokens
|
||||
// are low (20k / 200k = 10%). Should NOT trigger compaction because
|
||||
// the ratio is based on prompt tokens, not total.
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
timedOut: true,
|
||||
lastAssistant: {
|
||||
usage: { input: 20000, total: 150000 },
|
||||
} as never,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedPiAgent(overflowBaseRunParams);
|
||||
|
||||
// Despite high total tokens, low prompt tokens mean no compaction
|
||||
expect(mockedCompactDirect).not.toHaveBeenCalled();
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("timed out");
|
||||
});
|
||||
});
|
||||
@ -65,6 +65,7 @@ import {
|
||||
import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js";
|
||||
import { derivePromptTokens, normalizeUsage, type UsageLike } from "../usage.js";
|
||||
import { redactRunIdentifier, resolveRunWorkspaceDir } from "../workspace-run.js";
|
||||
import { runPostCompactionSideEffects } from "./compact.js";
|
||||
import { buildEmbeddedCompactionRuntimeContext } from "./compaction-runtime-context.js";
|
||||
import { runContextEngineMaintenance } from "./context-engine-maintenance.js";
|
||||
import { resolveGlobalLane, resolveSessionLane } from "./lanes.js";
|
||||
@ -816,6 +817,7 @@ export async function runEmbeddedPiAgent(
|
||||
}
|
||||
};
|
||||
|
||||
const MAX_TIMEOUT_COMPACTION_ATTEMPTS = 2;
|
||||
const MAX_OVERFLOW_COMPACTION_ATTEMPTS = 3;
|
||||
const MAX_RUN_LOOP_ITERATIONS = resolveMaxRunRetryIterations(profileCandidates.length);
|
||||
let overflowCompactionAttempts = 0;
|
||||
@ -828,6 +830,7 @@ export async function runEmbeddedPiAgent(
|
||||
let autoCompactionCount = 0;
|
||||
let runLoopIterations = 0;
|
||||
let overloadFailoverAttempts = 0;
|
||||
let timeoutCompactionAttempts = 0;
|
||||
const maybeMarkAuthProfileFailure = async (failure: {
|
||||
profileId?: string;
|
||||
reason?: AuthProfileFailureReason | null;
|
||||
@ -882,6 +885,51 @@ export async function runEmbeddedPiAgent(
|
||||
ensureContextEnginesInitialized();
|
||||
const contextEngine = await resolveContextEngine(params.config);
|
||||
try {
|
||||
// When the engine owns compaction, compactEmbeddedPiSessionDirect is
|
||||
// bypassed. Fire lifecycle hooks here so recovery paths still notify
|
||||
// subscribers like memory extensions and usage trackers.
|
||||
const runOwnsCompactionBeforeHook = async (reason: string) => {
|
||||
if (
|
||||
contextEngine.info.ownsCompaction !== true ||
|
||||
!hookRunner?.hasHooks("before_compaction")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await hookRunner.runBeforeCompaction(
|
||||
{ messageCount: -1, sessionFile: params.sessionFile },
|
||||
hookCtx,
|
||||
);
|
||||
} catch (hookErr) {
|
||||
log.warn(`before_compaction hook failed during ${reason}: ${String(hookErr)}`);
|
||||
}
|
||||
};
|
||||
const runOwnsCompactionAfterHook = async (
|
||||
reason: string,
|
||||
compactResult: Awaited<ReturnType<typeof contextEngine.compact>>,
|
||||
) => {
|
||||
if (
|
||||
contextEngine.info.ownsCompaction !== true ||
|
||||
!compactResult.ok ||
|
||||
!compactResult.compacted ||
|
||||
!hookRunner?.hasHooks("after_compaction")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await hookRunner.runAfterCompaction(
|
||||
{
|
||||
messageCount: -1,
|
||||
compactedCount: -1,
|
||||
tokenCount: compactResult.result?.tokensAfter,
|
||||
sessionFile: params.sessionFile,
|
||||
},
|
||||
hookCtx,
|
||||
);
|
||||
} catch (hookErr) {
|
||||
log.warn(`after_compaction hook failed during ${reason}: ${String(hookErr)}`);
|
||||
}
|
||||
};
|
||||
let authRetryPending = false;
|
||||
// Hoisted so the retry-limit error path can use the most recent API total.
|
||||
let lastTurnTotal: number | undefined;
|
||||
@ -1048,6 +1096,90 @@ export async function runEmbeddedPiAgent(
|
||||
? lastAssistant.errorMessage?.trim() || formattedAssistantErrorText
|
||||
: undefined;
|
||||
|
||||
// ── Timeout-triggered compaction ──────────────────────────────────
|
||||
// When the LLM times out with high context usage, compact before
|
||||
// retrying to break the death spiral of repeated timeouts.
|
||||
if (timedOut && !timedOutDuringCompaction) {
|
||||
// Only consider prompt-side tokens here. API totals include output
|
||||
// tokens, which can make a long generation look like high context
|
||||
// pressure even when the prompt itself was small.
|
||||
const lastTurnPromptTokens = derivePromptTokens(lastRunPromptUsage);
|
||||
const tokenUsedRatio =
|
||||
lastTurnPromptTokens != null && ctxInfo.tokens > 0
|
||||
? lastTurnPromptTokens / ctxInfo.tokens
|
||||
: 0;
|
||||
if (timeoutCompactionAttempts >= MAX_TIMEOUT_COMPACTION_ATTEMPTS) {
|
||||
log.warn(
|
||||
`[timeout-compaction] already attempted timeout compaction ${timeoutCompactionAttempts} time(s); falling through to failover rotation`,
|
||||
);
|
||||
} else if (tokenUsedRatio > 0.65) {
|
||||
const timeoutDiagId = createCompactionDiagId();
|
||||
timeoutCompactionAttempts++;
|
||||
log.warn(
|
||||
`[timeout-compaction] LLM timed out with high prompt token usage (${Math.round(tokenUsedRatio * 100)}%); ` +
|
||||
`attempting compaction before retry (attempt ${timeoutCompactionAttempts}/${MAX_TIMEOUT_COMPACTION_ATTEMPTS}) diagId=${timeoutDiagId}`,
|
||||
);
|
||||
let timeoutCompactResult: Awaited<ReturnType<typeof contextEngine.compact>>;
|
||||
await runOwnsCompactionBeforeHook("timeout recovery");
|
||||
try {
|
||||
timeoutCompactResult = await contextEngine.compact({
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionFile: params.sessionFile,
|
||||
tokenBudget: ctxInfo.tokens,
|
||||
force: true,
|
||||
compactionTarget: "budget",
|
||||
runtimeContext: {
|
||||
sessionKey: params.sessionKey,
|
||||
messageChannel: params.messageChannel,
|
||||
messageProvider: params.messageProvider,
|
||||
agentAccountId: params.agentAccountId,
|
||||
authProfileId: lastProfileId,
|
||||
workspaceDir: resolvedWorkspace,
|
||||
agentDir,
|
||||
config: params.config,
|
||||
skillsSnapshot: params.skillsSnapshot,
|
||||
senderIsOwner: params.senderIsOwner,
|
||||
provider,
|
||||
model: modelId,
|
||||
runId: params.runId,
|
||||
thinkLevel,
|
||||
reasoningLevel: params.reasoningLevel,
|
||||
bashElevated: params.bashElevated,
|
||||
extraSystemPrompt: params.extraSystemPrompt,
|
||||
ownerNumbers: params.ownerNumbers,
|
||||
trigger: "timeout_recovery",
|
||||
diagId: timeoutDiagId,
|
||||
attempt: timeoutCompactionAttempts,
|
||||
maxAttempts: MAX_TIMEOUT_COMPACTION_ATTEMPTS,
|
||||
},
|
||||
});
|
||||
} catch (compactErr) {
|
||||
log.warn(
|
||||
`[timeout-compaction] contextEngine.compact() threw during timeout recovery for ${provider}/${modelId}: ${String(compactErr)}`,
|
||||
);
|
||||
timeoutCompactResult = { ok: false, compacted: false, reason: String(compactErr) };
|
||||
}
|
||||
await runOwnsCompactionAfterHook("timeout recovery", timeoutCompactResult);
|
||||
if (timeoutCompactResult.compacted) {
|
||||
autoCompactionCount += 1;
|
||||
await runPostCompactionSideEffects({
|
||||
config: params.config,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionFile: params.sessionFile,
|
||||
});
|
||||
log.info(
|
||||
`[timeout-compaction] compaction succeeded for ${provider}/${modelId}; retrying prompt`,
|
||||
);
|
||||
continue;
|
||||
} else {
|
||||
log.warn(
|
||||
`[timeout-compaction] compaction did not reduce context for ${provider}/${modelId}; falling through to normal handling`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const contextOverflowError = !aborted
|
||||
? (() => {
|
||||
if (promptError) {
|
||||
@ -1113,24 +1245,7 @@ export async function runEmbeddedPiAgent(
|
||||
`context overflow detected (attempt ${overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); attempting auto-compaction for ${provider}/${modelId}`,
|
||||
);
|
||||
let compactResult: Awaited<ReturnType<typeof contextEngine.compact>>;
|
||||
// When the engine owns compaction, hooks are not fired inside
|
||||
// compactEmbeddedPiSessionDirect (which is bypassed). Fire them
|
||||
// here so subscribers (memory extensions, usage trackers) are
|
||||
// notified even on overflow-recovery compactions.
|
||||
const overflowEngineOwnsCompaction = contextEngine.info.ownsCompaction === true;
|
||||
const overflowHookRunner = overflowEngineOwnsCompaction ? hookRunner : null;
|
||||
if (overflowHookRunner?.hasHooks("before_compaction")) {
|
||||
try {
|
||||
await overflowHookRunner.runBeforeCompaction(
|
||||
{ messageCount: -1, sessionFile: params.sessionFile },
|
||||
hookCtx,
|
||||
);
|
||||
} catch (hookErr) {
|
||||
log.warn(
|
||||
`before_compaction hook failed during overflow recovery: ${String(hookErr)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await runOwnsCompactionBeforeHook("overflow recovery");
|
||||
try {
|
||||
const overflowCompactionRuntimeContext = {
|
||||
...buildEmbeddedCompactionRuntimeContext({
|
||||
@ -1193,27 +1308,7 @@ export async function runEmbeddedPiAgent(
|
||||
);
|
||||
compactResult = { ok: false, compacted: false, reason: String(compactErr) };
|
||||
}
|
||||
if (
|
||||
compactResult.ok &&
|
||||
compactResult.compacted &&
|
||||
overflowHookRunner?.hasHooks("after_compaction")
|
||||
) {
|
||||
try {
|
||||
await overflowHookRunner.runAfterCompaction(
|
||||
{
|
||||
messageCount: -1,
|
||||
compactedCount: -1,
|
||||
tokenCount: compactResult.result?.tokensAfter,
|
||||
sessionFile: params.sessionFile,
|
||||
},
|
||||
hookCtx,
|
||||
);
|
||||
} catch (hookErr) {
|
||||
log.warn(
|
||||
`after_compaction hook failed during overflow recovery: ${String(hookErr)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await runOwnsCompactionAfterHook("overflow recovery", compactResult);
|
||||
if (compactResult.compacted) {
|
||||
autoCompactionCount += 1;
|
||||
log.info(`auto-compaction succeeded for ${provider}/${modelId}; retrying prompt`);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user