diff --git a/src/agents/pi-embedded-runner/compact.ts b/src/agents/pi-embedded-runner/compact.ts index dd5806421a0..8f7e2141e39 100644 --- a/src/agents/pi-embedded-runner/compact.ts +++ b/src/agents/pi-embedded-runner/compact.ts @@ -361,7 +361,7 @@ function syncPostCompactionSessionMemory(params: { return Promise.resolve(); } -async function runPostCompactionSideEffects(params: { +export async function runPostCompactionSideEffects(params: { config?: OpenClawConfig; sessionKey?: string; sessionFile: string; diff --git a/src/agents/pi-embedded-runner/run.overflow-compaction.harness.ts b/src/agents/pi-embedded-runner/run.overflow-compaction.harness.ts index 10c13dfe6fc..5c6f0394bf0 100644 --- a/src/agents/pi-embedded-runner/run.overflow-compaction.harness.ts +++ b/src/agents/pi-embedded-runner/run.overflow-compaction.harness.ts @@ -62,6 +62,7 @@ export const mockedContextEngine = { export const mockedContextEngineCompact = mockedContextEngine.compact; export const mockedCompactDirect = mockedContextEngine.compact; +export const mockedRunPostCompactionSideEffects = vi.fn(async () => {}); export const mockedEnsureRuntimePluginsLoaded = vi.fn<(params?: unknown) => void>(); export const mockedPrepareProviderRuntimeAuth = vi.fn(async () => undefined); export const mockedRunEmbeddedAttempt = @@ -137,6 +138,15 @@ export const mockedIsLikelyContextOverflowError = vi.fn((msg?: string) => { export const mockedPickFallbackThinkingLevel = vi.fn<(params?: unknown) => ThinkLevel | null>( () => null, ); +export const mockedGetApiKeyForModel = vi.fn( + async ({ profileId }: { profileId?: string } = {}) => ({ + apiKey: "test-key", + profileId: profileId ?? "test-profile", + source: "test", + mode: "api-key" as const, + }), +); +export const mockedResolveAuthProfileOrder = vi.fn(() => [] as string[]); export const overflowBaseRunParams = { sessionId: "test-session", @@ -226,6 +236,19 @@ export function resetRunOverflowCompactionHarnessMocks(): void { }); mockedPickFallbackThinkingLevel.mockReset(); mockedPickFallbackThinkingLevel.mockReturnValue(null); + mockedGetApiKeyForModel.mockReset(); + mockedGetApiKeyForModel.mockImplementation( + async ({ profileId }: { profileId?: string } = {}) => ({ + apiKey: "test-key", + profileId: profileId ?? "test-profile", + source: "test", + mode: "api-key", + }), + ); + mockedResolveAuthProfileOrder.mockReset(); + mockedResolveAuthProfileOrder.mockReturnValue([]); + mockedRunPostCompactionSideEffects.mockReset(); + mockedRunPostCompactionSideEffects.mockResolvedValue(undefined); } export async function loadRunOverflowCompactionHarness(): Promise<{ @@ -329,12 +352,8 @@ export async function loadRunOverflowCompactionHarness(): Promise<{ vi.doMock("../model-auth.js", () => ({ applyLocalNoAuthHeaderOverride: vi.fn((model: unknown) => model), ensureAuthProfileStore: vi.fn(() => ({})), - getApiKeyForModel: vi.fn(async () => ({ - apiKey: "test-key", - profileId: "test-profile", - source: "test", - })), - resolveAuthProfileOrder: vi.fn(() => []), + getApiKeyForModel: mockedGetApiKeyForModel, + resolveAuthProfileOrder: mockedResolveAuthProfileOrder, })); vi.doMock("../models-config.js", () => ({ @@ -399,6 +418,10 @@ export async function loadRunOverflowCompactionHarness(): Promise<{ sessionLikelyHasOversizedToolResults: mockedSessionLikelyHasOversizedToolResults, })); + vi.doMock("./compact.js", () => ({ + runPostCompactionSideEffects: mockedRunPostCompactionSideEffects, + })); + vi.doMock("./utils.js", () => ({ describeUnknownError: vi.fn((err: unknown) => { if (err instanceof Error) { diff --git a/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts b/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts index 56b4fbf0186..17d1c7b9c25 100644 --- a/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts +++ b/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts @@ -46,6 +46,7 @@ describe("runEmbeddedPiAgent overflow compaction trigger routing", () => { mockedGlobalHookRunner.runBeforeAgentStart.mockReset(); mockedGlobalHookRunner.runBeforeCompaction.mockReset(); mockedGlobalHookRunner.runAfterCompaction.mockReset(); + mockedPickFallbackThinkingLevel.mockReset(); mockedContextEngine.info.ownsCompaction = false; mockedCompactDirect.mockResolvedValue({ ok: false, @@ -66,6 +67,7 @@ describe("runEmbeddedPiAgent overflow compaction trigger routing", () => { truncatedCount: 0, reason: "no oversized tool results", }); + mockedPickFallbackThinkingLevel.mockReturnValue(undefined); mockedGlobalHookRunner.hasHooks.mockImplementation(() => false); }); diff --git a/src/agents/pi-embedded-runner/run.timeout-triggered-compaction.test.ts b/src/agents/pi-embedded-runner/run.timeout-triggered-compaction.test.ts new file mode 100644 index 00000000000..d74b689e695 --- /dev/null +++ b/src/agents/pi-embedded-runner/run.timeout-triggered-compaction.test.ts @@ -0,0 +1,520 @@ +import { beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { makeAttemptResult, makeCompactionSuccess } from "./run.overflow-compaction.fixture.js"; +import { + loadRunOverflowCompactionHarness, + mockedCoerceToFailoverError, + mockedDescribeFailoverError, + mockedGlobalHookRunner, + mockedResolveFailoverStatus, + mockedCompactDirect, + mockedContextEngine, + mockedRunEmbeddedAttempt, + mockedGetApiKeyForModel, + mockedPickFallbackThinkingLevel, + mockedResolveAuthProfileOrder, + resetRunOverflowCompactionHarnessMocks, + mockedSessionLikelyHasOversizedToolResults, + mockedTruncateOversizedToolResultsInSession, + mockedRunPostCompactionSideEffects, + overflowBaseRunParams, +} from "./run.overflow-compaction.harness.js"; + +let runEmbeddedPiAgent: typeof import("./run.js").runEmbeddedPiAgent; + +const useTwoAuthProfiles = () => { + mockedResolveAuthProfileOrder.mockReturnValue(["profile-a", "profile-b"]); + mockedGetApiKeyForModel.mockImplementation(async ({ profileId } = {}) => ({ + apiKey: `test-key-${profileId ?? "profile-a"}`, + profileId: profileId ?? "profile-a", + source: "test", + mode: "api-key", + })); +}; + +describe("timeout-triggered compaction", () => { + beforeAll(async () => { + ({ runEmbeddedPiAgent } = await loadRunOverflowCompactionHarness()); + }); + + beforeEach(() => { + resetRunOverflowCompactionHarnessMocks(); + mockedRunEmbeddedAttempt.mockReset(); + mockedCompactDirect.mockReset(); + mockedCoerceToFailoverError.mockReset(); + mockedDescribeFailoverError.mockReset(); + mockedResolveFailoverStatus.mockReset(); + mockedSessionLikelyHasOversizedToolResults.mockReset(); + mockedTruncateOversizedToolResultsInSession.mockReset(); + mockedGlobalHookRunner.runBeforeAgentStart.mockReset(); + mockedGlobalHookRunner.runBeforeCompaction.mockReset(); + mockedGlobalHookRunner.runAfterCompaction.mockReset(); + mockedPickFallbackThinkingLevel.mockReset(); + mockedRunPostCompactionSideEffects.mockReset(); + mockedRunPostCompactionSideEffects.mockResolvedValue(undefined); + mockedContextEngine.info.ownsCompaction = false; + mockedCompactDirect.mockResolvedValue({ + ok: false, + compacted: false, + reason: "nothing to compact", + }); + mockedCoerceToFailoverError.mockReturnValue(null); + mockedDescribeFailoverError.mockImplementation((err: unknown) => ({ + message: err instanceof Error ? err.message : String(err), + reason: undefined, + status: undefined, + code: undefined, + })); + mockedSessionLikelyHasOversizedToolResults.mockReturnValue(false); + mockedTruncateOversizedToolResultsInSession.mockResolvedValue({ + truncated: false, + truncatedCount: 0, + reason: "no oversized tool results", + }); + mockedPickFallbackThinkingLevel.mockReturnValue(undefined); + mockedGlobalHookRunner.hasHooks.mockImplementation(() => false); + mockedGetApiKeyForModel.mockImplementation(async ({ profileId }) => ({ + apiKey: "test-key", + profileId: profileId ?? "test-profile", + source: "test", + mode: "api-key", + })); + mockedResolveAuthProfileOrder.mockReturnValue([]); + }); + + it("attempts compaction when LLM times out with high prompt token usage (>65%)", async () => { + // First attempt: timeout with high prompt usage (150k / 200k = 75%) + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ); + // Compaction succeeds + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "timeout recovery compaction", + tokensBefore: 150000, + tokensAfter: 80000, + }), + ); + // Retry after compaction succeeds + mockedRunEmbeddedAttempt.mockResolvedValueOnce(makeAttemptResult({ promptError: null })); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + expect(mockedCompactDirect).toHaveBeenCalledTimes(1); + expect(mockedCompactDirect).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: "test-session", + sessionFile: "/tmp/session.json", + tokenBudget: 200000, + force: true, + compactionTarget: "budget", + runtimeContext: expect.objectContaining({ + trigger: "timeout_recovery", + attempt: 1, + maxAttempts: 2, + }), + }), + ); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.meta.error).toBeUndefined(); + }); + + it("retries the prompt after successful timeout compaction", async () => { + // First attempt: timeout with high prompt usage + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 160000 }, + } as never, + }), + ); + // Compaction succeeds + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "compacted for timeout", + tokensBefore: 160000, + tokensAfter: 60000, + }), + ); + // Second attempt succeeds + mockedRunEmbeddedAttempt.mockResolvedValueOnce(makeAttemptResult({ promptError: null })); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + // Verify the loop continued (retry happened) + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + // Post-compaction side effects (transcript update, memory sync) should fire + expect(mockedRunPostCompactionSideEffects).toHaveBeenCalledTimes(1); + expect(mockedRunPostCompactionSideEffects).toHaveBeenCalledWith( + expect.objectContaining({ + sessionFile: "/tmp/session.json", + }), + ); + expect(result.meta.error).toBeUndefined(); + }); + + it("falls through to normal handling when timeout compaction fails", async () => { + // Timeout with high prompt usage + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ); + // Compaction does not reduce context + mockedCompactDirect.mockResolvedValueOnce({ + ok: false, + compacted: false, + reason: "nothing to compact", + }); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + // Compaction was attempted but failed → falls through to timeout error payload + expect(mockedCompactDirect).toHaveBeenCalledTimes(1); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); + + it("does not attempt compaction when prompt token usage is low", async () => { + // Timeout with low prompt usage (20k / 200k = 10%) + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 20000 }, + } as never, + }), + ); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + // No compaction attempt for low usage + expect(mockedCompactDirect).not.toHaveBeenCalled(); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); + + it("does not attempt compaction for low-context timeouts on later retries", async () => { + mockedPickFallbackThinkingLevel.mockReturnValueOnce("low"); + mockedRunEmbeddedAttempt + .mockResolvedValueOnce( + makeAttemptResult({ + promptError: new Error("unsupported reasoning mode"), + }), + ) + .mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 20000 }, + } as never, + }), + ); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(mockedCompactDirect).not.toHaveBeenCalled(); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); + + it("still attempts compaction for timed-out attempts that set aborted", async () => { + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + aborted: true, + lastAssistant: { + usage: { input: 180000 }, + } as never, + }), + ); + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "timeout recovery compaction", + tokensBefore: 180000, + tokensAfter: 90000, + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce(makeAttemptResult({ promptError: null })); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + expect(mockedCompactDirect).toHaveBeenCalledTimes(1); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.meta.error).toBeUndefined(); + }); + + it("does not attempt compaction when timedOutDuringCompaction is true", async () => { + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + timedOutDuringCompaction: true, + lastAssistant: { + usage: { input: 180000 }, + } as never, + }), + ); + + await runEmbeddedPiAgent(overflowBaseRunParams); + + // timedOutDuringCompaction skips timeout-triggered compaction + expect(mockedCompactDirect).not.toHaveBeenCalled(); + }); + + it("falls through to failover rotation after max timeout compaction attempts", async () => { + // First attempt: timeout with high prompt usage (150k / 200k = 75%) + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ); + // First compaction succeeds + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "timeout recovery compaction 1", + tokensBefore: 150000, + tokensAfter: 80000, + }), + ); + // Second attempt after compaction: also times out with high usage + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 140000 }, + } as never, + }), + ); + // Second compaction also succeeds + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "timeout recovery compaction 2", + tokensBefore: 140000, + tokensAfter: 70000, + }), + ); + // Third attempt after second compaction: still times out + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 130000 }, + } as never, + }), + ); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + // Both compaction attempts used; third timeout falls through. + expect(mockedCompactDirect).toHaveBeenCalledTimes(2); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3); + // Falls through to timeout error payload (failover rotation path) + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); + + it("catches thrown errors from contextEngine.compact during timeout recovery", async () => { + // Timeout with high prompt usage + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ); + // Compaction throws + mockedCompactDirect.mockRejectedValueOnce(new Error("engine crashed")); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + // Should not crash — falls through to normal timeout handling + expect(mockedCompactDirect).toHaveBeenCalledTimes(1); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); + + it("fires compaction hooks during timeout recovery for ownsCompaction engines", async () => { + mockedContextEngine.info.ownsCompaction = true; + mockedGlobalHookRunner.hasHooks.mockImplementation( + (hookName) => hookName === "before_compaction" || hookName === "after_compaction", + ); + mockedRunEmbeddedAttempt + .mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 160000 }, + } as never, + }), + ) + .mockResolvedValueOnce(makeAttemptResult({ promptError: null })); + mockedCompactDirect.mockResolvedValueOnce({ + ok: true, + compacted: true, + result: { + summary: "engine-owned timeout compaction", + tokensAfter: 70, + }, + }); + + await runEmbeddedPiAgent(overflowBaseRunParams); + + expect(mockedGlobalHookRunner.runBeforeCompaction).toHaveBeenCalledWith( + { messageCount: -1, sessionFile: "/tmp/session.json" }, + expect.objectContaining({ + sessionKey: "test-key", + }), + ); + expect(mockedGlobalHookRunner.runAfterCompaction).toHaveBeenCalledWith( + { + messageCount: -1, + compactedCount: -1, + tokenCount: 70, + sessionFile: "/tmp/session.json", + }, + expect.objectContaining({ + sessionKey: "test-key", + }), + ); + }); + + it("counts compacted:false timeout compactions against the retry cap across profile rotation", async () => { + useTwoAuthProfiles(); + // Attempt 1 (profile-a): timeout → compaction #1 fails → rotate to profile-b + mockedRunEmbeddedAttempt + .mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + aborted: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ) + // Attempt 2 (profile-b): timeout → compaction #2 fails → cap exhausted → rotation + .mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + aborted: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ); + mockedCompactDirect + .mockResolvedValueOnce({ + ok: false, + compacted: false, + reason: "nothing to compact", + }) + .mockResolvedValueOnce({ + ok: false, + compacted: false, + reason: "nothing to compact", + }); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + expect(mockedCompactDirect).toHaveBeenCalledTimes(2); + expect(mockedCompactDirect).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + runtimeContext: expect.objectContaining({ + authProfileId: "profile-a", + attempt: 1, + maxAttempts: 2, + }), + }), + ); + expect(mockedCompactDirect).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + runtimeContext: expect.objectContaining({ + authProfileId: "profile-b", + attempt: 2, + maxAttempts: 2, + }), + }), + ); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); + + it("counts thrown timeout compactions against the retry cap across profile rotation", async () => { + useTwoAuthProfiles(); + // Attempt 1 (profile-a): timeout → compaction #1 throws → rotate to profile-b + mockedRunEmbeddedAttempt + .mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + aborted: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ) + // Attempt 2 (profile-b): timeout → compaction #2 throws → cap exhausted → rotation + .mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + aborted: true, + lastAssistant: { + usage: { input: 150000 }, + } as never, + }), + ); + mockedCompactDirect + .mockRejectedValueOnce(new Error("engine crashed")) + .mockRejectedValueOnce(new Error("engine crashed again")); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + expect(mockedCompactDirect).toHaveBeenCalledTimes(2); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(mockedRunEmbeddedAttempt).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ authProfileId: "profile-a" }), + ); + expect(mockedRunEmbeddedAttempt).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ authProfileId: "profile-b" }), + ); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); + + it("uses prompt/input tokens for ratio, not total tokens", async () => { + // Timeout where total tokens are high (150k) but input/prompt tokens + // are low (20k / 200k = 10%). Should NOT trigger compaction because + // the ratio is based on prompt tokens, not total. + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + timedOut: true, + lastAssistant: { + usage: { input: 20000, total: 150000 }, + } as never, + }), + ); + + const result = await runEmbeddedPiAgent(overflowBaseRunParams); + + // Despite high total tokens, low prompt tokens mean no compaction + expect(mockedCompactDirect).not.toHaveBeenCalled(); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("timed out"); + }); +}); diff --git a/src/agents/pi-embedded-runner/run.ts b/src/agents/pi-embedded-runner/run.ts index 0c66203992f..29cfdbaaf14 100644 --- a/src/agents/pi-embedded-runner/run.ts +++ b/src/agents/pi-embedded-runner/run.ts @@ -65,6 +65,7 @@ import { import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js"; import { derivePromptTokens, normalizeUsage, type UsageLike } from "../usage.js"; import { redactRunIdentifier, resolveRunWorkspaceDir } from "../workspace-run.js"; +import { runPostCompactionSideEffects } from "./compact.js"; import { buildEmbeddedCompactionRuntimeContext } from "./compaction-runtime-context.js"; import { runContextEngineMaintenance } from "./context-engine-maintenance.js"; import { resolveGlobalLane, resolveSessionLane } from "./lanes.js"; @@ -816,6 +817,7 @@ export async function runEmbeddedPiAgent( } }; + const MAX_TIMEOUT_COMPACTION_ATTEMPTS = 2; const MAX_OVERFLOW_COMPACTION_ATTEMPTS = 3; const MAX_RUN_LOOP_ITERATIONS = resolveMaxRunRetryIterations(profileCandidates.length); let overflowCompactionAttempts = 0; @@ -828,6 +830,7 @@ export async function runEmbeddedPiAgent( let autoCompactionCount = 0; let runLoopIterations = 0; let overloadFailoverAttempts = 0; + let timeoutCompactionAttempts = 0; const maybeMarkAuthProfileFailure = async (failure: { profileId?: string; reason?: AuthProfileFailureReason | null; @@ -882,6 +885,51 @@ export async function runEmbeddedPiAgent( ensureContextEnginesInitialized(); const contextEngine = await resolveContextEngine(params.config); try { + // When the engine owns compaction, compactEmbeddedPiSessionDirect is + // bypassed. Fire lifecycle hooks here so recovery paths still notify + // subscribers like memory extensions and usage trackers. + const runOwnsCompactionBeforeHook = async (reason: string) => { + if ( + contextEngine.info.ownsCompaction !== true || + !hookRunner?.hasHooks("before_compaction") + ) { + return; + } + try { + await hookRunner.runBeforeCompaction( + { messageCount: -1, sessionFile: params.sessionFile }, + hookCtx, + ); + } catch (hookErr) { + log.warn(`before_compaction hook failed during ${reason}: ${String(hookErr)}`); + } + }; + const runOwnsCompactionAfterHook = async ( + reason: string, + compactResult: Awaited>, + ) => { + if ( + contextEngine.info.ownsCompaction !== true || + !compactResult.ok || + !compactResult.compacted || + !hookRunner?.hasHooks("after_compaction") + ) { + return; + } + try { + await hookRunner.runAfterCompaction( + { + messageCount: -1, + compactedCount: -1, + tokenCount: compactResult.result?.tokensAfter, + sessionFile: params.sessionFile, + }, + hookCtx, + ); + } catch (hookErr) { + log.warn(`after_compaction hook failed during ${reason}: ${String(hookErr)}`); + } + }; let authRetryPending = false; // Hoisted so the retry-limit error path can use the most recent API total. let lastTurnTotal: number | undefined; @@ -1048,6 +1096,90 @@ export async function runEmbeddedPiAgent( ? lastAssistant.errorMessage?.trim() || formattedAssistantErrorText : undefined; + // ── Timeout-triggered compaction ────────────────────────────────── + // When the LLM times out with high context usage, compact before + // retrying to break the death spiral of repeated timeouts. + if (timedOut && !timedOutDuringCompaction) { + // Only consider prompt-side tokens here. API totals include output + // tokens, which can make a long generation look like high context + // pressure even when the prompt itself was small. + const lastTurnPromptTokens = derivePromptTokens(lastRunPromptUsage); + const tokenUsedRatio = + lastTurnPromptTokens != null && ctxInfo.tokens > 0 + ? lastTurnPromptTokens / ctxInfo.tokens + : 0; + if (timeoutCompactionAttempts >= MAX_TIMEOUT_COMPACTION_ATTEMPTS) { + log.warn( + `[timeout-compaction] already attempted timeout compaction ${timeoutCompactionAttempts} time(s); falling through to failover rotation`, + ); + } else if (tokenUsedRatio > 0.65) { + const timeoutDiagId = createCompactionDiagId(); + timeoutCompactionAttempts++; + log.warn( + `[timeout-compaction] LLM timed out with high prompt token usage (${Math.round(tokenUsedRatio * 100)}%); ` + + `attempting compaction before retry (attempt ${timeoutCompactionAttempts}/${MAX_TIMEOUT_COMPACTION_ATTEMPTS}) diagId=${timeoutDiagId}`, + ); + let timeoutCompactResult: Awaited>; + await runOwnsCompactionBeforeHook("timeout recovery"); + try { + timeoutCompactResult = await contextEngine.compact({ + sessionId: params.sessionId, + sessionKey: params.sessionKey, + sessionFile: params.sessionFile, + tokenBudget: ctxInfo.tokens, + force: true, + compactionTarget: "budget", + runtimeContext: { + sessionKey: params.sessionKey, + messageChannel: params.messageChannel, + messageProvider: params.messageProvider, + agentAccountId: params.agentAccountId, + authProfileId: lastProfileId, + workspaceDir: resolvedWorkspace, + agentDir, + config: params.config, + skillsSnapshot: params.skillsSnapshot, + senderIsOwner: params.senderIsOwner, + provider, + model: modelId, + runId: params.runId, + thinkLevel, + reasoningLevel: params.reasoningLevel, + bashElevated: params.bashElevated, + extraSystemPrompt: params.extraSystemPrompt, + ownerNumbers: params.ownerNumbers, + trigger: "timeout_recovery", + diagId: timeoutDiagId, + attempt: timeoutCompactionAttempts, + maxAttempts: MAX_TIMEOUT_COMPACTION_ATTEMPTS, + }, + }); + } catch (compactErr) { + log.warn( + `[timeout-compaction] contextEngine.compact() threw during timeout recovery for ${provider}/${modelId}: ${String(compactErr)}`, + ); + timeoutCompactResult = { ok: false, compacted: false, reason: String(compactErr) }; + } + await runOwnsCompactionAfterHook("timeout recovery", timeoutCompactResult); + if (timeoutCompactResult.compacted) { + autoCompactionCount += 1; + await runPostCompactionSideEffects({ + config: params.config, + sessionKey: params.sessionKey, + sessionFile: params.sessionFile, + }); + log.info( + `[timeout-compaction] compaction succeeded for ${provider}/${modelId}; retrying prompt`, + ); + continue; + } else { + log.warn( + `[timeout-compaction] compaction did not reduce context for ${provider}/${modelId}; falling through to normal handling`, + ); + } + } + } + const contextOverflowError = !aborted ? (() => { if (promptError) { @@ -1113,24 +1245,7 @@ export async function runEmbeddedPiAgent( `context overflow detected (attempt ${overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); attempting auto-compaction for ${provider}/${modelId}`, ); let compactResult: Awaited>; - // When the engine owns compaction, hooks are not fired inside - // compactEmbeddedPiSessionDirect (which is bypassed). Fire them - // here so subscribers (memory extensions, usage trackers) are - // notified even on overflow-recovery compactions. - const overflowEngineOwnsCompaction = contextEngine.info.ownsCompaction === true; - const overflowHookRunner = overflowEngineOwnsCompaction ? hookRunner : null; - if (overflowHookRunner?.hasHooks("before_compaction")) { - try { - await overflowHookRunner.runBeforeCompaction( - { messageCount: -1, sessionFile: params.sessionFile }, - hookCtx, - ); - } catch (hookErr) { - log.warn( - `before_compaction hook failed during overflow recovery: ${String(hookErr)}`, - ); - } - } + await runOwnsCompactionBeforeHook("overflow recovery"); try { const overflowCompactionRuntimeContext = { ...buildEmbeddedCompactionRuntimeContext({ @@ -1193,27 +1308,7 @@ export async function runEmbeddedPiAgent( ); compactResult = { ok: false, compacted: false, reason: String(compactErr) }; } - if ( - compactResult.ok && - compactResult.compacted && - overflowHookRunner?.hasHooks("after_compaction") - ) { - try { - await overflowHookRunner.runAfterCompaction( - { - messageCount: -1, - compactedCount: -1, - tokenCount: compactResult.result?.tokensAfter, - sessionFile: params.sessionFile, - }, - hookCtx, - ); - } catch (hookErr) { - log.warn( - `after_compaction hook failed during overflow recovery: ${String(hookErr)}`, - ); - } - } + await runOwnsCompactionAfterHook("overflow recovery", compactResult); if (compactResult.compacted) { autoCompactionCount += 1; log.info(`auto-compaction succeeded for ${provider}/${modelId}; retrying prompt`);