From 098d276fe02062fb3ecff9c6090cf7d7e0452846 Mon Sep 17 00:00:00 2001 From: Stephen Schoettler Date: Sun, 1 Mar 2026 18:42:02 -0800 Subject: [PATCH] fix(delivery-queue): increment retryCount on deferred entries when time budget exceeded When delivery recovery ran out of the 60s time budget, remaining pending entries were silently deferred to the next restart with no retryCount increment. This caused them to loop forever across restarts, never hitting MAX_RETRIES and never moving to failed/. Fix: call failDelivery() on each remaining entry before breaking out of the recovery loop (both the deadline check and the backoff-exceeds-deadline check). This increments retryCount so that entries eventually exhaust MAX_RETRIES and are permanently skipped. Fixes #24353 --- src/infra/outbound/delivery-queue.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/infra/outbound/delivery-queue.ts b/src/infra/outbound/delivery-queue.ts index 1e954ea8e39..9b09aef4550 100644 --- a/src/infra/outbound/delivery-queue.ts +++ b/src/infra/outbound/delivery-queue.ts @@ -303,8 +303,19 @@ export async function recoverPendingDeliveries(opts: { for (const entry of pending) { const now = Date.now(); if (now >= deadline) { - const deferred = pending.length - recovered - failed - skippedMaxRetries - deferredBackoff; - opts.log.warn(`Recovery time budget exceeded — ${deferred} entries deferred to next restart`); + // Increment retryCount on remaining entries so they eventually hit MAX_RETRIES + const remaining = pending.slice(pending.indexOf(entry)); + for (const r of remaining) { + try { + await failDelivery(r.id, "Recovery time budget exceeded — deferred", opts.stateDir); + } catch { + /* best-effort */ + } + } + const deferred = remaining.length; + opts.log.warn( + `Recovery time budget exceeded — ${deferred} entries deferred (retryCount incremented)`, + ); break; } if (entry.retryCount >= MAX_RETRIES) {