From 53aca43ee79278388c794ddef5ddb6049906b8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A8=81=E5=93=A5Wego?= Date: Wed, 18 Mar 2026 08:22:02 +0800 Subject: [PATCH 1/7] gateway: tail-read session transcripts to prevent chat.history freezes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readSessionMessages() previously called fs.readFileSync to load the entire JSONL transcript synchronously, then split and JSON.parse every line. On large sessions — or when a single record is unusually large — this blocked the Node.js event loop long enough to freeze the Gateway WebSocket and make the Web UI / WebChat appear hung. Root cause confirmed via a 447 KB single-line JSONL record that reproduced the hang reliably; moving the file aside restored UI responsiveness immediately. Fix: two-layer guard in readSessionMessages(): 1. Tail-read: files larger than 2 MB are read from the trailing 2 MB only (partial first line is discarded). This covers the common case where a session grows gradually over many turns. 2. Per-line cap: lines longer than 200 KB are skipped without parsing. A normal assistant reply is well under 50 KB; anything beyond 200 KB is a runaway prompt or model output that would stall JSON.parse and bloat the UI. The 200 KB threshold catches the confirmed 447 KB case that was too small to trigger the previous 1 MB guard. readSessionTitleFieldsFromTranscript() already uses head/tail chunk reads and is unaffected. --- src/gateway/session-utils.fs.ts | 45 ++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index 53be7392d10..f2a1833c99c 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -82,14 +82,53 @@ export function readSessionMessages( return []; } - const lines = fs.readFileSync(filePath, "utf-8").split(/\r?\n/); + // NOTE: This is on the Gateway hot path (chat.history). Reading + splitting an entire transcript + // file can freeze the UI when a session grows large (or when a single JSONL record is huge). + // We therefore tail-read large files and apply a per-line size guard. + const MAX_TAIL_BYTES = 2 * 1024 * 1024; // 2MB tail is plenty for the last ~100-1000 messages + // 200KB per line: a normal assistant reply is well under 50KB. Anything larger is a runaway + // prompt/response that would only stall JSON.parse and bloat the UI — skip it entirely. + // (The confirmed 447KB line causing Gateway freezes is caught by this threshold.) + const MAX_LINE_CHARS = 200 * 1024; + + let content = ""; + try { + const stat = fs.statSync(filePath); + if (stat.size > MAX_TAIL_BYTES) { + const fd = fs.openSync(filePath, "r"); + try { + const start = Math.max(0, stat.size - MAX_TAIL_BYTES); + const buf = Buffer.allocUnsafe(stat.size - start); + fs.readSync(fd, buf, 0, buf.length, start); + content = buf.toString("utf-8"); + // If we started mid-line, drop the partial first line. + const firstNewline = content.indexOf("\n"); + if (firstNewline >= 0 && start > 0) { + content = content.slice(firstNewline + 1); + } + } finally { + fs.closeSync(fd); + } + } else { + content = fs.readFileSync(filePath, "utf-8"); + } + } catch { + return []; + } + + const lines = content.split(/\r?\n/); const messages: unknown[] = []; for (const line of lines) { - if (!line.trim()) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + if (trimmed.length > MAX_LINE_CHARS) { + // Skip lines that are too large to safely parse on the RPC path. continue; } try { - const parsed = JSON.parse(line); + const parsed = JSON.parse(trimmed); if (parsed?.message) { messages.push(parsed.message); continue; From 61714bc92063193992d5a2b88508497a31c81203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A8=81=E5=93=A5Wego?= Date: Wed, 18 Mar 2026 10:30:46 +0800 Subject: [PATCH 2/7] gateway: fix readSync TOCTOU race; warn on oversized-line skip --- src/gateway/session-utils.fs.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index f2a1833c99c..c086267178a 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -99,9 +99,14 @@ export function readSessionMessages( try { const start = Math.max(0, stat.size - MAX_TAIL_BYTES); const buf = Buffer.allocUnsafe(stat.size - start); - fs.readSync(fd, buf, 0, buf.length, start); - content = buf.toString("utf-8"); + // Capture bytesRead: if the file shrank between statSync and readSync (TOCTOU), + // readSync returns fewer bytes than buf.length — slice to avoid feeding + // uninitialized memory into the UTF-8 / JSON pipeline. + const bytesRead = fs.readSync(fd, buf, 0, buf.length, start); + content = buf.toString("utf-8", 0, bytesRead); // If we started mid-line, drop the partial first line. + // Note: messages before the 2 MB boundary are intentionally omitted to keep + // this RPC fast; the UI will show the most recent history only. const firstNewline = content.indexOf("\n"); if (firstNewline >= 0 && start > 0) { content = content.slice(firstNewline + 1); @@ -125,6 +130,9 @@ export function readSessionMessages( } if (trimmed.length > MAX_LINE_CHARS) { // Skip lines that are too large to safely parse on the RPC path. + console.warn( + `[session-utils] skipping oversized line in session ${sessionId}: ${trimmed.length} chars (max ${MAX_LINE_CHARS})`, + ); continue; } try { From fe3d456edf319e6b4b667a0e962b307619b12c04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A8=81=E5=93=A5Wego?= Date: Wed, 18 Mar 2026 10:56:38 +0800 Subject: [PATCH 3/7] gateway: raise MAX_TAIL_BYTES to 18 MB (3x chat.history budget) --- src/gateway/session-utils.fs.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index c086267178a..6d44c5f7fe6 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -85,7 +85,12 @@ export function readSessionMessages( // NOTE: This is on the Gateway hot path (chat.history). Reading + splitting an entire transcript // file can freeze the UI when a session grows large (or when a single JSONL record is huge). // We therefore tail-read large files and apply a per-line size guard. - const MAX_TAIL_BYTES = 2 * 1024 * 1024; // 2MB tail is plenty for the last ~100-1000 messages + // + // MAX_TAIL_BYTES must exceed the chat.history response budget (6 MB) with enough headroom + // that the file-read layer never drops records that would fit in the response. Set to 3× + // the 6 MB response budget so truncation always happens at the response-cap layer in + // chat.ts (which the caller can observe), never silently here. + const MAX_TAIL_BYTES = 18 * 1024 * 1024; // 3× the 6 MB chat.history response budget // 200KB per line: a normal assistant reply is well under 50KB. Anything larger is a runaway // prompt/response that would only stall JSON.parse and bloat the UI — skip it entirely. // (The confirmed 447KB line causing Gateway freezes is caught by this threshold.) From e80516a8f0f28dddc1df6d0908f8cf7729394c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A8=81=E5=93=A5Wego?= Date: Thu, 19 Mar 2026 09:25:05 +0800 Subject: [PATCH 4/7] gateway: avoid dropping first tail record; emit placeholder for oversized lines --- src/gateway/session-utils.fs.ts | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index 6d44c5f7fe6..61618f42f9f 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -109,12 +109,23 @@ export function readSessionMessages( // uninitialized memory into the UTF-8 / JSON pipeline. const bytesRead = fs.readSync(fd, buf, 0, buf.length, start); content = buf.toString("utf-8", 0, bytesRead); + // If we started mid-line, drop the partial first line. - // Note: messages before the 2 MB boundary are intentionally omitted to keep + // Note: messages before the MAX_TAIL_BYTES boundary are intentionally omitted to keep // this RPC fast; the UI will show the most recent history only. - const firstNewline = content.indexOf("\n"); - if (firstNewline >= 0 && start > 0) { - content = content.slice(firstNewline + 1); + if (start > 0) { + // Only drop the first line if we actually started in the middle of a record. + // If the byte before `start` is a newline, `content` begins at a record boundary + // and we must not discard the first complete record. + const prev = Buffer.allocUnsafe(1); + const prevRead = fs.readSync(fd, prev, 0, 1, start - 1); + const prevChar = prevRead == 1 ? prev.toString("utf-8", 0, 1) : ""; + if (prevChar !== "\n") { + const firstNewline = content.indexOf("\n"); + if (firstNewline >= 0) { + content = content.slice(firstNewline + 1); + } + } } } finally { fs.closeSync(fd); @@ -134,10 +145,20 @@ export function readSessionMessages( continue; } if (trimmed.length > MAX_LINE_CHARS) { - // Skip lines that are too large to safely parse on the RPC path. + // Preserve history semantics: emit a placeholder entry instead of dropping the line entirely. + // (chat.history will still apply response-byte caps downstream.) console.warn( - `[session-utils] skipping oversized line in session ${sessionId}: ${trimmed.length} chars (max ${MAX_LINE_CHARS})`, + `[session-utils] oversized transcript line in session ${sessionId}: ${trimmed.length} chars (max ${MAX_LINE_CHARS}); emitting placeholder`, ); + messages.push({ + role: "system", + content: [{ type: "text", text: "[chat.history omitted: message too large]" }], + timestamp: Date.now(), + __openclaw: { + kind: "oversized_transcript_line", + sizeChars: trimmed.length, + }, + }); continue; } try { From 8647a2e391503bdcb2d668950066d7677c9ed9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A8=81=E5=93=A5Wego?= Date: Thu, 19 Mar 2026 09:55:57 +0800 Subject: [PATCH 5/7] gateway: grow transcript tail + preserve role for oversized lines --- src/gateway/session-utils.fs.ts | 115 +++++++++++++++++++++++--------- 1 file changed, 82 insertions(+), 33 deletions(-) diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index 61618f42f9f..e92e3c3a8c9 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -82,56 +82,78 @@ export function readSessionMessages( return []; } + // NOTE: This is on the Gateway hot path (chat.history). Reading + splitting an entire transcript // file can freeze the UI when a session grows large (or when a single JSONL record is huge). // We therefore tail-read large files and apply a per-line size guard. // - // MAX_TAIL_BYTES must exceed the chat.history response budget (6 MB) with enough headroom - // that the file-read layer never drops records that would fit in the response. Set to 3× - // the 6 MB response budget so truncation always happens at the response-cap layer in - // chat.ts (which the caller can observe), never silently here. - const MAX_TAIL_BYTES = 18 * 1024 * 1024; // 3× the 6 MB chat.history response budget + // Important: the downstream chat.history handler applies a 6 MB response budget *after* + // sanitization/stripping, so the raw transcript can be substantially larger on disk while still + // fitting in the response. To avoid silently dropping history that would fit post-sanitization, + // we grow the tail window until we either (a) start at byte 0, (b) have enough line coverage, or + // (c) hit a hard cap. + const INITIAL_TAIL_BYTES = 18 * 1024 * 1024; // 3× the 6 MB chat.history response budget + const MAX_TAIL_BYTES_CAP = 128 * 1024 * 1024; // hard cap to avoid huge reads on the RPC hot path + const MIN_TAIL_LINES_TARGET = 1500; // heuristic: enough for ~1000 recent messages + headroom + // 200KB per line: a normal assistant reply is well under 50KB. Anything larger is a runaway - // prompt/response that would only stall JSON.parse and bloat the UI — skip it entirely. - // (The confirmed 447KB line causing Gateway freezes is caught by this threshold.) + // prompt/response that would only stall JSON.parse and bloat the UI. const MAX_LINE_CHARS = 200 * 1024; let content = ""; try { const stat = fs.statSync(filePath); - if (stat.size > MAX_TAIL_BYTES) { + + if (stat.size <= INITIAL_TAIL_BYTES) { + content = fs.readFileSync(filePath, "utf-8"); + } else { const fd = fs.openSync(filePath, "r"); try { - const start = Math.max(0, stat.size - MAX_TAIL_BYTES); - const buf = Buffer.allocUnsafe(stat.size - start); - // Capture bytesRead: if the file shrank between statSync and readSync (TOCTOU), - // readSync returns fewer bytes than buf.length — slice to avoid feeding - // uninitialized memory into the UTF-8 / JSON pipeline. - const bytesRead = fs.readSync(fd, buf, 0, buf.length, start); - content = buf.toString("utf-8", 0, bytesRead); + let tailBytes = Math.min(stat.size, INITIAL_TAIL_BYTES); + while (true) { + const start = Math.max(0, stat.size - tailBytes); + const buf = Buffer.allocUnsafe(stat.size - start); - // If we started mid-line, drop the partial first line. - // Note: messages before the MAX_TAIL_BYTES boundary are intentionally omitted to keep - // this RPC fast; the UI will show the most recent history only. - if (start > 0) { - // Only drop the first line if we actually started in the middle of a record. - // If the byte before `start` is a newline, `content` begins at a record boundary - // and we must not discard the first complete record. - const prev = Buffer.allocUnsafe(1); - const prevRead = fs.readSync(fd, prev, 0, 1, start - 1); - const prevChar = prevRead == 1 ? prev.toString("utf-8", 0, 1) : ""; - if (prevChar !== "\n") { - const firstNewline = content.indexOf("\n"); - if (firstNewline >= 0) { - content = content.slice(firstNewline + 1); + // Capture bytesRead: if the file shrank between statSync and readSync (TOCTOU), + // readSync returns fewer bytes than buf.length — slice to avoid feeding uninitialized + // memory into the UTF-8 / JSON pipeline. + const bytesRead = fs.readSync(fd, buf, 0, buf.length, start); + content = buf.toString("utf-8", 0, bytesRead); + + + // If we started mid-line, drop the partial first line. + // Note: messages before the tail boundary are intentionally omitted to keep this RPC fast; + // the UI will show the most recent history only. + if (start > 0) { + // Only drop the first line if we actually started in the middle of a record. + // If the byte before `start` is a newline, `content` begins at a record boundary. + const prev = Buffer.allocUnsafe(1); + const prevRead = fs.readSync(fd, prev, 0, 1, start - 1); + const prevChar = prevRead === 1 ? prev.toString("utf-8", 0, 1) : ""; + if (prevChar !== "\n") { + const firstNewline = content.indexOf("\n"); + if (firstNewline >= 0) { + content = content.slice(firstNewline + 1); + } } } + + const newlineCount = (content.match(/\n/g) || []).length; + const lineCount = newlineCount + 1; + + if ( + start === 0 || + lineCount >= MIN_TAIL_LINES_TARGET || + tailBytes >= MAX_TAIL_BYTES_CAP + ) { + break; + } + + tailBytes = Math.min(MAX_TAIL_BYTES_CAP, tailBytes * 2); } } finally { fs.closeSync(fd); } - } else { - content = fs.readFileSync(filePath, "utf-8"); } } catch { return []; @@ -150,13 +172,40 @@ export function readSessionMessages( console.warn( `[session-utils] oversized transcript line in session ${sessionId}: ${trimmed.length} chars (max ${MAX_LINE_CHARS}); emitting placeholder`, ); + // Best-effort: preserve original role/timestamp without JSON.parse (which would stall). + const roleMatch = trimmed.match(/"role"\s*:\s*"([^"]+)"/); + const roleCandidate = roleMatch?.[1]; + const role = + roleCandidate === "user" || + roleCandidate === "assistant" || + roleCandidate === "tool" || + roleCandidate === "system" + ? roleCandidate + : "assistant"; + + let timestamp = Date.now(); + const tsNum = trimmed.match(/"timestamp"\s*:\s*(\d{10,13})/); + if (tsNum?.[1]) { + const n = Number(tsNum[1]); + if (Number.isFinite(n)) { + timestamp = n; + } + } else { + const tsIso = trimmed.match(/"timestamp"\s*:\s*"([^"]+)"/); + const d = tsIso?.[1] ? Date.parse(tsIso[1]) : Number.NaN; + if (Number.isFinite(d)) { + timestamp = d; + } + } + messages.push({ - role: "system", + role, content: [{ type: "text", text: "[chat.history omitted: message too large]" }], - timestamp: Date.now(), + timestamp, __openclaw: { kind: "oversized_transcript_line", sizeChars: trimmed.length, + guessed: true, }, }); continue; From d68c53ae36367092c671a127f1951adc2bed84f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A8=81=E5=93=A5Wego?= Date: Thu, 19 Mar 2026 10:15:44 +0800 Subject: [PATCH 6/7] gateway: improve oversized transcript placeholder semantics --- src/gateway/session-utils.fs.ts | 46 ++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index e92e3c3a8c9..ceaf8a1413b 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -146,6 +146,11 @@ export function readSessionMessages( lineCount >= MIN_TAIL_LINES_TARGET || tailBytes >= MAX_TAIL_BYTES_CAP ) { + if (tailBytes >= MAX_TAIL_BYTES_CAP && start > 0 && lineCount < MIN_TAIL_LINES_TARGET) { + console.warn( + `[session-utils] transcript tail cap hit for session ${sessionId}: read ${tailBytes} bytes (cap ${MAX_TAIL_BYTES_CAP}) but only recovered ~${lineCount} lines; history may be truncated`, + ); + } break; } @@ -173,7 +178,10 @@ export function readSessionMessages( `[session-utils] oversized transcript line in session ${sessionId}: ${trimmed.length} chars (max ${MAX_LINE_CHARS}); emitting placeholder`, ); // Best-effort: preserve original role/timestamp without JSON.parse (which would stall). - const roleMatch = trimmed.match(/"role"\s*:\s*"([^"]+)"/); + // Regex scans are limited to a prefix so we don't do O(n) work over tens/hundreds of MB. + const scan = trimmed.slice(0, 1_000_000); + + const roleMatch = scan.match(/"role"\s*:\s*"([^"]+)"/); const roleCandidate = roleMatch?.[1]; const role = roleCandidate === "user" || @@ -184,28 +192,58 @@ export function readSessionMessages( : "assistant"; let timestamp = Date.now(); - const tsNum = trimmed.match(/"timestamp"\s*:\s*(\d{10,13})/); + const tsNum = scan.match(/"timestamp"\s*:\s*(\d{10,13})/); if (tsNum?.[1]) { const n = Number(tsNum[1]); if (Number.isFinite(n)) { timestamp = n; } } else { - const tsIso = trimmed.match(/"timestamp"\s*:\s*"([^"]+)"/); + const tsIso = scan.match(/"timestamp"\s*:\s*"([^"]+)"/); const d = tsIso?.[1] ? Date.parse(tsIso[1]) : Number.NaN; if (Number.isFinite(d)) { timestamp = d; } } + // If the record is oversized mainly due to inline media (e.g. image blocks with `data:`), + // downstream sanitization would normally strip that payload. Preserve some semantics here + // without parsing the full JSON. + const looksLikeInlineMedia = /"type"\s*:\s*"image"/.test(scan) || /"data"\s*:\s*"data:/.test(scan); + + let placeholderText = "[chat.history omitted: message too large]"; + const textSnips: string[] = []; + if (looksLikeInlineMedia) { + placeholderText = "[chat.history omitted: inline media too large]"; + + // Best-effort: pull out a few text snippets so the user still sees *something* meaningful + // even if the record is oversized due to inline media payload. + const re = /"text"\s*:\s*"((?:\\.|[^"\\]){0,3000})"/g; + let match: RegExpExecArray | null; + while (textSnips.length < 4 && (match = re.exec(scan))) { + const raw = match[1]; + const unescaped = raw.replace(/\\n/g, "\n").replace(/\\t/g, "\t"); + const flat = unescaped.replace(/\s+/g, " ").trim(); + if (flat) { + textSnips.push(flat); + } + } + } + + const blocks: Array<{ type: string; text: string }> = [{ type: "text", text: placeholderText }]; + if (textSnips.length) { + blocks.push({ type: "text", text: `Context (best-effort): ${textSnips.join(" | ")}` }); + } + messages.push({ role, - content: [{ type: "text", text: "[chat.history omitted: message too large]" }], + content: blocks, timestamp, __openclaw: { kind: "oversized_transcript_line", sizeChars: trimmed.length, guessed: true, + looksLikeInlineMedia, }, }); continue; From 33cdd8e32b1d107e8d6ede8bf4c8aeb7f299ce5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A8=81=E5=93=A5Wego?= Date: Thu, 19 Mar 2026 10:54:50 +0800 Subject: [PATCH 7/7] gateway: measure transcript line guard in UTF-8 bytes --- src/gateway/session-utils.fs.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index ceaf8a1413b..cd12e2977a9 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -96,9 +96,9 @@ export function readSessionMessages( const MAX_TAIL_BYTES_CAP = 128 * 1024 * 1024; // hard cap to avoid huge reads on the RPC hot path const MIN_TAIL_LINES_TARGET = 1500; // heuristic: enough for ~1000 recent messages + headroom - // 200KB per line: a normal assistant reply is well under 50KB. Anything larger is a runaway + // 200KB per line (UTF-8 bytes): a normal assistant reply is well under 50KB. Anything larger is a runaway // prompt/response that would only stall JSON.parse and bloat the UI. - const MAX_LINE_CHARS = 200 * 1024; + const MAX_LINE_BYTES = 200 * 1024; let content = ""; try { @@ -171,11 +171,12 @@ export function readSessionMessages( if (!trimmed) { continue; } - if (trimmed.length > MAX_LINE_CHARS) { + const trimmedBytes = Buffer.byteLength(trimmed, "utf-8"); + if (trimmedBytes > MAX_LINE_BYTES) { // Preserve history semantics: emit a placeholder entry instead of dropping the line entirely. // (chat.history will still apply response-byte caps downstream.) console.warn( - `[session-utils] oversized transcript line in session ${sessionId}: ${trimmed.length} chars (max ${MAX_LINE_CHARS}); emitting placeholder`, + `[session-utils] oversized transcript line in session ${sessionId}: ${trimmedBytes} bytes (max ${MAX_LINE_BYTES}); emitting placeholder`, ); // Best-effort: preserve original role/timestamp without JSON.parse (which would stall). // Regex scans are limited to a prefix so we don't do O(n) work over tens/hundreds of MB. @@ -241,6 +242,7 @@ export function readSessionMessages( timestamp, __openclaw: { kind: "oversized_transcript_line", + sizeBytes: trimmedBytes, sizeChars: trimmed.length, guessed: true, looksLikeInlineMedia,