openclaw/src/slack/monitor/message-handler.ts
Dennis Rankin a28a4b1b61
feat: detect stale Slack sockets and auto-restart (#30153)
* feat: detect stale Slack sockets and auto-restart

Slack Socket Mode connections can silently stop delivering events while
still appearing connected (health checks pass, WebSocket stays open).
This "half-dead socket" problem causes messages to go unanswered.

This commit adds two layers of protection:

1. **Event liveness tracking**: Every inbound Slack event (messages,
   reactions, member joins/leaves, channel events, pins) now calls
   `setStatus({ lastEventAt, lastInboundAt })` to update the channel
   account snapshot with the timestamp of the last received event.

2. **Health monitor stale socket detection**: The channel health monitor
   now checks `lastEventAt` against a configurable threshold (default
   30 minutes). If a channel has been running longer than the threshold
   and hasn't received any events in that window, it is flagged as
   unhealthy and automatically restarted — the same way disconnected
   or crashed channels are already handled.

The restart reason is logged as "stale-socket" for observability, and
the existing cooldown/rate-limit logic (3 restarts/hour max) prevents
restart storms.

* Slack: gate liveness tracking to accepted events

---------

Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com>
2026-03-01 10:58:21 -06:00

123 lines
4.3 KiB
TypeScript

import { hasControlCommand } from "../../auto-reply/command-detection.js";
import {
createInboundDebouncer,
resolveInboundDebounceMs,
} from "../../auto-reply/inbound-debounce.js";
import type { ResolvedSlackAccount } from "../accounts.js";
import type { SlackMessageEvent } from "../types.js";
import { stripSlackMentionsForCommandDetection } from "./commands.js";
import type { SlackMonitorContext } from "./context.js";
import { dispatchPreparedSlackMessage } from "./message-handler/dispatch.js";
import { prepareSlackMessage } from "./message-handler/prepare.js";
import { createSlackThreadTsResolver } from "./thread-resolution.js";
export type SlackMessageHandler = (
message: SlackMessageEvent,
opts: { source: "message" | "app_mention"; wasMentioned?: boolean },
) => Promise<void>;
export function createSlackMessageHandler(params: {
ctx: SlackMonitorContext;
account: ResolvedSlackAccount;
/** Called on each inbound event to update liveness tracking. */
trackEvent?: () => void;
}): SlackMessageHandler {
const { ctx, account, trackEvent } = params;
const debounceMs = resolveInboundDebounceMs({ cfg: ctx.cfg, channel: "slack" });
const threadTsResolver = createSlackThreadTsResolver({ client: ctx.app.client });
const debouncer = createInboundDebouncer<{
message: SlackMessageEvent;
opts: { source: "message" | "app_mention"; wasMentioned?: boolean };
}>({
debounceMs,
buildKey: (entry) => {
const senderId = entry.message.user ?? entry.message.bot_id;
if (!senderId) {
return null;
}
const messageTs = entry.message.ts ?? entry.message.event_ts;
// If Slack flags a thread reply but omits thread_ts, isolate it from root debouncing.
const threadKey = entry.message.thread_ts
? `${entry.message.channel}:${entry.message.thread_ts}`
: entry.message.parent_user_id && messageTs
? `${entry.message.channel}:maybe-thread:${messageTs}`
: entry.message.channel;
return `slack:${ctx.accountId}:${threadKey}:${senderId}`;
},
shouldDebounce: (entry) => {
const text = entry.message.text ?? "";
if (!text.trim()) {
return false;
}
if (entry.message.files && entry.message.files.length > 0) {
return false;
}
const textForCommandDetection = stripSlackMentionsForCommandDetection(text);
return !hasControlCommand(textForCommandDetection, ctx.cfg);
},
onFlush: async (entries) => {
const last = entries.at(-1);
if (!last) {
return;
}
const combinedText =
entries.length === 1
? (last.message.text ?? "")
: entries
.map((entry) => entry.message.text ?? "")
.filter(Boolean)
.join("\n");
const combinedMentioned = entries.some((entry) => Boolean(entry.opts.wasMentioned));
const syntheticMessage: SlackMessageEvent = {
...last.message,
text: combinedText,
};
const prepared = await prepareSlackMessage({
ctx,
account,
message: syntheticMessage,
opts: {
...last.opts,
wasMentioned: combinedMentioned || last.opts.wasMentioned,
},
});
if (!prepared) {
return;
}
if (entries.length > 1) {
const ids = entries.map((entry) => entry.message.ts).filter(Boolean) as string[];
if (ids.length > 0) {
prepared.ctxPayload.MessageSids = ids;
prepared.ctxPayload.MessageSidFirst = ids[0];
prepared.ctxPayload.MessageSidLast = ids[ids.length - 1];
}
}
await dispatchPreparedSlackMessage(prepared);
},
onError: (err) => {
ctx.runtime.error?.(`slack inbound debounce flush failed: ${String(err)}`);
},
});
return async (message, opts) => {
if (opts.source === "message" && message.type !== "message") {
return;
}
if (
opts.source === "message" &&
message.subtype &&
message.subtype !== "file_share" &&
message.subtype !== "bot_message"
) {
return;
}
if (ctx.markMessageSeen(message.channel, message.ts)) {
return;
}
trackEvent?.();
const resolvedMessage = await threadTsResolver.resolve({ message, source: opts.source });
await debouncer.enqueue({ message: resolvedMessage, opts });
};
}