From 6b794dac4ed42fff2daf6a657f002a0adac90e55 Mon Sep 17 00:00:00 2001 From: kumarabhirup Date: Thu, 5 Mar 2026 15:35:51 -0800 Subject: [PATCH] feat(telemetry): add structure-preserving privacy redaction Redacts text content and tool arguments while keeping roles, tool names, and message ordering visible in PostHog. --- extensions/posthog-analytics/lib/privacy.ts | 135 ++++++++++++++++++++ src/telemetry/privacy.test.ts | 133 ++++++++++++++++++- 2 files changed, 267 insertions(+), 1 deletion(-) diff --git a/extensions/posthog-analytics/lib/privacy.ts b/extensions/posthog-analytics/lib/privacy.ts index 0f6cd112cc5..87072bfd090 100644 --- a/extensions/posthog-analytics/lib/privacy.ts +++ b/extensions/posthog-analytics/lib/privacy.ts @@ -81,3 +81,138 @@ export function sanitizeForCapture( if (privacyMode) return REDACTED; return stripSecrets(value); } + +/** + * Redact a tool_calls array while preserving tool name, id, and type metadata. + * Only arguments are redacted. + */ +function redactToolCalls(toolCalls: unknown[]): unknown[] { + return toolCalls.map((tc: any) => { + if (!tc || typeof tc !== "object") return tc; + const out: Record = { + id: tc.id, + type: tc.type ?? "function", + }; + if (tc.function && typeof tc.function === "object") { + out.function = { + name: tc.function.name, + arguments: REDACTED, + }; + } + if (tc.name) out.name = tc.name; + return out; + }); +} + +/** + * Redact Anthropic-format content blocks while preserving tool metadata. + * Text blocks get redacted; toolCall blocks keep their name but redact arguments. + */ +function redactContentBlocks(blocks: unknown[]): unknown[] { + return blocks.map((block: any) => { + if (!block || typeof block !== "object") return block; + if (block.type === "text") { + return { type: "text", text: REDACTED }; + } + if (block.type === "toolCall") { + return { + type: "toolCall", + id: block.id ?? block.toolCallId, + name: block.name, + arguments: REDACTED, + }; + } + if (block.type === "tool_use") { + return { + type: "tool_use", + id: block.id, + name: block.name, + input: REDACTED, + }; + } + if (block.type === "thinking") { + return { type: "thinking", text: REDACTED }; + } + return { type: block.type }; + }); +} + +/** + * Structure-preserving message redaction for PostHog message-array fields. + * Preserves: role, tool names, tool_call IDs, message ordering, tool types. + * Redacts: text content, tool arguments, tool results. + */ +export function redactMessagesStructured(messages: unknown): unknown { + if (!Array.isArray(messages)) return messages; + return messages.map((msg: any) => { + if (!msg || typeof msg !== "object") return msg; + const out: Record = { role: msg.role }; + + if (msg.name) out.name = msg.name; + if (msg.tool_call_id) out.tool_call_id = msg.tool_call_id; + if (msg.toolCallId) out.toolCallId = msg.toolCallId; + if (msg.toolName) out.toolName = msg.toolName; + if (msg.isError != null) out.isError = msg.isError; + if (msg.stopReason) out.stopReason = msg.stopReason; + if (msg.model) out.model = msg.model; + if (msg.provider) out.provider = msg.provider; + + if (Array.isArray(msg.content)) { + out.content = redactContentBlocks(msg.content); + } else { + out.content = REDACTED; + } + + if (Array.isArray(msg.tool_calls)) { + out.tool_calls = redactToolCalls(msg.tool_calls); + } + + return out; + }); +} + +/** + * Structure-preserving redaction for normalized OpenAI-format output choices. + * Keeps role, tool_calls[].function.name, tool_calls[].type. + * Redacts text content and tool arguments. + */ +export function redactOutputChoicesStructured(choices: unknown): unknown { + if (!Array.isArray(choices)) return choices; + return choices.map((choice: any) => { + if (!choice || typeof choice !== "object") return choice; + const out: Record = { + role: choice.role, + content: choice.content != null ? REDACTED : null, + }; + if (Array.isArray(choice.tool_calls)) { + out.tool_calls = redactToolCalls(choice.tool_calls); + } + return out; + }); +} + +/** + * Sanitize messages for PostHog capture, preserving structure in privacy mode. + * Privacy on: redacts text content/arguments but keeps role, tool names, ordering. + * Privacy off: only strips credential patterns. + */ +export function sanitizeMessages( + messages: unknown, + privacyMode: boolean, +): unknown { + if (privacyMode) return redactMessagesStructured(messages); + return stripSecrets(messages); +} + +/** + * Sanitize normalized output choices for PostHog capture, preserving structure. + * Privacy on: redacts text content/arguments but keeps role, tool names. + * Privacy off: only strips credential patterns. + */ +export function sanitizeOutputChoices( + choices: unknown, + privacyMode: boolean, +): unknown { + if (privacyMode) return redactOutputChoicesStructured(choices); + return stripSecrets(choices); +} diff --git a/src/telemetry/privacy.test.ts b/src/telemetry/privacy.test.ts index 233e7b8de0d..aa371322202 100644 --- a/src/telemetry/privacy.test.ts +++ b/src/telemetry/privacy.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { stripSecrets, redactMessages, sanitizeForCapture } from "../../extensions/posthog-analytics/lib/privacy.js"; +import { stripSecrets, redactMessages, sanitizeForCapture, redactMessagesStructured, redactOutputChoicesStructured, sanitizeMessages, sanitizeOutputChoices } from "../../extensions/posthog-analytics/lib/privacy.js"; // --------------------------------------------------------------------------- // stripSecrets — security-critical: prevents credential leakage @@ -208,3 +208,134 @@ describe("sanitizeForCapture", () => { expect(result.data).toBe("visible"); }); }); + +// --------------------------------------------------------------------------- +// redactMessagesStructured — preserves message/tool structure in privacy mode +// --------------------------------------------------------------------------- + +describe("redactMessagesStructured", () => { + it("preserves role and ordering while redacting text content (enables PostHog conversation view)", () => { + const messages = [ + { role: "user", content: "secret question" }, + { role: "assistant", content: "secret answer" }, + { role: "user", content: "follow-up" }, + ]; + const result = redactMessagesStructured(messages) as Array>; + expect(result).toHaveLength(3); + expect(result[0]).toEqual({ role: "user", content: "[REDACTED]" }); + expect(result[1]).toEqual({ role: "assistant", content: "[REDACTED]" }); + expect(result[2]).toEqual({ role: "user", content: "[REDACTED]" }); + }); + + it("preserves tool_calls[].function.name while redacting arguments (tool type visible in privacy mode)", () => { + const messages = [ + { + role: "assistant", + content: "Let me run that.", + tool_calls: [ + { id: "call_1", type: "function", function: { name: "exec", arguments: '{"cmd":"ls -la"}' } }, + { id: "call_2", type: "function", function: { name: "read", arguments: '{"path":"/etc/passwd"}' } }, + ], + }, + ]; + const result = redactMessagesStructured(messages) as Array>; + expect(result[0].role).toBe("assistant"); + expect(result[0].content).toBe("[REDACTED]"); + const tc = result[0].tool_calls as Array>; + expect(tc).toHaveLength(2); + expect((tc[0] as any).function.name).toBe("exec"); + expect((tc[0] as any).function.arguments).toBe("[REDACTED]"); + expect((tc[1] as any).function.name).toBe("read"); + }); + + it("preserves Anthropic toolCall content blocks with name visible (type visible regardless of privacy)", () => { + const messages = [ + { + role: "assistant", + content: [ + { type: "text", text: "Running command..." }, + { type: "toolCall", id: "tc1", name: "exec", arguments: { command: "ls" } }, + ], + }, + ]; + const result = redactMessagesStructured(messages) as Array>; + const blocks = result[0].content as Array>; + expect(blocks).toHaveLength(2); + expect(blocks[0]).toEqual({ type: "text", text: "[REDACTED]" }); + expect(blocks[1].type).toBe("toolCall"); + expect(blocks[1].name).toBe("exec"); + expect(blocks[1].arguments).toBe("[REDACTED]"); + }); + + it("preserves tool result metadata (name, tool_call_id) while redacting content", () => { + const messages = [ + { role: "tool", name: "exec", tool_call_id: "call_1", content: "file1.txt\nfile2.txt" }, + ]; + const result = redactMessagesStructured(messages) as Array>; + expect(result[0].role).toBe("tool"); + expect(result[0].name).toBe("exec"); + expect(result[0].tool_call_id).toBe("call_1"); + expect(result[0].content).toBe("[REDACTED]"); + }); + + it("preserves model and provider metadata on assistant messages", () => { + const messages = [ + { role: "assistant", content: "hello", model: "claude-4", provider: "anthropic" }, + ]; + const result = redactMessagesStructured(messages) as Array>; + expect(result[0].model).toBe("claude-4"); + expect(result[0].provider).toBe("anthropic"); + expect(result[0].content).toBe("[REDACTED]"); + }); + + it("returns non-array input unchanged", () => { + expect(redactMessagesStructured(null)).toBe(null); + expect(redactMessagesStructured("string")).toBe("string"); + }); +}); + +describe("redactOutputChoicesStructured", () => { + it("preserves tool_calls[].function.name while redacting text and arguments", () => { + const choices = [ + { + role: "assistant", + content: "Here you go", + tool_calls: [{ id: "c1", type: "function", function: { name: "web_search", arguments: '{"q":"test"}' } }], + }, + ]; + const result = redactOutputChoicesStructured(choices) as Array>; + expect(result[0].content).toBe("[REDACTED]"); + const tc = result[0].tool_calls as Array>; + expect((tc[0] as any).function.name).toBe("web_search"); + expect((tc[0] as any).function.arguments).toBe("[REDACTED]"); + }); + + it("sets content to null when original content is null (no-text tool-only responses)", () => { + const choices = [ + { role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "exec" } }] }, + ]; + const result = redactOutputChoicesStructured(choices) as Array>; + expect(result[0].content).toBe(null); + }); +}); + +describe("sanitizeMessages", () => { + it("uses structural redaction when privacy is on (preserves role/tool structure)", () => { + const messages = [ + { role: "user", content: "secret" }, + { role: "assistant", content: "response", tool_calls: [{ id: "c", type: "function", function: { name: "exec", arguments: "{}" } }] }, + ]; + const result = sanitizeMessages(messages, true) as Array>; + expect(result[0].role).toBe("user"); + expect(result[0].content).toBe("[REDACTED]"); + expect(result[1].role).toBe("assistant"); + const tc = result[1].tool_calls as any[]; + expect(tc[0].function.name).toBe("exec"); + }); + + it("uses secret stripping when privacy is off (preserves full content)", () => { + const messages = [{ role: "user", content: "hello" }]; + const result = sanitizeMessages(messages, false) as Array>; + expect(result[0].content).toBe("hello"); + }); +});