diff --git a/.cursor/plans/dench_filesystem_crm_integration.plan.md b/.cursor/plans/dench_filesystem_crm_integration.plan.md index 36baec5d76a..d2f3698bd23 100644 --- a/.cursor/plans/dench_filesystem_crm_integration.plan.md +++ b/.cursor/plans/dench_filesystem_crm_integration.plan.md @@ -538,18 +538,41 @@ Identical patterns, but with SQL examples instead of tool call examples: - NEVER modify `workspace_context.yaml` -- it is read-only context from Dench - Members list is authoritative for user-type field resolution -### Section 11: Post-Mutation Pipeline (new) +### Section 11: Post-Mutation Checklist (MANDATORY -- revised after agent testing) -After any schema mutation (create/update/delete object, field, or document), run a 3-step pipeline: +**Problem identified:** In testing, the agent correctly executed SQL (object + fields + entries) but skipped the filesystem projection (.object.yaml) and sometimes the PIVOT view. Root cause: the original skill mentioned these as afterthoughts ("Then project the filesystem...") with no concrete template or examples. The agent follows examples literally -- if examples only show SQL, it only does SQL. -1. **Regenerate views**: `CREATE OR REPLACE VIEW v_{object}` for any affected objects -2. **Project filesystem**: Sync the `knowledge/` directory structure from DuckDB (mkdir/rmdir for objects, write `.object.yaml` summaries, move `.md` files if nesting changed) -3. **Sync to S3**: Run `dench/sync.sh` to persist workspace.duckdb + knowledge/ to S3 -4. **Regenerate WORKSPACE.md**: Human-readable summary of all objects, fields, entry counts, and views +**Fix:** Every workflow example now uses an explicit 3-step structure. The post-mutation section is now a checklist, not a description. + +After creating/modifying an OBJECT or FIELDS: + +- `CREATE OR REPLACE VIEW v_{object_name}` -- regenerate PIVOT view +- `mkdir -p dench/knowledge/{object_name}/` -- create directory +- Write `.object.yaml` with id, name, description, icon, default_view, entry_count, and full field list +- Update WORKSPACE.md + +After adding ENTRIES: + +- Update `entry_count` in `.object.yaml` +- Verify: `SELECT * FROM v_{object} LIMIT 5` + +After deleting an OBJECT: + +- `DROP VIEW IF EXISTS v_{object_name}` +- `rm -rf dench/knowledge/{object_name}/` + +The skill now includes: + +- A concrete `.object.yaml` template with example content (previously missing entirely) +- Full bash commands for generating `.object.yaml` from DuckDB queries +- "Step 1 / Step 2 / Step 3" structure in every workflow example (SQL, Filesystem, Verify) +- Critical Reminders section leads with "NEVER SKIP FILESYSTEM PROJECTION" and "THREE STEPS, EVERY TIME" ### Section 12: Critical Reminders (adapted from ``) -- Handle the ENTIRE CRM operation from analysis to SQL execution to summary +- Handle the ENTIRE CRM operation from analysis to SQL execution **to filesystem projection** to summary +- **NEVER SKIP FILESYSTEM PROJECTION**: After any object mutation, create/update `.object.yaml` AND the `v_{object}` view. If missing, the object is invisible in the sidebar. +- **THREE STEPS, EVERY TIME**: (1) SQL transaction, (2) filesystem projection, (3) verify - Always check existing data before creating (SELECT before INSERT, or ON CONFLICT) - Search proactively to provide better UX (PIVOT with filters) - Never assume field names -- always verify with `SELECT * FROM fields WHERE object_id = ?` @@ -561,6 +584,7 @@ After any schema mutation (create/update/delete object, field, or document), run - KANBAN BOARDS: `default_view = 'kanban'`, auto-create Status and Assigned To fields - PROTECTED OBJECTS: Never delete objects listed in `workspace_context.yaml` `protected_objects` - ONE EXEC CALL: Batch related SQL in a single transaction whenever possible -- this is the entire point of the filesystem-first approach +- ENTRY COUNT: After adding entries, update `entry_count` in `.object.yaml` --- diff --git a/apps/web/app/api/workspace/db/introspect/route.ts b/apps/web/app/api/workspace/db/introspect/route.ts new file mode 100644 index 00000000000..3c78a50cea0 --- /dev/null +++ b/apps/web/app/api/workspace/db/introspect/route.ts @@ -0,0 +1,93 @@ +import { safeResolvePath, duckdbQueryOnFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type TableInfo = { + table_name: string; + column_count: number; + estimated_row_count: number; + columns: Array<{ + name: string; + type: string; + is_nullable: boolean; + }>; +}; + +/** + * GET /api/workspace/db/introspect?path= + * + * Introspects a DuckDB / SQLite / generic DB file using the duckdb CLI. + * Returns the list of tables with their columns and approximate row counts. + */ +export async function GET(request: Request) { + const { searchParams } = new URL(request.url); + const relPath = searchParams.get("path"); + + if (!relPath) { + return Response.json( + { error: "Missing required `path` query parameter" }, + { status: 400 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + // Get all user tables (skip internal DuckDB catalogs) + const rawTables = duckdbQueryOnFile<{ + table_name: string; + table_type: string; + }>( + absPath, + "SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = 'main' ORDER BY table_name", + ); + + if (rawTables.length === 0) { + return Response.json({ tables: [], path: relPath }); + } + + const tables: TableInfo[] = []; + + for (const t of rawTables) { + // Fetch columns for this table + const cols = duckdbQueryOnFile<{ + column_name: string; + data_type: string; + is_nullable: string; + }>( + absPath, + `SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = 'main' AND table_name = '${t.table_name.replace(/'/g, "''")}' ORDER BY ordinal_position`, + ); + + // Get approximate row count + let rowCount = 0; + try { + const countResult = duckdbQueryOnFile<{ cnt: number }>( + absPath, + `SELECT count(*) as cnt FROM "${t.table_name.replace(/"/g, '""')}"`, + ); + rowCount = countResult[0]?.cnt ?? 0; + } catch { + // skip if we can't count + } + + tables.push({ + table_name: t.table_name, + column_count: cols.length, + estimated_row_count: rowCount, + columns: cols.map((c) => ({ + name: c.column_name, + type: c.data_type, + is_nullable: c.is_nullable === "YES", + })), + }); + } + + return Response.json({ tables, path: relPath }); +} diff --git a/apps/web/app/api/workspace/db/query/route.ts b/apps/web/app/api/workspace/db/query/route.ts new file mode 100644 index 00000000000..69c7011af79 --- /dev/null +++ b/apps/web/app/api/workspace/db/query/route.ts @@ -0,0 +1,56 @@ +import { safeResolvePath, duckdbQueryOnFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/db/query + * Body: { path: string, sql: string } + * + * Executes a read-only SQL query against a database file and returns JSON rows. + * Only SELECT statements are allowed for safety. + */ +export async function POST(request: Request) { + let body: { path?: string; sql?: string }; + try { + body = await request.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, sql } = body; + + if (!relPath || !sql) { + return Response.json( + { error: "Missing required `path` and `sql` fields" }, + { status: 400 }, + ); + } + + // Basic safety: only allow SELECT-like statements + const trimmedSql = sql.trim().toUpperCase(); + if ( + !trimmedSql.startsWith("SELECT") && + !trimmedSql.startsWith("PRAGMA") && + !trimmedSql.startsWith("DESCRIBE") && + !trimmedSql.startsWith("SHOW") && + !trimmedSql.startsWith("EXPLAIN") && + !trimmedSql.startsWith("WITH") + ) { + return Response.json( + { error: "Only read-only queries (SELECT, DESCRIBE, SHOW, EXPLAIN, WITH) are allowed" }, + { status: 403 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + const rows = duckdbQueryOnFile(absPath, sql); + return Response.json({ rows, sql }); +} diff --git a/apps/web/app/api/workspace/tree/route.ts b/apps/web/app/api/workspace/tree/route.ts index bb1a50623cc..46112699542 100644 --- a/apps/web/app/api/workspace/tree/route.ts +++ b/apps/web/app/api/workspace/tree/route.ts @@ -1,6 +1,6 @@ import { readdirSync, readFileSync, existsSync, type Dirent } from "node:fs"; import { join } from "node:path"; -import { resolveDenchRoot, parseSimpleYaml, duckdbQuery } from "@/lib/workspace"; +import { resolveDenchRoot, parseSimpleYaml, duckdbQuery, isDatabaseFile } from "@/lib/workspace"; export const dynamic = "force-dynamic"; export const runtime = "nodejs"; @@ -8,7 +8,7 @@ export const runtime = "nodejs"; export type TreeNode = { name: string; path: string; // relative to dench/ - type: "object" | "document" | "folder" | "file"; + type: "object" | "document" | "folder" | "file" | "database"; icon?: string; defaultView?: "table" | "kanban"; children?: TreeNode[]; @@ -118,11 +118,12 @@ function buildTree( } else if (entry.isFile()) { const ext = entry.name.split(".").pop()?.toLowerCase(); const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = isDatabaseFile(entry.name); nodes.push({ name: entry.name, path: relPath, - type: isDocument ? "document" : "file", + type: isDatabase ? "database" : isDocument ? "document" : "file", }); } } @@ -130,6 +131,14 @@ function buildTree( return nodes; } +/** Classify a top-level file's type. */ +function classifyFileType(name: string): TreeNode["type"] { + if (isDatabaseFile(name)) {return "database";} + const ext = name.split(".").pop()?.toLowerCase(); + if (ext === "md" || ext === "mdx") {return "document";} + return "file"; +} + export async function GET() { const root = resolveDenchRoot(); if (!root) { @@ -147,19 +156,17 @@ export async function GET() { tree.push(...buildTree(knowledgeDir, "knowledge", dbObjects)); } - // Add top-level files (WORKSPACE.md, workspace_context.yaml, etc.) + // Add top-level files (WORKSPACE.md, workspace_context.yaml, workspace.duckdb, etc.) try { const topLevel = readdirSync(root, { withFileTypes: true }); for (const entry of topLevel) { if (!entry.isFile()) {continue;} if (entry.name.startsWith(".")) {continue;} - const ext = entry.name.split(".").pop()?.toLowerCase(); - const isDocument = ext === "md" || ext === "mdx"; tree.push({ name: entry.name, path: entry.name, - type: isDocument ? "document" : "file", + type: classifyFileType(entry.name), }); } } catch { diff --git a/apps/web/app/components/sidebar.tsx b/apps/web/app/components/sidebar.tsx index 760103f1964..e0e2115db7b 100644 --- a/apps/web/app/components/sidebar.tsx +++ b/apps/web/app/components/sidebar.tsx @@ -27,7 +27,7 @@ type MemoryFile = { type TreeNode = { name: string; path: string; - type: "object" | "document" | "folder" | "file"; + type: "object" | "document" | "folder" | "file" | "database"; icon?: string; defaultView?: "table" | "kanban"; children?: TreeNode[]; @@ -231,7 +231,9 @@ function WorkspaceTreeNode({ ? "var(--color-accent)" : node.type === "document" ? "#60a5fa" - : "var(--color-text-muted)"; + : node.type === "database" + ? "#c084fc" + : "var(--color-text-muted)"; return (
@@ -241,7 +243,7 @@ function WorkspaceTreeNode({ onClick={() => { if (isExpandable) {onToggle(node.path);} // Navigate to workspace page for actionable items - if (node.type === "object" || node.type === "document" || node.type === "file") { + if (node.type === "object" || node.type === "document" || node.type === "file" || node.type === "database") { window.location.href = `/workspace?path=${encodeURIComponent(node.path)}`; } }} @@ -279,6 +281,12 @@ function WorkspaceTreeNode({ + ) : node.type === "database" ? ( + + + + + ) : ( diff --git a/apps/web/app/components/workspace/database-viewer.tsx b/apps/web/app/components/workspace/database-viewer.tsx new file mode 100644 index 00000000000..207d649f965 --- /dev/null +++ b/apps/web/app/components/workspace/database-viewer.tsx @@ -0,0 +1,918 @@ +"use client"; + +import { useEffect, useState, useCallback, useMemo } from "react"; + +// --- Types --- + +type ColumnInfo = { + name: string; + type: string; + is_nullable: boolean; +}; + +type TableInfo = { + table_name: string; + column_count: number; + estimated_row_count: number; + columns: ColumnInfo[]; +}; + +type SortState = { + column: string; + direction: "asc" | "desc"; +} | null; + +type DatabaseViewerProps = { + /** Relative path to the database file within the dench workspace */ + dbPath: string; + filename: string; +}; + +// --- Icons --- + +function DatabaseIcon({ size = 16 }: { size?: number }) { + return ( + + + + + + ); +} + +function TableIcon() { + return ( + + + + ); +} + +function ViewIcon() { + return ( + + + + + ); +} + +function ColumnIcon() { + return ( + + + + ); +} + +function PlayIcon() { + return ( + + + + ); +} + +function ChevronIcon({ direction }: { direction: "left" | "right" }) { + return ( + + {direction === "left" ? ( + + ) : ( + + )} + + ); +} + +function SortIndicator({ active, direction }: { active: boolean; direction: "asc" | "desc" }) { + return ( + + {direction === "asc" ? : } + + ); +} + +// --- Helpers --- + +function formatRowCount(n: number): string { + if (n >= 1_000_000) {return `${(n / 1_000_000).toFixed(1)}M`;} + if (n >= 1_000) {return `${(n / 1_000).toFixed(1)}K`;} + return String(n); +} + +/** Map DuckDB type names to short display labels + color hints */ +function typeDisplay(dtype: string): { label: string; color: string } { + const t = dtype.toUpperCase(); + if (t.includes("INT") || t.includes("BIGINT") || t.includes("SMALLINT") || t.includes("TINYINT") || t.includes("HUGEINT")) + {return { label: "int", color: "#c084fc" };} + if (t.includes("FLOAT") || t.includes("DOUBLE") || t.includes("DECIMAL") || t.includes("NUMERIC") || t.includes("REAL")) + {return { label: "float", color: "#c084fc" };} + if (t.includes("BOOL")) + {return { label: "bool", color: "#f59e0b" };} + if (t.includes("VARCHAR") || t.includes("TEXT") || t.includes("STRING") || t.includes("CHAR") || t === "UUID" || t === "BLOB") + {return { label: t.includes("UUID") ? "uuid" : "text", color: "#22c55e" };} + if (t.includes("TIMESTAMP") || t.includes("DATETIME")) + {return { label: "timestamp", color: "#60a5fa" };} + if (t.includes("DATE")) + {return { label: "date", color: "#60a5fa" };} + if (t.includes("TIME")) + {return { label: "time", color: "#60a5fa" };} + if (t.includes("JSON")) + {return { label: "json", color: "#fb923c" };} + return { label: dtype.toLowerCase(), color: "var(--color-text-muted)" }; +} + +// --- Main Component --- + +export function DatabaseViewer({ dbPath, filename }: DatabaseViewerProps) { + const [tables, setTables] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + // Selected table + const [selectedTable, setSelectedTable] = useState(null); + + // Table data + const [tableData, setTableData] = useState[]>([]); + const [dataLoading, setDataLoading] = useState(false); + const [sort, setSort] = useState(null); + + // Pagination + const [page, setPage] = useState(0); + const pageSize = 100; + + // Custom SQL query + const [queryMode, setQueryMode] = useState(false); + const [sqlInput, setSqlInput] = useState(""); + const [queryResult, setQueryResult] = useState[] | null>(null); + const [queryError, setQueryError] = useState(null); + const [queryRunning, setQueryRunning] = useState(false); + + // Schema panel toggle + const [showSchema, setShowSchema] = useState(false); + + // Fetch table list on mount + useEffect(() => { + let cancelled = false; + + async function introspect() { + setLoading(true); + setError(null); + try { + const res = await fetch( + `/api/workspace/db/introspect?path=${encodeURIComponent(dbPath)}`, + ); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.error || `HTTP ${res.status}`); + } + const data = await res.json(); + if (!cancelled) { + setTables(data.tables ?? []); + // Auto-select first table + if (data.tables?.length > 0) { + setSelectedTable(data.tables[0].table_name); + } + } + } catch (err) { + if (!cancelled) { + setError(err instanceof Error ? err.message : "Failed to introspect database"); + } + } finally { + if (!cancelled) {setLoading(false);} + } + } + + introspect(); + return () => { cancelled = true; }; + }, [dbPath]); + + // Fetch table data when selection or page changes + const fetchTableData = useCallback( + async (tableName: string, offset: number) => { + setDataLoading(true); + try { + const safeName = tableName.replace(/"/g, '""'); + const sql = `SELECT * FROM "${safeName}" LIMIT ${pageSize} OFFSET ${offset}`; + const res = await fetch("/api/workspace/db/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: dbPath, sql }), + }); + if (!res.ok) { + setTableData([]); + return; + } + const data = await res.json(); + setTableData(data.rows ?? []); + } catch { + setTableData([]); + } finally { + setDataLoading(false); + } + }, + [dbPath], + ); + + useEffect(() => { + if (selectedTable) { + setSort(null); + fetchTableData(selectedTable, page * pageSize); + } + }, [selectedTable, page, fetchTableData]); + + // Run custom query + const runQuery = useCallback(async () => { + if (!sqlInput.trim()) {return;} + setQueryRunning(true); + setQueryError(null); + setQueryResult(null); + try { + const res = await fetch("/api/workspace/db/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: dbPath, sql: sqlInput }), + }); + const data = await res.json(); + if (!res.ok) { + setQueryError(data.error || `HTTP ${res.status}`); + } else { + setQueryResult(data.rows ?? []); + } + } catch (err) { + setQueryError(err instanceof Error ? err.message : "Query failed"); + } finally { + setQueryRunning(false); + } + }, [dbPath, sqlInput]); + + // Get selected table info + const selectedTableInfo = useMemo( + () => tables.find((t) => t.table_name === selectedTable) ?? null, + [tables, selectedTable], + ); + + // Sort client-side + const sortedData = useMemo(() => { + const data = queryMode && queryResult ? queryResult : tableData; + if (!sort) {return data;} + return [...data].toSorted((a, b) => { + const aVal = String(a[sort.column] ?? ""); + const bVal = String(b[sort.column] ?? ""); + const cmp = aVal.localeCompare(bVal, undefined, { numeric: true }); + return sort.direction === "asc" ? cmp : -cmp; + }); + }, [queryMode, queryResult, tableData, sort]); + + const handleSort = (column: string) => { + setSort((prev) => { + if (prev?.column === column) { + return prev.direction === "asc" + ? { column, direction: "desc" } + : null; + } + return { column, direction: "asc" }; + }); + }; + + // Derive columns from data + const dataColumns = useMemo(() => { + const data = queryMode && queryResult ? queryResult : tableData; + if (data.length === 0) {return [];} + return Object.keys(data[0]); + }, [queryMode, queryResult, tableData]); + + // Detect database engine from filename + const dbEngine = useMemo(() => { + const ext = filename.split(".").pop()?.toLowerCase(); + if (ext === "duckdb") {return "DuckDB";} + if (ext === "sqlite" || ext === "sqlite3") {return "SQLite";} + if (ext === "postgres") {return "PostgreSQL";} + if (ext === "db") {return "Database";} + return "Database"; + }, [filename]); + + // --- Loading state --- + if (loading) { + return ( +
+
+ + Loading database... + +
+ ); + } + + // --- Error state --- + if (error) { + return ( +
+ +

+ Failed to open database +

+

+ {error} +

+
+ ); + } + + return ( +
+ {/* Left panel: Table list */} +
+ {/* Database header */} +
+ + + +
+
+ {filename} +
+
+ {dbEngine} · {tables.length} table{tables.length !== 1 ? "s" : ""} +
+
+
+ + {/* Table list */} +
+ {tables.length === 0 ? ( +
+ No tables found +
+ ) : ( + tables.map((t) => { + const isView = t.table_name.startsWith("v_"); + const isActive = selectedTable === t.table_name; + return ( + + ); + }) + )} +
+ + {/* Query mode toggle */} +
+ +
+
+ + {/* Right panel: Data / Query */} +
+ {queryMode ? ( + + ) : selectedTableInfo ? ( + setShowSchema(!showSchema)} + /> + ) : ( +
+

+ Select a table to view its data +

+
+ )} +
+
+ ); +} + +// --- Table Data Panel --- + +function TableDataPanel({ + table, + data, + dataLoading, + dataColumns, + sort, + onSort, + page, + pageSize, + onPageChange, + showSchema, + onToggleSchema, +}: { + table: TableInfo; + data: Record[]; + dataLoading: boolean; + dataColumns: string[]; + sort: SortState; + onSort: (col: string) => void; + page: number; + pageSize: number; + onPageChange: (page: number) => void; + showSchema: boolean; + onToggleSchema: () => void; +}) { + const totalRows = table.estimated_row_count; + const totalPages = Math.max(1, Math.ceil(totalRows / pageSize)); + + return ( +
+ {/* Table header bar */} +
+ + + + + {table.table_name} + + + {/* Stats */} +
+ + {table.estimated_row_count.toLocaleString()} rows + + + {table.column_count} columns + + +
+
+ + {/* Schema panel (collapsible) */} + {showSchema && ( +
+
+ {table.columns.map((col) => { + const display = typeDisplay(col.type); + return ( +
+ + + + + {col.name} + + + {display.label} + + {col.is_nullable && ( + + null + + )} +
+ ); + })} +
+
+ )} + + {/* Data table */} +
+ {dataLoading ? ( +
+
+
+ ) : data.length === 0 ? ( +
+ + + +

+ No data +

+
+ ) : ( + + )} +
+ + {/* Pagination */} + {totalRows > pageSize && ( +
+ + Page {page + 1} of {totalPages} + +
+ + +
+
+ )} +
+ ); +} + +// --- Query Panel --- + +function QueryPanel({ + sqlInput, + setSqlInput, + queryResult, + queryError, + queryRunning, + runQuery, + dataColumns, + sortedData, + sort, + onSort, +}: { + sqlInput: string; + setSqlInput: (v: string) => void; + queryResult: Record[] | null; + queryError: string | null; + queryRunning: boolean; + runQuery: () => void; + dataColumns: string[]; + sortedData: Record[]; + sort: SortState; + onSort: (col: string) => void; +}) { + return ( +
+ {/* SQL input */} +
+
+
+