diff --git a/.cursor/plans/cli-only-streaming-hardening_8cba61e1.plan.md b/.cursor/plans/cli-only-streaming-hardening_8cba61e1.plan.md new file mode 100644 index 00000000000..3bcb9887296 --- /dev/null +++ b/.cursor/plans/cli-only-streaming-hardening_8cba61e1.plan.md @@ -0,0 +1,102 @@ +--- +name: cli-only-streaming-hardening +overview: Harden the CLI-only web streaming refactor by fixing protocol-level flaws first, then replacing web WS consumers with managed CLI subscribe processes and adding guardrails for dedupe, lifecycle, and long-wait stability. +todos: + - id: fix-subscribe-cli-semantics + content: Make `agent --stream-json --subscribe-session-key` long-lived and session-filtered, with tests. + status: completed + - id: add-subscribe-spawner + content: Add `spawnAgentSubscribeProcess` helper in `apps/web/lib/agent-runner.ts` with profile/workspace env wiring. + status: completed + - id: parent-wait-cli-subscribe + content: Refactor `apps/web/lib/active-runs.ts` waiting flow to use managed subscribe child + globalSeq dedupe. + status: completed + - id: subagent-cli-subscribe + content: Refactor `apps/web/lib/subagent-runs.ts` fallback/rehydration to managed subscribe child + globalSeq dedupe. + status: completed + - id: remove-web-ws-client + content: Remove `apps/web/lib/gateway-events.ts` usages and delete file after typecheck passes. + status: completed + - id: sse-keepalive + content: Add keepalive behavior for long idle waiting streams. + status: completed + - id: verify-regressions + content: Run targeted tests/smoke checks for handoff, refresh, replay, and duplicate/cross-session safety. + status: completed +isProject: false +--- + +# CLI-Only Streaming Plan (Flaw-Hardened) + +## Critical flaws to fix before WS removal + +- The current subscribe CLI path in [src/commands/agent-via-gateway.ts](/Users/kumareth/Documents/projects/openclaw/src/commands/agent-via-gateway.ts) calls `callGateway(... expectFinal: false)` and exits after `agent.subscribe` response; it does not remain attached for live events. +- `agent.subscribe` clients still receive global `agent` broadcasts unless filtered client-side; without filtering, per-session subscribe children can ingest unrelated events and cause cross-session noise/duplication. +- Handoff/replay can duplicate already-buffered events unless consumers gate by `globalSeq` (`<= lastSeen` ignore). +- Long β€œwaiting for subagents” SSE windows in [apps/web/app/api/chat/stream/route.ts](/Users/kumareth/Documents/projects/openclaw/apps/web/app/api/chat/stream/route.ts) have no keepalive signal, increasing disconnect risk during quiet periods. + +## Revised implementation sequence + +1. **Stabilize subscribe transport semantics first** + +- Rework subscribe mode in [src/commands/agent-via-gateway.ts](/Users/kumareth/Documents/projects/openclaw/src/commands/agent-via-gateway.ts) to use a long-lived gateway client session (not one-shot `callGateway`) that: + - connects, + - sends `agent.subscribe { sessionKey, afterSeq }`, + - streams events until SIGTERM/SIGINT, + - emits only matching `sessionKey` events, + - exits cleanly with `aborted` on signal. +- Add targeted tests for subscribe staying alive and session-key filtering. + +2. **Add reusable CLI subscribe spawner** + +- In [apps/web/lib/agent-runner.ts](/Users/kumareth/Documents/projects/openclaw/apps/web/lib/agent-runner.ts), add `spawnAgentSubscribeProcess(sessionKey, afterSeq)` using: + - `node agent --stream-json --subscribe-session-key --after-seq ` + - same profile/workspace env wiring as `spawnAgentProcess`. + +3. **Replace parent waiting flow with subscribe child process** + +- In [apps/web/lib/active-runs.ts](/Users/kumareth/Documents/projects/openclaw/apps/web/lib/active-runs.ts): + - replace `subscribeToSessionKey(...)` usage with a managed subscribe child, + - parse NDJSON from subscribe child and route through existing parent event processor, + - dedupe using `globalSeq` (drop stale/replayed duplicates), + - store/cleanup process handle across finalize/abort/cleanup. + +4. **Replace subagent fallback/rehydration with subscribe child process** + +- In [apps/web/lib/subagent-runs.ts](/Users/kumareth/Documents/projects/openclaw/apps/web/lib/subagent-runs.ts): + - swap `subscribeToSessionKey(...)` for one managed subscribe child per running subagent session, + - feed NDJSON into existing `routeRawEvent`/transform path, + - use `lastGlobalSeq` dedupe and robust teardown on completion/error/cleanup. + +5. **Retire direct web WS client** + +- Remove [apps/web/lib/gateway-events.ts](/Users/kumareth/Documents/projects/openclaw/apps/web/lib/gateway-events.ts) imports/usages from web runtime. +- Delete file only after all references are gone and typecheck passes. + +6. **Long-wait stream resilience** + +- Add lightweight SSE keepalive comments/events while run status is `waiting-for-subagents` in [apps/web/app/api/chat/stream/route.ts](/Users/kumareth/Documents/projects/openclaw/apps/web/app/api/chat/stream/route.ts) or run subscription layer, so idle waits don’t silently time out. + +7. **Verification gates** + +- Run targeted checks for: + - parent run -> subagent spawn -> parent wait -> announcement turn -> finalize, + - page refresh during parent wait, + - page refresh during subagent live stream, + - no cross-session event bleed, + - no duplicate tool/lifecycle events after replay handoff. + +## Flow target + +```mermaid +flowchart TD + webRun[WebRunManager] --> parentCli[agent --stream-json main run] + parentCli --> ndjsonParent[Parent NDJSON events] + parentCli -->|parent exits while subagents running| waitState[waitingForSubagents] + waitState --> subscribeCliParent[agent --stream-json subscribe parentSessionKey] + subscribeCliParent --> ndjsonReplayParent[ReplayedPlusLive NDJSON] + subagentMgr[SubagentRunManager] --> subscribeCliSub[agent --stream-json subscribe subagentSessionKey] + subscribeCliSub --> ndjsonSub[Subagent NDJSON] + ndjsonReplayParent --> sse[API chat stream SSE] + ndjsonSub --> sse +``` diff --git a/.cursor/plans/dench_filesystem_crm_integration.plan.md b/.cursor/plans/dench_filesystem_crm_integration.plan.md new file mode 100644 index 00000000000..d2f3698bd23 --- /dev/null +++ b/.cursor/plans/dench_filesystem_crm_integration.plan.md @@ -0,0 +1,665 @@ +--- +name: Dench Filesystem CRM Integration +overview: Replace Dench's slow tool-by-tool CRM agent with a filesystem-first architecture where OpenClaw manages a DuckDB database and markdown documents in a dedicated workspace folder, synced via S3, and surfaced in Dench's sidebar. +todos: + - id: inject-skill-infra + content: "Phase 1: Add `inject: true` skill metadata support to OpenClaw (types.ts, workspace.ts, system-prompt.ts, run.ts)" + status: pending + - id: dench-skill + content: "Phase 2: Create skills/dench/SKILL.md with DuckDB schema, SQL patterns, CRM patterns, document management instructions" + status: pending + - id: s3-sync-script + content: "Phase 3: Create S3 sync script and sandbox startup hook for dench workspace persistence" + status: pending + - id: dench-sidebar + content: "Phase 4 (DEFERRED): Add workspace data source to Dench sidebar (tRPC endpoints + buildKnowledgeTree merge)" + status: pending + - id: lambda-sync + content: "Phase 4b (DEFERRED): S3-to-PostgreSQL Lambda sync for fast sidebar queries (or simpler: direct DuckDB read from S3)" + status: pending +isProject: false +--- + +# Dench Filesystem-First CRM via OpenClaw + +## Architecture Overview + +The current Dench CRM agent uses 15+ individual tRPC-backed tools (createObjectTool, createFieldTool, createEntryTool, searchEntriesTool, etc.) that each make a Prisma call. This is slow -- the agent often needs 10+ tool calls for a single user request. + +The new architecture gives OpenClaw a `dench/` workspace folder with a DuckDB database and markdown files. The agent generates SQL directly via `exec` (duckdb CLI) and writes documents as `.md` files. S3 syncs this data between sandbox sessions, and Dench's sidebar reads from S3/PostgreSQL. + +**Why DuckDB over SQLite:** + +- Native PIVOT/UNPIVOT -- essential for the EAV (Entity-Attribute-Value) pattern used by custom fields; every "show me entries as a table" query needs pivot +- PostgreSQL-compatible SQL dialect -- matches Dench's Supabase/Postgres, so generated SQL is portable +- Native JSON type -- clean handling of `enum_values`, `enum_colors`, field mappings (no JSON1 extension needed) +- Built-in CSV/Parquet import/export with auto-detection -- bulk CRM operations ("import 500 leads from CSV") +- FTS extension -- full-text search across entry fields +- `generate_series` + macros -- nanoid 32 generation in pure SQL +- ~50-100ms startup overhead is acceptable given agent tool-call overhead is already 200-500ms + +```mermaid +flowchart TB + subgraph sandbox [OpenClaw Sandbox] + agent[OpenClaw Agent] + skill[Dench Skill - always in context] + ctx[dench/workspace_context.yaml] + db[dench/workspace.duckdb] + docs[dench/documents/*.md] + agent --> skill + agent -->|"read-only context"| ctx + agent -->|"exec: duckdb"| db + agent -->|"write/edit"| docs + end + + subgraph s3layer [Persistence Layer] + s3["S3: dench/{orgId}/"] + end + + subgraph denchApp [Dench Web App] + sidebar[App Sidebar] + api[tRPC API] + pg[PostgreSQL - fs_files sync] + end + + db -->|sync on save| s3 + docs -->|sync on save| s3 + s3 -->|Lambda trigger| pg + pg --> api --> sidebar + s3 -->|download on sandbox start| db + s3 -->|download on sandbox start| docs +``` + +--- + +## Phase 1: Always-In-Context Skill Infrastructure (OpenClaw) + +Currently, all skills are lazy-loaded: only name + description appear in the system prompt, and the agent must `read` the SKILL.md to get full instructions. For the Dench CRM skill, we need the full content injected automatically. + +**Approach:** Add an `inject: true` flag to skill metadata. When set, the skill's full content is included in the system prompt alongside bootstrap files (in the "Project Context" section), not in the lazy-loaded skills list. + +**Files to modify:** + +- [src/agents/skills/types.ts](src/agents/skills/types.ts) -- Add `inject?: boolean` to `OpenClawSkillMetadata` +- [src/agents/skills/workspace.ts](src/agents/skills/workspace.ts) -- In `buildWorkspaceSkillSnapshot()`, separate injected skills from lazy-loaded skills. Return injected skill contents alongside the prompt. +- [src/agents/system-prompt.ts](src/agents/system-prompt.ts) -- Accept injected skill content and include it in the "Project Context" / "Workspace Files" section (similar to how bootstrap files like AGENTS.md are included via `contextFiles`) +- [src/agents/pi-embedded-runner/run.ts](src/agents/pi-embedded-runner/run.ts) -- Pass injected skill content through to the system prompt builder + +**Key change in `buildWorkspaceSkillSnapshot`:** + +```typescript +// New return type addition +export type SkillSnapshot = { + prompt: string; // lazy-loaded skills prompt (XML) + injectedContent?: string; // always-in-context skill content (concatenated) + skills: Array<{ name: string; primaryEnv?: string }>; + // ... +}; +``` + +--- + +## Phase 2: Dench CRM Skill (OpenClaw) + +Create `skills/dench/SKILL.md` with: + +- Full DuckDB schema reference +- nanoid 32 macro for ID generation (matching Dench's Supabase nanoid IDs) +- SQL patterns for all CRUD operations including PIVOT for table views +- Document management instructions +- Workspace structure documentation +- CRM patterns (contact, lead, deal, etc.) ported from the [Dench CRM agent prompt](file:///Users/kumareth/Documents/projects/dench/src/lib/agents/crm-agent.ts) + +**New file:** `skills/dench/SKILL.md` + +**Skill frontmatter:** + +```yaml +--- +name: dench +description: Manage Dench CRM workspace - create objects, fields, entries via DuckDB and documents as markdown files +metadata: + openclaw: + inject: true + emoji: "πŸ“Š" +--- +``` + +**Workspace directory structure managed by the skill:** + +``` +~/.openclaw/workspace/dench/ + workspace_context.yaml # READ-ONLY context (org, members, integrations, defaults) + workspace.duckdb # DuckDB database (CRM data) + documents/ # Markdown documents (nested by path) + getting-started.md + projects/ + project-alpha.md + exports/ # Generated CSV/Excel exports + WORKSPACE.md # Auto-generated schema documentation +``` + +**workspace_context.yaml** -- read-only context the agent consumes on startup (written by Dench, never by the agent): + +```yaml +# Dench Workspace Context (READ-ONLY) +# This file is generated by Dench and synced via S3. +# The agent reads this for organizational context but MUST NOT modify it. +# Changes flow from Dench UI -> S3 -> this file (on sandbox init). + +workspace: + version: 1 + +# Organization identity (synced from Dench on sandbox init) +organization: + id: "org_abc123" + name: "Acme Corp" + slug: "acme-corp" + business: + name: "Acme Corporation" + type: "saas" # saas, agency, ecommerce, services, etc. + industry: "Technology" + website: "https://acme.com" + +# Team members -- needed for "user" type fields (e.g. "Assigned To") +# Agent uses these IDs when creating entries with user-type fields. +members: + - id: "usr_abc123" + name: "John Doe" + email: "john@acme.com" + role: owner + - id: "usr_def456" + name: "Jane Smith" + email: "jane@acme.com" + role: admin + - id: "usr_ghi789" + name: "Bob Wilson" + email: "bob@acme.com" + role: member + +# Protected objects -- cannot be deleted/renamed by agent +# Mirrors Dench's base-objects.ts immutable list +protected_objects: + - name: "people" + description: "Contact records" + icon: "users" + - name: "companies" + description: "Company records" + icon: "building-2" + +# Connected integrations -- agent reads this for sync context +# Populated by Dench when sandbox initializes from S3 +integrations: + connections: [] + # Example when connected: + # - app_key: "salesforce" + # app_name: "Salesforce" + # connection_id: "conn_xyz" + # synced_objects: + # - external_resource: "Lead" + # local_object: "lead" + # sync_direction: bidirectional # import, export, bidirectional + # sync_frequency: hourly # realtime, hourly, daily, manual + # field_mappings: + # "FirstName": "Full Name" + # "Email": "Email Address" + +# Enrichment configuration +enrichment: + enabled: false + provider: "aviato" # aviato, apollo + auto_enrich: false # Auto-enrich new entries on creation + +# CRM defaults +defaults: + default_view: table # table, kanban + date_format: "YYYY-MM-DD" + naming_convention: singular_lowercase # Object names: "lead" not "Leads" + +# S3 persistence +sync: + s3_bucket: "dench-workspaces" + s3_prefix: "" # Set to org_id on init + frequency: on_write # on_write, manual + last_synced_at: null + +# Credit account (for enrichment, AI operations) +credits: + allowance_balance: 0 + topup_balance: 0 +``` + +**Why YAML for context (not DuckDB):** This is read-only context the agent consumes once at startup -- never writes. The agent can `cat workspace_context.yaml` to understand the full org context instantly. Members list means no separate query to resolve user-type field assignments. Integrations give awareness of sync relationships without querying external APIs. This follows the Fintool pattern where user state (preferences, watchlists) lives as YAML in S3 while dense queryable data lives in the database. The data flow is one-way: Dench UI -> S3 -> workspace_context.yaml (on sandbox init). The agent never writes back to this file. + +**DuckDB schema** (initialized by agent on first use via `duckdb dench/workspace.duckdb`): + +```sql +-- nanoid 32 macro: generates 32-char IDs matching Dench's Supabase nanoid format +CREATE OR REPLACE MACRO nanoid32() AS ( + SELECT string_agg( + substr('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_-', + (floor(random() * 64) + 1)::int, 1), '') + FROM generate_series(1, 32) +); + +CREATE TABLE IF NOT EXISTS objects ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + name VARCHAR NOT NULL, + description VARCHAR, + icon VARCHAR, + default_view VARCHAR DEFAULT 'table', -- 'table' or 'kanban' + parent_document_id VARCHAR, + sort_order INTEGER DEFAULT 0, + source_app VARCHAR, + immutable BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(name) +); + +CREATE TABLE IF NOT EXISTS fields ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + object_id VARCHAR NOT NULL REFERENCES objects(id) ON DELETE CASCADE, + name VARCHAR NOT NULL, + description VARCHAR, + type VARCHAR NOT NULL, -- text, number, email, phone, boolean, date, richtext, user, relation, enum + required BOOLEAN DEFAULT false, + default_value VARCHAR, + related_object_id VARCHAR REFERENCES objects(id) ON DELETE SET NULL, + relationship_type VARCHAR, -- one_to_one, one_to_many, many_to_one, many_to_many + enum_values JSON, -- ["New","In Progress","Done"] + enum_colors JSON, -- ["#ef4444","#f59e0b","#22c55e"] + enum_multiple BOOLEAN DEFAULT false, + sort_order INTEGER DEFAULT 0, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(object_id, name) +); + +CREATE TABLE IF NOT EXISTS entries ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + object_id VARCHAR NOT NULL REFERENCES objects(id) ON DELETE CASCADE, + sort_order INTEGER DEFAULT 0, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS entry_fields ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + entry_id VARCHAR NOT NULL REFERENCES entries(id) ON DELETE CASCADE, + field_id VARCHAR NOT NULL REFERENCES fields(id) ON DELETE CASCADE, + value VARCHAR, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(entry_id, field_id) +); + +CREATE TABLE IF NOT EXISTS statuses ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + object_id VARCHAR NOT NULL REFERENCES objects(id) ON DELETE CASCADE, + name VARCHAR NOT NULL, + color VARCHAR DEFAULT '#94a3b8', + sort_order INTEGER DEFAULT 0, + is_default BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(object_id, name) +); + +CREATE TABLE IF NOT EXISTS documents ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + title VARCHAR DEFAULT 'Untitled', + icon VARCHAR, + cover_image VARCHAR, + file_path VARCHAR NOT NULL UNIQUE, -- relative path in documents/ dir + parent_id VARCHAR REFERENCES documents(id) ON DELETE CASCADE, + parent_object_id VARCHAR REFERENCES objects(id) ON DELETE CASCADE, + sort_order INTEGER DEFAULT 0, + is_published BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +-- Full-text search index (DuckDB FTS extension) +INSTALL fts; LOAD fts; +``` + +**Auto-generated views** -- after every object/field mutation, the agent regenerates a PIVOT view for each object. These are stored queries (zero data duplication) that make the EAV pattern invisible: + +```sql +-- Auto-generated after creating the "leads" object and its fields +CREATE OR REPLACE VIEW v_leads AS +PIVOT ( + SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = (SELECT id FROM objects WHERE name = 'leads') +) ON field_name USING first(value); + +-- Now query like a normal table: +SELECT * FROM v_leads WHERE "Status" = 'New' ORDER BY "Full Name" LIMIT 50; +SELECT "Status", COUNT(*) FROM v_leads GROUP BY "Status"; +SELECT * FROM v_leads WHERE "Email Address" LIKE '%@gmail.com'; +``` + +Views are regenerated (not data, just the query definition) whenever fields are added/removed/renamed. Naming convention: `v_{object_name}` (e.g., `v_leads`, `v_companies`, `v_people`). + +**Filesystem directory structure** -- auto-projected from DuckDB after schema mutations. Represents the sidebar's nested knowledge tree. NO entry data in the filesystem (DuckDB is sole source of truth for entries): + +``` +dench/ + workspace.duckdb # SOLE source of truth for all structured data + views + workspace_context.yaml # Read-only org context + knowledge/ # Root of knowledge tree (= sidebar root) + people/ # Object "people" (directory = object node in sidebar) + .object.yaml # Lightweight metadata projection (id, icon, view, field list) + onboarding-guide.md # Document nested UNDER the people object + companies/ # Object "companies" + .object.yaml + projects/ # Document "Projects" (directory with children) + projects.md # Document content + tasks/ # Object nested UNDER the projects document + .object.yaml + roadmap.md # Sibling document + sales/ + sales.md + leads/ # Object nested under sales document + .object.yaml + follow-up-playbook.md # Document nested under leads object + exports/ # CSV/Parquet exports (on-demand, not auto-generated) + WORKSPACE.md # Auto-generated schema summary +``` + +**Source of truth rules:** + +- **Entries (rows)**: DuckDB ONLY (queried via `v_{object}` views). Never duplicated to filesystem. +- **Fields (columns)**: DuckDB. Summary projected to `.object.yaml` (read-only). +- **Objects (tables)**: DuckDB. Projected as directories + `.object.yaml` (read-only). Queryable via auto-generated views. +- **Document metadata**: DuckDB. Projected as directory structure (read-only). +- **Document content**: Filesystem (`.md` files). DuckDB stores `file_path` reference only. +- **Nesting/ordering**: DuckDB. Projected as directory hierarchy (read-only). + +**Key DuckDB advantages leveraged:** + +- **PIVOT views**: Auto-generated `v_{object}` views make EAV invisible -- query like normal tables +- **Native JSON**: `enum_values` and `enum_colors` are native JSON columns, no string parsing needed +- **CSV import**: `COPY v_leads TO 'exports/leads.csv';` or import with `COPY ... FROM 'import.csv' (AUTO_DETECT true);` +- **PostgreSQL dialect**: Generated SQL is directly portable to Dench's Supabase/Postgres + +**Skill content structure** -- the full SKILL.md incorporates and adapts every section from Dench's existing CRM agent prompt ([src/lib/agents/crm-agent.ts](file:///Users/kumareth/Documents/projects/dench/src/lib/agents/crm-agent.ts)), rewritten for DuckDB/filesystem execution instead of tool calls: + +### Section 1: Role and Workspace Startup + +- Role: Dench CRM Management Agent operating via DuckDB and filesystem +- On every conversation: read `dench/workspace_context.yaml` (READ-ONLY) for org context, members, integrations, protected objects +- Initialize DuckDB if not exists: `duckdb dench/workspace.duckdb < schema.sql` +- Database path: `dench/workspace.duckdb`, documents path: `dench/documents/` + +### Section 2: Primary Responsibilities (adapted from ``) + +- **Request analysis**: Same as original -- extract intent, identify entities/objects/fields/relationships, transform vague requests into structured SQL +- **Object creation**: Instead of `createObjectTool`, generate `INSERT INTO objects` SQL. Naming convention: singular, lowercase (e.g., "lead", "customer"). Check existing with `SELECT` first. For kanban: auto-create Status (enum) and Assigned To (user) fields in same transaction +- **Field management**: Instead of `createFieldTool`, generate `INSERT INTO fields` SQL. Field types: text, number, email, phone, boolean, date, richtext, user, relation, enum. Use `INSERT ... ON CONFLICT (object_id, name) DO UPDATE` for idempotency +- **Entry creation**: Instead of `createEntryTool`, generate `INSERT INTO entries` + `INSERT INTO entry_fields` in a transaction. Resolve field names to field IDs via `SELECT id FROM fields WHERE object_id = ? AND name = ?` +- **Entry search**: Instead of `searchEntriesTool`, generate PIVOT queries with WHERE/LIKE/ORDER BY. Use DuckDB FTS for full-text search. Operators: `=` (equals), `LIKE '%...%'` (contains), `LIKE '...%'` (startsWith), `LIKE '%...'` (endsWith), `IS NULL` (isEmpty), `IS NOT NULL` (isNotEmpty) + +### Section 3: SQL Operation Guide (replaces ``) + +Each former tool maps to SQL patterns: + +- **createObjectTool -> INSERT object**: `INSERT INTO objects (name, description, icon, default_view) VALUES (...) ON CONFLICT (name) DO NOTHING RETURNING *;` +- **createFieldTool -> INSERT field**: `INSERT INTO fields (object_id, name, type, required, enum_values, enum_colors, related_object_id, relationship_type, sort_order) VALUES (...) ON CONFLICT (object_id, name) DO UPDATE SET ...;` +- **createEntryTool -> INSERT entry + entry_fields**: Transaction with `INSERT INTO entries` then `INSERT INTO entry_fields` for each field value +- **getObjectTool -> SELECT object + fields**: `SELECT o.*, json_group_array(json_object('id', f.id, 'name', f.name, 'type', f.type)) as fields FROM objects o LEFT JOIN fields f ON f.object_id = o.id WHERE o.id = ? GROUP BY o.id;` +- **getObjectsTool -> SELECT all objects**: `SELECT o.*, COUNT(e.id) as entry_count FROM objects o LEFT JOIN entries e ON e.object_id = o.id GROUP BY o.id ORDER BY o.sort_order;` +- **getOrganizationMembersTool -> READ workspace_context.yaml**: Members list is in `workspace_context.yaml` under `members:`. Read with `cat dench/workspace_context.yaml` and extract the members section. User fields store member IDs like `usr_abc123`. +- **searchEntriesTool -> query the auto-generated view**: + ```sql + -- Simple: query the pre-built PIVOT view like a normal table + SELECT * FROM v_leads WHERE "Status" = 'New' ORDER BY created_at DESC LIMIT 50; + SELECT * FROM v_leads WHERE "Email Address" LIKE '%@gmail.com'; + SELECT * FROM v_leads WHERE "Full Name" ILIKE '%john%'; + SELECT "Status", COUNT(*) FROM v_leads GROUP BY "Status"; + ``` + Views are auto-generated per object (`v_{object_name}`) so the agent never writes raw PIVOT queries for reads. +- **updateObjectTool -> UPDATE object**: `UPDATE objects SET name = ?, description = ?, updated_at = now() WHERE id = ?;` +- **updateFieldTool -> UPDATE field**: `UPDATE fields SET ... WHERE id = ?;` +- **updateEntryTool -> UPSERT entry_fields**: `INSERT INTO entry_fields (entry_id, field_id, value) VALUES (?, ?, ?) ON CONFLICT (entry_id, field_id) DO UPDATE SET value = excluded.value, updated_at = now();` +- **deleteObjectTool -> DELETE cascade**: `DELETE FROM objects WHERE id = ? AND immutable = false;` (cascades to fields, entries, entry_fields via FK) +- **deleteFieldTool -> DELETE field**: `DELETE FROM fields WHERE id = ?;` (cascades to entry_fields) +- **deleteEntryTool -> DELETE entry**: `DELETE FROM entries WHERE id = ?;` (cascades to entry_fields) +- **createManyEntriesTool -> Batch INSERT**: Wrap multiple entry+entry_fields inserts in `BEGIN TRANSACTION; ... COMMIT;` +- **Bulk import from CSV**: `COPY ... FROM 'import.csv' (AUTO_DETECT true);` with field mapping + +### Section 4: Execution Workflows (adapted from ``) + +Same workflow selection principle: choose the minimal workflow needed based on user intent. + +- **Create New CRM Structure**: `SELECT` to check existence -> `INSERT objects` -> `INSERT fields` (all in one `exec` call with multi-statement SQL in a transaction) +- **Search and Display**: Generate PIVOT query with appropriate WHERE/ORDER BY/LIMIT +- **Add New Entries**: `SELECT` object+fields -> `INSERT entries` + `INSERT entry_fields` in transaction +- **Update Existing Data**: PIVOT query to find -> `UPDATE` entry_fields +- **Quick Information**: `SELECT` with aggregate counts +- **Bulk Operations**: Multi-row INSERT in transaction, report counts +- **Data Cleanup**: PIVOT query with filters -> `DELETE` matching entries + +Key differences from original: + +1. **All steps happen in a single `exec` call** with multi-statement SQL, not 10+ separate tool calls +2. **After schema mutations**: regenerate the `v_{object}` view and project the filesystem directory structure +3. **Reads use views**: `SELECT * FROM v_leads` instead of raw PIVOT queries + +Example of creating a full CRM structure in one shot: + +```sql +BEGIN TRANSACTION; +INSERT INTO objects (name, description, icon, default_view) VALUES ('lead', 'Sales leads', 'user-plus', 'table') ON CONFLICT (name) DO NOTHING; +INSERT INTO fields (object_id, name, type, required, sort_order) VALUES + ((SELECT id FROM objects WHERE name = 'lead'), 'Full Name', 'text', true, 0), + ((SELECT id FROM objects WHERE name = 'lead'), 'Email Address', 'email', true, 1), + ((SELECT id FROM objects WHERE name = 'lead'), 'Phone Number', 'phone', false, 2) +ON CONFLICT (object_id, name) DO NOTHING; +INSERT INTO fields (object_id, name, type, enum_values, enum_colors, sort_order) VALUES + ((SELECT id FROM objects WHERE name = 'lead'), 'Status', 'enum', + '["New","Contacted","Qualified","Converted"]'::JSON, + '["#94a3b8","#3b82f6","#f59e0b","#22c55e"]'::JSON, 3) +ON CONFLICT (object_id, name) DO NOTHING; + +-- Auto-generate the PIVOT view for this object +CREATE OR REPLACE VIEW v_lead AS +PIVOT ( + SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = (SELECT id FROM objects WHERE name = 'lead') +) ON field_name USING first(value); + +COMMIT; +-- Then: project filesystem structure (mkdir knowledge/lead/, write .object.yaml) +``` + +### Section 5: CRM Patterns (adapted from ``) + +Identical patterns, but with SQL examples instead of tool call examples: + +- **Contact/Customer**: Full Name (text, required), Email Address (email, required), Phone Number (phone), Company (relation), Notes (richtext) +- **Lead/Prospect**: Full Name, Email, Phone, Status (enum: New/Contacted/Qualified/Converted), Source (enum), Score (number), Assigned To (user), Notes (richtext) +- **Company/Organization**: Company Name (text, required), Industry (enum), Website (text), Type (enum), Notes (richtext) +- **Deal/Opportunity**: Deal Name, Amount (number), Stage (enum: Discovery/Proposal/Negotiation/Closed Won/Closed Lost), Close Date (date), Probability (number), Primary Contact (relation), Assigned To (user), Notes (richtext) +- **Case/Project**: Case Number, Title, Client (relation), Status (enum), Priority (enum), Due Date (date), Assigned To (user), Notes (richtext) +- **Property/Asset**: Address, Property Type (enum), Price (number), Status (enum), Square Footage (number), Bedrooms (number), Notes (richtext) +- **Task/Activity** (kanban): Title, Description, Assigned To (user), Due Date, Status (enum: In Queue/In Progress/Done), Priority (enum), Notes (richtext). Use `default_view = 'kanban'` and auto-create Status + Assigned To fields. + +### Section 6: Field Type Reference (adapted from ``) + +- **text**: General text data, names, descriptions, addresses. Stored as VARCHAR. +- **email**: Email addresses. Stored as VARCHAR. Agent validates format. +- **phone**: Phone numbers. Stored as VARCHAR. Agent normalizes format. +- **number**: Numeric values (prices, quantities, scores). Stored as VARCHAR in entry_fields (EAV), cast with `::NUMERIC` in queries. +- **boolean**: Yes/no flags. Stored as "true"/"false" strings. +- **date**: Dates. Stored as ISO 8601 strings. Cast with `::DATE` in queries. +- **richtext**: Rich text for Notes fields. Content stored as entry_field value (plain text / markdown). Displayed in Notion-style editor in Dench UI. +- **user**: Person/assignee fields. Stores member ID (e.g., "usr_abc123") from `workspace_context.yaml` members list. ALWAYS resolve member name -> ID before inserting. +- **enum**: Dropdown/select fields. Field definition stores `enum_values` as JSON array. Entry stores the selected value string. `enum_colors` parallel array for styling. `enum_multiple = true` for multi-select (value stored as JSON array string). +- **relation**: Links to entries in another object. Field stores `related_object_id` and `relationship_type`. Entry stores the related entry ID(s). `many_to_one` for single select, `many_to_many` for multi-select (stored as JSON array). + +### Section 7: Naming Conventions and Data Handling (adapted from `` and ``) + +- Object names: singular, lowercase, one word ("lead" not "Leads") +- Field names: human-readable, proper capitalization ("Email Address" not "email") +- Be descriptive: "Phone Number" not "Phone" +- Be consistent within an object +- Validate email formats, normalize phone numbers, use ISO 8601 dates +- Trim whitespace from all values +- Check for duplicates before creating entries (SELECT before INSERT) + +### Section 8: Error Handling (adapted from ``) + +- `UNIQUE constraint` on INSERT -> item already exists, treat as SUCCESS (use `ON CONFLICT DO NOTHING` or `DO UPDATE`) +- Protected object deletion -> check `immutable` column and `protected_objects` in workspace_context.yaml +- Field type mismatch -> warn user before changing type on field with existing data +- Missing required fields -> validate before INSERT, report which fields are missing + +### Section 9: Document Management (new -- not in original CRM agent) + +- Create document: `write` tool to create `dench/documents/.md` + `INSERT INTO documents` with metadata +- Document content is the .md file; DuckDB `documents` table tracks metadata (title, icon, nesting, order) +- Cross-nesting: documents under objects (`parent_object_id`), objects under documents (`parent_document_id`) +- Document tree mirrors filesystem: `dench/documents/projects/alpha.md` -> `file_path = 'projects/alpha.md'` + +### Section 10: Protected Objects and Read-Only Context (new) + +- Read `protected_objects` from `workspace_context.yaml` on startup +- NEVER delete, rename, or modify immutable objects (People, Companies) +- NEVER modify `workspace_context.yaml` -- it is read-only context from Dench +- Members list is authoritative for user-type field resolution + +### Section 11: Post-Mutation Checklist (MANDATORY -- revised after agent testing) + +**Problem identified:** In testing, the agent correctly executed SQL (object + fields + entries) but skipped the filesystem projection (.object.yaml) and sometimes the PIVOT view. Root cause: the original skill mentioned these as afterthoughts ("Then project the filesystem...") with no concrete template or examples. The agent follows examples literally -- if examples only show SQL, it only does SQL. + +**Fix:** Every workflow example now uses an explicit 3-step structure. The post-mutation section is now a checklist, not a description. + +After creating/modifying an OBJECT or FIELDS: + +- `CREATE OR REPLACE VIEW v_{object_name}` -- regenerate PIVOT view +- `mkdir -p dench/knowledge/{object_name}/` -- create directory +- Write `.object.yaml` with id, name, description, icon, default_view, entry_count, and full field list +- Update WORKSPACE.md + +After adding ENTRIES: + +- Update `entry_count` in `.object.yaml` +- Verify: `SELECT * FROM v_{object} LIMIT 5` + +After deleting an OBJECT: + +- `DROP VIEW IF EXISTS v_{object_name}` +- `rm -rf dench/knowledge/{object_name}/` + +The skill now includes: + +- A concrete `.object.yaml` template with example content (previously missing entirely) +- Full bash commands for generating `.object.yaml` from DuckDB queries +- "Step 1 / Step 2 / Step 3" structure in every workflow example (SQL, Filesystem, Verify) +- Critical Reminders section leads with "NEVER SKIP FILESYSTEM PROJECTION" and "THREE STEPS, EVERY TIME" + +### Section 12: Critical Reminders (adapted from ``) + +- Handle the ENTIRE CRM operation from analysis to SQL execution **to filesystem projection** to summary +- **NEVER SKIP FILESYSTEM PROJECTION**: After any object mutation, create/update `.object.yaml` AND the `v_{object}` view. If missing, the object is invisible in the sidebar. +- **THREE STEPS, EVERY TIME**: (1) SQL transaction, (2) filesystem projection, (3) verify +- Always check existing data before creating (SELECT before INSERT, or ON CONFLICT) +- Search proactively to provide better UX (PIVOT with filters) +- Never assume field names -- always verify with `SELECT * FROM fields WHERE object_id = ?` +- Extract ALL data from user messages +- NOTES FIELDS: type "richtext", displayed in Notion editor +- USER FIELDS: Resolve member name to ID from workspace_context.yaml BEFORE inserting +- ENUM FIELDS: type "enum" with `enum_values` JSON array +- RELATION FIELDS: type "relation" with `related_object_id` +- KANBAN BOARDS: `default_view = 'kanban'`, auto-create Status and Assigned To fields +- PROTECTED OBJECTS: Never delete objects listed in `workspace_context.yaml` `protected_objects` +- ONE EXEC CALL: Batch related SQL in a single transaction whenever possible -- this is the entire point of the filesystem-first approach +- ENTRY COUNT: After adding entries, update `entry_count` in `.object.yaml` + +--- + +## Phase 3: S3 Sync Layer + +**Two sync directions:** + +1. **Sandbox -> S3:** After every database write or document save, sync changed files to S3 +2. **S3 -> Sandbox:** On sandbox startup, download the latest dench/ folder from S3 + +**Option A (simpler): Script-based sync** + +- Add a `dench/sync.sh` script that the agent calls after mutations +- Uses AWS CLI: `aws s3 sync dench/ s3://dench-workspaces/{orgId}/` +- Credentials injected via ABAC (short-lived, scoped to org prefix) +- Skill instructs the agent to run sync after writes + +**Option B (more robust): inotifywait + background sync** + +- Background process watches dench/ for changes +- Debounced sync to S3 (e.g., 5s after last change) +- More reliable but more infrastructure + +**Recommendation:** Start with Option A. The agent is already making exec calls; one more `aws s3 sync` is trivial. + +**New file:** `skills/dench/sync.sh` (bundled with the skill, referenced by SKILL.md) + +**Sandbox startup hook:** + +- Add to OpenClaw's sandbox initialization: download dench workspace from S3 if it exists +- Could be a `BOOTSTRAP.md` instruction or a sandbox pre-warm step + +--- + +## Phase 4: Dench Sidebar Integration + +The Dench sidebar currently reads from Prisma (`getAll` objects + `getAll` documents). It needs a new data source for workspace-managed data. + +**Approach:** Add a new tRPC endpoint that reads from S3 (or the synced PostgreSQL `fs_files` table) and returns the same tree structure the sidebar expects. + +**Files to modify in Dench:** + +- [src/lib/trpc/routers/objects.ts](file:///Users/kumareth/Documents/projects/dench/src/lib/trpc/routers/objects.ts) -- Add `getWorkspaceObjects` endpoint that queries the synced SQLite data (either by reading dench.db from S3, or from a PostgreSQL materialized view) +- [src/lib/trpc/routers/documents.ts](file:///Users/kumareth/Documents/projects/dench/src/lib/trpc/routers/documents.ts) -- Add `getWorkspaceDocuments` endpoint +- [src/components/app-sidebar.tsx](file:///Users/kumareth/Documents/projects/dench/src/components/app-sidebar.tsx) -- Merge workspace data into the `buildKnowledgeTree` function + +**S3 -> PostgreSQL sync (Lambda):** + +- When workspace.duckdb is uploaded to S3, a Lambda function: + 1. Downloads the DuckDB file + 2. Reads objects, fields, entries, documents tables + 3. Upserts into PostgreSQL tables (or a dedicated `workspace_objects`, `workspace_documents` table) +- This mirrors the Fintool pattern: S3 source of truth, Lambda sync, PG for fast queries + +**Alternative (simpler for v1):** Dench backend downloads workspace.duckdb from S3 on demand and queries it directly using `duckdb` Node.js bindings (`@duckdb/node-api`). No Lambda needed initially. + +--- + +## Phase 5: Real-Time Updates (Future) + +- WebSocket notifications when S3 objects change (via S3 Event Notifications -> SNS -> WebSocket) +- Dench sidebar auto-refreshes when workspace data changes +- Bidirectional: Dench UI edits write back to S3, agent picks up changes on next sandbox start + +--- + +## Key Design Decisions + +- **Three-layer storage** (following Fintool pattern): + - **YAML** (`workspace_context.yaml`): Read-only workspace identity, team members, integrations, defaults. Written by Dench, consumed by agent. Never modified by the agent -- data flows one-way from Dench UI -> S3 -> this file. + - **DuckDB** (`workspace.duckdb`): Dense, relational CRM data (objects, fields, entries, statuses). Queryable via SQL with PIVOT, JOIN, FTS. + - **Markdown** (`documents/*.md`): Rich document content. Agent uses `write`/`edit` tools directly. DuckDB tracks metadata (title, icon, nesting) while the file system holds the content. +- **DuckDB over SQLite**: Native PIVOT/UNPIVOT is essential for the EAV data model (rendering custom-field entries as tables). PostgreSQL-compatible SQL dialect means generated SQL is portable to Dench's Supabase/Postgres. Native JSON type eliminates string parsing for enum_values/colors. Built-in CSV/Parquet import with auto-detection enables bulk CRM operations. FTS extension provides full-text search. ~50-100ms startup overhead is acceptable. +- **nanoid 32 IDs**: Matches Dench's Supabase PostgreSQL nanoid format. Generated via a DuckDB macro using `generate_series` + `random()` over the standard nanoid alphabet (`0-9A-Za-z_-`). 32 chars provides 192 bits of entropy. +- **Members in YAML not DuckDB**: The agent needs member IDs for "user" type fields (like "Assigned To"). Putting members in workspace_context.yaml means the agent reads them once on startup without a separate SQL query. The list changes infrequently (team changes, not per-request). Agent reads only, never writes. +- **Integrations in YAML not DuckDB**: The agent needs to know what apps are connected and how fields map, but doesn't need to query this relationally. YAML gives the agent instant context about sync relationships. Agent reads only, never writes. +- **Skill inject vs bootstrap file**: Using skill metadata `inject: true` is cleaner than adding a new bootstrap file type. It keeps the Dench instructions modular and version-controlled in the skills directory. +- **One transaction per operation**: The skill instructs the agent to wrap multi-step operations in `BEGIN ... COMMIT` for atomicity. DuckDB supports full ACID transactions. diff --git a/.cursor/plans/file_chat_sidebar_368973cb.plan.md b/.cursor/plans/file_chat_sidebar_368973cb.plan.md new file mode 100644 index 00000000000..a5e14610732 --- /dev/null +++ b/.cursor/plans/file_chat_sidebar_368973cb.plan.md @@ -0,0 +1,138 @@ +--- +name: File chat sidebar +overview: Add a collapsible chat sidebar to the workspace file view that lets users chat with the agent about the currently open file, with file-scoped sessions stored separately from the main chat list. +todos: + - id: extract-chat-panel + content: Extract ChatPanel component from page.tsx with fileContext prop support + status: in_progress + - id: simplify-home-page + content: Simplify page.tsx to render Sidebar + ChatPanel + status: pending + - id: tag-file-sessions + content: Add filePath field to WebSessionMeta and filtering to GET /api/web-sessions + status: pending + - id: workspace-chat-sidebar + content: Add collapsible ChatPanel sidebar to workspace page with file context + status: pending + - id: live-reload + content: Re-fetch file content after agent finishes streaming to show edits live + status: pending +isProject: false +--- + +# File Chat Sidebar for Workspace + +## Architecture + +The workspace page layout changes from `[sidebar | content]` to `[sidebar | content | chat-panel]`. The chat panel reuses the same `useChat` + `ChatMessage` + session persistence logic from the main chat page. + +```mermaid +graph LR + subgraph workspace [Workspace Page Layout] + WS[WorkspaceSidebar_260px] --> MC[MainContent_flex1] + MC --> CP[ChatPanel_380px] + end + subgraph storage [Session Storage] + idx[index.json] -->|filePath field| fileScoped[File-scoped sessions] + idx -->|no filePath| globalSessions[Global sessions] + end + CP -->|POST /api/chat| agent[Agent Runner] + CP -->|file context in message| agent +``` + +## Step 1: Extract `ChatPanel` from `page.tsx` + +Extract the entire chat UI (messages list, input form, session management, `useChat`, streaming status) from [apps/web/app/page.tsx](apps/web/app/page.tsx) into a new reusable component: + +**New file:** `apps/web/app/components/chat-panel.tsx` + +```typescript +type ChatPanelProps = { + /** When set, scopes sessions to this file and prepends content as context */ + fileContext?: { path: string; content: string; filename: string }; + /** External session list (for sidebar session list) */ + sessions?: WebSessionMeta[]; + onSessionsChange?: () => void; + /** Compact mode for workspace sidebar (no lobster, smaller empty state) */ + compact?: boolean; +}; +``` + +- Internally uses `useChat` from `@ai-sdk/react` with `DefaultChatTransport` +- Manages its own `currentSessionId`, `savedMessageIdsRef`, `input`, etc. (same logic as `page.tsx`) +- When `fileContext` is provided: + - The first message in each session is prefixed with: `"[Context: file '{path}']\n\n{content}\n\n---\n\nUser question: {userText}"` -- subsequent messages just send `userText` as-is (the agent already has context from the conversation) + - Session creation passes `filePath` to the API +- Renders: header bar (with session title / status), scrollable message list using `ChatMessage`, error bar, input form + +Then **simplify `page.tsx**` to just: + +```tsx + + +``` + +## Step 2: Tag file sessions in web-sessions API + +Modify [apps/web/app/api/web-sessions/route.ts](apps/web/app/api/web-sessions/route.ts): + +- Add optional `filePath` to `WebSessionMeta` type +- `POST` accepts `filePath` in the body and stores it in the index +- `GET` accepts `?filePath=...` query param: + - If `filePath` is set: returns only sessions where `meta.filePath === filePath` + - If `filePath` is absent: returns only sessions where `meta.filePath` is falsy (excludes file-scoped sessions from the main list) + +This single change means the main chat sidebar automatically stops showing file sessions, with no other code changes needed. + +## Step 3: Add chat sidebar to workspace page + +Modify [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx): + +- Add a collapsible right panel (380px) that renders `ChatPanel` with `fileContext` +- The panel appears when a file or document is selected (content kinds: `"document"`, `"file"`) +- Add a toggle button in the breadcrumbs bar to show/hide the chat panel +- Pass the file's `content`, `path`, and `filename` as `fileContext` +- The panel shows a mini session list at top (file-scoped sessions only) and the chat below + +Layout becomes: + +``` ++------------------+------------------------+-------------------+ +| WorkspaceSidebar | Content (flex-1) | ChatPanel (380px) | +| (260px) | | [toggle] | ++------------------+------------------------+-------------------+ +``` + +## Step 4: File-scoped session list inside ChatPanel + +Inside `ChatPanel`, when `fileContext` is provided: + +- Fetch sessions filtered by `filePath` via `GET /api/web-sessions?filePath=...` +- Show a compact session list (just titles, clickable) above the messages area +- "New chat" button creates a new file-scoped session +- Selecting a session loads its messages (same logic as main page's `handleSessionSelect`) + +## Step 5: Live file reload after agent edits + +When the agent finishes streaming (status goes from `streaming` -> `ready`) and `fileContext` is provided: + +- Re-fetch the file content via `GET /api/workspace/file?path=...` +- Call a callback `onFileChanged?.(newContent)` so the workspace page can update `ContentState` without a full reload +- This makes edits appear live in the file viewer/document view next to the chat + +## Key files touched + +| File | Change | +| ---------------------------------------- | ------------------------------------ | +| `apps/web/app/components/chat-panel.tsx` | **New** -- extracted chat UI + logic | +| `apps/web/app/page.tsx` | Simplify to use `ChatPanel` | +| `apps/web/app/api/web-sessions/route.ts` | Add `filePath` field + filtering | +| `apps/web/app/workspace/page.tsx` | Add right chat sidebar with toggle | + +## What stays the same (reused as-is) + +- `ChatMessage` component -- no changes +- `ChainOfThought` component -- no changes +- `/api/chat` route + `agent-runner.ts` -- no changes (file context goes in the message text) +- `/api/web-sessions/[id]` and `/api/web-sessions/[id]/messages` routes -- no changes +- `useChat` from `@ai-sdk/react` -- same transport, same everything diff --git a/.cursor/plans/full_web_ui_redesign_9ad2e285.plan.md b/.cursor/plans/full_web_ui_redesign_9ad2e285.plan.md new file mode 100644 index 00000000000..afd0127c22f --- /dev/null +++ b/.cursor/plans/full_web_ui_redesign_9ad2e285.plan.md @@ -0,0 +1,324 @@ +--- +name: Full Web UI Redesign +overview: "Complete redesign of the OpenClaw web app (apps/web/) to match the Dench design system: switch from dark theme to light, adopt Instrument Serif + Inter fonts, port the Dench color palette and layout patterns, and rewrite every component and page from the ground up." +todos: + - id: foundation + content: "Phase 1: Rewrite globals.css (light theme, HSL tokens, font imports) and layout.tsx (next/font, remove dark mode)" + status: pending + - id: landing + content: "Phase 2: Rewrite app/page.tsx as Dench-style landing page (navbar, hero, demo sections, footer)" + status: pending + - id: layout-shell + content: "Phase 3: Create app-navbar.tsx, rewrite workspace/page.tsx layout with top navbar + sidebar grid" + status: pending + - id: sidebar + content: "Phase 4: Redesign workspace-sidebar.tsx and file-manager-tree.tsx to match Dench sidebar" + status: pending + - id: data-table + content: "Phase 5: Redesign object-table.tsx with Dench-style toolbar, sticky headers, pagination, enum badges" + status: pending + - id: kanban + content: "Phase 6: Redesign object-kanban.tsx with light cards, columns, board header" + status: pending + - id: entry-detail + content: "Phase 7: Redesign entry-detail-modal.tsx as right-panel slide-out with properties list" + status: pending + - id: dashboard-chat + content: "Phase 8a: Build dashboard view with greeting, centered chat input, suggestion chips, and animate-down-to-bottom Framer Motion layoutId transition" + status: pending + - id: chat + content: "Phase 8b: Restyle chat-panel.tsx, chat-message.tsx, chain-of-thought.tsx for light theme + bottom composer" + status: pending + - id: remaining + content: "Phase 9: Restyle all remaining components (breadcrumbs, document-view, file-viewer, database-viewer, empty-state, markdown, context-menu, slash-command, charts, etc.)" + status: pending + - id: deps + content: "Phase 10: Add framer-motion dependency, verify fonts work, test build" + status: pending +isProject: false +--- + +# Full Web UI Redesign β€” Dench Design System + +## Current State + +The OpenClaw web app is a **dark-themed** Next.js 15 app with: + +- Dark background (`#0a0a0a`), dark surfaces (`#141414`), orange accent (`#e85d3a`) +- Inter font only, no serif headings +- Minimal homepage (centered text + CTA) +- Workspace layout: left sidebar (260px) + content + optional chat panel +- Custom table/kanban/viewer components, all dark-styled +- Tailwind v4 (CSS-based config), no shadcn/ui + +## Target State (Dench Design) + +Per the screenshots and Dench source: + +- **Light theme** β€” `bg-neutral-50` layout, white cards, `bg-neutral-100` sidebar/navbar +- **Instrument Serif** for headings/titles, **Inter** for body text, **Lora** for branding +- **Top navbar** (grid 3-col, with Dashboard/Workflows/Integrations tabs, org logo, user menu) +- **Left sidebar** (260px, `bg-neutral-100`, collapsible knowledge tree with item counts) +- **Data tables** with: sticky header, column borders, search bar, filter/column controls, enum badges, relation chips, pagination +- **Kanban board** with rounded cards, priority badges, assignee avatars +- **Entry detail** right-panel slide-out with property list +- **Landing page** with hero section, demo sections, clean navigation bar +- **Dashboard chat UX** β€” centered greeting ("Good evening, Kumar?") in Instrument Serif + centered chat input with suggestion chips; on first message, the input animates down to a bottom-docked composer via Framer Motion shared `layoutId` spring transition +- HSL-based CSS variables (shadcn pattern), `--radius: 0.5rem`, neutral base color + +## Architecture Decision: Tailwind v4 + +The OpenClaw app uses **Tailwind v4** (CSS-based config via `@import "tailwindcss"`), while Dench uses Tailwind v3 (JS config). We will keep Tailwind v4 but port all design tokens into `globals.css` using `@theme` blocks and CSS custom properties. No downgrade needed. + +## Architecture Decision: Light + Dark Theme + +Dench is light-only. We will use Dench's light palette as the `:root` default AND create a custom dark palette under `.dark` (class-based toggle via ``). All components will use CSS variable references (e.g. `bg-background`, `text-foreground`, `border-border`) so they automatically adapt. No hardcoded hex/rgb in components. + +**Light palette** (from Dench): + +- `--background: 0 0% 96%` (neutral-50 feel) +- `--foreground: 0 0% 3.9%` +- `--card: 0 0% 100%` / `--card-foreground: 0 0% 3.9%` +- `--muted: 0 0% 96.1%` / `--muted-foreground: 0 0% 45.1%` +- `--border: 0 0% 89.8%` +- `--primary: 0 0% 9%` / `--primary-foreground: 0 0% 98%` +- `--accent: 0 0% 96.1%` / `--accent-foreground: 0 0% 9%` +- `--destructive: 0 84.2% 60.2%` + +**Dark palette** (custom, designed to complement Dench's light theme): + +- `--background: 0 0% 7%` (#121212 β€” rich near-black, not pure black) +- `--foreground: 0 0% 93%` (#ededed) +- `--card: 0 0% 10%` (#1a1a1a) / `--card-foreground: 0 0% 93%` +- `--muted: 0 0% 14%` (#242424) / `--muted-foreground: 0 0% 55%` (#8c8c8c) +- `--border: 0 0% 18%` (#2e2e2e) +- `--primary: 0 0% 93%` / `--primary-foreground: 0 0% 9%` +- `--accent: 0 0% 16%` (#292929) / `--accent-foreground: 0 0% 93%` +- `--destructive: 0 62% 55%` +- Sidebar: `--sidebar-bg: 0 0% 9%` (#171717) +- Navbar: similar to sidebar, subtle `border-b` at `--border` + +Sidebar/navbar in dark mode use a slightly elevated surface (`#171717`) rather than pure background, for depth. + +**Theme toggle:** add a sun/moon toggle button in the navbar (right side, near user avatar). Use `next-themes` or a simple `useEffect` + `localStorage` approach to persist preference and apply `.dark` class on ``. + +--- + +## Files to Change + +### Phase 1 β€” Foundation (Theme, Fonts, Layout Shell) + +**[app/globals.css](apps/web/app/globals.css)** β€” Complete rewrite: + +- `:root` block: Dench's light-theme HSL palette (background, foreground, card, primary, secondary, muted, accent, destructive, border, ring, sidebar, chart-1 through chart-5) +- `.dark` block: custom dark palette (see "Architecture Decision: Light + Dark Theme" above) β€” all same variable names, dark values +- Add `@theme` block for Tailwind v4 mapping CSS vars to utility classes (`bg-background`, `text-foreground`, `border-border`, `bg-card`, `text-muted-foreground`, etc.) +- Import Instrument Serif from Google Fonts +- Add `.font-instrument` utility class +- Port scrollbar, prose, editor, and slash-command styles using CSS variables (theme-aware, not hardcoded) +- Port workflow state colors (`--workflow-active`, `--workflow-processing`, `--workflow-idle`) + +**[app/layout.tsx](apps/web/app/layout.tsx)** β€” Rewrite: + +- Import Inter and Lora via `next/font/google` +- Set CSS variables `--font-corporate` and `--font-lora` +- Default to light: no `className="dark"` on `` (let theme provider handle it) +- Apply `font-corporate` to `` +- Add `suppressHydrationWarning` on `` for theme flash prevention +- Add inline script or `next-themes` `ThemeProvider` for class-based dark mode toggle with `localStorage` persistence +- Update metadata title/description to "Dench" branding + +**New: `app/hooks/use-theme.ts**` β€” Simple theme hook: + +- Read/write `localStorage` key `"theme"` (`"light"` | `"dark"` | `"system"`) +- Apply/remove `.dark` class on `document.documentElement` +- Expose `theme`, `setTheme`, `resolvedTheme` for components + +### Phase 2 β€” Landing Page + +**[app/page.tsx](apps/web/app/page.tsx)** β€” Full rewrite to match Dench landing: + +- Sticky navigation bar (logo "Dench" in `font-lora`, Login button in rounded-full blue pill) +- Hero section: "AI CRM" headline in `font-instrument font-bold`, subtext, "Get Started Free" CTA +- Full-width CRM demo area (window chrome with traffic-light dots, scaled mock table) +- Additional demo sections (workflow, kanban) β€” simplified versions +- Footer with copyright, links + +### Phase 3 β€” Workspace Layout Shell + +**[app/workspace/page.tsx](apps/web/app/workspace/page.tsx)** β€” Rewrite layout structure: + +- Add top `AppNavbar` component: `bg-neutral-100 border-b border-border shadow-[0_0_40px_rgba(0,0,0,0.05)]` + - Left: org logo + "Powered by Dench" + org name in `font-instrument` + - Center: tab navigation (Dashboard, Workflows, Integrations) with active state + - Right: credit display, notification bell, sun/moon theme toggle, user avatar dropdown +- Main area: `grid lg:grid-cols-[260px_1fr]` under navbar +- Full height: `h-[100dvh] flex flex-col bg-neutral-50` +- Content area: `overflow-y-auto overflow-x-hidden` +- Replace all inline `style={{}}` dark colors with Tailwind classes + +**New component: `app/components/workspace/app-navbar.tsx**` β€” Top navbar (extracted for reuse) + +### Phase 4 β€” Sidebar Redesign + +**[app/components/workspace/workspace-sidebar.tsx](apps/web/app/components/workspace/workspace-sidebar.tsx)** β€” Full rewrite: + +- Background: `bg-sidebar` with `border-r border-border` (light: neutral-100, dark: #171717 via CSS var) +- Shadow: theme-aware subtle shadow +- Header: "KNOWLEDGE" section label in uppercase `text-[11px] font-medium tracking-wider text-muted-foreground` +- Knowledge items: `text-[13px]`, hover `bg-accent`, `rounded-xl` +- Item badges showing entry counts in `bg-muted border border-border` pills +- Icons per item type (objects get custom icons, documents get doc icon) +- Collapsible sections: KNOWLEDGE, CHATS, TELEPHONY +- Bottom: "API Keys" link +- Remove all inline `style={{}}` dark colors + +**[app/components/workspace/file-manager-tree.tsx](apps/web/app/components/workspace/file-manager-tree.tsx)** β€” Restyle tree items: + +- Light-theme hover states, active states matching `bg-neutral-200` +- `text-[13px]` sizing, proper icon colors +- Drag-and-drop visual indicators in light theme + +### Phase 5 β€” Data Table Redesign + +**[app/components/workspace/object-table.tsx](apps/web/app/components/workspace/object-table.tsx)** β€” Complete rewrite to match Dench data-table: + +- Toolbar: object name in `font-instrument`, search input (`rounded-full shadow-[0_0_21px_0_rgba(0,0,0,0.07)]`), "Ask AI" button, Table/Board view toggle, refresh/import/filter/columns/+ Add buttons +- Table header: `sticky top-0 z-30 bg-card border-b-2 border-border/80`, sortable columns with sort arrows +- Table cells: `px-4 border-r border-border/30`, proper text truncation +- Enum badges: colored pill style matching Dench (translucent background + border) +- Relation chips: link icon + blue text +- Row hover: `hover:bg-muted/50` +- Pagination bar: "Showing 1 to N of N results", rows-per-page selector, page navigation +- "..." action menu per row (right column) + +### Phase 6 β€” Kanban Board Redesign + +**[app/components/workspace/object-kanban.tsx](apps/web/app/components/workspace/object-kanban.tsx)** β€” Rewrite: + +- Board header: view toggle (Board/Table), "Ask AI" button, search, "Group by" selector +- Columns: `bg-muted/50 rounded-2xl border border-border/60`, column title + count badge +- Cards: `bg-card rounded-xl border border-border/80 shadow-sm` +- Card content: title, field badges (objective, risk profile), date, assignee avatar +- "+ Add Item" at column bottom +- "Drop cards here" empty column placeholder + +### Phase 7 β€” Entry Detail Panel + +**[app/components/workspace/entry-detail-modal.tsx](apps/web/app/components/workspace/entry-detail-modal.tsx)** β€” Redesign as right-panel slide-out: + +- Takes ~40% of content width, pushes table left +- Header: icon + title in large font, "Created Jan 12, 2026 at 12:47 PM" subtitle +- "PROPERTIES" section label +- Property rows: label (uppercase text-xs text-muted-foreground) + value +- Relation fields show colored link chips +- Enum fields show colored badges (matching table) +- "Add a property" at bottom +- Close button (>> icon) top-right + +### Phase 8a β€” Dashboard Chat UX (Greeting + Animate-to-Bottom Input) + +This is the hero interaction on the workspace "Dashboard" tab β€” a centered greeting with a chat input that transitions into the bottom-docked composer after the first message. + +**How Dench implements it:** + +- `DashboardHeader`: time-based greeting ("Good morning/afternoon/evening, Name?") with staggered word-by-word Framer Motion entrance (`y:20 β†’ 0`, `blur(8px) β†’ blur(0)`) +- `DashboardChatbox`: centered TipTap input with placeholder "Build a workflow to automate your tasks", attach/voice/submit buttons, suggestion chips below (shuffled from a pool of ~27 templates, showing 7 in two rows) +- **Layout animation:** both the centered input and the bottom composer share a Framer Motion `layoutId="chat-thread-composer"`. When `showStartComposer` flips to false after the first message, Framer Motion automatically animates the input from center to bottom with `transition={{ type: "spring", stiffness: 260, damping: 30 }}` + +**New components to create:** + +`app/components/workspace/dashboard-view.tsx` β€” Dashboard home view: + +- Greeting in `font-instrument text-4xl` with time-based message + user name +- Word-by-word staggered Framer Motion entrance animation +- Centered chat input area below greeting + +`app/components/workspace/dashboard-chatbox.tsx` β€” Centered input + chips: + +- Rounded white card with subtle shadow, textarea/input with placeholder +- Attach (paperclip), voice (mic), submit (arrow) icon buttons +- Suggestion chip rows: 3 on first row, 4 on second row, each with icon + label + `rounded-xl` border +- Accepts `layoutId` prop for shared layout animation +- `mode` prop: `"dashboard"` (centered, with greeting) vs `"thread"` (same input but used within chat thread) +- Entry animation: `opacity: 0, y: 20` β†’ `opacity: 1, y: 0`, duration 0.8s + +**Modify [app/workspace/page.tsx](apps/web/app/workspace/page.tsx):** + +- When no content selected (Dashboard tab active), render `DashboardView` +- On chat submit: transition to chat thread view +- Use `LayoutGroup` from Framer Motion to wrap the dashboard + chat area +- Track `showStartComposer` state: when true, show centered `DashboardChatbox`; when false, show messages + bottom `ChatComposer` β€” both sharing the same `layoutId` + +**Prompt templates** (simplified set for OpenClaw): + +- Follow-up Emails, Calendly Prep, Zoom Recap, Facebook Leads, Calendar Sync, Salesforce Sync, Intercom Chat (matching the Dench screenshot chips) + +### Phase 8b β€” Chat & Message Restyling + +**[app/components/chat-panel.tsx](apps/web/app/components/chat-panel.tsx)** β€” Restyle: + +- Theme-aware background (`bg-card`), card-colored input area +- Input: rounded border, subtle shadow, consistent with dashboard chatbox style +- Bottom-docked composer with `layoutId` for shared animation +- Session tabs in light theme +- Tool call indicators in light theme +- Send button styling (rounded, neutral) + +**[app/components/chat-message.tsx](apps/web/app/components/chat-message.tsx)** β€” Restyle: + +- Theme-aware message bubbles (user: `bg-muted`, assistant: `bg-card`) +- Code blocks with `bg-muted` +- Markdown rendering in light theme +- Chain-of-thought styling update + +**[app/components/chain-of-thought.tsx](apps/web/app/components/chain-of-thought.tsx)** β€” Light theme + +### Phase 9 β€” Remaining Components + +All components below: replace every hardcoded color (`style={{}}`, hex, rgb) with semantic Tailwind utilities (`bg-background`, `text-foreground`, `border-border`, `bg-card`, `text-muted-foreground`, `bg-muted`, etc.) so they work in both light and dark: + +- **[breadcrumbs.tsx](apps/web/app/components/workspace/breadcrumbs.tsx)** β€” `text-muted-foreground`, `hover:text-foreground` +- **[document-view.tsx](apps/web/app/components/workspace/document-view.tsx)** β€” `bg-card` background, `border-border` +- **[file-viewer.tsx](apps/web/app/components/workspace/file-viewer.tsx)** β€” `bg-muted` code blocks, `text-foreground` +- **[database-viewer.tsx](apps/web/app/components/workspace/database-viewer.tsx)** β€” `bg-card` tables, `bg-muted` query editor +- **[empty-state.tsx](apps/web/app/components/workspace/empty-state.tsx)** β€” `text-muted-foreground` illustration +- **[markdown-content.tsx](apps/web/app/components/workspace/markdown-content.tsx)** β€” Prose styles via CSS vars +- **[markdown-editor.tsx](apps/web/app/components/workspace/markdown-editor.tsx)** β€” `bg-card` editor chrome +- **[context-menu.tsx](apps/web/app/components/workspace/context-menu.tsx)** β€” `bg-card` dropdown, `border-border` +- **[slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx)** β€” `bg-card` command palette +- **[inline-rename.tsx](apps/web/app/components/workspace/inline-rename.tsx)** β€” `bg-card` input, `border-border` +- **[knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx)** β€” Theme-aware tree styles +- **[charts/](apps/web/app/components/charts/)** β€” All chart components: CSS var chart colors, `bg-card` panels +- **[sidebar.tsx](apps/web/app/components/sidebar.tsx)** β€” Theme-aware (if still used) + +### Phase 10 β€” Package Dependencies + +**[package.json](apps/web/package.json)** β€” Add if needed: + +- `framer-motion` (for landing page + dashboard chat animations) +- `next-themes` (for dark/light toggle with `localStorage` + class-based switching, SSR-safe) +- Verify `next/font/google` is available (bundled with Next.js) + +--- + +## Key Design Tokens + +- **Radius:** `0.5rem` base +- **Primary font:** Inter via `next/font/google` +- **Heading font:** Instrument Serif via Google Fonts import +- **Brand font:** Lora via `next/font/google` +- **Sidebar width:** 260px +- **Shadows (light):** `shadow-[0_0_40px_rgba(0,0,0,0.05)]` (sidebar/navbar), `shadow-[0_0_21px_0_rgba(0,0,0,0.07)]` (search) +- **Shadows (dark):** `shadow-[0_0_40px_rgba(0,0,0,0.2)]` (sidebar/navbar), `shadow-[0_0_21px_0_rgba(0,0,0,0.15)]` (search) + +## Component Styling Rules (Theme-Safe) + +All components MUST use semantic CSS variable-backed utilities β€” never hardcoded colors: + +- `bg-background` / `bg-card` / `bg-muted` / `bg-accent` β€” not `bg-white`, `bg-neutral-50`, `bg-[#1a1a1a]` +- `text-foreground` / `text-muted-foreground` / `text-card-foreground` β€” not `text-black`, `text-gray-500` +- `border-border` β€” not `border-neutral-200`, `border-[#2e2e2e]` +- `bg-sidebar` for sidebar/navbar backgrounds +- For shadows that differ between themes: use a CSS variable `--shadow-subtle` / `--shadow-elevated` or conditional `dark:shadow-*` utilities +- Exceptions: Dench-specific decorative elements (landing page traffic-light dots, brand colors) can use fixed values diff --git a/.cursor/plans/interactive_subagent_panel_fb79f5b8.plan.md b/.cursor/plans/interactive_subagent_panel_fb79f5b8.plan.md new file mode 100644 index 00000000000..187efbe6561 --- /dev/null +++ b/.cursor/plans/interactive_subagent_panel_fb79f5b8.plan.md @@ -0,0 +1,132 @@ +--- +name: Interactive Subagent Panel +overview: Make subagents fully independent of the parent agent's event stream. Each subagent gets its own gateway subscription immediately on registration. Then make the subagent panel interactive with stop, send, queue matching the main chat, using unified API routes. +todos: + - id: decouple-subagent + content: "SubagentRunManager: subscribe immediately on registration, remove routeRawEvent/preRegBuffer/activateGatewayFallback" + status: pending + - id: remove-parent-routing + content: "active-runs.ts: remove subagent event routing from parent NDJSON stream" + status: pending + - id: srm-methods + content: "SubagentRunManager: add persistUserMessage(), reactivateSubagent(), abortSubagent(), spawnSubagentMessage()" + status: pending + - id: unify-chat-route + content: Extend POST /api/chat to dispatch to subagent flow when sessionKey is a subagent key + status: pending + - id: unify-stop-route + content: Extend POST /api/chat/stop to dispatch to SubagentRunManager when sessionKey is a subagent key + status: pending + - id: unify-stream-route + content: Extend GET /api/chat/stream to dispatch to SubagentRunManager when sessionKey is a subagent key + status: pending + - id: parser-turns + content: Extend createStreamParser to handle user-message events as turn boundaries + status: pending + - id: panel-rewrite + content: Rewrite SubagentPanel with ChatEditor, send/stop/queue, multi-turn conversation + status: pending +isProject: false +--- + +# Interactive Subagent Panel + +## Core Problem + +Subagent events piggyback on the parent agent's CLI NDJSON stream. When the parent finishes (spawns subagents then exits), the stream dies and subagent events stop flowing. The `activateGatewayFallback()` partially compensates but loses early events. + +The root cause is architectural: subagents are treated as appendages of the parent. They should be independent sessions. + +## Architecture Change + +A subagent is just an agent session. The only link to the parent is the completion announcement. Each subagent gets its own gateway subscription from the moment it's registered. + +```mermaid +flowchart TB + subgraph before [Current: Coupled] + GW1[Gateway] --> ParentCLI[Parent CLI stdout] + ParentCLI --> ARM1[ActiveRunManager] + ParentCLI -.->|"routeRawEvent
(filtered by runId, never arrives)"| SRM1[SubagentRunManager] + ARM1 -.->|"activateGatewayFallback
(after parent exits, loses early events)"| SRM1 + end + + subgraph after [New: Independent] + GW2[Gateway] --> ParentCLI2[Parent CLI stdout] + GW2 --> SubProc[Subscribe Process per subagent] + ParentCLI2 --> ARM2[ActiveRunManager] + SubProc --> SRM2[SubagentRunManager] + end +``` + +## Phase 1: Decouple Subagents + +### 1. SubagentRunManager ([subagent-runs.ts](apps/web/lib/subagent-runs.ts)) + +**In `registerSubagent()` (line 266-270)**: replace the comment with: + +```typescript +if (run.status === "running") { + startSubagentSubscribeStream(run); +} +``` + +Each subagent immediately gets its own subscribe process (`spawnAgentSubscribeProcess`) that connects to the gateway and streams events for that subagent's sessionKey. No dependency on the parent's stream. + +**Remove dead code:** + +- `routeRawEvent()` (lines 419-448) -- no longer called; events come from per-subagent subscribe processes +- `preRegBuffer` from the registry type and `getRegistry()` -- no pre-registration buffering needed; the subscribe process handles everything +- `activateGatewayFallback()` (lines 368-375) -- no longer needed; subscription starts at registration time + +### 2. active-runs.ts ([active-runs.ts](apps/web/lib/active-runs.ts)) + +**Remove subagent event routing from the parent NDJSON handler**: the block that checks `ev.sessionKey !== parentSessionKey` and calls `routeSubagentEvent()` -- delete it entirely. Parent NDJSON stream now only processes parent events. No imports of `routeRawEvent`, `ensureRegisteredFromDisk`, `hasActiveSubagent` from subagent-runs needed for routing. + +**Remove `activateGatewayFallback()` call** from the parent exit handler. + +**Keep**: the `waiting-for-subagents` state transition and `hasRunningSubagentsForParent()` check -- the parent still needs to know when all subagents finish so it can finalize. + +### 3. No CLI changes needed + +The `runId` filter in `src/commands/agent.ts` is correct -- the parent's NDJSON stream should only contain parent events. Subagent events flow independently through their own subscribe processes. + +## Phase 2: Unified API Routes + +Same primitive, same routes. Dispatch based on session key format (`:subagent:` vs `:web:`). + +### 4. SubagentRunManager: interactive methods + +- `**persistUserMessage(sessionKey, msg)**` -- append `{type: "user-message", text, id}` to event buffer + JSONL +- `**reactivateSubagent(sessionKey)**` -- set status to `"running"`, clear `endedAt`, restart subscribe process +- `**abortSubagent(sessionKey)**` -- spawn CLI `gateway call chat.abort`, mark `"error"`, signal subscribers +- `**spawnSubagentMessage(sessionKey, message)**` -- spawn CLI `gateway call agent --params '{"message":"...", "sessionKey":"...", "lane":"subagent", ...}'` + +### 5. Extend `POST /api/chat` ([route.ts](apps/web/app/api/chat/route.ts)) + +If `sessionKey` contains `:subagent:`: + +- Reject if running (409) +- `persistUserMessage()` + `reactivateSubagent()` + `spawnSubagentMessage()` +- Subscribe via `subscribeToSubagent(sessionKey, ..., { replay: false })` for SSE response + +Otherwise: existing parent flow. + +### 6. Extend `POST /api/chat/stop` ([stop/route.ts](apps/web/app/api/chat/stop/route.ts)) + +Accept `sessionKey`. If `:subagent:`: `abortSubagent()`. Otherwise: `abortRun()`. + +### 7. Extend `GET /api/chat/stream` ([stream/route.ts](apps/web/app/api/chat/stream/route.ts)) + +Accept `sessionKey`. If `:subagent:`: lazy-register from disk, `ensureSubagentStreamable()`, `subscribeToSubagent()`. Otherwise: existing parent flow. + +Remove `apps/web/app/api/chat/subagent-stream/route.ts` after migration. + +## Phase 3: Frontend + +### 8. Stream parser turn boundaries ([chat-panel.tsx](apps/web/app/components/chat-panel.tsx)) + +Add `user-message` to `ParsedPart` and `createStreamParser` for multi-turn subagent conversations. + +### 9. Rewrite SubagentPanel ([subagent-panel.tsx](apps/web/app/components/subagent-panel.tsx)) + +Full ChatPanel-like experience: ChatEditor, send/stop/queue buttons, AttachmentStrip, message queue, auto-scroll. Uses the unified routes with `sessionKey`. diff --git a/.cursor/plans/mention_search_and_entry_modals_8a47eefc.plan.md b/.cursor/plans/mention_search_and_entry_modals_8a47eefc.plan.md new file mode 100644 index 00000000000..0baa352a9d2 --- /dev/null +++ b/.cursor/plans/mention_search_and_entry_modals_8a47eefc.plan.md @@ -0,0 +1,245 @@ +--- +name: Mention Search and Entry Modals +overview: Overhaul the tiptap @ mention system to search files AND object entries with fast fuzzy matching, fix the broken link system with canonical internal URIs, and add query-param-based entry detail modals accessible from table rows and @ mention links. +todos: + - id: search-index-api + content: "Phase 1: Build GET /api/workspace/search-index endpoint -- returns flat JSON of all files + all entries from every DuckDB object with display fields" + status: completed + - id: fuse-search-hook + content: "Phase 2: Add fuse.js dep, create useSearchIndex() hook in lib/search-index.ts, build Fuse instance on fetch" + status: completed + - id: mention-upgrade + content: "Phase 2b: Rewrite createFileMention -> createWorkspaceMention in slash-command.tsx, wire up Fuse search, update CommandList for entry results" + status: completed + - id: link-utilities + content: "Phase 3: Create lib/workspace-links.ts with parseWorkspaceLink/buildEntryLink, define @entry/{object}/{id} format" + status: completed + - id: link-insert-fix + content: "Phase 3b: Update mention insert commands to use canonical link format; update markdown-editor.tsx link click handler to parse workspace links" + status: completed + - id: entry-api + content: "Phase 4: Build GET /api/workspace/objects/[name]/entries/[id] endpoint for single entry data" + status: completed + - id: entry-modal + content: "Phase 4b: Build EntryDetailModal component with full field display, relation links, close/escape handling" + status: completed + - id: query-param-routing + content: "Phase 4c: Wire ?entry= query param in workspace/page.tsx -- render modal, sync URL on open/close, handle initial load" + status: completed + - id: table-row-click + content: "Phase 4d: Make object-table.tsx rows clickable with onEntryClick prop, wire to entry param in parent" + status: completed + - id: fix-nav-bugs + content: "Phase 5: Fix path resolution in onNavigate (knowledge/ prefix handling), sync URL bar with activePath via router.replace()" + status: completed +isProject: false +--- + +# @ Mention Search, Link System, and Entry Detail Modals + +## Current Problems + +1. **@ mention only searches files** -- entries (rows) in objects (tables) are invisible to the search. Uses naive `String.includes()` with no fuzzy matching. +2. **File links are broken** -- `buildFileItems` in [slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx) sets `href: node.path` (e.g. `knowledge/leads`), but `onNavigate` in [workspace/page.tsx](apps/web/app/workspace/page.tsx) calls `findNode(tree, path)` which may not resolve correctly depending on whether the path includes `knowledge/` prefix. No canonical link format exists. +3. **No entry detail view** -- table rows in [object-table.tsx](apps/web/app/components/workspace/object-table.tsx) are not clickable. There is no modal, route, or UI to view a single entry. +4. **No URL-based navigation** -- navigating to content is purely callback-based (state updates). The URL only updates on initial load via `?path=`. Sharing a link to a specific entry is impossible. + +--- + +## Architecture + +```mermaid +flowchart TB + subgraph searchIndex [Search Index - API] + searchEndpoint["GET /api/workspace/search-index"] + duckdb["DuckDB: objects + entries + fields"] + tree["Filesystem: tree nodes"] + searchEndpoint --> duckdb + searchEndpoint --> tree + end + + subgraph clientSearch [Client-Side Fuzzy Search] + fuseIndex["Fuse.js index - built once on load"] + mentionPlugin["@ Mention Plugin"] + fuseIndex --> mentionPlugin + end + + subgraph routing [Query Param Routing] + pathParam["?path=knowledge/leads"] + entryParam["&entry=abc123"] + pathParam --> workspacePage["Workspace Page"] + entryParam --> entryModal["Entry Detail Modal"] + end + + searchEndpoint -->|"JSON: files + entries"| fuseIndex + mentionPlugin -->|"insert link"| internalLink["dench://entry/leads/abc123"] + internalLink -->|"onNavigate resolves"| routing +``` + +--- + +## Phase 1: Search Index API Endpoint + +**New file:** `apps/web/app/api/workspace/search-index/route.ts` + +Returns a flat JSON array of all searchable items -- both files and entry rows from every object. The client fetches this once on workspace load and rebuilds on tree changes (SSE watcher already triggers refreshes). + +```typescript +type SearchIndexItem = { + // Shared + id: string; // unique key (path for files, entryId for entries) + label: string; // display text (filename or display-field value) + sublabel?: string; // secondary text (path for files, object name for entries) + kind: "file" | "object" | "entry"; + icon?: string; + + // For entries + objectName?: string; + entryId?: string; + fields?: Record; // first 3-4 field key-value pairs for preview + + // For files/objects + path?: string; + nodeType?: "document" | "folder" | "file" | "report" | "database"; +}; +``` + +**Server implementation:** + +- Reuse existing `buildTree()` from [tree/route.ts](apps/web/app/api/workspace/tree/route.ts) for files +- For entries: query every object from DuckDB, resolve display fields, and return flattened entries with their first few field values as searchable text +- SQL: `SELECT * FROM v_{objectName} LIMIT 500` per object (capped for perf) +- Cache with short TTL or rely on client refetch on SSE tree-change events + +--- + +## Phase 2: Client-Side Fuzzy Search with Fuse.js + +**New file:** `apps/web/lib/search-index.ts` + +A React hook `useSearchIndex()` that: + +1. Fetches `/api/workspace/search-index` once on mount +2. Builds a `Fuse` instance over the items, keyed on `label`, `sublabel`, and `fields` values +3. Exposes a `search(query: string): SearchIndexItem[]` function +4. Rebuilds when tree changes (listen to same SSE watcher signal) + +**Fuse.js config:** + +- Keys: `["label", "sublabel", "objectName", "fields.*"]` with weighted scoring +- Threshold: ~0.4 (tolerant fuzzy) +- Max results: 20 + +**Update [slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx):** + +Replace `createFileMention(tree)` with `createWorkspaceMention(searchFn)`: + +- `items` callback uses the Fuse search function instead of `flattenTree` + `.includes()` +- Results are grouped: show files first, then entries grouped by object +- The `CommandList` component gets a minor update to show entry items differently (object badge, field preview) + +**Dep:** Add `fuse.js` to `apps/web/package.json` -- lightweight (~7KB gzipped), runs entirely client-side, no wasm. + +--- + +## Phase 3: Canonical Internal Link Format + +Define a URI scheme for internal workspace links, parsed in a single utility: + +**New file:** `apps/web/lib/workspace-links.ts` + +```typescript +// Canonical formats: +// Files: "knowledge/path/to/doc.md" (relative path, unchanged) +// Objects: "knowledge/leads" (relative path, unchanged) +// Entries: "@entry/leads/abc123" (new: @entry/{objectName}/{entryId}) + +type WorkspaceLink = + | { kind: "file"; path: string } + | { kind: "object"; objectName: string; path: string } + | { kind: "entry"; objectName: string; entryId: string }; + +function parseWorkspaceLink(href: string): WorkspaceLink; +function buildEntryLink(objectName: string, entryId: string): string; +function buildFileLink(path: string): string; +function workspaceLinkToUrl(link: WorkspaceLink): string; +// Returns: "/workspace?path=knowledge/leads" or "/workspace?path=knowledge/leads&entry=abc123" +``` + +**Update mention insert command** in `slash-command.tsx`: + +- File items: keep `href: node.path` (unchanged) +- Entry items: use `href: "@entry/{objectName}/{entryId}"`, display text = display field value + +**Update link click handler** in [markdown-editor.tsx](apps/web/app/components/workspace/markdown-editor.tsx) (lines 215-238): + +- Parse the href with `parseWorkspaceLink()` +- For file/object links: `onNavigate(link.path)` (as today, but path resolution fixed) +- For entry links: `onNavigate("@entry/leads/abc123")` -- parent resolves to modal + +**Update workspace page** `onNavigate` handler: + +- Parse the link, and if it's an `@entry/...` link, set query params to open the entry modal +- If it's a file/object link, do `findNode(tree, path)` as before but with proper path normalization + +--- + +## Phase 4: Entry Detail Modal with Query Param Routing + +**URL pattern:** `/workspace?path=knowledge/leads&entry=abc123` + +When `entry` query param is present, render an `EntryDetailModal` as an overlay on top of the current workspace content. + +**New file:** `apps/web/app/components/workspace/entry-detail-modal.tsx` + +- Full-screen overlay/side panel with entry data +- Fetches entry data from a new API endpoint or uses already-loaded object data +- Renders all fields with proper type-specific display (reuse `CellValue` from [object-table.tsx](apps/web/app/components/workspace/object-table.tsx)) +- Relation fields are clickable -- clicking opens the related entry's modal (updates `entry` param) +- Close button / Escape / clicking backdrop clears the `entry` param +- URL is shareable and bookmarkable + +**New API:** `GET /api/workspace/objects/[name]/entries/[id]` + +- Returns a single entry with all field values, resolved relation labels, and reverse relations +- Lightweight endpoint for the modal to fetch data without loading the full object + +**Update [workspace/page.tsx](apps/web/app/workspace/page.tsx):** + +- Read `entry` from `searchParams` +- When present, render `` on top of current content +- Update URL using `router.replace()` (shallow) when opening/closing modal -- no full page navigation +- On initial load, if both `path` and `entry` are set, load the object AND open the modal + +**Update [object-table.tsx](apps/web/app/components/workspace/object-table.tsx):** + +- Make table rows clickable +- `onClick` handler calls a new `onEntryClick(entryId)` prop +- Parent sets the `entry` query param, opening the modal + +--- + +## Phase 5: Fix Existing Link Navigation Bugs + +**Bug 1:** Path resolution mismatch -- `node.path` from tree includes `knowledge/` prefix but link resolution sometimes strips it. Fix: normalize paths in `findNode()` and `onNavigate()` using the new `parseWorkspaceLink()`. + +**Bug 2:** The `onNavigate` callback in `DocumentView` does `findNode(tree, path)` but the href from a markdown link may not exactly match a tree path (e.g., `leads` vs `knowledge/leads`). Fix: add fallback resolution that tries with/without `knowledge/` prefix, and tries matching the last segment against object names. + +**Bug 3:** No URL update when navigating -- selecting a sidebar item or clicking a link updates state but not the URL bar. Fix: use `router.replace()` to keep URL in sync with `activePath` so links are shareable. + +--- + +## Key Files to Modify + +- [apps/web/app/components/workspace/slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx) -- Replace file-only search with unified fuzzy search over files + entries +- [apps/web/app/components/workspace/markdown-editor.tsx](apps/web/app/components/workspace/markdown-editor.tsx) -- Fix link click handler to support entry links +- [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx) -- Add entry modal rendering, fix URL sync, fix `onNavigate` resolution +- [apps/web/app/components/workspace/object-table.tsx](apps/web/app/components/workspace/object-table.tsx) -- Make rows clickable with `onEntryClick` + +## New Files + +- `apps/web/app/api/workspace/search-index/route.ts` -- Search index endpoint +- `apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts` -- Single entry endpoint +- `apps/web/lib/workspace-links.ts` -- Link parsing/building utilities +- `apps/web/lib/search-index.ts` -- Client-side Fuse.js hook +- `apps/web/app/components/workspace/entry-detail-modal.tsx` -- Entry modal component diff --git a/.cursor/plans/reports_analytics_layer_d6cf8500.plan.md b/.cursor/plans/reports_analytics_layer_d6cf8500.plan.md new file mode 100644 index 00000000000..c00ac4740c3 --- /dev/null +++ b/.cursor/plans/reports_analytics_layer_d6cf8500.plan.md @@ -0,0 +1,258 @@ +--- +name: Reports Analytics Layer +overview: "Add a generative-UI reports feature to the Dench web app: the agent creates JSON report definitions with SQL queries and chart configs, rendered live via Recharts in both the workspace view and inline in chat as artifacts." +todos: + - id: recharts-dep + content: Add recharts dependency to apps/web/package.json + status: completed + - id: chart-panel + content: Create ChartPanel component supporting bar/line/area/pie/donut/radar/scatter/funnel via Recharts with CSS variable theming + status: completed + - id: filter-bar + content: Create FilterBar component with dateRange, select, multiSelect filter types; fetches options via SQL + status: completed + - id: report-viewer + content: "Create ReportViewer component: loads report config, executes panel SQL with filter injection, renders ChartPanel grid" + status: completed + - id: report-execute-api + content: Create POST /api/workspace/reports/execute route for SQL execution with filter clause injection + status: completed + - id: workspace-report-type + content: Add report content type to workspace/page.tsx ContentState + ContentRenderer; detect .report.json in tree API + status: completed + - id: knowledge-tree-report + content: Add report node type + icon to knowledge-tree.tsx and tree/route.ts + status: completed + - id: chat-artifact + content: Modify chat-message.tsx to detect report-json fenced blocks and render inline ReportCard + status: completed + - id: report-card + content: Create ReportCard component for compact inline chart rendering in chat with Pin/Open actions + status: completed + - id: sidebar-reports + content: Add Reports section to sidebar.tsx listing .report.json files from workspace tree + status: completed + - id: skill-update + content: Add Section 13 (Report Generation) to skills/dench/SKILL.md with schema, examples, and agent instructions + status: completed +isProject: false +--- + +# Reports / Analytics Layer for Dench Workspace + +## Architecture + +Reports are **JSON config files** (`.report.json`) that declare SQL queries to run against `workspace.duckdb` and how to visualize the results. The web app executes SQL at render time (live data), renders charts via Recharts, and supports interactive filters. + +Reports surface in **three places**: + +1. **Workspace view** -- full-page report dashboard when clicking a report in the tree +2. **Chat** -- inline chart artifact when the agent generates a report in conversation +3. **Sidebar** -- reports listed under a new "Reports" section + +```mermaid +flowchart LR + subgraph agent [Agent] + skill[Dench Skill] + skill -->|"write .report.json"| fs[Filesystem] + skill -->|"emit report block in text"| chat[Chat Stream] + end + + subgraph web [Web App] + fs --> treeAPI["/api/workspace/tree"] + fs --> reportAPI["/api/workspace/reports"] + reportAPI --> execAPI["/api/workspace/query"] + execAPI --> duckdb["workspace.duckdb"] + + treeAPI --> sidebar[Sidebar + KnowledgeTree] + reportAPI --> workspaceView[ReportViewer in Workspace] + chat --> chatUI["ChatMessage with inline ReportCard"] + chatUI --> execAPI + workspaceView --> recharts[Recharts Charts] + chatUI --> recharts + end +``` + +--- + +## Report Definition Format + +Stored as `.report.json` files in `dench/reports/` (or nested under any knowledge path). Agent generates these via the `write` tool. + +```json +{ + "version": 1, + "title": "Deals Pipeline", + "description": "Revenue breakdown by stage and rep", + "panels": [ + { + "id": "deals-by-stage", + "title": "Deal Count by Stage", + "type": "bar", + "sql": "SELECT \"Stage\", COUNT(*) as count FROM v_deal GROUP BY \"Stage\" ORDER BY count DESC", + "mapping": { "xAxis": "Stage", "yAxis": ["count"] }, + "size": "half" + }, + { + "id": "revenue-trend", + "title": "Revenue Over Time", + "type": "area", + "sql": "SELECT DATE_TRUNC('month', created_at) as month, SUM(\"Amount\"::NUMERIC) as revenue FROM v_deal GROUP BY month ORDER BY month", + "mapping": { "xAxis": "month", "yAxis": ["revenue"] }, + "size": "half" + }, + { + "id": "stage-distribution", + "title": "Stage Distribution", + "type": "pie", + "sql": "SELECT \"Stage\", COUNT(*) as count FROM v_deal GROUP BY \"Stage\"", + "mapping": { "nameKey": "Stage", "valueKey": "count" }, + "size": "third" + } + ], + "filters": [ + { + "id": "date-range", + "type": "dateRange", + "label": "Date Range", + "column": "created_at" + }, + { + "id": "assigned-to", + "type": "select", + "label": "Assigned To", + "sql": "SELECT DISTINCT \"Assigned To\" as value FROM v_deal WHERE \"Assigned To\" IS NOT NULL", + "column": "Assigned To" + } + ] +} +``` + +**Chart types supported:** `bar`, `line`, `area`, `pie`, `donut`, `radar`, `radialBar`, `scatter`, `funnel`. +**Panel sizes:** `full`, `half`, `third` (CSS grid layout). +**Filter types:** `dateRange`, `select`, `multiSelect`, `number`. + +When filters are active, they inject `WHERE` clauses into each panel's SQL before execution. + +--- + +## Phase 1: Recharts + Report Viewer Component + +**Add Recharts dependency:** + +- [apps/web/package.json](apps/web/package.json) -- add `recharts` to dependencies + +**New components** (all in `apps/web/app/components/charts/`): + +- `**chart-panel.tsx**` -- Wrapper that takes a panel config + data rows, renders the correct Recharts chart (Bar, Line, Area, Pie, etc.). Uses the app's CSS variable palette for theming (`--color-accent`, `--color-text`, `--color-border`). One component, switch on `panel.type`. +- `**filter-bar.tsx**` -- Horizontal filter strip. Reads filter configs from the report, fetches options for `select` type filters via SQL, renders date pickers / dropdowns. Emits active filter state upward. +- `**report-viewer.tsx**` -- Full report dashboard. Fetches report config (from file or prop), iterates panels, executes each panel's SQL (with filter injection), renders `ChartPanel` components in a CSS grid (`size` controls column span). Includes a header with title/description and the filter bar. + +--- + +## Phase 2: Workspace Integration + +**Extend content types** in [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx): + +- Add `report` to the `ContentState` union: `{ kind: "report"; reportPath: string; filename: string }` +- Add `report` case to `ContentRenderer` that renders `ReportViewer` +- In `loadContent`, detect `.report.json` files and load as `report` kind + +**Extend knowledge tree** in [apps/web/app/components/workspace/knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx): + +- Add `"report"` to the `TreeNode.type` union +- Add report icon (bar chart icon) to `NodeTypeIcon` + +**Extend tree API** in [apps/web/app/api/workspace/tree/route.ts](apps/web/app/api/workspace/tree/route.ts): + +- Detect `.report.json` files and assign them `type: "report"` in the tree + +**New API route** -- `apps/web/app/api/workspace/reports/execute/route.ts`: + +- POST `{ sql: string, filters?: FilterState }` -- injects filter WHERE clauses into SQL, executes via `duckdbQuery`, returns rows +- This is separate from the generic query endpoint because it handles filter injection safely + +--- + +## Phase 3: Chat Artifact (Inline Reports) + +**Report block convention in agent text:** +The agent emits a fenced code block with language `report-json` containing the report JSON. Example in the streamed text: + +```` +Here's your pipeline analysis: + +```report-json +{"version":1,"title":"Deals by Stage","panels":[...]} +```` + +The data shows most deals are in the Discovery stage. + +````` + +**Modify [apps/web/app/components/chat-message.tsx](apps/web/app/components/chat-message.tsx):** + +- In `groupParts`, detect text segments containing ````report-json ... ```` blocks +- Split text around report blocks into `text` and `report-artifact` segments +- New segment type: `{ type: "report-artifact"; config: ReportConfig }` + +**New component** -- `apps/web/app/components/charts/report-card.tsx`: + +- Compact inline report card rendered inside chat bubbles +- Shows report title + a subset of panels (auto-sized to fit chat width) +- "Open in Workspace" button that saves to filesystem + navigates +- "Pin" action to persist an ephemeral chat report as a `.report.json` file + +--- + +## Phase 4: Sidebar Reports Section + +**Modify [apps/web/app/components/sidebar.tsx](apps/web/app/components/sidebar.tsx):** + +- Add "Reports" as a new `SidebarSection` (between Workspace and Memories) +- Fetch report list from `/api/workspace/tree` (filter for `type: "report"` nodes) +- Each report links to `/workspace?path=reports/{name}.report.json` +- Show chart icon + report title + +--- + +## Phase 5: Dench Skill Updates + +**Modify [skills/dench/SKILL.md](skills/dench/SKILL.md):** + +- Add Section 13: Report Generation +- Document the `.report.json` format with full schema reference +- Provide example reports for common CRM analytics: + - Pipeline funnel (deals by stage) + - Revenue trend over time + - Lead source breakdown + - Activity/task completion rates + - Contact growth over time +- Instruct the agent to: + - Create reports in `reports/` directory + - Use the existing `v_{object}` PIVOT views in SQL queries + - Include relevant filters (date range, assignee, status) + - Emit `report-json` blocks in chat for inline rendering + - Choose appropriate chart types for the data shape +- Add a "Post-Report Checklist" matching the existing post-mutation pattern + +--- + +## File Summary + + +| Action | File | +| ------ | -------------------------------------------------------------------- | +| Modify | `apps/web/package.json` (add recharts) | +| Create | `apps/web/app/components/charts/chart-panel.tsx` | +| Create | `apps/web/app/components/charts/report-viewer.tsx` | +| Create | `apps/web/app/components/charts/report-card.tsx` | +| Create | `apps/web/app/components/charts/filter-bar.tsx` | +| Create | `apps/web/app/api/workspace/reports/execute/route.ts` | +| Modify | `apps/web/app/workspace/page.tsx` (add report content type) | +| Modify | `apps/web/app/components/chat-message.tsx` (detect report blocks) | +| Modify | `apps/web/app/components/sidebar.tsx` (Reports section) | +| Modify | `apps/web/app/components/workspace/knowledge-tree.tsx` (report node) | +| Modify | `apps/web/app/api/workspace/tree/route.ts` (detect .report.json) | +| Modify | `skills/dench/SKILL.md` (report generation instructions) | +````` diff --git a/.cursor/plans/sidebar_file_manager_02ed8b45.plan.md b/.cursor/plans/sidebar_file_manager_02ed8b45.plan.md new file mode 100644 index 00000000000..bc5d5719c90 --- /dev/null +++ b/.cursor/plans/sidebar_file_manager_02ed8b45.plan.md @@ -0,0 +1,282 @@ +--- +name: Sidebar File Manager +overview: Transform the workspace file tree sidebar (both on `/` home and `/workspace` pages) into a full-fledged file system manager with context menus, drag-and-drop file moves, create/delete/rename operations, system file locking, and live reactivity -- modeled after macOS Finder. +todos: + - id: api-system-files + content: Add isSystemFile() to lib/workspace.ts, extend safeResolvePath to support non-existent target paths + status: pending + - id: api-delete + content: Add DELETE handler to /api/workspace/file/route.ts with system file protection + status: pending + - id: api-rename + content: Create /api/workspace/rename/route.ts (POST) with validation and system file protection + status: pending + - id: api-move + content: Create /api/workspace/move/route.ts (POST) for drag-and-drop moves + status: pending + - id: api-mkdir + content: Create /api/workspace/mkdir/route.ts (POST) for creating new directories + status: pending + - id: api-copy + content: Create /api/workspace/copy/route.ts (POST) for duplicating files/folders + status: pending + - id: api-watch + content: Create /api/workspace/watch/route.ts SSE endpoint using chokidar for live file events + status: pending + - id: install-dndkit + content: Install @dnd-kit/core, @dnd-kit/sortable, @dnd-kit/utilities in apps/web + status: pending + - id: context-menu + content: "Build context-menu.tsx: portal-based right-click menu with Finder-like options, system file lock states" + status: pending + - id: inline-rename + content: "Build inline-rename.tsx: double-click/F2 to rename in-place with validation" + status: pending + - id: file-manager-tree + content: "Build file-manager-tree.tsx: unified DnD tree wrapping context menu, inline rename, drag-drop, keyboard shortcuts" + status: pending + - id: sse-hook + content: "Build useWorkspaceWatcher hook: SSE connection with debounced tree refetch and auto-reconnect" + status: pending + - id: integrate-workspace-sidebar + content: Replace KnowledgeTree with FileManagerTree in workspace-sidebar.tsx + status: pending + - id: integrate-home-sidebar + content: Replace WorkspaceTreeNode in sidebar.tsx with FileManagerTree (compact mode) + status: pending + - id: integrate-workspace-page + content: Wire workspace/page.tsx to useWorkspaceWatcher for live-reactive tree state + status: pending + - id: keyboard-shortcuts + content: Add keyboard navigation and file operation shortcuts to FileManagerTree + status: pending +isProject: false +--- + +# Full File System Manager Sidebar + +## Current State + +Two sidebar trees render workspace files read-only: + +- **Home sidebar** (`[apps/web/app/components/sidebar.tsx](apps/web/app/components/sidebar.tsx)`): `WorkspaceSection` / `WorkspaceTreeNode` -- compact tree inside collapsible section +- **Workspace sidebar** (`[apps/web/app/components/workspace/workspace-sidebar.tsx](apps/web/app/components/workspace/workspace-sidebar.tsx)`): wraps `KnowledgeTree` from `[knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx)` + +Both fetch from `GET /api/workspace/tree` (`[apps/web/app/api/workspace/tree/route.ts](apps/web/app/api/workspace/tree/route.ts)`). File read/write exists at `/api/workspace/file` (`[apps/web/app/api/workspace/file/route.ts](apps/web/app/api/workspace/file/route.ts)`). No delete, rename, move, mkdir, or copy endpoints exist. No context menus, drag-and-drop, or live refresh. + +## Architecture + +```mermaid +flowchart TD + subgraph frontend [Frontend Components] + ContextMenu[ContextMenu Component] + FileTree[FileTree - Unified Tree with DnD] + InlineRename[Inline Rename Input] + NewFileDialog[New File/Folder Prompt] + end + + subgraph api [API Routes - apps/web/app/api/workspace/] + TreeRoute[GET /tree] + FileRoute[GET+POST /file] + DeleteRoute[DELETE /file] + RenameRoute[POST /rename] + MoveRoute[POST /move] + MkdirRoute[POST /mkdir] + CopyRoute[POST /copy] + WatchRoute[GET /watch - SSE] + end + + subgraph fsLib [lib/workspace.ts] + SafeResolve[safeResolvePath] + IsSystemFile[isSystemFile] + end + + FileTree -->|right-click| ContextMenu + FileTree -->|drag-drop| MoveRoute + ContextMenu -->|"New File/Folder"| MkdirRoute + ContextMenu -->|Delete| DeleteRoute + ContextMenu -->|Rename| InlineRename + ContextMenu -->|Duplicate| CopyRoute + ContextMenu -->|"Move to..."| MoveRoute + InlineRename --> RenameRoute + NewFileDialog --> MkdirRoute + NewFileDialog --> FileRoute + + DeleteRoute --> SafeResolve + RenameRoute --> SafeResolve + MoveRoute --> SafeResolve + CopyRoute --> SafeResolve + DeleteRoute --> IsSystemFile + RenameRoute --> IsSystemFile + MoveRoute --> IsSystemFile + + WatchRoute -->|"SSE events"| FileTree +``` + +## 1. New Backend API Endpoints + +Add to `[apps/web/app/api/workspace/](apps/web/app/api/workspace/)`: + +`**DELETE /api/workspace/file**` - Delete a file or folder + +- Body: `{ path: string }` +- Reject if `isSystemFile(path)` returns true +- Use `fs.rmSync(absPath, { recursive: true })` for folders + +`**POST /api/workspace/rename**` - Rename a file or folder (new route file) + +- Body: `{ path: string, newName: string }` +- Reject system files; validate newName (no slashes, no `.` prefix for non-system) +- Use `fs.renameSync(oldAbs, newAbs)` where newAbs replaces only the basename + +`**POST /api/workspace/move**` - Move a file/folder to a new parent (new route file) + +- Body: `{ sourcePath: string, destinationDir: string }` +- Reject system files; validate destination exists and is a directory +- Use `fs.renameSync(srcAbs, join(destAbs, basename(srcAbs)))` + +`**POST /api/workspace/mkdir**` - Create a directory (new route file) + +- Body: `{ path: string }` +- Use `fs.mkdirSync(absPath, { recursive: true })` + +`**POST /api/workspace/copy**` - Duplicate a file or folder (new route file) + +- Body: `{ path: string, destinationPath?: string }` +- If no destination, auto-name as ` copy.` +- Use `fs.cpSync` for recursive copy + +`**GET /api/workspace/watch**` - SSE endpoint for live changes (new route file) + +- Uses `chokidar` to watch the dench workspace root +- Streams SSE events: `{ type: "add"|"change"|"unlink"|"addDir"|"unlinkDir", path: string }` +- Client reconnects on close; debounce events (200ms) + +### System File Protection + +Add to `[apps/web/lib/workspace.ts](apps/web/lib/workspace.ts)`: + +```typescript +const SYSTEM_FILE_PATTERNS = [ + /^\.object\.yaml$/, + /^workspace\.duckdb/, + /^workspace_context\.yaml$/, + /\.wal$/, + /\.tmp$/, +]; + +export function isSystemFile(relativePath: string): boolean { + const basename = relativePath.split("/").pop() ?? ""; + return SYSTEM_FILE_PATTERNS.some((p) => p.test(basename)); +} +``` + +Extend `safeResolvePath` to optionally accept a `mustExist: false` flag (currently returns null for non-existent paths, but mkdir/create need paths that don't exist yet). + +## 2. Context Menu Component + +Create `[apps/web/app/components/workspace/context-menu.tsx](apps/web/app/components/workspace/context-menu.tsx)`: + +- Pure CSS + React portal-based context menu (no library, matches the zero-dep approach) +- Positioned at cursor coordinates, clamped to viewport +- Closes on click-outside, Escape, or scroll +- Menu items with icons, keyboard shortcut hints, separators, and disabled states + +**Menu structure** (mirrors Finder): + +| For files | For folders | For empty area | +| ---------- | -------------------- | -------------- | +| Open | Open | New File | +| Rename | New File inside... | New Folder | +| Duplicate | New Folder inside... | Paste | +| Copy | Rename | --- | +| Move to... | Duplicate | --- | +| --- | Copy | --- | +| Get Info | Move to... | --- | +| --- | --- | --- | +| Delete | Delete | --- | + +System files (`.object.yaml`, `workspace.duckdb`) show the same menu but all mutating actions are **disabled** with a lock icon and "System file" tooltip. + +## 3. Drag-and-Drop for File Moves + +Install `@dnd-kit/core` + `@dnd-kit/sortable` + `@dnd-kit/utilities` (lightweight, ~15KB gzipped, React 19 compatible). + +Create `[apps/web/app/components/workspace/dnd-file-tree.tsx](apps/web/app/components/workspace/dnd-file-tree.tsx)`: + +- Each tree node is both a **draggable** and a **droppable** (folders accept drops) +- Drag overlay shows a ghost of the file/folder name with icon +- Drop targets highlight with accent border when hovered +- On drop: call `POST /api/workspace/move` with `{ sourcePath, destinationDir }` +- System files: draggable is **disabled** (visual lock indicator) +- Folder auto-expand on hover during drag (300ms delay) +- Drop validation: prevent dropping a folder into itself or its children + +## 4. Inline Rename + +- Double-click or press Enter/F2 on a selected node to enter rename mode +- Replace the label `` with a controlled `` pre-filled with the current name +- Commit on Enter or blur; cancel on Escape +- Call `POST /api/workspace/rename` on commit +- Shake animation + red border on validation error (empty name, invalid chars, name collision) + +## 5. Unified FileManagerTree Component + +Refactor `[knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx)` into a new `FileManagerTree` that wraps DnD + context menu + inline rename: + +``` +FileManagerTree (DndContext provider + SSE watcher) + └─ FileManagerNode (draggable + droppable + onContextMenu + double-click rename) + β”œβ”€ ChevronIcon + β”œβ”€ NodeIcon (+ lock badge for system files) + β”œβ”€ InlineRenameInput | Label + └─ Children (recursive) +``` + +Both sidebars (`sidebar.tsx` WorkspaceSection and `workspace-sidebar.tsx`) will use this unified component, passing a `compact` prop to control spacing/features for the home sidebar (e.g., hide "Get Info", simpler context menu). + +## 6. Live Reactivity (SSE) + +Create a `useWorkspaceWatcher` hook in `[apps/web/app/hooks/use-workspace-watcher.ts](apps/web/app/hooks/use-workspace-watcher.ts)`: + +- Opens an `EventSource` to `GET /api/workspace/watch` +- On any file event, debounces and refetches the tree from `/api/workspace/tree` +- Provides `tree`, `loading`, and `refresh()` to consumers +- Auto-reconnects with exponential backoff (1s, 2s, 4s, max 30s) +- Falls back to polling every 5s if SSE fails + +Both sidebars and the workspace page will use this hook for shared, reactive tree state. + +## 7. Keyboard Shortcuts + +In the `FileManagerTree`, attach keyboard handlers on the focused tree container: + +- **Delete / Backspace**: Delete selected item (with confirmation dialog) +- **Enter / F2**: Start inline rename +- **Cmd+C**: Copy path to clipboard +- **Cmd+D**: Duplicate +- **Cmd+N**: New file in current folder +- **Cmd+Shift+N**: New folder in current folder +- **Arrow keys**: Navigate tree up/down, expand/collapse with left/right +- **Space**: Quick Look / preview toggle (future) + +## Files Changed + +| File | Action | +| --------------------------------------------------------- | -------------------------------------------------------------- | +| `apps/web/package.json` | Add `@dnd-kit/core`, `@dnd-kit/sortable`, `@dnd-kit/utilities` | +| `apps/web/lib/workspace.ts` | Add `isSystemFile()`, extend `safeResolvePath` | +| `apps/web/app/api/workspace/file/route.ts` | Add `DELETE` handler | +| `apps/web/app/api/workspace/rename/route.ts` | New -- rename endpoint | +| `apps/web/app/api/workspace/move/route.ts` | New -- move endpoint | +| `apps/web/app/api/workspace/mkdir/route.ts` | New -- mkdir endpoint | +| `apps/web/app/api/workspace/copy/route.ts` | New -- copy endpoint | +| `apps/web/app/api/workspace/watch/route.ts` | New -- SSE file watcher | +| `apps/web/app/components/workspace/context-menu.tsx` | New -- right-click menu | +| `apps/web/app/components/workspace/file-manager-tree.tsx` | New -- unified DnD + context menu tree | +| `apps/web/app/components/workspace/inline-rename.tsx` | New -- inline rename input | +| `apps/web/app/hooks/use-workspace-watcher.ts` | New -- SSE watcher hook | +| `apps/web/app/components/workspace/knowledge-tree.tsx` | Refactor into file-manager-tree | +| `apps/web/app/components/workspace/workspace-sidebar.tsx` | Use new FileManagerTree | +| `apps/web/app/components/sidebar.tsx` | Use new FileManagerTree (compact mode) | +| `apps/web/app/workspace/page.tsx` | Use `useWorkspaceWatcher` hook | diff --git a/.cursor/plans/web_cron_dashboard_d0829ca4.plan.md b/.cursor/plans/web_cron_dashboard_d0829ca4.plan.md new file mode 100644 index 00000000000..3fc19e09145 --- /dev/null +++ b/.cursor/plans/web_cron_dashboard_d0829ca4.plan.md @@ -0,0 +1,214 @@ +--- +name: Web Cron Dashboard +overview: Add a "Cron" virtual folder to the web app sidebar that reads `~/.openclaw/cron/` data, showing all cron jobs, their heartbeat/scheduling status, run history, and full session transcripts with thinking for each run. +todos: + - id: types + content: Create shared cron types in apps/web/app/types/cron.ts + status: pending + - id: api-jobs + content: Create GET /api/cron/jobs route (reads jobs.json + heartbeat info) + status: pending + - id: api-runs + content: Create GET /api/cron/jobs/[jobId]/runs route (reads run log JSONL) + status: pending + - id: api-session + content: Create GET /api/cron/runs/[sessionId] route (reads session transcript JSONL with full thinking/tools) + status: pending + - id: cron-dashboard + content: Build CronDashboard component (heartbeat status, job list, timeline) + status: pending + - id: cron-job-detail + content: Build CronJobDetail component (job config, next run countdown, run history) + status: pending + - id: cron-run-chat + content: Build CronRunChat component (full session transcript with thinking/tools, reusing ChainOfThought patterns) + status: pending + - id: sidebar + content: Add ~cron virtual folder to workspace sidebar in page.tsx (fetch + tree + handleNodeSelect) + status: pending + - id: content-routing + content: Extend content state and main area renderer in workspace/page.tsx for cron-dashboard and cron-job kinds + status: pending +isProject: false +--- + +# Web Cron Dashboard + +## Architecture + +The cron system stores its data at: + +- **Job definitions**: `~/.openclaw/cron/jobs.json` (`CronStoreFile` with `CronJob[]`) +- **Run logs per job**: `~/.openclaw/cron/runs/{jobId}.jsonl` (`CronRunLogEntry` per line) +- **Run session transcripts**: `~/.openclaw/agents/{agentId}/sessions/{sessionId}.jsonl` + +The web app does NOT connect to the gateway directly -- it reads files from disk via Next.js API routes (same pattern as existing `/api/sessions`, `/api/web-sessions`). No gateway WebSocket integration needed. + +```mermaid +flowchart LR + subgraph disk [Disk Storage] + JobsJSON["~/.openclaw/cron/jobs.json"] + RunLogs["~/.openclaw/cron/runs/{jobId}.jsonl"] + Sessions["~/.openclaw/agents/{agentId}/sessions/{sessionId}.jsonl"] + end + subgraph api [Next.js API Routes] + CronJobsAPI["/api/cron/jobs"] + CronRunsAPI["/api/cron/jobs/{jobId}/runs"] + CronRunSessionAPI["/api/cron/runs/{sessionId}"] + end + subgraph ui [UI Components] + Sidebar["~cron virtual folder"] + Dashboard["CronDashboard"] + JobDetail["CronJobDetail"] + RunChat["CronRunChat"] + end + JobsJSON --> CronJobsAPI --> Dashboard + RunLogs --> CronRunsAPI --> JobDetail + Sessions --> CronRunSessionAPI --> RunChat + Dashboard --> Sidebar + JobDetail --> Sidebar + RunChat --> Sidebar +``` + +## 1. API Routes + +### `GET /api/cron/jobs` -- [apps/web/app/api/cron/jobs/route.ts](apps/web/app/api/cron/jobs/route.ts) + +Read `~/.openclaw/cron/jobs.json` and return all jobs with computed status fields. Also scan `~/.openclaw/cron/runs/` to get latest run info. Include heartbeat info by reading agent config from `~/.openclaw/config.yaml` (heartbeat interval, next due estimate). + +Returns: + +```typescript +{ + jobs: CronJob[], + heartbeat: { + intervalMs: number, + nextDueEstimateMs: number | null + }, + cronStatus: { + enabled: boolean, + nextWakeAtMs: number | null + } +} +``` + +### `GET /api/cron/jobs/[jobId]/runs` -- [apps/web/app/api/cron/jobs/[jobId]/runs/route.ts](apps/web/app/api/cron/jobs/[jobId]/runs/route.ts) + +Read `~/.openclaw/cron/runs/{jobId}.jsonl` using the same logic as `readCronRunLogEntries` in [src/cron/run-log.ts](src/cron/run-log.ts). Return entries with `?limit=N` support. + +Returns: + +```typescript +{ entries: CronRunLogEntry[] } +``` + +### `GET /api/cron/runs/[sessionId]` -- [apps/web/app/api/cron/runs/[sessionId]/route.ts](apps/web/app/api/cron/runs/[sessionId]/route.ts) + +Find and read `~/.openclaw/agents/*/sessions/{sessionId}.jsonl` (same search pattern as existing [apps/web/app/api/sessions/[sessionId]/route.ts](apps/web/app/api/sessions/[sessionId]/route.ts)). Parse JSONL into messages including `thinking` blocks, tool calls, and text -- return the full conversation with all parts preserved (not truncated like the existing session API). + +Returns: + +```typescript +{ + sessionId: string, + messages: Array<{ + id: string, + role: "user" | "assistant", + parts: Array< + | { type: "text", text: string } + | { type: "thinking", thinking: string } + | { type: "tool-call", toolName: string, args: unknown, output?: string } + >, + timestamp: string + }> +} +``` + +## 2. Sidebar Integration + +In [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx), add a `~cron` virtual folder alongside the existing `~chats` folder: + +- Fetch cron jobs from `/api/cron/jobs` on mount (same pattern as `fetchSessions`). +- Build a `~cron` `TreeNode` folder with children: + - Each job becomes a folder: `~cron/{jobId}` (name = job name, with status icon in the name string) + - Clicking `~cron` opens the cron dashboard view + - Clicking `~cron/{jobId}` opens the job detail view +- Handle `~cron` paths in `handleNodeSelect` (same pattern as `~chats`). + +```typescript +const cronFolder: TreeNode = { + name: "Cron", + path: "~cron", + type: "folder", + virtual: true, + children: cronJobs.map((j) => ({ + name: `${statusEmoji(j)} ${j.name}`, + path: `~cron/${j.id}`, + type: "folder", + virtual: true, + })), +}; +return [...tree, chatsFolder, cronFolder]; +``` + +## 3. UI Components + +### CronDashboard ([apps/web/app/components/cron/cron-dashboard.tsx](apps/web/app/components/cron/cron-dashboard.tsx)) + +Shown when user clicks `~cron` in sidebar. Displays: + +- **Heartbeat status card**: interval (e.g. "every 30m"), estimated next heartbeat as a live countdown, explanation of how heartbeat decides which cron jobs to run +- **Cron scheduler status**: enabled/disabled, next wake time as countdown +- **Jobs table/list**: all jobs with columns: + - Name, Schedule (human-readable), Status (running/ok/error/disabled/idle), Next run (countdown), Last run (time-ago + duration), Session target (main/isolated) + - Click a job row to navigate to its detail view +- **Timeline view**: visual timeline showing upcoming scheduled runs across all jobs (next 24h) + +### CronJobDetail ([apps/web/app/components/cron/cron-job-detail.tsx](apps/web/app/components/cron/cron-job-detail.tsx)) + +Shown when user clicks `~cron/{jobId}`. Displays: + +- **Job config header**: name, description, schedule (formatted), enabled status, session target, wake mode, payload summary +- **Next run countdown**: large countdown timer to next execution +- **Run history list**: each `CronRunLogEntry` as a card with: + - Timestamp, status badge (ok/error/skipped), duration, summary text + - Click to expand into full run chat (loads session transcript inline or navigates to run view) +- **Error details**: if `consecutiveErrors > 0`, show error streak and last error message + +### CronRunChat ([apps/web/app/components/cron/cron-run-chat.tsx](apps/web/app/components/cron/cron-run-chat.tsx)) + +Displayed inline when expanding a run in `CronJobDetail`. Renders the full session transcript: + +- Reuse `ChatMessage` and `ChainOfThought` components from [apps/web/app/components/chat-message.tsx](apps/web/app/components/chat-message.tsx) and [apps/web/app/components/chain-of-thought.tsx](apps/web/app/components/chain-of-thought.tsx) (or at minimum their rendering patterns) +- Show all thinking/reasoning blocks fully expanded (not collapsed, since these are historical runs) +- Show all tool calls with their inputs and outputs +- Show the system event / prompt that triggered the run +- Display run metadata at the top: model used, tokens consumed, duration + +## 4. Content Routing in Workspace Page + +Extend the `content` state in [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx) with new kinds: + +```typescript +type ContentState = + | { kind: "none" } + | { kind: "loading" } + | { kind: "document"; ... } + | { kind: "object"; ... } + | { kind: "cron-dashboard"; jobs: CronJob[]; heartbeat: HeartbeatInfo } + | { kind: "cron-job"; jobId: string; job: CronJob; runs: CronRunLogEntry[] } +``` + +In the main content area renderer, add cases for `cron-dashboard` and `cron-job` that render the corresponding components. + +## 5. Data Types (shared) + +Create [apps/web/app/types/cron.ts](apps/web/app/types/cron.ts) with client-side type definitions mirroring `CronJob`, `CronRunLogEntry`, `CronSchedule`, `CronJobState` etc. (copy from [src/cron/types.ts](src/cron/types.ts) and [src/cron/run-log.ts](src/cron/run-log.ts), stripped of server-only imports). + +## Key Design Decisions + +- **Read-only**: The cron UI is read-only (view jobs, runs, transcripts). No add/edit/delete/enable/disable from the web UI -- that stays in the CLI. +- **No gateway connection**: All data read directly from disk via API routes, consistent with the existing web app pattern. +- **Reuse existing components**: Chat rendering reuses `ChatMessage`/`ChainOfThought` patterns rather than building from scratch. +- **Live countdowns**: Use `setInterval` for countdown timers (next heartbeat, next cron run) with periodic refetch of job data to stay current. +- **Auto-refresh**: Poll `/api/cron/jobs` every ~30s to catch state changes from the running gateway. diff --git a/.cursor/plans/workspace_profile_support_7e8600ec.plan.md b/.cursor/plans/workspace_profile_support_7e8600ec.plan.md new file mode 100644 index 00000000000..4f76fc75b91 --- /dev/null +++ b/.cursor/plans/workspace_profile_support_7e8600ec.plan.md @@ -0,0 +1,124 @@ +--- +name: Workspace profile support +overview: Add full workspace profile and custom path support to the Ironclaw web app and the dench SKILL.md, so they respect OPENCLAW_PROFILE, OPENCLAW_HOME, OPENCLAW_STATE_DIR, and per-agent workspace config β€” matching the CLI's existing resolution logic. +todos: + - id: centralize-helpers + content: Add resolveOpenClawStateDir() to apps/web/lib/workspace.ts and update resolveWorkspaceRoot() with OPENCLAW_PROFILE + OPENCLAW_HOME + OPENCLAW_STATE_DIR support + status: pending + - id: fix-api-routes + content: Replace all hardcoded join(homedir(), '.openclaw', ...) in ~13 web app API routes/lib files with the new shared helpers + status: pending + - id: fix-empty-state-ui + content: Make empty-state.tsx show the resolved workspace path dynamically instead of hardcoded ~/.openclaw/workspace + status: pending + - id: fix-system-prompt + content: Replace hardcoded ~/.openclaw/web-chat/ in system-prompt.ts line 173 with a dynamic path from the state dir context + status: pending + - id: skill-substitution + content: Add workspace path substitution in buildWorkspaceSkillSnapshot() so injected SKILL.md content replaces ~/.openclaw/workspace with the actual resolved workspace dir + status: pending + - id: tree-api-profile + content: Expose active profile name in the tree API response so the UI can show profile-aware workspace labels + status: pending +isProject: false +--- + +# Full Workspace Profile and Custom Path Support + +## Problem + +The CLI core (`src/agents/workspace.ts`, `src/config/paths.ts`) already resolves workspace paths dynamically via `OPENCLAW_PROFILE`, `OPENCLAW_HOME`, `OPENCLAW_STATE_DIR`, and per-agent config β€” but the web app (`apps/web/`) and the injected dench skill (`skills/dench/SKILL.md`) hardcode `~/.openclaw` and `~/.openclaw/workspace` everywhere, ignoring profiles entirely. + +**35 hardcoded `~/.openclaw` references** in `SKILL.md`, and **~15 hardcoded paths** across the web app API routes and UI. + +## Approach + +### 1. Centralize path resolution in the web app + +Create two new helpers in [apps/web/lib/workspace.ts](apps/web/lib/workspace.ts) and update the existing `resolveWorkspaceRoot()`: + +- `**resolveOpenClawStateDir()**` β€” mirrors `src/config/paths.ts:resolveStateDir()` logic: checks `OPENCLAW_STATE_DIR` env var, then `OPENCLAW_HOME`, falls back to `~/.openclaw`. Returns the base state directory (e.g. `~/.openclaw`). +- **Update `resolveWorkspaceRoot()**`β€” add`OPENCLAW_PROFILE`awareness between the`OPENCLAW_WORKSPACE` check and the fallback: + 1. `OPENCLAW_WORKSPACE` env var (existing) + 2. `OPENCLAW_PROFILE` -> `/workspace-` (new) + 3. `/workspace` (existing, but now uses `resolveOpenClawStateDir()` instead of hardcoded `~/.openclaw`) + +```typescript +export function resolveOpenClawStateDir(): string { + const override = process.env.OPENCLAW_STATE_DIR?.trim(); + if (override) return override.startsWith("~") ? join(homedir(), override.slice(1)) : override; + const home = process.env.OPENCLAW_HOME?.trim() || homedir(); + return join(home, ".openclaw"); +} + +export function resolveWorkspaceRoot(): string | null { + const stateDir = resolveOpenClawStateDir(); + const profile = process.env.OPENCLAW_PROFILE?.trim(); + const candidates = [ + process.env.OPENCLAW_WORKSPACE, + profile && profile.toLowerCase() !== "default" ? join(stateDir, `workspace-${profile}`) : null, + join(stateDir, "workspace"), + ].filter(Boolean) as string[]; + + for (const dir of candidates) { + if (existsSync(dir)) return dir; + } + return null; +} +``` + +### 2. Replace all hardcoded paths in web app API routes + +Every file below uses `join(homedir(), ".openclaw", ...)` directly. Replace with calls to `resolveOpenClawStateDir()` or `resolveWorkspaceRoot()`: + +| File | What to change | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `apps/web/app/api/workspace/tree/route.ts` | `join(home, ".openclaw", "skills")` and `join(home, ".openclaw")` -> `resolveOpenClawStateDir()` | +| `apps/web/app/api/workspace/virtual-file/route.ts` | All 6 hardcoded paths in `resolveVirtualPath()` and `isSafePath()` -> derive from `resolveWorkspaceRoot()` and `resolveOpenClawStateDir()` | +| `apps/web/app/api/skills/route.ts` | `join(openclawDir, "skills")` and `join(openclawDir, "workspace", "skills")` -> use both helpers | +| `apps/web/app/api/sessions/route.ts` | `resolveOpenClawDir()` local helper -> use shared `resolveOpenClawStateDir()` | +| `apps/web/app/api/memories/route.ts` | `join(homedir(), ".openclaw", "workspace")` -> `resolveWorkspaceRoot()` | +| `apps/web/app/api/cron/jobs/route.ts` | Module-level `CRON_DIR` and `agentsDir` -> derive from `resolveOpenClawStateDir()` | +| `apps/web/app/api/cron/runs/search-transcript/route.ts` | agents dir -> `resolveOpenClawStateDir()` | +| `apps/web/app/api/cron/runs/[sessionId]/route.ts` | agents dir -> `resolveOpenClawStateDir()` | +| `apps/web/app/api/cron/jobs/[jobId]/runs/route.ts` | if hardcoded -> `resolveOpenClawStateDir()` | +| `apps/web/app/api/web-sessions/route.ts` | `WEB_CHAT_DIR` -> derive from `resolveOpenClawStateDir()` | +| `apps/web/app/api/web-sessions/[id]/route.ts` | same | +| `apps/web/app/api/web-sessions/[id]/messages/route.ts` | same | +| `apps/web/lib/active-runs.ts` | `WEB_CHAT_DIR` -> derive from `resolveOpenClawStateDir()` | + +### 3. Update the UI empty state + +In [apps/web/app/components/workspace/empty-state.tsx](apps/web/app/components/workspace/empty-state.tsx) (line 128): the hardcoded `~/.openclaw/workspace` display string should be dynamic. Two options: + +- **Option A**: Pass the resolved workspace path from the tree API response (it already returns `workspaceRoot`). The empty state can show that or a user-friendly tilde-collapsed version. +- **Option B**: Add an API endpoint or server component that returns the expected workspace path. + +Option A is simplest β€” the tree API already returns `openclawDir` and `workspaceRoot`. Thread the expected path into the empty state component. + +### 4. Fix hardcoded path in system prompt + +In [src/agents/system-prompt.ts](src/agents/system-prompt.ts) line 173: the hardcoded `~/.openclaw/web-chat/` should use the `workspaceDir` parameter (or derive from the state dir that's already available in the prompt builder context). Replace with a template string that references the actual state directory. + +### 5. Add workspace variable substitution for injected SKILL.md content + +The dench `SKILL.md` has **35 instances** of `~/.openclaw/workspace`. Since this content is injected verbatim into the system prompt via `readSkillContent()`, we need a substitution mechanism. + +In [src/agents/skills/workspace.ts](src/agents/skills/workspace.ts) around line 271 where `readSkillContent()` is called for injected skills: + +```typescript +// After reading content, substitute workspace path placeholders +const content = readSkillContent(entry.skill.filePath); +if (content) { + const resolved = content.replaceAll("~/.openclaw/workspace", workspaceDir); + injectedSkills.push({ name: entry.skill.name, content: resolved }); +} +``` + +This requires threading `workspaceDir` into `buildWorkspaceSkillSnapshot()` β€” which it already receives as its first argument. + +Then update `skills/dench/SKILL.md` to use `~/.openclaw/workspace` as a canonical placeholder (it already does), and the substitution will replace it with the actual resolved path at injection time. No changes needed to the SKILL.md content itself. + +### 6. Expose workspace info in the tree API response + +The tree API ([apps/web/app/api/workspace/tree/route.ts](apps/web/app/api/workspace/tree/route.ts)) already returns `workspaceRoot` and `openclawDir`. Consider also returning `profile` (from `OPENCLAW_PROFILE`) so the UI can display profile-aware context (e.g. "Workspace (staging)" in the sidebar). diff --git a/.cursor/worktrees.json b/.cursor/worktrees.json new file mode 100644 index 00000000000..c81f1214f24 --- /dev/null +++ b/.cursor/worktrees.json @@ -0,0 +1,3 @@ +{ + "setup-worktree": ["npm install"] +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0a965febb1c..e5a410a3107 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -47,80 +47,3 @@ updates: - minor - patch open-pull-requests-limit: 5 - - # Swift Package Manager - macOS app - - package-ecosystem: swift - directory: /apps/macos - schedule: - interval: weekly - cooldown: - default-days: 7 - groups: - swift-deps: - patterns: - - "*" - update-types: - - minor - - patch - open-pull-requests-limit: 5 - - # Swift Package Manager - shared MoltbotKit - - package-ecosystem: swift - directory: /apps/shared/MoltbotKit - schedule: - interval: weekly - cooldown: - default-days: 7 - groups: - swift-deps: - patterns: - - "*" - update-types: - - minor - - patch - open-pull-requests-limit: 5 - - # Swift Package Manager - Swabble - - package-ecosystem: swift - directory: /Swabble - schedule: - interval: weekly - cooldown: - default-days: 7 - groups: - swift-deps: - patterns: - - "*" - update-types: - - minor - - patch - open-pull-requests-limit: 5 - - # Gradle - Android app - - package-ecosystem: gradle - directory: /apps/android - schedule: - interval: weekly - cooldown: - default-days: 7 - groups: - android-deps: - patterns: - - "*" - update-types: - - minor - - patch - open-pull-requests-limit: 5 - - # Docker base images - - package-ecosystem: docker - directory: / - schedule: - interval: weekly - cooldown: - default-days: 7 - groups: - docker-images: - patterns: - - "*" - open-pull-requests-limit: 5 diff --git a/.github/workflows/auto-response.yml b/.github/workflows/auto-response.yml deleted file mode 100644 index 1502456a251..00000000000 --- a/.github/workflows/auto-response.yml +++ /dev/null @@ -1,224 +0,0 @@ -name: Auto response - -on: - issues: - types: [opened, edited, labeled] - pull_request_target: - types: [labeled] - -permissions: {} - -jobs: - auto-response: - permissions: - issues: write - pull-requests: write - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 - id: app-token - with: - app-id: "2729701" - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Handle labeled items - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - // Labels prefixed with "r:" are auto-response triggers. - const rules = [ - { - label: "r: skill", - close: true, - message: - "Thanks for the contribution! New skills should be published to [Clawhub](https://clawhub.ai) for everyone to use. We’re keeping the core lean on skills, so I’m closing this out.", - }, - { - label: "r: support", - close: true, - message: - "Please use [our support server](https://discord.gg/clawd) and ask in #help or #users-helping-users to resolve this, or follow the stuck FAQ at https://docs.openclaw.ai/help/faq#im-stuck-whats-the-fastest-way-to-get-unstuck.", - }, - { - label: "r: testflight", - close: true, - message: "Not available, build from source.", - }, - { - label: "r: third-party-extension", - close: true, - message: - "Please make this as a third-party plugin that you maintain yourself in your own repo. Docs: https://docs.openclaw.ai/plugin. Feel free to open a PR after to add it to our community plugins page: https://docs.openclaw.ai/plugins/community", - }, - { - label: "r: moltbook", - close: true, - lock: true, - lockReason: "off-topic", - message: - "OpenClaw is not affiliated with Moltbook, and issues related to Moltbook should not be submitted here.", - }, - ]; - - const triggerLabel = "trigger-response"; - const target = context.payload.issue ?? context.payload.pull_request; - if (!target) { - return; - } - - const labelSet = new Set( - (target.labels ?? []) - .map((label) => (typeof label === "string" ? label : label?.name)) - .filter((name) => typeof name === "string"), - ); - - const hasTriggerLabel = labelSet.has(triggerLabel); - if (hasTriggerLabel) { - labelSet.delete(triggerLabel); - try { - await github.rest.issues.removeLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: target.number, - name: triggerLabel, - }); - } catch (error) { - if (error?.status !== 404) { - throw error; - } - } - } - - const isLabelEvent = context.payload.action === "labeled"; - if (!hasTriggerLabel && !isLabelEvent) { - return; - } - - const issue = context.payload.issue; - if (issue) { - const title = issue.title ?? ""; - const body = issue.body ?? ""; - const haystack = `${title}\n${body}`.toLowerCase(); - const hasMoltbookLabel = labelSet.has("r: moltbook"); - const hasTestflightLabel = labelSet.has("r: testflight"); - const hasSecurityLabel = labelSet.has("security"); - if (title.toLowerCase().includes("security") && !hasSecurityLabel) { - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - labels: ["security"], - }); - labelSet.add("security"); - } - if (title.toLowerCase().includes("testflight") && !hasTestflightLabel) { - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - labels: ["r: testflight"], - }); - labelSet.add("r: testflight"); - } - if (haystack.includes("moltbook") && !hasMoltbookLabel) { - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - labels: ["r: moltbook"], - }); - labelSet.add("r: moltbook"); - } - } - - const invalidLabel = "invalid"; - const dirtyLabel = "dirty"; - const noisyPrMessage = - "Closing this PR because it looks dirty (too many unrelated or unexpected changes). This usually happens when a branch picks up unrelated commits or a merge went sideways. Please recreate the PR from a clean branch."; - - const pullRequest = context.payload.pull_request; - if (pullRequest) { - if (labelSet.has(dirtyLabel)) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - body: noisyPrMessage, - }); - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - state: "closed", - }); - return; - } - const labelCount = labelSet.size; - if (labelCount > 20) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - body: noisyPrMessage, - }); - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - state: "closed", - }); - return; - } - if (labelSet.has(invalidLabel)) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - state: "closed", - }); - return; - } - } - - if (issue && labelSet.has(invalidLabel)) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - state: "closed", - state_reason: "not_planned", - }); - return; - } - - const rule = rules.find((item) => labelSet.has(item.label)); - if (!rule) { - return; - } - - const issueNumber = target.number; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body: rule.message, - }); - - if (rule.close) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - state: "closed", - }); - } - - if (rule.lock) { - await github.rest.issues.lock({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - lock_reason: rule.lockReason ?? "resolved", - }); - } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb5b50a5ce..0f0b5bb56f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,186 +10,30 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - # Detect docs-only changes to skip heavy jobs (test, build, Windows, macOS, Android). - # Lint and format always run. Fail-safe: if detection fails, run everything. - docs-scope: - runs-on: blacksmith-16vcpu-ubuntu-2404 - outputs: - docs_only: ${{ steps.check.outputs.docs_only }} - docs_changed: ${{ steps.check.outputs.docs_changed }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false + # check: + # name: "check" + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # submodules: false + # + # - name: Setup Node environment + # uses: ./.github/actions/setup-node-env + # + # - name: Check types and lint and oxfmt + # run: pnpm check - - name: Detect docs-only changes - id: check - uses: ./.github/actions/detect-docs-changes - - # Detect which heavy areas are touched so PRs can skip unrelated expensive jobs. - # Push to main keeps broad coverage. - changed-scope: - needs: [docs-scope] - if: needs.docs-scope.outputs.docs_only != 'true' - runs-on: blacksmith-16vcpu-ubuntu-2404 - outputs: - run_node: ${{ steps.scope.outputs.run_node }} - run_macos: ${{ steps.scope.outputs.run_macos }} - run_android: ${{ steps.scope.outputs.run_android }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - - - name: Detect changed scopes - id: scope - shell: bash - run: | - set -euo pipefail - - if [ "${{ github.event_name }}" = "push" ]; then - BASE="${{ github.event.before }}" - else - BASE="${{ github.event.pull_request.base.sha }}" - fi - - CHANGED="$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo "UNKNOWN")" - if [ "$CHANGED" = "UNKNOWN" ] || [ -z "$CHANGED" ]; then - # Fail-safe: run broad checks if detection fails. - echo "run_node=true" >> "$GITHUB_OUTPUT" - echo "run_macos=true" >> "$GITHUB_OUTPUT" - echo "run_android=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - run_node=false - run_macos=false - run_android=false - has_non_docs=false - has_non_native_non_docs=false - - while IFS= read -r path; do - [ -z "$path" ] && continue - case "$path" in - docs/*|*.md|*.mdx) - continue - ;; - *) - has_non_docs=true - ;; - esac - - case "$path" in - # Generated protocol models are already covered by protocol:check and - # should not force the full native macOS lane. - apps/macos/Sources/OpenClawProtocol/*|apps/shared/OpenClawKit/Sources/OpenClawProtocol/*) - ;; - apps/macos/*|apps/ios/*|apps/shared/*|Swabble/*) - run_macos=true - ;; - esac - - case "$path" in - apps/android/*|apps/shared/*) - run_android=true - ;; - esac - - case "$path" in - src/*|test/*|extensions/*|packages/*|scripts/*|ui/*|.github/*|openclaw.mjs|package.json|pnpm-lock.yaml|pnpm-workspace.yaml|tsconfig*.json|vitest*.ts|tsdown.config.ts|.oxlintrc.json|.oxfmtrc.jsonc) - run_node=true - ;; - esac - - case "$path" in - apps/android/*|apps/ios/*|apps/macos/*|apps/shared/*|Swabble/*|appcast.xml) - ;; - *) - has_non_native_non_docs=true - ;; - esac - done <<< "$CHANGED" - - # If there are non-doc files outside native app trees, keep Node checks enabled. - if [ "$run_node" = false ] && [ "$has_non_docs" = true ] && [ "$has_non_native_non_docs" = true ]; then - run_node=true - fi - - echo "run_node=${run_node}" >> "$GITHUB_OUTPUT" - echo "run_macos=${run_macos}" >> "$GITHUB_OUTPUT" - echo "run_android=${run_android}" >> "$GITHUB_OUTPUT" - - # Build dist once for Node-relevant changes and share it with downstream jobs. - build-artifacts: - needs: [docs-scope, changed-scope, check] - if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Build dist - run: pnpm build - - - name: Upload dist artifact - uses: actions/upload-artifact@v4 - with: - name: dist-build - path: dist/ - retention-days: 1 - - # Validate npm pack contents after build (only on push to main, not PRs). - release-check: - needs: [docs-scope, build-artifacts] - if: github.event_name == 'push' && needs.docs-scope.outputs.docs_only != 'true' - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Download dist artifact - uses: actions/download-artifact@v4 - with: - name: dist-build - path: dist/ - - - name: Check release contents - run: pnpm release:check - - checks: - needs: [docs-scope, changed-scope, check] - if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') - runs-on: blacksmith-16vcpu-ubuntu-2404 + test: + runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - runtime: node - task: test command: pnpm canvas:a2ui:bundle && pnpm test - - runtime: node - task: protocol - command: pnpm protocol:check - runtime: bun - task: test command: pnpm canvas:a2ui:bundle && bunx vitest run --config vitest.unit.config.ts steps: - name: Skip bun lane on push @@ -208,537 +52,11 @@ jobs: with: install-bun: "${{ matrix.runtime == 'bun' }}" - - name: Configure vitest JSON reports - if: (github.event_name != 'push' || matrix.runtime != 'bun') && matrix.task == 'test' && matrix.runtime == 'node' - run: echo "OPENCLAW_VITEST_REPORT_DIR=$RUNNER_TEMP/vitest-reports" >> "$GITHUB_ENV" - - name: Configure Node test resources - if: (github.event_name != 'push' || matrix.runtime != 'bun') && matrix.task == 'test' && matrix.runtime == 'node' + if: matrix.runtime == 'node' run: | - # `pnpm test` runs `scripts/test-parallel.mjs`, which spawns multiple Node processes. - # Default heap limits have been too low on Linux CI (V8 OOM near 4GB). echo "OPENCLAW_TEST_WORKERS=2" >> "$GITHUB_ENV" - echo "OPENCLAW_TEST_MAX_OLD_SPACE_SIZE_MB=6144" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_MAX_OLD_SPACE_SIZE_MB=8192" >> "$GITHUB_ENV" - - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) - if: matrix.runtime != 'bun' || github.event_name != 'push' - run: ${{ matrix.command }} - - - name: Summarize slowest tests - if: (github.event_name != 'push' || matrix.runtime != 'bun') && matrix.task == 'test' && matrix.runtime == 'node' - run: | - node scripts/vitest-slowest.mjs --dir "$OPENCLAW_VITEST_REPORT_DIR" --top 50 --out "$RUNNER_TEMP/vitest-slowest.md" > /dev/null - echo "Slowest test summary written to $RUNNER_TEMP/vitest-slowest.md" - - - name: Upload vitest reports - if: (github.event_name != 'push' || matrix.runtime != 'bun') && matrix.task == 'test' && matrix.runtime == 'node' - uses: actions/upload-artifact@v4 - with: - name: vitest-reports-${{ runner.os }}-${{ matrix.runtime }} - path: | - ${{ env.OPENCLAW_VITEST_REPORT_DIR }} - ${{ runner.temp }}/vitest-slowest.md - - # Types, lint, and format check. - check: - name: "check" - needs: [docs-scope] - if: needs.docs-scope.outputs.docs_only != 'true' - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Check types and lint and oxfmt - run: pnpm check - - # Report-only dead-code scans. Runs after scope detection and stores machine-readable - # results as artifacts for later triage before we enable hard gates. - # Temporarily disabled in CI while we process initial findings. - deadcode: - name: dead-code report - needs: [docs-scope, changed-scope] - # if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') - if: false - runs-on: blacksmith-16vcpu-ubuntu-2404 - strategy: - fail-fast: false - matrix: - include: - - tool: knip - command: pnpm deadcode:report:ci:knip - - tool: ts-prune - command: pnpm deadcode:report:ci:ts-prune - - tool: ts-unused-exports - command: pnpm deadcode:report:ci:ts-unused - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Run ${{ matrix.tool }} dead-code scan - run: ${{ matrix.command }} - - - name: Upload dead-code results - uses: actions/upload-artifact@v4 - with: - name: dead-code-${{ matrix.tool }}-${{ github.run_id }} - path: .artifacts/deadcode - - # Validate docs (format, lint, broken links) only when docs files changed. - check-docs: - needs: [docs-scope] - if: needs.docs-scope.outputs.docs_changed == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Check docs - run: pnpm check:docs - - secrets: - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install detect-secrets - run: | - python -m pip install --upgrade pip - python -m pip install detect-secrets==1.5.0 - - - name: Detect secrets - run: | - if ! detect-secrets scan --baseline .secrets.baseline; then - echo "::error::Secret scanning failed. See docs/gateway/security.md#secret-scanning-detect-secrets" - exit 1 - fi - - checks-windows: - needs: [docs-scope, changed-scope, build-artifacts, check] - if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') - runs-on: blacksmith-16vcpu-windows-2025 - env: - NODE_OPTIONS: --max-old-space-size=4096 - # Keep total concurrency predictable on the 16 vCPU runner: - # `scripts/test-parallel.mjs` runs some vitest suites in parallel processes. - OPENCLAW_TEST_WORKERS: 2 - defaults: - run: - shell: bash - strategy: - fail-fast: false - matrix: - include: - - runtime: node - task: lint - command: pnpm lint - - runtime: node - task: test - command: pnpm canvas:a2ui:bundle && pnpm test - - runtime: node - task: protocol - command: pnpm protocol:check - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Try to exclude workspace from Windows Defender (best-effort) - shell: pwsh - run: | - $cmd = Get-Command Add-MpPreference -ErrorAction SilentlyContinue - if (-not $cmd) { - Write-Host "Add-MpPreference not available, skipping Defender exclusions." - exit 0 - } - - try { - # Defender sometimes intercepts process spawning (vitest workers). If this fails - # (eg hardened images), keep going and rely on worker limiting above. - Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" -ErrorAction Stop - Add-MpPreference -ExclusionProcess "node.exe" -ErrorAction Stop - Write-Host "Defender exclusions applied." - } catch { - Write-Warning "Failed to apply Defender exclusions, continuing. $($_.Exception.Message)" - } - - - name: Download dist artifact (lint lane) - if: matrix.task == 'lint' - uses: actions/download-artifact@v4 - with: - name: dist-build - path: dist/ - - - name: Verify dist artifact (lint lane) - if: matrix.task == 'lint' - run: | - set -euo pipefail - test -s dist/index.js - test -s dist/plugin-sdk/index.js - - - name: Setup Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Runtime versions - run: | - node -v - npm -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true - - - name: Configure vitest JSON reports - if: matrix.task == 'test' - run: echo "OPENCLAW_VITEST_REPORT_DIR=$RUNNER_TEMP/vitest-reports" >> "$GITHUB_ENV" - - - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) - run: ${{ matrix.command }} - - - name: Summarize slowest tests - if: matrix.task == 'test' - run: | - node scripts/vitest-slowest.mjs --dir "$OPENCLAW_VITEST_REPORT_DIR" --top 50 --out "$RUNNER_TEMP/vitest-slowest.md" > /dev/null - echo "Slowest test summary written to $RUNNER_TEMP/vitest-slowest.md" - - - name: Upload vitest reports - if: matrix.task == 'test' - uses: actions/upload-artifact@v4 - with: - name: vitest-reports-${{ runner.os }}-${{ matrix.runtime }} - path: | - ${{ env.OPENCLAW_VITEST_REPORT_DIR }} - ${{ runner.temp }}/vitest-slowest.md - - # Consolidated macOS job: runs TS tests + Swift lint/build/test sequentially - # on a single runner. GitHub limits macOS concurrent jobs to 5 per org; - # running 4 separate jobs per PR (as before) starved the queue. One job - # per PR allows 5 PRs to run macOS checks simultaneously. - macos: - needs: [docs-scope, changed-scope, check] - if: github.event_name == 'pull_request' && needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_macos == 'true' - runs-on: macos-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - # --- Run all checks sequentially (fast gates first) --- - - name: TS tests (macOS) - env: - NODE_OPTIONS: --max-old-space-size=4096 - run: pnpm test - - # --- Xcode/Swift setup --- - - name: Select Xcode 26.1 - run: | - sudo xcode-select -s /Applications/Xcode_26.1.app - xcodebuild -version - - - name: Install XcodeGen / SwiftLint / SwiftFormat - run: brew install xcodegen swiftlint swiftformat - - - name: Show toolchain - run: | - sw_vers - xcodebuild -version - swift --version - - - name: Swift lint - run: | - swiftlint --config .swiftlint.yml - swiftformat --lint apps/macos/Sources --config .swiftformat - - - name: Cache SwiftPM - uses: actions/cache@v4 - with: - path: ~/Library/Caches/org.swift.swiftpm - key: ${{ runner.os }}-swiftpm-${{ hashFiles('apps/macos/Package.resolved') }} - restore-keys: | - ${{ runner.os }}-swiftpm- - - - name: Swift build (release) - run: | - set -euo pipefail - for attempt in 1 2 3; do - if swift build --package-path apps/macos --configuration release; then - exit 0 - fi - echo "swift build failed (attempt $attempt/3). Retrying…" - sleep $((attempt * 20)) - done - exit 1 - - - name: Swift test - run: | - set -euo pipefail - for attempt in 1 2 3; do - if swift test --package-path apps/macos --parallel --enable-code-coverage --show-codecov-path; then - exit 0 - fi - echo "swift test failed (attempt $attempt/3). Retrying…" - sleep $((attempt * 20)) - done - exit 1 - - ios: - if: false # ignore iOS in CI for now - runs-on: macos-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Select Xcode 26.1 - run: | - sudo xcode-select -s /Applications/Xcode_26.1.app - xcodebuild -version - - - name: Install XcodeGen - run: brew install xcodegen - - - name: Install SwiftLint / SwiftFormat - run: brew install swiftlint swiftformat - - - name: Show toolchain - run: | - sw_vers - xcodebuild -version - swift --version - - - name: Generate iOS project - run: | - cd apps/ios - xcodegen generate - - - name: iOS tests - run: | - set -euo pipefail - RESULT_BUNDLE_PATH="$RUNNER_TEMP/Clawdis-iOS.xcresult" - DEST_ID="$( - python3 - <<'PY' - import json - import subprocess - import sys - import uuid - - def sh(args: list[str]) -> str: - return subprocess.check_output(args, text=True).strip() - - # Prefer an already-created iPhone simulator if it exists. - devices = json.loads(sh(["xcrun", "simctl", "list", "devices", "-j"])) - candidates: list[tuple[str, str]] = [] - for runtime, devs in (devices.get("devices") or {}).items(): - for dev in devs or []: - if not dev.get("isAvailable"): - continue - name = str(dev.get("name") or "") - udid = str(dev.get("udid") or "") - if not udid or not name.startswith("iPhone"): - continue - candidates.append((name, udid)) - - candidates.sort(key=lambda it: (0 if "iPhone 16" in it[0] else 1, it[0])) - if candidates: - print(candidates[0][1]) - sys.exit(0) - - # Otherwise, create one from the newest available iOS runtime. - runtimes = json.loads(sh(["xcrun", "simctl", "list", "runtimes", "-j"])).get("runtimes") or [] - ios = [rt for rt in runtimes if rt.get("platform") == "iOS" and rt.get("isAvailable")] - if not ios: - print("No available iOS runtimes found.", file=sys.stderr) - sys.exit(1) - - def version_key(rt: dict) -> tuple[int, ...]: - parts: list[int] = [] - for p in str(rt.get("version") or "0").split("."): - try: - parts.append(int(p)) - except ValueError: - parts.append(0) - return tuple(parts) - - ios.sort(key=version_key, reverse=True) - runtime = ios[0] - runtime_id = str(runtime.get("identifier") or "") - if not runtime_id: - print("Missing iOS runtime identifier.", file=sys.stderr) - sys.exit(1) - - supported = runtime.get("supportedDeviceTypes") or [] - iphones = [dt for dt in supported if dt.get("productFamily") == "iPhone"] - if not iphones: - print("No iPhone device types for iOS runtime.", file=sys.stderr) - sys.exit(1) - - iphones.sort( - key=lambda dt: ( - 0 if "iPhone 16" in str(dt.get("name") or "") else 1, - str(dt.get("name") or ""), - ) - ) - device_type_id = str(iphones[0].get("identifier") or "") - if not device_type_id: - print("Missing iPhone device type identifier.", file=sys.stderr) - sys.exit(1) - - sim_name = f"CI iPhone {uuid.uuid4().hex[:8]}" - udid = sh(["xcrun", "simctl", "create", sim_name, device_type_id, runtime_id]) - if not udid: - print("Failed to create iPhone simulator.", file=sys.stderr) - sys.exit(1) - print(udid) - PY - )" - echo "Using iOS Simulator id: $DEST_ID" - xcodebuild test \ - -project apps/ios/Clawdis.xcodeproj \ - -scheme Clawdis \ - -destination "platform=iOS Simulator,id=$DEST_ID" \ - -resultBundlePath "$RESULT_BUNDLE_PATH" \ - -enableCodeCoverage YES - - - name: iOS coverage summary - run: | - set -euo pipefail - RESULT_BUNDLE_PATH="$RUNNER_TEMP/Clawdis-iOS.xcresult" - xcrun xccov view --report --only-targets "$RESULT_BUNDLE_PATH" - - - name: iOS coverage gate (43%) - run: | - set -euo pipefail - RESULT_BUNDLE_PATH="$RUNNER_TEMP/Clawdis-iOS.xcresult" - RESULT_BUNDLE_PATH="$RESULT_BUNDLE_PATH" python3 - <<'PY' - import json - import os - import subprocess - import sys - - target_name = "Clawdis.app" - minimum = 0.43 - - report = json.loads( - subprocess.check_output( - ["xcrun", "xccov", "view", "--report", "--json", os.environ["RESULT_BUNDLE_PATH"]], - text=True, - ) - ) - - target_coverage = None - for target in report.get("targets", []): - if target.get("name") == target_name: - target_coverage = float(target["lineCoverage"]) - break - - if target_coverage is None: - print(f"Could not find coverage for target: {target_name}") - sys.exit(1) - - print(f"{target_name} line coverage: {target_coverage * 100:.2f}% (min {minimum * 100:.2f}%)") - if target_coverage + 1e-12 < minimum: - sys.exit(1) - PY - - android: - needs: [docs-scope, changed-scope, check] - if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_android == 'true') - runs-on: blacksmith-16vcpu-ubuntu-2404 - strategy: - fail-fast: false - matrix: - include: - - task: test - command: ./gradlew --no-daemon :app:testDebugUnitTest - - task: build - command: ./gradlew --no-daemon :app:assembleDebug - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Setup Java - uses: actions/setup-java@v4 - with: - distribution: temurin - # setup-android's sdkmanager currently crashes on JDK 21 in CI. - java-version: 17 - - - name: Setup Android SDK - uses: android-actions/setup-android@v3 - with: - accept-android-sdk-licenses: false - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - with: - gradle-version: 8.11.1 - - - name: Install Android SDK packages - run: | - yes | sdkmanager --licenses >/dev/null - sdkmanager --install \ - "platform-tools" \ - "platforms;android-36" \ - "build-tools;36.0.0" - - - name: Run Android ${{ matrix.task }} - working-directory: apps/android + - name: Run tests (${{ matrix.runtime }}) run: ${{ matrix.command }} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml deleted file mode 100644 index fc0d97d4091..00000000000 --- a/.github/workflows/docker-release.yml +++ /dev/null @@ -1,198 +0,0 @@ -name: Docker Release - -on: - push: - branches: - - main - tags: - - "v*" - paths-ignore: - - "docs/**" - - "**/*.md" - - "**/*.mdx" - - ".agents/**" - - "skills/**" - -concurrency: - group: docker-release-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - # Build amd64 image - build-amd64: - runs-on: blacksmith-16vcpu-ubuntu-2404 - permissions: - packages: write - contents: read - outputs: - image-digest: ${{ steps.build.outputs.digest }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Resolve image tags (amd64) - id: tags - shell: bash - env: - IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - run: | - set -euo pipefail - tags=() - if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - tags+=("${IMAGE}:main-amd64") - fi - if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then - version="${GITHUB_REF#refs/tags/v}" - tags+=("${IMAGE}:${version}-amd64") - fi - if [[ ${#tags[@]} -eq 0 ]]; then - echo "::error::No amd64 tags resolved for ref ${GITHUB_REF}" - exit 1 - fi - { - echo "value<> "$GITHUB_OUTPUT" - - - name: Build and push amd64 image - id: build - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64 - tags: ${{ steps.tags.outputs.value }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:amd64 - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:amd64,mode=max - provenance: false - push: true - - # Build arm64 image - build-arm64: - runs-on: blacksmith-16vcpu-ubuntu-2404-arm - permissions: - packages: write - contents: read - outputs: - image-digest: ${{ steps.build.outputs.digest }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Resolve image tags (arm64) - id: tags - shell: bash - env: - IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - run: | - set -euo pipefail - tags=() - if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - tags+=("${IMAGE}:main-arm64") - fi - if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then - version="${GITHUB_REF#refs/tags/v}" - tags+=("${IMAGE}:${version}-arm64") - fi - if [[ ${#tags[@]} -eq 0 ]]; then - echo "::error::No arm64 tags resolved for ref ${GITHUB_REF}" - exit 1 - fi - { - echo "value<> "$GITHUB_OUTPUT" - - - name: Build and push arm64 image - id: build - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/arm64 - tags: ${{ steps.tags.outputs.value }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:arm64 - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:arm64,mode=max - provenance: false - push: true - - # Create multi-platform manifest - create-manifest: - runs-on: blacksmith-16vcpu-ubuntu-2404 - permissions: - packages: write - contents: read - needs: [build-amd64, build-arm64] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Resolve manifest tags - id: tags - shell: bash - env: - IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - run: | - set -euo pipefail - tags=() - if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - tags+=("${IMAGE}:main") - fi - if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then - version="${GITHUB_REF#refs/tags/v}" - tags+=("${IMAGE}:${version}") - fi - if [[ ${#tags[@]} -eq 0 ]]; then - echo "::error::No manifest tags resolved for ref ${GITHUB_REF}" - exit 1 - fi - { - echo "value<> "$GITHUB_OUTPUT" - - - name: Create and push manifest - shell: bash - run: | - set -euo pipefail - mapfile -t tags <<< "${{ steps.tags.outputs.value }}" - args=() - for tag in "${tags[@]}"; do - [ -z "$tag" ] && continue - args+=("-t" "$tag") - done - docker buildx imagetools create "${args[@]}" \ - ${{ needs.build-amd64.outputs.image-digest }} \ - ${{ needs.build-arm64.outputs.image-digest }} diff --git a/.github/workflows/install-smoke.yml b/.github/workflows/install-smoke.yml deleted file mode 100644 index 03e87db82b9..00000000000 --- a/.github/workflows/install-smoke.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Install Smoke - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -concurrency: - group: install-smoke-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - docs-scope: - runs-on: blacksmith-16vcpu-ubuntu-2404 - outputs: - docs_only: ${{ steps.check.outputs.docs_only }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Detect docs-only changes - id: check - uses: ./.github/actions/detect-docs-changes - - install-smoke: - needs: [docs-scope] - if: needs.docs-scope.outputs.docs_only != 'true' - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - name: Checkout CLI - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Install pnpm deps (minimal) - run: pnpm install --ignore-scripts --frozen-lockfile - - - name: Run installer docker tests - env: - CLAWDBOT_INSTALL_URL: https://openclaw.ai/install.sh - CLAWDBOT_INSTALL_CLI_URL: https://openclaw.ai/install-cli.sh - CLAWDBOT_NO_ONBOARD: "1" - CLAWDBOT_INSTALL_SMOKE_SKIP_CLI: "1" - CLAWDBOT_INSTALL_SMOKE_SKIP_NONROOT: ${{ github.event_name == 'pull_request' && '1' || '0' }} - CLAWDBOT_INSTALL_SMOKE_SKIP_PREVIOUS: "1" - run: pnpm test:install:smoke diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index 9ac44dfa6b6..00000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,519 +0,0 @@ -name: Labeler - -on: - pull_request_target: - types: [opened, synchronize, reopened] - issues: - types: [opened] - workflow_dispatch: - inputs: - max_prs: - description: "Maximum number of open PRs to process (0 = all)" - required: false - default: "200" - per_page: - description: "PRs per page (1-100)" - required: false - default: "50" - -permissions: {} - -jobs: - label: - permissions: - contents: read - pull-requests: write - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 - id: app-token - with: - app-id: "2729701" - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5 - with: - configuration-path: .github/labeler.yml - repo-token: ${{ steps.app-token.outputs.token }} - sync-labels: true - - name: Apply PR size label - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - const pullRequest = context.payload.pull_request; - if (!pullRequest) { - return; - } - - const sizeLabels = ["size: XS", "size: S", "size: M", "size: L", "size: XL"]; - const labelColor = "b76e79"; - - for (const label of sizeLabels) { - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - }); - } catch (error) { - if (error?.status !== 404) { - throw error; - } - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - color: labelColor, - }); - } - } - - const files = await github.paginate(github.rest.pulls.listFiles, { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pullRequest.number, - per_page: 100, - }); - - const excludedLockfiles = new Set(["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lockb"]); - const totalChangedLines = files.reduce((total, file) => { - const path = file.filename ?? ""; - if (path === "docs.acp.md" || path.startsWith("docs/") || excludedLockfiles.has(path)) { - return total; - } - return total + (file.additions ?? 0) + (file.deletions ?? 0); - }, 0); - - let targetSizeLabel = "size: XL"; - if (totalChangedLines < 50) { - targetSizeLabel = "size: XS"; - } else if (totalChangedLines < 200) { - targetSizeLabel = "size: S"; - } else if (totalChangedLines < 500) { - targetSizeLabel = "size: M"; - } else if (totalChangedLines < 1000) { - targetSizeLabel = "size: L"; - } - - const currentLabels = await github.paginate(github.rest.issues.listLabelsOnIssue, { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - per_page: 100, - }); - - for (const label of currentLabels) { - const name = label.name ?? ""; - if (!sizeLabels.includes(name)) { - continue; - } - if (name === targetSizeLabel) { - continue; - } - await github.rest.issues.removeLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - name, - }); - } - - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - labels: [targetSizeLabel], - }); - - name: Apply maintainer or trusted-contributor label - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - const login = context.payload.pull_request?.user?.login; - if (!login) { - return; - } - - const repo = `${context.repo.owner}/${context.repo.repo}`; - const trustedLabel = "trusted-contributor"; - const experiencedLabel = "experienced-contributor"; - const trustedThreshold = 4; - const experiencedThreshold = 10; - - let isMaintainer = false; - try { - const membership = await github.rest.teams.getMembershipForUserInOrg({ - org: context.repo.owner, - team_slug: "maintainer", - username: login, - }); - isMaintainer = membership?.data?.state === "active"; - } catch (error) { - if (error?.status !== 404) { - throw error; - } - } - - if (isMaintainer) { - await github.rest.issues.addLabels({ - ...context.repo, - issue_number: context.payload.pull_request.number, - labels: ["maintainer"], - }); - return; - } - - const mergedQuery = `repo:${repo} is:pr is:merged author:${login}`; - let mergedCount = 0; - try { - const merged = await github.rest.search.issuesAndPullRequests({ - q: mergedQuery, - per_page: 1, - }); - mergedCount = merged?.data?.total_count ?? 0; - } catch (error) { - if (error?.status !== 422) { - throw error; - } - core.warning(`Skipping merged search for ${login}; treating as 0.`); - } - - if (mergedCount >= experiencedThreshold) { - await github.rest.issues.addLabels({ - ...context.repo, - issue_number: context.payload.pull_request.number, - labels: [experiencedLabel], - }); - return; - } - - if (mergedCount >= trustedThreshold) { - await github.rest.issues.addLabels({ - ...context.repo, - issue_number: context.payload.pull_request.number, - labels: [trustedLabel], - }); - } - - backfill-pr-labels: - if: github.event_name == 'workflow_dispatch' - permissions: - contents: read - pull-requests: write - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 - id: app-token - with: - app-id: "2729701" - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Backfill PR labels - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - const owner = context.repo.owner; - const repo = context.repo.repo; - const repoFull = `${owner}/${repo}`; - const inputs = context.payload.inputs ?? {}; - const maxPrsInput = inputs.max_prs ?? "200"; - const perPageInput = inputs.per_page ?? "50"; - const parsedMaxPrs = Number.parseInt(maxPrsInput, 10); - const parsedPerPage = Number.parseInt(perPageInput, 10); - const maxPrs = Number.isFinite(parsedMaxPrs) ? parsedMaxPrs : 200; - const perPage = Number.isFinite(parsedPerPage) ? Math.min(100, Math.max(1, parsedPerPage)) : 50; - const processAll = maxPrs <= 0; - const maxCount = processAll ? Number.POSITIVE_INFINITY : Math.max(1, maxPrs); - - const sizeLabels = ["size: XS", "size: S", "size: M", "size: L", "size: XL"]; - const labelColor = "b76e79"; - const trustedLabel = "trusted-contributor"; - const experiencedLabel = "experienced-contributor"; - const trustedThreshold = 4; - const experiencedThreshold = 10; - - const contributorCache = new Map(); - - async function ensureSizeLabels() { - for (const label of sizeLabels) { - try { - await github.rest.issues.getLabel({ - owner, - repo, - name: label, - }); - } catch (error) { - if (error?.status !== 404) { - throw error; - } - await github.rest.issues.createLabel({ - owner, - repo, - name: label, - color: labelColor, - }); - } - } - } - - async function resolveContributorLabel(login) { - if (contributorCache.has(login)) { - return contributorCache.get(login); - } - - let isMaintainer = false; - try { - const membership = await github.rest.teams.getMembershipForUserInOrg({ - org: owner, - team_slug: "maintainer", - username: login, - }); - isMaintainer = membership?.data?.state === "active"; - } catch (error) { - if (error?.status !== 404) { - throw error; - } - } - - if (isMaintainer) { - contributorCache.set(login, "maintainer"); - return "maintainer"; - } - - const mergedQuery = `repo:${repoFull} is:pr is:merged author:${login}`; - let mergedCount = 0; - try { - const merged = await github.rest.search.issuesAndPullRequests({ - q: mergedQuery, - per_page: 1, - }); - mergedCount = merged?.data?.total_count ?? 0; - } catch (error) { - if (error?.status !== 422) { - throw error; - } - core.warning(`Skipping merged search for ${login}; treating as 0.`); - } - - let label = null; - if (mergedCount >= experiencedThreshold) { - label = experiencedLabel; - } else if (mergedCount >= trustedThreshold) { - label = trustedLabel; - } - - contributorCache.set(login, label); - return label; - } - - async function applySizeLabel(pullRequest, currentLabels, labelNames) { - const files = await github.paginate(github.rest.pulls.listFiles, { - owner, - repo, - pull_number: pullRequest.number, - per_page: 100, - }); - - const excludedLockfiles = new Set(["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lockb"]); - const totalChangedLines = files.reduce((total, file) => { - const path = file.filename ?? ""; - if (path === "docs.acp.md" || path.startsWith("docs/") || excludedLockfiles.has(path)) { - return total; - } - return total + (file.additions ?? 0) + (file.deletions ?? 0); - }, 0); - - let targetSizeLabel = "size: XL"; - if (totalChangedLines < 50) { - targetSizeLabel = "size: XS"; - } else if (totalChangedLines < 200) { - targetSizeLabel = "size: S"; - } else if (totalChangedLines < 500) { - targetSizeLabel = "size: M"; - } else if (totalChangedLines < 1000) { - targetSizeLabel = "size: L"; - } - - for (const label of currentLabels) { - const name = label.name ?? ""; - if (!sizeLabels.includes(name)) { - continue; - } - if (name === targetSizeLabel) { - continue; - } - await github.rest.issues.removeLabel({ - owner, - repo, - issue_number: pullRequest.number, - name, - }); - labelNames.delete(name); - } - - if (!labelNames.has(targetSizeLabel)) { - await github.rest.issues.addLabels({ - owner, - repo, - issue_number: pullRequest.number, - labels: [targetSizeLabel], - }); - labelNames.add(targetSizeLabel); - } - } - - async function applyContributorLabel(pullRequest, labelNames) { - const login = pullRequest.user?.login; - if (!login) { - return; - } - - const label = await resolveContributorLabel(login); - if (!label) { - return; - } - - if (labelNames.has(label)) { - return; - } - - await github.rest.issues.addLabels({ - owner, - repo, - issue_number: pullRequest.number, - labels: [label], - }); - labelNames.add(label); - } - - await ensureSizeLabels(); - - let page = 1; - let processed = 0; - - while (processed < maxCount) { - const remaining = maxCount - processed; - const pageSize = processAll ? perPage : Math.min(perPage, remaining); - const { data: pullRequests } = await github.rest.pulls.list({ - owner, - repo, - state: "open", - per_page: pageSize, - page, - }); - - if (pullRequests.length === 0) { - break; - } - - for (const pullRequest of pullRequests) { - if (!processAll && processed >= maxCount) { - break; - } - - const currentLabels = await github.paginate(github.rest.issues.listLabelsOnIssue, { - owner, - repo, - issue_number: pullRequest.number, - per_page: 100, - }); - - const labelNames = new Set( - currentLabels.map((label) => label.name).filter((name) => typeof name === "string"), - ); - - await applySizeLabel(pullRequest, currentLabels, labelNames); - await applyContributorLabel(pullRequest, labelNames); - - processed += 1; - } - - if (pullRequests.length < pageSize) { - break; - } - - page += 1; - } - - core.info(`Processed ${processed} pull requests.`); - - label-issues: - permissions: - issues: write - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 - id: app-token - with: - app-id: "2729701" - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Apply maintainer or trusted-contributor label - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - const login = context.payload.issue?.user?.login; - if (!login) { - return; - } - - const repo = `${context.repo.owner}/${context.repo.repo}`; - const trustedLabel = "trusted-contributor"; - const experiencedLabel = "experienced-contributor"; - const trustedThreshold = 4; - const experiencedThreshold = 10; - - let isMaintainer = false; - try { - const membership = await github.rest.teams.getMembershipForUserInOrg({ - org: context.repo.owner, - team_slug: "maintainer", - username: login, - }); - isMaintainer = membership?.data?.state === "active"; - } catch (error) { - if (error?.status !== 404) { - throw error; - } - } - - if (isMaintainer) { - await github.rest.issues.addLabels({ - ...context.repo, - issue_number: context.payload.issue.number, - labels: ["maintainer"], - }); - return; - } - - const mergedQuery = `repo:${repo} is:pr is:merged author:${login}`; - let mergedCount = 0; - try { - const merged = await github.rest.search.issuesAndPullRequests({ - q: mergedQuery, - per_page: 1, - }); - mergedCount = merged?.data?.total_count ?? 0; - } catch (error) { - if (error?.status !== 422) { - throw error; - } - core.warning(`Skipping merged search for ${login}; treating as 0.`); - } - - if (mergedCount >= experiencedThreshold) { - await github.rest.issues.addLabels({ - ...context.repo, - issue_number: context.payload.issue.number, - labels: [experiencedLabel], - }); - return; - } - - if (mergedCount >= trustedThreshold) { - await github.rest.issues.addLabels({ - ...context.repo, - issue_number: context.payload.issue.number, - labels: [trustedLabel], - }); - } diff --git a/.github/workflows/sandbox-common-smoke.yml b/.github/workflows/sandbox-common-smoke.yml deleted file mode 100644 index 26c0dcc106f..00000000000 --- a/.github/workflows/sandbox-common-smoke.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Sandbox Common Smoke - -on: - push: - branches: [main] - paths: - - Dockerfile.sandbox - - Dockerfile.sandbox-common - - scripts/sandbox-common-setup.sh - pull_request: - paths: - - Dockerfile.sandbox - - Dockerfile.sandbox-common - - scripts/sandbox-common-setup.sh - -concurrency: - group: sandbox-common-smoke-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - sandbox-common-smoke: - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - - - name: Build minimal sandbox base (USER sandbox) - shell: bash - run: | - set -euo pipefail - - docker build -t openclaw-sandbox-smoke-base:bookworm-slim - <<'EOF' - FROM debian:bookworm-slim - RUN useradd --create-home --shell /bin/bash sandbox - USER sandbox - WORKDIR /home/sandbox - EOF - - - name: Build sandbox-common image (root for installs, sandbox at runtime) - shell: bash - run: | - set -euo pipefail - - BASE_IMAGE="openclaw-sandbox-smoke-base:bookworm-slim" \ - TARGET_IMAGE="openclaw-sandbox-common-smoke:bookworm-slim" \ - PACKAGES="ca-certificates" \ - INSTALL_PNPM=0 \ - INSTALL_BUN=0 \ - INSTALL_BREW=0 \ - FINAL_USER=sandbox \ - scripts/sandbox-common-setup.sh - - u="$(docker run --rm openclaw-sandbox-common-smoke:bookworm-slim sh -lc 'id -un')" - test "$u" = "sandbox" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 6248a93dce7..00000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Stale - -on: - schedule: - - cron: "17 3 * * *" - workflow_dispatch: - -permissions: {} - -jobs: - stale: - permissions: - issues: write - pull-requests: write - runs-on: blacksmith-16vcpu-ubuntu-2404 - steps: - - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 - id: app-token - with: - app-id: "2729701" - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Mark stale issues and pull requests - uses: actions/stale@v9 - with: - repo-token: ${{ steps.app-token.outputs.token }} - days-before-issue-stale: 7 - days-before-issue-close: 5 - days-before-pr-stale: 5 - days-before-pr-close: 3 - stale-issue-label: stale - stale-pr-label: stale - exempt-issue-labels: enhancement,maintainer,pinned,security,no-stale - exempt-pr-labels: maintainer,no-stale - operations-per-run: 10000 - exempt-all-assignees: true - remove-stale-when-updated: true - stale-issue-message: | - This issue has been automatically marked as stale due to inactivity. - Please add updates or it will be closed. - stale-pr-message: | - This pull request has been automatically marked as stale due to inactivity. - Please add updates or it will be closed. - close-issue-message: | - Closing due to inactivity. - If this is still an issue, please retry on the latest OpenClaw release and share updated details. - If you are absolutely sure it still happens on the latest release, open a new issue with fresh repro steps. - close-issue-reason: not_planned - close-pr-message: | - Closing due to inactivity. - If you believe this PR should be revived, post in #pr-thunderdome-dangerzone on Discord to talk to a maintainer. - That channel is the escape hatch for high-quality PRs that get auto-closed. diff --git a/.gitignore b/.gitignore index 120ff08b835..7f5cae53122 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ bun.lockb coverage __pycache__/ *.pyc -.tsbuildinfo +*.tsbuildinfo .pnpm-store .worktrees/ .DS_Store @@ -83,6 +83,10 @@ USER.md .tgz .idea +# Next.js +**/.next/ +next-env.d.ts + # local tooling .serena/ diff --git a/README.md b/README.md index 72f362418d7..c19cff4f396 100644 --- a/README.md +++ b/README.md @@ -1,580 +1,421 @@ -# 🦞 OpenClaw β€” Personal AI Assistant -

- - - OpenClaw - + 

+ β–ˆβ–ˆβ•—β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•—  β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•— β–ˆβ–ˆβ–ˆβ•—   β–ˆβ–ˆβ•— β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•—β–ˆβ–ˆβ•—      β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•— β–ˆβ–ˆβ•—    β–ˆβ–ˆβ•—
+ β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•”β•β•β–ˆβ–ˆβ•—β–ˆβ–ˆβ•”β•β•β•β–ˆβ–ˆβ•—β–ˆβ–ˆβ–ˆβ–ˆβ•—  β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•”β•β•β•β•β•β–ˆβ–ˆβ•‘     β–ˆβ–ˆβ•”β•β•β–ˆβ–ˆβ•—β–ˆβ–ˆβ•‘    β–ˆβ–ˆβ•‘
+ β–ˆβ–ˆβ•‘β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•”β•β–ˆβ–ˆβ•‘   β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•”β–ˆβ–ˆβ•— β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•‘     β–ˆβ–ˆβ•‘     β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•‘β–ˆβ–ˆβ•‘ β–ˆβ•— β–ˆβ–ˆβ•‘
+ β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•”β•β•β–ˆβ–ˆβ•—β–ˆβ–ˆβ•‘   β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•‘β•šβ–ˆβ–ˆβ•—β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•‘     β–ˆβ–ˆβ•‘     β–ˆβ–ˆβ•”β•β•β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•‘β–ˆβ–ˆβ–ˆβ•—β–ˆβ–ˆβ•‘
+ β–ˆβ–ˆβ•‘β–ˆβ–ˆβ•‘  β–ˆβ–ˆβ•‘β•šβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•”β•β–ˆβ–ˆβ•‘ β•šβ–ˆβ–ˆβ–ˆβ–ˆβ•‘β•šβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•—β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ•—β–ˆβ–ˆβ•‘  β–ˆβ–ˆβ•‘β•šβ–ˆβ–ˆβ–ˆβ•”β–ˆβ–ˆβ–ˆβ•”β•
+ β•šβ•β•β•šβ•β•  β•šβ•β• β•šβ•β•β•β•β•β• β•šβ•β•  β•šβ•β•β•β• β•šβ•β•β•β•β•β•β•šβ•β•β•β•β•β•β•β•šβ•β•  β•šβ•β• β•šβ•β•β•β•šβ•β•β•
+

- EXFOLIATE! EXFOLIATE! + AI CRM, hosted locally on your Mac.

- CI status - GitHub release + Chat with your database. Automate outreach. Enrich leads. All from a single prompt. +

+ +

+ npm version Discord MIT License

-**OpenClaw** is a _personal AI assistant_ you run on your own devices. -It answers you on the channels you already use (WhatsApp, Telegram, Slack, Discord, Google Chat, Signal, iMessage, Microsoft Teams, WebChat), plus extension channels like BlueBubbles, Matrix, Zalo, and Zalo Personal. It can speak and listen on macOS/iOS/Android, and can render a live Canvas you control. The Gateway is just the control plane β€” the product is the assistant. +

+ Website Β· Docs Β· OpenClaw Framework Β· Discord Β· Skills Store +

-If you want a personal, single-user assistant that feels local, fast, and always-on, this is it. +--- -[Website](https://openclaw.ai) Β· [Docs](https://docs.openclaw.ai) Β· [Vision](VISION.md) Β· [DeepWiki](https://deepwiki.com/openclaw/openclaw) Β· [Getting Started](https://docs.openclaw.ai/start/getting-started) Β· [Updating](https://docs.openclaw.ai/install/updating) Β· [Showcase](https://docs.openclaw.ai/start/showcase) Β· [FAQ](https://docs.openclaw.ai/start/faq) Β· [Wizard](https://docs.openclaw.ai/start/wizard) Β· [Nix](https://github.com/openclaw/nix-openclaw) Β· [Docker](https://docs.openclaw.ai/install/docker) Β· [Discord](https://discord.gg/clawd) +## Install -Preferred setup: run the onboarding wizard (`openclaw onboard`) in your terminal. -The wizard guides you step by step through setting up the gateway, workspace, channels, and skills. The CLI wizard is the recommended path and works on **macOS, Linux, and Windows (via WSL2; strongly recommended)**. -Works with npm, pnpm, or bun. -New install? Start here: [Getting started](https://docs.openclaw.ai/start/getting-started) - -## Sponsors - -| OpenAI | Blacksmith | -| ----------------------------------------------------------------- | ---------------------------------------------------------------------------- | -| [![OpenAI](docs/assets/sponsors/openai.svg)](https://openai.com/) | [![Blacksmith](docs/assets/sponsors/blacksmith.svg)](https://blacksmith.sh/) | - -**Subscriptions (OAuth):** - -- **[Anthropic](https://www.anthropic.com/)** (Claude Pro/Max) -- **[OpenAI](https://openai.com/)** (ChatGPT/Codex) - -Model note: while any model is supported, I strongly recommend **Anthropic Pro/Max (100/200) + Opus 4.6** for long‑context strength and better prompt‑injection resistance. See [Onboarding](https://docs.openclaw.ai/start/onboarding). - -## Models (selection + auth) - -- Models config + CLI: [Models](https://docs.openclaw.ai/concepts/models) -- Auth profile rotation (OAuth vs API keys) + fallbacks: [Model failover](https://docs.openclaw.ai/concepts/model-failover) - -## Install (recommended) - -Runtime: **Node β‰₯22**. +**Runtime: Node 22+** ```bash -npm install -g openclaw@latest -# or: pnpm add -g openclaw@latest - -openclaw onboard --install-daemon +npm i -g ironclaw +ironclaw onboard --install-daemon ``` -The wizard installs the Gateway daemon (launchd/systemd user service) so it stays running. +Opens at `localhost:3100`. That's it. -## Quick start (TL;DR) - -Runtime: **Node β‰₯22**. - -Full beginner guide (auth, pairing, channels): [Getting started](https://docs.openclaw.ai/start/getting-started) - -```bash -openclaw onboard --install-daemon - -openclaw gateway --port 18789 --verbose - -# Send a message -openclaw message send --to +1234567890 --message "Hello from OpenClaw" - -# Talk to the assistant (optionally deliver back to any connected channel: WhatsApp/Telegram/Slack/Discord/Google Chat/Signal/iMessage/BlueBubbles/Microsoft Teams/Matrix/Zalo/Zalo Personal/WebChat) -openclaw agent --message "Ship checklist" --thinking high -``` - -Upgrading? [Updating guide](https://docs.openclaw.ai/install/updating) (and run `openclaw doctor`). - -## Development channels - -- **stable**: tagged releases (`vYYYY.M.D` or `vYYYY.M.D-`), npm dist-tag `latest`. -- **beta**: prerelease tags (`vYYYY.M.D-beta.N`), npm dist-tag `beta` (macOS app may be missing). -- **dev**: moving head of `main`, npm dist-tag `dev` (when published). - -Switch channels (git + npm): `openclaw update --channel stable|beta|dev`. -Details: [Development channels](https://docs.openclaw.ai/install/development-channels). - -## From source (development) - -Prefer `pnpm` for builds from source. Bun is optional for running TypeScript directly. - -```bash -git clone https://github.com/openclaw/openclaw.git -cd openclaw - -pnpm install -pnpm ui:build # auto-installs UI deps on first run -pnpm build - -pnpm openclaw onboard --install-daemon - -# Dev loop (auto-reload on TS changes) -pnpm gateway:watch -``` - -Note: `pnpm openclaw ...` runs TypeScript directly (via `tsx`). `pnpm build` produces `dist/` for running via Node / the packaged `openclaw` binary. - -## Security defaults (DM access) - -OpenClaw connects to real messaging surfaces. Treat inbound DMs as **untrusted input**. - -Full security guide: [Security](https://docs.openclaw.ai/gateway/security) - -Default behavior on Telegram/WhatsApp/Signal/iMessage/Microsoft Teams/Discord/Google Chat/Slack: - -- **DM pairing** (`dmPolicy="pairing"` / `channels.discord.dmPolicy="pairing"` / `channels.slack.dmPolicy="pairing"`; legacy: `channels.discord.dm.policy`, `channels.slack.dm.policy`): unknown senders receive a short pairing code and the bot does not process their message. -- Approve with: `openclaw pairing approve ` (then the sender is added to a local allowlist store). -- Public inbound DMs require an explicit opt-in: set `dmPolicy="open"` and include `"*"` in the channel allowlist (`allowFrom` / `channels.discord.allowFrom` / `channels.slack.allowFrom`; legacy: `channels.discord.dm.allowFrom`, `channels.slack.dm.allowFrom`). - -Run `openclaw doctor` to surface risky/misconfigured DM policies. - -## Highlights - -- **[Local-first Gateway](https://docs.openclaw.ai/gateway)** β€” single control plane for sessions, channels, tools, and events. -- **[Multi-channel inbox](https://docs.openclaw.ai/channels)** β€” WhatsApp, Telegram, Slack, Discord, Google Chat, Signal, BlueBubbles (iMessage), iMessage (legacy), Microsoft Teams, Matrix, Zalo, Zalo Personal, WebChat, macOS, iOS/Android. -- **[Multi-agent routing](https://docs.openclaw.ai/gateway/configuration)** β€” route inbound channels/accounts/peers to isolated agents (workspaces + per-agent sessions). -- **[Voice Wake](https://docs.openclaw.ai/nodes/voicewake) + [Talk Mode](https://docs.openclaw.ai/nodes/talk)** β€” always-on speech for macOS/iOS/Android with ElevenLabs. -- **[Live Canvas](https://docs.openclaw.ai/platforms/mac/canvas)** β€” agent-driven visual workspace with [A2UI](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui). -- **[First-class tools](https://docs.openclaw.ai/tools)** β€” browser, canvas, nodes, cron, sessions, and Discord/Slack actions. -- **[Companion apps](https://docs.openclaw.ai/platforms/macos)** β€” macOS menu bar app + iOS/Android [nodes](https://docs.openclaw.ai/nodes). -- **[Onboarding](https://docs.openclaw.ai/start/wizard) + [skills](https://docs.openclaw.ai/tools/skills)** β€” wizard-driven setup with bundled/managed/workspace skills. - -## Star History - -[![Star History Chart](https://api.star-history.com/svg?repos=openclaw/openclaw&type=date&legend=top-left)](https://www.star-history.com/#openclaw/openclaw&type=date&legend=top-left) - -## Everything we built so far - -### Core platform - -- [Gateway WS control plane](https://docs.openclaw.ai/gateway) with sessions, presence, config, cron, webhooks, [Control UI](https://docs.openclaw.ai/web), and [Canvas host](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui). -- [CLI surface](https://docs.openclaw.ai/tools/agent-send): gateway, agent, send, [wizard](https://docs.openclaw.ai/start/wizard), and [doctor](https://docs.openclaw.ai/gateway/doctor). -- [Pi agent runtime](https://docs.openclaw.ai/concepts/agent) in RPC mode with tool streaming and block streaming. -- [Session model](https://docs.openclaw.ai/concepts/session): `main` for direct chats, group isolation, activation modes, queue modes, reply-back. Group rules: [Groups](https://docs.openclaw.ai/concepts/groups). -- [Media pipeline](https://docs.openclaw.ai/nodes/images): images/audio/video, transcription hooks, size caps, temp file lifecycle. Audio details: [Audio](https://docs.openclaw.ai/nodes/audio). - -### Channels - -- [Channels](https://docs.openclaw.ai/channels): [WhatsApp](https://docs.openclaw.ai/channels/whatsapp) (Baileys), [Telegram](https://docs.openclaw.ai/channels/telegram) (grammY), [Slack](https://docs.openclaw.ai/channels/slack) (Bolt), [Discord](https://docs.openclaw.ai/channels/discord) (discord.js), [Google Chat](https://docs.openclaw.ai/channels/googlechat) (Chat API), [Signal](https://docs.openclaw.ai/channels/signal) (signal-cli), [BlueBubbles](https://docs.openclaw.ai/channels/bluebubbles) (iMessage, recommended), [iMessage](https://docs.openclaw.ai/channels/imessage) (legacy imsg), [Microsoft Teams](https://docs.openclaw.ai/channels/msteams) (extension), [Matrix](https://docs.openclaw.ai/channels/matrix) (extension), [Zalo](https://docs.openclaw.ai/channels/zalo) (extension), [Zalo Personal](https://docs.openclaw.ai/channels/zalouser) (extension), [WebChat](https://docs.openclaw.ai/web/webchat). -- [Group routing](https://docs.openclaw.ai/concepts/group-messages): mention gating, reply tags, per-channel chunking and routing. Channel rules: [Channels](https://docs.openclaw.ai/channels). - -### Apps + nodes - -- [macOS app](https://docs.openclaw.ai/platforms/macos): menu bar control plane, [Voice Wake](https://docs.openclaw.ai/nodes/voicewake)/PTT, [Talk Mode](https://docs.openclaw.ai/nodes/talk) overlay, [WebChat](https://docs.openclaw.ai/web/webchat), debug tools, [remote gateway](https://docs.openclaw.ai/gateway/remote) control. -- [iOS node](https://docs.openclaw.ai/platforms/ios): [Canvas](https://docs.openclaw.ai/platforms/mac/canvas), [Voice Wake](https://docs.openclaw.ai/nodes/voicewake), [Talk Mode](https://docs.openclaw.ai/nodes/talk), camera, screen recording, Bonjour pairing. -- [Android node](https://docs.openclaw.ai/platforms/android): [Canvas](https://docs.openclaw.ai/platforms/mac/canvas), [Talk Mode](https://docs.openclaw.ai/nodes/talk), camera, screen recording, optional SMS. -- [macOS node mode](https://docs.openclaw.ai/nodes): system.run/notify + canvas/camera exposure. - -### Tools + automation - -- [Browser control](https://docs.openclaw.ai/tools/browser): dedicated openclaw Chrome/Chromium, snapshots, actions, uploads, profiles. -- [Canvas](https://docs.openclaw.ai/platforms/mac/canvas): [A2UI](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui) push/reset, eval, snapshot. -- [Nodes](https://docs.openclaw.ai/nodes): camera snap/clip, screen record, [location.get](https://docs.openclaw.ai/nodes/location-command), notifications. -- [Cron + wakeups](https://docs.openclaw.ai/automation/cron-jobs); [webhooks](https://docs.openclaw.ai/automation/webhook); [Gmail Pub/Sub](https://docs.openclaw.ai/automation/gmail-pubsub). -- [Skills platform](https://docs.openclaw.ai/tools/skills): bundled, managed, and workspace skills with install gating + UI. - -### Runtime + safety - -- [Channel routing](https://docs.openclaw.ai/concepts/channel-routing), [retry policy](https://docs.openclaw.ai/concepts/retry), and [streaming/chunking](https://docs.openclaw.ai/concepts/streaming). -- [Presence](https://docs.openclaw.ai/concepts/presence), [typing indicators](https://docs.openclaw.ai/concepts/typing-indicators), and [usage tracking](https://docs.openclaw.ai/concepts/usage-tracking). -- [Models](https://docs.openclaw.ai/concepts/models), [model failover](https://docs.openclaw.ai/concepts/model-failover), and [session pruning](https://docs.openclaw.ai/concepts/session-pruning). -- [Security](https://docs.openclaw.ai/gateway/security) and [troubleshooting](https://docs.openclaw.ai/channels/troubleshooting). - -### Ops + packaging - -- [Control UI](https://docs.openclaw.ai/web) + [WebChat](https://docs.openclaw.ai/web/webchat) served directly from the Gateway. -- [Tailscale Serve/Funnel](https://docs.openclaw.ai/gateway/tailscale) or [SSH tunnels](https://docs.openclaw.ai/gateway/remote) with token/password auth. -- [Nix mode](https://docs.openclaw.ai/install/nix) for declarative config; [Docker](https://docs.openclaw.ai/install/docker)-based installs. -- [Doctor](https://docs.openclaw.ai/gateway/doctor) migrations, [logging](https://docs.openclaw.ai/logging). - -## How it works (short) +Three steps total: ``` -WhatsApp / Telegram / Slack / Discord / Google Chat / Signal / iMessage / BlueBubbles / Microsoft Teams / Matrix / Zalo / Zalo Personal / WebChat +1. npm i -g ironclaw +2. ironclaw onboard +3. ironclaw gateway start +``` + +--- + +## What is Ironclaw? + +Ironclaw is a personal AI agent and CRM that runs locally on your machine. It connects to every messaging channel you use, manages structured data through DuckDB, browses the web with your Chrome profile, and gives you a full web UI for pipeline management, analytics, and document management. + +Built on [OpenClaw](https://github.com/openclaw/openclaw) with **Vercel AI SDK v6** as the LLM orchestration layer. + +**One prompt does everything:** + +- "Find YC W26 founders building AI companies" β†’ scrapes YC directory + LinkedIn, returns 127 matches +- "Enrich all contacts with LinkedIn and email" β†’ fills in profiles with 98% coverage +- "Send personalized messages to qualified leads" β†’ crafts and sends custom outreach +- "Show me pipeline stats for this quarter" β†’ generates interactive charts from live data +- "Set up weekly follow-up sequences for all leads" β†’ creates automation rules that run in the background + +--- + +## Use Cases + +### Find Leads + +Type a prompt, Ironclaw scrapes the web using your actual Chrome profile (all your auth sessions, cookies, history). It logs into LinkedIn, browses YC batches, pulls company data. No separate login, no API keys for browsing. + +### Enrich Data + +Point it at your contacts table. It fills in LinkedIn URLs, email addresses, education, company info. Enrichment runs in bulk with real-time progress. + +### Send Outreach + +Personalized LinkedIn messages, cold emails, follow-up sequences. Each message is customized per lead. You see status (Sent, Sending, Queued) in real time. + +### Analyze Pipeline + +Ask for analytics in plain English. Ironclaw queries your DuckDB workspace and generates interactive Recharts dashboards inline. Pipeline funnels, outreach activity charts, conversion rates, donut breakdowns. + +### Automate Everything + +Cron jobs that run on schedule. Follow-up if no reply after 3 days. Move leads to Qualified when they reply. Weekly pipeline reports every Monday. Alert on high-intent replies. + +--- + +## Core Capabilities + +### Uses Your Chrome Profile + +Unlike other AI tools, Ironclaw copies your existing Chrome profile with all your auth sessions, cookies, and history. It logs into LinkedIn, scrapes YC batches, and sends messages as you. No separate browser login needed. + +### Chat with Your Database + +Ask questions in plain English. Ironclaw translates to SQL, queries your local DuckDB, and returns structured results. Like having a data analyst on speed dial. + +``` +You: "How many founders have we contacted from YC W26?" + +β†’ SELECT "Status", COUNT(*) as count FROM v_founders GROUP BY "Status"; + +You've contacted 67 of 200 founders. 31 are qualified, 13 converted. +Reply rate is 34%. +``` + +### Coding Agent with Diffs + +Ironclaw writes code. Review changes in a rich diff viewer before applying. Config changes, automation scripts, data transformations. All with diffs you approve. + +### Your Second Brain + +Full access to your Mac: files, apps, documents. It remembers context across sessions via persistent memory files. Learns your preferences. Proactively handles tasks during heartbeat checks. + +--- + +## Web UI (Dench) + +The web app runs at `localhost:3100` and includes: + +- **Chat panel** with streaming responses, chain-of-thought reasoning display, and markdown rendering +- **Workspace sidebar** with file manager tree, knowledge base, and database viewer +- **Object tables** powered by TanStack, with sorting, filtering, row selection, and bulk operations +- **Entry detail modals** with field editing and media previews +- **Kanban boards** with drag-and-drop that auto-update as leads reply +- **Interactive report cards** with chart panels (bar, line, area, pie, donut, funnel, scatter, radar) and filter bars +- **Document editor** with embedded live charts +- **Media viewer** supporting images, video, audio, and PDFs + +--- + +## Multi-Channel Inbox + +One agent, every channel. Connect any messaging platform. Your AI agent responds everywhere, managed from a single terminal. + +| Channel | Setup | +| ------------------- | ------------------------------------------------------------- | +| **WhatsApp** | `ironclaw channels login` + set `channels.whatsapp.allowFrom` | +| **Telegram** | Set `TELEGRAM_BOT_TOKEN` or `channels.telegram.botToken` | +| **Slack** | Set `SLACK_BOT_TOKEN` + `SLACK_APP_TOKEN` | +| **Discord** | Set `DISCORD_BOT_TOKEN` or `channels.discord.token` | +| **Signal** | Requires `signal-cli` + `channels.signal` config | +| **iMessage** | Via BlueBubbles (recommended) or legacy macOS integration | +| **Microsoft Teams** | Configure Teams app + Bot Framework | +| **Google Chat** | Chat API integration | +| **Matrix** | Extension channel | +| **WebChat** | Built-in, uses Gateway WebSocket directly | + +``` + WhatsApp Β· Telegram Β· Slack Β· Discord + Signal Β· iMessage Β· Teams Β· WebChat β”‚ β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Gateway β”‚ -β”‚ (control plane) β”‚ -β”‚ ws://127.0.0.1:18789 β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”œβ”€ Pi agent (RPC) - β”œβ”€ CLI (openclaw …) - β”œβ”€ WebChat UI - β”œβ”€ macOS app - └─ iOS / Android nodes + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Ironclaw Gateway β”‚ + β”‚ ws://127.0.0.1:18789 β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ β”‚ + β–Ό β–Ό β–Ό + AI SDK Web UI CLI + Engine (Dench) (ironclaw) ``` -## Key subsystems +--- -- **[Gateway WebSocket network](https://docs.openclaw.ai/concepts/architecture)** β€” single WS control plane for clients, tools, and events (plus ops: [Gateway runbook](https://docs.openclaw.ai/gateway)). -- **[Tailscale exposure](https://docs.openclaw.ai/gateway/tailscale)** β€” Serve/Funnel for the Gateway dashboard + WS (remote access: [Remote](https://docs.openclaw.ai/gateway/remote)). -- **[Browser control](https://docs.openclaw.ai/tools/browser)** β€” openclaw‑managed Chrome/Chromium with CDP control. -- **[Canvas + A2UI](https://docs.openclaw.ai/platforms/mac/canvas)** β€” agent‑driven visual workspace (A2UI host: [Canvas/A2UI](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui)). -- **[Voice Wake](https://docs.openclaw.ai/nodes/voicewake) + [Talk Mode](https://docs.openclaw.ai/nodes/talk)** β€” always‑on speech and continuous conversation. -- **[Nodes](https://docs.openclaw.ai/nodes)** β€” Canvas, camera snap/clip, screen record, `location.get`, notifications, plus macOS‑only `system.run`/`system.notify`. +## Integrations -## Tailscale access (Gateway dashboard) +Import your data from anywhere: Google Drive, Notion, Salesforce, HubSpot, Gmail, Calendar, Obsidian, Slack, LinkedIn, Asana, Monday, ClickUp, PostHog, Sheets, Apple Notes, GitHub, and 50+ more via the Skills Store. -OpenClaw can auto-configure Tailscale **Serve** (tailnet-only) or **Funnel** (public) while the Gateway stays bound to loopback. Configure `gateway.tailscale.mode`: +--- -- `off`: no Tailscale automation (default). -- `serve`: tailnet-only HTTPS via `tailscale serve` (uses Tailscale identity headers by default). -- `funnel`: public HTTPS via `tailscale funnel` (requires shared password auth). +## Skills Platform -Notes: +Extend your agent with a single command. Browse skills from [skills.sh](https://skills.sh) and [ClawHub](https://clawhub.com). -- `gateway.bind` must stay `loopback` when Serve/Funnel is enabled (OpenClaw enforces this). -- Serve can be forced to require a password by setting `gateway.auth.mode: "password"` or `gateway.auth.allowTailscale: false`. -- Funnel refuses to start unless `gateway.auth.mode: "password"` is set. -- Optional: `gateway.tailscale.resetOnExit` to undo Serve/Funnel on shutdown. +```bash +npx skills add vercel-labs/agent-browser +``` -Details: [Tailscale guide](https://docs.openclaw.ai/gateway/tailscale) Β· [Web surfaces](https://docs.openclaw.ai/web) +Popular skills: -## Remote Gateway (Linux is great) +| Skill | Description | Installs | +| ----------------------- | ---------------------------------------------------------- | -------- | +| `crm-automation` | CRM workflow automation, lead scoring, pipeline management | 18.2K | +| `linkedin-outreach` | Automated LinkedIn prospecting and follow-up sequences | 14.8K | +| `lead-enrichment` | Enrich contacts with LinkedIn, email, and company data | 12.1K | +| `email-sequences` | Multi-step cold email campaigns with personalization | 9.7K | +| `agent-browser` | Browser automation and web scraping for agents | 35.8K | +| `web-design-guidelines` | Best practices for modern web design | 99.4K | +| `frontend-design` | Expert frontend engineering patterns | 68.9K | +| `typescript-expert` | Advanced TypeScript patterns and best practices | 15.1K | -It’s perfectly fine to run the Gateway on a small Linux instance. Clients (macOS app, CLI, WebChat) can connect over **Tailscale Serve/Funnel** or **SSH tunnels**, and you can still pair device nodes (macOS/iOS/Android) to execute device‑local actions when needed. +Or write your own. Skills are just a `SKILL.md` file with instructions + optional scripts. -- **Gateway host** runs the exec tool and channel connections by default. -- **Device nodes** run device‑local actions (`system.run`, camera, screen recording, notifications) via `node.invoke`. - In short: exec runs where the Gateway lives; device actions run where the device lives. +--- -Details: [Remote access](https://docs.openclaw.ai/gateway/remote) Β· [Nodes](https://docs.openclaw.ai/nodes) Β· [Security](https://docs.openclaw.ai/gateway/security) +## Analytics & Reports -## macOS permissions via the Gateway protocol +Ask "show me pipeline analytics" and get interactive charts generated from your live DuckDB data. -The macOS app can run in **node mode** and advertises its capabilities + permission map over the Gateway WebSocket (`node.list` / `node.describe`). Clients can then execute local actions via `node.invoke`: +- **Outreach Activity** β€” area charts tracking LinkedIn and email volume over time +- **Pipeline Breakdown** β€” donut charts showing lead distribution by status +- **Conversion Funnel** β€” stage-by-stage conversion rates with overall percentage +- **Deal Pipeline** β€” bar charts, funnel views, revenue by stage +- **Custom Reports** β€” save as `.report.json` files that render as live dashboards -- `system.run` runs a local command and returns stdout/stderr/exit code; set `needsScreenRecording: true` to require screen-recording permission (otherwise you’ll get `PERMISSION_MISSING`). -- `system.notify` posts a user notification and fails if notifications are denied. -- `canvas.*`, `camera.*`, `screen.record`, and `location.get` are also routed via `node.invoke` and follow TCC permission status. +Reports use the `report-json` format and render inline in chat as interactive Recharts components. -Elevated bash (host permissions) is separate from macOS TCC: +--- -- Use `/elevated on|off` to toggle per‑session elevated access when enabled + allowlisted. -- Gateway persists the per‑session toggle via `sessions.patch` (WS method) alongside `thinkingLevel`, `verboseLevel`, `model`, `sendPolicy`, and `groupActivation`. +## Kanban Pipeline -Details: [Nodes](https://docs.openclaw.ai/nodes) Β· [macOS app](https://docs.openclaw.ai/platforms/macos) Β· [Gateway protocol](https://docs.openclaw.ai/concepts/architecture) +Drag-and-drop kanban boards that auto-update as leads reply. Ironclaw moves cards through your pipeline automatically. -## Agent to Agent (sessions\_\* tools) +Columns like New Lead β†’ Contacted β†’ Qualified β†’ Demo Scheduled β†’ Closed map to your sales process. Each card shows the lead name, company, and last action taken. -- Use these to coordinate work across sessions without jumping between chat surfaces. -- `sessions_list` β€” discover active sessions (agents) and their metadata. -- `sessions_history` β€” fetch transcript logs for a session. -- `sessions_send` β€” message another session; optional reply‑back ping‑pong + announce step (`REPLY_SKIP`, `ANNOUNCE_SKIP`). +--- -Details: [Session tools](https://docs.openclaw.ai/concepts/session-tool) +## Documents, Reports & Cron Jobs -## Skills registry (ClawHub) +### Documents -ClawHub is a minimal skill registry. With ClawHub enabled, the agent can search for skills automatically and pull in new ones as needed. +Rich markdown documents with embedded live charts. SOPs, playbooks, onboarding guides. Documents nest under objects or stand alone in the file tree. -[ClawHub](https://clawhub.com) +### Cron Jobs -## Chat commands +Scheduled automations that run in the background: -Send these in WhatsApp/Telegram/Slack/Google Chat/Microsoft Teams/WebChat (group commands are owner-only): +| Job | Schedule | Description | +| ---------------------- | -------------- | ------------------------------------ | +| Weekly pipeline report | `0 9 * * MON` | Auto-generates pipeline summary | +| Lead enrichment sync | Every 6h | Enriches new contacts | +| Email follow-up check | Every 30m | Checks for replies needing follow-up | +| Inbox digest | `0 8,18 * * *` | Morning and evening inbox summary | +| Competitor monitoring | `0 6 * * *` | Tracks competitor activity | +| CRM backup to S3 | `0 2 * * *` | Nightly workspace backup | -- `/status` β€” compact session status (model + tokens, cost when available) -- `/new` or `/reset` β€” reset the session -- `/compact` β€” compact session context (summary) -- `/think ` β€” off|minimal|low|medium|high|xhigh (GPT-5.2 + Codex models only) -- `/verbose on|off` -- `/usage off|tokens|full` β€” per-response usage footer -- `/restart` β€” restart the gateway (owner-only in groups) -- `/activation mention|always` β€” group activation toggle (groups only) +```bash +ironclaw cron list +``` -## Apps (optional) +--- -The Gateway alone delivers a great experience. All apps are optional and add extra features. +## Gateway -If you plan to build/run companion apps, follow the platform runbooks below. +The Gateway is the local-first WebSocket control plane that routes everything: -### macOS (OpenClaw.app) (optional) +- **Sessions** β€” main sessions for DMs, isolated sessions for group chats, sub-agent sessions for background tasks +- **Channels** β€” route inbound messages from any platform to the right session +- **Tools** β€” browser control, canvas, nodes, cron, messaging, file operations +- **Events** β€” webhooks, Gmail Pub/Sub, cron triggers, heartbeats +- **Multi-agent routing** β€” route channels/accounts/peers to isolated agents with separate workspaces -- Menu bar control for the Gateway and health. -- Voice Wake + push-to-talk overlay. -- WebChat + debug tools. -- Remote gateway control over SSH. +### Session Model -Note: signed builds required for macOS permissions to stick across rebuilds (see `docs/mac/permissions.md`). +- `main` β€” direct 1:1 chats with persistent context +- `group` β€” isolated per-group sessions with mention gating +- `isolated` β€” sub-agent sessions for background tasks (cron jobs, spawned work) -### iOS node (optional) +### Security -- Pairs as a node via the Bridge. -- Voice trigger forwarding + Canvas surface. -- Controlled via `openclaw nodes …`. +- **DM pairing** enabled by default. Unknown senders get a pairing code. +- Approve with `ironclaw pairing approve ` +- Non-main sessions can be sandboxed in Docker +- Run `ironclaw doctor` to audit DM policies -Runbook: [iOS connect](https://docs.openclaw.ai/platforms/ios). +--- -### Android node (optional) +## Companion Apps -- Pairs via the same Bridge + pairing flow as iOS. -- Exposes Canvas, Camera, and Screen capture commands. -- Runbook: [Android connect](https://docs.openclaw.ai/platforms/android). +- **macOS** β€” menu bar app with Voice Wake, Push-to-Talk, Talk Mode overlay, WebChat, and debug tools +- **iOS** β€” Canvas, Voice Wake, Talk Mode, camera, screen recording, Bonjour pairing +- **Android** β€” Canvas, Talk Mode, camera, screen recording, optional SMS -## Agent workspace + skills - -- Workspace root: `~/.openclaw/workspace` (configurable via `agents.defaults.workspace`). -- Injected prompt files: `AGENTS.md`, `SOUL.md`, `TOOLS.md`. -- Skills: `~/.openclaw/workspace/skills//SKILL.md`. +--- ## Configuration -Minimal `~/.openclaw/openclaw.json` (model + defaults): +Config lives at `~/.openclaw/openclaw.json`: -```json5 -{ - agent: { - model: "anthropic/claude-opus-4-6", - }, -} +Supports all latest and greatest mainstream LLM models. BYOK. + +--- + +## Chat Commands + +Send these in any connected channel: + +| Command | Description | +| ----------------------------- | ------------------------------- | +| `/status` | Session status (model + tokens) | +| `/new` or `/reset` | Reset the session | +| `/compact` | Compact session context | +| `/think ` | Set thinking level | +| `/verbose on\|off` | Toggle verbose output | +| `/usage off\|tokens\|full` | Per-response usage footer | +| `/restart` | Restart the gateway | +| `/activation mention\|always` | Group activation toggle | + +--- + +## DuckDB Workspace + +All structured data lives in a local DuckDB database. Objects, fields, entries, relations. EAV pattern with auto-generated PIVOT views so you query like normal tables: + +```sql +SELECT * FROM v_leads WHERE "Status" = 'New' ORDER BY created_at DESC LIMIT 50; +SELECT "Status", COUNT(*) FROM v_leads GROUP BY "Status"; ``` -[Full configuration reference (all keys + examples).](https://docs.openclaw.ai/gateway/configuration) +Features: -## Security model (important) +- Custom objects with typed fields (text, email, phone, number, boolean, date, enum, relation, user) +- Full-text search +- Bulk import/export (CSV, Parquet) +- Automatic view generation +- Kanban support with drag-and-drop -- **Default:** tools run on the host for the **main** session, so the agent has full access when it’s just you. -- **Group/channel safety:** set `agents.defaults.sandbox.mode: "non-main"` to run **non‑main sessions** (groups/channels) inside per‑session Docker sandboxes; bash then runs in Docker for those sessions. -- **Sandbox defaults:** allowlist `bash`, `process`, `read`, `write`, `edit`, `sessions_list`, `sessions_history`, `sessions_send`, `sessions_spawn`; denylist `browser`, `canvas`, `nodes`, `cron`, `discord`, `gateway`. +--- -Details: [Security guide](https://docs.openclaw.ai/gateway/security) Β· [Docker + sandboxing](https://docs.openclaw.ai/install/docker) Β· [Sandbox config](https://docs.openclaw.ai/gateway/configuration) +## Quick Start -### [WhatsApp](https://docs.openclaw.ai/channels/whatsapp) +```bash +# Install +npm i -g ironclaw -- Link the device: `pnpm openclaw channels login` (stores creds in `~/.openclaw/credentials`). -- Allowlist who can talk to the assistant via `channels.whatsapp.allowFrom`. -- If `channels.whatsapp.groups` is set, it becomes a group allowlist; include `"*"` to allow all. +# Run onboarding wizard +ironclaw onboard --install-daemon -### [Telegram](https://docs.openclaw.ai/channels/telegram) +# Start the gateway +ironclaw gateway start -- Set `TELEGRAM_BOT_TOKEN` or `channels.telegram.botToken` (env wins). -- Optional: set `channels.telegram.groups` (with `channels.telegram.groups."*".requireMention`); when set, it is a group allowlist (include `"*"` to allow all). Also `channels.telegram.allowFrom` or `channels.telegram.webhookUrl` + `channels.telegram.webhookSecret` as needed. +# Open the web UI +open http://localhost:3100 -```json5 -{ - channels: { - telegram: { - botToken: "123456:ABCDEF", - }, - }, -} +# Talk to your agent from CLI +ironclaw agent --message "Summarize my inbox" --thinking high + +# Send a message +ironclaw message send --to +1234567890 --message "Hello from Ironclaw" ``` -### [Slack](https://docs.openclaw.ai/channels/slack) +--- -- Set `SLACK_BOT_TOKEN` + `SLACK_APP_TOKEN` (or `channels.slack.botToken` + `channels.slack.appToken`). +## From Source -### [Discord](https://docs.openclaw.ai/channels/discord) +```bash +git clone https://github.com/kumarabhirup/ironclaw.git +cd ironclaw -- Set `DISCORD_BOT_TOKEN` or `channels.discord.token` (env wins). -- Optional: set `commands.native`, `commands.text`, or `commands.useAccessGroups`, plus `channels.discord.allowFrom`, `channels.discord.guilds`, or `channels.discord.mediaMaxMb` as needed. +pnpm install +pnpm build -```json5 -{ - channels: { - discord: { - token: "1234abcd", - }, - }, -} +pnpm dev onboard --install-daemon ``` -### [Signal](https://docs.openclaw.ai/channels/signal) +Web UI development: -- Requires `signal-cli` and a `channels.signal` config section. - -### [BlueBubbles (iMessage)](https://docs.openclaw.ai/channels/bluebubbles) - -- **Recommended** iMessage integration. -- Configure `channels.bluebubbles.serverUrl` + `channels.bluebubbles.password` and a webhook (`channels.bluebubbles.webhookPath`). -- The BlueBubbles server runs on macOS; the Gateway can run on macOS or elsewhere. - -### [iMessage (legacy)](https://docs.openclaw.ai/channels/imessage) - -- Legacy macOS-only integration via `imsg` (Messages must be signed in). -- If `channels.imessage.groups` is set, it becomes a group allowlist; include `"*"` to allow all. - -### [Microsoft Teams](https://docs.openclaw.ai/channels/msteams) - -- Configure a Teams app + Bot Framework, then add a `msteams` config section. -- Allowlist who can talk via `msteams.allowFrom`; group access via `msteams.groupAllowFrom` or `msteams.groupPolicy: "open"`. - -### [WebChat](https://docs.openclaw.ai/web/webchat) - -- Uses the Gateway WebSocket; no separate WebChat port/config. - -Browser control (optional): - -```json5 -{ - browser: { - enabled: true, - color: "#FF4500", - }, -} +```bash +cd apps/web +pnpm install +pnpm dev ``` -## Docs +--- -Use these when you’re past the onboarding flow and want the deeper reference. +## Project Structure -- [Start with the docs index for navigation and β€œwhat’s where.”](https://docs.openclaw.ai) -- [Read the architecture overview for the gateway + protocol model.](https://docs.openclaw.ai/concepts/architecture) -- [Use the full configuration reference when you need every key and example.](https://docs.openclaw.ai/gateway/configuration) -- [Run the Gateway by the book with the operational runbook.](https://docs.openclaw.ai/gateway) -- [Learn how the Control UI/Web surfaces work and how to expose them safely.](https://docs.openclaw.ai/web) -- [Understand remote access over SSH tunnels or tailnets.](https://docs.openclaw.ai/gateway/remote) -- [Follow the onboarding wizard flow for a guided setup.](https://docs.openclaw.ai/start/wizard) -- [Wire external triggers via the webhook surface.](https://docs.openclaw.ai/automation/webhook) -- [Set up Gmail Pub/Sub triggers.](https://docs.openclaw.ai/automation/gmail-pubsub) -- [Learn the macOS menu bar companion details.](https://docs.openclaw.ai/platforms/mac/menu-bar) -- [Platform guides: Windows (WSL2)](https://docs.openclaw.ai/platforms/windows), [Linux](https://docs.openclaw.ai/platforms/linux), [macOS](https://docs.openclaw.ai/platforms/macos), [iOS](https://docs.openclaw.ai/platforms/ios), [Android](https://docs.openclaw.ai/platforms/android) -- [Debug common failures with the troubleshooting guide.](https://docs.openclaw.ai/channels/troubleshooting) -- [Review security guidance before exposing anything.](https://docs.openclaw.ai/gateway/security) +``` +src/ Core CLI, commands, gateway, agent, media pipeline +apps/web/ Next.js web UI (Dench) +apps/ios/ iOS companion node +apps/android/ Android companion node +apps/macos/ macOS menu bar app +extensions/ Channel plugins (MS Teams, Matrix, Zalo, voice-call) +docs/ Documentation +scripts/ Build, deploy, and utility scripts +skills/ Workspace skills +``` -## Advanced docs (discovery + control) +--- -- [Discovery + transports](https://docs.openclaw.ai/gateway/discovery) -- [Bonjour/mDNS](https://docs.openclaw.ai/gateway/bonjour) -- [Gateway pairing](https://docs.openclaw.ai/gateway/pairing) -- [Remote gateway README](https://docs.openclaw.ai/gateway/remote-gateway-readme) -- [Control UI](https://docs.openclaw.ai/web/control-ui) -- [Dashboard](https://docs.openclaw.ai/web/dashboard) +## Development -## Operations & troubleshooting +```bash +pnpm install # Install deps +pnpm build # Type-check + build +pnpm check # Lint + format check +pnpm test # Run tests (vitest) +pnpm test:coverage # Tests with coverage +pnpm dev # Dev mode (auto-reload) +``` -- [Health checks](https://docs.openclaw.ai/gateway/health) -- [Gateway lock](https://docs.openclaw.ai/gateway/gateway-lock) -- [Background process](https://docs.openclaw.ai/gateway/background-process) -- [Browser troubleshooting (Linux)](https://docs.openclaw.ai/tools/browser-linux-troubleshooting) -- [Logging](https://docs.openclaw.ai/logging) +--- -## Deep dives +## Upstream -- [Agent loop](https://docs.openclaw.ai/concepts/agent-loop) -- [Presence](https://docs.openclaw.ai/concepts/presence) -- [TypeBox schemas](https://docs.openclaw.ai/concepts/typebox) -- [RPC adapters](https://docs.openclaw.ai/reference/rpc) -- [Queue](https://docs.openclaw.ai/concepts/queue) +Ironclaw is built on [OpenClaw](https://github.com/openclaw/openclaw). To sync with upstream: -## Workspace & skills +```bash +git remote add upstream https://github.com/openclaw/openclaw.git +git fetch upstream +git merge upstream/main +``` -- [Skills config](https://docs.openclaw.ai/tools/skills-config) -- [Default AGENTS](https://docs.openclaw.ai/reference/AGENTS.default) -- [Templates: AGENTS](https://docs.openclaw.ai/reference/templates/AGENTS) -- [Templates: BOOTSTRAP](https://docs.openclaw.ai/reference/templates/BOOTSTRAP) -- [Templates: IDENTITY](https://docs.openclaw.ai/reference/templates/IDENTITY) -- [Templates: SOUL](https://docs.openclaw.ai/reference/templates/SOUL) -- [Templates: TOOLS](https://docs.openclaw.ai/reference/templates/TOOLS) -- [Templates: USER](https://docs.openclaw.ai/reference/templates/USER) +--- -## Platform internals +## Open Source -- [macOS dev setup](https://docs.openclaw.ai/platforms/mac/dev-setup) -- [macOS menu bar](https://docs.openclaw.ai/platforms/mac/menu-bar) -- [macOS voice wake](https://docs.openclaw.ai/platforms/mac/voicewake) -- [iOS node](https://docs.openclaw.ai/platforms/ios) -- [Android node](https://docs.openclaw.ai/platforms/android) -- [Windows (WSL2)](https://docs.openclaw.ai/platforms/windows) -- [Linux app](https://docs.openclaw.ai/platforms/linux) +MIT Licensed. Fork it, extend it, make it yours. -## Email hooks (Gmail) - -- [docs.openclaw.ai/gmail-pubsub](https://docs.openclaw.ai/automation/gmail-pubsub) - -## Molty - -OpenClaw was built for **Molty**, a space lobster AI assistant. 🦞 -by Peter Steinberger and the community. - -- [openclaw.ai](https://openclaw.ai) -- [soul.md](https://soul.md) -- [steipete.me](https://steipete.me) -- [@openclaw](https://x.com/openclaw) - -## Community - -See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines, maintainers, and how to submit PRs. -AI/vibe-coded PRs welcome! πŸ€– - -Special thanks to [Mario Zechner](https://mariozechner.at/) for his support and for -[pi-mono](https://github.com/badlogic/pi-mono). -Special thanks to Adam Doppelt for lobster.bot. - -Thanks to all clawtributors: - -

- steipete sktbrd cpojer joshp123 sebslight Mariano Belinky Takhoffman tyler6204 quotentiroler Verite Igiraneza - bohdanpodvirnyi gumadeiras iHildy jaydenfyi joaohlisboa rodrigouroz Glucksberg mneves75 MatthieuBizien MaudeBot - vignesh07 vincentkoc smartprogrammer93 advaitpaliwal HenryLoenwind rahthakor vrknetha abdelsfane radek-paclt joshavant - christianklotz zerone0x ranausmanai Tobias Bischoff heyhudson czekaj ethanpalm mukhtharcm yinghaosang aether-ai-agent - nabbilkhan Mrseenz maxsumrall coygeek xadenryan VACInc juanpablodlc conroywhitney buerbaumer Bridgerz - hsrvc magimetal openclaw-bot meaningfool mudrii JustasM ENCHIGO patelhiren NicholasSpisak claude - jonisjongithub abhisekbasu1 theonejvo Blakeshannon jamesgroat Marvae BunsDev shakkernerd gejifeng akoscz - divanoli ryan-crabbe nyanjou Sam Padilla dantelex SocialNerd42069 solstead natefikru daveonkels LeftX - Yida-Dev Masataka Shinohara arosstale riccardogiorato lc0rp adam91holt mousberg BillChirico shadril238 CharlieGreenman - hougangdev orlyjamie McRolly NWANGWU durenzidu JustYannicc Minidoracat magendary jessy2027 mteam88 hirefrank - M00N7682 dbhurley Eng. Juan Combetto Harrington-bot TSavo Lalit Singh julianengel jscaldwell55 bradleypriest TsekaLuk - benithors Shailesh loiie45e El-Fitz benostein pvtclawn thewilloftheshadow nachx639 0xRaini Taylor Asplund - Paul van Oorschot sreekaransrinath buddyh gupsammy AI-Reviewer-QS Stefan Galescu WalterSumbon nachoiacovino xinhuagu brandonwise - rodbland2021 Vasanth Rao Naik Sabavat fagemx petter-b leszekszpunar davidrudduck Jackten scald pycckuu Parker Todd Brooks - simonemacario omair445 AnonO6 Tanwa Arpornthip andranik-sahakyan davidguttman sleontenko denysvitali Tom Ron popomore - Patrick Barletta shayan919293 δΈεšδΊ†η‘ε€§θ§‰ Lucky Michael Lee sircrumpet peschee dakshaymehta nicolasstanley davidiach - nonggia.liang seheepeak danielwanwx hudson-rivera misterdas Shuai-DaiDai dominicnunez obviyus lploc94 sfo2001 - lutr0 dirbalak cathrynlavery kiranjd danielz1z Iranb cdorsey AdeboyeDN j2h4u Alg0rix - Skyler Miao peetzweg/ TideFinder Clawborn emanuelst bsormagec Diaspar4u evanotero Nate OscarMinjarez - webvijayi garnetlyx jlowin liebertar Max rhuanssauro joshrad-dev osolmaz adityashaw2 CashWilliams - sheeek asklee-klawd h0tp-ftw constansino Mitsuyuki Osabe onutc ryan artuskg Solvely-Colin mcaxtr - HirokiKobayashi-R taw0002 Kimitaka Watanabe Lilo Rajat Joshi Yuting Lin Neo Thorfinn wu-tian807 crimeacs - manuelhettich mcinteerj unisone bjesuiter Manik Vahsith alexgleason Nicholas Stephen Brian King mahanandhi andreesg - connorshea dinakars777 divisonofficer Flash-LHR Protocol Zero kyleok Limitless slonce70 grp06 robbyczgw-cla - JayMishra-source ngutman ide-rea badlogic lailoo amitbiswal007 azade-c John-Rood Iron9521 roshanasingh4 - tosh-hamburg dlauer ezhikkk Shivam Kumar Raut jabezborja Mykyta Bozhenko YuriNachos Josh Phillips Wangnov jadilson12 - 康熙 akramcodez clawdinator[bot] emonty kaizen403 Whoaa512 chriseidhof wangai-studio ysqander Yurii Chukhlib - 17jmumford aj47 google-labs-jules[bot] hyf0-agent Kenny Lee Lukavyi Operative-001 superman32432432 DylanWoodAkers Hisleren - widingmarcus-cyber antons austinm911 boris721 damoahdominic dan-dr doodlewind GHesericsu HeimdallStrategy imfing - jalehman jarvis-medmatic kkarimi mahmoudashraf93 pkrmf Randy Torres Ryan Lisse sumleo Yeom-JinHo zisisp - akyourowngames aldoeliacim Dithilli dougvk erikpr1994 fal3 Ghost jonasjancarik Keith the Silly Goose koala73 - L36 Server Marc mitschabaude-bot mkbehr Oren Rain shtse8 sibbl thesomewhatyou zats - chrisrodz echoVic Friederike Seiler gabriel-trigo ghsmc iamadig ibrahimq21 irtiq7 jeann2013 jogelin - Jonathan D. Rhyne (DJ-D) Joshua Mitchell Justin Ling kelvinCB Kit manmal MattQ Milofax mitsuhiko neist - pejmanjohn Ralph rmorse rubyrunsstuff rybnikov Steve (OpenClaw) suminhthanh svkozak wes-davis 24601 - AkashKobal ameno- awkoy BinHPdev bonald Chris Taylor dawondyifraw dguido Django Navarro evalexpr - henrino3 humanwritten hyojin joeykrug justinhuangcode larlyssa liuy ludd50155 Mark Liu natedenh - odysseus0 pcty-nextgen-service-account pi0 Roopak Nijhara Sean McLellan Syhids tmchow Ubuntu uli-will-code xiaose - Aaron Konyer aaronveklabs Aditya Singh andreabadesso Andrii battman21 BinaryMuse cash-echo-bot CJWTRUST Clawd - Clawdbot ClawdFx cordx56 danballance Elarwei001 EnzeD erik-agens Evizero fcatuhe gildo - Grynn hanxiao Ignacio itsjaydesu ivancasco ivanrvpereira Jarvis jayhickey jeffersonwarrior jeffersonwarrior - jverdi kentaro loeclos longmaba Marco Marandiz MarvinCui mjrussell odnxe optimikelabs oswalpalash - p6l-richard philipp-spiess Pocket Clawd RamiNoodle733 Raymond Berger Rob Axelsen Sash Catanzarite sauerdaniel Sriram Naidu Thota T5-AndyML - thejhinvirtuoso travisp VAC william arzt Yao yudshj zknicker ε°Ήε‡― {Suksham-sharma} 0oAstro - 8BlT Abdul535 abhaymundhara abhijeet117 aduk059 afurm aisling404 akari-musubi alejandro maza Alex-Alaniz - alexanderatallah alexstyl AlexZhangji amabito andrewting19 anisoptera araa47 arthyn Asleep123 Ayush Ojha - Ayush10 baccula beefiker bennewton999 bguidolim blacksmith-sh[bot] bqcfjwhz85-arch bravostation Buddy (AI) caelum0x - calvin-hpnet championswimmer chenglun.hu Chloe-VP Claw Clawdbot Maintainers cristip73 danielcadenhead dario-github DarwinsBuddy - David-Marsh-Photo davidbors-snyk dcantu96 dependabot[bot] Developer Dimitrios Ploutarchos Drake Thomsen dvrshil dxd5001 dylanneve1 - elliotsecops EmberCF ereid7 eternauta1337 f-trycua fan Felix Krause foeken frankekn fujiwara-tofu-shop - ganghyun kim gaowanqi08141999 gerardward2007 gitpds gtsifrikas habakan HassanFleyah HazAT hcl headswim - hlbbbbbbb Hubert hugobarauna hyaxia iamEvanYT ikari ikari-pl Iron ironbyte-rgb Ítalo Souza - Jamie Openshaw Jane Jarvis Deploy jarvis89757 jasonftl jasonsschin Jefferson Nunn jg-noncelogic jigar joeynyc - Jon Uleis Josh Long justyannicc Karim Naguib Kasper Neist Christjansen Keshav Rao Kevin Lin Kira knocte Knox - Kristijan Jovanovski Kyle Chen Latitude Bot Levi Figueira Liu Weizhan Lloyd Loganaden Velvindron lsh411 Lucas Kim Luka Zhang - LukΓ‘Ε‘ Loukota Lukin mac mimi mac26ai MackDing Mahsum Aktas Marc Beaupre Marcus Neves Mario Zechner Markus Buhatem Koch - Martin Púčik Martin SchΓΌrrer MarvinDontPanic Mateusz Michalik Matias Wainsten Matt Ezell Matt mini Matthew Dicembrino Mauro Bolis mcwigglesmcgee - meaadore1221-afk Mert Γ‡iΓ§ekΓ§i Michael Verrilli Miles minghinmatthewlam Mourad Boustani Mr. Guy Mustafa Tag Eldeen myfunc Nate - Nathaniel Kelner Netanel Draiman niceysam Nick Lamb Nick Taylor Nikolay Petrov NM nobrainer-tech Noctivoro norunners - Ocean Vael Ogulcan Celik Oleg Kossoy Olshansk Omar Khaleel OpenClaw Agent Ozgur Polat Pablo Nunez Palash Oswal pasogott - Patrick Shao Paul Pamment Paulo Portella Peter Lee Petra Donka Pham Nam pierreeurope pip-nomel plum-dawg pookNast - Pratham Dubey Quentin rafaelreis-r Raikan10 Ramin Shirali Hossein Zade Randy Torres Raphael Borg Ellul Vincenti Ratul Sarna Richard Pinedo Rick Qian - robhparker Rohan Nagpal Rohan Patil rohanpatriot Rolf Fredheim Rony Kelner Ryan Nelson Samrat Jha Santosh Sascha Reuter - Saurabh.Chopade saurav470 seans-openclawbot SecondThread seewhy Senol Dogan Sergiy Dybskiy Shadow shatner Shaun Loo - Shaun Mason Shiva Prasad Shrinija Kummari Siddhant Jain Simon Kelly SK Heavy Industries sldkfoiweuaranwdlaiwyeoaw Soumyadeep Ghosh Spacefish spiceoogway - Stephen Chen Steve succ985 Suksham Sunwoo Yu Suvin Nimnaka Swader swizzmagik Tag techboss - testingabc321 tewatia The Admiral therealZpoint-bot tian Xiao Tim Krase Timo Lins Tom McKenzie Tom Peri Tomas Hajek - Tomsun28 Tonic Travis Hinton Travis Irby Tulsi Prasad Ty Sabs Tyler uos-status Vai Varun Kruthiventi - Vibe Kanban Victor Castell victor-wu.eth vikpos Vincent VintLin Vladimir Peshekhonov void Vultr-Clawd Admin William Stock - williamtwomey Wimmie Winry Winston wolfred Xin Xinhe Hu Xu Haoran Yash Yaxuan42 - Yazin Yevhen Bobrov Yi Wang ymat19 Yuan Chen Yuanhai Zach Knickerbocker Zaf (via OpenClaw) zhixian 石川 θ«’ - 0xJonHoldsCrypto aaronn Alphonse-arianee atalovesyou Azade carlulsoe ddyo Erik hrdwdmrbl jiulingyun - kitze latitudeki5223 loukotal Manuel Maly minghinmatthewlam MSch odrobnik pcty-nextgen-ios-builder rafaelreis-r ratulsarna - reeltimeapps rhjoh ronak-guliani snopoke thesash timkrase +

+ GitHub stars

diff --git a/apps/web/app/api/chat/active/route.ts b/apps/web/app/api/chat/active/route.ts new file mode 100644 index 00000000000..9d7afac2ade --- /dev/null +++ b/apps/web/app/api/chat/active/route.ts @@ -0,0 +1,13 @@ +/** + * GET /api/chat/active + * + * Returns the session IDs of all currently running agent sessions. + * Used by the sidebar to show streaming indicators. + */ +import { getRunningSessionIds } from "@/lib/active-runs"; + +export const runtime = "nodejs"; + +export function GET() { + return Response.json({ sessionIds: getRunningSessionIds() }); +} diff --git a/apps/web/app/api/chat/chat.test.ts b/apps/web/app/api/chat/chat.test.ts new file mode 100644 index 00000000000..22e5a51c918 --- /dev/null +++ b/apps/web/app/api/chat/chat.test.ts @@ -0,0 +1,269 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Mock active-runs module +vi.mock("@/lib/active-runs", () => ({ + startRun: vi.fn(), + hasActiveRun: vi.fn(() => false), + subscribeToRun: vi.fn(), + persistUserMessage: vi.fn(), + abortRun: vi.fn(() => false), + getActiveRun: vi.fn(), + getRunningSessionIds: vi.fn(() => []), +})); + +// Mock workspace module +vi.mock("@/lib/workspace", () => ({ + resolveAgentWorkspacePrefix: vi.fn(() => null), +})); + +describe("Chat API routes", () => { + beforeEach(() => { + vi.resetModules(); + // Re-wire mocks + vi.mock("@/lib/active-runs", () => ({ + startRun: vi.fn(), + hasActiveRun: vi.fn(() => false), + subscribeToRun: vi.fn(), + persistUserMessage: vi.fn(), + abortRun: vi.fn(() => false), + getActiveRun: vi.fn(), + getRunningSessionIds: vi.fn(() => []), + })); + vi.mock("@/lib/workspace", () => ({ + resolveAgentWorkspacePrefix: vi.fn(() => null), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── POST /api/chat ────────────────────────────────────────────── + + describe("POST /api/chat", () => { + it("returns 400 when no user message text", async () => { + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ role: "user", parts: [{ type: "text", text: "" }] }], + }), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("returns 409 when active run exists for session", async () => { + const { hasActiveRun } = await import("@/lib/active-runs"); + vi.mocked(hasActiveRun).mockReturnValue(true); + + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ role: "user", parts: [{ type: "text", text: "hello" }] }], + sessionId: "s1", + }), + }); + const res = await POST(req); + expect(res.status).toBe(409); + }); + + it("starts a run and returns streaming response", async () => { + const { startRun, hasActiveRun, subscribeToRun } = await import("@/lib/active-runs"); + vi.mocked(hasActiveRun).mockReturnValue(false); + vi.mocked(subscribeToRun).mockReturnValue(() => {}); + + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [ + { id: "m1", role: "user", parts: [{ type: "text", text: "hello" }] }, + ], + sessionId: "s1", + }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + expect(res.headers.get("Content-Type")).toBe("text/event-stream"); + expect(startRun).toHaveBeenCalled(); + }); + + it("persists user message when sessionId provided", async () => { + const { hasActiveRun, subscribeToRun, persistUserMessage } = await import("@/lib/active-runs"); + vi.mocked(hasActiveRun).mockReturnValue(false); + vi.mocked(subscribeToRun).mockReturnValue(() => {}); + + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [ + { id: "m1", role: "user", parts: [{ type: "text", text: "hi" }] }, + ], + sessionId: "s1", + }), + }); + await POST(req); + expect(persistUserMessage).toHaveBeenCalledWith("s1", expect.objectContaining({ id: "m1" })); + }); + + it("resolves workspace file paths in message", async () => { + const { resolveAgentWorkspacePrefix } = await import("@/lib/workspace"); + vi.mocked(resolveAgentWorkspacePrefix).mockReturnValue("workspace"); + const { startRun, hasActiveRun, subscribeToRun } = await import("@/lib/active-runs"); + vi.mocked(hasActiveRun).mockReturnValue(false); + vi.mocked(subscribeToRun).mockReturnValue(() => {}); + + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [ + { + id: "m1", + role: "user", + parts: [{ type: "text", text: "[Context: workspace file 'doc.md']" }], + }, + ], + sessionId: "s1", + }), + }); + await POST(req); + expect(startRun).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining("workspace/doc.md"), + }), + ); + }); + }); + + // ─── POST /api/chat/stop ──────────────────────────────────────── + + describe("POST /api/chat/stop", () => { + it("returns 400 when sessionId missing", async () => { + const { POST } = await import("./stop/route.js"); + const req = new Request("http://localhost/api/chat/stop", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("aborts run and returns result", async () => { + const { abortRun } = await import("@/lib/active-runs"); + vi.mocked(abortRun).mockReturnValue(true); + + const { POST } = await import("./stop/route.js"); + const req = new Request("http://localhost/api/chat/stop", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionId: "s1" }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.aborted).toBe(true); + }); + + it("returns aborted=false for unknown session", async () => { + const { abortRun } = await import("@/lib/active-runs"); + vi.mocked(abortRun).mockReturnValue(false); + + const { POST } = await import("./stop/route.js"); + const req = new Request("http://localhost/api/chat/stop", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionId: "nonexistent" }), + }); + const res = await POST(req); + const json = await res.json(); + expect(json.aborted).toBe(false); + }); + + it("handles invalid JSON body gracefully", async () => { + const { POST } = await import("./stop/route.js"); + const req = new Request("http://localhost/api/chat/stop", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "not json", + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + }); + + // ─── GET /api/chat/active ──────────────────────────────────────── + + describe("GET /api/chat/active", () => { + it("returns empty sessionIds when no active runs", async () => { + const { GET } = await import("./active/route.js"); + const res = GET(); + const json = await res.json(); + expect(json.sessionIds).toEqual([]); + }); + + it("returns active session IDs", async () => { + const { getRunningSessionIds } = await import("@/lib/active-runs"); + vi.mocked(getRunningSessionIds).mockReturnValue(["s1", "s2"]); + + const { GET } = await import("./active/route.js"); + const res = GET(); + const json = await res.json(); + expect(json.sessionIds).toEqual(["s1", "s2"]); + }); + }); + + // ─── GET /api/chat/stream ─────────────────────────────────────── + + describe("GET /api/chat/stream", () => { + it("returns 400 when sessionId is missing", async () => { + const { GET } = await import("./stream/route.js"); + const req = new Request("http://localhost/api/chat/stream"); + const res = await GET(req); + expect(res.status).toBe(400); + }); + + it("returns 404 when no run exists for session", async () => { + const { getActiveRun } = await import("@/lib/active-runs"); + vi.mocked(getActiveRun).mockReturnValue(undefined); + + const { GET } = await import("./stream/route.js"); + const req = new Request("http://localhost/api/chat/stream?sessionId=nonexistent"); + const res = await GET(req); + expect(res.status).toBe(404); + }); + + it("returns SSE stream for active run", async () => { + const { getActiveRun, subscribeToRun } = await import("@/lib/active-runs"); + vi.mocked(getActiveRun).mockReturnValue({ status: "running" } as never); + vi.mocked(subscribeToRun).mockReturnValue(() => {}); + + const { GET } = await import("./stream/route.js"); + const req = new Request("http://localhost/api/chat/stream?sessionId=s1"); + const res = await GET(req); + expect(res.status).toBe(200); + expect(res.headers.get("Content-Type")).toBe("text/event-stream"); + expect(res.headers.get("X-Run-Active")).toBe("true"); + }); + + it("returns X-Run-Active=false for completed run", async () => { + const { getActiveRun, subscribeToRun } = await import("@/lib/active-runs"); + vi.mocked(getActiveRun).mockReturnValue({ status: "completed" } as never); + vi.mocked(subscribeToRun).mockReturnValue(() => {}); + + const { GET } = await import("./stream/route.js"); + const req = new Request("http://localhost/api/chat/stream?sessionId=s1"); + const res = await GET(req); + expect(res.headers.get("X-Run-Active")).toBe("false"); + }); + }); +}); diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts new file mode 100644 index 00000000000..7af4818542e --- /dev/null +++ b/apps/web/app/api/chat/route.ts @@ -0,0 +1,223 @@ +import type { UIMessage } from "ai"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveAgentWorkspacePrefix } from "@/lib/workspace"; +import { + startRun, + hasActiveRun, + subscribeToRun, + persistUserMessage, + type SseEvent as ParentSseEvent, +} from "@/lib/active-runs"; +import { + hasActiveSubagent, + isSubagentRunning, + ensureRegisteredFromDisk, + subscribeToSubagent, + persistUserMessage as persistSubagentUserMessage, + reactivateSubagent, + spawnSubagentMessage, + type SseEvent as SubagentSseEvent, +} from "@/lib/subagent-runs"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +// Force Node.js runtime (required for child_process) +export const runtime = "nodejs"; + +// Allow streaming responses up to 10 minutes +export const maxDuration = 600; + +function deriveSubagentParentSessionId(sessionKey: string): string { + const registryPath = join(resolveOpenClawStateDir(), "subagents", "runs.json"); + if (!existsSync(registryPath)) {return "";} + try { + const raw = JSON.parse(readFileSync(registryPath, "utf-8")) as { + runs?: Record>; + }; + for (const entry of Object.values(raw.runs ?? {})) { + if (entry.childSessionKey !== sessionKey) {continue;} + const requester = typeof entry.requesterSessionKey === "string" ? entry.requesterSessionKey : ""; + const match = requester.match(/^agent:[^:]+:web:(.+)$/); + return match?.[1] ?? ""; + } + } catch { + // ignore + } + return ""; +} + +function ensureSubagentRegistered(sessionKey: string): boolean { + if (hasActiveSubagent(sessionKey)) {return true;} + const parentWebSessionId = deriveSubagentParentSessionId(sessionKey); + return ensureRegisteredFromDisk(sessionKey, parentWebSessionId); +} + +export async function POST(req: Request) { + const { + messages, + sessionId, + sessionKey, + }: { messages: UIMessage[]; sessionId?: string; sessionKey?: string } = await req.json(); + + // Extract the latest user message text + const lastUserMessage = messages.filter((m) => m.role === "user").pop(); + const userText = + lastUserMessage?.parts + ?.filter( + (p): p is { type: "text"; text: string } => + p.type === "text", + ) + .map((p) => p.text) + .join("\n") ?? ""; + + if (!userText.trim()) { + return new Response("No message provided", { status: 400 }); + } + + const isSubagentSession = typeof sessionKey === "string" && sessionKey.includes(":subagent:"); + + // Reject if a run is already active for this session. + if (!isSubagentSession && sessionId && hasActiveRun(sessionId)) { + return new Response("Active run in progress", { status: 409 }); + } + if (isSubagentSession && isSubagentRunning(sessionKey)) { + return new Response("Active subagent run in progress", { status: 409 }); + } + + // Resolve workspace file paths to be agent-cwd-relative. + let agentMessage = userText; + const wsPrefix = resolveAgentWorkspacePrefix(); + if (wsPrefix) { + agentMessage = userText.replace( + /\[Context: workspace file '([^']+)'\]/, + `[Context: workspace file '${wsPrefix}/$1']`, + ); + } + + // Persist the user message server-side so it survives a page reload + // even if the client never gets a chance to save. + if (isSubagentSession && sessionKey && lastUserMessage) { + if (!ensureSubagentRegistered(sessionKey)) { + return new Response("Subagent not found", { status: 404 }); + } + persistSubagentUserMessage(sessionKey, { + id: lastUserMessage.id, + text: userText, + }); + } else if (sessionId && lastUserMessage) { + persistUserMessage(sessionId, { + id: lastUserMessage.id, + content: userText, + parts: lastUserMessage.parts as unknown[], + }); + } + + // Start the agent run (decoupled from this HTTP connection). + // The child process will keep running even if this response is cancelled. + if (isSubagentSession && sessionKey) { + if (!reactivateSubagent(sessionKey)) { + return new Response("Subagent not found", { status: 404 }); + } + if (!spawnSubagentMessage(sessionKey, agentMessage)) { + return new Response("Failed to start subagent run", { status: 500 }); + } + } else if (sessionId) { + try { + startRun({ + sessionId, + message: agentMessage, + agentSessionId: sessionId, + }); + } catch (err) { + return new Response( + err instanceof Error ? err.message : String(err), + { status: 500 }, + ); + } + } + + // Stream SSE events to the client using the AI SDK v6 wire format. + const encoder = new TextEncoder(); + let closed = false; + let unsubscribe: (() => void) | null = null; + + const stream = new ReadableStream({ + start(controller) { + if (!sessionId && !sessionKey) { + // No session β€” shouldn't happen but close gracefully. + controller.close(); + return; + } + + unsubscribe = isSubagentSession && sessionKey + ? subscribeToSubagent( + sessionKey, + (event: SubagentSseEvent | null) => { + if (closed) {return;} + if (event === null) { + closed = true; + try { + controller.close(); + } catch { + /* already closed */ + } + return; + } + try { + const json = JSON.stringify(event); + controller.enqueue(encoder.encode(`data: ${json}\n\n`)); + } catch { + /* ignore enqueue errors on closed stream */ + } + }, + { replay: false }, + ) + : subscribeToRun( + sessionId as string, + (event: ParentSseEvent | null) => { + if (closed) {return;} + if (event === null) { + // Run completed β€” close the SSE stream. + closed = true; + try { + controller.close(); + } catch { + /* already closed */ + } + return; + } + try { + const json = JSON.stringify(event); + controller.enqueue( + encoder.encode(`data: ${json}\n\n`), + ); + } catch { + /* ignore enqueue errors on closed stream */ + } + }, + // Don't replay β€” we just created the run, the buffer is empty. + { replay: false }, + ); + + if (!unsubscribe) { + // Race: run was cleaned up between startRun and subscribe. + closed = true; + controller.close(); + } + }, + cancel() { + // Client disconnected β€” unsubscribe but keep the run alive. + // The ActiveRunManager continues buffering + persisting in the background. + closed = true; + unsubscribe?.(); + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + }, + }); +} diff --git a/apps/web/app/api/chat/stop/route.ts b/apps/web/app/api/chat/stop/route.ts new file mode 100644 index 00000000000..02b1a66e488 --- /dev/null +++ b/apps/web/app/api/chat/stop/route.ts @@ -0,0 +1,60 @@ +/** + * POST /api/chat/stop + * + * Abort an active agent run. Called by the Stop button. + * The child process is sent SIGTERM and the run transitions to "error" state. + */ +import { abortRun } from "@/lib/active-runs"; +import { + abortSubagent, + hasActiveSubagent, + isSubagentRunning, + ensureRegisteredFromDisk, +} from "@/lib/subagent-runs"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const runtime = "nodejs"; + +function deriveSubagentParentSessionId(sessionKey: string): string { + const registryPath = join(resolveOpenClawStateDir(), "subagents", "runs.json"); + if (!existsSync(registryPath)) {return "";} + try { + const raw = JSON.parse(readFileSync(registryPath, "utf-8")) as { + runs?: Record>; + }; + for (const entry of Object.values(raw.runs ?? {})) { + if (entry.childSessionKey !== sessionKey) {continue;} + const requester = typeof entry.requesterSessionKey === "string" ? entry.requesterSessionKey : ""; + const match = requester.match(/^agent:[^:]+:web:(.+)$/); + return match?.[1] ?? ""; + } + } catch { + // ignore + } + return ""; +} + +export async function POST(req: Request) { + const body: { sessionId?: string; sessionKey?: string } = await req + .json() + .catch(() => ({})); + + const isSubagentSession = typeof body.sessionKey === "string" && body.sessionKey.includes(":subagent:"); + if (isSubagentSession && body.sessionKey) { + if (!hasActiveSubagent(body.sessionKey)) { + const parentWebSessionId = deriveSubagentParentSessionId(body.sessionKey); + ensureRegisteredFromDisk(body.sessionKey, parentWebSessionId); + } + const aborted = isSubagentRunning(body.sessionKey) ? abortSubagent(body.sessionKey) : false; + return Response.json({ aborted }); + } + + if (!body.sessionId) { + return new Response("sessionId or subagent sessionKey required", { status: 400 }); + } + + const aborted = abortRun(body.sessionId); + return Response.json({ aborted }); +} diff --git a/apps/web/app/api/chat/stream/route.ts b/apps/web/app/api/chat/stream/route.ts new file mode 100644 index 00000000000..c1c16fd563e --- /dev/null +++ b/apps/web/app/api/chat/stream/route.ts @@ -0,0 +1,201 @@ +/** + * GET /api/chat/stream?sessionId=xxx + * + * Reconnect to an active (or recently-completed) agent run. + * Replays all buffered SSE events from the start of the run, then + * streams live events until the run finishes. + * + * Returns 404 if no run exists for the given session. + */ +import { + getActiveRun, + subscribeToRun, + type SseEvent as ParentSseEvent, +} from "@/lib/active-runs"; +import { + subscribeToSubagent, + hasActiveSubagent, + isSubagentRunning, + ensureRegisteredFromDisk, + ensureSubagentStreamable, + type SseEvent as SubagentSseEvent, +} from "@/lib/subagent-runs"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const runtime = "nodejs"; +export const maxDuration = 600; + +function deriveSubagentParentSessionId(sessionKey: string): string { + const registryPath = join(resolveOpenClawStateDir(), "subagents", "runs.json"); + if (!existsSync(registryPath)) {return "";} + try { + const raw = JSON.parse(readFileSync(registryPath, "utf-8")) as { + runs?: Record>; + }; + for (const entry of Object.values(raw.runs ?? {})) { + if (entry.childSessionKey !== sessionKey) {continue;} + const requester = typeof entry.requesterSessionKey === "string" ? entry.requesterSessionKey : ""; + const match = requester.match(/^agent:[^:]+:web:(.+)$/); + return match?.[1] ?? ""; + } + } catch { + // ignore + } + return ""; +} + +export async function GET(req: Request) { + const url = new URL(req.url); + const sessionId = url.searchParams.get("sessionId"); + const sessionKey = url.searchParams.get("sessionKey"); + const isSubagentSession = typeof sessionKey === "string" && sessionKey.includes(":subagent:"); + + if (!sessionId && !sessionKey) { + return new Response("sessionId or subagent sessionKey required", { status: 400 }); + } + + if (isSubagentSession && sessionKey) { + if (!hasActiveSubagent(sessionKey)) { + const parentWebSessionId = deriveSubagentParentSessionId(sessionKey); + const registered = ensureRegisteredFromDisk(sessionKey, parentWebSessionId); + if (!registered && !hasActiveSubagent(sessionKey)) { + return Response.json({ active: false }, { status: 404 }); + } + } + ensureSubagentStreamable(sessionKey); + const isActive = isSubagentRunning(sessionKey); + const encoder = new TextEncoder(); + let closed = false; + let unsubscribe: (() => void) | null = null; + + const stream = new ReadableStream({ + start(controller) { + unsubscribe = subscribeToSubagent( + sessionKey, + (event: SubagentSseEvent | null) => { + if (closed) {return;} + if (event === null) { + closed = true; + try { + controller.close(); + } catch { + /* already closed */ + } + return; + } + try { + const json = JSON.stringify(event); + controller.enqueue(encoder.encode(`data: ${json}\n\n`)); + } catch { + /* ignore enqueue errors on closed stream */ + } + }, + { replay: true }, + ); + + if (!unsubscribe) { + closed = true; + controller.close(); + } + }, + cancel() { + closed = true; + unsubscribe?.(); + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Run-Active": isActive ? "true" : "false", + }, + }); + } + const run = getActiveRun(sessionId as string); + if (!run) { + return Response.json({ active: false }, { status: 404 }); + } + + const encoder = new TextEncoder(); + let closed = false; + let unsubscribe: (() => void) | null = null; + let keepalive: ReturnType | null = null; + + const stream = new ReadableStream({ + start(controller) { + // Keep idle SSE connections alive while waiting for subagent announcements. + keepalive = setInterval(() => { + if (closed) {return;} + try { + controller.enqueue(encoder.encode(": keepalive\n\n")); + } catch { + /* ignore enqueue errors on closed stream */ + } + }, 15_000); + + // subscribeToRun with replay=true replays the full event buffer + // synchronously, then subscribes for live events. + unsubscribe = subscribeToRun( + sessionId as string, + (event: ParentSseEvent | null) => { + if (closed) {return;} + if (event === null) { + // Run completed β€” close the SSE stream. + closed = true; + if (keepalive) { + clearInterval(keepalive); + keepalive = null; + } + try { + controller.close(); + } catch { + /* already closed */ + } + return; + } + try { + const json = JSON.stringify(event); + controller.enqueue( + encoder.encode(`data: ${json}\n\n`), + ); + } catch { + /* ignore enqueue errors on closed stream */ + } + }, + { replay: true }, + ); + + if (!unsubscribe) { + // Run was cleaned up between getActiveRun and subscribe. + closed = true; + if (keepalive) { + clearInterval(keepalive); + keepalive = null; + } + controller.close(); + } + }, + cancel() { + // Client disconnected β€” unsubscribe only (don't kill the run). + closed = true; + if (keepalive) { + clearInterval(keepalive); + keepalive = null; + } + unsubscribe?.(); + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Run-Active": run.status === "running" || run.status === "waiting-for-subagents" ? "true" : "false", + }, + }); +} diff --git a/apps/web/app/api/chat/subagents/route.ts b/apps/web/app/api/chat/subagents/route.ts new file mode 100644 index 00000000000..35f3ab6f811 --- /dev/null +++ b/apps/web/app/api/chat/subagents/route.ts @@ -0,0 +1,64 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const runtime = "nodejs"; + +type RegistryEntry = { + runId: string; + childSessionKey: string; + requesterSessionKey: string; + task: string; + label?: string; + createdAt?: number; + endedAt?: number; + outcome?: { status: string; error?: string }; +}; + +function readSubagentRegistry(): RegistryEntry[] { + const registryPath = join(resolveOpenClawStateDir(), "subagents", "runs.json"); + if (!existsSync(registryPath)) {return [];} + + try { + const raw = JSON.parse(readFileSync(registryPath, "utf-8")); + if (!raw || typeof raw !== "object") {return [];} + const runs = raw.runs; + if (!runs || typeof runs !== "object") {return [];} + return Object.values(runs); + } catch { + return []; + } +} + +export async function GET(req: Request) { + const url = new URL(req.url); + const sessionId = url.searchParams.get("sessionId"); + + if (!sessionId) { + return Response.json({ error: "sessionId required" }, { status: 400 }); + } + + const webSessionKey = `agent:main:web:${sessionId}`; + const entries = readSubagentRegistry(); + + const subagents = entries + .filter((e) => e.requesterSessionKey === webSessionKey) + .map((e) => ({ + sessionKey: e.childSessionKey, + runId: e.runId, + task: e.task, + label: e.label || undefined, + status: resolveStatus(e), + startedAt: e.createdAt, + endedAt: e.endedAt, + })) + .toSorted((a, b) => (a.startedAt ?? 0) - (b.startedAt ?? 0)); + + return Response.json({ subagents }); +} + +function resolveStatus(e: RegistryEntry): "running" | "completed" | "error" { + if (typeof e.endedAt !== "number") {return "running";} + if (e.outcome?.status === "error") {return "error";} + return "completed"; +} diff --git a/apps/web/app/api/cron/cron.test.ts b/apps/web/app/api/cron/cron.test.ts new file mode 100644 index 00000000000..48d3e71ce6c --- /dev/null +++ b/apps/web/app/api/cron/cron.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Mock node:fs +vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + statSync: vi.fn(() => ({ mtimeMs: Date.now() })), +})); + +// Mock node:os +vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), +})); + +describe("Cron API routes", () => { + beforeEach(() => { + vi.resetModules(); + vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + statSync: vi.fn(() => ({ mtimeMs: Date.now() })), + })); + vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── GET /api/cron/jobs ───────────────────────────────────────── + + describe("GET /api/cron/jobs", () => { + it("returns empty jobs when no config file", async () => { + const { GET } = await import("./jobs/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.jobs).toEqual([]); + }); + + it("returns jobs from config file", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + const cronStore = { + version: 1, + jobs: [ + { id: "j1", name: "Daily sync", schedule: "0 8 * * *", enabled: true, command: "sync" }, + ], + }; + vi.mocked(mockReadFile).mockReturnValue(JSON.stringify(cronStore) as never); + + const { GET } = await import("./jobs/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.jobs).toHaveLength(1); + expect(json.jobs[0].name).toBe("Daily sync"); + }); + + it("handles corrupt jobs file gracefully", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReadFile).mockReturnValue("not json" as never); + + const { GET } = await import("./jobs/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.jobs).toEqual([]); + }); + }); + + // ─── GET /api/cron/jobs/[jobId]/runs ──────────────────────────── + + describe("GET /api/cron/jobs/[jobId]/runs", () => { + it("returns empty entries when no runs file", async () => { + const { GET } = await import("./jobs/[jobId]/runs/route.js"); + const res = await GET( + new Request("http://localhost/api/cron/jobs/j1/runs"), + { params: Promise.resolve({ jobId: "j1" }) }, + ); + const json = await res.json(); + expect(json.entries).toEqual([]); + }); + + it("returns run entries from jsonl file", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + const lines = [ + JSON.stringify({ ts: 1000, jobId: "j1", action: "finished", status: "completed", summary: "Done" }), + JSON.stringify({ ts: 2000, jobId: "j1", action: "finished", status: "completed", summary: "In progress" }), + ].join("\n"); + vi.mocked(mockReadFile).mockReturnValue(lines as never); + + const { GET } = await import("./jobs/[jobId]/runs/route.js"); + const res = await GET( + new Request("http://localhost/api/cron/jobs/j1/runs"), + { params: Promise.resolve({ jobId: "j1" }) }, + ); + const json = await res.json(); + expect(json.entries.length).toBeGreaterThan(0); + }); + + it("respects limit query param", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + const lines = Array.from({ length: 50 }, (_, i) => + JSON.stringify({ ts: i, status: "completed" }), + ).join("\n"); + vi.mocked(mockReadFile).mockReturnValue(lines as never); + + const { GET } = await import("./jobs/[jobId]/runs/route.js"); + const res = await GET( + new Request("http://localhost/api/cron/jobs/j1/runs?limit=5"), + { params: Promise.resolve({ jobId: "j1" }) }, + ); + const json = await res.json(); + expect(json.entries.length).toBeLessThanOrEqual(5); + }); + }); + + // ─── GET /api/cron/runs/[sessionId] ───────────────────────────── + + describe("GET /api/cron/runs/[sessionId]", () => { + it("returns 404 when session not found", async () => { + const { GET } = await import("./runs/[sessionId]/route.js"); + const res = await GET( + new Request("http://localhost"), + { params: Promise.resolve({ sessionId: "nonexistent" }) }, + ); + expect(res.status).toBe(404); + }); + }); + + // ─── GET /api/cron/runs/search-transcript ─────────────────────── + + describe("GET /api/cron/runs/search-transcript", () => { + it("returns 400 when missing required params", async () => { + const { GET } = await import("./runs/search-transcript/route.js"); + const req = new Request("http://localhost/api/cron/runs/search-transcript"); + const res = await GET(req); + expect(res.status).toBe(400); + }); + + it("returns 404 when no transcript found", async () => { + const { GET } = await import("./runs/search-transcript/route.js"); + const req = new Request("http://localhost/api/cron/runs/search-transcript?jobId=j1&runAtMs=1000"); + const res = await GET(req); + expect(res.status).toBe(404); + }); + }); +}); diff --git a/apps/web/app/api/cron/jobs/[jobId]/runs/route.ts b/apps/web/app/api/cron/jobs/[jobId]/runs/route.ts new file mode 100644 index 00000000000..05fa62cea44 --- /dev/null +++ b/apps/web/app/api/cron/jobs/[jobId]/runs/route.ts @@ -0,0 +1,86 @@ +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +const CRON_DIR = join(resolveOpenClawStateDir(), "cron"); + +type CronRunLogEntry = { + ts: number; + jobId: string; + action: "finished"; + status?: string; + error?: string; + summary?: string; + sessionId?: string; + sessionKey?: string; + runAtMs?: number; + durationMs?: number; + nextRunAtMs?: number; +}; + +/** Read run log entries from a JSONL file, returning most recent first (then reversed). */ +function readRunLog(filePath: string, limit: number): CronRunLogEntry[] { + if (!existsSync(filePath)) {return [];} + try { + const raw = readFileSync(filePath, "utf-8"); + if (!raw.trim()) {return [];} + const lines = raw.split("\n"); + const parsed: CronRunLogEntry[] = []; + for (let i = lines.length - 1; i >= 0 && parsed.length < limit; i--) { + const line = lines[i]?.trim(); + if (!line) {continue;} + try { + const obj = JSON.parse(line) as Partial; + if (!obj || typeof obj !== "object") {continue;} + if (obj.action !== "finished") {continue;} + if (typeof obj.jobId !== "string" || !obj.jobId.trim()) {continue;} + if (typeof obj.ts !== "number" || !Number.isFinite(obj.ts)) {continue;} + const entry: CronRunLogEntry = { + ts: obj.ts, + jobId: obj.jobId, + action: "finished", + status: obj.status, + error: obj.error, + summary: obj.summary, + runAtMs: obj.runAtMs, + durationMs: obj.durationMs, + nextRunAtMs: obj.nextRunAtMs, + }; + if (typeof obj.sessionId === "string" && obj.sessionId.trim()) { + entry.sessionId = obj.sessionId; + } + if (typeof obj.sessionKey === "string" && obj.sessionKey.trim()) { + entry.sessionKey = obj.sessionKey; + } + parsed.push(entry); + } catch { + // skip malformed lines + } + } + return parsed.toReversed(); + } catch { + return []; + } +} + +/** GET /api/cron/jobs/[jobId]/runs -- list run log entries for a cron job */ +export async function GET( + request: Request, + { params }: { params: Promise<{ jobId: string }> }, +) { + const { jobId } = await params; + if (!jobId) { + return Response.json({ error: "Job ID required" }, { status: 400 }); + } + + const url = new URL(request.url); + const limitParam = url.searchParams.get("limit"); + const limit = Math.max(1, Math.min(500, Number(limitParam) || 100)); + + const logPath = join(CRON_DIR, "runs", `${jobId}.jsonl`); + const entries = readRunLog(logPath, limit); + + return Response.json({ entries }); +} diff --git a/apps/web/app/api/cron/jobs/route.ts b/apps/web/app/api/cron/jobs/route.ts new file mode 100644 index 00000000000..0a77df08fb2 --- /dev/null +++ b/apps/web/app/api/cron/jobs/route.ts @@ -0,0 +1,99 @@ +import { readFileSync, existsSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +const CRON_DIR = join(resolveOpenClawStateDir(), "cron"); +const JOBS_FILE = join(CRON_DIR, "jobs.json"); + +type CronStoreFile = { + version: 1; + jobs: Array>; +}; + +/** Read cron jobs.json, returning empty array if missing or invalid. */ +function readJobsFile(): Array> { + if (!existsSync(JOBS_FILE)) {return [];} + try { + const raw = readFileSync(JOBS_FILE, "utf-8"); + const parsed = JSON.parse(raw) as CronStoreFile; + if (parsed && Array.isArray(parsed.jobs)) {return parsed.jobs;} + return []; + } catch { + return []; + } +} + +/** Compute next wake time from job states (minimum nextRunAtMs among enabled jobs). */ +function computeNextWakeAtMs(jobs: Array>): number | null { + let min: number | null = null; + for (const job of jobs) { + if (job.enabled !== true) {continue;} + const state = job.state as Record | undefined; + if (!state) {continue;} + const next = state.nextRunAtMs; + if (typeof next === "number" && Number.isFinite(next)) { + if (min === null || next < min) {min = next;} + } + } + return min; +} + +/** Read heartbeat config from ~/.openclaw/config.yaml (best-effort). */ +function readHeartbeatInfo(): { intervalMs: number; nextDueEstimateMs: number | null } { + const defaults = { intervalMs: 30 * 60_000, nextDueEstimateMs: null as number | null }; + + // Try to read agent session stores to estimate next heartbeat from lastRunMs + try { + const agentsDir = join(resolveOpenClawStateDir(), "agents"); + if (!existsSync(agentsDir)) {return defaults;} + + const agentDirs = readdirSync(agentsDir, { withFileTypes: true }); + let latestHeartbeat: number | null = null; + + for (const d of agentDirs) { + if (!d.isDirectory()) {continue;} + const storePath = join(agentsDir, d.name, "sessions", "sessions.json"); + if (!existsSync(storePath)) {continue;} + try { + const raw = readFileSync(storePath, "utf-8"); + const store = JSON.parse(raw) as Record; + // Look for the main agent session (shortest key, most recently updated) + for (const [key, entry] of Object.entries(store)) { + if (key.startsWith("agent:") && !key.includes(":cron:") && entry.updatedAt) { + if (latestHeartbeat === null || entry.updatedAt > latestHeartbeat) { + latestHeartbeat = entry.updatedAt; + } + } + } + } catch { + // skip + } + } + + if (latestHeartbeat) { + defaults.nextDueEstimateMs = latestHeartbeat + defaults.intervalMs; + } + } catch { + // ignore + } + + return defaults; +} + +/** GET /api/cron/jobs -- list all cron jobs with heartbeat & status info */ +export async function GET() { + const jobs = readJobsFile(); + const heartbeat = readHeartbeatInfo(); + const nextWakeAtMs = computeNextWakeAtMs(jobs); + + return Response.json({ + jobs, + heartbeat, + cronStatus: { + enabled: jobs.length > 0, + nextWakeAtMs, + }, + }); +} diff --git a/apps/web/app/api/cron/runs/[sessionId]/route.ts b/apps/web/app/api/cron/runs/[sessionId]/route.ts new file mode 100644 index 00000000000..5d6369c4c1a --- /dev/null +++ b/apps/web/app/api/cron/runs/[sessionId]/route.ts @@ -0,0 +1,154 @@ +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +type MessagePart = + | { type: "text"; text: string } + | { type: "thinking"; thinking: string } + | { type: "tool-call"; toolName: string; toolCallId: string; args?: unknown; output?: string }; + +type ParsedMessage = { + id: string; + role: "user" | "assistant" | "system"; + parts: MessagePart[]; + timestamp: string; +}; + +/** Search agent session directories for a session file by ID. */ +function findSessionFile(sessionId: string): string | null { + const agentsDir = join(resolveOpenClawStateDir(), "agents"); + if (!existsSync(agentsDir)) {return null;} + + try { + const agentDirs = readdirSync(agentsDir, { withFileTypes: true }); + for (const agentDir of agentDirs) { + if (!agentDir.isDirectory()) {continue;} + const sessionFile = join(agentsDir, agentDir.name, "sessions", `${sessionId}.jsonl`); + if (existsSync(sessionFile)) {return sessionFile;} + } + } catch { + // ignore + } + return null; +} + +/** Parse a JSONL session transcript into structured messages with thinking and tool calls. */ +function parseSessionTranscript(content: string): ParsedMessage[] { + const lines = content.trim().split("\n").filter((l) => l.trim()); + const messages: ParsedMessage[] = []; + + // Track tool calls for linking invocations with results + const pendingToolCalls = new Map(); + + for (const line of lines) { + try { + const entry = JSON.parse(line); + + if (entry.type === "message" && entry.message) { + const msg = entry.message; + const role = msg.role as "user" | "assistant" | "system"; + const parts: MessagePart[] = []; + + if (Array.isArray(msg.content)) { + for (const part of msg.content) { + if (part.type === "text" && typeof part.text === "string" && part.text.trim()) { + parts.push({ type: "text", text: part.text }); + } else if (part.type === "thinking" && typeof part.thinking === "string" && part.thinking.trim()) { + parts.push({ type: "thinking", thinking: part.thinking }); + } else if (part.type === "tool_use" || part.type === "tool-call") { + const toolName = part.name ?? part.toolName ?? "unknown"; + const toolCallId = part.id ?? part.toolCallId ?? `tool-${Date.now()}`; + pendingToolCalls.set(toolCallId, { toolName, args: part.input ?? part.args }); + parts.push({ + type: "tool-call", + toolName, + toolCallId, + args: part.input ?? part.args, + }); + } else if (part.type === "tool_result" || part.type === "tool-result") { + const toolCallId = part.tool_use_id ?? part.toolCallId ?? ""; + const pending = pendingToolCalls.get(toolCallId); + const outputText = typeof part.content === "string" + ? part.content + : Array.isArray(part.content) + ? part.content.filter((c: { type: string }) => c.type === "text").map((c: { text: string }) => c.text).join("\n") + : typeof part.output === "string" + ? part.output + : JSON.stringify(part.output ?? part.content ?? ""); + + if (pending) { + // Merge output into existing tool-call part + const existingMsg = messages[messages.length - 1]; + if (existingMsg) { + const tc = existingMsg.parts.find( + (p) => p.type === "tool-call" && (p as { toolCallId: string }).toolCallId === toolCallId, + ); + if (tc && tc.type === "tool-call") { + (tc as { output?: string }).output = outputText.slice(0, 5000); + continue; + } + } + parts.push({ + type: "tool-call", + toolName: pending.toolName, + toolCallId, + args: pending.args, + output: outputText.slice(0, 5000), + }); + } else { + parts.push({ + type: "tool-call", + toolName: "tool", + toolCallId, + output: outputText.slice(0, 5000), + }); + } + } + } + } else if (typeof msg.content === "string" && msg.content.trim()) { + parts.push({ type: "text", text: msg.content }); + } + + if (parts.length > 0) { + messages.push({ + id: entry.id ?? `msg-${messages.length}`, + role, + parts, + timestamp: entry.timestamp ?? new Date(entry.ts ?? Date.now()).toISOString(), + }); + } + } + } catch { + // skip malformed lines + } + } + + return messages; +} + +/** GET /api/cron/runs/[sessionId] -- get full session transcript for a cron run */ +export async function GET( + _request: Request, + { params }: { params: Promise<{ sessionId: string }> }, +) { + const { sessionId } = await params; + if (!sessionId) { + return Response.json({ error: "Session ID required" }, { status: 400 }); + } + + const sessionFile = findSessionFile(sessionId); + if (!sessionFile) { + return Response.json({ error: "Session not found" }, { status: 404 }); + } + + try { + const content = readFileSync(sessionFile, "utf-8"); + const messages = parseSessionTranscript(content); + return Response.json({ sessionId, messages }); + } catch (error) { + console.error("Error reading cron session:", error); + return Response.json({ error: "Failed to read session" }, { status: 500 }); + } +} diff --git a/apps/web/app/api/cron/runs/search-transcript/route.ts b/apps/web/app/api/cron/runs/search-transcript/route.ts new file mode 100644 index 00000000000..2b2a91be019 --- /dev/null +++ b/apps/web/app/api/cron/runs/search-transcript/route.ts @@ -0,0 +1,315 @@ +import { readFileSync, readdirSync, existsSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +const AGENTS_DIR = join(resolveOpenClawStateDir(), "agents"); + +type MessagePart = + | { type: "text"; text: string } + | { type: "thinking"; thinking: string } + | { type: "tool-call"; toolName: string; toolCallId: string; args?: unknown; output?: string }; + +type ParsedMessage = { + id: string; + role: "user" | "assistant" | "system"; + parts: MessagePart[]; + timestamp: string; +}; + +/** + * Search for the actual agent transcript for a cron run. + * + * For main-target cron runs, the agent response lives in the main session + * transcript files. This endpoint searches session files for the cron payload + * text near the run timestamp and returns the matching conversation + * (user message + assistant response). + */ + +/** Try to find a cron-specific session from sessions.json. */ +function findCronSessionId(jobId: string): string | null { + if (!existsSync(AGENTS_DIR)) {return null;} + try { + const agentDirs = readdirSync(AGENTS_DIR, { withFileTypes: true }); + for (const agentDir of agentDirs) { + if (!agentDir.isDirectory()) {continue;} + const sessionsJsonPath = join(AGENTS_DIR, agentDir.name, "sessions", "sessions.json"); + if (!existsSync(sessionsJsonPath)) {continue;} + try { + const store = JSON.parse(readFileSync(sessionsJsonPath, "utf-8")); + // Look for cron session key matching this job + for (const [key, entry] of Object.entries(store)) { + if (key.includes(`:cron:${jobId}`) && !key.includes(":run:")) { + const sessionId = (entry as { sessionId?: string })?.sessionId; + if (typeof sessionId === "string" && sessionId.trim()) { + // Verify the session file actually exists + const sessionFile = join(AGENTS_DIR, agentDir.name, "sessions", `${sessionId}.jsonl`); + if (existsSync(sessionFile)) { + return sessionId; + } + } + } + } + } catch { + // skip malformed sessions.json + } + } + } catch { + // ignore + } + return null; +} + +/** Find session files that might contain the cron run's transcript. */ +function findCandidateSessionFiles(runAtMs: number): string[] { + const candidates: Array<{ path: string; mtimeMs: number }> = []; + if (!existsSync(AGENTS_DIR)) {return [];} + + try { + const agentDirs = readdirSync(AGENTS_DIR, { withFileTypes: true }); + for (const agentDir of agentDirs) { + if (!agentDir.isDirectory()) {continue;} + const sessionsDir = join(AGENTS_DIR, agentDir.name, "sessions"); + if (!existsSync(sessionsDir)) {continue;} + try { + const files = readdirSync(sessionsDir); + for (const file of files) { + if (!file.endsWith(".jsonl")) {continue;} + const filePath = join(sessionsDir, file); + try { + const stat = statSync(filePath); + // Only consider files modified within Β±2 hours of the run + const windowMs = 2 * 60 * 60 * 1000; + if (Math.abs(stat.mtimeMs - runAtMs) < windowMs) { + candidates.push({ path: filePath, mtimeMs: stat.mtimeMs }); + } + } catch { + // skip + } + } + } catch { + // skip + } + } + } catch { + // ignore + } + + // Sort by closest modification time to runAtMs + candidates.sort((a, b) => Math.abs(a.mtimeMs - runAtMs) - Math.abs(b.mtimeMs - runAtMs)); + + // Limit to 10 most likely candidates + return candidates.slice(0, 10).map((c) => c.path); +} + +/** Parse message entries from a JSONL transcript, optionally filtered by time range. */ +function parseMessagesInRange( + content: string, + opts?: { afterMs?: number; beforeMs?: number }, +): ParsedMessage[] { + const lines = content.trim().split("\n").filter((l) => l.trim()); + const messages: ParsedMessage[] = []; + const pendingToolCalls = new Map(); + + for (const line of lines) { + try { + const entry = JSON.parse(line); + if (entry.type !== "message" || !entry.message) {continue;} + + // Filter by timestamp if provided + if (opts?.afterMs || opts?.beforeMs) { + const ts = entry.timestamp ? new Date(entry.timestamp).getTime() : (entry.ts ?? 0); + if (opts.afterMs && ts < opts.afterMs) {continue;} + if (opts.beforeMs && ts > opts.beforeMs) {continue;} + } + + const msg = entry.message; + const role = msg.role as "user" | "assistant" | "system"; + const parts: MessagePart[] = []; + + if (Array.isArray(msg.content)) { + for (const part of msg.content) { + if (part.type === "text" && typeof part.text === "string" && part.text.trim()) { + parts.push({ type: "text", text: part.text }); + } else if (part.type === "thinking" && typeof part.thinking === "string" && part.thinking.trim()) { + parts.push({ type: "thinking", thinking: part.thinking }); + } else if (part.type === "tool_use" || part.type === "tool-call") { + const toolName = part.name ?? part.toolName ?? "unknown"; + const toolCallId = part.id ?? part.toolCallId ?? `tool-${Date.now()}`; + pendingToolCalls.set(toolCallId, { toolName, args: part.input ?? part.args }); + parts.push({ type: "tool-call", toolName, toolCallId, args: part.input ?? part.args }); + } else if (part.type === "tool_result" || part.type === "tool-result") { + const toolCallId = part.tool_use_id ?? part.toolCallId ?? ""; + const pending = pendingToolCalls.get(toolCallId); + const outputText = typeof part.content === "string" + ? part.content + : Array.isArray(part.content) + ? part.content.filter((c: { type: string }) => c.type === "text").map((c: { text: string }) => c.text).join("\n") + : typeof part.output === "string" + ? part.output + : JSON.stringify(part.output ?? part.content ?? ""); + + if (pending) { + const existingMsg = messages[messages.length - 1]; + if (existingMsg) { + const tc = existingMsg.parts.find( + (p) => p.type === "tool-call" && (p as { toolCallId: string }).toolCallId === toolCallId, + ); + if (tc && tc.type === "tool-call") { + (tc as { output?: string }).output = outputText.slice(0, 5000); + continue; + } + } + parts.push({ type: "tool-call", toolName: pending.toolName, toolCallId, args: pending.args, output: outputText.slice(0, 5000) }); + } else { + parts.push({ type: "tool-call", toolName: "tool", toolCallId, output: outputText.slice(0, 5000) }); + } + } + } + } else if (typeof msg.content === "string" && msg.content.trim()) { + parts.push({ type: "text", text: msg.content }); + } + + if (parts.length > 0) { + messages.push({ + id: entry.id ?? `msg-${messages.length}`, + role, + parts, + timestamp: entry.timestamp ?? new Date(entry.ts ?? Date.now()).toISOString(), + }); + } + } catch { + // skip malformed lines + } + } + + return messages; +} + +/** Extract text content from message parts. */ +function getMessageText(msg: ParsedMessage): string { + return msg.parts + .filter((p): p is { type: "text"; text: string } => p.type === "text") + .map((p) => p.text) + .join("\n"); +} + +/** + * Search session files for the cron run's conversation. + * Matches by finding a user message containing the summary text near runAtMs, + * then returns that message + all following messages until the next user message. + */ +function searchForRunTranscript( + sessionFiles: string[], + summary: string, + runAtMs: number, +): { messages: ParsedMessage[]; sessionFile: string } | null { + // Use a distinctive portion of the summary for matching (first 80 chars) + const searchText = summary.slice(0, 80); + // Search window: from 5s before run to 10 minutes after (heartbeat delay) + const afterMs = runAtMs - 5_000; + const beforeMs = runAtMs + 10 * 60_000; + + for (const filePath of sessionFiles) { + try { + const content = readFileSync(filePath, "utf-8"); + if (!content.includes(searchText.slice(0, 40))) { + // Quick pre-check: skip files that don't contain the text at all + continue; + } + + const allMessages = parseMessagesInRange(content); + + // Find user messages containing the summary text within the time window + for (let i = 0; i < allMessages.length; i++) { + const msg = allMessages[i]; + if (msg.role !== "user") {continue;} + + const msgTs = new Date(msg.timestamp).getTime(); + if (msgTs < afterMs || msgTs > beforeMs) {continue;} + + const text = getMessageText(msg); + if (!text.includes(searchText.slice(0, 40))) {continue;} + + // Found the user message! Collect it + all following messages + // until the next user message (the full agent turn). + const conversation: ParsedMessage[] = [msg]; + for (let j = i + 1; j < allMessages.length; j++) { + const next = allMessages[j]; + if (next.role === "user") {break;} + conversation.push(next); + } + + return { messages: conversation, sessionFile: filePath }; + } + } catch { + // skip unreadable files + } + } + + return null; +} + +/** + * GET /api/cron/runs/search-transcript?jobId=X&runAtMs=Y&summary=Z + * + * Search for the actual agent transcript for a cron run that doesn't have + * a direct sessionId. Tries: + * 1. Sessions.json lookup for a cron-specific session + * 2. Time-based search of session files near the run timestamp + */ +export async function GET(request: Request) { + const url = new URL(request.url); + const jobId = url.searchParams.get("jobId"); + const runAtMsStr = url.searchParams.get("runAtMs"); + const summary = url.searchParams.get("summary"); + + if (!jobId || !runAtMsStr) { + return Response.json({ error: "jobId and runAtMs are required" }, { status: 400 }); + } + + const runAtMs = Number(runAtMsStr); + if (!Number.isFinite(runAtMs)) { + return Response.json({ error: "Invalid runAtMs" }, { status: 400 }); + } + + // Strategy 1: Look for a cron-specific session in sessions.json + const cronSessionId = findCronSessionId(jobId); + if (cronSessionId) { + try { + const agentDirs = readdirSync(AGENTS_DIR, { withFileTypes: true }); + for (const agentDir of agentDirs) { + if (!agentDir.isDirectory()) {continue;} + const sessionFile = join(AGENTS_DIR, agentDir.name, "sessions", `${cronSessionId}.jsonl`); + if (!existsSync(sessionFile)) {continue;} + + const content = readFileSync(sessionFile, "utf-8"); + const messages = parseMessagesInRange(content); + if (messages.length > 0) { + return Response.json({ + sessionId: cronSessionId, + messages, + source: "cron-session", + }); + } + } + } catch { + // fall through to search + } + } + + // Strategy 2: Search session files near the run timestamp + if (summary) { + const candidates = findCandidateSessionFiles(runAtMs); + const result = searchForRunTranscript(candidates, summary, runAtMs); + if (result) { + return Response.json({ + messages: result.messages, + source: "main-session-search", + }); + } + } + + return Response.json({ error: "Transcript not found" }, { status: 404 }); +} diff --git a/apps/web/app/api/memories/route.ts b/apps/web/app/api/memories/route.ts new file mode 100644 index 00000000000..d9b0ee2e1f6 --- /dev/null +++ b/apps/web/app/api/memories/route.ts @@ -0,0 +1,66 @@ +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir, resolveWorkspaceRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +type MemoryFile = { + name: string; + path: string; + sizeBytes: number; +}; + +export async function GET() { + const stateDir = resolveOpenClawStateDir(); + const workspaceDir = resolveWorkspaceRoot() ?? join(stateDir, "workspace"); + let mainMemory: string | null = null; + const dailyLogs: MemoryFile[] = []; + + // Read main MEMORY.md + for (const filename of ["MEMORY.md", "memory.md"]) { + const memPath = join(workspaceDir, filename); + if (existsSync(memPath)) { + try { + mainMemory = readFileSync(memPath, "utf-8"); + } catch { + // skip unreadable + } + break; + } + } + + // Scan daily log files + const memoryDir = join(workspaceDir, "memory"); + if (existsSync(memoryDir)) { + try { + const entries = readdirSync(memoryDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".md")) { + continue; + } + const filePath = join(memoryDir, entry.name); + try { + const content = readFileSync(filePath, "utf-8"); + dailyLogs.push({ + name: entry.name, + path: filePath, + sizeBytes: Buffer.byteLength(content, "utf-8"), + }); + } catch { + // skip + } + } + } catch { + // dir unreadable + } + } + + // Sort daily logs by name (date-based filenames sort chronologically) + dailyLogs.sort((a, b) => b.name.localeCompare(a.name)); + + return Response.json({ + mainMemory, + dailyLogs, + workspaceDir, + }); +} diff --git a/apps/web/app/api/profiles/route.test.ts b/apps/web/app/api/profiles/route.test.ts new file mode 100644 index 00000000000..c64108838d6 --- /dev/null +++ b/apps/web/app/api/profiles/route.test.ts @@ -0,0 +1,212 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import type { Dirent } from "node:fs"; + +vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), +})); + +vi.mock("node:child_process", () => ({ + execSync: vi.fn(() => ""), + exec: vi.fn( + ( + _cmd: string, + _opts: unknown, + cb: (err: Error | null, result: { stdout: string }) => void, + ) => { + cb(null, { stdout: "" }); + }, + ), +})); + +vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), +})); + +import { join } from "node:path"; + +function makeDirent(name: string, isDir: boolean): Dirent { + return { + name, + isDirectory: () => isDir, + isFile: () => !isDir, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isFIFO: () => false, + isSocket: () => false, + isSymbolicLink: () => false, + path: "", + parentPath: "", + } as Dirent; +} + +describe("profiles API", () => { + const originalEnv = { ...process.env }; + const STATE_DIR = join("/home/testuser", ".openclaw"); + + beforeEach(() => { + vi.resetModules(); + vi.restoreAllMocks(); + process.env = { ...originalEnv }; + delete process.env.OPENCLAW_PROFILE; + delete process.env.OPENCLAW_HOME; + delete process.env.OPENCLAW_WORKSPACE; + delete process.env.OPENCLAW_STATE_DIR; + + vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + })); + vi.mock("node:child_process", () => ({ + execSync: vi.fn(() => ""), + exec: vi.fn( + ( + _cmd: string, + _opts: unknown, + cb: (err: Error | null, result: { stdout: string }) => void, + ) => { + cb(null, { stdout: "" }); + }, + ), + })); + vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), + })); + }); + + afterEach(() => { + process.env = originalEnv; + }); + + // ─── GET /api/profiles ──────────────────────────────────────────── + + describe("GET /api/profiles", () => { + async function callGet() { + const { GET } = await import("./route.js"); + return GET(); + } + + it("returns profiles list with default profile", async () => { + const response = await callGet(); + expect(response.status).toBe(200); + const json = await response.json(); + expect(json.profiles).toBeDefined(); + expect(json.profiles.length).toBeGreaterThanOrEqual(1); + expect(json.profiles[0].name).toBe("default"); + }); + + it("returns activeProfile", async () => { + const response = await callGet(); + const json = await response.json(); + expect(json.activeProfile).toBe("default"); + }); + + it("returns stateDir", async () => { + const response = await callGet(); + const json = await response.json(); + expect(json.stateDir).toBe(STATE_DIR); + }); + + it("discovers workspace- directories", async () => { + const { existsSync: es, readdirSync: rds } = await import("node:fs"); + vi.mocked(es).mockImplementation((p) => { + const s = String(p); + return ( + s === STATE_DIR || + s === join(STATE_DIR, "workspace-dev") + ); + }); + vi.mocked(rds).mockReturnValue([ + makeDirent("workspace-dev", true), + ] as unknown as Dirent[]); + + const response = await callGet(); + const json = await response.json(); + const names = json.profiles.map((p: { name: string }) => p.name); + expect(names).toContain("dev"); + }); + }); + + // ─── POST /api/profiles/switch ──────────────────────────────────── + + describe("POST /api/profiles/switch", () => { + async function callSwitch(body: Record) { + const { POST } = await import("./switch/route.js"); + const req = new Request("http://localhost/api/profiles/switch", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + return POST(req); + } + + it("switches to named profile", async () => { + const { writeFileSync: wfs } = await import("node:fs"); + const { existsSync: es } = await import("node:fs"); + vi.mocked(es).mockReturnValue(true); + + const response = await callSwitch({ profile: "work" }); + expect(response.status).toBe(200); + const json = await response.json(); + expect(json.activeProfile).toBe("work"); + + const writeCalls = vi.mocked(wfs).mock.calls; + const stateWrite = writeCalls.find((c) => + (c[0] as string).includes(".ironclaw-ui-state.json"), + ); + expect(stateWrite).toBeDefined(); + }); + + it("'default' clears the override", async () => { + const { existsSync: es } = await import("node:fs"); + vi.mocked(es).mockReturnValue(true); + + const response = await callSwitch({ profile: "default" }); + expect(response.status).toBe(200); + const json = await response.json(); + expect(json.activeProfile).toBe("default"); + }); + + it("rejects missing profile name", async () => { + const response = await callSwitch({}); + expect(response.status).toBe(400); + const json = await response.json(); + expect(json.error).toContain("Missing profile name"); + }); + + it("rejects invalid profile name characters", async () => { + const response = await callSwitch({ profile: "bad name!" }); + expect(response.status).toBe(400); + const json = await response.json(); + expect(json.error).toContain("Invalid profile name"); + }); + + it("returns workspace root after switching", async () => { + const { existsSync: es } = await import("node:fs"); + const wsDir = join(STATE_DIR, "workspace-dev"); + vi.mocked(es).mockImplementation((p) => { + const s = String(p); + return s === wsDir || s.includes(".openclaw"); + }); + + const response = await callSwitch({ profile: "dev" }); + const json = await response.json(); + expect(json.workspaceRoot).toBeDefined(); + }); + + it("returns stateDir in response", async () => { + const { existsSync: es } = await import("node:fs"); + vi.mocked(es).mockReturnValue(true); + + const response = await callSwitch({ profile: "test" }); + const json = await response.json(); + expect(json.stateDir).toBe(STATE_DIR); + }); + }); +}); diff --git a/apps/web/app/api/profiles/route.ts b/apps/web/app/api/profiles/route.ts new file mode 100644 index 00000000000..f4a55e6458f --- /dev/null +++ b/apps/web/app/api/profiles/route.ts @@ -0,0 +1,16 @@ +import { discoverProfiles, getEffectiveProfile, resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function GET() { + const profiles = discoverProfiles(); + const activeProfile = getEffectiveProfile(); + const stateDir = resolveOpenClawStateDir(); + + return Response.json({ + profiles, + activeProfile: activeProfile || "default", + stateDir, + }); +} diff --git a/apps/web/app/api/profiles/switch/route.ts b/apps/web/app/api/profiles/switch/route.ts new file mode 100644 index 00000000000..3180cee59aa --- /dev/null +++ b/apps/web/app/api/profiles/switch/route.ts @@ -0,0 +1,34 @@ +import { setUIActiveProfile, getEffectiveProfile, resolveWorkspaceRoot, resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST(req: Request) { + const body = (await req.json()) as { profile?: string }; + const profileName = body.profile?.trim(); + + if (!profileName) { + return Response.json({ error: "Missing profile name" }, { status: 400 }); + } + + // Validate profile name: letters, numbers, hyphens, underscores only + if (profileName !== "default" && !/^[a-zA-Z0-9_-]+$/.test(profileName)) { + return Response.json( + { error: "Invalid profile name. Use letters, numbers, hyphens, or underscores." }, + { status: 400 }, + ); + } + + // "default" clears the override + setUIActiveProfile(profileName === "default" ? null : profileName); + + const activeProfile = getEffectiveProfile(); + const workspaceRoot = resolveWorkspaceRoot(); + const stateDir = resolveOpenClawStateDir(); + + return Response.json({ + activeProfile: activeProfile || "default", + workspaceRoot, + stateDir, + }); +} diff --git a/apps/web/app/api/sessions/[sessionId]/route.ts b/apps/web/app/api/sessions/[sessionId]/route.ts new file mode 100644 index 00000000000..39de17f10b3 --- /dev/null +++ b/apps/web/app/api/sessions/[sessionId]/route.ts @@ -0,0 +1,129 @@ +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +type JSONLMessage = { + type: string; + id: string; + parentId: string | null; + timestamp: string; + message?: { + role: "user" | "assistant"; + content: Array< + | { type: "text"; text: string } + | { type: "image"; data: string } + | { type: "thinking"; thinking: string; thinkingSignature?: string } + >; + timestamp?: number; + }; + customType?: string; + data?: unknown; +}; + +function findSessionFile(sessionId: string): string | null { + const openclawDir = resolveOpenClawStateDir(); + const agentsDir = join(openclawDir, "agents"); + + if (!existsSync(agentsDir)) { + return null; + } + + try { + const agentDirs = readdirSync(agentsDir, { withFileTypes: true }); + for (const agentDir of agentDirs) { + if (!agentDir.isDirectory()) { + continue; + } + + const sessionFile = join( + agentsDir, + agentDir.name, + "sessions", + `${sessionId}.jsonl` + ); + + if (existsSync(sessionFile)) { + return sessionFile; + } + } + } catch { + // ignore errors + } + + return null; +} + +export async function GET( + _request: Request, + { params }: { params: Promise<{ sessionId: string }> } +) { + const { sessionId } = await params; + + if (!sessionId) { + return Response.json({ error: "Session ID required" }, { status: 400 }); + } + + const sessionFile = findSessionFile(sessionId); + + if (!sessionFile) { + return Response.json({ error: "Session not found" }, { status: 404 }); + } + + try { + const content = readFileSync(sessionFile, "utf-8"); + const lines = content + .trim() + .split("\n") + .filter((line) => line.trim()); + + const messages: Array<{ + id: string; + role: "user" | "assistant"; + content: string; + timestamp: string; + }> = []; + + for (const line of lines) { + try { + const entry = JSON.parse(line) as JSONLMessage; + + if (entry.type === "message" && entry.message) { + // Extract text content from the message + const textContent = entry.message.content + .filter((part) => part.type === "text" || part.type === "thinking") + .map((part) => { + if (part.type === "text") { + return part.text; + } + if (part.type === "thinking") { + return `[Thinking: ${part.thinking.slice(0, 100)}...]`; + } + return ""; + }) + .join("\n"); + + if (textContent) { + messages.push({ + id: entry.id, + role: entry.message.role, + content: textContent, + timestamp: entry.timestamp, + }); + } + } + } catch { + // skip malformed lines + } + } + + return Response.json({ sessionId, messages }); + } catch (error) { + console.error("Error reading session:", error); + return Response.json( + { error: "Failed to read session" }, + { status: 500 } + ); + } +} diff --git a/apps/web/app/api/sessions/route.ts b/apps/web/app/api/sessions/route.ts new file mode 100644 index 00000000000..b6d84d2a02c --- /dev/null +++ b/apps/web/app/api/sessions/route.ts @@ -0,0 +1,98 @@ +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +type SessionEntry = { + sessionId: string; + updatedAt: number; + label?: string; + displayName?: string; + channel?: string; + model?: string; + modelProvider?: string; + thinkingLevel?: string; + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + contextTokens?: number; + compactionCount?: number; +}; + +type SessionRow = { + key: string; + sessionId: string; + updatedAt: number; + label?: string; + displayName?: string; + channel?: string; + model?: string; + modelProvider?: string; + thinkingLevel?: string; + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + contextTokens?: number; +}; + +export async function GET() { + const openclawDir = resolveOpenClawStateDir(); + const agentsDir = join(openclawDir, "agents"); + + if (!existsSync(agentsDir)) { + return Response.json({ agents: [], sessions: [] }); + } + + const allSessions: SessionRow[] = []; + const agentIds: string[] = []; + + try { + const entries = readdirSync(agentsDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) { + continue; + } + agentIds.push(entry.name); + + const storePath = join(agentsDir, entry.name, "sessions", "sessions.json"); + if (!existsSync(storePath)) { + continue; + } + + try { + const raw = readFileSync(storePath, "utf-8"); + const store = JSON.parse(raw) as Record; + for (const [key, session] of Object.entries(store)) { + if (!session || typeof session !== "object") { + continue; + } + allSessions.push({ + key, + sessionId: session.sessionId, + updatedAt: session.updatedAt, + label: session.label, + displayName: session.displayName, + channel: session.channel, + model: session.model, + modelProvider: session.modelProvider, + thinkingLevel: session.thinkingLevel, + inputTokens: session.inputTokens, + outputTokens: session.outputTokens, + totalTokens: session.totalTokens, + contextTokens: session.contextTokens, + }); + } + } catch { + // skip unreadable store files + } + } + } catch { + // agents dir unreadable + } + + // Sort by updatedAt descending + allSessions.sort((a, b) => (b.updatedAt ?? 0) - (a.updatedAt ?? 0)); + + return Response.json({ agents: agentIds, sessions: allSessions }); +} diff --git a/apps/web/app/api/sessions/sessions.test.ts b/apps/web/app/api/sessions/sessions.test.ts new file mode 100644 index 00000000000..64ab449639a --- /dev/null +++ b/apps/web/app/api/sessions/sessions.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Mock node:fs +vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + statSync: vi.fn(() => ({ mtimeMs: Date.now() })), +})); + +// Mock node:os +vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), +})); + +describe("Sessions, Memories & Skills API", () => { + beforeEach(() => { + vi.resetModules(); + vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + statSync: vi.fn(() => ({ mtimeMs: Date.now() })), + })); + vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── GET /api/sessions ────────────────────────────────────────── + + describe("GET /api/sessions", () => { + it("returns empty agents and sessions when no dir exists", async () => { + const { GET } = await import("./route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.agents).toEqual([]); + expect(json.sessions).toEqual([]); + }); + + it("returns sessions from agent directories", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile, readdirSync: mockReaddir } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReaddir).mockImplementation((dir) => { + const s = String(dir); + if (s.endsWith("agents")) {return ["main" as never];} + if (s.endsWith("sessions")) {return ["sessions.json" as never];} + return []; + }); + const sessionsData = { + "s1": { label: "Chat 1", displayName: "Chat 1", channel: "webchat", updatedAt: Date.now() }, + }; + vi.mocked(mockReadFile).mockReturnValue(JSON.stringify(sessionsData) as never); + + const { GET } = await import("./route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.sessions.length).toBeGreaterThanOrEqual(0); + }); + }); + + // ─── GET /api/sessions/[sessionId] ────────────────────────────── + + describe("GET /api/sessions/[sessionId]", () => { + it("returns 404 when session not found", async () => { + const { GET } = await import("./[sessionId]/route.js"); + const res = await GET( + new Request("http://localhost"), + { params: Promise.resolve({ sessionId: "nonexistent" }) }, + ); + expect(res.status).toBe(404); + }); + + it("returns 404 for non-existent session ID", async () => { + const { GET } = await import("./[sessionId]/route.js"); + const res = await GET( + new Request("http://localhost"), + { params: Promise.resolve({ sessionId: "missing-id" }) }, + ); + expect(res.status).toBe(404); + }); + }); + + // ─── GET /api/memories ────────────────────────────────────────── + + describe("GET /api/memories", () => { + it("returns null mainMemory when no memory file exists", async () => { + const { existsSync: mockExists } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(false); + + const { GET } = await import("../memories/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.mainMemory).toBeNull(); + }); + + it("returns memory content when file exists", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile, readdirSync: mockReaddir } = await import("node:fs"); + vi.mocked(mockExists).mockImplementation((p) => { + const s = String(p); + if (s.endsWith("MEMORY.md") || s.endsWith("memory.md")) {return true;} + return false; + }); + vi.mocked(mockReadFile).mockReturnValue("# My memories\n\n- Remember X" as never); + vi.mocked(mockReaddir).mockReturnValue([]); + + const { GET } = await import("../memories/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.mainMemory).toContain("memories"); + }); + }); + + // ─── GET /api/skills ──────────────────────────────────────────── + + describe("GET /api/skills", () => { + it("returns empty skills when no skills directories exist", async () => { + const { GET } = await import("../skills/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.skills).toEqual([]); + }); + + it("returns skills from directory", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile, readdirSync: mockReaddir } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReaddir).mockImplementation((dir) => { + const s = String(dir); + if (s.endsWith("skills")) {return ["my-skill" as never];} + return []; + }); + vi.mocked(mockReadFile).mockReturnValue("---\nname: My Skill\n---\n# Skill content" as never); + + const { GET } = await import("../skills/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.skills.length).toBeGreaterThanOrEqual(0); + }); + }); +}); diff --git a/apps/web/app/api/skills/route.ts b/apps/web/app/api/skills/route.ts new file mode 100644 index 00000000000..d773eb81859 --- /dev/null +++ b/apps/web/app/api/skills/route.ts @@ -0,0 +1,85 @@ +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveOpenClawStateDir, resolveWorkspaceRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +type SkillEntry = { + name: string; + description: string; + emoji?: string; + source: string; + filePath: string; +}; + +/** Parse YAML frontmatter from a SKILL.md file (lightweight, no deps). */ +function parseSkillFrontmatter(content: string): { + name?: string; + description?: string; + emoji?: string; +} { + const match = content.match(/^---\s*\n([\s\S]*?)\n---/); + if (!match) return {}; + + const yaml = match[1]; + const result: Record = {}; + for (const line of yaml.split("\n")) { + const kv = line.match(/^(\w+)\s*:\s*(.+)/); + if (kv) { + result[kv[1]] = kv[2].replace(/^["']|["']$/g, "").trim(); + } + } + return { + name: result.name, + description: result.description, + emoji: result.emoji, + }; +} + +function scanSkillDir(dir: string, source: string): SkillEntry[] { + const skills: SkillEntry[] = []; + if (!existsSync(dir)) return skills; + + try { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const skillMdPath = join(dir, entry.name, "SKILL.md"); + if (!existsSync(skillMdPath)) continue; + + try { + const content = readFileSync(skillMdPath, "utf-8"); + const meta = parseSkillFrontmatter(content); + skills.push({ + name: meta.name ?? entry.name, + description: meta.description ?? "", + emoji: meta.emoji, + source, + filePath: skillMdPath, + }); + } catch { + // skip unreadable skill files + } + } + } catch { + // dir unreadable + } + + return skills; +} + +export async function GET() { + const stateDir = resolveOpenClawStateDir(); + const workspaceRoot = resolveWorkspaceRoot() ?? join(stateDir, "workspace"); + + const managedSkills = scanSkillDir(join(stateDir, "skills"), "managed"); + const workspaceSkills = scanSkillDir( + join(workspaceRoot, "skills"), + "workspace", + ); + + const allSkills = [...workspaceSkills, ...managedSkills]; + allSkills.sort((a, b) => a.name.localeCompare(b.name)); + + return Response.json({ skills: allSkills }); +} diff --git a/apps/web/app/api/web-sessions/[id]/messages/route.ts b/apps/web/app/api/web-sessions/[id]/messages/route.ts new file mode 100644 index 00000000000..ba0912c5173 --- /dev/null +++ b/apps/web/app/api/web-sessions/[id]/messages/route.ts @@ -0,0 +1,103 @@ +import { + readFileSync, + writeFileSync, + existsSync, + mkdirSync, +} from "node:fs"; +import { join } from "node:path"; +import { resolveWebChatDir } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; + +type IndexEntry = { + id: string; + title: string; + createdAt: number; + updatedAt: number; + messageCount: number; +}; + +/** + * POST /api/web-sessions/[id]/messages β€” append or upsert messages. + * + * Uses upsert semantics: if a message with the same `id` already exists + * in the session JSONL, it is replaced in-place. Otherwise the message + * is appended. This supports both the client's post-stream save and the + * server-side incremental persistence from the ActiveRunManager. + */ +export async function POST( + request: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const { id } = await params; + const chatDir = resolveWebChatDir(); + const filePath = join(chatDir, `${id}.jsonl`); + const indexPath = join(chatDir, "index.json"); + + // Auto-create the session directory if it doesn't exist yet + if (!existsSync(chatDir)) { + mkdirSync(chatDir, { recursive: true }); + } + if (!existsSync(filePath)) { + writeFileSync(filePath, ""); + } + + const { messages, title } = await request.json(); + + if (!Array.isArray(messages) || messages.length === 0) { + return Response.json({ error: "messages array required" }, { status: 400 }); + } + + // Read existing lines for upsert checks. + const existing = readFileSync(filePath, "utf-8"); + const lines = existing.split("\n").filter((l) => l.trim()); + let newCount = 0; + + for (const msg of messages) { + const msgId = typeof msg.id === "string" ? msg.id : undefined; + let found = false; + + if (msgId) { + for (let i = 0; i < lines.length; i++) { + try { + const parsed = JSON.parse(lines[i]); + if (parsed.id === msgId) { + // Replace the existing line in-place. + lines[i] = JSON.stringify(msg); + found = true; + break; + } + } catch { + /* keep malformed lines as-is */ + } + } + } + + if (!found) { + lines.push(JSON.stringify(msg)); + newCount++; + } + } + + writeFileSync(filePath, lines.join("\n") + "\n"); + + // Update index metadata + try { + if (existsSync(indexPath)) { + const index: IndexEntry[] = JSON.parse( + readFileSync(indexPath, "utf-8"), + ); + const session = index.find((s) => s.id === id); + if (session) { + session.updatedAt = Date.now(); + if (newCount > 0) {session.messageCount += newCount;} + if (title) {session.title = title;} + writeFileSync(indexPath, JSON.stringify(index, null, 2)); + } + } + } catch { + // index update is best-effort + } + + return Response.json({ ok: true }); +} diff --git a/apps/web/app/api/web-sessions/[id]/route.ts b/apps/web/app/api/web-sessions/[id]/route.ts new file mode 100644 index 00000000000..83f98a72bb4 --- /dev/null +++ b/apps/web/app/api/web-sessions/[id]/route.ts @@ -0,0 +1,94 @@ +import { readFileSync, existsSync, unlinkSync } from "node:fs"; +import { join } from "node:path"; +import { resolveWebChatDir } from "@/lib/workspace"; +import { readIndex, writeIndex } from "../shared"; + +export const dynamic = "force-dynamic"; + +export type ChatLine = { + id: string; + role: "user" | "assistant"; + /** Plain text summary (always present, used for sidebar / backward compat). */ + content: string; + /** Full UIMessage parts array β€” reasoning, tool calls, outputs, text. + * Present for sessions saved after the rich-persistence update; + * absent for older sessions (fall back to `content` as a text part). */ + parts?: Array>; + timestamp: string; +}; + +/** GET /api/web-sessions/[id] β€” read all messages for a web chat session */ +export async function GET( + _request: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const { id } = await params; + const filePath = join(resolveWebChatDir(), `${id}.jsonl`); + + if (!existsSync(filePath)) { + return Response.json({ error: "Session not found" }, { status: 404 }); + } + + const content = readFileSync(filePath, "utf-8"); + const messages: ChatLine[] = content + .trim() + .split("\n") + .filter((line) => line.trim()) + .map((line) => { + try { + return JSON.parse(line) as ChatLine; + } catch { + return null; + } + }) + .filter((m): m is ChatLine => m !== null); + + return Response.json({ id, messages }); +} + +/** DELETE /api/web-sessions/[id] β€” delete a web chat session */ +export async function DELETE( + _request: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const { id } = await params; + + const sessions = readIndex(); + const idx = sessions.findIndex((s) => s.id === id); + if (idx === -1) { + return Response.json({ error: "Session not found" }, { status: 404 }); + } + + sessions.splice(idx, 1); + writeIndex(sessions); + + const filePath = join(resolveWebChatDir(), `${id}.jsonl`); + if (existsSync(filePath)) { + unlinkSync(filePath); + } + + return Response.json({ ok: true }); +} + +/** PATCH /api/web-sessions/[id] β€” update session metadata (e.g. rename) */ +export async function PATCH( + request: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const { id } = await params; + const body = await request.json().catch(() => ({})); + + const sessions = readIndex(); + const session = sessions.find((s) => s.id === id); + if (!session) { + return Response.json({ error: "Session not found" }, { status: 404 }); + } + + if (typeof body.title === "string") { + session.title = body.title; + } + session.updatedAt = Date.now(); + writeIndex(sessions); + + return Response.json({ session }); +} diff --git a/apps/web/app/api/web-sessions/route.ts b/apps/web/app/api/web-sessions/route.ts new file mode 100644 index 00000000000..858de33ae8a --- /dev/null +++ b/apps/web/app/api/web-sessions/route.ts @@ -0,0 +1,45 @@ +import { writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { type WebSessionMeta, ensureDir, readIndex, writeIndex } from "./shared"; + +export { type WebSessionMeta }; + +export const dynamic = "force-dynamic"; + +/** GET /api/web-sessions β€” list web chat sessions. + * ?filePath=... β†’ returns only sessions scoped to that file. + * No filePath β†’ returns only global (non-file) sessions. */ +export async function GET(req: Request) { + const url = new URL(req.url); + const filePath = url.searchParams.get("filePath"); + + const all = readIndex(); + const sessions = filePath + ? all.filter((s) => s.filePath === filePath) + : all.filter((s) => !s.filePath); + + return Response.json({ sessions }); +} + +/** POST /api/web-sessions β€” create a new web chat session */ +export async function POST(req: Request) { + const body = await req.json().catch(() => ({})); + const id = randomUUID(); + const session: WebSessionMeta = { + id, + title: body.title || "New Chat", + createdAt: Date.now(), + updatedAt: Date.now(), + messageCount: 0, + ...(body.filePath ? { filePath: body.filePath } : {}), + }; + + const sessions = readIndex(); + sessions.unshift(session); + writeIndex(sessions); + + const dir = ensureDir(); + writeFileSync(`${dir}/${id}.jsonl`, ""); + + return Response.json({ session }); +} diff --git a/apps/web/app/api/web-sessions/shared.ts b/apps/web/app/api/web-sessions/shared.ts new file mode 100644 index 00000000000..03073ed9c50 --- /dev/null +++ b/apps/web/app/api/web-sessions/shared.ts @@ -0,0 +1,88 @@ +import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { resolveWebChatDir } from "@/lib/workspace"; + +export type WebSessionMeta = { + id: string; + title: string; + createdAt: number; + updatedAt: number; + messageCount: number; + /** When set, this session is scoped to a specific workspace file. */ + filePath?: string; +}; + +export function ensureDir() { + const dir = resolveWebChatDir(); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + return dir; +} + +/** + * Read the session index, auto-discovering any orphaned .jsonl files + * that aren't in the index (e.g. from profile switches or missing index). + */ +export function readIndex(): WebSessionMeta[] { + const dir = ensureDir(); + const indexFile = join(dir, "index.json"); + let index: WebSessionMeta[] = []; + if (existsSync(indexFile)) { + try { + index = JSON.parse(readFileSync(indexFile, "utf-8")); + } catch { + index = []; + } + } + + // Scan for orphaned .jsonl files not in the index + try { + const indexed = new Set(index.map((s) => s.id)); + const files = readdirSync(dir).filter((f) => f.endsWith(".jsonl")); + let dirty = false; + for (const file of files) { + const id = file.replace(/\.jsonl$/, ""); + if (indexed.has(id)) {continue;} + + const fp = join(dir, file); + const stat = statSync(fp); + let title = "New Chat"; + let messageCount = 0; + try { + const content = readFileSync(fp, "utf-8"); + const lines = content.split("\n").filter((l) => l.trim()); + messageCount = lines.length; + for (const line of lines) { + const parsed = JSON.parse(line); + if (parsed.role === "user" && parsed.content) { + const text = String(parsed.content); + title = text.length > 60 ? text.slice(0, 60) + "..." : text; + break; + } + } + } catch { /* best-effort */ } + + index.push({ + id, + title, + createdAt: stat.birthtimeMs || stat.mtimeMs, + updatedAt: stat.mtimeMs, + messageCount, + }); + dirty = true; + } + + if (dirty) { + index.sort((a, b) => b.updatedAt - a.updatedAt); + writeFileSync(indexFile, JSON.stringify(index, null, 2)); + } + } catch { /* best-effort */ } + + return index; +} + +export function writeIndex(sessions: WebSessionMeta[]) { + const dir = ensureDir(); + writeFileSync(join(dir, "index.json"), JSON.stringify(sessions, null, 2)); +} diff --git a/apps/web/app/api/web-sessions/web-sessions.test.ts b/apps/web/app/api/web-sessions/web-sessions.test.ts new file mode 100644 index 00000000000..1f5bec18ea3 --- /dev/null +++ b/apps/web/app/api/web-sessions/web-sessions.test.ts @@ -0,0 +1,304 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Mock node:fs +vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => "[]"), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + appendFileSync: vi.fn(), +})); + +// Mock node:os +vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), +})); + +// Mock node:crypto +vi.mock("node:crypto", () => ({ + randomUUID: vi.fn(() => "test-uuid-1234"), +})); + +describe("Web Sessions API", () => { + beforeEach(() => { + vi.resetModules(); + vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => "[]"), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + appendFileSync: vi.fn(), + })); + vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), + })); + vi.mock("node:crypto", () => ({ + randomUUID: vi.fn(() => "test-uuid-1234"), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── GET /api/web-sessions ────────────────────────────────────── + + describe("GET /api/web-sessions", () => { + it("returns empty sessions when no index exists", async () => { + const { GET } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions"); + const res = await GET(req); + const json = await res.json(); + expect(json.sessions).toEqual([]); + }); + + it("returns global sessions when no filePath param", async () => { + const { readFileSync: mockReadFile, existsSync: mockExists } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + const sessions = [ + { id: "s1", title: "Chat 1", createdAt: 1, updatedAt: 1, messageCount: 0 }, + { id: "s2", title: "File Chat", createdAt: 2, updatedAt: 2, messageCount: 1, filePath: "doc.md" }, + ]; + vi.mocked(mockReadFile).mockReturnValue(JSON.stringify(sessions) as never); + + const { GET } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions"); + const res = await GET(req); + const json = await res.json(); + expect(json.sessions).toHaveLength(1); + expect(json.sessions[0].id).toBe("s1"); + }); + + it("filters sessions by filePath param", async () => { + const { readFileSync: mockReadFile, existsSync: mockExists } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + const sessions = [ + { id: "s1", title: "Global", createdAt: 1, updatedAt: 1, messageCount: 0 }, + { id: "s2", title: "Doc Chat", createdAt: 2, updatedAt: 2, messageCount: 1, filePath: "doc.md" }, + ]; + vi.mocked(mockReadFile).mockReturnValue(JSON.stringify(sessions) as never); + + const { GET } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions?filePath=doc.md"); + const res = await GET(req); + const json = await res.json(); + expect(json.sessions).toHaveLength(1); + expect(json.sessions[0].filePath).toBe("doc.md"); + }); + + it("returns empty when no matching filePath sessions", async () => { + const { readFileSync: mockReadFile, existsSync: mockExists } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReadFile).mockReturnValue("[]" as never); + + const { GET } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions?filePath=nonexistent.md"); + const res = await GET(req); + const json = await res.json(); + expect(json.sessions).toEqual([]); + }); + }); + + // ─── POST /api/web-sessions ──────────────────────────────────── + + describe("POST /api/web-sessions", () => { + it("creates a new session with default title", async () => { + const { writeFileSync: mockWrite } = await import("node:fs"); + + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req); + const json = await res.json(); + expect(json.session.id).toBe("test-uuid-1234"); + expect(json.session.title).toBe("New Chat"); + expect(json.session.messageCount).toBe(0); + expect(mockWrite).toHaveBeenCalled(); + }); + + it("creates session with custom title", async () => { + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ title: "My Chat" }), + }); + const res = await POST(req); + const json = await res.json(); + expect(json.session.title).toBe("My Chat"); + }); + + it("creates file-scoped session with filePath", async () => { + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ title: "File Chat", filePath: "readme.md" }), + }); + const res = await POST(req); + const json = await res.json(); + expect(json.session.filePath).toBe("readme.md"); + }); + + it("handles invalid JSON body gracefully", async () => { + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "not json", + }); + const res = await POST(req); + const json = await res.json(); + // Falls back to default title + expect(json.session.title).toBe("New Chat"); + }); + + it("creates jsonl file for new session", async () => { + const { writeFileSync: mockWrite } = await import("node:fs"); + + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/web-sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + await POST(req); + // Should write at least the index.json and the empty .jsonl + expect(mockWrite).toHaveBeenCalled(); + // Verify that one of the calls is to the jsonl file + const calls = mockWrite.mock.calls.map((c) => String(c[0])); + expect(calls.some((c) => c.endsWith(".jsonl"))).toBe(true); + }); + }); + + // ─── GET /api/web-sessions/[id] ──────────────────────────────── + + describe("GET /api/web-sessions/[id]", () => { + it("returns session messages", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + const lines = [ + JSON.stringify({ id: "m1", role: "user", content: "hello" }), + JSON.stringify({ id: "m2", role: "assistant", content: "hi" }), + ].join("\n"); + vi.mocked(mockReadFile).mockReturnValue(lines as never); + + const { GET } = await import("./[id]/route.js"); + const res = await GET( + new Request("http://localhost/api/web-sessions/s1"), + { params: Promise.resolve({ id: "s1" }) }, + ); + const json = await res.json(); + expect(json.id).toBe("s1"); + expect(json.messages).toHaveLength(2); + }); + + it("returns 404 when session file does not exist", async () => { + const { existsSync: mockExists } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(false); + + const { GET } = await import("./[id]/route.js"); + const res = await GET( + new Request("http://localhost/api/web-sessions/nonexistent"), + { params: Promise.resolve({ id: "nonexistent" }) }, + ); + expect(res.status).toBe(404); + }); + + it("handles empty session file", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReadFile).mockReturnValue("" as never); + + const { GET } = await import("./[id]/route.js"); + const res = await GET( + new Request("http://localhost/api/web-sessions/s1"), + { params: Promise.resolve({ id: "s1" }) }, + ); + const json = await res.json(); + expect(json.messages).toEqual([]); + }); + }); + + // ─── POST /api/web-sessions/[id]/messages ────────────────────── + + describe("POST /api/web-sessions/[id]/messages", () => { + it("appends messages to session file", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile, writeFileSync: _mockWrite } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReadFile).mockImplementation((p) => { + const s = String(p); + if (s.endsWith("index.json")) { + return JSON.stringify([{ id: "s1", title: "Chat", createdAt: 1, updatedAt: 1, messageCount: 0 }]) as never; + } + return "" as never; + }); + + const { POST } = await import("./[id]/messages/route.js"); + const req = new Request("http://localhost/api/web-sessions/s1/messages", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ id: "m1", role: "user", content: "hello" }], + }), + }); + const res = await POST(req, { params: Promise.resolve({ id: "s1" }) }); + const json = await res.json(); + expect(json.ok).toBe(true); + }); + + it("auto-creates session file if missing", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile, writeFileSync: _mockWrite } = await import("node:fs"); + vi.mocked(mockExists).mockImplementation((p) => { + const s = String(p); + if (s.endsWith(".jsonl")) {return false;} + return true; + }); + vi.mocked(mockReadFile).mockImplementation((p) => { + const s = String(p); + if (s.endsWith("index.json")) {return "[]" as never;} + return "" as never; + }); + + const { POST } = await import("./[id]/messages/route.js"); + const req = new Request("http://localhost/api/web-sessions/new-s/messages", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ id: "m1", role: "user", content: "first message" }], + }), + }); + const res = await POST(req, { params: Promise.resolve({ id: "new-s" }) }); + expect(res.status).toBe(200); + }); + + it("updates session title when provided", async () => { + const { existsSync: mockExists, readFileSync: mockReadFile, writeFileSync: mockWrite } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReadFile).mockImplementation((p) => { + const s = String(p); + if (s.endsWith("index.json")) { + return JSON.stringify([{ id: "s1", title: "Old Title", createdAt: 1, updatedAt: 1, messageCount: 0 }]) as never; + } + return "" as never; + }); + + const { POST } = await import("./[id]/messages/route.js"); + const req = new Request("http://localhost/api/web-sessions/s1/messages", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ id: "m1", role: "user", content: "hello" }], + title: "New Title", + }), + }); + const res = await POST(req, { params: Promise.resolve({ id: "s1" }) }); + expect(res.status).toBe(200); + // Verify index was written with new title + expect(mockWrite).toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/web/app/api/workspace/assets/[...path]/route.ts b/apps/web/app/api/workspace/assets/[...path]/route.ts new file mode 100644 index 00000000000..1c7869b0ef5 --- /dev/null +++ b/apps/web/app/api/workspace/assets/[...path]/route.ts @@ -0,0 +1,57 @@ +import { readFileSync, existsSync } from "node:fs"; +import { extname } from "node:path"; +import { safeResolvePath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const MIME_MAP: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".svg": "image/svg+xml", + ".bmp": "image/bmp", + ".ico": "image/x-icon", +}; + +/** + * GET /api/workspace/assets/ + * Serves an image file from the workspace's assets/ directory. + */ +export async function GET( + _req: Request, + { params }: { params: Promise<{ path: string[] }> }, +) { + const segments = (await params).path; + if (!segments || segments.length === 0) { + return new Response("Not found", { status: 404 }); + } + + const relPath = "assets/" + segments.join("/"); + const ext = extname(relPath).toLowerCase(); + + // Only serve known image types + const mime = MIME_MAP[ext]; + if (!mime) { + return new Response("Unsupported file type", { status: 400 }); + } + + const absPath = safeResolvePath(relPath); + if (!absPath || !existsSync(absPath)) { + return new Response("Not found", { status: 404 }); + } + + try { + const buffer = readFileSync(absPath); + return new Response(buffer, { + headers: { + "Content-Type": mime, + "Cache-Control": "public, max-age=31536000, immutable", + }, + }); + } catch { + return new Response("Read error", { status: 500 }); + } +} diff --git a/apps/web/app/api/workspace/browse-file/route.ts b/apps/web/app/api/workspace/browse-file/route.ts new file mode 100644 index 00000000000..eeed22d9857 --- /dev/null +++ b/apps/web/app/api/workspace/browse-file/route.ts @@ -0,0 +1,109 @@ +import { readFileSync, existsSync, statSync } from "node:fs"; +import { resolve, normalize } from "node:path"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** MIME types for common file extensions. */ +const MIME_MAP: Record = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + svg: "image/svg+xml", + mp4: "video/mp4", + webm: "video/webm", + mp3: "audio/mpeg", + wav: "audio/wav", + ogg: "audio/ogg", + pdf: "application/pdf", + html: "text/html", + htm: "text/html", +}; + +/** Extensions recognized as code files for syntax-highlighted viewing. */ +const CODE_EXTENSIONS = new Set([ + "ts", "tsx", "js", "jsx", "mjs", "cjs", "py", "rb", "go", "rs", + "java", "kt", "swift", "c", "cpp", "h", "hpp", "cs", "css", "scss", + "less", "html", "htm", "xml", "json", "jsonc", "toml", "sh", "bash", + "zsh", "fish", "ps1", "sql", "graphql", "gql", "dockerfile", "makefile", + "r", "lua", "php", "vue", "svelte", "diff", "patch", "ini", "env", + "tf", "proto", "zig", "elixir", "ex", "erl", "hs", "scala", "clj", "dart", +]); + +export async function GET(req: Request) { + const url = new URL(req.url); + const filePath = url.searchParams.get("path"); + const raw = url.searchParams.get("raw") === "true"; + + if (!filePath) { + return Response.json( + { error: "Missing 'path' query parameter" }, + { status: 400 }, + ); + } + + // Normalize and resolve to prevent traversal + const resolved = resolve(normalize(filePath)); + + if (!existsSync(resolved)) { + return Response.json( + { error: "File not found" }, + { status: 404 }, + ); + } + + try { + const stat = statSync(resolved); + if (!stat.isFile()) { + return Response.json( + { error: "Path is not a file" }, + { status: 400 }, + ); + } + } catch { + return Response.json( + { error: "Cannot stat file" }, + { status: 500 }, + ); + } + + // Raw mode: return binary content with appropriate MIME type + if (raw) { + try { + const buffer = readFileSync(resolved); + const ext = resolved.split(".").pop()?.toLowerCase() ?? ""; + const mime = MIME_MAP[ext] ?? "application/octet-stream"; + return new Response(buffer, { + headers: { + "Content-Type": mime, + "Content-Length": String(buffer.length), + }, + }); + } catch { + return Response.json( + { error: "Cannot read file" }, + { status: 500 }, + ); + } + } + + // Text mode: return content and type metadata (same shape as /api/workspace/file) + try { + const content = readFileSync(resolved, "utf-8"); + const ext = resolved.split(".").pop()?.toLowerCase(); + + let type: "markdown" | "yaml" | "code" | "text" = "text"; + if (ext === "md" || ext === "mdx") {type = "markdown";} + else if (ext === "yaml" || ext === "yml") {type = "yaml";} + else if (CODE_EXTENSIONS.has(ext ?? "")) {type = "code";} + + return Response.json({ content, type }); + } catch { + return Response.json( + { error: "Cannot read file" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/browse/route.ts b/apps/web/app/api/workspace/browse/route.ts new file mode 100644 index 00000000000..bba02cbddd7 --- /dev/null +++ b/apps/web/app/api/workspace/browse/route.ts @@ -0,0 +1,134 @@ +import { readdirSync, statSync, type Dirent } from "node:fs"; +import { join, dirname, resolve } from "node:path"; +import { homedir } from "node:os"; +import { resolveWorkspaceRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type BrowseNode = { + name: string; + path: string; // absolute path + type: "folder" | "file" | "document" | "database"; + children?: BrowseNode[]; + symlink?: boolean; +}; + +/** Directories to skip when browsing the filesystem. */ +const SKIP_DIRS = new Set(["node_modules", ".git", ".Trash", "__pycache__", ".cache"]); + +/** Resolve a dirent's effective type, following symlinks to their target. */ +function resolveEntryType(entry: Dirent, absPath: string): "directory" | "file" | null { + if (entry.isDirectory()) {return "directory";} + if (entry.isFile()) {return "file";} + if (entry.isSymbolicLink()) { + try { + const st = statSync(absPath); + if (st.isDirectory()) {return "directory";} + if (st.isFile()) {return "file";} + } catch { + // Broken symlink + } + } + return null; +} + +/** Build a depth-limited tree from an absolute directory. */ +function buildBrowseTree( + absDir: string, + maxDepth: number, + currentDepth = 0, + showHidden = false, +): BrowseNode[] { + if (currentDepth >= maxDepth) {return [];} + + let entries: Dirent[]; + try { + entries = readdirSync(absDir, { withFileTypes: true }); + } catch { + return []; + } + + const filtered = entries + .filter((e) => showHidden || !e.name.startsWith(".")) + .filter((e) => { + const absPath = join(absDir, e.name); + const t = resolveEntryType(e, absPath); + return !(t === "directory" && SKIP_DIRS.has(e.name)); + }); + + const sorted = filtered.toSorted((a, b) => { + const absA = join(absDir, a.name); + const absB = join(absDir, b.name); + const typeA = resolveEntryType(a, absA); + const typeB = resolveEntryType(b, absB); + const dirA = typeA === "directory"; + const dirB = typeB === "directory"; + if (dirA && !dirB) {return -1;} + if (!dirA && dirB) {return 1;} + return a.name.localeCompare(b.name); + }); + + const nodes: BrowseNode[] = []; + + for (const entry of sorted) { + const absPath = join(absDir, entry.name); + const isSymlink = entry.isSymbolicLink(); + const effectiveType = resolveEntryType(entry, absPath); + + if (effectiveType === "directory") { + const children = buildBrowseTree(absPath, maxDepth, currentDepth + 1, showHidden); + nodes.push({ + name: entry.name, + path: absPath, + type: "folder", + children: children.length > 0 ? children : undefined, + ...(isSymlink && { symlink: true }), + }); + } else if (effectiveType === "file") { + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = ext === "duckdb" || ext === "sqlite" || ext === "sqlite3" || ext === "db"; + + nodes.push({ + name: entry.name, + path: absPath, + type: isDatabase ? "database" : isDocument ? "document" : "file", + ...(isSymlink && { symlink: true }), + }); + } + } + + return nodes; +} + +export async function GET(req: Request) { + const url = new URL(req.url); + let dir = url.searchParams.get("dir"); + const showHidden = url.searchParams.get("showHidden") === "1"; + + if (!dir) { + dir = resolveWorkspaceRoot(); + } + + if (!dir) { + return Response.json( + { entries: [], currentDir: "/", parentDir: null }, + ); + } + + if (dir.startsWith("~")) { + dir = join(homedir(), dir.slice(1)); + } + + const resolved = resolve(dir); + + const entries = buildBrowseTree(resolved, 3, 0, showHidden); + const parentDir = resolved === "/" ? null : dirname(resolved); + + return Response.json({ + entries, + currentDir: resolved, + parentDir, + }); +} diff --git a/apps/web/app/api/workspace/context/route.ts b/apps/web/app/api/workspace/context/route.ts new file mode 100644 index 00000000000..c9af98b8f19 --- /dev/null +++ b/apps/web/app/api/workspace/context/route.ts @@ -0,0 +1,116 @@ +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveWorkspaceRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export type WorkspaceContext = { + exists: boolean; + organization?: { + id?: string; + name?: string; + slug?: string; + }; + members?: Array<{ + id: string; + name: string; + email: string; + role: string; + }>; + defaults?: { + default_view?: string; + date_format?: string; + naming_convention?: string; + }; +}; + +/** + * Parse workspace_context.yaml with basic YAML extraction. + * Handles the specific structure defined by the workspace skill. + */ +function parseWorkspaceContext(content: string): WorkspaceContext { + const ctx: WorkspaceContext = { exists: true }; + + // Extract organization block + const orgMatch = content.match( + /organization:\s*\n((?:\s{2,}.+\n)*)/, + ); + if (orgMatch) { + const orgBlock = orgMatch[1]; + const org: Record = {}; + for (const line of orgBlock.split("\n")) { + const kv = line.match(/^\s+(\w+)\s*:\s*"?([^"\n]+)"?/); + if (kv) {org[kv[1]] = kv[2].trim();} + } + ctx.organization = { + id: org.id, + name: org.name, + slug: org.slug, + }; + } + + // Extract members list + const membersMatch = content.match( + /members:\s*\n((?:\s{2,}.+\n)*)/, + ); + if (membersMatch) { + const membersBlock = membersMatch[1]; + const members: WorkspaceContext["members"] = []; + let current: Record = {}; + + for (const line of membersBlock.split("\n")) { + const itemStart = line.match(/^\s+-\s+(\w+)\s*:\s*"?([^"\n]+)"?/); + const propLine = line.match(/^\s+(\w+)\s*:\s*"?([^"\n]+)"?/); + + if (itemStart) { + if (current.id) {members.push(current as never);} + current = { [itemStart[1]]: itemStart[2].trim() }; + } else if (propLine && !line.trim().startsWith("-")) { + current[propLine[1]] = propLine[2].trim(); + } + } + if (current.id) {members.push(current as never);} + ctx.members = members; + } + + // Extract defaults block + const defaultsMatch = content.match( + /defaults:\s*\n((?:\s{2,}.+\n)*)/, + ); + if (defaultsMatch) { + const defaultsBlock = defaultsMatch[1]; + const defaults: Record = {}; + for (const line of defaultsBlock.split("\n")) { + const kv = line.match(/^\s+(\w[\w_]*)\s*:\s*(.+)/); + if (kv) {defaults[kv[1]] = kv[2].trim();} + } + ctx.defaults = { + default_view: defaults.default_view, + date_format: defaults.date_format, + naming_convention: defaults.naming_convention, + }; + } + + return ctx; +} + +export async function GET() { + const root = resolveWorkspaceRoot(); + if (!root) { + return Response.json({ exists: false } satisfies WorkspaceContext); + } + + const ctxPath = join(root, "workspace_context.yaml"); + if (!existsSync(ctxPath)) { + return Response.json({ exists: true } satisfies WorkspaceContext); + } + + try { + const content = readFileSync(ctxPath, "utf-8"); + const parsed = parseWorkspaceContext(content); + return Response.json(parsed); + } catch { + return Response.json({ exists: true } satisfies WorkspaceContext); + } +} diff --git a/apps/web/app/api/workspace/copy/route.ts b/apps/web/app/api/workspace/copy/route.ts new file mode 100644 index 00000000000..6431f3eafe4 --- /dev/null +++ b/apps/web/app/api/workspace/copy/route.ts @@ -0,0 +1,77 @@ +import { cpSync, existsSync, statSync } from "node:fs"; +import { dirname, basename, extname } from "node:path"; +import { safeResolvePath, safeResolveNewPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/copy + * Body: { path: string, destinationPath?: string } + * + * Duplicates a file or folder. If no destinationPath is provided, + * creates a copy next to the original with " copy" appended. + */ +export async function POST(req: Request) { + let body: { path?: string; destinationPath?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, destinationPath } = body; + if (!relPath || typeof relPath !== "string") { + return Response.json( + { error: "Missing 'path' field" }, + { status: 400 }, + ); + } + + const srcAbs = safeResolvePath(relPath); + if (!srcAbs) { + return Response.json( + { error: "Source not found or path traversal rejected" }, + { status: 404 }, + ); + } + + let destRelPath: string; + if (destinationPath && typeof destinationPath === "string") { + destRelPath = destinationPath; + } else { + // Auto-generate "name copy.ext" or "name copy" for folders + const name = basename(relPath); + const dir = dirname(relPath); + const ext = extname(name); + const stem = ext ? name.slice(0, -ext.length) : name; + const copyName = ext ? `${stem} copy${ext}` : `${stem} copy`; + destRelPath = dir === "." ? copyName : `${dir}/${copyName}`; + } + + const destAbs = safeResolveNewPath(destRelPath); + if (!destAbs) { + return Response.json( + { error: "Invalid destination path" }, + { status: 400 }, + ); + } + + if (existsSync(destAbs)) { + return Response.json( + { error: "Destination already exists" }, + { status: 409 }, + ); + } + + try { + const isDir = statSync(srcAbs).isDirectory(); + cpSync(srcAbs, destAbs, { recursive: isDir }); + return Response.json({ ok: true, sourcePath: relPath, newPath: destRelPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Copy failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/db.test.ts b/apps/web/app/api/workspace/db.test.ts new file mode 100644 index 00000000000..fe2a17cc2bf --- /dev/null +++ b/apps/web/app/api/workspace/db.test.ts @@ -0,0 +1,273 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Mock workspace (include ALL exports used by the routes) +vi.mock("@/lib/workspace", () => ({ + safeResolvePath: vi.fn(() => null), + resolveWorkspaceRoot: vi.fn(() => null), + resolveDuckdbBin: vi.fn(() => null), + duckdbPath: vi.fn(() => null), + duckdbQuery: vi.fn(() => []), + duckdbQueryAsync: vi.fn(async () => []), + duckdbQueryOnFile: vi.fn(() => []), + duckdbQueryOnFileAsync: vi.fn(async () => []), + duckdbExecOnFile: vi.fn(() => true), + discoverDuckDBPaths: vi.fn(() => []), + isDatabaseFile: vi.fn(() => false), +})); + +// Mock report-filters +vi.mock("@/lib/report-filters", () => ({ + buildFilterClauses: vi.fn(() => []), + injectFilters: vi.fn((sql: string) => sql), + checkSqlSafety: vi.fn(() => null), +})); + +describe("Workspace DB & Reports API", () => { + beforeEach(() => { + vi.resetModules(); + vi.mock("@/lib/workspace", () => ({ + safeResolvePath: vi.fn(() => null), + resolveWorkspaceRoot: vi.fn(() => null), + resolveDuckdbBin: vi.fn(() => null), + duckdbPath: vi.fn(() => null), + duckdbQuery: vi.fn(() => []), + duckdbQueryAsync: vi.fn(async () => []), + duckdbQueryOnFile: vi.fn(() => []), + duckdbQueryOnFileAsync: vi.fn(async () => []), + duckdbExecOnFile: vi.fn(() => true), + discoverDuckDBPaths: vi.fn(() => []), + isDatabaseFile: vi.fn(() => false), + })); + vi.mock("@/lib/report-filters", () => ({ + buildFilterClauses: vi.fn(() => []), + injectFilters: vi.fn((sql: string) => sql), + checkSqlSafety: vi.fn(() => null), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── POST /api/workspace/db/query ─────────────────────────────── + + describe("POST /api/workspace/db/query", () => { + it("returns 400 for missing sql", async () => { + const { POST } = await import("./db/query/route.js"); + const req = new Request("http://localhost/api/workspace/db/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "test.duckdb" }), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("returns 400 for missing path", async () => { + const { POST } = await import("./db/query/route.js"); + const req = new Request("http://localhost/api/workspace/db/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: "SELECT 1" }), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("rejects mutation queries with 403", async () => { + const { safeResolvePath } = await import("@/lib/workspace"); + vi.mocked(safeResolvePath).mockReturnValue("/ws/test.duckdb"); + + const { POST } = await import("./db/query/route.js"); + const req = new Request("http://localhost/api/workspace/db/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "test.duckdb", sql: "DROP TABLE users" }), + }); + const res = await POST(req); + expect(res.status).toBe(403); + }); + + it("executes query and returns rows", async () => { + const { safeResolvePath, duckdbQueryOnFile } = await import("@/lib/workspace"); + vi.mocked(safeResolvePath).mockReturnValue("/ws/test.duckdb"); + vi.mocked(duckdbQueryOnFile).mockReturnValue([{ id: 1, name: "test" }]); + + const { POST } = await import("./db/query/route.js"); + const req = new Request("http://localhost/api/workspace/db/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "test.duckdb", sql: "SELECT * FROM t" }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.rows).toEqual([{ id: 1, name: "test" }]); + }); + + it("returns empty rows for empty result", async () => { + const { safeResolvePath, duckdbQueryOnFile } = await import("@/lib/workspace"); + vi.mocked(safeResolvePath).mockReturnValue("/ws/test.duckdb"); + vi.mocked(duckdbQueryOnFile).mockReturnValue([]); + + const { POST } = await import("./db/query/route.js"); + const req = new Request("http://localhost/api/workspace/db/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "test.duckdb", sql: "SELECT * FROM empty" }), + }); + const res = await POST(req); + const json = await res.json(); + expect(json.rows).toEqual([]); + }); + }); + + // ─── GET /api/workspace/db/introspect ─────────────────────────── + + describe("GET /api/workspace/db/introspect", () => { + it("returns 400 for missing path", async () => { + const { GET } = await import("./db/introspect/route.js"); + const req = new Request("http://localhost/api/workspace/db/introspect"); + const res = await GET(req); + expect(res.status).toBe(400); + }); + + it("returns 404 when file not found", async () => { + const { safeResolvePath } = await import("@/lib/workspace"); + vi.mocked(safeResolvePath).mockReturnValue(null); + + const { GET } = await import("./db/introspect/route.js"); + const req = new Request("http://localhost/api/workspace/db/introspect?path=missing.duckdb"); + const res = await GET(req); + expect(res.status).toBe(404); + }); + + it("returns schema when database exists", async () => { + const { safeResolvePath, resolveDuckdbBin, duckdbQueryOnFile } = await import("@/lib/workspace"); + vi.mocked(safeResolvePath).mockReturnValue("/ws/test.duckdb"); + vi.mocked(resolveDuckdbBin).mockReturnValue("/opt/homebrew/bin/duckdb"); + vi.mocked(duckdbQueryOnFile).mockReturnValue([ + { table_name: "users", column_name: "id", data_type: "INTEGER", is_nullable: "NO" }, + ]); + + const { GET } = await import("./db/introspect/route.js"); + const req = new Request("http://localhost/api/workspace/db/introspect?path=test.duckdb"); + const res = await GET(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.tables).toBeDefined(); + }); + }); + + // ─── POST /api/workspace/reports/execute ──────────────────────── + + describe("POST /api/workspace/reports/execute", () => { + it("returns 400 for missing sql", async () => { + const { POST } = await import("./reports/execute/route.js"); + const req = new Request("http://localhost/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("rejects mutation SQL with 403", async () => { + const { checkSqlSafety } = await import("@/lib/report-filters"); + vi.mocked(checkSqlSafety).mockReturnValue("Only SELECT queries allowed"); + + const { POST } = await import("./reports/execute/route.js"); + const req = new Request("http://localhost/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: "DROP TABLE users" }), + }); + const res = await POST(req); + expect(res.status).toBe(403); + }); + + it("executes report query successfully", async () => { + const { checkSqlSafety } = await import("@/lib/report-filters"); + vi.mocked(checkSqlSafety).mockReturnValue(null); + const { duckdbQuery } = await import("@/lib/workspace"); + vi.mocked(duckdbQuery).mockReturnValue([{ count: 42 }]); + + const { POST } = await import("./reports/execute/route.js"); + const req = new Request("http://localhost/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: "SELECT COUNT(*) as count FROM v_deals" }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.rows).toEqual([{ count: 42 }]); + }); + + it("applies filters to SQL", async () => { + const { checkSqlSafety, buildFilterClauses, injectFilters } = await import("@/lib/report-filters"); + vi.mocked(checkSqlSafety).mockReturnValue(null); + vi.mocked(buildFilterClauses).mockReturnValue(['"Status" = \'Active\'']); + vi.mocked(injectFilters).mockReturnValue("SELECT * FROM filtered"); + const { duckdbQuery } = await import("@/lib/workspace"); + vi.mocked(duckdbQuery).mockReturnValue([{ count: 10 }]); + + const { POST } = await import("./reports/execute/route.js"); + const req = new Request("http://localhost/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sql: "SELECT * FROM v_deals", + filters: [{ id: "s", column: "Status", value: { type: "select", value: "Active" } }], + }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + expect(buildFilterClauses).toHaveBeenCalled(); + expect(injectFilters).toHaveBeenCalled(); + }); + }); + + // ─── POST /api/workspace/query ───────────────────────────────── + + describe("POST /api/workspace/query", () => { + it("returns 400 for missing sql", async () => { + const { POST } = await import("./query/route.js"); + const req = new Request("http://localhost/api/workspace/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("executes query and returns rows", async () => { + const { duckdbQuery } = await import("@/lib/workspace"); + vi.mocked(duckdbQuery).mockReturnValue([{ id: 1 }]); + + const { POST } = await import("./query/route.js"); + const req = new Request("http://localhost/api/workspace/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: "SELECT 1 as id" }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.rows).toEqual([{ id: 1 }]); + }); + + it("rejects mutation SQL with 403", async () => { + const { POST } = await import("./query/route.js"); + const req = new Request("http://localhost/api/workspace/query", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: "DELETE FROM users" }), + }); + const res = await POST(req); + expect(res.status).toBe(403); + }); + }); +}); diff --git a/apps/web/app/api/workspace/db/introspect/route.ts b/apps/web/app/api/workspace/db/introspect/route.ts new file mode 100644 index 00000000000..409c3fb3e96 --- /dev/null +++ b/apps/web/app/api/workspace/db/introspect/route.ts @@ -0,0 +1,98 @@ +import { safeResolvePath, duckdbQueryOnFile, resolveDuckdbBin } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type TableInfo = { + table_name: string; + column_count: number; + estimated_row_count: number; + columns: Array<{ + name: string; + type: string; + is_nullable: boolean; + }>; +}; + +/** + * GET /api/workspace/db/introspect?path= + * + * Introspects a DuckDB / SQLite / generic DB file using the duckdb CLI. + * Returns the list of tables with their columns and approximate row counts. + */ +export async function GET(request: Request) { + const { searchParams } = new URL(request.url); + const relPath = searchParams.get("path"); + + if (!relPath) { + return Response.json( + { error: "Missing required `path` query parameter" }, + { status: 400 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + // Check if DuckDB CLI binary is available + if (!resolveDuckdbBin()) { + return Response.json({ tables: [], path: relPath, duckdb_available: false }); + } + + // Get all user tables (skip internal DuckDB catalogs) + const rawTables = duckdbQueryOnFile<{ + table_name: string; + table_type: string; + }>( + absPath, + "SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = 'main' ORDER BY table_name", + ); + + if (rawTables.length === 0) { + return Response.json({ tables: [], path: relPath }); + } + + const tables: TableInfo[] = []; + + for (const t of rawTables) { + // Fetch columns for this table + const cols = duckdbQueryOnFile<{ + column_name: string; + data_type: string; + is_nullable: string; + }>( + absPath, + `SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = 'main' AND table_name = '${t.table_name.replace(/'/g, "''")}' ORDER BY ordinal_position`, + ); + + // Get approximate row count + let rowCount = 0; + try { + const countResult = duckdbQueryOnFile<{ cnt: number }>( + absPath, + `SELECT count(*) as cnt FROM "${t.table_name.replace(/"/g, '""')}"`, + ); + rowCount = countResult[0]?.cnt ?? 0; + } catch { + // skip if we can't count + } + + tables.push({ + table_name: t.table_name, + column_count: cols.length, + estimated_row_count: rowCount, + columns: cols.map((c) => ({ + name: c.column_name, + type: c.data_type, + is_nullable: c.is_nullable === "YES", + })), + }); + } + + return Response.json({ tables, path: relPath }); +} diff --git a/apps/web/app/api/workspace/db/query/route.ts b/apps/web/app/api/workspace/db/query/route.ts new file mode 100644 index 00000000000..69c7011af79 --- /dev/null +++ b/apps/web/app/api/workspace/db/query/route.ts @@ -0,0 +1,56 @@ +import { safeResolvePath, duckdbQueryOnFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/db/query + * Body: { path: string, sql: string } + * + * Executes a read-only SQL query against a database file and returns JSON rows. + * Only SELECT statements are allowed for safety. + */ +export async function POST(request: Request) { + let body: { path?: string; sql?: string }; + try { + body = await request.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, sql } = body; + + if (!relPath || !sql) { + return Response.json( + { error: "Missing required `path` and `sql` fields" }, + { status: 400 }, + ); + } + + // Basic safety: only allow SELECT-like statements + const trimmedSql = sql.trim().toUpperCase(); + if ( + !trimmedSql.startsWith("SELECT") && + !trimmedSql.startsWith("PRAGMA") && + !trimmedSql.startsWith("DESCRIBE") && + !trimmedSql.startsWith("SHOW") && + !trimmedSql.startsWith("EXPLAIN") && + !trimmedSql.startsWith("WITH") + ) { + return Response.json( + { error: "Only read-only queries (SELECT, DESCRIBE, SHOW, EXPLAIN, WITH) are allowed" }, + { status: 403 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + const rows = duckdbQueryOnFile(absPath, sql); + return Response.json({ rows, sql }); +} diff --git a/apps/web/app/api/workspace/file-ops.test.ts b/apps/web/app/api/workspace/file-ops.test.ts new file mode 100644 index 00000000000..5a6bcdb94d6 --- /dev/null +++ b/apps/web/app/api/workspace/file-ops.test.ts @@ -0,0 +1,292 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Mock node:fs +vi.mock("node:fs", () => ({ + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + rmSync: vi.fn(), + statSync: vi.fn(() => ({ isDirectory: () => false })), + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + renameSync: vi.fn(), + cpSync: vi.fn(), + copyFileSync: vi.fn(), +})); + +// Mock workspace utilities +vi.mock("@/lib/workspace", () => ({ + readWorkspaceFile: vi.fn(), + safeResolvePath: vi.fn(), + safeResolveNewPath: vi.fn(), + isSystemFile: vi.fn(() => false), + resolveWorkspaceRoot: vi.fn(() => "/ws"), +})); + +describe("Workspace File Operations API", () => { + beforeEach(() => { + vi.resetModules(); + vi.mock("node:fs", () => ({ + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + rmSync: vi.fn(), + statSync: vi.fn(() => ({ isDirectory: () => false })), + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + renameSync: vi.fn(), + cpSync: vi.fn(), + copyFileSync: vi.fn(), + })); + vi.mock("@/lib/workspace", () => ({ + readWorkspaceFile: vi.fn(), + safeResolvePath: vi.fn(), + safeResolveNewPath: vi.fn(), + isSystemFile: vi.fn(() => false), + resolveWorkspaceRoot: vi.fn(() => "/ws"), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── GET /api/workspace/file ──────────────────────────────────── + + describe("GET /api/workspace/file", () => { + it("returns 400 when path param is missing", async () => { + const { GET } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file"); + const res = await GET(req); + expect(res.status).toBe(400); + const json = await res.json(); + expect(json.error).toContain("path"); + }); + + it("returns file content when found", async () => { + const { readWorkspaceFile } = await import("@/lib/workspace"); + vi.mocked(readWorkspaceFile).mockReturnValue({ content: "# Hello", type: "markdown" }); + + const { GET } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file?path=doc.md"); + const res = await GET(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.content).toBe("# Hello"); + expect(json.type).toBe("markdown"); + }); + + it("returns 404 when file not found", async () => { + const { readWorkspaceFile } = await import("@/lib/workspace"); + vi.mocked(readWorkspaceFile).mockReturnValue(null); + + const { GET } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file?path=missing.md"); + const res = await GET(req); + expect(res.status).toBe(404); + }); + }); + + // ─── POST /api/workspace/file ─────────────────────────────────── + + describe("POST /api/workspace/file", () => { + it("writes file content successfully", async () => { + const { safeResolveNewPath } = await import("@/lib/workspace"); + vi.mocked(safeResolveNewPath).mockReturnValue("/ws/doc.md"); + const { writeFileSync: mockWrite, mkdirSync: mockMkdir } = await import("node:fs"); + + const { POST } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "doc.md", content: "# Hello" }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.ok).toBe(true); + expect(mockMkdir).toHaveBeenCalled(); + expect(mockWrite).toHaveBeenCalled(); + }); + + it("returns 400 for missing path", async () => { + const { POST } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content: "text" }), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("returns 400 for missing content", async () => { + const { POST } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "doc.md" }), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("returns 400 for path traversal", async () => { + const { safeResolveNewPath } = await import("@/lib/workspace"); + vi.mocked(safeResolveNewPath).mockReturnValue(null); + + const { POST } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "../etc/passwd", content: "hack" }), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("returns 400 for invalid JSON body", async () => { + const { POST } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "not json", + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("returns 500 on write error", async () => { + const { safeResolveNewPath } = await import("@/lib/workspace"); + vi.mocked(safeResolveNewPath).mockReturnValue("/ws/doc.md"); + const { writeFileSync: mockWrite } = await import("node:fs"); + vi.mocked(mockWrite).mockImplementation(() => { throw new Error("EACCES"); }); + + const { POST } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "doc.md", content: "text" }), + }); + const res = await POST(req); + expect(res.status).toBe(500); + }); + }); + + // ─── DELETE /api/workspace/file ───────────────────────────────── + + describe("DELETE /api/workspace/file", () => { + it("deletes file successfully", async () => { + const { safeResolvePath, isSystemFile } = await import("@/lib/workspace"); + vi.mocked(safeResolvePath).mockReturnValue("/ws/file.txt"); + vi.mocked(isSystemFile).mockReturnValue(false); + + const { DELETE } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "DELETE", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "file.txt" }), + }); + const res = await DELETE(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.ok).toBe(true); + }); + + it("returns 403 for system file", async () => { + const { isSystemFile } = await import("@/lib/workspace"); + vi.mocked(isSystemFile).mockReturnValue(true); + + const { DELETE } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "DELETE", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: ".object.yaml" }), + }); + const res = await DELETE(req); + expect(res.status).toBe(403); + }); + + it("returns 404 when file not found", async () => { + const { safeResolvePath, isSystemFile } = await import("@/lib/workspace"); + vi.mocked(isSystemFile).mockReturnValue(false); + vi.mocked(safeResolvePath).mockReturnValue(null); + + const { DELETE } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "DELETE", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "nonexistent.txt" }), + }); + const res = await DELETE(req); + expect(res.status).toBe(404); + }); + + it("returns 400 for missing path", async () => { + const { DELETE } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "DELETE", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await DELETE(req); + expect(res.status).toBe(400); + }); + + it("returns 400 for invalid JSON body", async () => { + const { DELETE } = await import("./file/route.js"); + const req = new Request("http://localhost/api/workspace/file", { + method: "DELETE", + headers: { "Content-Type": "application/json" }, + body: "not json", + }); + const res = await DELETE(req); + expect(res.status).toBe(400); + }); + }); + + // ─── POST /api/workspace/mkdir ────────────────────────────────── + + describe("POST /api/workspace/mkdir", () => { + it("creates directory successfully", async () => { + const { safeResolveNewPath } = await import("@/lib/workspace"); + vi.mocked(safeResolveNewPath).mockReturnValue("/ws/new-folder"); + + const { POST } = await import("./mkdir/route.js"); + const req = new Request("http://localhost/api/workspace/mkdir", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "new-folder" }), + }); + const res = await POST(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.ok).toBe(true); + }); + + it("returns 400 for missing path", async () => { + const { POST } = await import("./mkdir/route.js"); + const req = new Request("http://localhost/api/workspace/mkdir", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + + it("returns 400 for traversal attempt", async () => { + const { safeResolveNewPath } = await import("@/lib/workspace"); + vi.mocked(safeResolveNewPath).mockReturnValue(null); + + const { POST } = await import("./mkdir/route.js"); + const req = new Request("http://localhost/api/workspace/mkdir", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "../../etc" }), + }); + const res = await POST(req); + expect(res.status).toBe(400); + }); + }); +}); diff --git a/apps/web/app/api/workspace/file/route.ts b/apps/web/app/api/workspace/file/route.ts new file mode 100644 index 00000000000..a096c66f9b7 --- /dev/null +++ b/apps/web/app/api/workspace/file/route.ts @@ -0,0 +1,121 @@ +import { writeFileSync, mkdirSync, rmSync, statSync } from "node:fs"; +import { dirname } from "node:path"; +import { readWorkspaceFile, safeResolvePath, safeResolveNewPath, isSystemFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function GET(req: Request) { + const url = new URL(req.url); + const path = url.searchParams.get("path"); + + if (!path) { + return Response.json( + { error: "Missing 'path' query parameter" }, + { status: 400 }, + ); + } + + const file = readWorkspaceFile(path); + if (!file) { + return Response.json( + { error: "File not found or access denied" }, + { status: 404 }, + ); + } + + return Response.json(file); +} + +/** + * POST /api/workspace/file + * Body: { path: string, content: string } + * + * Writes a file to the workspace. Creates parent directories as needed. + */ +export async function POST(req: Request) { + let body: { path?: string; content?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, content } = body; + if (!relPath || typeof relPath !== "string" || typeof content !== "string") { + return Response.json( + { error: "Missing 'path' and 'content' fields" }, + { status: 400 }, + ); + } + + // Use safeResolveNewPath (not safeResolvePath) because the file may not exist yet + const absPath = safeResolveNewPath(relPath); + if (!absPath) { + return Response.json( + { error: "Invalid path or path traversal rejected" }, + { status: 400 }, + ); + } + + try { + mkdirSync(dirname(absPath), { recursive: true }); + writeFileSync(absPath, content, "utf-8"); + return Response.json({ ok: true, path: relPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Write failed" }, + { status: 500 }, + ); + } +} + +/** + * DELETE /api/workspace/file + * Body: { path: string } + * + * Deletes a file or folder from the workspace. + * System files (.object.yaml, workspace.duckdb, etc.) are protected. + */ +export async function DELETE(req: Request) { + let body: { path?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath } = body; + if (!relPath || typeof relPath !== "string") { + return Response.json( + { error: "Missing 'path' field" }, + { status: 400 }, + ); + } + + if (isSystemFile(relPath)) { + return Response.json( + { error: "Cannot delete system file" }, + { status: 403 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + try { + const stat = statSync(absPath); + rmSync(absPath, { recursive: stat.isDirectory() }); + return Response.json({ ok: true, path: relPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Delete failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/init/route.test.ts b/apps/web/app/api/workspace/init/route.test.ts new file mode 100644 index 00000000000..4829de2cfbc --- /dev/null +++ b/apps/web/app/api/workspace/init/route.test.ts @@ -0,0 +1,219 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + copyFileSync: vi.fn(), +})); + +vi.mock("node:child_process", () => ({ + execSync: vi.fn(() => ""), + exec: vi.fn( + ( + _cmd: string, + _opts: unknown, + cb: (err: Error | null, result: { stdout: string }) => void, + ) => { + cb(null, { stdout: "" }); + }, + ), +})); + +vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), +})); + +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +describe("POST /api/workspace/init", () => { + const originalEnv = { ...process.env }; + const STATE_DIR = join("/home/testuser", ".openclaw"); + + beforeEach(() => { + vi.resetModules(); + vi.restoreAllMocks(); + process.env = { ...originalEnv }; + delete process.env.OPENCLAW_PROFILE; + delete process.env.OPENCLAW_HOME; + delete process.env.OPENCLAW_WORKSPACE; + delete process.env.OPENCLAW_STATE_DIR; + + vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => false), + readFileSync: vi.fn(() => ""), + readdirSync: vi.fn(() => []), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + copyFileSync: vi.fn(), + })); + vi.mock("node:child_process", () => ({ + execSync: vi.fn(() => ""), + exec: vi.fn( + ( + _cmd: string, + _opts: unknown, + cb: (err: Error | null, result: { stdout: string }) => void, + ) => { + cb(null, { stdout: "" }); + }, + ), + })); + vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), + })); + }); + + afterEach(() => { + process.env = originalEnv; + }); + + async function callInit(body: Record) { + const { POST } = await import("./route.js"); + const req = new Request("http://localhost/api/workspace/init", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + return POST(req); + } + + it("creates default workspace directory", async () => { + const mockMkdir = vi.mocked(mkdirSync); + const response = await callInit({}); + expect(response.status).toBe(200); + expect(mockMkdir).toHaveBeenCalledWith( + join(STATE_DIR, "workspace"), + { recursive: true }, + ); + const json = await response.json(); + expect(json.profile).toBe("default"); + expect(json.workspaceDir).toBe(join(STATE_DIR, "workspace")); + }); + + it("creates profile-specific workspace directory", async () => { + const mockMkdir = vi.mocked(mkdirSync); + const response = await callInit({ profile: "work" }); + expect(response.status).toBe(200); + expect(mockMkdir).toHaveBeenCalledWith( + join(STATE_DIR, "workspace-work"), + { recursive: true }, + ); + const json = await response.json(); + expect(json.profile).toBe("work"); + }); + + it("rejects invalid profile names", async () => { + const response = await callInit({ profile: "invalid profile!" }); + expect(response.status).toBe(400); + const json = await response.json(); + expect(json.error).toContain("Invalid profile name"); + }); + + it("allows alphanumeric, hyphens, and underscores in profile names", async () => { + const response = await callInit({ profile: "my-work_1" }); + expect(response.status).toBe(200); + const json = await response.json(); + expect(json.profile).toBe("my-work_1"); + }); + + it("accepts 'default' as profile name", async () => { + const response = await callInit({ profile: "default" }); + expect(response.status).toBe(200); + const json = await response.json(); + expect(json.workspaceDir).toBe(join(STATE_DIR, "workspace")); + }); + + it("seeds bootstrap files when seedBootstrap is not false", async () => { + const mockWrite = vi.mocked(writeFileSync); + await callInit({}); + const writtenPaths = mockWrite.mock.calls.map((c) => c[0] as string); + const bootstrapFiles = writtenPaths.filter( + (p) => + p.endsWith("AGENTS.md") || + p.endsWith("SOUL.md") || + p.endsWith("TOOLS.md") || + p.endsWith("IDENTITY.md") || + p.endsWith("USER.md") || + p.endsWith("HEARTBEAT.md") || + p.endsWith("BOOTSTRAP.md"), + ); + expect(bootstrapFiles.length).toBeGreaterThan(0); + }); + + it("returns seeded files list", async () => { + const response = await callInit({}); + const json = await response.json(); + expect(Array.isArray(json.seededFiles)).toBe(true); + }); + + it("skips bootstrap seeding when seedBootstrap is false", async () => { + const mockWrite = vi.mocked(writeFileSync); + const callsBefore = mockWrite.mock.calls.length; + await callInit({ seedBootstrap: false }); + const bootstrapWrites = mockWrite.mock.calls + .slice(callsBefore) + .filter((c) => { + const p = c[0] as string; + return p.endsWith(".md") && !p.endsWith("workspace-state.json"); + }); + expect(bootstrapWrites).toHaveLength(0); + }); + + it("does not overwrite existing bootstrap files (idempotent)", async () => { + const mockExist = vi.mocked(existsSync); + const wsDir = join(STATE_DIR, "workspace"); + mockExist.mockImplementation((p) => { + const s = String(p); + return s === join(wsDir, "AGENTS.md") || s === join(wsDir, "SOUL.md"); + }); + + const response = await callInit({}); + const json = await response.json(); + expect(json.seededFiles).not.toContain("AGENTS.md"); + expect(json.seededFiles).not.toContain("SOUL.md"); + }); + + it("handles custom workspace path", async () => { + const mockMkdir = vi.mocked(mkdirSync); + const response = await callInit({ + profile: "custom", + path: "/my/custom/workspace", + }); + expect(response.status).toBe(200); + expect(mockMkdir).toHaveBeenCalledWith("/my/custom/workspace", { + recursive: true, + }); + const json = await response.json(); + expect(json.workspaceDir).toBe("/my/custom/workspace"); + }); + + it("resolves tilde in custom path", async () => { + const mockMkdir = vi.mocked(mkdirSync); + await callInit({ profile: "tilde", path: "~/my-workspace" }); + expect(mockMkdir).toHaveBeenCalledWith( + join("/home/testuser", "my-workspace"), + { recursive: true }, + ); + }); + + it("auto-switches to new profile after creation", async () => { + const response = await callInit({ profile: "newprofile" }); + const json = await response.json(); + expect(json.activeProfile).toBe("newprofile"); + }); + + it("handles mkdir failure with 500", async () => { + const mockMkdir = vi.mocked(mkdirSync); + mockMkdir.mockImplementation(() => { + throw new Error("EACCES: permission denied"); + }); + const response = await callInit({ profile: "fail" }); + expect(response.status).toBe(500); + const json = await response.json(); + expect(json.error).toContain("Failed to create workspace directory"); + }); +}); diff --git a/apps/web/app/api/workspace/init/route.ts b/apps/web/app/api/workspace/init/route.ts new file mode 100644 index 00000000000..eef98d4c744 --- /dev/null +++ b/apps/web/app/api/workspace/init/route.ts @@ -0,0 +1,332 @@ +import { existsSync, mkdirSync, writeFileSync, readFileSync, copyFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { homedir } from "node:os"; +import { resolveOpenClawStateDir, setUIActiveProfile, getEffectiveProfile, resolveWorkspaceRoot, registerWorkspacePath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +// --------------------------------------------------------------------------- +// Bootstrap file names (must match src/agents/workspace.ts) +// --------------------------------------------------------------------------- + +const BOOTSTRAP_FILENAMES = [ + "AGENTS.md", + "SOUL.md", + "TOOLS.md", + "IDENTITY.md", + "USER.md", + "HEARTBEAT.md", + "BOOTSTRAP.md", +] as const; + +// Minimal fallback content used when templates can't be loaded from disk +const FALLBACK_CONTENT: Record = { + "AGENTS.md": "# AGENTS.md - Your Workspace\n\nThis folder is home. Treat it that way.\n", + "SOUL.md": "# SOUL.md - Who You Are\n\nDescribe the personality and behavior of your agent here.\n", + "TOOLS.md": "# TOOLS.md - Local Notes\n\nSkills define how tools work. This file is for your specifics.\n", + "IDENTITY.md": "# IDENTITY.md - Who Am I?\n\nFill this in during your first conversation.\n", + "USER.md": "# USER.md - About Your Human\n\nDescribe yourself and how you'd like the agent to interact with you.\n", + "HEARTBEAT.md": "# HEARTBEAT.md\n\n# Keep this file empty (or with only comments) to skip heartbeat API calls.\n", + "BOOTSTRAP.md": "# BOOTSTRAP.md - Hello, World\n\nYou just woke up. Time to figure out who you are.\n", +}; + +// --------------------------------------------------------------------------- +// CRM seed objects (mirrors src/agents/workspace-seed.ts) +// --------------------------------------------------------------------------- + +type SeedField = { + name: string; + type: string; + required?: boolean; + enumValues?: string[]; +}; + +type SeedObject = { + id: string; + name: string; + description: string; + icon: string; + defaultView: string; + entryCount: number; + fields: SeedField[]; +}; + +const SEED_OBJECTS: SeedObject[] = [ + { + id: "seed_obj_people_00000000000000", + name: "people", + description: "Contact management", + icon: "users", + defaultView: "table", + entryCount: 5, + fields: [ + { name: "Full Name", type: "text", required: true }, + { name: "Email Address", type: "email", required: true }, + { name: "Phone Number", type: "phone" }, + { name: "Company", type: "text" }, + { name: "Status", type: "enum", enumValues: ["Active", "Inactive", "Lead"] }, + { name: "Notes", type: "richtext" }, + ], + }, + { + id: "seed_obj_company_0000000000000", + name: "company", + description: "Company tracking", + icon: "building-2", + defaultView: "table", + entryCount: 3, + fields: [ + { name: "Company Name", type: "text", required: true }, + { + name: "Industry", + type: "enum", + enumValues: ["Technology", "Finance", "Healthcare", "Education", "Retail", "Other"], + }, + { name: "Website", type: "text" }, + { name: "Type", type: "enum", enumValues: ["Client", "Partner", "Vendor", "Prospect"] }, + { name: "Notes", type: "richtext" }, + ], + }, + { + id: "seed_obj_task_000000000000000", + name: "task", + description: "Task tracking board", + icon: "check-square", + defaultView: "kanban", + entryCount: 5, + fields: [ + { name: "Title", type: "text", required: true }, + { name: "Description", type: "text" }, + { name: "Status", type: "enum", enumValues: ["In Queue", "In Progress", "Done"] }, + { name: "Priority", type: "enum", enumValues: ["Low", "Medium", "High"] }, + { name: "Due Date", type: "date" }, + { name: "Notes", type: "richtext" }, + ], + }, +]; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function stripFrontMatter(content: string): string { + if (!content.startsWith("---")) {return content;} + const endIndex = content.indexOf("\n---", 3); + if (endIndex === -1) {return content;} + return content.slice(endIndex + "\n---".length).replace(/^\s+/, ""); +} + +/** Try multiple candidate paths to find the monorepo root. */ +function resolveProjectRoot(): string | null { + const marker = join("docs", "reference", "templates", "AGENTS.md"); + const cwd = process.cwd(); + + // CWD is the repo root (standalone builds) + if (existsSync(join(cwd, marker))) {return cwd;} + + // CWD is apps/web/ (dev mode) + const fromApps = resolve(cwd, "..", ".."); + if (existsSync(join(fromApps, marker))) {return fromApps;} + + return null; +} + +function loadTemplateContent(filename: string, projectRoot: string | null): string { + if (projectRoot) { + const templatePath = join(projectRoot, "docs", "reference", "templates", filename); + try { + const raw = readFileSync(templatePath, "utf-8"); + return stripFrontMatter(raw); + } catch { + // fall through to fallback + } + } + return FALLBACK_CONTENT[filename] ?? ""; +} + +function generateObjectYaml(obj: SeedObject): string { + const lines: string[] = [ + `id: "${obj.id}"`, + `name: "${obj.name}"`, + `description: "${obj.description}"`, + `icon: "${obj.icon}"`, + `default_view: "${obj.defaultView}"`, + `entry_count: ${obj.entryCount}`, + "fields:", + ]; + + for (const field of obj.fields) { + lines.push(` - name: "${field.name}"`); + lines.push(` type: ${field.type}`); + if (field.required) {lines.push(" required: true");} + if (field.enumValues) {lines.push(` values: ${JSON.stringify(field.enumValues)}`);} + } + + return lines.join("\n") + "\n"; +} + +function generateWorkspaceMd(objects: SeedObject[]): string { + const lines: string[] = ["# Workspace Schema", "", "Auto-generated summary of the workspace database.", ""]; + for (const obj of objects) { + lines.push(`## ${obj.name}`, ""); + lines.push(`- **Description**: ${obj.description}`); + lines.push(`- **View**: \`${obj.defaultView}\``); + lines.push(`- **Entries**: ${obj.entryCount}`); + lines.push("- **Fields**:"); + for (const field of obj.fields) { + const req = field.required ? " (required)" : ""; + const vals = field.enumValues ? ` β€” ${field.enumValues.join(", ")}` : ""; + lines.push(` - ${field.name} (\`${field.type}\`)${req}${vals}`); + } + lines.push(""); + } + return lines.join("\n"); +} + +function writeIfMissing(filePath: string, content: string): boolean { + if (existsSync(filePath)) {return false;} + try { + writeFileSync(filePath, content, { encoding: "utf-8", flag: "wx" }); + return true; + } catch { + return false; + } +} + +function seedDuckDB(workspaceDir: string, projectRoot: string | null): boolean { + const destPath = join(workspaceDir, "workspace.duckdb"); + if (existsSync(destPath)) {return false;} + + if (!projectRoot) {return false;} + + const seedDb = join(projectRoot, "assets", "seed", "workspace.duckdb"); + if (!existsSync(seedDb)) {return false;} + + try { + copyFileSync(seedDb, destPath); + } catch { + return false; + } + + // Create filesystem projections for CRM objects + for (const obj of SEED_OBJECTS) { + const objDir = join(workspaceDir, obj.name); + mkdirSync(objDir, { recursive: true }); + writeIfMissing(join(objDir, ".object.yaml"), generateObjectYaml(obj)); + } + + writeIfMissing(join(workspaceDir, "WORKSPACE.md"), generateWorkspaceMd(SEED_OBJECTS)); + + return true; +} + +// --------------------------------------------------------------------------- +// Route handler +// --------------------------------------------------------------------------- + +export async function POST(req: Request) { + const body = (await req.json()) as { + profile?: string; + path?: string; + seedBootstrap?: boolean; + }; + + const profileName = body.profile?.trim() || null; + + if (profileName && profileName !== "default" && !/^[a-zA-Z0-9_-]+$/.test(profileName)) { + return Response.json( + { error: "Invalid profile name. Use letters, numbers, hyphens, or underscores." }, + { status: 400 }, + ); + } + + // Determine workspace directory + let workspaceDir: string; + if (body.path?.trim()) { + workspaceDir = body.path.trim(); + if (workspaceDir.startsWith("~")) { + workspaceDir = join(homedir(), workspaceDir.slice(1)); + } + workspaceDir = resolve(workspaceDir); + } else { + const stateDir = resolveOpenClawStateDir(); + if (profileName && profileName !== "default") { + workspaceDir = join(stateDir, `workspace-${profileName}`); + } else { + workspaceDir = join(stateDir, "workspace"); + } + } + + try { + mkdirSync(workspaceDir, { recursive: true }); + } catch (err) { + return Response.json( + { error: `Failed to create workspace directory: ${(err as Error).message}` }, + { status: 500 }, + ); + } + + const seedBootstrap = body.seedBootstrap !== false; + const seeded: string[] = []; + + if (seedBootstrap) { + const projectRoot = resolveProjectRoot(); + + // Seed all bootstrap files from templates + for (const filename of BOOTSTRAP_FILENAMES) { + const filePath = join(workspaceDir, filename); + if (!existsSync(filePath)) { + const content = loadTemplateContent(filename, projectRoot); + if (writeIfMissing(filePath, content)) { + seeded.push(filename); + } + } + } + + // Seed DuckDB + CRM object projections + if (seedDuckDB(workspaceDir, projectRoot)) { + seeded.push("workspace.duckdb"); + for (const obj of SEED_OBJECTS) { + seeded.push(`${obj.name}/.object.yaml`); + } + } + + // Write workspace state so the gateway knows seeding was done + const stateDir = join(workspaceDir, ".openclaw"); + const statePath = join(stateDir, "workspace-state.json"); + if (!existsSync(statePath)) { + try { + mkdirSync(stateDir, { recursive: true }); + const state = { + version: 1, + bootstrapSeededAt: new Date().toISOString(), + duckdbSeededAt: existsSync(join(workspaceDir, "workspace.duckdb")) + ? new Date().toISOString() + : undefined, + }; + writeFileSync(statePath, JSON.stringify(state, null, 2) + "\n", "utf-8"); + } catch { + // Best-effort state tracking + } + } + } + + // Remember custom-path workspaces in the registry + if (body.path?.trim() && profileName) { + registerWorkspacePath(profileName, workspaceDir); + } + + // Switch to the new profile + if (profileName) { + setUIActiveProfile(profileName === "default" ? null : profileName); + } + + return Response.json({ + workspaceDir, + profile: profileName || "default", + activeProfile: getEffectiveProfile() || "default", + seededFiles: seeded, + workspaceRoot: resolveWorkspaceRoot(), + }); +} diff --git a/apps/web/app/api/workspace/mkdir/route.ts b/apps/web/app/api/workspace/mkdir/route.ts new file mode 100644 index 00000000000..fcbe54acb73 --- /dev/null +++ b/apps/web/app/api/workspace/mkdir/route.ts @@ -0,0 +1,71 @@ +import { mkdirSync, existsSync } from "node:fs"; +import { resolve, normalize } from "node:path"; +import { safeResolveNewPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/mkdir + * Body: { path: string; absolute?: boolean } + * + * Creates a new directory. By default paths are resolved relative to the + * workspace root. When `absolute` is true the path is treated as a + * filesystem-absolute path (used by the directory picker for workspace + * creation outside the current workspace). + */ +export async function POST(req: Request) { + let body: { path?: string; absolute?: boolean }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: rawPath, absolute: useAbsolute } = body; + if (!rawPath || typeof rawPath !== "string") { + return Response.json( + { error: "Missing 'path' field" }, + { status: 400 }, + ); + } + + let absPath: string | null; + + if (useAbsolute) { + const normalized = normalize(rawPath); + if (normalized.includes("/../") || normalized.includes("/..")) { + return Response.json( + { error: "Path traversal rejected" }, + { status: 400 }, + ); + } + absPath = resolve(normalized); + } else { + absPath = safeResolveNewPath(rawPath); + } + + if (!absPath) { + return Response.json( + { error: "Invalid path or path traversal rejected" }, + { status: 400 }, + ); + } + + if (existsSync(absPath)) { + return Response.json( + { error: "Directory already exists" }, + { status: 409 }, + ); + } + + try { + mkdirSync(absPath, { recursive: true }); + return Response.json({ ok: true, path: absPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "mkdir failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/move/route.ts b/apps/web/app/api/workspace/move/route.ts new file mode 100644 index 00000000000..a2670a60240 --- /dev/null +++ b/apps/web/app/api/workspace/move/route.ts @@ -0,0 +1,93 @@ +import { renameSync, existsSync, statSync } from "node:fs"; +import { join, basename } from "node:path"; +import { safeResolvePath, isSystemFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/move + * Body: { sourcePath: string, destinationDir: string } + * + * Moves a file or folder into a different directory. + * System files are protected from moving. + */ +export async function POST(req: Request) { + let body: { sourcePath?: string; destinationDir?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { sourcePath, destinationDir } = body; + if (!sourcePath || typeof sourcePath !== "string" || !destinationDir || typeof destinationDir !== "string") { + return Response.json( + { error: "Missing 'sourcePath' and 'destinationDir' fields" }, + { status: 400 }, + ); + } + + if (isSystemFile(sourcePath)) { + return Response.json( + { error: "Cannot move system file" }, + { status: 403 }, + ); + } + + const srcAbs = safeResolvePath(sourcePath); + if (!srcAbs) { + return Response.json( + { error: "Source not found or path traversal rejected" }, + { status: 404 }, + ); + } + + const destDirAbs = safeResolvePath(destinationDir); + if (!destDirAbs) { + return Response.json( + { error: "Destination not found or path traversal rejected" }, + { status: 404 }, + ); + } + + // Destination must be a directory + if (!statSync(destDirAbs).isDirectory()) { + return Response.json( + { error: "Destination is not a directory" }, + { status: 400 }, + ); + } + + // Prevent moving a folder into itself or its children + const srcAbsNorm = srcAbs + "/"; + if (destDirAbs.startsWith(srcAbsNorm) || destDirAbs === srcAbs) { + return Response.json( + { error: "Cannot move a folder into itself" }, + { status: 400 }, + ); + } + + const itemName = basename(srcAbs); + const destAbs = join(destDirAbs, itemName); + + if (existsSync(destAbs)) { + return Response.json( + { error: `'${itemName}' already exists in destination` }, + { status: 409 }, + ); + } + + // Build new relative path + const newRelPath = destinationDir === "." ? itemName : `${destinationDir}/${itemName}`; + + try { + renameSync(srcAbs, destAbs); + return Response.json({ ok: true, oldPath: sourcePath, newPath: newRelPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Move failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/objects.test.ts b/apps/web/app/api/workspace/objects.test.ts new file mode 100644 index 00000000000..ce2da6a62cf --- /dev/null +++ b/apps/web/app/api/workspace/objects.test.ts @@ -0,0 +1,392 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Mock node:child_process +vi.mock("node:child_process", () => ({ + execSync: vi.fn(() => ""), +})); + +// Mock workspace +vi.mock("@/lib/workspace", () => ({ + duckdbPath: vi.fn(() => null), + duckdbQueryOnFile: vi.fn(() => []), + duckdbExecOnFile: vi.fn(() => true), + findDuckDBForObject: vi.fn(() => null), + parseRelationValue: vi.fn((v: string | null) => (v ? [v] : [])), + resolveDuckdbBin: vi.fn(() => null), + discoverDuckDBPaths: vi.fn(() => []), +})); + +describe("Workspace Objects API", () => { + beforeEach(() => { + vi.resetModules(); + vi.mock("node:child_process", () => ({ + execSync: vi.fn(() => ""), + })); + vi.mock("@/lib/workspace", () => ({ + duckdbPath: vi.fn(() => null), + duckdbQueryOnFile: vi.fn(() => []), + duckdbExecOnFile: vi.fn(() => true), + findDuckDBForObject: vi.fn(() => null), + parseRelationValue: vi.fn((v: string | null) => (v ? [v] : [])), + resolveDuckdbBin: vi.fn(() => null), + discoverDuckDBPaths: vi.fn(() => []), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── GET /api/workspace/objects/[name] ────────────────────────── + + describe("GET /api/workspace/objects/[name]", () => { + it("returns 503 when DuckDB CLI not installed", async () => { + const { resolveDuckdbBin } = await import("@/lib/workspace"); + vi.mocked(resolveDuckdbBin).mockReturnValue(null); + + const { GET } = await import("./objects/[name]/route.js"); + const res = await GET( + new Request("http://localhost/api/workspace/objects/bad-name!"), + { params: Promise.resolve({ name: "bad-name!" }) }, + ); + expect(res.status).toBe(503); + }); + + it("returns 400 for invalid object name (when duckdb available)", async () => { + const { resolveDuckdbBin } = await import("@/lib/workspace"); + vi.mocked(resolveDuckdbBin).mockReturnValue("/opt/homebrew/bin/duckdb"); + + const { GET } = await import("./objects/[name]/route.js"); + const res = await GET( + new Request("http://localhost/api/workspace/objects/bad!name"), + { params: Promise.resolve({ name: "bad!name" }) }, + ); + expect(res.status).toBe(400); + }); + + it("returns 404 when object not found", async () => { + const { findDuckDBForObject, resolveDuckdbBin, duckdbPath: mockDuckdbPath } = await import("@/lib/workspace"); + vi.mocked(resolveDuckdbBin).mockReturnValue("/opt/homebrew/bin/duckdb"); + vi.mocked(findDuckDBForObject).mockReturnValue(null); + vi.mocked(mockDuckdbPath).mockReturnValue(null); + + const { GET } = await import("./objects/[name]/route.js"); + const res = await GET( + new Request("http://localhost/api/workspace/objects/nonexistent"), + { params: Promise.resolve({ name: "nonexistent" }) }, + ); + expect(res.status).toBe(404); + }); + + it("returns object schema and entries when found", async () => { + const { findDuckDBForObject, duckdbQueryOnFile, resolveDuckdbBin, discoverDuckDBPaths } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + vi.mocked(resolveDuckdbBin).mockReturnValue("/opt/homebrew/bin/duckdb"); + vi.mocked(discoverDuckDBPaths).mockReturnValue(["/ws/workspace.duckdb"]); + + // Mock different queries with a call counter + let queryCall = 0; + vi.mocked(duckdbQueryOnFile).mockImplementation(() => { + queryCall++; + if (queryCall === 1) { + // Object row + return [{ id: "obj1", name: "leads", description: "Leads object", icon: "star" }]; + } + if (queryCall === 2) { + // Fields + return [ + { id: "f1", name: "name", type: "text", sort_order: 0 }, + { id: "f2", name: "status", type: "enum", sort_order: 1, enum_values: '["New","Active"]' }, + ]; + } + if (queryCall === 3) { + // Statuses + return []; + } + // Entries and subsequent queries + return []; + }); + + const { GET } = await import("./objects/[name]/route.js"); + const res = await GET( + new Request("http://localhost/api/workspace/objects/leads"), + { params: Promise.resolve({ name: "leads" }) }, + ); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.object).toBeDefined(); + expect(json.fields).toBeDefined(); + }); + + it("accepts underscored names", async () => { + const { findDuckDBForObject } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue(null); + + const { GET } = await import("./objects/[name]/route.js"); + const res = await GET( + new Request("http://localhost/api/workspace/objects/my_object"), + { params: Promise.resolve({ name: "my_object" }) }, + ); + // 404 because findDuckDBForObject returns null, but name validation passes + expect(res.status).toBe(404); + }); + }); + + // ─── POST /api/workspace/objects/[name]/entries ───────────────── + + describe("POST /api/workspace/objects/[name]/entries", () => { + it("returns 400 for invalid object name", async () => { + const { POST } = await import("./objects/[name]/entries/route.js"); + const req = new Request("http://localhost/api/workspace/objects/bad!/entries", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req, { params: Promise.resolve({ name: "bad!" }) }); + expect(res.status).toBe(400); + }); + + it("returns 404 when DuckDB not found", async () => { + const { findDuckDBForObject } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue(null); + + const { POST } = await import("./objects/[name]/entries/route.js"); + const req = new Request("http://localhost/api/workspace/objects/leads/entries", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req, { params: Promise.resolve({ name: "leads" }) }); + expect(res.status).toBe(404); + }); + + it("creates entry successfully", async () => { + const { findDuckDBForObject, duckdbQueryOnFile, duckdbExecOnFile } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + + let queryCall = 0; + vi.mocked(duckdbQueryOnFile).mockImplementation(() => { + queryCall++; + if (queryCall === 1) {return [{ id: "obj1" }];} // object lookup + if (queryCall === 2) {return [{ id: "new-entry-uuid" }];} // uuid generation + return []; + }); + vi.mocked(duckdbExecOnFile).mockReturnValue(true); + + const { POST } = await import("./objects/[name]/entries/route.js"); + const req = new Request("http://localhost/api/workspace/objects/leads/entries", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fields: { name: "Acme Corp" } }), + }); + const res = await POST(req, { params: Promise.resolve({ name: "leads" }) }); + expect(res.status).toBe(201); + const json = await res.json(); + expect(json.ok).toBe(true); + expect(json.entryId).toBeDefined(); + }); + + it("returns 404 when object not found in DB", async () => { + const { findDuckDBForObject, duckdbQueryOnFile } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + vi.mocked(duckdbQueryOnFile).mockReturnValue([]); // object not found + + const { POST } = await import("./objects/[name]/entries/route.js"); + const req = new Request("http://localhost/api/workspace/objects/missing/entries", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + const res = await POST(req, { params: Promise.resolve({ name: "missing" }) }); + expect(res.status).toBe(404); + }); + }); + + // ─── GET /api/workspace/objects/[name]/entries/[id] ───────────── + + describe("GET /api/workspace/objects/[name]/entries/[id]", () => { + it("returns 400 for invalid object name", async () => { + const { GET } = await import("./objects/[name]/entries/[id]/route.js"); + const res = await GET( + new Request("http://localhost"), + { params: Promise.resolve({ name: "bad!", id: "123" }) }, + ); + expect(res.status).toBe(400); + }); + + it("returns 404 when DuckDB not found", async () => { + const { findDuckDBForObject } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue(null); + + const { GET } = await import("./objects/[name]/entries/[id]/route.js"); + const res = await GET( + new Request("http://localhost"), + { params: Promise.resolve({ name: "leads", id: "123" }) }, + ); + expect(res.status).toBe(404); + }); + + it("returns entry details when found", async () => { + const { findDuckDBForObject, duckdbQueryOnFile } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + + let queryCall = 0; + vi.mocked(duckdbQueryOnFile).mockImplementation(() => { + queryCall++; + if (queryCall === 1) {return [{ id: "obj1" }];} // object + if (queryCall === 2) {return [{ id: "f1", name: "name", type: "text" }];} // fields + if (queryCall === 3) {return [{ entry_id: "e1", field_name: "name", value: "Acme", created_at: "2025-01-01", updated_at: "2025-01-01" }];} // EAV + return []; + }); + + const { GET } = await import("./objects/[name]/entries/[id]/route.js"); + const res = await GET( + new Request("http://localhost"), + { params: Promise.resolve({ name: "leads", id: "e1" }) }, + ); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.entry).toBeDefined(); + }); + }); + + // ─── PATCH /api/workspace/objects/[name]/entries/[id] ─────────── + + describe("PATCH /api/workspace/objects/[name]/entries/[id]", () => { + it("returns 400 for invalid object name", async () => { + const { PATCH } = await import("./objects/[name]/entries/[id]/route.js"); + const req = new Request("http://localhost", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fields: {} }), + }); + const res = await PATCH(req, { params: Promise.resolve({ name: "bad!", id: "123" }) }); + expect(res.status).toBe(400); + }); + + it("returns 404 when DuckDB not found", async () => { + const { findDuckDBForObject } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue(null); + + const { PATCH } = await import("./objects/[name]/entries/[id]/route.js"); + const req = new Request("http://localhost", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fields: { name: "Updated" } }), + }); + const res = await PATCH(req, { params: Promise.resolve({ name: "leads", id: "e1" }) }); + expect(res.status).toBe(404); + }); + + it("updates entry fields", async () => { + const { findDuckDBForObject, duckdbQueryOnFile, duckdbExecOnFile } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + + let queryCall = 0; + vi.mocked(duckdbQueryOnFile).mockImplementation(() => { + queryCall++; + if (queryCall === 1) {return [{ id: "obj1" }];} // object + if (queryCall === 2) {return [{ id: "f1", name: "name", type: "text" }];} // fields + return []; + }); + vi.mocked(duckdbExecOnFile).mockReturnValue(true); + + const { PATCH } = await import("./objects/[name]/entries/[id]/route.js"); + const req = new Request("http://localhost", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fields: { name: "Updated Corp" } }), + }); + const res = await PATCH(req, { params: Promise.resolve({ name: "leads", id: "e1" }) }); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.ok).toBe(true); + }); + }); + + // ─── DELETE /api/workspace/objects/[name]/entries/[id] ────────── + + describe("DELETE /api/workspace/objects/[name]/entries/[id]", () => { + it("returns 400 for invalid object name", async () => { + const { DELETE } = await import("./objects/[name]/entries/[id]/route.js"); + const res = await DELETE( + new Request("http://localhost", { method: "DELETE" }), + { params: Promise.resolve({ name: "bad!", id: "123" }) }, + ); + expect(res.status).toBe(400); + }); + + it("returns 404 when DuckDB not found", async () => { + const { findDuckDBForObject } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue(null); + + const { DELETE } = await import("./objects/[name]/entries/[id]/route.js"); + const res = await DELETE( + new Request("http://localhost", { method: "DELETE" }), + { params: Promise.resolve({ name: "leads", id: "e1" }) }, + ); + expect(res.status).toBe(404); + }); + + it("deletes entry successfully", async () => { + const { findDuckDBForObject, duckdbQueryOnFile, duckdbExecOnFile } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + vi.mocked(duckdbQueryOnFile).mockReturnValue([{ id: "obj1" }]); + vi.mocked(duckdbExecOnFile).mockReturnValue(true); + + const { DELETE } = await import("./objects/[name]/entries/[id]/route.js"); + const res = await DELETE( + new Request("http://localhost", { method: "DELETE" }), + { params: Promise.resolve({ name: "leads", id: "e1" }) }, + ); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.ok).toBe(true); + }); + }); + + // ─── POST /api/workspace/objects/[name]/entries/bulk-delete ───── + + describe("POST /api/workspace/objects/[name]/entries/bulk-delete", () => { + it("returns 400 for invalid object name", async () => { + const { POST } = await import("./objects/[name]/entries/bulk-delete/route.js"); + const req = new Request("http://localhost", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ids: ["e1"] }), + }); + const res = await POST(req, { params: Promise.resolve({ name: "bad!" }) }); + expect(res.status).toBe(400); + }); + + it("returns 400 for empty entryIds", async () => { + const { findDuckDBForObject } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + + const { POST } = await import("./objects/[name]/entries/bulk-delete/route.js"); + const req = new Request("http://localhost", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ entryIds: [] }), + }); + const res = await POST(req, { params: Promise.resolve({ name: "leads" }) }); + expect(res.status).toBe(400); + }); + + it("deletes multiple entries", async () => { + const { findDuckDBForObject, duckdbQueryOnFile, duckdbExecOnFile } = await import("@/lib/workspace"); + vi.mocked(findDuckDBForObject).mockReturnValue("/ws/workspace.duckdb"); + vi.mocked(duckdbQueryOnFile).mockReturnValue([{ id: "obj1" }]); + vi.mocked(duckdbExecOnFile).mockReturnValue(true); + + const { POST } = await import("./objects/[name]/entries/bulk-delete/route.js"); + const req = new Request("http://localhost", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ entryIds: ["e1", "e2", "e3"] }), + }); + const res = await POST(req, { params: Promise.resolve({ name: "leads" }) }); + expect(res.status).toBe(200); + }); + }); +}); diff --git a/apps/web/app/api/workspace/objects/[name]/display-field/route.ts b/apps/web/app/api/workspace/objects/[name]/display-field/route.ts new file mode 100644 index 00000000000..4e0e8b34e1e --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/display-field/route.ts @@ -0,0 +1,83 @@ +import { duckdbQueryOnFile, duckdbExecOnFile, findDuckDBForObject } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * PATCH /api/workspace/objects/[name]/display-field + * Set which field is used as the display label for entries of this object. + * Body: { displayField: string } + */ +export async function PATCH( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB database not found" }, + { status: 404 }, + ); + } + + const body = await req.json(); + const { displayField } = body; + + if (typeof displayField !== "string" || !displayField.trim()) { + return Response.json( + { error: "displayField must be a non-empty string" }, + { status: 400 }, + ); + } + + // Ensure display_field column exists + duckdbExecOnFile(dbFile, + "ALTER TABLE objects ADD COLUMN IF NOT EXISTS display_field VARCHAR", + ); + + // Verify the object exists + const objects = duckdbQueryOnFile<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${name}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + + // Verify the field exists on this object + const escapedField = displayField.replace(/'/g, "''"); + const fieldCheck = duckdbQueryOnFile<{ id: string }>(dbFile, + `SELECT id FROM fields WHERE object_id = '${objects[0].id}' AND name = '${escapedField}' LIMIT 1`, + ); + if (fieldCheck.length === 0) { + return Response.json( + { error: `Field '${displayField}' not found on object '${name}'` }, + { status: 400 }, + ); + } + + // Update the display_field + const success = duckdbExecOnFile(dbFile, + `UPDATE objects SET display_field = '${escapedField}', updated_at = now() WHERE name = '${name}'`, + ); + + if (!success) { + return Response.json( + { error: "Failed to update display field" }, + { status: 500 }, + ); + } + + return Response.json({ ok: true, displayField }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts b/apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts new file mode 100644 index 00000000000..6a6ee61cb36 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts @@ -0,0 +1,515 @@ +import { + duckdbQueryOnFile, + duckdbExecOnFile, + findDuckDBForObject, + discoverDuckDBPaths, + parseRelationValue, +} from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +// --- Types --- + +type ObjectRow = { + id: string; + name: string; + description?: string; + icon?: string; + default_view?: string; + display_field?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; + description?: string; + required?: boolean; + enum_values?: string; + enum_colors?: string; + enum_multiple?: boolean; + related_object_id?: string; + relationship_type?: string; + sort_order?: number; +}; + +// --- Helpers --- + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +function tryParseJson(value: unknown): unknown { + if (typeof value !== "string") { + return value; + } + try { + return JSON.parse(value); + } catch { + return value; + } +} + +function resolveDisplayField( + obj: ObjectRow, + fields: FieldRow[], +): string { + if (obj.display_field) { + return obj.display_field; + } + const nameField = fields.find( + (f) => + /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) { + return nameField.name; + } + const textField = fields.find((f) => f.type === "text"); + if (textField) { + return textField.name; + } + return fields[0]?.name ?? "id"; +} + +/** Scoped query shorthand. */ +function q>(db: string, sql: string): T[] { + return duckdbQueryOnFile(db, sql); +} + +// --- Route handlers --- + +/** + * GET /api/workspace/objects/[name]/entries/[id] + * Returns a single entry with all field values, relation labels, and reverse relations. + */ +export async function GET( + _req: Request, + { params }: { params: Promise<{ name: string; id: string }> }, +) { + const { name, id } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + if (!id || id.length > 64) { + return Response.json( + { error: "Invalid entry ID" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + // Fetch object + const objects = q(dbFile, + `SELECT * FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const obj = objects[0]; + + // Fetch fields + const fields = q(dbFile, + `SELECT * FROM fields WHERE object_id = '${sqlEscape(obj.id)}' ORDER BY sort_order`, + ); + + // Fetch entry field values + const entryRows = q<{ + entry_id: string; + created_at: string; + updated_at: string; + field_name: string; + value: string | null; + }>(dbFile, + `SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.id = '${sqlEscape(id)}' + AND e.object_id = '${sqlEscape(obj.id)}'`, + ); + + if (entryRows.length === 0) { + const exists = q<{ cnt: number }>(dbFile, + `SELECT COUNT(*) as cnt FROM entries WHERE id = '${sqlEscape(id)}' AND object_id = '${sqlEscape(obj.id)}'`, + ); + if (!exists[0] || exists[0].cnt === 0) { + return Response.json( + { error: "Entry not found" }, + { status: 404 }, + ); + } + } + + // Pivot into a single record + const entry: Record = { entry_id: id }; + for (const row of entryRows) { + entry.created_at ??= row.created_at; + entry.updated_at ??= row.updated_at; + if (row.field_name) { + entry[row.field_name] = row.value; + } + } + + // Parse enum JSON strings in fields + const parsedFields = fields.map((f) => ({ + ...f, + enum_values: f.enum_values + ? tryParseJson(f.enum_values) + : undefined, + enum_colors: f.enum_colors + ? tryParseJson(f.enum_colors) + : undefined, + })); + + // Resolve relation labels for this entry + const relationLabels: Record> = {}; + const relatedObjectNames: Record = {}; + + const relationFields = fields.filter( + (f) => f.type === "relation" && f.related_object_id, + ); + + for (const rf of relationFields) { + const relatedObjs = q(dbFile, + `SELECT * FROM objects WHERE id = '${sqlEscape(rf.related_object_id!)}' LIMIT 1`, + ); + if (relatedObjs.length === 0) { + continue; + } + const relObj = relatedObjs[0]; + relatedObjectNames[rf.name] = relObj.name; + + const val = entry[rf.name]; + if (val == null || val === "") { + relationLabels[rf.name] = {}; + continue; + } + + const valStr = + typeof val === "object" && val !== null + ? JSON.stringify(val) + : typeof val === "string" + ? val + : typeof val === "number" || typeof val === "boolean" + ? String(val) + : ""; + const ids = parseRelationValue(valStr); + if (ids.length === 0) { + relationLabels[rf.name] = {}; + continue; + } + + const relFields = q(dbFile, + `SELECT * FROM fields WHERE object_id = '${sqlEscape(relObj.id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField(relObj, relFields); + + const idList = ids + .map((i) => `'${sqlEscape(i)}'`) + .join(","); + const displayRows = q<{ entry_id: string; value: string }>(dbFile, + `SELECT e.id as entry_id, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.id IN (${idList}) + AND f.object_id = '${sqlEscape(relObj.id)}' + AND f.name = '${sqlEscape(displayFieldName)}'`, + ); + + const labelMap: Record = {}; + for (const row of displayRows) { + labelMap[row.entry_id] = row.value || row.entry_id; + } + for (const i of ids) { + if (!labelMap[i]) { + labelMap[i] = i; + } + } + relationLabels[rf.name] = labelMap; + } + + // Enrich fields with related object names + const enrichedFields = parsedFields.map((f) => ({ + ...f, + related_object_name: + f.type === "relation" + ? relatedObjectNames[f.name] + : undefined, + })); + + // Find reverse relations for this entry (search across all DBs) + const reverseRelations = findReverseRelationsForEntry(obj.id, id); + + const effectiveDisplayField = resolveDisplayField(obj, fields); + + return Response.json({ + object: obj, + fields: enrichedFields, + entry, + relationLabels, + reverseRelations, + effectiveDisplayField, + }); +} + +/** + * PATCH /api/workspace/objects/[name]/entries/[id] + * Update field values for an entry. + * Body: { fields: { [fieldName]: newValue } } + */ +export async function PATCH( + req: Request, + { params }: { params: Promise<{ name: string; id: string }> }, +) { + const { name, id } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + // Find object + const objects = q<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Verify entry exists + const exists = q<{ cnt: number }>(dbFile, + `SELECT COUNT(*) as cnt FROM entries WHERE id = '${sqlEscape(id)}' AND object_id = '${sqlEscape(objectId)}'`, + ); + if (!exists[0] || exists[0].cnt === 0) { + return Response.json( + { error: "Entry not found" }, + { status: 404 }, + ); + } + + const body = await req.json(); + const fieldUpdates: Record = body.fields ?? {}; + + // Get field IDs by name + const dbFields = q<{ id: string; name: string }>(dbFile, + `SELECT id, name FROM fields WHERE object_id = '${sqlEscape(objectId)}'`, + ); + const fieldMap = new Map(dbFields.map((f) => [f.name, f.id])); + + let updatedCount = 0; + for (const [fieldName, value] of Object.entries(fieldUpdates)) { + const fieldId = fieldMap.get(fieldName); + if (!fieldId) {continue;} + + const escapedValue = + value == null ? "NULL" : `'${sqlEscape(String(value))}'`; + + // Try update first, then insert if no rows affected + const existingRows = q<{ cnt: number }>(dbFile, + `SELECT COUNT(*) as cnt FROM entry_fields WHERE entry_id = '${sqlEscape(id)}' AND field_id = '${sqlEscape(fieldId)}'`, + ); + + if (existingRows[0]?.cnt > 0) { + duckdbExecOnFile(dbFile, + `UPDATE entry_fields SET value = ${escapedValue} WHERE entry_id = '${sqlEscape(id)}' AND field_id = '${sqlEscape(fieldId)}'`, + ); + } else { + duckdbExecOnFile(dbFile, + `INSERT INTO entry_fields (entry_id, field_id, value) VALUES ('${sqlEscape(id)}', '${sqlEscape(fieldId)}', ${escapedValue})`, + ); + } + updatedCount++; + } + + // Touch updated_at on the entry + const now = new Date().toISOString(); + duckdbExecOnFile(dbFile, + `UPDATE entries SET updated_at = '${now}' WHERE id = '${sqlEscape(id)}'`, + ); + + return Response.json({ ok: true, updatedCount }); +} + +/** + * DELETE /api/workspace/objects/[name]/entries/[id] + * Delete a single entry and its field values. + */ +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ name: string; id: string }> }, +) { + const { name, id } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + // Find object + const objects = q<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Delete field values first, then entry + duckdbExecOnFile(dbFile, + `DELETE FROM entry_fields WHERE entry_id = '${sqlEscape(id)}'`, + ); + duckdbExecOnFile(dbFile, + `DELETE FROM entries WHERE id = '${sqlEscape(id)}' AND object_id = '${sqlEscape(objectId)}'`, + ); + + return Response.json({ ok: true }); +} + +// --- Reverse relations for a single entry --- + +type ReverseRelation = { + fieldName: string; + sourceObjectName: string; + sourceObjectId: string; + displayField: string; + links: Array<{ id: string; label: string }>; +}; + +/** + * Find reverse relations for a single entry, searching across ALL discovered databases. + */ +function findReverseRelationsForEntry( + objectId: string, + entryId: string, +): ReverseRelation[] { + const dbPaths = discoverDuckDBPaths(); + const result: ReverseRelation[] = []; + + for (const db of dbPaths) { + const reverseFields = q<{ + id: string; + name: string; + object_id: string; + source_object_name: string; + }>(db, + `SELECT f.id, f.name, f.object_id, o.name as source_object_name + FROM fields f + JOIN objects o ON o.id = f.object_id + WHERE f.type = 'relation' + AND f.related_object_id = '${sqlEscape(objectId)}'`, + ); + + for (const rrf of reverseFields) { + const refRows = q<{ + source_entry_id: string; + target_value: string; + }>(db, + `SELECT ef.entry_id as source_entry_id, ef.value as target_value + FROM entry_fields ef + WHERE ef.field_id = '${sqlEscape(rrf.id)}' + AND ef.value IS NOT NULL + AND ef.value != ''`, + ); + + const matchingSourceIds: string[] = []; + for (const row of refRows) { + const targetIds = parseRelationValue(row.target_value); + if (targetIds.includes(entryId)) { + matchingSourceIds.push(row.source_entry_id); + } + } + + if (matchingSourceIds.length === 0) { + continue; + } + + const sourceObj = q(db, + `SELECT * FROM objects WHERE id = '${sqlEscape(rrf.object_id)}' LIMIT 1`, + ); + if (sourceObj.length === 0) { + continue; + } + + const sourceFields = q(db, + `SELECT * FROM fields WHERE object_id = '${sqlEscape(rrf.object_id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField(sourceObj[0], sourceFields); + + const idList = matchingSourceIds + .map((i) => `'${sqlEscape(i)}'`) + .join(","); + const displayRows = q<{ entry_id: string; value: string }>(db, + `SELECT ef.entry_id, ef.value + FROM entry_fields ef + JOIN fields f ON f.id = ef.field_id + WHERE ef.entry_id IN (${idList}) + AND f.name = '${sqlEscape(displayFieldName)}' + AND f.object_id = '${sqlEscape(rrf.object_id)}'`, + ); + + const displayMap: Record = {}; + for (const row of displayRows) { + displayMap[row.entry_id] = row.value || row.entry_id; + } + + const links = matchingSourceIds.map((sid) => ({ + id: sid, + label: displayMap[sid] || sid, + })); + + result.push({ + fieldName: rrf.name, + sourceObjectName: rrf.source_object_name, + sourceObjectId: rrf.object_id, + displayField: displayFieldName, + links, + }); + } + } + + return result; +} diff --git a/apps/web/app/api/workspace/objects/[name]/entries/bulk-delete/route.ts b/apps/web/app/api/workspace/objects/[name]/entries/bulk-delete/route.ts new file mode 100644 index 00000000000..26a99459867 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/entries/bulk-delete/route.ts @@ -0,0 +1,74 @@ +import { duckdbExecOnFile, duckdbQueryOnFile, findDuckDBForObject } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * POST /api/workspace/objects/[name]/entries/bulk-delete + * Delete multiple entries at once. + * Body: { entryIds: string[] } + */ +export async function POST( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + const body = await req.json(); + const entryIds: string[] = body.entryIds; + + if (!Array.isArray(entryIds) || entryIds.length === 0) { + return Response.json( + { error: "entryIds must be a non-empty array" }, + { status: 400 }, + ); + } + + // Validate object exists + const objects = duckdbQueryOnFile<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + const idList = entryIds + .map((id) => `'${sqlEscape(id)}'`) + .join(","); + + // Delete field values first, then entries + duckdbExecOnFile(dbFile, + `DELETE FROM entry_fields WHERE entry_id IN (${idList})`, + ); + duckdbExecOnFile(dbFile, + `DELETE FROM entries WHERE id IN (${idList}) AND object_id = '${sqlEscape(objectId)}'`, + ); + + return Response.json({ + ok: true, + deletedCount: entryIds.length, + }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/entries/options/route.ts b/apps/web/app/api/workspace/objects/[name]/entries/options/route.ts new file mode 100644 index 00000000000..5f332e31e62 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/entries/options/route.ts @@ -0,0 +1,91 @@ +import { duckdbQueryOnFile, findDuckDBForObject } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type ObjectRow = { + id: string; + name: string; + display_field?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; +}; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +function resolveDisplayField( + obj: ObjectRow, + fields: FieldRow[], +): string { + if (obj.display_field) {return obj.display_field;} + const nameField = fields.find( + (f) => /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) {return nameField.name;} + const textField = fields.find((f) => f.type === "text"); + if (textField) {return textField.name;} + return fields[0]?.name ?? "id"; +} + +/** + * GET /api/workspace/objects/[name]/entries/options + * Returns lightweight { options: [{ id, label }] } for relation dropdowns. + * Supports optional ?q= search parameter. + */ +export async function GET( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json({ error: "Invalid object name" }, { status: 400 }); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json({ error: "DuckDB not found" }, { status: 404 }); + } + + const objects = duckdbQueryOnFile(dbFile, + `SELECT * FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json({ error: `Object '${name}' not found` }, { status: 404 }); + } + const obj = objects[0]; + + const fields = duckdbQueryOnFile(dbFile, + `SELECT * FROM fields WHERE object_id = '${sqlEscape(obj.id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField(obj, fields); + + // Optional search filter + const url = new URL(req.url); + const query = url.searchParams.get("q")?.trim() ?? ""; + + // Fetch entries with their display field value + const rows = duckdbQueryOnFile<{ entry_id: string; label: string | null }>(dbFile, + `SELECT e.id as entry_id, ef.value as label + FROM entries e + LEFT JOIN entry_fields ef ON ef.entry_id = e.id + LEFT JOIN fields f ON f.id = ef.field_id AND f.name = '${sqlEscape(displayFieldName)}' + WHERE e.object_id = '${sqlEscape(obj.id)}' + ${query ? `AND (ef.value IS NOT NULL AND LOWER(ef.value) LIKE '%${sqlEscape(query.toLowerCase())}%')` : ""} + ORDER BY ef.value ASC NULLS LAST + LIMIT 200`, + ); + + const options = rows.map((r) => ({ + id: r.entry_id, + label: r.label || r.entry_id, + })); + + return Response.json({ options, displayField: displayFieldName }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/entries/route.ts b/apps/web/app/api/workspace/objects/[name]/entries/route.ts new file mode 100644 index 00000000000..3cf5c962526 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/entries/route.ts @@ -0,0 +1,97 @@ +import { duckdbExecOnFile, duckdbQueryOnFile, findDuckDBForObject } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * POST /api/workspace/objects/[name]/entries + * Create a new entry with optional field values. + * Body: { fields?: Record } + */ +export async function POST( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + // Find object + const objects = duckdbQueryOnFile<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Generate UUID for the new entry + const idRows = duckdbQueryOnFile<{ id: string }>(dbFile, + "SELECT uuid()::VARCHAR as id", + ); + const entryId = idRows[0]?.id; + if (!entryId) { + return Response.json( + { error: "Failed to generate UUID" }, + { status: 500 }, + ); + } + + // Create entry + const now = new Date().toISOString(); + const ok = duckdbExecOnFile(dbFile, + `INSERT INTO entries (id, object_id, created_at, updated_at) VALUES ('${sqlEscape(entryId)}', '${sqlEscape(objectId)}', '${now}', '${now}')`, + ); + if (!ok) { + return Response.json( + { error: "Failed to create entry" }, + { status: 500 }, + ); + } + + // Insert field values if provided + let body: { fields?: Record } = {}; + try { + body = await req.json(); + } catch { + // no body is fine + } + + if (body.fields && typeof body.fields === "object") { + // Get field IDs by name + const dbFields = duckdbQueryOnFile<{ id: string; name: string }>(dbFile, + `SELECT id, name FROM fields WHERE object_id = '${sqlEscape(objectId)}'`, + ); + const fieldMap = new Map(dbFields.map((f) => [f.name, f.id])); + + for (const [fieldName, value] of Object.entries(body.fields)) { + const fieldId = fieldMap.get(fieldName); + if (!fieldId || value == null) {continue;} + duckdbExecOnFile(dbFile, + `INSERT INTO entry_fields (entry_id, field_id, value) VALUES ('${sqlEscape(entryId)}', '${sqlEscape(fieldId)}', '${sqlEscape(String(value))}')`, + ); + } + } + + return Response.json({ entryId, ok: true }, { status: 201 }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/enum-rename/route.ts b/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/enum-rename/route.ts new file mode 100644 index 00000000000..b5d07c14a22 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/enum-rename/route.ts @@ -0,0 +1,116 @@ +import { duckdbExecOnFile, duckdbQueryOnFile, findDuckDBForObject } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * PATCH /api/workspace/objects/[name]/fields/[fieldId]/enum-rename + * Rename an enum value across the field definition and all entries. + * Body: { oldValue: string, newValue: string } + */ +export async function PATCH( + req: Request, + { + params, + }: { params: Promise<{ name: string; fieldId: string }> }, +) { + const { name, fieldId } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + const body = await req.json(); + const oldValue: string = body.oldValue; + const newValue: string = body.newValue; + + if (!oldValue || !newValue || typeof oldValue !== "string" || typeof newValue !== "string") { + return Response.json( + { error: "oldValue and newValue are required" }, + { status: 400 }, + ); + } + if (oldValue.trim() === newValue.trim()) { + return Response.json({ ok: true, changed: 0 }); + } + + // Validate object exists + const objects = duckdbQueryOnFile<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Validate field exists and is an enum + const fields = duckdbQueryOnFile<{ id: string; enum_values: string | null; enum_colors: string | null }>(dbFile, + `SELECT id, enum_values, enum_colors FROM fields WHERE id = '${sqlEscape(fieldId)}' AND object_id = '${sqlEscape(objectId)}'`, + ); + if (fields.length === 0) { + return Response.json( + { error: "Field not found" }, + { status: 404 }, + ); + } + + const field = fields[0]; + let enumValues: string[]; + try { + enumValues = field.enum_values ? JSON.parse(field.enum_values) : []; + } catch { + return Response.json( + { error: "Invalid enum_values in field" }, + { status: 500 }, + ); + } + + const idx = enumValues.indexOf(oldValue.trim()); + if (idx === -1) { + return Response.json( + { error: `Enum value '${oldValue}' not found` }, + { status: 404 }, + ); + } + + // Check for duplicate + if (enumValues.includes(newValue.trim())) { + return Response.json( + { error: `Enum value '${newValue}' already exists` }, + { status: 409 }, + ); + } + + // Update enum_values array + enumValues[idx] = newValue.trim(); + const newEnumJson = JSON.stringify(enumValues); + + duckdbExecOnFile(dbFile, + `UPDATE fields SET enum_values = '${sqlEscape(newEnumJson)}' WHERE id = '${sqlEscape(fieldId)}'`, + ); + + // Update all entry_fields with the old value to the new value + const updatedEntries = duckdbExecOnFile(dbFile, + `UPDATE entry_fields SET value = '${sqlEscape(newValue.trim())}' WHERE field_id = '${sqlEscape(fieldId)}' AND value = '${sqlEscape(oldValue.trim())}'`, + ); + + return Response.json({ ok: true, updated: updatedEntries }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/route.ts b/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/route.ts new file mode 100644 index 00000000000..595a1c11b1e --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/route.ts @@ -0,0 +1,98 @@ +import { duckdbExecOnFile, duckdbQueryOnFile, findDuckDBForObject } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * PATCH /api/workspace/objects/[name]/fields/[fieldId] + * Rename a field. + * Body: { name: string } + */ +export async function PATCH( + req: Request, + { + params, + }: { params: Promise<{ name: string; fieldId: string }> }, +) { + const { name, fieldId } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + const body = await req.json(); + const newName: string = body.name; + + if ( + !newName || + typeof newName !== "string" || + newName.trim().length === 0 + ) { + return Response.json( + { error: "Name is required" }, + { status: 400 }, + ); + } + + // Validate object exists + const objects = duckdbQueryOnFile<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Validate field exists and belongs to this object + const fieldExists = duckdbQueryOnFile<{ cnt: number }>(dbFile, + `SELECT COUNT(*) as cnt FROM fields WHERE id = '${sqlEscape(fieldId)}' AND object_id = '${sqlEscape(objectId)}'`, + ); + if (!fieldExists[0] || fieldExists[0].cnt === 0) { + return Response.json( + { error: "Field not found" }, + { status: 404 }, + ); + } + + // Check for duplicate name + const duplicateCheck = duckdbQueryOnFile<{ cnt: number }>(dbFile, + `SELECT COUNT(*) as cnt FROM fields WHERE object_id = '${sqlEscape(objectId)}' AND name = '${sqlEscape(newName.trim())}' AND id != '${sqlEscape(fieldId)}'`, + ); + if (duplicateCheck[0]?.cnt > 0) { + return Response.json( + { error: "A field with that name already exists" }, + { status: 409 }, + ); + } + + const ok = duckdbExecOnFile(dbFile, + `UPDATE fields SET name = '${sqlEscape(newName.trim())}' WHERE id = '${sqlEscape(fieldId)}'`, + ); + + if (!ok) { + return Response.json( + { error: "Failed to rename field" }, + { status: 500 }, + ); + } + + return Response.json({ ok: true }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/fields/reorder/route.ts b/apps/web/app/api/workspace/objects/[name]/fields/reorder/route.ts new file mode 100644 index 00000000000..f5818aec715 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/fields/reorder/route.ts @@ -0,0 +1,66 @@ +import { duckdbExecOnFile, duckdbQueryOnFile, findDuckDBForObject } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * PATCH /api/workspace/objects/[name]/fields/reorder + * Reorder fields by updating sort_order. + * Body: { fieldOrder: string[] } β€” array of field IDs in desired order + */ +export async function PATCH( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + const body = await req.json(); + const fieldOrder: string[] = body.fieldOrder; + + if (!Array.isArray(fieldOrder) || fieldOrder.length === 0) { + return Response.json( + { error: "fieldOrder must be a non-empty array" }, + { status: 400 }, + ); + } + + // Validate object exists + const objects = duckdbQueryOnFile<{ id: string }>(dbFile, + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Update sort_order for each field + for (let i = 0; i < fieldOrder.length; i++) { + duckdbExecOnFile(dbFile, + `UPDATE fields SET sort_order = ${i} WHERE id = '${sqlEscape(fieldOrder[i])}' AND object_id = '${sqlEscape(objectId)}'`, + ); + } + + return Response.json({ ok: true }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/route.ts b/apps/web/app/api/workspace/objects/[name]/route.ts new file mode 100644 index 00000000000..d5ffd5619b8 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/route.ts @@ -0,0 +1,491 @@ +import { duckdbPath, parseRelationValue, resolveDuckdbBin, findDuckDBForObject, duckdbQueryOnFile, discoverDuckDBPaths, getObjectViews } from "@/lib/workspace"; +import { deserializeFilters, buildWhereClause, buildOrderByClause, type FieldMeta } from "@/lib/object-filters"; +import { execSync } from "node:child_process"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type ObjectRow = { + id: string; + name: string; + description?: string; + icon?: string; + default_view?: string; + display_field?: string; + immutable?: boolean; + created_at?: string; + updated_at?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; + description?: string; + required?: boolean; + enum_values?: string; + enum_colors?: string; + enum_multiple?: boolean; + related_object_id?: string; + relationship_type?: string; + sort_order?: number; +}; + +type StatusRow = { + id: string; + name: string; + color?: string; + sort_order?: number; + is_default?: boolean; +}; + +type EavRow = { + entry_id: string; + created_at: string; + updated_at: string; + field_name: string; + value: string | null; +}; + +// --- Schema migration (idempotent, runs once per process) --- + +const migratedDbs = new Set(); + +/** Ensure the display_field column exists on a specific DB file. */ +function ensureDisplayFieldColumn(dbFile: string) { + if (migratedDbs.has(dbFile)) {return;} + const bin = resolveDuckdbBin(); + if (!bin) {return;} + try { + execSync( + `'${bin}' '${dbFile}' 'ALTER TABLE objects ADD COLUMN IF NOT EXISTS display_field VARCHAR'`, + { encoding: "utf-8", timeout: 5_000, shell: "/bin/sh" }, + ); + } catch { + // migration might fail on DBs that don't have the objects table β€” skip + } + migratedDbs.add(dbFile); +} + +// --- Helpers --- + +/** Scoped query helper: queries a specific DB file. */ +function q>(dbFile: string, sql: string): T[] { + return duckdbQueryOnFile(dbFile, sql); +} + +/** + * Pivot raw EAV rows into one object per entry with field names as keys. + */ +function pivotEavRows(rows: EavRow[]): Record[] { + const grouped = new Map>(); + + for (const row of rows) { + let entry = grouped.get(row.entry_id); + if (!entry) { + entry = { + entry_id: row.entry_id, + created_at: row.created_at, + updated_at: row.updated_at, + }; + grouped.set(row.entry_id, entry); + } + if (row.field_name) { + entry[row.field_name] = row.value; + } + } + + return Array.from(grouped.values()); +} + +function tryParseJson(value: unknown): unknown { + if (typeof value !== "string") {return value;} + try { + return JSON.parse(value); + } catch { + return value; + } +} + +/** SQL-escape a string (double single-quotes). */ +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * Determine the display field for an object. + * Priority: explicit display_field > heuristic (name/title) > first text field > first field. + */ +function resolveDisplayField( + obj: ObjectRow, + objFields: FieldRow[], +): string { + if (obj.display_field) {return obj.display_field;} + + // Heuristic: look for name/title fields + const nameField = objFields.find( + (f) => + /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) {return nameField.name;} + + // Fallback: first text field + const textField = objFields.find((f) => f.type === "text"); + if (textField) {return textField.name;} + + // Ultimate fallback: first field + return objFields[0]?.name ?? "id"; +} + +/** + * Resolve relation field values to human-readable display labels. + * All queries target the same DB file where the object lives. + */ +function resolveRelationLabels( + dbFile: string, + fields: FieldRow[], + entries: Record[], +): { + labels: Record>; + relatedObjectNames: Record; +} { + const labels: Record> = {}; + const relatedObjectNames: Record = {}; + + const relationFields = fields.filter( + (f) => f.type === "relation" && f.related_object_id, + ); + + for (const rf of relationFields) { + const relatedObjs = q(dbFile, + `SELECT * FROM objects WHERE id = '${sqlEscape(rf.related_object_id!)}' LIMIT 1`, + ); + if (relatedObjs.length === 0) {continue;} + const relObj = relatedObjs[0]; + relatedObjectNames[rf.name] = relObj.name; + + const relFields = q(dbFile, + `SELECT * FROM fields WHERE object_id = '${sqlEscape(relObj.id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField(relObj, relFields); + + const entryIds = new Set(); + for (const entry of entries) { + const val = entry[rf.name]; + if (val == null || val === "") { + continue; + } + const valStr = + typeof val === "object" && val !== null + ? JSON.stringify(val) + : typeof val === "string" + ? val + : typeof val === "number" || typeof val === "boolean" + ? String(val) + : ""; + for (const id of parseRelationValue(valStr)) { + entryIds.add(id); + } + } + + if (entryIds.size === 0) { + labels[rf.name] = {}; + continue; + } + + const idList = Array.from(entryIds) + .map((id) => `'${sqlEscape(id)}'`) + .join(","); + const displayRows = q<{ entry_id: string; value: string }>(dbFile, + `SELECT e.id as entry_id, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.id IN (${idList}) + AND f.object_id = '${sqlEscape(relObj.id)}' + AND f.name = '${sqlEscape(displayFieldName)}'`, + ); + + const labelMap: Record = {}; + for (const row of displayRows) { + labelMap[row.entry_id] = row.value || row.entry_id; + } + for (const id of entryIds) { + if (!labelMap[id]) {labelMap[id] = id;} + } + + labels[rf.name] = labelMap; + } + + return { labels, relatedObjectNames }; +} + +type ReverseRelation = { + fieldName: string; + sourceObjectName: string; + sourceObjectId: string; + displayField: string; + entries: Record>; +}; + +/** + * Find reverse relations: other objects with relation fields pointing TO this object. + * Searches across ALL discovered databases to catch cross-DB relations. + */ +function findReverseRelations(objectId: string): ReverseRelation[] { + const dbPaths = discoverDuckDBPaths(); + const result: ReverseRelation[] = []; + + for (const db of dbPaths) { + const reverseFields = q< + FieldRow & { source_object_id: string; source_object_name: string } + >(db, + `SELECT f.*, f.object_id as source_object_id, o.name as source_object_name + FROM fields f + JOIN objects o ON o.id = f.object_id + WHERE f.type = 'relation' + AND f.related_object_id = '${sqlEscape(objectId)}'`, + ); + + for (const rrf of reverseFields) { + const sourceObjs = q(db, + `SELECT * FROM objects WHERE id = '${sqlEscape(rrf.source_object_id)}' LIMIT 1`, + ); + if (sourceObjs.length === 0) {continue;} + + const sourceFields = q(db, + `SELECT * FROM fields WHERE object_id = '${sqlEscape(rrf.source_object_id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField(sourceObjs[0], sourceFields); + + const refRows = q<{ source_entry_id: string; target_value: string }>(db, + `SELECT ef.entry_id as source_entry_id, ef.value as target_value + FROM entry_fields ef + WHERE ef.field_id = '${sqlEscape(rrf.id)}' + AND ef.value IS NOT NULL + AND ef.value != ''`, + ); + + if (refRows.length === 0) {continue;} + + const sourceEntryIds = [...new Set(refRows.map((r) => r.source_entry_id))]; + const idList = sourceEntryIds.map((id) => `'${sqlEscape(id)}'`).join(","); + const displayRows = q<{ entry_id: string; value: string }>(db, + `SELECT ef.entry_id, ef.value + FROM entry_fields ef + JOIN fields f ON f.id = ef.field_id + WHERE ef.entry_id IN (${idList}) + AND f.name = '${sqlEscape(displayFieldName)}' + AND f.object_id = '${sqlEscape(rrf.source_object_id)}'`, + ); + + const displayMap: Record = {}; + for (const row of displayRows) { + displayMap[row.entry_id] = row.value || row.entry_id; + } + + const entriesMap: Record> = {}; + for (const row of refRows) { + const targetIds = parseRelationValue(row.target_value); + for (const targetId of targetIds) { + if (!entriesMap[targetId]) {entriesMap[targetId] = [];} + entriesMap[targetId].push({ + id: row.source_entry_id, + label: displayMap[row.source_entry_id] || row.source_entry_id, + }); + } + } + + result.push({ + fieldName: rrf.name, + sourceObjectName: rrf.source_object_name, + sourceObjectId: rrf.source_object_id, + displayField: displayFieldName, + entries: entriesMap, + }); + } + } + + return result; +} + +// --- Route handler --- + +export async function GET( + _req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!resolveDuckdbBin()) { + return Response.json( + { error: "DuckDB CLI is not installed", code: "DUCKDB_NOT_INSTALLED" }, + { status: 503 }, + ); + } + + // Sanitize name to prevent injection (only allow alphanumeric + underscore + hyphen) + if (!/^[a-zA-Z_][a-zA-Z0-9_-]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + // Find which DuckDB file contains this object (searches all discovered DBs) + const dbFile = findDuckDBForObject(name); + if (!dbFile) { + // Fall back to primary DB check for a friendlier error message + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB database not found" }, + { status: 404 }, + ); + } + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + + // Ensure display_field column exists on this specific DB + ensureDisplayFieldColumn(dbFile); + + // All queries below target the specific DB that owns this object + const objects = q(dbFile, + `SELECT * FROM objects WHERE name = '${name}' LIMIT 1`, + ); + + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + + const obj = objects[0]; + + const fields = q(dbFile, + `SELECT * FROM fields WHERE object_id = '${obj.id}' ORDER BY sort_order`, + ); + + const statuses = q(dbFile, + `SELECT * FROM statuses WHERE object_id = '${obj.id}' ORDER BY sort_order`, + ); + + // --- Parse filter/sort/pagination query params --- + const url = new URL(_req.url); + const filtersParam = url.searchParams.get("filters"); + const sortParam = url.searchParams.get("sort"); + const searchParam = url.searchParams.get("search"); + const pageParam = url.searchParams.get("page"); + const pageSizeParam = url.searchParams.get("pageSize"); + + const filterGroup = filtersParam ? deserializeFilters(filtersParam) : undefined; + const fieldsMeta: FieldMeta[] = fields.map((f) => ({ name: f.name, type: f.type })); + + // Build WHERE clause from filters + let whereClause = ""; + if (filterGroup) { + const where = buildWhereClause(filterGroup, fieldsMeta); + if (where) {whereClause = ` WHERE ${where}`;} + } + + // Build ORDER BY clause + let orderByClause = " ORDER BY created_at DESC"; + if (sortParam) { + try { + const sortRules = JSON.parse(sortParam); + const orderBy = buildOrderByClause(sortRules); + if (orderBy) {orderByClause = ` ORDER BY ${orderBy}`;} + } catch { + // keep default sort + } + } + + // Pagination + const page = Math.max(1, Number(pageParam) || 1); + const pageSize = Math.min(5000, Math.max(1, Number(pageSizeParam) || 100)); + const offset = (page - 1) * pageSize; + const limitClause = ` LIMIT ${pageSize} OFFSET ${offset}`; + + // Full-text search across text fields + if (searchParam && searchParam.trim()) { + const textFields = fields.filter((f) => ["text", "richtext", "email"].includes(f.type)); + if (textFields.length > 0) { + const searchConditions = textFields + .map((f) => `LOWER(CAST("${f.name.replace(/"/g, '""')}" AS VARCHAR)) LIKE '%${sqlEscape(searchParam.toLowerCase())}%'`) + .join(" OR "); + whereClause = whereClause + ? `${whereClause} AND (${searchConditions})` + : ` WHERE (${searchConditions})`; + } + } + + // Try the PIVOT view first, then fall back to raw EAV query + client-side pivot + let entries: Record[] = []; + let totalCount = 0; + + try { + // Get total count with same WHERE clause but no LIMIT/OFFSET + const countResult = q<{ cnt: number }>(dbFile, + `SELECT COUNT(*) as cnt FROM v_${name}${whereClause}`, + ); + totalCount = countResult[0]?.cnt ?? 0; + + const pivotEntries = q(dbFile, + `SELECT * FROM v_${name}${whereClause}${orderByClause}${limitClause}`, + ); + entries = pivotEntries; + } catch { + // Pivot view might not exist or filter SQL may not apply; fall back + const rawRows = q(dbFile, + `SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = '${obj.id}' + ORDER BY e.created_at DESC + LIMIT 5000`, + ); + entries = pivotEavRows(rawRows); + } + + const parsedFields = fields.map((f) => ({ + ...f, + enum_values: f.enum_values ? tryParseJson(f.enum_values) : undefined, + enum_colors: f.enum_colors ? tryParseJson(f.enum_colors) : undefined, + })); + + const { labels: relationLabels, relatedObjectNames } = + resolveRelationLabels(dbFile, fields, entries); + + const enrichedFields = parsedFields.map((f) => ({ + ...f, + related_object_name: + f.type === "relation" ? relatedObjectNames[f.name] : undefined, + })); + + const reverseRelations = findReverseRelations(obj.id); + + const effectiveDisplayField = resolveDisplayField(obj, fields); + + // Include saved views from .object.yaml + const { views: savedViews, activeView } = getObjectViews(name); + + return Response.json({ + object: obj, + fields: enrichedFields, + statuses, + entries, + relationLabels, + reverseRelations, + effectiveDisplayField, + savedViews, + activeView, + totalCount, + page, + pageSize, + }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/views/route.ts b/apps/web/app/api/workspace/objects/[name]/views/route.ts new file mode 100644 index 00000000000..32e6ae53b0b --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/views/route.ts @@ -0,0 +1,61 @@ +import { NextResponse } from "next/server"; +import { getObjectViews, saveObjectViews } from "@/lib/workspace"; +import type { SavedView } from "@/lib/object-filters"; + +type Params = { params: Promise<{ name: string }> }; + +/** + * GET /api/workspace/objects/[name]/views + * + * Returns saved views and active_view from the object's .object.yaml. + */ +export async function GET(_req: Request, ctx: Params) { + const { name } = await ctx.params; + const objectName = decodeURIComponent(name); + + try { + const { views, activeView } = getObjectViews(objectName); + return NextResponse.json({ views, activeView }); + } catch (err) { + return NextResponse.json( + { error: `Failed to read views: ${err instanceof Error ? err.message : String(err)}` }, + { status: 500 }, + ); + } +} + +/** + * PUT /api/workspace/objects/[name]/views + * + * Save views and active_view to the object's .object.yaml. + * Body: { views: SavedView[], activeView?: string } + */ +export async function PUT(req: Request, ctx: Params) { + const { name } = await ctx.params; + const objectName = decodeURIComponent(name); + + try { + const body = (await req.json()) as { + views?: SavedView[]; + activeView?: string; + }; + + const views = body.views ?? []; + const activeView = body.activeView; + + const ok = saveObjectViews(objectName, views, activeView); + if (!ok) { + return NextResponse.json( + { error: "Object directory not found" }, + { status: 404 }, + ); + } + + return NextResponse.json({ ok: true }); + } catch (err) { + return NextResponse.json( + { error: `Failed to save views: ${err instanceof Error ? err.message : String(err)}` }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/open-file/route.ts b/apps/web/app/api/workspace/open-file/route.ts new file mode 100644 index 00000000000..402f0e2f527 --- /dev/null +++ b/apps/web/app/api/workspace/open-file/route.ts @@ -0,0 +1,97 @@ +import { exec } from "node:child_process"; +import { existsSync } from "node:fs"; +import { resolve, normalize } from "node:path"; +import { homedir } from "node:os"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/open-file + * Opens a file or directory using the system's default application. + * On macOS this uses `open`, on Linux `xdg-open`. + */ +export async function POST(req: Request) { + let body: { path?: string; reveal?: boolean }; + try { + body = await req.json(); + } catch { + return Response.json( + { error: "Invalid JSON body" }, + { status: 400 }, + ); + } + + const rawPath = body.path; + if (!rawPath || typeof rawPath !== "string") { + return Response.json( + { error: "Missing 'path' in request body" }, + { status: 400 }, + ); + } + + // Expand ~ to home directory + const expanded = rawPath.startsWith("~/") + ? rawPath.replace(/^~/, homedir()) + : rawPath; + + let resolved = resolve(normalize(expanded)); + + // If the file doesn't exist and looks like a bare filename, try to locate it + // using macOS Spotlight (mdfind). + if (!existsSync(resolved) && !rawPath.includes("/")) { + const found = await new Promise((res) => { + exec( + `mdfind -name ${JSON.stringify(rawPath)} | head -1`, + (err, stdout) => { + if (err || !stdout.trim()) {res(null);} + else {res(stdout.trim().split("\n")[0]);} + }, + ); + }); + if (found && existsSync(found)) { + resolved = found; + } + } + + if (!existsSync(resolved)) { + return Response.json( + { error: "File not found", path: resolved }, + { status: 404 }, + ); + } + + const platform = process.platform; + const reveal = body.reveal === true; + + let cmd: string; + if (platform === "darwin") { + // macOS: use `open` β€” `-R` reveals in Finder instead of opening + cmd = reveal + ? `open -R ${JSON.stringify(resolved)}` + : `open ${JSON.stringify(resolved)}`; + } else if (platform === "linux") { + // Linux: xdg-open (no reveal equivalent) + cmd = `xdg-open ${JSON.stringify(resolved)}`; + } else { + return Response.json( + { error: `Unsupported platform: ${platform}` }, + { status: 400 }, + ); + } + + return new Promise((res) => { + exec(cmd, (error) => { + if (error) { + res( + Response.json( + { error: `Failed to open file: ${error.message}` }, + { status: 500 }, + ), + ); + } else { + res(Response.json({ ok: true, path: resolved })); + } + }); + }); +} diff --git a/apps/web/app/api/workspace/path-info/route.ts b/apps/web/app/api/workspace/path-info/route.ts new file mode 100644 index 00000000000..e06c0e60469 --- /dev/null +++ b/apps/web/app/api/workspace/path-info/route.ts @@ -0,0 +1,88 @@ +import { exec } from "node:child_process"; +import { existsSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, normalize, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * GET /api/workspace/path-info?path=... + * Resolves and inspects a filesystem path for in-app preview routing. + */ +export async function GET(req: Request) { + const url = new URL(req.url); + const rawPath = url.searchParams.get("path"); + + if (!rawPath) { + return Response.json( + { error: "Missing 'path' query parameter" }, + { status: 400 }, + ); + } + + let candidatePath = rawPath; + + // Convert file:// URLs into local paths first. + if (candidatePath.startsWith("file://")) { + try { + candidatePath = fileURLToPath(candidatePath); + } catch { + return Response.json( + { error: "Invalid file URL" }, + { status: 400 }, + ); + } + } + + // Expand "~/..." to the current user's home directory. + const expandedPath = candidatePath.startsWith("~/") + ? candidatePath.replace(/^~/, homedir()) + : candidatePath; + let resolvedPath = resolve(normalize(expandedPath)); + + // If the path doesn't exist and looks like a bare filename, try to locate it + // using macOS Spotlight (mdfind). + if (!existsSync(resolvedPath) && !rawPath.includes("/")) { + const found = await new Promise((res) => { + exec( + `mdfind -name ${JSON.stringify(rawPath)} | head -1`, + (err, stdout) => { + if (err || !stdout.trim()) {res(null);} + else {res(stdout.trim().split("\n")[0]);} + }, + ); + }); + if (found && existsSync(found)) { + resolvedPath = found; + } + } + + if (!existsSync(resolvedPath)) { + return Response.json( + { error: "Path not found", path: resolvedPath }, + { status: 404 }, + ); + } + + try { + const stat = statSync(resolvedPath); + const type = stat.isDirectory() + ? "directory" + : stat.isFile() + ? "file" + : "other"; + + return Response.json({ + path: resolvedPath, + name: basename(resolvedPath) || resolvedPath, + type, + }); + } catch { + return Response.json( + { error: "Cannot stat path", path: resolvedPath }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/query/route.ts b/apps/web/app/api/workspace/query/route.ts new file mode 100644 index 00000000000..2df1dcc2962 --- /dev/null +++ b/apps/web/app/api/workspace/query/route.ts @@ -0,0 +1,43 @@ +import { duckdbQuery } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST(req: Request) { + let body: { sql?: string }; + try { + body = await req.json(); + } catch { + return Response.json( + { error: "Invalid JSON body" }, + { status: 400 }, + ); + } + + const { sql } = body; + if (!sql || typeof sql !== "string") { + return Response.json( + { error: "Missing 'sql' field in request body" }, + { status: 400 }, + ); + } + + // Basic SQL safety: reject obviously dangerous statements + const upper = sql.toUpperCase().trim(); + if ( + upper.startsWith("DROP") || + upper.startsWith("DELETE") || + upper.startsWith("INSERT") || + upper.startsWith("UPDATE") || + upper.startsWith("ALTER") || + upper.startsWith("CREATE") + ) { + return Response.json( + { error: "Only SELECT queries are allowed" }, + { status: 403 }, + ); + } + + const rows = duckdbQuery(sql); + return Response.json({ rows }); +} diff --git a/apps/web/app/api/workspace/raw-file/route.ts b/apps/web/app/api/workspace/raw-file/route.ts new file mode 100644 index 00000000000..25c0084fe13 --- /dev/null +++ b/apps/web/app/api/workspace/raw-file/route.ts @@ -0,0 +1,124 @@ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { safeResolvePath, resolveWorkspaceRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const MIME_MAP: Record = { + // Images + jpg: "image/jpeg", + jpeg: "image/jpeg", + png: "image/png", + gif: "image/gif", + webp: "image/webp", + svg: "image/svg+xml", + ico: "image/x-icon", + bmp: "image/bmp", + tiff: "image/tiff", + tif: "image/tiff", + avif: "image/avif", + heic: "image/heic", + heif: "image/heif", + // Video + mp4: "video/mp4", + webm: "video/webm", + mov: "video/quicktime", + avi: "video/x-msvideo", + mkv: "video/x-matroska", + // Audio + mp3: "audio/mpeg", + wav: "audio/wav", + ogg: "audio/ogg", + m4a: "audio/mp4", + // Documents + pdf: "application/pdf", + html: "text/html", + htm: "text/html", +}; + +/** + * Resolve a file path, trying multiple strategies: + * 1. Absolute path β€” the agent may read files from anywhere on the local machine + * (Photos library, Downloads, etc.), so we serve any readable absolute path. + * 2. Workspace-relative via safeResolvePath + * 3. Bare filename β€” search common workspace subdirectories + * + * Security note: this is a local-only dev server; it never runs in production. + */ +function resolveFile(path: string): string | null { + // 1. Absolute path β€” serve directly if it exists on disk + if (path.startsWith("/")) { + const abs = resolve(path); + if (existsSync(abs)) {return abs;} + // Fall through to workspace-relative in case the leading / is accidental + } + + // 2. Standard workspace-relative resolution + const resolved = safeResolvePath(path); + if (resolved) {return resolved;} + + // 3. Try common subdirectories in case the path is a bare filename + const root = resolveWorkspaceRoot(); + if (!root) {return null;} + const rootAbs = resolve(root); + const basename = path.split("/").pop() ?? path; + if (basename === path) { + const subdirs = [ + "assets", + "knowledge", + "manufacturing", + "uploads", + "files", + "images", + "media", + "reports", + "exports", + ]; + for (const sub of subdirs) { + const candidate = resolve(root, sub, basename); + if ( + candidate.startsWith(rootAbs) && + existsSync(candidate) + ) { + return candidate; + } + } + } + + return null; +} + +/** + * GET /api/workspace/raw-file?path=... + * Serves a workspace file with the correct Content-Type for inline display. + * Used by the chain-of-thought component to render images, videos, and PDFs. + */ +export async function GET(req: Request) { + const url = new URL(req.url); + const path = url.searchParams.get("path"); + + if (!path) { + return new Response("Missing path", { status: 400 }); + } + + const absolute = resolveFile(path); + if (!absolute) { + return new Response("Not found", { status: 404 }); + } + + const ext = path.split(".").pop()?.toLowerCase() ?? ""; + const contentType = MIME_MAP[ext] ?? "application/octet-stream"; + + try { + const buffer = readFileSync(absolute); + return new Response(buffer, { + headers: { + "Content-Type": contentType, + "Cache-Control": "public, max-age=3600", + }, + }); + } catch { + return new Response("Read error", { status: 500 }); + } +} diff --git a/apps/web/app/api/workspace/rename/route.ts b/apps/web/app/api/workspace/rename/route.ts new file mode 100644 index 00000000000..43066d2c316 --- /dev/null +++ b/apps/web/app/api/workspace/rename/route.ts @@ -0,0 +1,84 @@ +import { renameSync, existsSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { safeResolvePath, safeResolveNewPath, isSystemFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/rename + * Body: { path: string, newName: string } + * + * Renames a file or folder within the same directory. + * System files are protected from renaming. + */ +export async function POST(req: Request) { + let body: { path?: string; newName?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, newName } = body; + if (!relPath || typeof relPath !== "string" || !newName || typeof newName !== "string") { + return Response.json( + { error: "Missing 'path' and 'newName' fields" }, + { status: 400 }, + ); + } + + if (isSystemFile(relPath)) { + return Response.json( + { error: "Cannot rename system file" }, + { status: 403 }, + ); + } + + // Validate newName: no slashes, no empty, no traversal + if (newName.includes("/") || newName.includes("\\") || newName.trim() === "") { + return Response.json( + { error: "Invalid file name" }, + { status: 400 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "Source not found or path traversal rejected" }, + { status: 404 }, + ); + } + + const parentDir = dirname(absPath); + const newAbsPath = join(parentDir, newName); + + // Ensure the new path stays within workspace + const parentRel = dirname(relPath); + const newRelPath = parentRel === "." ? newName : `${parentRel}/${newName}`; + const validated = safeResolveNewPath(newRelPath); + if (!validated) { + return Response.json( + { error: "Invalid destination path" }, + { status: 400 }, + ); + } + + if (existsSync(newAbsPath)) { + return Response.json( + { error: `A file named '${newName}' already exists` }, + { status: 409 }, + ); + } + + try { + renameSync(absPath, newAbsPath); + return Response.json({ ok: true, oldPath: relPath, newPath: newRelPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Rename failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/reports/execute/route.ts b/apps/web/app/api/workspace/reports/execute/route.ts new file mode 100644 index 00000000000..fb6b8ed23a7 --- /dev/null +++ b/apps/web/app/api/workspace/reports/execute/route.ts @@ -0,0 +1,54 @@ +import { duckdbQuery } from "@/lib/workspace"; +import { buildFilterClauses, injectFilters, checkSqlSafety } from "@/lib/report-filters"; +import type { FilterEntry } from "@/lib/report-filters"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/reports/execute + * + * Body: { sql: string, filters?: FilterEntry[] } + * + * Executes a report panel's SQL query with optional filter injection. + * Only SELECT-compatible queries are allowed. + */ +export async function POST(req: Request) { + let body: { + sql?: string; + filters?: FilterEntry[]; + }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { sql, filters } = body; + if (!sql || typeof sql !== "string") { + return Response.json( + { error: "Missing 'sql' field in request body" }, + { status: 400 }, + ); + } + + // Basic SQL safety: reject mutation statements + const safetyError = checkSqlSafety(sql); + if (safetyError) { + return Response.json({ error: safetyError }, { status: 403 }); + } + + // Build filter clauses and inject into SQL + const filterClauses = buildFilterClauses(filters); + const finalSql = injectFilters(sql, filterClauses); + + try { + const rows = duckdbQuery(finalSql); + return Response.json({ rows, sql: finalSql }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Query execution failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/search-index/route.ts b/apps/web/app/api/workspace/search-index/route.ts new file mode 100644 index 00000000000..41902b942fa --- /dev/null +++ b/apps/web/app/api/workspace/search-index/route.ts @@ -0,0 +1,289 @@ +import { readdirSync, readFileSync, existsSync, type Dirent } from "node:fs"; +import { join } from "node:path"; +import { + resolveWorkspaceRoot, + parseSimpleYaml, + duckdbQueryAllAsync, + discoverDuckDBPaths, + duckdbQueryOnFileAsync, + isDatabaseFile, +} from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** Safely convert an unknown DB value to a display string. */ +function dbStr(val: unknown): string { + if (val == null) {return "";} + if (typeof val === "object") {return JSON.stringify(val);} + return String(val as string | number | boolean); +} + +// --- Types --- + +export type SearchIndexItem = { + /** Unique key: relative path for files, entryId for entries */ + id: string; + /** Primary display text (filename or display-field value) */ + label: string; + /** Secondary text (path for files, object name for entries) */ + sublabel?: string; + /** Item kind for grouping and icons */ + kind: "file" | "object" | "entry"; + /** Icon hint */ + icon?: string; + + // Entry-specific + objectName?: string; + entryId?: string; + /** First few field key-value pairs for search and preview */ + fields?: Record; + + // File/object-specific + path?: string; + nodeType?: "document" | "folder" | "file" | "report" | "database"; +}; + +// --- DB types --- + +type ObjectRow = { + id: string; + name: string; + description?: string; + icon?: string; + default_view?: string; + display_field?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; + sort_order?: number; +}; + +type EavRow = { + entry_id: string; + created_at: string; + updated_at: string; + field_name: string; + value: string | null; +}; + +// --- Helpers --- + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** Determine the display field (same heuristic as the objects route). */ +function resolveDisplayField(obj: ObjectRow, fields: FieldRow[]): string { + if (obj.display_field) {return obj.display_field;} + + const nameField = fields.find( + (f) => /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) {return nameField.name;} + + const textField = fields.find((f) => f.type === "text"); + if (textField) {return textField.name;} + + return fields[0]?.name ?? "id"; +} + +/** Flatten a tree recursively to produce file/object search items. */ +function flattenTree( + absDir: string, + relBase: string, + dbObjects: Map, + items: SearchIndexItem[], +) { + let entries: Dirent[]; + try { + entries = readdirSync(absDir, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (entry.name.startsWith(".")) {continue;} + + const absPath = join(absDir, entry.name); + const relPath = relBase ? `${relBase}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + const dbObj = dbObjects.get(entry.name); + // Check for .object.yaml + const yamlPath = join(absPath, ".object.yaml"); + const hasYaml = existsSync(yamlPath); + + if (dbObj || hasYaml) { + let icon: string | undefined; + if (hasYaml) { + try { + const parsed = parseSimpleYaml( + readFileSync(yamlPath, "utf-8"), + ); + icon = parsed.icon as string | undefined; + } catch { + // ignore + } + } + + items.push({ + id: relPath, + label: entry.name, + sublabel: relPath, + kind: "object", + icon: icon ?? dbObj?.icon, + path: relPath, + nodeType: undefined, + }); + } else { + // Regular folder -- don't add as item, but recurse + } + + flattenTree(absPath, relPath, dbObjects, items); + } else if (entry.isFile()) { + const isReport = entry.name.endsWith(".report.json"); + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = isDatabaseFile(entry.name); + + items.push({ + id: relPath, + label: entry.name.replace(/\.md$/, ""), + sublabel: relPath, + kind: "file", + path: relPath, + nodeType: isReport + ? "report" + : isDatabase + ? "database" + : isDocument + ? "document" + : "file", + }); + } + } +} + +/** + * Fetch all entries from all objects across ALL discovered DuckDB files. + * Deduplicates objects by name (shallower DBs win). + */ +async function buildEntryItems(): Promise { + const items: SearchIndexItem[] = []; + const dbPaths = discoverDuckDBPaths(); + if (dbPaths.length === 0) {return [];} + + // Collect all objects across DBs, deduplicating by name (shallowest wins) + const seenNames = new Set(); + const objectsWithDb: Array<{ obj: ObjectRow; dbPath: string }> = []; + + for (const dbPath of dbPaths) { + const objs = await duckdbQueryOnFileAsync(dbPath, + "SELECT * FROM objects ORDER BY name", + ); + for (const obj of objs) { + if (seenNames.has(obj.name)) {continue;} + seenNames.add(obj.name); + objectsWithDb.push({ obj, dbPath }); + } + } + + for (const { obj, dbPath } of objectsWithDb) { + const fields = await duckdbQueryOnFileAsync(dbPath, + `SELECT * FROM fields WHERE object_id = '${sqlEscape(obj.id)}' ORDER BY sort_order`, + ); + const displayField = resolveDisplayField(obj, fields); + const previewFields = fields + .filter((f) => !["relation", "richtext"].includes(f.type)) + .slice(0, 4); + + // Try PIVOT view first, then raw EAV (on the same DB) + let entries: Record[] = await duckdbQueryOnFileAsync(dbPath, + `SELECT * FROM v_${obj.name} ORDER BY created_at DESC LIMIT 500`, + ); + + if (entries.length === 0) { + const rawRows = await duckdbQueryOnFileAsync(dbPath, + `SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = '${sqlEscape(obj.id)}' + ORDER BY e.created_at DESC + LIMIT 2500`, + ); + + const grouped = new Map>(); + for (const row of rawRows) { + let entry = grouped.get(row.entry_id); + if (!entry) { + entry = { entry_id: row.entry_id }; + grouped.set(row.entry_id, entry); + } + if (row.field_name) {entry[row.field_name] = row.value;} + } + entries = Array.from(grouped.values()); + } + + for (const entry of entries) { + const entryId = dbStr(entry.entry_id); + if (!entryId) {continue;} + + const displayValue = dbStr(entry[displayField]); + const fieldPreview: Record = {}; + for (const f of previewFields) { + const val = entry[f.name]; + if (val != null && val !== "") { + fieldPreview[f.name] = dbStr(val); + } + } + + items.push({ + id: `entry:${obj.name}:${entryId}`, + label: displayValue || `(${obj.name} entry)`, + sublabel: obj.name, + kind: "entry", + icon: obj.icon, + objectName: obj.name, + entryId, + fields: Object.keys(fieldPreview).length > 0 ? fieldPreview : undefined, + }); + } + } + + return items; +} + +// --- Route handler --- + +export async function GET() { + const items: SearchIndexItem[] = []; + + // 1. Files + objects from tree + const root = resolveWorkspaceRoot(); + if (root) { + // Aggregate objects from ALL discovered DuckDB files (shallower wins) + const dbObjects = new Map(); + const objs = await duckdbQueryAllAsync( + "SELECT * FROM objects", + "name", + ); + for (const o of objs) {dbObjects.set(o.name, o);} + + // Scan workspace root (the workspace folder IS the knowledge base) + flattenTree(root, "", dbObjects, items); + } + + // 2. Entries from all objects across all discovered DBs + const dbPaths = discoverDuckDBPaths(); + if (dbPaths.length > 0) { + items.push(...await buildEntryItems()); + } + + return Response.json({ items }); +} diff --git a/apps/web/app/api/workspace/suggest-files/route.ts b/apps/web/app/api/workspace/suggest-files/route.ts new file mode 100644 index 00000000000..57bd1dfd45b --- /dev/null +++ b/apps/web/app/api/workspace/suggest-files/route.ts @@ -0,0 +1,434 @@ +import { readdirSync, readFileSync, existsSync, type Dirent } from "node:fs"; +import { join, dirname, resolve, basename } from "node:path"; +import { homedir } from "node:os"; +import { + resolveWorkspaceRoot, + duckdbQueryAllAsync, + discoverDuckDBPaths, + duckdbQueryOnFileAsync, + parseSimpleYaml, +} from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type SuggestItem = { + name: string; + path: string; + type: "folder" | "file" | "document" | "database" | "object" | "entry"; + /** Icon hint (emoji) for objects/entries */ + icon?: string; + /** Object name that owns this entry */ + objectName?: string; + /** DB entry ID */ + entryId?: string; +}; + +const SKIP_DIRS = new Set([ + "node_modules", + ".git", + ".Trash", + "__pycache__", + ".cache", + ".DS_Store", +]); + +/** List entries in a directory, sorted folders-first then alphabetically. */ +function listDir(absDir: string, filter?: string): SuggestItem[] { + let entries: Dirent[]; + try { + entries = readdirSync(absDir, { withFileTypes: true }); + } catch { + return []; + } + + const lowerFilter = filter?.toLowerCase(); + + const sorted = entries + .filter((e) => !e.name.startsWith(".")) + .filter((e) => !(e.isDirectory() && SKIP_DIRS.has(e.name))) + .filter((e) => !lowerFilter || e.name.toLowerCase().includes(lowerFilter)) + .toSorted((a, b) => { + if (a.isDirectory() && !b.isDirectory()) {return -1;} + if (!a.isDirectory() && b.isDirectory()) {return 1;} + return a.name.localeCompare(b.name); + }); + + const items: SuggestItem[] = []; + for (const entry of sorted) { + if (items.length >= 30) {break;} + const absPath = join(absDir, entry.name); + + if (entry.isDirectory()) { + items.push({ name: entry.name, path: absPath, type: "folder" }); + } else if (entry.isFile()) { + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = + ext === "duckdb" || ext === "sqlite" || ext === "sqlite3" || ext === "db"; + items.push({ + name: entry.name, + path: absPath, + type: isDatabase ? "database" : isDocument ? "document" : "file", + }); + } + } + return items; +} + +/** Recursively search for files matching a query, up to a limit. */ +function searchFiles( + absDir: string, + query: string, + results: SuggestItem[], + maxResults: number, + depth = 0, +): void { + if (depth > 6 || results.length >= maxResults) {return;} + + let entries: Dirent[]; + try { + entries = readdirSync(absDir, { withFileTypes: true }); + } catch { + return; + } + + const lowerQuery = query.toLowerCase(); + + for (const entry of entries) { + if (results.length >= maxResults) {return;} + if (entry.name.startsWith(".")) {continue;} + if (entry.isDirectory() && SKIP_DIRS.has(entry.name)) {continue;} + + const absPath = join(absDir, entry.name); + + if (entry.isFile() && entry.name.toLowerCase().includes(lowerQuery)) { + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = + ext === "duckdb" || ext === "sqlite" || ext === "sqlite3" || ext === "db"; + results.push({ + name: entry.name, + path: absPath, + type: isDatabase ? "database" : isDocument ? "document" : "file", + }); + } else if ( + entry.isDirectory() && + entry.name.toLowerCase().includes(lowerQuery) + ) { + results.push({ name: entry.name, path: absPath, type: "folder" }); + } + + if (entry.isDirectory()) { + searchFiles(absPath, query, results, maxResults, depth + 1); + } + } +} + +/** + * Resolve a user-typed path query into a directory to list and an optional filter. + * + * Examples: + * "../" β†’ list parent of workspace root + * "/" β†’ list filesystem root + * "~/" β†’ list home dir + * "~/Doc" β†’ list home dir, filter "Doc" + * "src/utils" β†’ list /src, filter "utils" + * "foo.ts" β†’ search by filename + */ +function resolvePath( + raw: string, + workspaceRoot: string, +): { dir: string; filter?: string } | null { + const home = homedir(); + + if (raw.startsWith("~/")) { + const rest = raw.slice(2); + if (!rest || rest.endsWith("/")) { + // List the directory + const dir = rest ? resolve(home, rest) : home; + return { dir }; + } + // Has a trailing segment β†’ list parent, filter by segment + const dir = resolve(home, dirname(rest)); + return { dir, filter: basename(rest) }; + } + + if (raw.startsWith("/")) { + if (raw === "/") {return { dir: "/" };} + if (raw.endsWith("/")) { + return { dir: resolve(raw) }; + } + const dir = dirname(resolve(raw)); + return { dir, filter: basename(raw) }; + } + + if (raw.startsWith("../") || raw === "..") { + const resolved = resolve(workspaceRoot, raw); + if (raw.endsWith("/") || raw === "..") { + return { dir: resolved }; + } + return { dir: dirname(resolved), filter: basename(resolved) }; + } + + if (raw.startsWith("./")) { + const rest = raw.slice(2); + if (!rest || rest.endsWith("/")) { + const dir = rest ? resolve(workspaceRoot, rest) : workspaceRoot; + return { dir }; + } + const dir = resolve(workspaceRoot, dirname(rest)); + return { dir, filter: basename(rest) }; + } + + // Contains a slash β†’ treat as relative path from workspace + if (raw.includes("/")) { + if (raw.endsWith("/")) { + return { dir: resolve(workspaceRoot, raw) }; + } + const dir = resolve(workspaceRoot, dirname(raw)); + return { dir, filter: basename(raw) }; + } + + // No path separator β†’ this is a filename search + return null; +} + +// --------------------------------------------------------------------------- +// DuckDB object & entry search +// --------------------------------------------------------------------------- + +type ObjectRow = { + id: string; + name: string; + description?: string; + icon?: string; + display_field?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; + sort_order?: number; +}; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +function resolveDisplayField(obj: ObjectRow, fields: FieldRow[]): string { + if (obj.display_field) {return obj.display_field;} + const nameField = fields.find( + (f) => /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) {return nameField.name;} + const textField = fields.find((f) => f.type === "text"); + if (textField) {return textField.name;} + return fields[0]?.name ?? "id"; +} + +/** Read icon from .object.yaml if present. */ +function readObjectIcon(workspaceRoot: string, objName: string): string | undefined { + // Walk workspace to find a folder matching objName that has .object.yaml + function walk(dir: string, depth: number): string | undefined { + if (depth > 4) {return undefined;} + try { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory() || entry.name.startsWith(".")) {continue;} + if (entry.name === objName) { + const yamlPath = join(dir, entry.name, ".object.yaml"); + if (existsSync(yamlPath)) { + const parsed = parseSimpleYaml(readFileSync(yamlPath, "utf-8")); + if (parsed.icon) {return dbStr(parsed.icon);} + } + } + const found = walk(join(dir, entry.name), depth + 1); + if (found) {return found;} + } + } catch { /* skip */ } + return undefined; + } + return walk(workspaceRoot, 0); +} + +/** Search objects by name (case-insensitive substring). */ +async function searchObjects( + query: string, + workspaceRoot: string, + max: number, +): Promise { + const sql = query + ? `SELECT * FROM objects WHERE LOWER(name) LIKE LOWER('%${sqlEscape(query)}%') ORDER BY name LIMIT ${max}` + : `SELECT * FROM objects ORDER BY name LIMIT ${max}`; + const objects = await duckdbQueryAllAsync(sql, "name"); + + const items: SuggestItem[] = []; + for (const obj of objects) { + const yamlIcon = readObjectIcon(workspaceRoot, obj.name); + items.push({ + name: obj.name, + path: `workspace:object:${obj.name}`, + type: "object", + icon: yamlIcon ?? obj.icon, + }); + } + return items; +} + +/** Safely convert an unknown DB value to a display string. */ +function dbStr(val: unknown): string { + if (val == null) {return "";} + if (typeof val === "object") {return JSON.stringify(val);} + return String(val as string | number | boolean); +} + +/** + * Search entries across all objects using a single UNION ALL query per DB. + * Each object's pivot view (v_) is searched by display field with ILIKE. + * This avoids spawning N DuckDB CLI processes per object. + */ +async function searchEntries( + query: string, + max: number, +): Promise { + const dbPaths = discoverDuckDBPaths(); + if (dbPaths.length === 0 || !query) {return [];} + + const items: SuggestItem[] = []; + const seenObjects = new Set(); + const likePattern = `%${sqlEscape(query)}%`; + + for (const dbPath of dbPaths) { + if (items.length >= max) {break;} + + // Step 1: get objects + display fields in a single query + type ObjFieldRow = ObjectRow & { field_name: string; field_type: string }; + const objFields = await duckdbQueryOnFileAsync( + dbPath, + `SELECT o.*, f.name as field_name, f.type as field_type + FROM objects o + LEFT JOIN fields f ON f.object_id = o.id + ORDER BY o.name, f.sort_order`, + ); + + // Group fields by object and resolve display fields + const objectMap = new Map(); + const fieldsByObj = new Map(); + for (const row of objFields) { + if (seenObjects.has(row.name)) {continue;} + if (!fieldsByObj.has(row.id)) {fieldsByObj.set(row.id, []);} + if (row.field_name) { + fieldsByObj.get(row.id)!.push({ + id: row.id, + name: row.field_name, + type: row.field_type, + }); + } + if (!objectMap.has(row.name)) { + const fields = fieldsByObj.get(row.id) ?? []; + objectMap.set(row.name, { + obj: row, + displayField: resolveDisplayField(row, fields), + }); + } + } + + // Re-resolve display fields now that all fields are collected + for (const [name, entry] of objectMap) { + const fields = fieldsByObj.get(entry.obj.id) ?? []; + entry.displayField = resolveDisplayField(entry.obj, fields); + seenObjects.add(name); + } + + if (objectMap.size === 0) {continue;} + + // Step 2: build a single UNION ALL query searching all pivot views + // Wrap each SELECT in parens so per-view LIMIT is valid DuckDB syntax + const unionParts: string[] = []; + for (const [name, { displayField }] of objectMap) { + const safeDisplay = sqlEscape(displayField); + unionParts.push( + `(SELECT '${sqlEscape(name)}' as _obj_name, entry_id, "${safeDisplay}" as _display + FROM v_${name} + WHERE LOWER(CAST("${safeDisplay}" AS VARCHAR)) LIKE LOWER('${likePattern}') + LIMIT ${max})`, + ); + } + + if (unionParts.length === 0) {continue;} + + type EntryHit = { _obj_name: string; entry_id: string; _display: string }; + const hits = await duckdbQueryOnFileAsync( + dbPath, + `${unionParts.join(" UNION ALL ")} LIMIT ${max}`, + ); + + for (const hit of hits) { + if (items.length >= max) {return items;} + if (!hit.entry_id || !hit._display) {continue;} + const objInfo = objectMap.get(hit._obj_name); + items.push({ + name: String(hit._display), + path: `workspace:entry:${hit._obj_name}:${hit.entry_id}`, + type: "entry", + icon: objInfo?.obj.icon, + objectName: hit._obj_name, + entryId: hit.entry_id, + }); + } + } + + return items; +} + +export async function GET(req: Request) { + const url = new URL(req.url); + const pathQuery = url.searchParams.get("path"); + const searchQuery = url.searchParams.get("q"); + const workspaceRoot = resolveWorkspaceRoot() ?? homedir(); + + // Search mode: find files, objects, and entries by name + if (searchQuery) { + // File search: workspace only (skip expensive home dir traversal) + const fileResults: SuggestItem[] = []; + searchFiles(workspaceRoot, searchQuery, fileResults, 15); + + // DuckDB search: objects and entries (sequential to avoid lock contention) + const objectResults = await searchObjects(searchQuery, workspaceRoot, 10); + const entryResults = await searchEntries(searchQuery, 15); + + // Deduplicate: if an object matches, remove the duplicate folder + const objectNames = new Set(objectResults.map((o) => o.name)); + const dedupedFiles = fileResults.filter( + (f) => !(f.type === "folder" && objectNames.has(f.name)), + ); + + // Merge: objects first, then entries, then files + const items = [...objectResults, ...entryResults, ...dedupedFiles].slice(0, 30); + return Response.json({ items }); + } + + // Browse mode: resolve path and list directory + if (pathQuery) { + const resolved = resolvePath(pathQuery, workspaceRoot); + if (!resolved) { + const results: SuggestItem[] = []; + searchFiles(workspaceRoot, pathQuery, results, 20); + return Response.json({ items: results }); + } + const items = listDir(resolved.dir, resolved.filter); + return Response.json({ items }); + } + + // Default: list workspace root + all objects + const fileItems = listDir(workspaceRoot); + const objectItems = await searchObjects("", workspaceRoot, 20); + // Deduplicate: if an object also appears as a folder, keep the object version + const objectNames = new Set(objectItems.map((o) => o.name)); + const dedupedFiles = fileItems.filter( + (f) => !(f.type === "folder" && objectNames.has(f.name)), + ); + return Response.json({ items: [...objectItems, ...dedupedFiles] }); +} diff --git a/apps/web/app/api/workspace/thumbnail/route.ts b/apps/web/app/api/workspace/thumbnail/route.ts new file mode 100644 index 00000000000..22b298ead14 --- /dev/null +++ b/apps/web/app/api/workspace/thumbnail/route.ts @@ -0,0 +1,69 @@ +import { existsSync, readFileSync, mkdirSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { join, basename } from "node:path"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { safeResolvePath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const THUMB_DIR = join(tmpdir(), "ironclaw-thumbs"); +mkdirSync(THUMB_DIR, { recursive: true }); + +/** + * Resolve a file path β€” supports absolute paths and workspace-relative paths. + */ +function resolveFile(path: string): string | null { + if (path.startsWith("/")) { + const abs = resolve(path); + if (existsSync(abs)) {return abs;} + } + return safeResolvePath(path) ?? null; +} + +/** + * GET /api/workspace/thumbnail?path=...&size=200 + * Uses macOS Quick Look (qlmanage) to generate a thumbnail image. + * Returns the thumbnail as image/png. + */ +export async function GET(req: Request) { + const url = new URL(req.url); + const path = url.searchParams.get("path"); + const size = url.searchParams.get("size") ?? "200"; + + if (!path) { + return new Response("Missing path", { status: 400 }); + } + + const absolute = resolveFile(path); + if (!absolute) { + return new Response("Not found", { status: 404 }); + } + + // The thumbnail output filename is .png + const thumbName = `${basename(absolute)}.png`; + const thumbPath = join(THUMB_DIR, thumbName); + + try { + // Generate thumbnail using macOS Quick Look + execSync( + `qlmanage -t -s ${parseInt(size, 10)} -o "${THUMB_DIR}" "${absolute}" 2>/dev/null`, + { timeout: 5000 }, + ); + + if (!existsSync(thumbPath)) { + return new Response("Thumbnail generation failed", { status: 500 }); + } + + const buffer = readFileSync(thumbPath); + return new Response(buffer, { + headers: { + "Content-Type": "image/png", + "Cache-Control": "public, max-age=3600", + }, + }); + } catch { + return new Response("Thumbnail generation failed", { status: 500 }); + } +} diff --git a/apps/web/app/api/workspace/tree-browse.test.ts b/apps/web/app/api/workspace/tree-browse.test.ts new file mode 100644 index 00000000000..3d908100e8e --- /dev/null +++ b/apps/web/app/api/workspace/tree-browse.test.ts @@ -0,0 +1,233 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import type { Dirent } from "node:fs"; + +// Mock node:fs +vi.mock("node:fs", () => ({ + readdirSync: vi.fn(() => []), + readFileSync: vi.fn(() => ""), + existsSync: vi.fn(() => false), + statSync: vi.fn(() => ({ isDirectory: () => false, size: 100 })), +})); + +// Mock node:os +vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), +})); + +// Mock workspace +vi.mock("@/lib/workspace", () => ({ + resolveWorkspaceRoot: vi.fn(() => null), + parseSimpleYaml: vi.fn(() => ({})), + duckdbQueryAll: vi.fn(() => []), + duckdbQueryAllAsync: vi.fn(async () => []), + isDatabaseFile: vi.fn(() => false), + discoverDuckDBPaths: vi.fn(() => []), + resolveDuckdbBin: vi.fn(() => null), + safeResolvePath: vi.fn(() => null), +})); + +function makeDirent(name: string, isDir: boolean): Dirent { + return { + name, + isDirectory: () => isDir, + isFile: () => !isDir, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isFIFO: () => false, + isSocket: () => false, + isSymbolicLink: () => false, + path: "", + parentPath: "", + } as Dirent; +} + +describe("Workspace Tree & Browse API", () => { + beforeEach(() => { + vi.resetModules(); + vi.mock("node:fs", () => ({ + readdirSync: vi.fn(() => []), + readFileSync: vi.fn(() => ""), + existsSync: vi.fn(() => false), + statSync: vi.fn(() => ({ isDirectory: () => false, size: 100 })), + })); + vi.mock("node:os", () => ({ + homedir: vi.fn(() => "/home/testuser"), + })); + vi.mock("@/lib/workspace", () => ({ + resolveWorkspaceRoot: vi.fn(() => null), + parseSimpleYaml: vi.fn(() => ({})), + duckdbQueryAll: vi.fn(() => []), + duckdbQueryAllAsync: vi.fn(async () => []), + isDatabaseFile: vi.fn(() => false), + discoverDuckDBPaths: vi.fn(() => []), + resolveDuckdbBin: vi.fn(() => null), + safeResolvePath: vi.fn(() => null), + })); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ─── GET /api/workspace/tree ──────────────────────────────────── + + describe("GET /api/workspace/tree", () => { + it("returns tree with exists=false when no workspace root", async () => { + const { GET } = await import("./tree/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.exists).toBe(false); + expect(json.tree).toEqual([]); + }); + + it("returns tree with workspace files", async () => { + const { resolveWorkspaceRoot } = await import("@/lib/workspace"); + vi.mocked(resolveWorkspaceRoot).mockReturnValue("/ws"); + const { readdirSync: mockReaddir, existsSync: mockExists } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReaddir).mockImplementation((dir) => { + if (String(dir) === "/ws") { + return [ + makeDirent("knowledge", true), + makeDirent("readme.md", false), + ] as unknown as Dirent[]; + } + return [] as unknown as Dirent[]; + }); + + const { GET } = await import("./tree/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.exists).toBe(true); + expect(json.tree.length).toBeGreaterThan(0); + }); + + it("includes workspaceRoot in response", async () => { + const { resolveWorkspaceRoot } = await import("@/lib/workspace"); + vi.mocked(resolveWorkspaceRoot).mockReturnValue("/ws"); + const { existsSync: mockExists } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + + const { GET } = await import("./tree/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.workspaceRoot).toBe("/ws"); + }); + }); + + // ─── GET /api/workspace/browse ────────────────────────────────── + + describe("GET /api/workspace/browse", () => { + it("returns directory listing", async () => { + const { existsSync: mockExists, readdirSync: mockReaddir, statSync: mockStat } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReaddir).mockReturnValue([ + makeDirent("file.txt", false), + makeDirent("subfolder", true), + ] as unknown as Dirent[]); + vi.mocked(mockStat).mockReturnValue({ isDirectory: () => false, size: 100 } as never); + + const { GET } = await import("./browse/route.js"); + const req = new Request("http://localhost/api/workspace/browse?dir=/tmp/test"); + const res = await GET(req); + const json = await res.json(); + expect(json.entries).toBeDefined(); + expect(json.currentDir).toBeDefined(); + }); + + it("returns parentDir for nested directories", async () => { + const { existsSync: mockExists, readdirSync: mockReaddir, statSync: mockStat } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReaddir).mockReturnValue([]); + vi.mocked(mockStat).mockReturnValue({ isDirectory: () => true, size: 0 } as never); + + const { GET } = await import("./browse/route.js"); + const req = new Request("http://localhost/api/workspace/browse?dir=/tmp/test/sub"); + const res = await GET(req); + const json = await res.json(); + expect(json.parentDir).toBeDefined(); + }); + }); + + // ─── GET /api/workspace/suggest-files ──────────────────────────── + + describe("GET /api/workspace/suggest-files", () => { + it("returns suggestions when workspace exists", async () => { + const { resolveWorkspaceRoot } = await import("@/lib/workspace"); + vi.mocked(resolveWorkspaceRoot).mockReturnValue("/ws"); + const { existsSync: mockExists, readdirSync: mockReaddir } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReaddir).mockReturnValue([ + makeDirent("doc.md", false), + ] as unknown as Dirent[]); + + const { GET } = await import("./suggest-files/route.js"); + const req = new Request("http://localhost/api/workspace/suggest-files?q=doc"); + const res = await GET(req); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.items).toBeDefined(); + }); + }); + + // ─── GET /api/workspace/context ────────────────────────────────── + + describe("GET /api/workspace/context", () => { + it("returns exists=false when no workspace root", async () => { + const { resolveWorkspaceRoot } = await import("@/lib/workspace"); + vi.mocked(resolveWorkspaceRoot).mockReturnValue(null); + + const { GET } = await import("./context/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.exists).toBe(false); + }); + + it("returns context when workspace_context.yaml exists", async () => { + const { resolveWorkspaceRoot, parseSimpleYaml } = await import("@/lib/workspace"); + vi.mocked(resolveWorkspaceRoot).mockReturnValue("/ws"); + vi.mocked(parseSimpleYaml).mockReturnValue({ org_name: "Acme", org_slug: "acme" }); + const { existsSync: mockExists, readFileSync: mockReadFile } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReadFile).mockReturnValue("org_name: Acme" as never); + + const { GET } = await import("./context/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.exists).toBe(true); + }); + }); + + // ─── GET /api/workspace/search-index ───────────────────────────── + + describe("GET /api/workspace/search-index", () => { + it("returns empty items when no workspace", async () => { + const { resolveWorkspaceRoot } = await import("@/lib/workspace"); + vi.mocked(resolveWorkspaceRoot).mockReturnValue(null); + + const { GET } = await import("./search-index/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.items).toEqual([]); + }); + + it("returns file items from workspace tree", async () => { + const { resolveWorkspaceRoot, duckdbQueryAll } = await import("@/lib/workspace"); + vi.mocked(resolveWorkspaceRoot).mockReturnValue("/ws"); + vi.mocked(duckdbQueryAll).mockReturnValue([]); + const { existsSync: mockExists, readdirSync: mockReaddir } = await import("node:fs"); + vi.mocked(mockExists).mockReturnValue(true); + vi.mocked(mockReaddir).mockImplementation((dir) => { + if (String(dir) === "/ws") { + return [makeDirent("readme.md", false)] as unknown as Dirent[]; + } + return [] as unknown as Dirent[]; + }); + + const { GET } = await import("./search-index/route.js"); + const res = await GET(); + const json = await res.json(); + expect(json.items.length).toBeGreaterThanOrEqual(0); + }); + }); +}); diff --git a/apps/web/app/api/workspace/tree/route.ts b/apps/web/app/api/workspace/tree/route.ts new file mode 100644 index 00000000000..c603d38b4eb --- /dev/null +++ b/apps/web/app/api/workspace/tree/route.ts @@ -0,0 +1,264 @@ +import { readdirSync, readFileSync, existsSync, statSync, type Dirent } from "node:fs"; +import { join } from "node:path"; +import { resolveWorkspaceRoot, resolveOpenClawStateDir, getEffectiveProfile, parseSimpleYaml, duckdbQueryAll, isDatabaseFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export type TreeNode = { + name: string; + path: string; // relative to workspace root (or ~skills/ for virtual nodes) + type: "object" | "document" | "folder" | "file" | "database" | "report"; + icon?: string; + defaultView?: "table" | "kanban"; + children?: TreeNode[]; + /** Virtual nodes live outside the main workspace (e.g. Skills, Memories). */ + virtual?: boolean; + /** True when the entry is a symbolic link. */ + symlink?: boolean; +}; + +type DbObject = { + name: string; + icon?: string; + default_view?: string; +}; + +/** Read .object.yaml metadata from a directory if it exists. */ +function readObjectMeta( + dirPath: string, +): { icon?: string; defaultView?: string } | null { + const yamlPath = join(dirPath, ".object.yaml"); + if (!existsSync(yamlPath)) {return null;} + + try { + const content = readFileSync(yamlPath, "utf-8"); + const parsed = parseSimpleYaml(content); + return { + icon: parsed.icon as string | undefined, + defaultView: parsed.default_view as string | undefined, + }; + } catch { + return null; + } +} + +/** + * Query ALL discovered DuckDB files for objects so we can identify object + * directories even when .object.yaml files are missing. + * Shallower databases win on name conflicts (parent priority). + */ +function loadDbObjects(): Map { + const map = new Map(); + const rows = duckdbQueryAll( + "SELECT name, icon, default_view FROM objects", + "name", + ); + for (const row of rows) { + map.set(row.name, row); + } + return map; +} + +/** Resolve a dirent's effective type, following symlinks to their target. */ +function resolveEntryType(entry: Dirent, absPath: string): "directory" | "file" | null { + if (entry.isDirectory()) {return "directory";} + if (entry.isFile()) {return "file";} + if (entry.isSymbolicLink()) { + try { + const st = statSync(absPath); + if (st.isDirectory()) {return "directory";} + if (st.isFile()) {return "file";} + } catch { + // Broken symlink -- skip + } + } + return null; +} + +/** Recursively build a tree from a workspace directory. */ +function buildTree( + absDir: string, + relativeBase: string, + dbObjects: Map, + showHidden = false, +): TreeNode[] { + const nodes: TreeNode[] = []; + + let entries: Dirent[]; + try { + entries = readdirSync(absDir, { withFileTypes: true }); + } catch { + return nodes; + } + + const filtered = entries.filter((e) => { + // .object.yaml is always needed for metadata; also shown as a node when showHidden is on + if (e.name === ".object.yaml") {return true;} + if (e.name.startsWith(".")) {return showHidden;} + return true; + }); + + // Sort: directories first, then files, alphabetical within each group + const sorted = filtered.toSorted((a, b) => { + const absA = join(absDir, a.name); + const absB = join(absDir, b.name); + const typeA = resolveEntryType(a, absA); + const typeB = resolveEntryType(b, absB); + const dirA = typeA === "directory"; + const dirB = typeB === "directory"; + if (dirA && !dirB) {return -1;} + if (!dirA && dirB) {return 1;} + return a.name.localeCompare(b.name); + }); + + for (const entry of sorted) { + // .object.yaml is consumed for metadata; only show it as a visible node when revealing hidden files + if (entry.name === ".object.yaml" && !showHidden) {continue;} + + const absPath = join(absDir, entry.name); + const relPath = relativeBase + ? `${relativeBase}/${entry.name}` + : entry.name; + + const isSymlink = entry.isSymbolicLink(); + const effectiveType = resolveEntryType(entry, absPath); + + if (effectiveType === "directory") { + const objectMeta = readObjectMeta(absPath); + const dbObject = dbObjects.get(entry.name); + const children = buildTree(absPath, relPath, dbObjects, showHidden); + + if (objectMeta || dbObject) { + nodes.push({ + name: entry.name, + path: relPath, + type: "object", + icon: objectMeta?.icon ?? dbObject?.icon, + defaultView: + ((objectMeta?.defaultView ?? dbObject?.default_view) as + | "table" + | "kanban") ?? "table", + children: children.length > 0 ? children : undefined, + ...(isSymlink && { symlink: true }), + }); + } else { + nodes.push({ + name: entry.name, + path: relPath, + type: "folder", + children: children.length > 0 ? children : undefined, + ...(isSymlink && { symlink: true }), + }); + } + } else if (effectiveType === "file") { + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isReport = entry.name.endsWith(".report.json"); + const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = isDatabaseFile(entry.name); + + nodes.push({ + name: entry.name, + path: relPath, + type: isReport ? "report" : isDatabase ? "database" : isDocument ? "document" : "file", + ...(isSymlink && { symlink: true }), + }); + } + } + + return nodes; +} + +// --- Virtual folder builders --- + +/** Parse YAML frontmatter from a SKILL.md file (lightweight). */ +function parseSkillFrontmatter(content: string): { name?: string; emoji?: string } { + const match = content.match(/^---\s*\n([\s\S]*?)\n---/); + if (!match) {return {};} + const yaml = match[1]; + const result: Record = {}; + for (const line of yaml.split("\n")) { + const kv = line.match(/^(\w+)\s*:\s*(.+)/); + if (kv) {result[kv[1]] = kv[2].replace(/^["']|["']$/g, "").trim();} + } + return { name: result.name, emoji: result.emoji }; +} + +/** Build a virtual "Skills" folder from /skills/. */ +function buildSkillsVirtualFolder(): TreeNode | null { + const stateDir = resolveOpenClawStateDir(); + const dirs = [ + join(stateDir, "skills"), + ]; + + const children: TreeNode[] = []; + const seen = new Set(); + + for (const dir of dirs) { + if (!existsSync(dir)) {continue;} + try { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory() || seen.has(entry.name)) {continue;} + const skillMdPath = join(dir, entry.name, "SKILL.md"); + if (!existsSync(skillMdPath)) {continue;} + + seen.add(entry.name); + let displayName = entry.name; + try { + const content = readFileSync(skillMdPath, "utf-8"); + const meta = parseSkillFrontmatter(content); + if (meta.name) {displayName = meta.name;} + if (meta.emoji) {displayName = `${meta.emoji} ${displayName}`;} + } catch { + // skip + } + + children.push({ + name: displayName, + path: `~skills/${entry.name}/SKILL.md`, + type: "document", + virtual: true, + }); + } + } catch { + // dir unreadable + } + } + + if (children.length === 0) {return null;} + children.sort((a, b) => a.name.localeCompare(b.name)); + + return { + name: "Skills", + path: "~skills", + type: "folder", + virtual: true, + children, + }; +} + + +export async function GET(req: Request) { + const url = new URL(req.url); + const showHidden = url.searchParams.get("showHidden") === "1"; + + const openclawDir = resolveOpenClawStateDir(); + const profile = getEffectiveProfile(); + const root = resolveWorkspaceRoot(); + if (!root) { + const tree: TreeNode[] = []; + const skillsFolder = buildSkillsVirtualFolder(); + if (skillsFolder) {tree.push(skillsFolder);} + return Response.json({ tree, exists: false, workspaceRoot: null, openclawDir, profile }); + } + + const dbObjects = loadDbObjects(); + + const tree = buildTree(root, "", dbObjects, showHidden); + + const skillsFolder = buildSkillsVirtualFolder(); + if (skillsFolder) {tree.push(skillsFolder);} + + return Response.json({ tree, exists: true, workspaceRoot: root, openclawDir, profile }); +} diff --git a/apps/web/app/api/workspace/upload/route.ts b/apps/web/app/api/workspace/upload/route.ts new file mode 100644 index 00000000000..04909bf2e62 --- /dev/null +++ b/apps/web/app/api/workspace/upload/route.ts @@ -0,0 +1,73 @@ +import { writeFileSync, mkdirSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { resolveWorkspaceRoot, safeResolveNewPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const MAX_SIZE = 25 * 1024 * 1024; // 25 MB + +/** + * POST /api/workspace/upload + * Accepts multipart form data with a "file" field. + * Saves to assets/- inside the workspace. + * Returns { ok, path } where path is workspace-relative. + */ +export async function POST(req: Request) { + const root = resolveWorkspaceRoot(); + if (!root) { + return Response.json( + { error: "Workspace not found" }, + { status: 500 }, + ); + } + + let formData: FormData; + try { + formData = await req.formData(); + } catch { + return Response.json({ error: "Invalid form data" }, { status: 400 }); + } + + const file = formData.get("file"); + if (!file || !(file instanceof File)) { + return Response.json( + { error: "Missing 'file' field" }, + { status: 400 }, + ); + } + + // Validate size + if (file.size > MAX_SIZE) { + return Response.json( + { error: "File is too large (max 25 MB)" }, + { status: 400 }, + ); + } + + // Build a safe filename: timestamp + sanitized original name + const safeName = file.name + .replace(/[^a-zA-Z0-9._-]/g, "_") + .replace(/_{2,}/g, "_"); + const relPath = join("assets", `${Date.now()}-${safeName}`); + + const absPath = safeResolveNewPath(relPath); + if (!absPath) { + return Response.json( + { error: "Invalid path" }, + { status: 400 }, + ); + } + + try { + mkdirSync(dirname(absPath), { recursive: true }); + const buffer = Buffer.from(await file.arrayBuffer()); + writeFileSync(absPath, buffer); + return Response.json({ ok: true, path: relPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Upload failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/virtual-file/route.ts b/apps/web/app/api/workspace/virtual-file/route.ts new file mode 100644 index 00000000000..33b17f0ae6c --- /dev/null +++ b/apps/web/app/api/workspace/virtual-file/route.ts @@ -0,0 +1,170 @@ +import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { join, dirname, resolve, normalize } from "node:path"; +import { resolveOpenClawStateDir, resolveWorkspaceRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * Resolve a virtual path (~skills/... or ~memories/...) to an absolute filesystem path. + * Returns null if the path is invalid or tries to escape. + */ +function resolveVirtualPath(virtualPath: string): string | null { + const stateDir = resolveOpenClawStateDir(); + const workspaceDir = resolveWorkspaceRoot() ?? join(stateDir, "workspace"); + + if (virtualPath.startsWith("~skills/")) { + // ~skills//SKILL.md + const rest = virtualPath.slice("~skills/".length); + // Validate: must be /SKILL.md + const parts = rest.split("/"); + if (parts.length !== 2 || parts[1] !== "SKILL.md" || !parts[0]) { + return null; + } + const skillName = parts[0]; + // Prevent path traversal + if (skillName.includes("..") || skillName.includes("/")) { + return null; + } + + // Check workspace skills first, then managed skills + const candidates = [ + join(workspaceDir, "skills", skillName, "SKILL.md"), + join(stateDir, "skills", skillName, "SKILL.md"), + ]; + for (const candidate of candidates) { + if (existsSync(candidate)) { + return candidate; + } + } + // Default to workspace skills dir for new files + return candidates[0]; + } + + if (virtualPath.startsWith("~memories/")) { + const rest = virtualPath.slice("~memories/".length); + // Prevent path traversal + if (rest.includes("..") || rest.includes("/")) { + return null; + } + + if (rest === "MEMORY.md") { + // Check both casing + for (const filename of ["MEMORY.md", "memory.md"]) { + const candidate = join(workspaceDir, filename); + if (existsSync(candidate)) { + return candidate; + } + } + // Default to MEMORY.md for new files + return join(workspaceDir, "MEMORY.md"); + } + + // Daily log: must be a .md file in the memory/ subdirectory + if (!rest.endsWith(".md")) { + return null; + } + return join(workspaceDir, "memory", rest); + } + + if (virtualPath.startsWith("~workspace/")) { + const rest = virtualPath.slice("~workspace/".length); + // Only allow direct filenames (no subdirectories, no traversal) + if (!rest || rest.includes("..") || rest.includes("/")) { + return null; + } + return join(workspaceDir, rest); + } + + return null; +} + +/** + * Double-check that the resolved path stays within expected directories. + */ +function isSafePath(absPath: string): boolean { + const stateDir = resolveOpenClawStateDir(); + const workspaceDir = resolveWorkspaceRoot() ?? join(stateDir, "workspace"); + const normalized = normalize(resolve(absPath)); + const allowed = [ + normalize(join(stateDir, "skills")), + normalize(join(workspaceDir, "skills")), + normalize(workspaceDir), + ]; + return allowed.some((dir) => normalized.startsWith(dir)); +} + +/** Extensions recognized as code files for syntax-highlighted viewing. */ +const VIRTUAL_CODE_EXTENSIONS = new Set([ + "ts", "tsx", "js", "jsx", "mjs", "cjs", "py", "rb", "go", "rs", + "java", "kt", "swift", "c", "cpp", "h", "hpp", "cs", "css", "scss", + "less", "html", "htm", "xml", "json", "jsonc", "toml", "sh", "bash", + "zsh", "fish", "ps1", "sql", "graphql", "gql", "diff", "patch", + "ini", "env", "tf", "proto", "zig", "lua", "php", +]); + +export async function GET(req: Request) { + const url = new URL(req.url); + const path = url.searchParams.get("path"); + + if (!path) { + return Response.json({ error: "Missing 'path' query parameter" }, { status: 400 }); + } + + const absPath = resolveVirtualPath(path); + if (!absPath || !isSafePath(absPath)) { + return Response.json({ error: "Invalid virtual path" }, { status: 400 }); + } + + if (!existsSync(absPath)) { + return Response.json({ error: "File not found" }, { status: 404 }); + } + + try { + const content = readFileSync(absPath, "utf-8"); + const ext = absPath.split(".").pop()?.toLowerCase(); + let type: "markdown" | "yaml" | "code" | "text" = "text"; + if (ext === "md" || ext === "mdx") {type = "markdown";} + else if (ext === "yaml" || ext === "yml") {type = "yaml";} + else if (VIRTUAL_CODE_EXTENSIONS.has(ext ?? "")) {type = "code";} + return Response.json({ content, type }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Read failed" }, + { status: 500 }, + ); + } +} + +export async function POST(req: Request) { + let body: { path?: string; content?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: virtualPath, content } = body; + if (!virtualPath || typeof virtualPath !== "string" || typeof content !== "string") { + return Response.json( + { error: "Missing 'path' and 'content' fields" }, + { status: 400 }, + ); + } + + const absPath = resolveVirtualPath(virtualPath); + if (!absPath || !isSafePath(absPath)) { + return Response.json({ error: "Invalid virtual path" }, { status: 400 }); + } + + try { + mkdirSync(dirname(absPath), { recursive: true }); + writeFileSync(absPath, content, "utf-8"); + return Response.json({ ok: true, path: virtualPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Write failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/watch/route.ts b/apps/web/app/api/workspace/watch/route.ts new file mode 100644 index 00000000000..9bb11bb50ef --- /dev/null +++ b/apps/web/app/api/workspace/watch/route.ts @@ -0,0 +1,148 @@ +import { resolveWorkspaceRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +// --------------------------------------------------------------------------- +// Singleton watcher: one chokidar instance shared across all SSE connections. +// Uses polling (no native fs.watch FDs) so it doesn't compete with Next.js's +// own dev watcher for the macOS per-process file-descriptor limit. +// --------------------------------------------------------------------------- + +type Listener = (type: string, relPath: string) => void; + +let listeners = new Set(); +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let sharedWatcher: any = null; +let sharedRoot: string | null = null; +let _watcherReady = false; + +async function ensureWatcher(root: string) { + if (sharedWatcher && sharedRoot === root) {return;} + + // Root changed (e.g. profile switch) -- close the old watcher first. + if (sharedWatcher) { + await sharedWatcher.close(); + sharedWatcher = null; + sharedRoot = null; + _watcherReady = false; + } + + try { + const chokidar = await import("chokidar"); + sharedRoot = root; + sharedWatcher = chokidar.watch(root, { + ignoreInitial: true, + usePolling: true, + interval: 1500, + binaryInterval: 3000, + ignored: [ + /(^|[\\/])node_modules([\\/]|$)/, + /(^|[\\/])\.git([\\/]|$)/, + /(^|[\\/])\.next([\\/]|$)/, + /(^|[\\/])dist([\\/]|$)/, + /\.duckdb\.wal$/, + /\.duckdb\.tmp$/, + ], + depth: 5, + }); + + sharedWatcher.on("all", (eventType: string, filePath: string) => { + const rel = filePath.startsWith(root) + ? filePath.slice(root.length + 1) + : filePath; + for (const fn of listeners) {fn(eventType, rel);} + }); + + sharedWatcher.once("ready", () => {_watcherReady = true;}); + + sharedWatcher.on("error", () => { + // Swallow; polling mode shouldn't hit EMFILE but be safe. + }); + } catch { + // chokidar unavailable -- listeners simply won't fire. + } +} + +function stopWatcherIfIdle() { + if (listeners.size > 0 || !sharedWatcher) {return;} + sharedWatcher.close(); + sharedWatcher = null; + sharedRoot = null; + _watcherReady = false; +} + +/** + * GET /api/workspace/watch + * + * Server-Sent Events endpoint that watches the workspace for file changes. + * Falls back gracefully if chokidar is unavailable. + */ +export async function GET(req: Request) { + const root = resolveWorkspaceRoot(); + if (!root) { + return new Response("Workspace not found", { status: 404 }); + } + + const encoder = new TextEncoder(); + let closed = false; + let heartbeat: ReturnType | null = null; + let debounceTimer: ReturnType | null = null; + + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue(encoder.encode("event: connected\ndata: {}\n\n")); + + const listener: Listener = (_type, _rel) => { + if (closed) {return;} + if (debounceTimer) {clearTimeout(debounceTimer);} + debounceTimer = setTimeout(() => { + if (closed) {return;} + try { + const data = JSON.stringify({ type: _type, path: _rel }); + controller.enqueue(encoder.encode(`event: change\ndata: ${data}\n\n`)); + } catch { /* stream closed */ } + }, 300); + }; + + heartbeat = setInterval(() => { + if (closed) {return;} + try { + controller.enqueue(encoder.encode(": heartbeat\n\n")); + } catch { /* closed */ } + }, 30_000); + + function teardown() { + if (closed) {return;} + closed = true; + listeners.delete(listener); + if (heartbeat) {clearInterval(heartbeat);} + if (debounceTimer) {clearTimeout(debounceTimer);} + stopWatcherIfIdle(); + } + + req.signal.addEventListener("abort", teardown, { once: true }); + + listeners.add(listener); + await ensureWatcher(root); + + if (!sharedWatcher) { + controller.enqueue( + encoder.encode("event: error\ndata: {\"error\":\"File watching unavailable\"}\n\n"), + ); + } + }, + cancel() { + closed = true; + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }, + }); +} diff --git a/apps/web/app/components/chain-of-thought.tsx b/apps/web/app/components/chain-of-thought.tsx new file mode 100644 index 00000000000..5f7fa9d32fe --- /dev/null +++ b/apps/web/app/components/chain-of-thought.tsx @@ -0,0 +1,1726 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { motion, AnimatePresence } from "framer-motion"; +import { DiffCard } from "./diff-viewer"; + +/* ─── Diff synthesis from edit tool args ─── */ + +/** + * Build a unified diff string from old_string/new_string pairs. + * This provides a visual diff even when the tool result doesn't include one. + */ +function buildSyntheticDiff(filePath: string, oldStr: string, newStr: string): string { + const oldLines = oldStr.split("\n"); + const newLines = newStr.split("\n"); + const lines: string[] = [ + `--- a/${filePath}`, + `+++ b/${filePath}`, + `@@ -1,${oldLines.length} +1,${newLines.length} @@`, + ]; + for (const line of oldLines) { + lines.push(`-${line}`); + } + for (const line of newLines) { + lines.push(`+${line}`); + } + return lines.join("\n"); +} + +/* ─── Public types ─── */ + +export type ChainPart = + | { kind: "reasoning"; text: string; isStreaming: boolean } + | { + kind: "tool"; + toolName: string; + toolCallId: string; + status: "running" | "done" | "error"; + args?: Record; + output?: Record; + errorText?: string; + } +; + +/* ─── Media / file type helpers ─── */ + +const IMAGE_EXTS = new Set([ + "jpg", + "jpeg", + "png", + "gif", + "webp", + "svg", + "bmp", + "avif", + "heic", + "heif", + "tiff", + "tif", + "ico", +]); +const VIDEO_EXTS = new Set([ + "mp4", + "webm", + "mov", + "avi", + "mkv", +]); +const PDF_EXTS = new Set(["pdf"]); +const AUDIO_EXTS = new Set(["mp3", "wav", "ogg", "m4a"]); + +type MediaKind = "image" | "video" | "pdf" | "audio" | null; + +function getFileExt(path: string): string { + return (path.split(".").pop() ?? "").toLowerCase(); +} + +function detectMedia(path: string): MediaKind { + const ext = getFileExt(path); + if (IMAGE_EXTS.has(ext)) {return "image";} + if (VIDEO_EXTS.has(ext)) {return "video";} + if (PDF_EXTS.has(ext)) {return "pdf";} + if (AUDIO_EXTS.has(ext)) {return "audio";} + return null; +} + +function rawFileUrl(path: string): string { + return `/api/workspace/raw-file?path=${encodeURIComponent(path)}`; +} + +/** Resolve a media URL β€” use raw URL directly if it's already HTTP */ +function resolveMediaUrl(path: string): string { + if (path.startsWith("http://") || path.startsWith("https://")) { + return path; + } + return rawFileUrl(path); +} + +/** Regex to find file paths with media extensions in free text */ +const MEDIA_FILE_RE = + /(?:^|[\s"'(=])(((?:\/|\.\/)?[\w.\-/\\]+)\.(?:jpe?g|png|gif|webp|svg|bmp|avif|heic|heif|tiff?|ico|mp4|webm|mov|avi|mkv|mp3|wav|ogg|m4a|pdf))\b/i; + +const PATH_KEYS = [ + "path", + "file", + "file_path", + "filePath", + "filename", + "url", + "src", + "name", + "target", +]; + +/** + * Extract the file path from tool args and/or output. + * Searches standard keys, then all string values, then output text. + */ +function getFilePath( + args?: Record, + output?: Record, +): string | null { + // 1. Check standard keys in args + if (args) { + for (const key of PATH_KEYS) { + const v = args[key]; + if (typeof v === "string" && v.length > 0) {return v;} + } + } + + // 2. Check standard keys in output + if (output) { + for (const key of PATH_KEYS) { + const v = output[key]; + if (typeof v === "string" && v.length > 0 && looksLikePath(v)) + {return v;} + } + } + + // 3. Scan all string values in args for file-like paths + if (args) { + const found = findPathInValues(args); + if (found) {return found;} + } + + // 4. Extract from output text + if (output?.text && typeof output.text === "string") { + const m = output.text.match(MEDIA_FILE_RE); + if (m) {return m[1];} + } + + // 5. Scan output values too + if (output) { + const found = findPathInValues(output); + if (found) {return found;} + } + + return null; +} + +/** Check if a string looks like a file path (has an extension, no spaces) */ +function looksLikePath(s: string): boolean { + return ( + s.length > 2 && + s.length < 500 && + /\.\w{1,5}$/.test(s) && + !s.includes(" ") + ); +} + +/** Search all string values in an object for a path-like string */ +function findPathInValues(obj: Record): string | null { + for (const val of Object.values(obj)) { + if (typeof val === "string" && looksLikePath(val)) { + return val; + } + } + return null; +} + +/* ─── Domain / URL extraction helpers ─── */ + +const URL_RE = /https?:\/\/[^\s"'<>,;)}\]]+/gi; + +function extractDomains(text: string): string[] { + const urls = text.match(URL_RE) ?? []; + const domains = new Set(); + for (const url of urls) { + try { + const hostname = new URL(url).hostname; + if (hostname && !hostname.includes("localhost")) { + domains.add(hostname); + } + } catch { + /* skip */ + } + } + return [...domains].slice(0, 8); +} + +function faviconUrl(domain: string): string { + return `https://www.google.com/s2/favicons?sz=64&domain_url=${encodeURIComponent(domain)}`; +} + +/* ─── Format tool args for display ─── */ + +/** Render tool arguments as a compact readable string. */ +function formatArgs(args: Record): string { + const lines: string[] = []; + for (const [key, value] of Object.entries(args)) { + if (value === undefined || value === null) {continue;} + const str = + typeof value === "string" + ? value + : JSON.stringify(value, null, 2); + lines.push(`${key}: ${str}`); + } + const joined = lines.join("\n"); + return joined.length > 2000 ? joined.slice(0, 2000) + "\n..." : joined; +} + +/* ─── Classify tool steps ─── */ + +type StepKind = + | "search" + | "fetch" + | "read" + | "exec" + | "write" + | "image" + | "generic"; + +function classifyTool( + name: string, + args?: Record, +): StepKind { + const n = name.toLowerCase().replace(/[_-]/g, ""); + if ( + [ + "websearch", + "search", + "googlesearch", + "bingsearch", + "browsersearch", + "tavily", + ].some((k) => n.includes(k)) + ) + {return "search";} + + // Browser tool β€” classify based on the action being performed + if (n === "browser") { + const action = + typeof args?.action === "string" + ? args.action.toLowerCase() + : ""; + if (action === "open" || action === "navigate") {return "fetch";} + if (action === "screenshot") {return "image";} + return "generic"; // act/snapshot/status etc. have no URL + } + + if ( + ["fetchurl", "fetch", "browse", "browseurl", "webfetch"].some( + (k) => n.includes(k), + ) + ) + {return "fetch";} + if ( + ["read", "file", "readfile", "getfile"].some( + (k) => n.includes(k), + ) + ) + {return "read";} + if ( + [ + "bash", + "shell", + "execute", + "exec", + "terminal", + "command", + "run", + ].some((k) => n.includes(k)) + ) + {return "exec";} + if ( + [ + "write", + "create", + "edit", + "str_replace", + "save", + "patch", + ].some((k) => n.includes(k)) + ) + {return "write";} + if ( + [ + "image", + "screenshot", + "photo", + "picture", + "dalle", + "generateimage", + ].some((k) => n.includes(k)) + ) + {return "image";} + return "generic"; +} + +function buildStepLabel( + kind: StepKind, + toolName: string, + args?: Record, + output?: Record, +): string { + const strVal = (key: string) => { + const v = args?.[key]; + return typeof v === "string" && v.length > 0 ? v : null; + }; + + switch (kind) { + case "search": { + const q = + strVal("query") ?? + strVal("search_query") ?? + strVal("search") ?? + strVal("q"); + return q ? `Searching for ${q}` : "Searching..."; + } + case "fetch": { + const u = + strVal("url") ?? + strVal("targetUrl") ?? + strVal("path") ?? + strVal("src"); + if (u) { + try { + return `Fetching ${new URL(u).hostname}`; + } catch { + return `Fetching ${u}`; + } + } + // Fallback: check output for the URL (web_fetch results include url/finalUrl) + const outUrl = + (typeof output?.finalUrl === "string" && output.finalUrl) || + (typeof output?.url === "string" && output.url); + if (outUrl) { + try { + return `Fetched ${new URL(outUrl).hostname}`; + } catch { + return `Fetched ${outUrl}`; + } + } + return "Fetching page"; + } + case "read": { + const p = getFilePath(args, output); + if (p) { + const short = p.split("/").pop() ?? p; + return short.startsWith("http") + ? `Fetching ${short}` + : `Reading ${short}`; + } + return "Reading file"; + } + case "exec": { + const cmd = strVal("command") ?? strVal("cmd"); + if (cmd) {return `Running: ${cmd}`;} + return "Running command"; + } + case "write": { + const p = strVal("path") ?? strVal("file") ?? strVal("file_path"); + if (p) { + const short = p.split("/").pop() ?? p; + return `Editing ${short}`; + } + return "Editing file"; + } + case "image": + return strVal("description") + ? `Generating image: ${strVal("description")!}` + : "Generating image"; + default: { + // For generic/unknown tools, build a descriptive label from args + const name = toolName + .replace(/_/g, " ") + .replace(/\b\w/g, (c) => c.toUpperCase()) + .trim(); + if (args) { + // Try common arg patterns for a meaningful summary + const desc = + strVal("command") ?? + strVal("cmd") ?? + strVal("query") ?? + strVal("path") ?? + strVal("url") ?? + strVal("message") ?? + strVal("description") ?? + strVal("input") ?? + strVal("text"); + if (desc) {return `${name}: ${desc}`;} + } + return name || "Tool"; + } + } +} + +/** Extract domains from tool output for search steps */ +function getSearchDomains( + output?: Record, +): string[] { + if (!output) {return [];} + const text = typeof output.text === "string" ? output.text : ""; + const results = output.results; + const citations = output.citations; + let combined = text; + if (Array.isArray(results)) { + for (const r of results) { + if (typeof r === "string") { + combined += ` ${r}`; + } else if (typeof r === "object" && r !== null) { + const obj = r as Record; + if (typeof obj.url === "string") + {combined += ` ${obj.url}`;} + if (typeof obj.link === "string") + {combined += ` ${obj.link}`;} + } + } + } + if (Array.isArray(citations)) { + for (const c of citations) { + // Citations can be plain URL strings or objects with a url field + if (typeof c === "string") { + combined += ` ${c}`; + } else if (typeof c === "object" && c !== null) { + const obj = c as Record; + if (typeof obj.url === "string") + {combined += ` ${obj.url}`;} + } + } + } + // Scan all remaining string values in the output for URLs we may have missed + for (const val of Object.values(output)) { + if (typeof val === "string" && val !== text && val.includes("http")) { + combined += ` ${val}`; + } + } + return extractDomains(combined); +} + +/** Extract domain(s) from fetch/browser tool args and/or output */ +function getFetchDomains( + args?: Record, + output?: Record, +): string[] { + const domains = new Set(); + // Check args for URL (web_fetch uses "url", browser tool uses "targetUrl") + for (const key of ["url", "targetUrl", "path", "src"]) { + const v = args?.[key]; + if (typeof v === "string" && v.startsWith("http")) { + try { + const hostname = new URL(v).hostname; + if (hostname && !hostname.includes("localhost")) { + domains.add(hostname); + } + } catch { + /* skip */ + } + } + } + // Check output for URL / finalUrl + for (const key of ["url", "finalUrl", "targetUrl"]) { + const v = output?.[key]; + if (typeof v === "string" && v.startsWith("http")) { + try { + const hostname = new URL(v).hostname; + if (hostname && !hostname.includes("localhost")) { + domains.add(hostname); + } + } catch { + /* skip */ + } + } + } + return [...domains].slice(0, 4); +} + +/* ─── Group consecutive media reads ─── */ + +type ToolPart = Extract; + +type VisualItem = + | { type: "tool"; tool: ToolPart } + | { + type: "media-group"; + mediaKind: "image" | "video" | "pdf" | "audio"; + items: Array<{ path: string; tool: ToolPart }>; + } + | { + type: "fetch-group"; + items: ToolPart[]; + }; + +function groupToolSteps(tools: ToolPart[]): VisualItem[] { + const result: VisualItem[] = []; + let i = 0; + while (i < tools.length) { + const tool = tools[i]; + const kind = classifyTool(tool.toolName, tool.args); + // Check both args AND output for the file path + const filePath = getFilePath(tool.args, tool.output); + const media = filePath ? detectMedia(filePath) : null; + + // If this is a media read, look for consecutive media reads of the same kind + if (kind === "read" && media && filePath) { + const group: Array<{ path: string; tool: ToolPart }> = [ + { path: filePath, tool }, + ]; + let j = i + 1; + while (j < tools.length) { + const next = tools[j]; + const nextKind = classifyTool(next.toolName, next.args); + const nextPath = getFilePath(next.args, next.output); + const nextMedia = nextPath ? detectMedia(nextPath) : null; + if (nextKind === "read" && nextMedia === media && nextPath) { + group.push({ path: nextPath, tool: next }); + j++; + } else { + break; + } + } + result.push({ + type: "media-group", + mediaKind: media, + items: group, + }); + i = j; + } else if (kind === "fetch") { + // Group consecutive fetch tools into a single compact card + const group: ToolPart[] = [tool]; + let j = i + 1; + while (j < tools.length) { + const next = tools[j]; + const nextKind = classifyTool(next.toolName, next.args); + if (nextKind === "fetch") { + group.push(next); + j++; + } else { + break; + } + } + if (group.length > 1) { + result.push({ type: "fetch-group", items: group }); + } else { + result.push({ type: "tool", tool }); + } + i = j; + } else { + result.push({ type: "tool", tool }); + i++; + } + } + return result; +} + +/* ─── Main component ─── */ + +export function ChainOfThought({ parts, isStreaming }: { parts: ChainPart[]; isStreaming?: boolean }) { + const [isOpen, setIsOpen] = useState(!!isStreaming); + + const isActive = parts.some( + (p) => + (p.kind === "reasoning" && p.isStreaming) || + (p.kind === "tool" && p.status === "running"), + ); + + /* ─── Live elapsed-time tracking ─── */ + const startRef = useRef(null); + const [elapsed, setElapsed] = useState(0); + + useEffect(() => { + if (isActive && startRef.current === null) { + startRef.current = Date.now(); + } + }, [isActive]); + + useEffect(() => { + if (!isActive) {return;} + const tick = () => { + if (startRef.current !== null) { + setElapsed(Math.floor((Date.now() - startRef.current) / 1000)); + } + }; + tick(); + const id = setInterval(tick, 1000); + return () => clearInterval(id); + }, [isActive]); + + const formatDuration = useCallback((s: number) => { + if (s < 60) {return `${s}s`;} + const m = Math.floor(s / 60); + const rem = s % 60; + return rem > 0 ? `${m}m ${rem}s` : `${m}m`; + }, []); + + // Collapse only when the parent stream truly ends β€” not on intermediate + // isActive flickers (e.g. gap between reasoning end and tool start). + const wasStreamingRef = useRef(false); + useEffect(() => { + if (isStreaming) { + wasStreamingRef.current = true; + } else if (wasStreamingRef.current && parts.length > 0) { + wasStreamingRef.current = false; + setIsOpen(false); + } + }, [isStreaming, parts.length]); + + const reasoningText = parts + .filter( + (p): p is Extract => + p.kind === "reasoning", + ) + .map((p) => p.text) + .join(""); + const isReasoningStreaming = parts.some( + (p) => p.kind === "reasoning" && p.isStreaming, + ); + + const tools = parts.filter( + (p): p is ToolPart => p.kind === "tool", + ); + const visualItems = groupToolSteps(tools); + + const headerLabel = isActive + ? elapsed > 0 + ? `Thinking for ${formatDuration(elapsed)}` + : "Thinking..." + : elapsed > 0 + ? `Thought for ${formatDuration(elapsed)}` + : "Thought"; + + return ( +
+ {/* Header trigger */} + + + {/* Collapsible content */} + + {isOpen && ( + +
+ {/* Timeline connector line */} +
+ + {reasoningText && ( + +
+ + + + +
+
+ +
+ + )} + {visualItems.map((item, idx) => { + if (item.type === "media-group") { + return ( + + + + ); + } + if (item.type === "fetch-group") { + return ( + + + + ); + } + return ( + + + + ); + })} + +
+ + )} + +
+ ); +} + +/* ─── Reasoning block ─── */ + +function ReasoningBlock({ + text, + isStreaming: _isStreaming, +}: { + text: string; + isStreaming: boolean; +}) { + return ( +
+
+ {text} +
+
+ ); +} + +/* ─── Fetch group (consecutive web fetches in one compact card) ─── */ + +function FetchGroup({ items }: { items: ToolPart[] }) { + const anyRunning = items.some((t) => t.status === "running"); + const doneCount = items.filter((t) => t.status === "done").length; + + return ( +
+
+ + + +
+
+
+ + {anyRunning + ? `Fetching ${items.length} sources...` + : `Fetched ${items.length} sources`} + + {!anyRunning && ( + + {doneCount} {doneCount === 1 ? "result" : "results"} + + )} +
+ +
+ ); +} + +/** Extract domain and full URL from fetch tool args/output */ +function getFetchDomainAndUrl( + args?: Record, + output?: Record, +): { domain: string | null; url: string | null } { + for (const key of ["url", "targetUrl", "path", "src"]) { + const v = args?.[key]; + if (typeof v === "string" && v.startsWith("http")) { + try { + return { domain: new URL(v).hostname, url: v }; + } catch { + /* skip */ + } + } + } + for (const key of ["url", "finalUrl", "targetUrl"]) { + const v = output?.[key]; + if (typeof v === "string" && v.startsWith("http")) { + try { + return { domain: new URL(v).hostname, url: v }; + } catch { + /* skip */ + } + } + } + return { domain: null, url: null }; +} + +/* ─── Media group (images, videos, PDFs, audio) ─── */ + +function MediaGroup({ + mediaKind, + items, +}: { + mediaKind: "image" | "video" | "pdf" | "audio"; + items: Array<{ path: string; tool: ToolPart }>; +}) { + const [expanded, setExpanded] = useState(false); + const anyRunning = items.some( + (i) => i.tool.status === "running", + ); + + // Show completed items progressively β€” don't wait for allDone + const completedItems = items.filter( + (i) => i.tool.status === "done", + ); + const doneCount = completedItems.length; + + const label = anyRunning + ? `Reading ${items.length} ${mediaKind}${items.length > 1 ? "s" : ""}...` + : mediaKind === "image" + ? items.length === 1 + ? `Read 1 image` + : `Read ${items.length} images` + : mediaKind === "video" + ? items.length === 1 + ? `Read 1 video` + : `Read ${items.length} videos` + : mediaKind === "pdf" + ? items.length === 1 + ? `Read 1 PDF` + : `Read ${items.length} PDFs` + : items.length === 1 + ? `Read 1 audio file` + : `Read ${items.length} audio files`; + + // Show up to 6 thumbnails by default, expandable + const PREVIEW_COUNT = 6; + const displayItems = expanded + ? completedItems + : completedItems.slice(0, PREVIEW_COUNT); + const hasMore = + completedItems.length > PREVIEW_COUNT && !expanded; + + return ( +
+
+ + + +
+
+
+ {label} +
+ + {/* Image thumbnail grid β€” show progressively as items complete */} + {doneCount > 0 && mediaKind === "image" && ( +
+ {displayItems.map((item) => ( + + ))} + {anyRunning && ( +
+ + + +
+ )} + {hasMore && ( + + )} +
+ )} + + {/* Video inline */} + {doneCount > 0 && mediaKind === "video" && ( +
+ {displayItems.map((item) => ( +
+ )} + + {/* PDF links */} + {doneCount > 0 && mediaKind === "pdf" && ( +
+ {displayItems.map((item) => { + const filename = + item.path.split("/").pop() ?? + item.path; + return ( + + + + {filename} + + + ); + })} +
+ )} + + {/* Audio inline */} + {doneCount > 0 && mediaKind === "audio" && ( +
+ {displayItems.map((item) => ( +
+ )} +
+
+ ); +} + +/** Image thumbnail with error fallback */ +function MediaThumb({ + path, + single, +}: { + path: string; + single: boolean; +}) { + const [error, setError] = useState(false); + const filename = path.split("/").pop() ?? path; + const url = resolveMediaUrl(path); + const w = single ? 200 : 80; + const h = single ? 150 : 80; + + if (error) { + return ( +
+ {filename} +
+ ); + } + + return ( + + {/* eslint-disable-next-line @next/next/no-img-element */} + {filename} setError(true)} + /> + + ); +} + +/* ─── Tool step (non-media) ─── */ + +function ToolStep({ + toolName, + status, + args, + output, + errorText, +}: { + toolName: string; + status: "running" | "done" | "error"; + args?: Record; + output?: Record; + errorText?: string; +}) { + const kind = classifyTool(toolName, args); + // Show output by default for exec/command tools β€” these are the most + // useful to see inline. Other tools default to collapsed. + const [showOutput, setShowOutput] = useState(kind === "exec" || kind === "generic"); + // Auto-expand diffs for write tool steps + const [showDiff, setShowDiff] = useState(true); + const label = buildStepLabel(kind, toolName, args, output); + const domains = + kind === "search" + ? getSearchDomains(output) + : kind === "fetch" + ? getFetchDomains(args, output) + : []; + const outputText = + typeof output?.text === "string" ? output.text : undefined; + + // Detect diff data from edit/write tool results. + // Priority: output.diff (from edit tool), then synthesize from args. + const diffText = (() => { + if (kind !== "write" || status !== "done") {return undefined;} + // 1. Direct diff from tool result (edit tool returns this) + if (typeof output?.diff === "string") {return output.diff;} + // 2. Synthesize from edit args (old_string/new_string or oldText/newText) + const oldStr = + typeof args?.old_string === "string" ? args.old_string : + typeof args?.oldText === "string" ? args.oldText : null; + const newStr = + typeof args?.new_string === "string" ? args.new_string : + typeof args?.newText === "string" ? args.newText : null; + if (oldStr !== null && newStr !== null) { + const path = typeof args?.path === "string" ? args.path : + typeof args?.file_path === "string" ? args.file_path : "file"; + return buildSyntheticDiff(path, oldStr, newStr); + } + return undefined; + })(); + + // For single-file reads that are media, render inline preview + const filePath = getFilePath(args, output); + const media = filePath ? detectMedia(filePath) : null; + const isSingleMedia = kind === "read" && media && status === "done"; + + return ( +
+
+ {status === "error" ? ( + + ) : ( + + + + )} +
+ +
+
+ {label} + {/* Exit code badge for exec tools */} + {kind === "exec" && status === "done" && output?.exitCode !== undefined && ( + + exit {typeof output.exitCode === "object" && output.exitCode != null ? JSON.stringify(output.exitCode) : typeof output.exitCode === "number" ? String(output.exitCode) : typeof output.exitCode === "string" ? output.exitCode : ""} + + )} +
+ + {/* Inline diff for edit/write tool steps */} + {diffText && status === "done" && ( +
+ + {showDiff && ( + + )} +
+ )} + + {/* Single media inline preview (when not grouped) */} + {isSingleMedia && filePath && media === "image" && ( +
+ +
+ )} + + {isSingleMedia && filePath && media === "video" && ( +
+
+ ); +} + +/* ─── Domain badge with favicon ─── */ + +function DomainBadge({ domain }: { domain: string }) { + const short = domain.replace(/^www\./, ""); + return ( + + {/* eslint-disable-next-line @next/next/no-img-element */} + + {short} + + ); +} + +/* ─── Step icons ─── */ + +function StepIcon({ kind }: { kind: StepKind }) { + const color = "var(--color-text-muted)"; + const size = 16; + + switch (kind) { + case "search": + return ( + + + + + ); + case "fetch": + return ( + + + + + + ); + case "read": + return ( + + + + + ); + case "exec": + return ( + + + + + ); + case "write": + return ( + + + + + ); + case "image": + return ( + + + + + + ); + default: + return ( + + + + ); + } +} + +function ErrorCircleIcon() { + return ( + + + + + + ); +} + +function PdfIcon() { + return ( + + + + + + + + ); +} + +/* ─── Header icons ─── */ + +function ThinkingIcon({ className }: { className?: string }) { + return ( + + + + + + ); +} + +function ChevronIcon({ className }: { className?: string }) { + return ( + + + + ); +} diff --git a/apps/web/app/components/charts/chart-panel.tsx b/apps/web/app/components/charts/chart-panel.tsx new file mode 100644 index 00000000000..ecba8e87a20 --- /dev/null +++ b/apps/web/app/components/charts/chart-panel.tsx @@ -0,0 +1,426 @@ +"use client"; + +import { useMemo } from "react"; +import { + BarChart, + Bar, + LineChart, + Line, + AreaChart, + Area, + PieChart, + Pie, + Cell, + RadarChart, + Radar, + PolarGrid, + PolarAngleAxis, + PolarRadiusAxis, + ScatterChart, + Scatter, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + Legend, + ResponsiveContainer, + FunnelChart, + Funnel, + LabelList, +} from "recharts"; +import type { PanelConfig } from "./types"; + +// --- Color palette derived from CSS variables + accessible defaults --- + +const CHART_PALETTE = [ + "#2563eb", // accent + "#60a5fa", // blue + "#22c55e", // green + "#f59e0b", // amber + "#c084fc", // purple + "#fb923c", // orange + "#14b8a6", // teal + "#f43f5e", // rose + "#a78bfa", // violet + "#38bdf8", // sky +]; + +type ChartPanelProps = { + config: PanelConfig; + data: Record[]; + /** Compact mode for inline chat cards */ + compact?: boolean; +}; + +// --- Shared tooltip/axis styles --- + +const axisStyle = { + fontSize: 11, + fill: "var(--color-text-muted)", +}; + +const gridStyle = { + stroke: "var(--color-border-strong)", + strokeDasharray: "3 3", +}; + +function tooltipStyle() { + return { + contentStyle: { + background: "var(--color-surface)", + border: "1px solid var(--color-border)", + borderRadius: 8, + fontSize: 12, + color: "var(--color-text)", + }, + itemStyle: { color: "var(--color-text)" }, + labelStyle: { color: "var(--color-text-muted)", marginBottom: 4 }, + }; +} + +// --- Formatters --- + +/** Safe string conversion for chart values (handles objects via JSON.stringify). */ +function toDisplayStr(val: unknown): string { + if (val == null) {return "";} + if (typeof val === "object") {return JSON.stringify(val);} + if (typeof val === "string") {return val;} + if (typeof val === "number" || typeof val === "boolean") {return String(val);} + // symbol, bigint, function β€” val is narrowed (object already handled above) + // eslint-disable-next-line @typescript-eslint/no-base-to-string + return String(val); +} + +function formatValue(val: unknown): string { + if (val === null || val === undefined) {return "";} + if (typeof val === "object") {return JSON.stringify(val);} + if (typeof val === "number") { + if (Math.abs(val) >= 1_000_000) {return `${(val / 1_000_000).toFixed(1)}M`;} + if (Math.abs(val) >= 1_000) {return `${(val / 1_000).toFixed(1)}K`;} + return Number.isInteger(val) ? String(val) : val.toFixed(2); + } + return toDisplayStr(val); +} + +function formatLabel(val: unknown): string { + if (val === null || val === undefined) {return "";} + const str = toDisplayStr(val); + // Truncate long date strings + if (str.length > 16 && !isNaN(Date.parse(str))) { + return str.slice(0, 10); + } + // Truncate long labels + if (str.length > 20) {return str.slice(0, 18) + "...";} + return str; +} + +// --- Chart renderers --- + +function CartesianChart({ + config, + data, + compact, + ChartComponent, + SeriesComponent, + areaProps, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; + ChartComponent: typeof BarChart ; + SeriesComponent: typeof Bar | typeof Line | typeof Area; + areaProps?: Record; +}) { + const { mapping } = config; + const xKey = mapping.xAxis ?? Object.keys(data[0] ?? {})[0] ?? "x"; + const yKeys = mapping.yAxis ?? Object.keys(data[0] ?? {}).filter((k) => k !== xKey); + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + return ( + + + + + + + {yKeys.length > 1 && !compact && } + {yKeys.map((key, i) => { + const color = colors[i % colors.length]; + const props: Record = { + key, + dataKey: key, + fill: color, + stroke: color, + name: key, + ...areaProps, + }; + if (SeriesComponent === Bar) { + props.radius = [4, 4, 0, 0]; + props.maxBarSize = 48; + } + if (SeriesComponent === Line) { + props.strokeWidth = 2; + props.dot = { r: 3, fill: color }; + props.activeDot = { r: 5 }; + } + if (SeriesComponent === Area) { + props.fillOpacity = 0.15; + props.strokeWidth = 2; + } + // @ts-expect-error - dynamic component props + return ; + })} + + + ); +} + +function PieDonutChart({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping, type } = config; + const nameKey = mapping.nameKey ?? Object.keys(data[0] ?? {})[0] ?? "name"; + const valueKey = mapping.valueKey ?? Object.keys(data[0] ?? {})[1] ?? "value"; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + const innerRadius = type === "donut" ? "50%" : 0; + + return ( + + + { + const p = props as Record; + const name = p.name; + const percent = typeof p.percent === "number" ? p.percent : 0; + return `${formatLabel(name)} ${(percent * 100).toFixed(0)}%`; + }) as never} + labelLine={!compact} + style={{ fontSize: 11 }} + > + {data.map((_, i) => ( + + ))} + + + {!compact && } + + + ); +} + +function RadarChartPanel({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping } = config; + const nameKey = mapping.xAxis ?? mapping.nameKey ?? Object.keys(data[0] ?? {})[0] ?? "name"; + const valueKeys = mapping.yAxis ?? [Object.keys(data[0] ?? {})[1] ?? "value"]; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + return ( + + + + + + {valueKeys.map((key, i) => ( + + ))} + + {!compact && valueKeys.length > 1 && } + + + ); +} + +function ScatterChartPanel({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping } = config; + const xKey = mapping.xAxis ?? Object.keys(data[0] ?? {})[0] ?? "x"; + const yKeys = mapping.yAxis ?? [Object.keys(data[0] ?? {})[1] ?? "y"]; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + return ( + + + + + + + {yKeys.map((key, i) => ( + + ))} + {!compact && yKeys.length > 1 && } + + + ); +} + +function FunnelChartPanel({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping } = config; + const nameKey = mapping.nameKey ?? Object.keys(data[0] ?? {})[0] ?? "name"; + const valueKey = mapping.valueKey ?? Object.keys(data[0] ?? {})[1] ?? "value"; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + // Funnel expects data with fill colors + const funnelData = data.map((row, i) => ({ + ...row, + fill: colors[i % colors.length], + })); + + return ( + + + + + + + + + ); +} + +// --- Main ChartPanel component --- + +export function ChartPanel({ config, data, compact }: ChartPanelProps) { + // Coerce numeric values for Recharts + const processedData = useMemo(() => { + if (!data || data.length === 0) {return [];} + const { mapping } = config; + const numericKeys = new Set([ + ...(mapping.yAxis ?? []), + ...(mapping.valueKey ? [mapping.valueKey] : []), + ]); + + return data.map((row) => { + const out: Record = { ...row }; + for (const key of numericKeys) { + if (key in out) { + const v = out[key]; + if (typeof v === "string" && v !== "" && !isNaN(Number(v))) { + out[key] = Number(v); + } + } + } + return out; + }); + }, [data, config]); + + if (processedData.length === 0) { + return ( +
+ No data +
+ ); + } + + switch (config.type) { + case "bar": + return ; + case "line": + return ; + case "area": + return ; + case "pie": + return ; + case "donut": + return ; + case "radar": + case "radialBar": + return ; + case "scatter": + return ; + case "funnel": + return ; + default: + return ; + } +} diff --git a/apps/web/app/components/charts/filter-bar.tsx b/apps/web/app/components/charts/filter-bar.tsx new file mode 100644 index 00000000000..c1d8a13715b --- /dev/null +++ b/apps/web/app/components/charts/filter-bar.tsx @@ -0,0 +1,349 @@ +"use client"; + +import { useEffect, useState, useCallback } from "react"; +import type { FilterConfig, FilterState, FilterValue } from "./types"; + +type FilterBarProps = { + filters: FilterConfig[]; + value: FilterState; + onChange: (state: FilterState) => void; +}; + +// --- Icons --- + +function FilterIcon() { + return ( + + + + ); +} + +function XIcon() { + return ( + + + + ); +} + +// --- Individual filter components --- + +function DateRangeFilter({ + filter, + value, + onChange, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; +}) { + const current = value?.type === "dateRange" ? value : { type: "dateRange" as const }; + + return ( +
+ + onChange({ ...current, from: e.target.value || undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + colorScheme: "dark", + }} + /> + to + onChange({ ...current, to: e.target.value || undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + colorScheme: "dark", + }} + /> +
+ ); +} + +function SelectFilter({ + filter, + value, + onChange, + options, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; + options: string[]; +}) { + const current = value?.type === "select" ? value.value : undefined; + + return ( +
+ + +
+ ); +} + +function MultiSelectFilter({ + filter, + value, + onChange, + options, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; + options: string[]; +}) { + const current = value?.type === "multiSelect" ? (value.values ?? []) : []; + + const toggleOption = (opt: string) => { + const next = current.includes(opt) + ? current.filter((v) => v !== opt) + : [...current, opt]; + onChange({ type: "multiSelect", values: next.length > 0 ? next : undefined }); + }; + + return ( +
+ +
+ {options.map((opt) => { + const selected = current.includes(opt); + return ( + + ); + })} +
+
+ ); +} + +function NumberFilter({ + filter, + value, + onChange, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; +}) { + const current = value?.type === "number" ? value : { type: "number" as const }; + + return ( +
+ + onChange({ ...current, min: e.target.value ? Number(e.target.value) : undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none w-20" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + }} + /> + to + onChange({ ...current, max: e.target.value ? Number(e.target.value) : undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none w-20" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + }} + /> +
+ ); +} + +// --- Main FilterBar --- + +export function FilterBar({ filters, value, onChange }: FilterBarProps) { + // Fetch options for select/multiSelect filters + const [optionsMap, setOptionsMap] = useState>({}); + + const fetchOptions = useCallback(async () => { + const toFetch = filters.filter( + (f) => (f.type === "select" || f.type === "multiSelect") && f.sql, + ); + if (toFetch.length === 0) {return;} + + const results: Record = {}; + await Promise.all( + toFetch.map(async (f) => { + try { + const res = await fetch("/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: f.sql }), + }); + if (!res.ok) {return;} + const data = await res.json(); + const rows: Record[] = data.rows ?? []; + // Extract the first column's values as options + const opts = rows + .map((r) => { + const vals = Object.values(r); + const v = vals[0]; + if (v == null) {return null;} + if (typeof v === "object") {return JSON.stringify(v);} + // eslint-disable-next-line @typescript-eslint/no-base-to-string -- v narrowed, object handled above + return typeof v === "string" ? v : (typeof v === "number" || typeof v === "boolean" ? String(v) : String(v)); + }) + .filter((v): v is string => v !== null); + results[f.id] = opts; + } catch { + // skip failed option fetches + } + }), + ); + setOptionsMap(results); + }, [filters]); + + useEffect(() => { + void fetchOptions(); + }, [fetchOptions]); + + const handleFilterChange = useCallback( + (filterId: string, v: FilterValue) => { + onChange({ ...value, [filterId]: v }); + }, + [value, onChange], + ); + + const hasActiveFilters = Object.values(value).some((v) => { + if (!v) {return false;} + if (v.type === "dateRange") {return v.from || v.to;} + if (v.type === "select") {return v.value;} + if (v.type === "multiSelect") {return v.values && v.values.length > 0;} + if (v.type === "number") {return v.min !== undefined || v.max !== undefined;} + return false; + }); + + const clearFilters = () => onChange({}); + + if (filters.length === 0) {return null;} + + return ( +
+ + + Filters + + + {filters.map((filter) => { + const fv = value[filter.id]; + switch (filter.type) { + case "dateRange": + return ( + handleFilterChange(filter.id, v)} + /> + ); + case "select": + return ( + handleFilterChange(filter.id, v)} + options={optionsMap[filter.id] ?? []} + /> + ); + case "multiSelect": + return ( + handleFilterChange(filter.id, v)} + options={optionsMap[filter.id] ?? []} + /> + ); + case "number": + return ( + handleFilterChange(filter.id, v)} + /> + ); + default: + return null; + } + })} + + {hasActiveFilters && ( + + )} +
+ ); +} diff --git a/apps/web/app/components/charts/report-card.tsx b/apps/web/app/components/charts/report-card.tsx new file mode 100644 index 00000000000..12e4d7a33a0 --- /dev/null +++ b/apps/web/app/components/charts/report-card.tsx @@ -0,0 +1,464 @@ +"use client"; + +import { useEffect, useState, useCallback } from "react"; +import { motion, AnimatePresence } from "framer-motion"; +import { ChartPanel } from "./chart-panel"; +import type { ReportConfig, PanelConfig } from "./types"; + +type ReportCardProps = { + config: ReportConfig; +}; + +// --- Icons --- + +function ChartBarIcon() { + return ( + + + + + + ); +} + +function ExpandIcon() { + return ( + + + + + + + ); +} + +function CollapseIcon() { + return ( + + + + + + + ); +} + +function PinIcon() { + return ( + + + + + ); +} + +function RefreshIcon() { + return ( + + + + + + + ); +} + +// --- Panel data state --- + +type PanelData = { + rows: Record[]; + loading: boolean; + error?: string; +}; + +// --- Grid size helpers --- + +function panelColSpan(size?: string): string { + switch (size) { + case "full": + return "col-span-6"; + case "third": + return "col-span-2"; + case "half": + default: + return "col-span-3"; + } +} + +// --- Main ReportCard --- + +export function ReportCard({ config }: ReportCardProps) { + const [panelData, setPanelData] = useState>({}); + const [pinning, setPinning] = useState(false); + const [pinned, setPinned] = useState(false); + const [expanded, setExpanded] = useState(false); + + // In compact mode show at most 2 panels; expanded shows all + const visiblePanels = expanded ? config.panels : config.panels.slice(0, 2); + const hasMore = config.panels.length > 2; + + // Execute panel SQL queries + const executePanels = useCallback(async (panels: PanelConfig[]) => { + const initial: Record = {}; + for (const panel of panels) { + initial[panel.id] = { rows: [], loading: true }; + } + setPanelData((prev) => ({ ...prev, ...initial })); + + await Promise.all( + panels.map(async (panel) => { + try { + const res = await fetch("/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: panel.sql }), + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { rows: [], loading: false, error: data.error || `HTTP ${res.status}` }, + })); + return; + } + const data = await res.json(); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { rows: data.rows ?? [], loading: false }, + })); + } catch (err) { + setPanelData((prev) => ({ + ...prev, + [panel.id]: { rows: [], loading: false, error: err instanceof Error ? err.message : "Failed" }, + })); + } + }), + ); + }, []); + + // Load initial compact panels + useEffect(() => { + void executePanels(config.panels.slice(0, 2)); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // When expanding, fetch any panels not yet loaded + const handleToggleExpand = useCallback(() => { + setExpanded((prev) => { + const next = !prev; + if (next && hasMore) { + const unloaded = config.panels.filter((p) => !panelData[p.id]); + if (unloaded.length > 0) { + void executePanels(unloaded); + } + } + return next; + }); + }, [hasMore, config.panels, panelData, executePanels]); + + // Refresh all visible panels + const handleRefresh = useCallback(() => { + void executePanels(expanded ? config.panels : config.panels.slice(0, 2)); + }, [expanded, config.panels, executePanels]); + + // Pin report to workspace /reports directory + const handlePin = async () => { + setPinning(true); + try { + const slug = config.title + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40); + const filename = `${slug}.report.json`; + + await fetch("/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: `reports/${filename}`, + content: JSON.stringify(config, null, 2), + }), + }); + setPinned(true); + } catch { + // silently fail + } finally { + setPinning(false); + } + }; + + return ( +
+ {/* Header */} +
+
+ + + + + {config.title} + + + {config.panels.length} chart{config.panels.length !== 1 ? "s" : ""} + +
+ +
+ {expanded && ( + + )} + {!pinned ? ( + + ) : ( + + + + + Pinned + + )} + +
+
+ + {/* Description */} + {config.description && ( +
+

+ {config.description} +

+
+ )} + + + {expanded ? ( + /* ── Expanded: full grid with all panels ── */ + +
+ {config.panels.map((panel) => ( + + ))} +
+ + ) : ( + /* ── Compact: max 2 panels ── */ + +
1 ? "grid-cols-2" : "grid-cols-1"}`}> + {visiblePanels.map((panel) => ( + + ))} +
+ + {/* More panels indicator */} + {hasMore && ( + + )} + + )} + +
+ ); +} + +// --- Compact panel card for inline rendering --- + +function CompactPanelCard({ + panel, + data, +}: { + panel: PanelConfig; + data?: PanelData; +}) { + return ( +
+
+

+ {panel.title} +

+
+
+ {data?.loading ? ( +
+
+
+ ) : data?.error ? ( +
+

+ {data.error} +

+
+ ) : ( + + )} +
+
+ ); +} + +// --- Expanded panel card for full report view --- + +function ExpandedPanelCard({ + panel, + data, +}: { + panel: PanelConfig; + data?: PanelData; +}) { + const colSpan = panelColSpan(panel.size); + + return ( +
+
+

+ {panel.title} +

+ {data && !data.loading && !data.error && ( + + {data.rows.length} rows + + )} +
+
+ {data?.loading ? ( +
+
+
+ ) : data?.error ? ( +
+

+ Query error +

+

+ {data.error} +

+
+ ) : ( + + )} +
+
+ ); +} diff --git a/apps/web/app/components/charts/report-viewer.tsx b/apps/web/app/components/charts/report-viewer.tsx new file mode 100644 index 00000000000..72690e76e1d --- /dev/null +++ b/apps/web/app/components/charts/report-viewer.tsx @@ -0,0 +1,407 @@ +"use client"; + +import { useEffect, useState, useCallback, useMemo } from "react"; +import { ChartPanel } from "./chart-panel"; +import { FilterBar } from "./filter-bar"; +import type { ReportConfig, FilterState, PanelConfig, FilterConfig } from "./types"; + +type ReportViewerProps = { + /** Report config object (inline or loaded) */ + config?: ReportConfig; + /** Path to load report config from filesystem */ + reportPath?: string; +}; + +// --- Icons --- + +function ChartBarIcon({ size = 20 }: { size?: number }) { + return ( + + + + + + ); +} + +function RefreshIcon() { + return ( + + + + + + + ); +} + +// --- Helpers --- + +type PanelData = { + panelId: string; + rows: Record[]; + loading: boolean; + error?: string; +}; + +/** Build filter entries for the API from active filter state + filter configs. */ +function buildFilterEntries( + filterState: FilterState, + filterConfigs: FilterConfig[], +): Array<{ id: string; column: string; value: FilterState[string] }> { + const entries: Array<{ id: string; column: string; value: FilterState[string] }> = []; + for (const fc of filterConfigs) { + const v = filterState[fc.id]; + if (!v) {continue;} + // Only include if the filter has an active value + const hasValue = + (v.type === "dateRange" && (v.from || v.to)) || + (v.type === "select" && v.value) || + (v.type === "multiSelect" && v.values && v.values.length > 0) || + (v.type === "number" && (v.min !== undefined || v.max !== undefined)); + if (hasValue) { + entries.push({ id: fc.id, column: fc.column, value: v }); + } + } + return entries; +} + +// --- Grid size helpers --- + +function panelColSpan(size?: string): string { + switch (size) { + case "full": + return "col-span-6"; + case "third": + return "col-span-2"; + case "half": + default: + return "col-span-3"; + } +} + +// --- Main ReportViewer --- + +export function ReportViewer({ config: propConfig, reportPath }: ReportViewerProps) { + const [config, setConfig] = useState(propConfig ?? null); + const [configLoading, setConfigLoading] = useState(!propConfig && !!reportPath); + const [configError, setConfigError] = useState(null); + const [panelData, setPanelData] = useState>({}); + const [filterState, setFilterState] = useState({}); + const [refreshKey, setRefreshKey] = useState(0); + + // Load report config from filesystem if path provided + useEffect(() => { + if (propConfig) { + setConfig(propConfig); + return; + } + if (!reportPath) {return;} + + let cancelled = false; + setConfigLoading(true); + setConfigError(null); + + fetch(`/api/workspace/file?path=${encodeURIComponent(reportPath)}`) + .then(async (res) => { + if (!res.ok) {throw new Error(`Failed to load report: HTTP ${res.status}`);} + const data = await res.json(); + if (cancelled) {return;} + try { + const parsed = JSON.parse(data.content) as ReportConfig; + setConfig(parsed); + } catch { + throw new Error("Invalid report JSON"); + } + }) + .catch((err) => { + if (!cancelled) { + setConfigError(err instanceof Error ? err.message : "Failed to load report"); + } + }) + .finally(() => { + if (!cancelled) {setConfigLoading(false);} + }); + + return () => { cancelled = true; }; + }, [propConfig, reportPath]); + + // Execute all panel SQL queries when config or filters change + const executeAllPanels = useCallback(async () => { + if (!config) {return;} + + const filterEntries = buildFilterEntries(filterState, config.filters ?? []); + + // Mark all panels as loading + const initialState: Record = {}; + for (const panel of config.panels) { + initialState[panel.id] = { panelId: panel.id, rows: [], loading: true }; + } + setPanelData(initialState); + + // Execute all panels in parallel + await Promise.all( + config.panels.map(async (panel) => { + try { + const res = await fetch("/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sql: panel.sql, + filters: filterEntries.length > 0 ? filterEntries : undefined, + }), + }); + + if (!res.ok) { + const data = await res.json().catch(() => ({})); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { + panelId: panel.id, + rows: [], + loading: false, + error: data.error || `HTTP ${res.status}`, + }, + })); + return; + } + + const data = await res.json(); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { + panelId: panel.id, + rows: data.rows ?? [], + loading: false, + }, + })); + } catch (err) { + setPanelData((prev) => ({ + ...prev, + [panel.id]: { + panelId: panel.id, + rows: [], + loading: false, + error: err instanceof Error ? err.message : "Query failed", + }, + })); + } + }), + ); + }, [config, filterState]); + + // Re-execute when config, filters, or refresh key changes + useEffect(() => { + void executeAllPanels(); + }, [executeAllPanels, refreshKey]); + + const totalRows = useMemo(() => { + return Object.values(panelData).reduce((sum, pd) => sum + pd.rows.length, 0); + }, [panelData]); + + // --- Loading state --- + if (configLoading) { + return ( +
+
+ + Loading report... + +
+ ); + } + + // --- Error state --- + if (configError) { + return ( +
+ +

+ Failed to load report +

+

+ {configError} +

+
+ ); + } + + if (!config) { + return ( +
+

+ No report configuration found +

+
+ ); + } + + return ( +
+ {/* Report header */} +
+
+
+
+ + + +

+ {config.title} +

+
+ {config.description && ( +

+ {config.description} +

+ )} +
+ +
+ + {config.panels.length} panel{config.panels.length !== 1 ? "s" : ""} + + + {totalRows} rows + + +
+
+
+ + {/* Filters */} + {config.filters && config.filters.length > 0 && ( + + )} + + {/* Panel grid */} +
+
+ {config.panels.map((panel) => ( + + ))} +
+
+
+ ); +} + +// --- Individual panel card --- + +function PanelCard({ + panel, + data, +}: { + panel: PanelConfig; + data?: PanelData; +}) { + const colSpan = panelColSpan(panel.size); + + return ( +
+ {/* Panel header */} +
+

+ {panel.title} +

+ {data && !data.loading && !data.error && ( + + {data.rows.length} rows + + )} +
+ + {/* Chart area */} +
+ {data?.loading ? ( +
+
+
+ ) : data?.error ? ( +
+

+ Query error +

+

+ {data.error} +

+
+ ) : ( + + )} +
+
+ ); +} diff --git a/apps/web/app/components/charts/types.ts b/apps/web/app/components/charts/types.ts new file mode 100644 index 00000000000..150dfdc3d36 --- /dev/null +++ b/apps/web/app/components/charts/types.ts @@ -0,0 +1,64 @@ +/** Shared types for the report/analytics system. */ + +export type ChartType = + | "bar" + | "line" + | "area" + | "pie" + | "donut" + | "radar" + | "radialBar" + | "scatter" + | "funnel"; + +export type PanelSize = "full" | "half" | "third"; + +export type PanelMapping = { + /** Key for x-axis or category axis */ + xAxis?: string; + /** One or more keys for y-axis values (supports stacked/multi-series) */ + yAxis?: string[]; + /** Key used as label for pie/donut/funnel */ + nameKey?: string; + /** Key used as value for pie/donut/funnel */ + valueKey?: string; + /** Custom colors for series (hex). Falls back to palette. */ + colors?: string[]; +}; + +export type PanelConfig = { + id: string; + title: string; + type: ChartType; + sql: string; + mapping: PanelMapping; + size?: PanelSize; +}; + +export type FilterType = "dateRange" | "select" | "multiSelect" | "number"; + +export type FilterConfig = { + id: string; + type: FilterType; + label: string; + column: string; + /** SQL to fetch available options (for select/multiSelect) */ + sql?: string; +}; + +export type ReportConfig = { + version: number; + title: string; + description?: string; + panels: PanelConfig[]; + filters?: FilterConfig[]; +}; + +/** Active filter values keyed by filter ID */ +export type FilterState = Record; + +export type FilterValue = + | { type: "dateRange"; from?: string; to?: string } + | { type: "select"; value?: string } + | { type: "multiSelect"; values?: string[] } + | { type: "number"; min?: number; max?: number }; diff --git a/apps/web/app/components/chat-message.tsx b/apps/web/app/components/chat-message.tsx new file mode 100644 index 00000000000..b8b55e2028e --- /dev/null +++ b/apps/web/app/components/chat-message.tsx @@ -0,0 +1,940 @@ +"use client"; + +import dynamic from "next/dynamic"; +import type { UIMessage } from "ai"; +import { memo, useMemo, useState } from "react"; +import { motion, AnimatePresence } from "framer-motion"; +import type { Components } from "react-markdown"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import { ChainOfThought, type ChainPart } from "./chain-of-thought"; +import { splitReportBlocks, hasReportBlocks } from "@/lib/report-blocks"; +import { splitDiffBlocks, hasDiffBlocks } from "@/lib/diff-blocks"; +import type { ReportConfig } from "./charts/types"; +import { DiffCard } from "./diff-viewer"; +import { SyntaxBlock } from "./syntax-block"; + +// Lazy-load ReportCard (uses Recharts which is heavy) +const ReportCard = dynamic( + () => + import("./charts/report-card").then((m) => ({ + default: m.ReportCard, + })), + { + ssr: false, + loading: () => ( +
+ ), + }, +); + +/* ─── Silent-reply leak filter ─── */ + +const _SILENT_TOKEN = "NO_REPLY"; + +function isLeakedSilentToken(text: string): boolean { + const t = text.trim(); + if (!t) {return false;} + if (new RegExp(`^${_SILENT_TOKEN}\\W*$`).test(t)) {return true;} + if (_SILENT_TOKEN.startsWith(t) && t.length >= 2 && t.length < _SILENT_TOKEN.length) {return true;} + return false; +} + +/* ─── Part grouping ─── */ + +type MessageSegment = + | { type: "text"; text: string } + | { type: "chain"; parts: ChainPart[] } + | { type: "report-artifact"; config: ReportConfig } + | { type: "diff-artifact"; diff: string } + | { type: "subagent-card"; task: string; label?: string; status: "running" | "done" | "error" }; + +/** Map AI SDK tool state string to a simplified status */ +function toolStatus(state: string): "running" | "done" | "error" { + if (state === "output-available") { + return "done"; + } + if (state === "error") { + return "error"; + } + return "running"; +} + +/** + * Group consecutive non-text parts (reasoning + tools) into chain-of-thought + * blocks, with text parts standing alone between them. + */ +function groupParts(parts: UIMessage["parts"]): MessageSegment[] { + const segments: MessageSegment[] = []; + let chain: ChainPart[] = []; + + const flush = (textFollows?: boolean) => { + if (chain.length > 0) { + // If text content follows this chain, all tools must have + // completed β€” force any stuck "running" tools to "done". + if (textFollows) { + for (const cp of chain) { + if (cp.kind === "tool" && cp.status === "running") { + cp.status = "done"; + } + } + } + segments.push({ type: "chain", parts: [...chain] }); + chain = []; + } + }; + + for (const part of parts) { + if (part.type === "text") { + const text = (part as { type: "text"; text: string }).text; + if (isLeakedSilentToken(text)) { continue; } + flush(true); + if (hasReportBlocks(text)) { + segments.push( + ...(splitReportBlocks(text) as MessageSegment[]), + ); + } else if (hasDiffBlocks(text)) { + for (const seg of splitDiffBlocks(text)) { + if (seg.type === "diff-artifact") { + segments.push({ type: "diff-artifact", diff: seg.diff }); + } else { + segments.push({ type: "text", text: seg.text }); + } + } + } else { + segments.push({ type: "text", text }); + } + } else if (part.type === "reasoning") { + const rp = part as { + type: "reasoning"; + text: string; + state?: string; + }; + // Skip lifecycle/compaction status labels β€” they add noise + // (e.g. "Preparing response...", "Optimizing session context...") + const statusLabels = [ + "Preparing response...", + "Optimizing session context...", + ]; + const isStatus = statusLabels.some((l) => + rp.text.startsWith(l), + ); + if (!isStatus) { + chain.push({ + kind: "reasoning", + text: rp.text, + isStreaming: rp.state === "streaming", + }); + } + } else if (part.type === "dynamic-tool") { + const tp = part as { + type: "dynamic-tool"; + toolName: string; + toolCallId: string; + state: string; + input?: unknown; + output?: unknown; + }; + if (tp.toolName === "sessions_spawn") { + flush(true); + const args = asRecord(tp.input); + const task = typeof args?.task === "string" ? args.task : "Subagent task"; + const label = typeof args?.label === "string" ? args.label : undefined; + segments.push({ type: "subagent-card", task, label, status: toolStatus(tp.state) }); + } else { + chain.push({ + kind: "tool", + toolName: tp.toolName, + toolCallId: tp.toolCallId, + status: toolStatus(tp.state), + args: asRecord(tp.input), + output: asRecord(tp.output), + }); + } + } else if (part.type.startsWith("tool-")) { + // Handles both live SSE parts (input/output fields) and + // persisted JSONL parts (args/result fields from tool-invocation) + const tp = part as { + type: string; + toolCallId: string; + toolName?: string; + state?: string; + title?: string; + input?: unknown; + output?: unknown; + // Persisted JSONL format uses args/result instead + args?: unknown; + result?: unknown; + errorText?: string; + }; + const resolvedToolName = tp.title ?? tp.toolName ?? part.type.replace("tool-", ""); + if (resolvedToolName === "sessions_spawn") { + flush(true); + const args = asRecord(tp.input) ?? asRecord(tp.args); + const task = typeof args?.task === "string" ? args.task : "Subagent task"; + const label = typeof args?.label === "string" ? args.label : undefined; + const resolvedState = + tp.state ?? + (tp.errorText ? "error" : ("result" in tp || "output" in tp) ? "output-available" : "input-available"); + segments.push({ type: "subagent-card", task, label, status: toolStatus(resolvedState) }); + } else { + // Persisted tool-invocation parts have no state field but + // include result/output/errorText to indicate completion. + const resolvedState = + tp.state ?? + (tp.errorText ? "error" : ("result" in tp || "output" in tp) ? "output-available" : "input-available"); + chain.push({ + kind: "tool", + toolName: resolvedToolName, + toolCallId: tp.toolCallId, + status: toolStatus(resolvedState), + args: asRecord(tp.input) ?? asRecord(tp.args), + output: asRecord(tp.output) ?? asRecord(tp.result), + }); + } + } + } + + flush(); + return segments; +} + +/** Safely cast unknown to Record if it's a non-null object */ +function asRecord( + val: unknown, +): Record | undefined { + if (val && typeof val === "object" && !Array.isArray(val)) { + return val as Record; + } + return undefined; +} + +/* ─── Attachment parsing for sent messages ─── */ + +function parseAttachments( + text: string, +): { paths: string[]; message: string } | null { + const match = text.match(/\[Attached files: (.+?)\]/); + if (!match) {return null;} + const afterIdx = (match.index ?? 0) + match[0].length; + const message = text.slice(afterIdx).trim(); + const paths = match[1] + .split(", ") + .map((p) => p.trim()) + .filter(Boolean); + return { paths, message }; +} + +function getCategoryFromPath( + filePath: string, +): "image" | "video" | "audio" | "pdf" | "code" | "document" | "other" { + const ext = filePath.split(".").pop()?.toLowerCase() ?? ""; + if ( + [ + "jpg", "jpeg", "png", "gif", "webp", "svg", "bmp", + "ico", "tiff", "heic", + ].includes(ext) + ) + {return "image";} + if (["mp4", "webm", "mov", "avi", "mkv", "flv"].includes(ext)) + {return "video";} + if (["mp3", "wav", "ogg", "aac", "flac", "m4a"].includes(ext)) + {return "audio";} + if (ext === "pdf") {return "pdf";} + if ( + [ + "js", "ts", "tsx", "jsx", "py", "rb", "go", "rs", + "java", "cpp", "c", "h", "css", "html", "json", + "yaml", "yml", "toml", "md", "sh", "bash", "sql", + "swift", "kt", + ].includes(ext) + ) + {return "code";} + if ( + [ + "doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt", + "rtf", "csv", "pages", "numbers", "key", + ].includes(ext) + ) + {return "document";} + return "other"; +} + +function _shortenPath(path: string): string { + return path + .replace(/^\/Users\/[^/]+/, "~") + .replace(/^\/home\/[^/]+/, "~") + .replace(/^[A-Z]:\\Users\\[^\\]+/, "~"); +} + +const _attachCategoryMeta: Record = { + image: { bg: "rgba(16, 185, 129, 0.15)", fg: "#10b981" }, + video: { bg: "rgba(139, 92, 246, 0.15)", fg: "#8b5cf6" }, + audio: { bg: "rgba(245, 158, 11, 0.15)", fg: "#f59e0b" }, + pdf: { bg: "rgba(239, 68, 68, 0.15)", fg: "#ef4444" }, + code: { bg: "rgba(59, 130, 246, 0.15)", fg: "#3b82f6" }, + document: { bg: "rgba(107, 114, 128, 0.15)", fg: "#6b7280" }, + other: { bg: "rgba(107, 114, 128, 0.10)", fg: "#9ca3af" }, +}; + +function _AttachFileIcon({ category }: { category: string }) { + const props = { + width: 14, + height: 14, + viewBox: "0 0 24 24", + fill: "none", + stroke: "currentColor", + strokeWidth: 2, + strokeLinecap: "round" as const, + strokeLinejoin: "round" as const, + }; + switch (category) { + case "image": + return ( + + + + + + ); + case "video": + return ( + + + + + ); + case "audio": + return ( + + + + + + ); + case "pdf": + return ( + + + + + + + ); + case "code": + return ( + + + + + ); + case "document": + return ( + + + + + + + + ); + default: + return ( + + + + + ); + } +} + +function AttachedFilesCard({ paths }: { paths: string[] }) { + return ( +
+ {paths.map((filePath, i) => { + const category = getCategoryFromPath(filePath); + const src = category === "image" + ? `/api/workspace/raw-file?path=${encodeURIComponent(filePath)}` + : `/api/workspace/thumbnail?path=${encodeURIComponent(filePath)}&size=200`; + const ext = filePath.split(".").pop()?.toUpperCase() ?? ""; + + return ( +
+ {filePath.split("/").pop() { (e.currentTarget as HTMLImageElement).style.display = "none"; }} + /> + {category !== "image" && ( + + {ext} + + )} +
+ ); + })} +
+ ); +} + +/* ─── File path detection for clickable inline code ─── */ + +/** + * Detect whether an inline code string looks like a local file/directory path. + * Matches anything starting with: + * ~/ (home-relative) + * / (absolute) + * ./ (current-dir-relative) + * ../ (parent-dir-relative) + * Must contain at least one `/` separator to distinguish from plain commands. + */ +function looksLikeFilePath(text: string): boolean { + const t = text.trim(); + if (!t || t.length < 3 || t.length > 500) {return false;} + // Full path prefix + if (t.startsWith("~/") || t.startsWith("/") || t.startsWith("./") || t.startsWith("../")) { + const afterPrefix = t.startsWith("~/") ? t.slice(2) : + t.startsWith("../") ? t.slice(3) : + t.startsWith("./") ? t.slice(2) : + t.slice(1); + return afterPrefix.includes("/") || afterPrefix.includes("."); + } + // Bare filename with a known extension (e.g. "Rachapoom-Passport.pdf") + const fileExtPattern = /\.(pdf|docx?|xlsx?|pptx?|csv|txt|rtf|pages|numbers|key|md|json|yaml|yml|toml|xml|html?|css|jsx?|tsx?|py|rb|go|rs|java|cpp|c|h|sh|sql|swift|kt|png|jpe?g|gif|webp|svg|bmp|ico|heic|tiff|mp[34]|webm|mov|avi|mkv|flv|wav|ogg|aac|flac|m4a|zip|tar|gz|dmg)$/i; + if (fileExtPattern.test(t) && !t.includes(" ")) { + return true; + } + return false; +} + +/** Check if text looks like a filename (allows spaces, used for bold text). */ +function looksLikeFileName(text: string): boolean { + const t = text.trim(); + if (!t || t.length < 3 || t.length > 300) {return false;} + const fileExtPattern = /\.(pdf|docx?|xlsx?|pptx?|csv|txt|rtf|pages|numbers|key|md|json|yaml|yml|toml|xml|html?|css|jsx?|tsx?|py|rb|go|rs|java|cpp|c|h|sh|sql|swift|kt|png|jpe?g|gif|webp|svg|bmp|ico|heic|tiff|mp[34]|webm|mov|avi|mkv|flv|wav|ogg|aac|flac|m4a|zip|tar|gz|dmg)$/i; + return fileExtPattern.test(t); +} + +/** Open a file path using the system default application. */ +async function openFilePath(path: string, reveal = false) { + try { + const res = await fetch("/api/workspace/open-file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path, reveal }), + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + console.error("Failed to open file:", data); + } + } catch (err) { + console.error("Failed to open file:", err); + } +} + +type FilePathClickHandler = ( + path: string, +) => Promise | boolean | void; + +/** Convert file:// URLs to local paths for in-app preview routing. */ +function normalizePathReference(value: string): string { + const trimmed = value.trim(); + if (!trimmed.startsWith("file://")) { + return trimmed; + } + try { + const url = new URL(trimmed); + if (url.protocol !== "file:") { + return trimmed; + } + const decoded = decodeURIComponent(url.pathname); + // Windows file URLs are /C:/... in URL form + if (/^\/[A-Za-z]:\//.test(decoded)) { + return decoded.slice(1); + } + return decoded; + } catch { + return trimmed; + } +} + +/** Clickable file path inline code element */ +function FilePathCode({ + path, + children, + onFilePathClick, +}: { + path: string; + children: React.ReactNode; + onFilePathClick?: FilePathClickHandler; +}) { + const [status, setStatus] = useState<"idle" | "opening" | "error">("idle"); + + const handleClick = async (e: React.MouseEvent) => { + e.preventDefault(); + setStatus("opening"); + try { + if (onFilePathClick) { + const handled = await onFilePathClick(path); + if (handled === false) { + setStatus("error"); + setTimeout(() => setStatus("idle"), 2000); + return; + } + setStatus("idle"); + } else { + const res = await fetch("/api/workspace/open-file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path }), + }); + if (!res.ok) { + setStatus("error"); + setTimeout(() => setStatus("idle"), 2000); + } else { + setStatus("idle"); + } + } + } catch { + setStatus("error"); + setTimeout(() => setStatus("idle"), 2000); + } + }; + + const handleContextMenu = async (e: React.MouseEvent) => { + // Right-click reveals in Finder instead of opening + e.preventDefault(); + await openFilePath(path, true); + }; + + return ( + + {children} + + ); +} + +/* ─── Markdown component overrides for chat ─── */ + +function createMarkdownComponents( + onFilePathClick?: FilePathClickHandler, +): Components { + return { + // Open external links in new tab; intercept local file-path links + a: ({ href, children, ...props }) => { + const rawHref = typeof href === "string" ? href : ""; + const normalizedHref = normalizePathReference(rawHref); + const isExternal = + rawHref && (rawHref.startsWith("http://") || rawHref.startsWith("https://") || rawHref.startsWith("//")); + const isWorkspaceAppLink = rawHref.startsWith("/workspace"); + const isLocalPathLink = + !isWorkspaceAppLink && + (Boolean(rawHref.startsWith("file://")) || + looksLikeFilePath(normalizedHref)); + return ( + { + if (!isLocalPathLink || !onFilePathClick) {return;} + e.preventDefault(); + void onFilePathClick(normalizedHref); + }} + > + {children} + + ); + }, + // Route local image paths through raw-file API so workspace images render + img: ({ src, alt, ...props }) => { + const resolvedSrc = typeof src === "string" && !src.startsWith("http://") && !src.startsWith("https://") && !src.startsWith("data:") + ? `/api/workspace/raw-file?path=${encodeURIComponent(src)}` + : src; + return ( + // eslint-disable-next-line @next/next/no-img-element + {alt + ); + }, + // Syntax-highlighted fenced code blocks + pre: ({ children, ...props }) => { + const child = Array.isArray(children) ? children[0] : children; + if ( + child && + typeof child === "object" && + "type" in child && + (child as { type?: string }).type === "code" + ) { + const codeEl = child as { + props?: { + className?: string; + children?: string; + }; + }; + const className = codeEl.props?.className ?? ""; + const langMatch = className.match(/language-(\w+)/); + const lang = langMatch?.[1] ?? ""; + const code = + typeof codeEl.props?.children === "string" + ? codeEl.props.children.replace(/\n$/, "") + : ""; + + // Diff language: render as DiffCard + if (lang === "diff") { + return ; + } + + // Known language: syntax-highlight with shiki + if (lang) { + return ( +
+
+ {lang} +
+ +
+ ); + } + } + // Fallback: default pre rendering + return 
{children}
; + }, + // Inline code β€” detect file paths and make them clickable + code: ({ children, className, ...props }) => { + // If this code has a language class, it's inside a 
 and
+			// will be handled by the pre override above. Just return raw.
+			if (className?.startsWith("language-")) {
+				return (
+					
+						{children}
+					
+				);
+			}
+
+			// Check if the inline code content looks like a file path
+			const text = typeof children === "string" ? children : "";
+			const normalizedText = normalizePathReference(text);
+			if (normalizedText && looksLikeFilePath(normalizedText)) {
+				return (
+					
+						{children}
+					
+				);
+			}
+
+			// Regular inline code
+			return {children};
+		},
+		// Bold text β€” detect filenames and make them clickable
+		strong: ({ children, ...props }) => {
+			const text = typeof children === "string" ? children
+				: Array.isArray(children) ? children.filter((c) => typeof c === "string").join("")
+				: "";
+			if (text && looksLikeFileName(text)) {
+				return (
+					
+						
+							{children}
+						
+					
+				);
+			}
+			return {children};
+		},
+	};
+}
+
+/* ─── Chat message ─── */
+
+export const ChatMessage = memo(function ChatMessage({ message, isStreaming, onSubagentClick, onFilePathClick }: { message: UIMessage; isStreaming?: boolean; onSubagentClick?: (task: string) => void; onFilePathClick?: FilePathClickHandler }) {
+	const isUser = message.role === "user";
+	const segments = groupParts(message.parts);
+	const markdownComponents = useMemo(
+		() => createMarkdownComponents(onFilePathClick),
+		[onFilePathClick],
+	);
+
+	if (isUser) {
+		// User: right-aligned subtle pill
+		const textContent = segments
+			.filter(
+				(s): s is { type: "text"; text: string } =>
+					s.type === "text",
+			)
+			.map((s) => s.text)
+			.join("\n");
+
+		// Parse attachment prefix from sent messages
+		const attachmentInfo = parseAttachments(textContent);
+
+		if (attachmentInfo) {
+			return (
+				

+					{/* Attachment previews β€” standalone above the text bubble */}
+					
+					{/* Text bubble */}
+					{attachmentInfo.message && (
+						

+							

+								{attachmentInfo.message}
+							

+						

+					)}
+				

+			);
+		}
+
+		return (
+			

+				

+					

+						{textContent}
+					

+				

+			

+		);
+	}
+
+	// Find the last text segment index for streaming optimization
+	const lastTextIdx = isStreaming
+		? segments.reduce((acc, s, i) => (s.type === "text" ? i : acc), -1)
+		: -1;
+
+	// Assistant: free-flowing text, left-aligned, NO bubble
+	return (
+		

+			
+			{segments.map((segment, index) => {
+				if (segment.type === "text") {
+					// Detect agent error messages
+					const errorMatch = segment.text.match(
+						/^\[error\]\s*([\s\S]*)$/,
+					);
+					if (errorMatch) {
+						return (
+							

+								
+								
+									{errorMatch[1].trim()}
+								
+							

+						);
+					}
+
+					// During streaming, render the active text as plain text
+					// to avoid expensive ReactMarkdown re-parses on every token.
+					// Switch to full markdown once streaming ends.
+					if (index === lastTextIdx) {
+						return (
+							
+								{segment.text}
+							
+						);
+					}
+
+				return (
+			
+				
+					{segment.text}
+				
+			
+				);
+				}
+			if (segment.type === "report-artifact") {
+				return (
+					
+						
+					
+				);
+			}
+		if (segment.type === "diff-artifact") {
+			return (
+				
+					
+				
+			);
+		}
+		if (segment.type === "subagent-card") {
+			const truncatedTask = segment.task.length > 80 ? segment.task.slice(0, 80) + "..." : segment.task;
+			const isRunning = segment.status === "running";
+			return (
+				
+					
+				
+			);
+		}
+			return (
+				
+					
+				
+			);
+			})}
+			
+		

+	);
+});
diff --git a/apps/web/app/components/chat-panel.tsx b/apps/web/app/components/chat-panel.tsx
new file mode 100644
index 00000000000..06c6e26ea99
--- /dev/null
+++ b/apps/web/app/components/chat-panel.tsx
@@ -0,0 +1,2146 @@
+"use client";
+
+import { useChat } from "@ai-sdk/react";
+import { DefaultChatTransport, type UIMessage } from "ai";
+import {
+	forwardRef,
+	useCallback,
+	useEffect,
+	useImperativeHandle,
+	useMemo,
+	useRef,
+	useState,
+} from "react";
+import { ChatMessage } from "./chat-message";
+import {
+	FilePickerModal,
+	type SelectedFile,
+} from "./file-picker-modal";
+import { ChatEditor, type ChatEditorHandle } from "./tiptap/chat-editor";
+import {
+	DropdownMenu,
+	DropdownMenuContent,
+	DropdownMenuItem,
+	DropdownMenuTrigger,
+} from "./ui/dropdown-menu";
+import { UnicodeSpinner } from "./unicode-spinner";
+
+// ── Attachment types & helpers ──
+
+type AttachedFile = {
+	id: string;
+	name: string;
+	path: string;
+	/** True while the file is still uploading to the server. */
+	uploading?: boolean;
+	/** Local blob URL for instant preview before upload completes. */
+	localUrl?: string;
+};
+
+function getFileCategory(
+	name: string,
+): "image" | "video" | "audio" | "pdf" | "code" | "document" | "other" {
+	const ext = name.split(".").pop()?.toLowerCase() ?? "";
+	if (
+		[
+			"jpg", "jpeg", "png", "gif", "webp", "svg", "bmp",
+			"ico", "tiff", "heic",
+		].includes(ext)
+	)
+		{return "image";}
+	if (["mp4", "webm", "mov", "avi", "mkv", "flv"].includes(ext))
+		{return "video";}
+	if (["mp3", "wav", "ogg", "aac", "flac", "m4a"].includes(ext))
+		{return "audio";}
+	if (ext === "pdf") {return "pdf";}
+	if (
+		[
+			"js", "ts", "tsx", "jsx", "py", "rb", "go", "rs", "java",
+			"cpp", "c", "h", "css", "html", "json", "yaml", "yml",
+			"toml", "md", "sh", "bash", "sql", "swift", "kt",
+		].includes(ext)
+	)
+		{return "code";}
+	if (
+		[
+			"doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt",
+			"rtf", "csv", "pages", "numbers", "key",
+		].includes(ext)
+	)
+		{return "document";}
+	return "other";
+}
+
+function shortenPath(path: string): string {
+	return path
+		.replace(/^\/Users\/[^/]+/, "~")
+		.replace(/^\/home\/[^/]+/, "~")
+		.replace(/^[A-Z]:\\Users\\[^\\]+/, "~");
+}
+
+const categoryMeta: Record = {
+	image: { bg: "rgba(16, 185, 129, 0.12)", fg: "#10b981" },
+	video: { bg: "rgba(139, 92, 246, 0.12)", fg: "#8b5cf6" },
+	audio: { bg: "rgba(245, 158, 11, 0.12)", fg: "#f59e0b" },
+	pdf: { bg: "rgba(239, 68, 68, 0.12)", fg: "#ef4444" },
+	code: { bg: "rgba(59, 130, 246, 0.12)", fg: "#3b82f6" },
+	document: { bg: "rgba(107, 114, 128, 0.12)", fg: "#6b7280" },
+	other: { bg: "rgba(107, 114, 128, 0.08)", fg: "#9ca3af" },
+};
+
+function FileTypeIcon({ category }: { category: string }) {
+	const props = {
+		width: 16,
+		height: 16,
+		viewBox: "0 0 24 24",
+		fill: "none",
+		stroke: "currentColor",
+		strokeWidth: 2,
+		strokeLinecap: "round" as const,
+		strokeLinejoin: "round" as const,
+	};
+	switch (category) {
+		case "image":
+			return (
+				
+					
+					
+					
+				
+			);
+		case "video":
+			return (
+				
+					
+					
+				
+			);
+		case "audio":
+			return (
+				
+					
+					
+					
+				
+			);
+		case "pdf":
+			return (
+				
+					
+					
+					
+					
+				
+			);
+		case "code":
+			return (
+				
+					
+					
+				
+			);
+		case "document":
+			return (
+				
+					
+					
+					
+					
+					
+				
+			);
+		default:
+			return (
+				
+					
+					
+				
+			);
+	}
+}
+
+function QueueItem({
+	msg,
+	idx,
+	onEdit,
+	onSendNow,
+	onRemove,
+}: {
+	msg: QueuedMessage;
+	idx: number;
+	onEdit: (id: string, text: string) => void;
+	onSendNow: (id: string) => void;
+	onRemove: (id: string) => void;
+}) {
+	const [editing, setEditing] = useState(false);
+	const [draft, setDraft] = useState(msg.text);
+	const inputRef = useRef(null);
+
+	const autoResize = () => {
+		const el = inputRef.current;
+		if (!el) {return;}
+		el.style.height = "auto";
+		el.style.height = `${el.scrollHeight}px`;
+	};
+
+	useEffect(() => {
+		if (editing) {
+			inputRef.current?.focus();
+			const len = inputRef.current?.value.length ?? 0;
+			inputRef.current?.setSelectionRange(len, len);
+			autoResize();
+		}
+	}, [editing]);
+
+	const commitEdit = () => {
+		const trimmed = draft.trim();
+		if (trimmed && trimmed !== msg.text) {
+			onEdit(msg.id, trimmed);
+		} else {
+			setDraft(msg.text);
+		}
+		setEditing(false);
+	};
+
+	return (
+		
 0 ? "border-t" : ""}`}
+			style={idx > 0 ? { borderColor: "var(--color-border)" } : undefined}
+		>
+			
+				{idx + 1}
+			
+			{editing ? (
+