diff --git a/.agents/archive/PR_WORKFLOW_V1.md b/.agents/archive/PR_WORKFLOW_V1.md new file mode 100644 index 00000000000..1cb6ab653b5 --- /dev/null +++ b/.agents/archive/PR_WORKFLOW_V1.md @@ -0,0 +1,181 @@ +# PR Workflow for Maintainers + +Please read this in full and do not skip sections. +This is the single source of truth for the maintainer PR workflow. + +## Triage order + +Process PRs **oldest to newest**. Older PRs are more likely to have merge conflicts and stale dependencies; resolving them first keeps the queue healthy and avoids snowballing rebase pain. + +## Working rule + +Skills execute workflow. Maintainers provide judgment. +Always pause between skills to evaluate technical direction, not just command success. + +These three skills must be used in order: + +1. `review-pr` — review only, produce findings +2. `prepare-pr` — rebase, fix, gate, push to PR head branch +3. `merge-pr` — squash-merge, verify MERGED state, clean up + +They are necessary, but not sufficient. Maintainers must steer between steps and understand the code before moving forward. + +Treat PRs as reports first, code second. +If submitted code is low quality, ignore it and implement the best solution for the problem. + +Do not continue if you cannot verify the problem is real or test the fix. + +## Coding Agent + +Use ChatGPT 5.3 Codex High. Fall back to 5.2 Codex High or 5.3 Codex Medium if necessary. + +## PR quality bar + +- Do not trust PR code by default. +- Do not merge changes you cannot validate with a reproducible problem and a tested fix. +- Keep types strict. Do not use `any` in implementation code. +- Keep external-input boundaries typed and validated, including CLI input, environment variables, network payloads, and tool output. +- Keep implementations properly scoped. Fix root causes, not local symptoms. +- Identify and reuse canonical sources of truth so behavior does not drift across the codebase. +- Harden changes. Always evaluate security impact and abuse paths. +- Understand the system before changing it. Never make the codebase messier just to clear a PR queue. + +## Rebase and conflict resolution + +Before any substantive review or prep work, **always rebase the PR branch onto current `main` and resolve merge conflicts first**. A PR that cannot cleanly rebase is not ready for review — fix conflicts before evaluating correctness. + +- During `prepare-pr`: rebase onto `main` as the first step, before fixing findings or running gates. +- If conflicts are complex or touch areas you do not understand, stop and escalate. +- Prefer **rebase** for linear history; **squash** when commit history is messy or unhelpful. + +## Commit and changelog rules + +- Create commits with `scripts/committer "" `; avoid manual `git add`/`git commit` so staging stays scoped. +- Follow concise, action-oriented commit messages (e.g., `CLI: add verbose flag to send`). +- During `prepare-pr`, use this commit subject format: `fix: (openclaw#) thanks @`. +- Group related changes; avoid bundling unrelated refactors. +- Changelog workflow: keep the latest released version at the top (no `Unreleased`); after publishing, bump the version and start a new top section. +- When working on a PR: add a changelog entry with the PR number and thank the contributor. +- When working on an issue: reference the issue in the changelog entry. +- Pure test additions/fixes generally do **not** need a changelog entry unless they alter user-facing behavior or the user asks for one. + +## Co-contributor and clawtributors + +- If we squash, add the PR author as a co-contributor in the commit body using a `Co-authored-by:` trailer. +- When maintainer prepares and merges the PR, add the maintainer as an additional `Co-authored-by:` trailer too. +- Avoid `--auto` merges for maintainer landings. Merge only after checks are green so the maintainer account is the actor and attribution is deterministic. +- For squash merges, set `--author-email` to a reviewer-owned email with fallback candidates; if merge fails due to author-email validation, retry once with the next candidate. +- If you review a PR and later do work on it, land via merge/squash (no direct-main commits) and always add the PR author as a co-contributor. +- When merging a PR: leave a PR comment that explains exactly what we did, include the SHA hashes, and record the comment URL in the final report. +- When merging a PR from a new contributor: run `bun scripts/update-clawtributors.ts` to add their avatar to the README "Thanks to all clawtributors" list, then commit the regenerated README. + +## Review mode vs landing mode + +- **Review mode (PR link only):** read `gh pr view`/`gh pr diff`; **do not** switch branches; **do not** change code. +- **Landing mode (exception path):** use only when normal `review-pr -> prepare-pr -> merge-pr` flow cannot safely preserve attribution or cannot satisfy branch protection. Create an integration branch from `main`, bring in PR commits (**prefer rebase** for linear history; **merge allowed** when complexity/conflicts make it safer), apply fixes, add changelog (+ thanks + PR #), run full gate **locally before committing** (`pnpm build && pnpm check && pnpm test`), commit, merge back to `main`, then `git switch main` (never stay on a topic branch after landing). Important: the contributor needs to be in the git graph after this! + +## Pre-review safety checks + +- Before starting a review when a GH Issue/PR is pasted: use an isolated `.worktrees/pr-` checkout from `origin/main`. Do not require a clean main checkout, and do not run `git pull` in a dirty main checkout. +- PR review calls: prefer a single `gh pr view --json ...` to batch metadata/comments; run `gh pr diff` only when needed. +- PRs should summarize scope, note testing performed, and mention any user-facing changes or new flags. +- Read `docs/help/submitting-a-pr.md` ([Submitting a PR](https://docs.openclaw.ai/help/submitting-a-pr)) for what we expect from contributors. + +## Unified workflow + +Entry criteria: + +- PR URL/number is known. +- Problem statement is clear enough to attempt reproduction. +- A realistic verification path exists (tests, integration checks, or explicit manual validation). + +### 1) `review-pr` + +Purpose: + +- Review only: correctness, value, security risk, tests, docs, and changelog impact. +- Produce structured findings and a recommendation. + +Expected output: + +- Recommendation: ready, needs work, needs discussion, or close. +- `.local/review.md` with actionable findings. + +Maintainer checkpoint before `prepare-pr`: + +``` +What problem are they trying to solve? +What is the most optimal implementation? +Can we fix up everything? +Do we have any questions? +``` + +Stop and escalate instead of continuing if: + +- The problem cannot be reproduced or confirmed. +- The proposed PR scope does not match the stated problem. +- The design introduces unresolved security or trust-boundary concerns. + +### 2) `prepare-pr` + +Purpose: + +- Make the PR merge-ready on its head branch. +- Rebase onto current `main` first, then fix blocker/important findings, then run gates. +- In fresh worktrees, bootstrap dependencies before local gates (`pnpm install --frozen-lockfile`). + +Expected output: + +- Updated code and tests on the PR head branch. +- `.local/prep.md` with changes, verification, and current HEAD SHA. +- Final status: `PR is ready for /mergepr`. + +Maintainer checkpoint before `merge-pr`: + +``` +Is this the most optimal implementation? +Is the code properly scoped? +Is the code properly reusing existing logic in the codebase? +Is the code properly typed? +Is the code hardened? +Do we have enough tests? +Do we need regression tests? +Are tests using fake timers where appropriate? (e.g., debounce/throttle, retry backoff, timeout branches, delayed callbacks, polling loops) +Do not add performative tests, ensure tests are real and there are no regressions. +Do you see any follow-up refactors we should do? +Take your time, fix it properly, refactor if necessary. +Did any changes introduce any potential security vulnerabilities? +``` + +Stop and escalate instead of continuing if: + +- You cannot verify behavior changes with meaningful tests or validation. +- Fixing findings requires broad architecture changes outside safe PR scope. +- Security hardening requirements remain unresolved. + +### 3) `merge-pr` + +Purpose: + +- Merge only after review and prep artifacts are present and checks are green. +- Use deterministic squash merge flow (`--match-head-commit` + explicit subject/body with co-author trailer), then verify the PR ends in `MERGED` state. +- If no required checks are configured on the PR, treat that as acceptable and continue after branch-up-to-date validation. + +Go or no-go checklist before merge: + +- All BLOCKER and IMPORTANT findings are resolved. +- Verification is meaningful and regression risk is acceptably low. +- Docs and changelog are updated when required. +- Required CI checks are green and the branch is not behind `main`. + +Expected output: + +- Successful merge commit and recorded merge SHA. +- Worktree cleanup after successful merge. +- Comment on PR indicating merge was successful. + +Maintainer checkpoint after merge: + +- Were any refactors intentionally deferred and now need follow-up issue(s)? +- Did this reveal broader architecture or test gaps we should address? +- Run `bun scripts/update-clawtributors.ts` if the contributor is new. diff --git a/.agents/archive/merge-pr-v1/SKILL.md b/.agents/archive/merge-pr-v1/SKILL.md new file mode 100644 index 00000000000..0956699eb55 --- /dev/null +++ b/.agents/archive/merge-pr-v1/SKILL.md @@ -0,0 +1,304 @@ +--- +name: merge-pr +description: Merge a GitHub PR via squash after /prepare-pr. Use when asked to merge a ready PR. Do not push to main or modify code. Ensure the PR ends in MERGED state and clean up worktrees after success. +--- + +# Merge PR + +## Overview + +Merge a prepared PR via deterministic squash merge (`--match-head-commit` + explicit co-author trailer), then clean up the worktree after success. + +## Inputs + +- Ask for PR number or URL. +- If missing, use `.local/prep.env` from the worktree if present. +- If ambiguous, ask. + +## Safety + +- Use `gh pr merge --squash` as the only path to `main`. +- Do not run `git push` at all during merge. +- Do not use `gh pr merge --auto` for maintainer landings. +- Do not run gateway stop commands. Do not kill processes. Do not touch port 18792. + +## Execution Rule + +- Execute the workflow. Do not stop after printing the TODO checklist. +- If delegating, require the delegate to run commands and capture outputs. + +## Known Footguns + +- If you see "fatal: not a git repository", you are in the wrong directory. Move to the repo root and retry. +- Read `.local/review.md`, `.local/prep.md`, and `.local/prep.env` in the worktree. Do not skip. +- Always merge with `--match-head-commit "$PREP_HEAD_SHA"` to prevent racing stale or changed heads. +- Clean up `.worktrees/pr-` only after confirmed `MERGED`. + +## Completion Criteria + +- Ensure `gh pr merge` succeeds. +- Ensure PR state is `MERGED`, never `CLOSED`. +- Record the merge SHA. +- Leave a PR comment with merge SHA and prepared head SHA, and capture the comment URL. +- Run cleanup only after merge success. + +## First: Create a TODO Checklist + +Create a checklist of all merge steps, print it, then continue and execute the commands. + +## Setup: Use a Worktree + +Use an isolated worktree for all merge work. + +```sh +repo_root=$(git rev-parse --show-toplevel) +cd "$repo_root" +gh auth status + +WORKTREE_DIR=".worktrees/pr-" +cd "$WORKTREE_DIR" +``` + +Run all commands inside the worktree directory. + +## Load Local Artifacts (Mandatory) + +Expect these files from earlier steps: + +- `.local/review.md` from `/review-pr` +- `.local/prep.md` from `/prepare-pr` +- `.local/prep.env` from `/prepare-pr` + +```sh +ls -la .local || true + +for required in .local/review.md .local/prep.md .local/prep.env; do + if [ ! -f "$required" ]; then + echo "Missing $required. Stop and run /review-pr then /prepare-pr." + exit 1 + fi +done + +sed -n '1,120p' .local/review.md +sed -n '1,120p' .local/prep.md +source .local/prep.env +``` + +## Steps + +1. Identify PR meta and verify prepared SHA still matches + +```sh +pr_meta_json=$(gh pr view --json number,title,state,isDraft,author,headRefName,headRefOid,baseRefName,headRepository,body) +printf '%s\n' "$pr_meta_json" | jq '{number,title,state,isDraft,author:.author.login,head:.headRefName,headSha:.headRefOid,base:.baseRefName,headRepo:.headRepository.nameWithOwner,body}' +pr_title=$(printf '%s\n' "$pr_meta_json" | jq -r .title) +pr_number=$(printf '%s\n' "$pr_meta_json" | jq -r .number) +pr_head_sha=$(printf '%s\n' "$pr_meta_json" | jq -r .headRefOid) +contrib=$(printf '%s\n' "$pr_meta_json" | jq -r .author.login) +is_draft=$(printf '%s\n' "$pr_meta_json" | jq -r .isDraft) + +if [ "$is_draft" = "true" ]; then + echo "ERROR: PR is draft. Stop and run /prepare-pr after draft is cleared." + exit 1 +fi + +if [ "$pr_head_sha" != "$PREP_HEAD_SHA" ]; then + echo "ERROR: PR head changed after /prepare-pr (expected $PREP_HEAD_SHA, got $pr_head_sha). Re-run /prepare-pr." + exit 1 +fi +``` + +2. Run sanity checks + +Stop if any are true: + +- PR is a draft. +- Required checks are failing. +- Branch is behind main. + +If checks are pending, wait for completion before merging. Do not use `--auto`. +If no required checks are configured, continue. + +```sh +gh pr checks --required --watch --fail-fast || true +checks_json=$(gh pr checks --required --json name,bucket,state 2>/tmp/gh-checks.err || true) +if [ -z "$checks_json" ]; then + checks_json='[]' +fi +required_count=$(printf '%s\n' "$checks_json" | jq 'length') +if [ "$required_count" -eq 0 ]; then + echo "No required checks configured for this PR." +fi +printf '%s\n' "$checks_json" | jq -r '.[] | "\(.bucket)\t\(.name)\t\(.state)"' + +failed_required=$(printf '%s\n' "$checks_json" | jq '[.[] | select(.bucket=="fail")] | length') +pending_required=$(printf '%s\n' "$checks_json" | jq '[.[] | select(.bucket=="pending")] | length') +if [ "$failed_required" -gt 0 ]; then + echo "Required checks are failing, run /prepare-pr." + exit 1 +fi +if [ "$pending_required" -gt 0 ]; then + echo "Required checks are still pending, retry /merge-pr when green." + exit 1 +fi + +git fetch origin main +git fetch origin pull//head:pr- --force +git merge-base --is-ancestor origin/main pr- || (echo "PR branch is behind main, run /prepare-pr" && exit 1) +``` + +If anything is failing or behind, stop and say to run `/prepare-pr`. + +3. Merge PR with explicit attribution metadata + +```sh +reviewer=$(gh api user --jq .login) +reviewer_id=$(gh api user --jq .id) +coauthor_email=${COAUTHOR_EMAIL:-"$contrib@users.noreply.github.com"} +if [ -z "$coauthor_email" ] || [ "$coauthor_email" = "null" ]; then + contrib_id=$(gh api users/$contrib --jq .id) + coauthor_email="${contrib_id}+${contrib}@users.noreply.github.com" +fi + +gh_email=$(gh api user --jq '.email // ""' || true) +git_email=$(git config user.email || true) +mapfile -t reviewer_email_candidates < <( + printf '%s\n' \ + "$gh_email" \ + "$git_email" \ + "${reviewer_id}+${reviewer}@users.noreply.github.com" \ + "${reviewer}@users.noreply.github.com" | awk 'NF && !seen[$0]++' +) +[ "${#reviewer_email_candidates[@]}" -gt 0 ] || { echo "ERROR: could not resolve reviewer author email"; exit 1; } +reviewer_email="${reviewer_email_candidates[0]}" + +cat > .local/merge-body.txt < /prepare-pr -> /merge-pr. + +Prepared head SHA: $PREP_HEAD_SHA +Co-authored-by: $contrib <$coauthor_email> +Co-authored-by: $reviewer <$reviewer_email> +Reviewed-by: @$reviewer +EOF + +run_merge() { + local email="$1" + local stderr_file + stderr_file=$(mktemp) + if gh pr merge \ + --squash \ + --delete-branch \ + --match-head-commit "$PREP_HEAD_SHA" \ + --author-email "$email" \ + --subject "$pr_title (#$pr_number)" \ + --body-file .local/merge-body.txt \ + 2> >(tee "$stderr_file" >&2) + then + rm -f "$stderr_file" + return 0 + fi + merge_err=$(cat "$stderr_file") + rm -f "$stderr_file" + return 1 +} + +merge_err="" +selected_merge_author_email="$reviewer_email" +if ! run_merge "$selected_merge_author_email"; then + if printf '%s\n' "$merge_err" | rg -qi 'author.?email|email.*associated|associated.*email|invalid.*email' && [ "${#reviewer_email_candidates[@]}" -ge 2 ]; then + selected_merge_author_email="${reviewer_email_candidates[1]}" + echo "Retrying once with fallback author email: $selected_merge_author_email" + run_merge "$selected_merge_author_email" || { echo "ERROR: merge failed after fallback retry"; exit 1; } + else + echo "ERROR: merge failed" + exit 1 + fi +fi +``` + +Retry is allowed exactly once when the error is clearly author-email validation. + +4. Verify PR state and capture merge SHA + +```sh +state=$(gh pr view --json state --jq .state) +if [ "$state" != "MERGED" ]; then + echo "Merge not finalized yet (state=$state), waiting up to 15 minutes..." + for _ in $(seq 1 90); do + sleep 10 + state=$(gh pr view --json state --jq .state) + if [ "$state" = "MERGED" ]; then + break + fi + done +fi + +if [ "$state" != "MERGED" ]; then + echo "ERROR: PR state is $state after waiting. Leave worktree and retry /merge-pr later." + exit 1 +fi + +merge_sha=$(gh pr view --json mergeCommit --jq '.mergeCommit.oid') +if [ -z "$merge_sha" ] || [ "$merge_sha" = "null" ]; then + echo "ERROR: merge commit SHA missing." + exit 1 +fi + +commit_body=$(gh api repos/:owner/:repo/commits/$merge_sha --jq .commit.message) +contrib=${contrib:-$(gh pr view --json author --jq .author.login)} +reviewer=${reviewer:-$(gh api user --jq .login)} +printf '%s\n' "$commit_body" | rg -q "^Co-authored-by: $contrib <" || { echo "ERROR: missing PR author co-author trailer"; exit 1; } +printf '%s\n' "$commit_body" | rg -q "^Co-authored-by: $reviewer <" || { echo "ERROR: missing reviewer co-author trailer"; exit 1; } + +echo "merge_sha=$merge_sha" +``` + +5. PR comment + +Use a multiline heredoc with interpolation enabled. + +```sh +ok=0 +comment_output="" +for _ in 1 2 3; do + if comment_output=$(gh pr comment -F - <" --force +git branch -D temp/pr- 2>/dev/null || true +git branch -D pr- 2>/dev/null || true +git branch -D pr--prep 2>/dev/null || true +``` + +## Guardrails + +- Worktree only. +- Do not close PRs. +- End in MERGED state. +- Clean up only after merge success. +- Never push to main. Use `gh pr merge --squash` only. +- Do not run `git push` at all in this command. diff --git a/.agents/archive/merge-pr-v1/agents/openai.yaml b/.agents/archive/merge-pr-v1/agents/openai.yaml new file mode 100644 index 00000000000..9c10ae4d271 --- /dev/null +++ b/.agents/archive/merge-pr-v1/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Merge PR" + short_description: "Merge GitHub PRs via squash" + default_prompt: "Use $merge-pr to merge a GitHub PR via squash after preparation." diff --git a/.agents/archive/prepare-pr-v1/SKILL.md b/.agents/archive/prepare-pr-v1/SKILL.md new file mode 100644 index 00000000000..91c4508a07a --- /dev/null +++ b/.agents/archive/prepare-pr-v1/SKILL.md @@ -0,0 +1,336 @@ +--- +name: prepare-pr +description: Prepare a GitHub PR for merge by rebasing onto main, fixing review findings, running gates, committing fixes, and pushing to the PR head branch. Use after /review-pr. Never merge or push to main. +--- + +# Prepare PR + +## Overview + +Prepare a PR head branch for merge with review fixes, green gates, and deterministic merge handoff artifacts. + +## Inputs + +- Ask for PR number or URL. +- If missing, use `.local/pr-meta.env` from the PR worktree if present. +- If ambiguous, ask. + +## Safety + +- Never push to `main` or `origin/main`. Push only to the PR head branch. +- Never run `git push` without explicit remote and branch. Do not run bare `git push`. +- Do not run gateway stop commands. Do not kill processes. Do not touch port 18792. +- Do not run `git clean -fdx`. +- Do not run `git add -A` or `git add .`. + +## Execution Rule + +- Execute the workflow. Do not stop after printing the TODO checklist. +- If delegating, require the delegate to run commands and capture outputs. + +## Completion Criteria + +- Rebase PR commits onto `origin/main`. +- Fix all BLOCKER and IMPORTANT items from `.local/review.md`. +- Commit prep changes with required subject format. +- Run required gates and pass (`pnpm test` may be skipped only for high-confidence docs-only changes). +- Push the updated HEAD back to the PR head branch. +- Write `.local/prep.md` and `.local/prep.env`. +- Output exactly: `PR is ready for /mergepr`. + +## First: Create a TODO Checklist + +Create a checklist of all prep steps, print it, then continue and execute the commands. + +## Setup: Use a Worktree + +Use an isolated worktree for all prep work. + +```sh +repo_root=$(git rev-parse --show-toplevel) +cd "$repo_root" +gh auth status + +WORKTREE_DIR=".worktrees/pr-" +if [ ! -d "$WORKTREE_DIR" ]; then + git fetch origin main + git worktree add "$WORKTREE_DIR" -b temp/pr- origin/main +fi +cd "$WORKTREE_DIR" +mkdir -p .local +``` + +Run all commands inside the worktree directory. + +## Load Review Artifacts (Mandatory) + +```sh +if [ ! -f .local/review.md ]; then + echo "Missing .local/review.md. Run /review-pr first and save findings." + exit 1 +fi + +if [ ! -f .local/pr-meta.env ]; then + echo "Missing .local/pr-meta.env. Run /review-pr first and save metadata." + exit 1 +fi + +sed -n '1,220p' .local/review.md +source .local/pr-meta.env +``` + +## Steps + +1. Identify PR meta with one API call + +```sh +pr_meta_json=$(gh pr view --json number,title,author,headRefName,headRefOid,baseRefName,headRepository,headRepositoryOwner,body) +printf '%s\n' "$pr_meta_json" | jq '{number,title,author:.author.login,head:.headRefName,headSha:.headRefOid,base:.baseRefName,headRepo:.headRepository.nameWithOwner,headRepoOwner:.headRepositoryOwner.login,headRepoName:.headRepository.name,body}' + +pr_number=$(printf '%s\n' "$pr_meta_json" | jq -r .number) +contrib=$(printf '%s\n' "$pr_meta_json" | jq -r .author.login) +head=$(printf '%s\n' "$pr_meta_json" | jq -r .headRefName) +pr_head_sha_before=$(printf '%s\n' "$pr_meta_json" | jq -r .headRefOid) +head_owner=$(printf '%s\n' "$pr_meta_json" | jq -r '.headRepositoryOwner.login // empty') +head_repo_name=$(printf '%s\n' "$pr_meta_json" | jq -r '.headRepository.name // empty') +head_repo_url=$(printf '%s\n' "$pr_meta_json" | jq -r '.headRepository.url // empty') + +if [ -n "${PR_HEAD:-}" ] && [ "$head" != "$PR_HEAD" ]; then + echo "ERROR: PR head branch changed from $PR_HEAD to $head. Re-run /review-pr." + exit 1 +fi +``` + +2. Fetch PR head and rebase on latest `origin/main` + +```sh +git fetch origin pull//head:pr- --force +git checkout -B pr--prep pr- +git fetch origin main +git rebase origin/main +``` + +If conflicts happen: + +- Resolve each conflicted file. +- Run `git add ` for each file. +- Run `git rebase --continue`. + +If the rebase gets confusing or you resolve conflicts 3 or more times, stop and report. + +3. Fix issues from `.local/review.md` + +- Fix all BLOCKER and IMPORTANT items. +- NITs are optional. +- Keep scope tight. + +Keep a running log in `.local/prep.md`: + +- List which review items you fixed. +- List which files you touched. +- Note behavior changes. + +4. Optional quick feedback tests before full gates + +Targeted tests are optional quick feedback, not a substitute for full gates. + +If running targeted tests in a fresh worktree: + +```sh +if [ ! -x node_modules/.bin/vitest ]; then + pnpm install --frozen-lockfile +fi +``` + +5. Commit prep fixes with required subject format + +Use `scripts/committer` with explicit file paths. + +Required subject format: + +- `fix: (openclaw#) thanks @` + +```sh +commit_msg="fix: (openclaw#$pr_number) thanks @$contrib" +scripts/committer "$commit_msg" ... +``` + +If there are no local changes, do not create a no-op commit. + +Post-commit validation (mandatory): + +```sh +subject=$(git log -1 --pretty=%s) +echo "$subject" | rg -q "openclaw#$pr_number" || { echo "ERROR: commit subject missing openclaw#$pr_number"; exit 1; } +echo "$subject" | rg -q "thanks @$contrib" || { echo "ERROR: commit subject missing thanks @$contrib"; exit 1; } +``` + +6. Decide verification mode and run required gates before pushing + +If you are highly confident the change is docs-only, you may skip `pnpm test`. + +High-confidence docs-only criteria (all must be true): + +- Every changed file is documentation-only (`docs/**`, `README*.md`, `CHANGELOG.md`, `*.md`, `*.mdx`, `mintlify.json`, `docs.json`). +- No code, runtime, test, dependency, or build config files changed (`src/**`, `extensions/**`, `apps/**`, `package.json`, lockfiles, TS/JS config, test files, scripts). +- `.local/review.md` does not call for non-doc behavior fixes. + +Suggested check: + +```sh +changed_files=$(git diff --name-only origin/main...HEAD) +non_docs=$(printf "%s\n" "$changed_files" | grep -Ev '^(docs/|README.*\.md$|CHANGELOG\.md$|.*\.md$|.*\.mdx$|mintlify\.json$|docs\.json$)' || true) + +docs_only=false +if [ -n "$changed_files" ] && [ -z "$non_docs" ]; then + docs_only=true +fi + +echo "docs_only=$docs_only" +``` + +Bootstrap dependencies in a fresh worktree before gates: + +```sh +if [ ! -d node_modules ]; then + pnpm install --frozen-lockfile +fi +``` + +Run required gates: + +```sh +pnpm build +pnpm check + +if [ "$docs_only" = "true" ]; then + echo "Docs-only change detected with high confidence; skipping pnpm test." | tee -a .local/prep.md +else + pnpm test +fi +``` + +Require all required gates to pass. If something fails, fix, commit, and rerun. Allow at most 3 fix-and-rerun cycles. + +7. Push safely to the PR head branch + +Build `prhead` from owner/name first, then validate remote branch SHA before push. + +```sh +if [ -n "$head_owner" ] && [ -n "$head_repo_name" ]; then + head_repo_push_url="https://github.com/$head_owner/$head_repo_name.git" +elif [ -n "$head_repo_url" ] && [ "$head_repo_url" != "null" ]; then + case "$head_repo_url" in + *.git) head_repo_push_url="$head_repo_url" ;; + *) head_repo_push_url="$head_repo_url.git" ;; + esac +else + echo "ERROR: unable to determine PR head repo push URL" + exit 1 +fi + +git remote add prhead "$head_repo_push_url" 2>/dev/null || git remote set-url prhead "$head_repo_push_url" + +echo "Pushing to branch: $head" +if [ "$head" = "main" ] || [ "$head" = "master" ]; then + echo "ERROR: head branch is main/master. This is wrong. Stopping." + exit 1 +fi + +remote_sha=$(git ls-remote prhead "refs/heads/$head" | awk '{print $1}') +if [ -z "$remote_sha" ]; then + echo "ERROR: remote branch refs/heads/$head not found on prhead" + exit 1 +fi +if [ "$remote_sha" != "$pr_head_sha_before" ]; then + echo "ERROR: expected remote SHA $pr_head_sha_before, got $remote_sha. Re-fetch metadata and rebase first." + exit 1 +fi + +git push --force-with-lease=refs/heads/$head:$pr_head_sha_before prhead HEAD:$head || push_failed=1 +``` + +If lease push fails because head moved, perform one automatic retry: + +```sh +if [ "${push_failed:-0}" = "1" ]; then + echo "Lease push failed, retrying once with fresh PR head..." + + pr_head_sha_before=$(gh pr view --json headRefOid --jq .headRefOid) + git fetch origin pull//head:pr--latest --force + git rebase pr--latest + + pnpm build + pnpm check + if [ "$docs_only" != "true" ]; then + pnpm test + fi + + git push --force-with-lease=refs/heads/$head:$pr_head_sha_before prhead HEAD:$head +fi +``` + +8. Verify PR head and base relation (Mandatory) + +```sh +prep_head_sha=$(git rev-parse HEAD) +pr_head_sha_after=$(gh pr view --json headRefOid --jq .headRefOid) + +if [ "$prep_head_sha" != "$pr_head_sha_after" ]; then + echo "ERROR: pushed head SHA does not match PR head SHA." + exit 1 +fi + +git fetch origin main +git fetch origin pull//head:pr--verify --force +git merge-base --is-ancestor origin/main pr--verify && echo "PR is up to date with main" || (echo "ERROR: PR is still behind main, rebase again" && exit 1) +git branch -D pr--verify 2>/dev/null || true +``` + +9. Write prep summary artifacts (Mandatory) + +Write `.local/prep.md` and `.local/prep.env` for merge handoff. + +```sh +contrib_id=$(gh api users/$contrib --jq .id) +coauthor_email="${contrib_id}+${contrib}@users.noreply.github.com" + +cat > .local/prep.env < origin/main +else + git worktree add "$WORKTREE_DIR" -b temp/pr- origin/main + cd "$WORKTREE_DIR" +fi + +# Create local scratch space that persists across /review-pr to /prepare-pr to /merge-pr +mkdir -p .local +``` + +Run all commands inside the worktree directory. +Start on `origin/main` so you can check for existing implementations before looking at PR code. + +## Steps + +1. Identify PR meta and context + +```sh +pr_meta_json=$(gh pr view --json number,title,state,isDraft,author,baseRefName,headRefName,headRefOid,headRepository,url,body,labels,assignees,reviewRequests,files,additions,deletions,statusCheckRollup) +printf '%s\n' "$pr_meta_json" | jq '{number,title,url,state,isDraft,author:.author.login,base:.baseRefName,head:.headRefName,headSha:.headRefOid,headRepo:.headRepository.nameWithOwner,additions,deletions,files:(.files|length),body}' + +cat > .local/pr-meta.env <" -S src packages apps ui || true +rg -n "" -S src packages apps ui || true + +git log --oneline --all --grep="" | head -20 +``` + +If it already exists, call it out as a BLOCKER or at least IMPORTANT. + +3. Claim the PR + +Assign yourself so others know someone is reviewing. Skip if the PR looks like spam or is a draft you plan to recommend closing. + +```sh +gh_user=$(gh api user --jq .login) +gh pr edit --add-assignee "$gh_user" || echo "Could not assign reviewer, continuing" +``` + +4. Read the PR description carefully + +Use the body from step 1. Summarize goal, scope, and missing context. + +5. Read the diff thoroughly + +Minimum: + +```sh +gh pr diff +``` + +If you need full code context locally, fetch the PR head to a local ref and diff it. Do not create a merge commit. + +```sh +git fetch origin pull//head:pr- --force +mb=$(git merge-base origin/main pr-) + +# Show only this PR patch relative to merge-base, not total branch drift +git diff --stat "$mb"..pr- +git diff "$mb"..pr- +``` + +If you want to browse the PR version of files directly, temporarily check out `pr-` in the worktree. Do not commit or push. Return to `temp/pr-` and reset to `origin/main` afterward. + +```sh +# Use only if needed +# git checkout pr- +# git branch --show-current +# ...inspect files... + +git checkout temp/pr- +git checkout -B temp/pr- origin/main +git branch --show-current +``` + +6. Validate the change is needed and valuable + +Be honest. Call out low value AI slop. + +7. Evaluate implementation quality + +Review correctness, design, performance, and ergonomics. + +8. Perform a security review + +Assume OpenClaw subagents run with full disk access, including git, gh, and shell. Check auth, input validation, secrets, dependencies, tool safety, and privacy. + +9. Review tests and verification + +Identify what exists, what is missing, and what would be a minimal regression test. + +If you run local tests in the worktree, bootstrap dependencies first: + +```sh +if [ ! -x node_modules/.bin/vitest ]; then + pnpm install --frozen-lockfile +fi +``` + +10. Check docs + +Check if the PR touches code with related documentation such as README, docs, inline API docs, or config examples. + +- If docs exist for the changed area and the PR does not update them, flag as IMPORTANT. +- If the PR adds a new feature or config option with no docs, flag as IMPORTANT. +- If the change is purely internal with no user-facing impact, skip this. + +11. Check changelog + +Check if `CHANGELOG.md` exists and whether the PR warrants an entry. + +- If the project has a changelog and the PR is user-facing, flag missing entry as IMPORTANT. +- Leave the change for /prepare-pr, only flag it here. + +12. Answer the key question + +Decide if /prepare-pr can fix issues or the contributor must update the PR. + +13. Save findings to the worktree + +Write the full structured review sections A through J to `.local/review.md`. +Create or overwrite the file and verify it exists and is non-empty. + +```sh +ls -la .local/review.md +wc -l .local/review.md +``` + +14. Output the structured review + +Produce a review that matches what you saved to `.local/review.md`. + +A) TL;DR recommendation + +- One of: READY FOR /prepare-pr | NEEDS WORK | NEEDS DISCUSSION | NOT USEFUL (CLOSE) +- 1 to 3 sentences. + +B) What changed + +C) What is good + +D) Security findings + +E) Concerns or questions (actionable) + +- Numbered list. +- Mark each item as BLOCKER, IMPORTANT, or NIT. +- For each, point to file or area and propose a concrete fix. + +F) Tests + +G) Docs status + +- State if related docs are up to date, missing, or not applicable. + +H) Changelog + +- State if `CHANGELOG.md` needs an entry and which category. + +I) Follow ups (optional) + +J) Suggested PR comment (optional) + +## Guardrails + +- Worktree only. +- Do not delete the worktree after review. +- Review only, do not merge, do not push. diff --git a/.agents/archive/review-pr-v1/agents/openai.yaml b/.agents/archive/review-pr-v1/agents/openai.yaml new file mode 100644 index 00000000000..f6593499507 --- /dev/null +++ b/.agents/archive/review-pr-v1/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Review PR" + short_description: "Review GitHub PRs without merging" + default_prompt: "Use $review-pr to perform a thorough, review-only GitHub PR review." diff --git a/.agents/skills/PR_WORKFLOW.md b/.agents/skills/PR_WORKFLOW.md index f75414181bd..b4de1c49ec5 100644 --- a/.agents/skills/PR_WORKFLOW.md +++ b/.agents/skills/PR_WORKFLOW.md @@ -1,17 +1,22 @@ -# PR Review Instructions +# PR Workflow for Maintainers Please read this in full and do not skip sections. +This is the single source of truth for the maintainer PR workflow. + +## Triage order + +Process PRs **oldest to newest**. Older PRs are more likely to have merge conflicts and stale dependencies; resolving them first keeps the queue healthy and avoids snowballing rebase pain. ## Working rule -Skills execute workflow, maintainers provide judgment. +Skills execute workflow. Maintainers provide judgment. Always pause between skills to evaluate technical direction, not just command success. These three skills must be used in order: -1. `review-pr` -2. `prepare-pr` -3. `merge-pr` +1. `review-pr` — review only, produce findings +2. `prepare-pr` — rebase, fix, gate, push to PR head branch +3. `merge-pr` — squash-merge, verify MERGED state, clean up They are necessary, but not sufficient. Maintainers must steer between steps and understand the code before moving forward. @@ -20,6 +25,65 @@ If submitted code is low quality, ignore it and implement the best solution for Do not continue if you cannot verify the problem is real or test the fix. +## Script-first contract + +Skill runs should invoke these wrappers automatically. You only need to run them manually when debugging or doing an explicit script-only run: + +- `scripts/pr-review ` +- `scripts/pr review-checkout-main ` or `scripts/pr review-checkout-pr ` while reviewing +- `scripts/pr review-guard ` before writing review outputs +- `scripts/pr review-validate-artifacts ` after writing outputs +- `scripts/pr-prepare init ` +- `scripts/pr-prepare validate-commit ` +- `scripts/pr-prepare gates ` +- `scripts/pr-prepare push ` +- Optional one-shot prepare: `scripts/pr-prepare run ` +- `scripts/pr-merge ` (verify-only; short form remains backward compatible) +- `scripts/pr-merge verify ` (verify-only) +- Optional one-shot merge: `scripts/pr-merge run ` + +These wrappers run shared preflight checks and generate deterministic artifacts. They are designed to work from repo root or PR worktree cwd. + +## Required artifacts + +- `.local/pr-meta.json` and `.local/pr-meta.env` from review init. +- `.local/review.md` and `.local/review.json` from review output. +- `.local/prep-context.env` and `.local/prep.md` from prepare. +- `.local/prep.env` from prepare completion. + +## Structured review handoff + +`review-pr` must write `.local/review.json`. +In normal skill runs this is handled automatically. Use `scripts/pr review-artifacts-init ` and `scripts/pr review-tests ...` manually only for debugging or explicit script-only runs. + +Minimum schema: + +```json +{ + "recommendation": "READY FOR /prepare-pr", + "findings": [ + { + "id": "F1", + "severity": "IMPORTANT", + "title": "Missing changelog entry", + "area": "CHANGELOG.md", + "fix": "Add a Fixes entry for PR #" + } + ], + "tests": { + "ran": ["pnpm test -- ..."], + "gaps": ["..."], + "result": "pass" + } +} +``` + +`prepare-pr` resolves all `BLOCKER` and `IMPORTANT` findings from this file. + +## Coding Agent + +Use ChatGPT 5.3 Codex High. Fall back to 5.2 Codex High or 5.3 Codex Medium if necessary. + ## PR quality bar - Do not trust PR code by default. @@ -31,6 +95,60 @@ Do not continue if you cannot verify the problem is real or test the fix. - Harden changes. Always evaluate security impact and abuse paths. - Understand the system before changing it. Never make the codebase messier just to clear a PR queue. +## Rebase and conflict resolution + +Before any substantive review or prep work, **always rebase the PR branch onto current `main` and resolve merge conflicts first**. A PR that cannot cleanly rebase is not ready for review — fix conflicts before evaluating correctness. + +- During `prepare-pr`: rebase onto `main` as the first step, before fixing findings or running gates. +- If conflicts are complex or touch areas you do not understand, stop and escalate. +- Prefer **rebase** for linear history; **squash** when commit history is messy or unhelpful. + +## Commit and changelog rules + +- In normal `prepare-pr` runs, commits are created via `scripts/committer "" `. Use it manually only when operating outside the skill flow; avoid manual `git add`/`git commit` so staging stays scoped. +- Follow concise, action-oriented commit messages (e.g., `CLI: add verbose flag to send`). +- During `prepare-pr`, use this commit subject format: `fix: (openclaw#) thanks @`. +- Group related changes; avoid bundling unrelated refactors. +- Changelog workflow: keep the latest released version at the top (no `Unreleased`); after publishing, bump the version and start a new top section. +- When working on a PR: add a changelog entry with the PR number and thank the contributor. +- When working on an issue: reference the issue in the changelog entry. +- Pure test additions/fixes generally do **not** need a changelog entry unless they alter user-facing behavior or the user asks for one. + +## Gate policy + +In fresh worktrees, dependency bootstrap is handled by wrappers before local gates. Manual equivalent: + +```sh +pnpm install --frozen-lockfile +``` + +Gate set: + +- Always: `pnpm build`, `pnpm check` +- `pnpm test` required unless high-confidence docs-only criteria pass. + +## Co-contributor and clawtributors + +- If we squash, add the PR author as a co-contributor in the commit body using a `Co-authored-by:` trailer. +- When maintainer prepares and merges the PR, add the maintainer as an additional `Co-authored-by:` trailer too. +- Avoid `--auto` merges for maintainer landings. Merge only after checks are green so the maintainer account is the actor and attribution is deterministic. +- For squash merges, set `--author-email` to a reviewer-owned email with fallback candidates; if merge fails due to author-email validation, retry once with the next candidate. +- If you review a PR and later do work on it, land via merge/squash (no direct-main commits) and always add the PR author as a co-contributor. +- When merging a PR: leave a PR comment that explains exactly what we did, include the SHA hashes, and record the comment URL in the final report. +- Manual post-merge step for new contributors: run `bun scripts/update-clawtributors.ts` to add their avatar to the README "Thanks to all clawtributors" list, then commit the regenerated README. + +## Review mode vs landing mode + +- **Review mode (PR link only):** read `gh pr view`/`gh pr diff`; **do not** switch branches; **do not** change code. +- **Landing mode (exception path):** use only when normal `review-pr -> prepare-pr -> merge-pr` flow cannot safely preserve attribution or cannot satisfy branch protection. Create an integration branch from `main`, bring in PR commits (**prefer rebase** for linear history; **merge allowed** when complexity/conflicts make it safer), apply fixes, add changelog (+ thanks + PR #), run full gate **locally before committing** (`pnpm build && pnpm check && pnpm test`), commit, merge back to `main`, then `git switch main` (never stay on a topic branch after landing). Important: the contributor needs to be in the git graph after this! + +## Pre-review safety checks + +- Before starting a review when a GH Issue/PR is pasted: `review-pr`/`scripts/pr-review` should create and use an isolated `.worktrees/pr-` checkout from `origin/main` automatically. Do not require a clean main checkout, and do not run `git pull` in a dirty main checkout. +- PR review calls: prefer a single `gh pr view --json ...` to batch metadata/comments; run `gh pr diff` only when needed. +- PRs should summarize scope, note testing performed, and mention any user-facing changes or new flags. +- Read `docs/help/submitting-a-pr.md` ([Submitting a PR](https://docs.openclaw.ai/help/submitting-a-pr)) for what we expect from contributors. + ## Unified workflow Entry criteria: @@ -56,7 +174,6 @@ Maintainer checkpoint before `prepare-pr`: ``` What problem are they trying to solve? What is the most optimal implementation? -Is the code properly scoped? Can we fix up everything? Do we have any questions? ``` @@ -72,27 +189,30 @@ Stop and escalate instead of continuing if: Purpose: - Make the PR merge-ready on its head branch. -- Rebase onto current `main`, fix blocker/important findings, and run gates. +- Rebase onto current `main` first, then fix blocker/important findings, then run gates. +- In fresh worktrees, bootstrap dependencies before local gates (`pnpm install --frozen-lockfile`). Expected output: - Updated code and tests on the PR head branch. - `.local/prep.md` with changes, verification, and current HEAD SHA. -- Final status: `PR is ready for /mergepr`. +- Final status: `PR is ready for /merge-pr`. Maintainer checkpoint before `merge-pr`: ``` Is this the most optimal implementation? Is the code properly scoped? +Is the code properly reusing existing logic in the codebase? Is the code properly typed? Is the code hardened? Do we have enough tests? -Are tests using fake timers where relevant? (e.g., debounce/throttle, retry backoff, timeout branches, delayed callbacks, polling loops) +Do we need regression tests? +Are tests using fake timers where appropriate? (e.g., debounce/throttle, retry backoff, timeout branches, delayed callbacks, polling loops) Do not add performative tests, ensure tests are real and there are no regressions. -Take your time, fix it properly, refactor if necessary. Do you see any follow-up refactors we should do? Did any changes introduce any potential security vulnerabilities? +Take your time, fix it properly, refactor if necessary. ``` Stop and escalate instead of continuing if: @@ -106,7 +226,8 @@ Stop and escalate instead of continuing if: Purpose: - Merge only after review and prep artifacts are present and checks are green. -- Use squash merge flow and verify the PR ends in `MERGED` state. +- Use deterministic squash merge flow (`--match-head-commit` + explicit subject/body with co-author trailer), then verify the PR ends in `MERGED` state. +- If no required checks are configured on the PR, treat that as acceptable and continue after branch-up-to-date validation. Go or no-go checklist before merge: @@ -119,8 +240,10 @@ Expected output: - Successful merge commit and recorded merge SHA. - Worktree cleanup after successful merge. +- Comment on PR indicating merge was successful. Maintainer checkpoint after merge: - Were any refactors intentionally deferred and now need follow-up issue(s)? - Did this reveal broader architecture or test gaps we should address? +- Run `bun scripts/update-clawtributors.ts` if the contributor is new. diff --git a/.agents/skills/merge-pr/SKILL.md b/.agents/skills/merge-pr/SKILL.md index 4d8f01f4cf8..ae89b1a2742 100644 --- a/.agents/skills/merge-pr/SKILL.md +++ b/.agents/skills/merge-pr/SKILL.md @@ -1,185 +1,98 @@ --- name: merge-pr -description: Merge a GitHub PR via squash after /preparepr. Use when asked to merge a ready PR. Do not push to main or modify code. Ensure the PR ends in MERGED state and clean up worktrees after success. +description: Script-first deterministic squash merge with strict required-check gating, head-SHA pinning, and reliable attribution/commenting. --- # Merge PR ## Overview -Merge a prepared PR via `gh pr merge --squash` and clean up the worktree after success. +Merge a prepared PR only after deterministic validation. ## Inputs - Ask for PR number or URL. -- If missing, auto-detect from conversation. -- If ambiguous, ask. +- If missing, use `.local/prep.env` from the PR worktree. ## Safety -- Use `gh pr merge --squash` as the only path to `main`. -- Do not run `git push` at all during merge. -- Do not run gateway stop commands. Do not kill processes. Do not touch port 18792. +- Never use `gh pr merge --auto` in this flow. +- Never run `git push` directly. +- Require `--match-head-commit` during merge. -## Execution Rule +## Execution Contract -- Execute the workflow. Do not stop after printing the TODO checklist. -- If delegating, require the delegate to run commands and capture outputs. - -## Known Footguns - -- If you see "fatal: not a git repository", you are in the wrong directory. Use `~/dev/openclaw` if available; otherwise ask user. -- Read `.local/review.md` and `.local/prep.md` in the worktree. Do not skip. -- Clean up the real worktree directory `.worktrees/pr-` only after a successful merge. -- Expect cleanup to remove `.local/` artifacts. - -## Completion Criteria - -- Ensure `gh pr merge` succeeds. -- Ensure PR state is `MERGED`, never `CLOSED`. -- Record the merge SHA. -- Run cleanup only after merge success. - -## First: Create a TODO Checklist - -Create a checklist of all merge steps, print it, then continue and execute the commands. - -## Setup: Use a Worktree - -Use an isolated worktree for all merge work. +1. Validate merge readiness: ```sh -cd ~/dev/openclaw -# Sanity: confirm you are in the repo -git rev-parse --show-toplevel - -WORKTREE_DIR=".worktrees/pr-" +scripts/pr-merge verify ``` -Run all commands inside the worktree directory. - -## Load Local Artifacts (Mandatory) - -Expect these files from earlier steps: - -- `.local/review.md` from `/reviewpr` -- `.local/prep.md` from `/preparepr` +Backward-compatible verify form also works: ```sh -ls -la .local || true - -if [ -f .local/review.md ]; then - echo "Found .local/review.md" - sed -n '1,120p' .local/review.md -else - echo "Missing .local/review.md. Stop and run /reviewpr, then /preparepr." - exit 1 -fi - -if [ -f .local/prep.md ]; then - echo "Found .local/prep.md" - sed -n '1,120p' .local/prep.md -else - echo "Missing .local/prep.md. Stop and run /preparepr first." - exit 1 -fi +scripts/pr-merge ``` +2. Run one-shot deterministic merge: + +```sh +scripts/pr-merge run +``` + +3. Ensure output reports: + +- `merge_sha=` +- `merge_author_email=` +- `comment_url=` + ## Steps -1. Identify PR meta +1. Validate artifacts ```sh -gh pr view --json number,title,state,isDraft,author,headRefName,baseRefName,headRepository,body --jq '{number,title,state,isDraft,author:.author.login,head:.headRefName,base:.baseRefName,headRepo:.headRepository.nameWithOwner,body}' -contrib=$(gh pr view --json author --jq .author.login) -head=$(gh pr view --json headRefName --jq .headRefName) -head_repo_url=$(gh pr view --json headRepository --jq .headRepository.url) +require=(.local/review.md .local/review.json .local/prep.md .local/prep.env) +for f in "${require[@]}"; do + [ -s "$f" ] || { echo "Missing artifact: $f"; exit 1; } +done ``` -2. Run sanity checks - -Stop if any are true: - -- PR is a draft. -- Required checks are failing. -- Branch is behind main. +2. Validate checks and branch status ```sh -# Checks -gh pr checks - -# Check behind main -git fetch origin main -git fetch origin pull//head:pr- -git merge-base --is-ancestor origin/main pr- || echo "PR branch is behind main, run /preparepr" +scripts/pr-merge verify +source .local/prep.env ``` -If anything is failing or behind, stop and say to run `/preparepr`. +`scripts/pr-merge` treats “no required checks configured” as acceptable (`[]`), but fails on any required `fail` or `pending`. -3. Merge PR and delete branch - -If checks are still running, use `--auto` to queue the merge. +3. Merge deterministically (wrapper-managed) ```sh -# Check status first -check_status=$(gh pr checks 2>&1) -if echo "$check_status" | grep -q "pending\|queued"; then - echo "Checks still running, using --auto to queue merge" - gh pr merge --squash --delete-branch --auto - echo "Merge queued. Monitor with: gh pr checks --watch" -else - gh pr merge --squash --delete-branch -fi +scripts/pr-merge run ``` -If merge fails, report the error and stop. Do not retry in a loop. -If the PR needs changes beyond what `/preparepr` already did, stop and say to run `/preparepr` again. +`scripts/pr-merge run` performs: -4. Get merge SHA +- deterministic squash merge pinned to `PREP_HEAD_SHA` +- reviewer merge author email selection with fallback candidates +- one retry only when merge fails due to author-email validation +- co-author trailers for PR author and reviewer +- post-merge verification of both co-author trailers on commit message +- PR comment retry (3 attempts), then comment URL extraction +- cleanup after confirmed `MERGED` + +4. Manual fallback (only if wrapper is unavailable) ```sh -merge_sha=$(gh pr view --json mergeCommit --jq '.mergeCommit.oid') -echo "merge_sha=$merge_sha" +scripts/pr merge-run ``` -5. Optional comment +5. Cleanup -Use a literal multiline string or heredoc for newlines. - -```sh -gh pr comment -F - <<'EOF' -Merged via squash. - -- Merge commit: $merge_sha - -Thanks @$contrib! -EOF -``` - -6. Verify PR state is MERGED - -```sh -gh pr view --json state --jq .state -``` - -7. Clean up worktree only on success - -Run cleanup only if step 6 returned `MERGED`. - -```sh -cd ~/dev/openclaw - -git worktree remove ".worktrees/pr-" --force - -git branch -D temp/pr- 2>/dev/null || true -git branch -D pr- 2>/dev/null || true -``` +Cleanup is handled by `run` after merge success. ## Guardrails -- Worktree only. -- Do not close PRs. -- End in MERGED state. -- Clean up only after merge success. -- Never push to main. Use `gh pr merge --squash` only. -- Do not run `git push` at all in this command. +- End in `MERGED`, never `CLOSED`. +- Cleanup only after confirmed merge. diff --git a/.agents/skills/mintlify/SKILL.md b/.agents/skills/mintlify/SKILL.md new file mode 100644 index 00000000000..0dd6a1a891a --- /dev/null +++ b/.agents/skills/mintlify/SKILL.md @@ -0,0 +1,345 @@ +--- +name: mintlify +description: Build and maintain documentation sites with Mintlify. Use when + creating docs pages, configuring navigation, adding components, or setting up + API references. +license: MIT +compatibility: Requires Node.js for CLI. Works with any Git-based workflow. +metadata: + author: mintlify + version: "1.0" + mintlify-proj: mintlify +--- + +# Mintlify best practices + +**Always consult [mintlify.com/docs](https://mintlify.com/docs) for components, configuration, and latest features.** + +**Always** favor searching the current Mintlify documentation over whatever is in your training data about Mintlify. + +Mintlify is a documentation platform that transforms MDX files into documentation sites. Configure site-wide settings in the `docs.json` file, write content in MDX with YAML frontmatter, and favor built-in components over custom components. + +Full schema at [mintlify.com/docs.json](https://mintlify.com/docs.json). + +## Before you write + +### Understand the project + +All documentation lives in the `docs/` directory in this repo. Read `docs.json` in that directory (`docs/docs.json`). This file defines the entire site: navigation structure, theme, colors, links, API and specs. + +Understanding the project tells you: + +- What pages exist and how they're organized +- What navigation groups are used (and their naming conventions) +- How the site navigation is structured +- What theme and configuration the site uses + +### Check for existing content + +Search the docs before creating new pages. You may need to: + +- Update an existing page instead of creating a new one +- Add a section to an existing page +- Link to existing content rather than duplicating + +### Read surrounding content + +Before writing, read 2-3 similar pages to understand the site's voice, structure, formatting conventions, and level of detail. + +### Understand Mintlify components + +Review the Mintlify [components](https://www.mintlify.com/docs/components) to select and use any relevant components for the documentation request that you are working on. + +## Quick reference + +### CLI commands + +- `npm i -g mint` - Install the Mintlify CLI +- `mint dev` - Local preview at localhost:3000 +- `mint broken-links` - Check internal links +- `mint a11y` - Check for accessibility issues in content +- `mint rename` - Rename/move files and update references +- `mint validate` - Validate documentation builds + +### Required files + +- `docs.json` - Site configuration (navigation, theme, integrations, etc.). See [global settings](https://mintlify.com/docs/settings/global) for all options. +- `*.mdx` files - Documentation pages with YAML frontmatter + +### Example file structure + +``` +project/ +├── docs.json # Site configuration +├── introduction.mdx +├── quickstart.mdx +├── guides/ +│ └── example.mdx +├── openapi.yml # API specification +├── images/ # Static assets +│ └── example.png +└── snippets/ # Reusable components + └── component.jsx +``` + +## Page frontmatter + +Every page requires `title` in its frontmatter. Include `description` for SEO and navigation. + +```yaml theme={null} +--- +title: "Clear, descriptive title" +description: "Concise summary for SEO and navigation." +--- +``` + +Optional frontmatter fields: + +- `sidebarTitle`: Short title for sidebar navigation. +- `icon`: Lucide or Font Awesome icon name, URL, or file path. +- `tag`: Label next to the page title in the sidebar (for example, "NEW"). +- `mode`: Page layout mode (`default`, `wide`, `custom`). +- `keywords`: Array of terms related to the page content for local search and SEO. +- Any custom YAML fields for use with personalization or conditional content. + +## File conventions + +- Match existing naming patterns in the directory +- If there are no existing files or inconsistent file naming patterns, use kebab-case: `getting-started.mdx`, `api-reference.mdx` +- Use root-relative paths without file extensions for internal links: `/getting-started/quickstart` +- Do not use relative paths (`../`) or absolute URLs for internal pages +- When you create a new page, add it to `docs.json` navigation or it won't appear in the sidebar + +## Organize content + +When a user asks about anything related to site-wide configurations, start by understanding the [global settings](https://www.mintlify.com/docs/organize/settings). See if a setting in the `docs.json` file can be updated to achieve what the user wants. + +### Navigation + +The `navigation` property in `docs.json` controls site structure. Choose one primary pattern at the root level, then nest others within it. + +**Choose your primary pattern:** + +| Pattern | When to use | +| ------------- | ---------------------------------------------------------------------------------------------- | +| **Groups** | Default. Single audience, straightforward hierarchy | +| **Tabs** | Distinct sections with different audiences (Guides vs API Reference) or content types | +| **Anchors** | Want persistent section links at sidebar top. Good for separating docs from external resources | +| **Dropdowns** | Multiple doc sections users switch between, but not distinct enough for tabs | +| **Products** | Multi-product company with separate documentation per product | +| **Versions** | Maintaining docs for multiple API/product versions simultaneously | +| **Languages** | Localized content | + +**Within your primary pattern:** + +- **Groups** - Organize related pages. Can nest groups within groups, but keep hierarchy shallow +- **Menus** - Add dropdown navigation within tabs for quick jumps to specific pages +- **`expanded: false`** - Collapse nested groups by default. Use for reference sections users browse selectively +- **`openapi`** - Auto-generate pages from OpenAPI spec. Add at group/tab level to inherit + +**Common combinations:** + +- Tabs containing groups (most common for docs with API reference) +- Products containing tabs (multi-product SaaS) +- Versions containing tabs (versioned API docs) +- Anchors containing groups (simple docs with external resource links) + +### Links and paths + +- **Internal links:** Root-relative, no extension: `/getting-started/quickstart` +- **Images:** Store in `/images`, reference as `/images/example.png` +- **External links:** Use full URLs, they open in new tabs automatically + +## Customize docs sites + +**What to customize where:** + +- **Brand colors, fonts, logo** → `docs.json`. See [global settings](https://mintlify.com/docs/settings/global) +- **Component styling, layout tweaks** → `custom.css` at project root +- **Dark mode** → Enabled by default. Only disable with `"appearance": "light"` in `docs.json` if brand requires it + +Start with `docs.json`. Only add `custom.css` when you need styling that config doesn't support. + +## Write content + +### Components + +The [components overview](https://mintlify.com/docs/components) organizes all components by purpose: structure content, draw attention, show/hide content, document APIs, link to pages, and add visual context. Start there to find the right component. + +**Common decision points:** + +| Need | Use | +| -------------------------- | ----------------------- | +| Hide optional details | `` | +| Long code examples | `` | +| User chooses one option | `` | +| Linked navigation cards | `` in `` | +| Sequential instructions | `` | +| Code in multiple languages | `` | +| API parameters | `` | +| API response fields | `` | + +**Callouts by severity:** + +- `` - Supplementary info, safe to skip +- `` - Helpful context such as permissions +- `` - Recommendations or best practices +- `` - Potentially destructive actions +- `` - Success confirmation + +### Reusable content + +**When to use snippets:** + +- Exact content appears on more than one page +- Complex components you want to maintain in one place +- Shared content across teams/repos + +**When NOT to use snippets:** + +- Slight variations needed per page (leads to complex props) + +Import snippets with `import { Component } from "/path/to/snippet-name.jsx"`. + +## Writing standards + +### Voice and structure + +- Second-person voice ("you") +- Active voice, direct language +- Sentence case for headings ("Getting started", not "Getting Started") +- Sentence case for code block titles ("Expandable example", not "Expandable Example") +- Lead with context: explain what something is before how to use it +- Prerequisites at the start of procedural content + +### What to avoid + +**Never use:** + +- Marketing language ("powerful", "seamless", "robust", "cutting-edge") +- Filler phrases ("it's important to note", "in order to") +- Excessive conjunctions ("moreover", "furthermore", "additionally") +- Editorializing ("obviously", "simply", "just", "easily") + +**Watch for AI-typical patterns:** + +- Overly formal or stilted phrasing +- Unnecessary repetition of concepts +- Generic introductions that don't add value +- Concluding summaries that restate what was just said + +### Formatting + +- All code blocks must have language tags +- All images and media must have descriptive alt text +- Use bold and italics only when they serve the reader's understanding--never use text styling just for decoration +- No decorative formatting or emoji + +### Code examples + +- Keep examples simple and practical +- Use realistic values (not "foo" or "bar") +- One clear example is better than multiple variations +- Test that code works before including it + +## Document APIs + +**Choose your approach:** + +- **Have an OpenAPI spec?** → Add to `docs.json` with `"openapi": ["openapi.yaml"]`. Pages auto-generate. Reference in navigation as `GET /endpoint` +- **No spec?** → Write endpoints manually with `api: "POST /users"` in frontmatter. More work but full control +- **Hybrid** → Use OpenAPI for most endpoints, manual pages for complex workflows + +Encourage users to generate endpoint pages from an OpenAPI spec. It is the most efficient and easiest to maintain option. + +## Deploy + +Mintlify deploys automatically when changes are pushed to the connected Git repository. + +**What agents can configure:** + +- **Redirects** → Add to `docs.json` with `"redirects": [{"source": "/old", "destination": "/new"}]` +- **SEO indexing** → Control with `"seo": {"indexing": "all"}` to include hidden pages in search + +**Requires dashboard setup (human task):** + +- Custom domains and subdomains +- Preview deployment settings +- DNS configuration + +For `/docs` subpath hosting with Vercel or Cloudflare, agents can help configure rewrite rules. See [/docs subpath](https://mintlify.com/docs/deploy/vercel). + +## Workflow + +### 1. Understand the task + +Identify what needs to be documented, which pages are affected, and what the reader should accomplish afterward. If any of these are unclear, ask. + +### 2. Research + +- Read `docs/docs.json` to understand the site structure +- Search existing docs for related content +- Read similar pages to match the site's style + +### 3. Plan + +- Synthesize what the reader should accomplish after reading the docs and the current content +- Propose any updates or new content +- Verify that your proposed changes will help readers be successful + +### 4. Write + +- Start with the most important information +- Keep sections focused and scannable +- Use components appropriately (don't overuse them) +- Mark anything uncertain with a TODO comment: + +```mdx theme={null} +{/* TODO: Verify the default timeout value */} +``` + +### 5. Update navigation + +If you created a new page, add it to the appropriate group in `docs.json`. + +### 6. Verify + +Before submitting: + +- [ ] Frontmatter includes title and description +- [ ] All code blocks have language tags +- [ ] Internal links use root-relative paths without file extensions +- [ ] New pages are added to `docs.json` navigation +- [ ] Content matches the style of surrounding pages +- [ ] No marketing language or filler phrases +- [ ] TODOs are clearly marked for anything uncertain +- [ ] Run `mint broken-links` to check links +- [ ] Run `mint validate` to find any errors + +## Edge cases + +### Migrations + +If a user asks about migrating to Mintlify, ask if they are using ReadMe or Docusaurus. If they are, use the [@mintlify/scraping](https://www.npmjs.com/package/@mintlify/scraping) CLI to migrate content. If they are using a different platform to host their documentation, help them manually convert their content to MDX pages using Mintlify components. + +### Hidden pages + +Any page that is not included in the `docs.json` navigation is hidden. Use hidden pages for content that should be accessible by URL or indexed for the assistant or search, but not discoverable through the sidebar navigation. + +### Exclude pages + +The `.mintignore` file is used to exclude files from a documentation repository from being processed. + +## Common gotchas + +1. **Component imports** - JSX components need explicit import, MDX components don't +2. **Frontmatter required** - Every MDX file needs `title` at minimum +3. **Code block language** - Always specify language identifier +4. **Never use `mint.json`** - `mint.json` is deprecated. Only ever use `docs.json` + +## Resources + +- [Documentation](https://mintlify.com/docs) +- [Configuration schema](https://mintlify.com/docs.json) +- [Feature requests](https://github.com/orgs/mintlify/discussions/categories/feature-requests) +- [Bugs and feedback](https://github.com/orgs/mintlify/discussions/categories/bugs-feedback) diff --git a/.agents/skills/prepare-pr/SKILL.md b/.agents/skills/prepare-pr/SKILL.md index 8a7450cc3d6..e219141eb79 100644 --- a/.agents/skills/prepare-pr/SKILL.md +++ b/.agents/skills/prepare-pr/SKILL.md @@ -1,248 +1,131 @@ --- name: prepare-pr -description: Prepare a GitHub PR for merge by rebasing onto main, fixing review findings, running gates, committing fixes, and pushing to the PR head branch. Use after /reviewpr. Never merge or push to main. +description: Script-first PR preparation with structured findings resolution, deterministic push safety, and explicit gate execution. --- # Prepare PR ## Overview -Prepare a PR branch for merge with review fixes, green gates, and an updated head branch. +Prepare the PR head branch for merge after `/review-pr`. ## Inputs - Ask for PR number or URL. -- If missing, auto-detect from conversation. -- If ambiguous, ask. +- If missing, use `.local/pr-meta.env` if present in the PR worktree. ## Safety -- Never push to `main` or `origin/main`. Push only to the PR head branch. -- Never run `git push` without specifying remote and branch explicitly. Do not run bare `git push`. -- Do not run gateway stop commands. Do not kill processes. Do not touch port 18792. +- Never push to `main`. +- Only push to PR head with explicit `--force-with-lease` against known head SHA. - Do not run `git clean -fdx`. -- Do not run `git add -A` or `git add .`. Stage only specific files changed. +- Wrappers are cwd-agnostic; run from repo root or PR worktree. -## Execution Rule +## Execution Contract -- Execute the workflow. Do not stop after printing the TODO checklist. -- If delegating, require the delegate to run commands and capture outputs. - -## Known Footguns - -- If you see "fatal: not a git repository", you are in the wrong directory. Use `~/dev/openclaw` if available; otherwise ask user. -- Do not run `git clean -fdx`. -- Do not run `git add -A` or `git add .`. - -## Completion Criteria - -- Rebase PR commits onto `origin/main`. -- Fix all BLOCKER and IMPORTANT items from `.local/review.md`. -- Run gates and pass. -- Commit prep changes. -- Push the updated HEAD back to the PR head branch. -- Write `.local/prep.md` with a prep summary. -- Output exactly: `PR is ready for /mergepr`. - -## First: Create a TODO Checklist - -Create a checklist of all prep steps, print it, then continue and execute the commands. - -## Setup: Use a Worktree - -Use an isolated worktree for all prep work. +1. Run setup: ```sh -cd ~/openclaw -# Sanity: confirm you are in the repo -git rev-parse --show-toplevel - -WORKTREE_DIR=".worktrees/pr-" +scripts/pr-prepare init ``` -Run all commands inside the worktree directory. +2. Resolve findings from structured review: -## Load Review Findings (Mandatory) +- `.local/review.json` is mandatory. +- Resolve all `BLOCKER` and `IMPORTANT` items. + +3. Commit with required subject format and validate it. + +4. Run gates via wrapper. + +5. Push via wrapper (includes pre-push remote verification, one automatic lease-retry path, and post-push API propagation retry). + +Optional one-shot path: ```sh -if [ -f .local/review.md ]; then - echo "Found review findings from /reviewpr" -else - echo "Missing .local/review.md. Run /reviewpr first and save findings." - exit 1 -fi - -# Read it -sed -n '1,200p' .local/review.md +scripts/pr-prepare run ``` ## Steps -1. Identify PR meta (author, head branch, head repo URL) +1. Setup and artifacts ```sh -gh pr view --json number,title,author,headRefName,baseRefName,headRepository,body --jq '{number,title,author:.author.login,head:.headRefName,base:.baseRefName,headRepo:.headRepository.nameWithOwner,body}' -contrib=$(gh pr view --json author --jq .author.login) -head=$(gh pr view --json headRefName --jq .headRefName) -head_repo_url=$(gh pr view --json headRepository --jq .headRepository.url) +scripts/pr-prepare init + +ls -la .local/review.md .local/review.json .local/pr-meta.env .local/prep-context.env +jq . .local/review.json >/dev/null ``` -2. Fetch the PR branch tip into a local ref +2. Resolve required findings + +List required items: ```sh -git fetch origin pull//head:pr- +jq -r '.findings[] | select(.severity=="BLOCKER" or .severity=="IMPORTANT") | "- [\(.severity)] \(.id): \(.title) => \(.fix)"' .local/review.json ``` -3. Rebase PR commits onto latest main +Fix all required findings. Keep scope tight. + +3. Update changelog/docs when required ```sh -# Move worktree to the PR tip first -git reset --hard pr- - -# Rebase onto current main -git fetch origin main -git rebase origin/main +jq -r '.changelog' .local/review.json +jq -r '.docs' .local/review.json ``` -If conflicts happen: +4. Commit scoped changes -- Resolve each conflicted file. -- Run `git add ` for each file. -- Run `git rebase --continue`. +Required commit subject format: -If the rebase gets confusing or you resolve conflicts 3 or more times, stop and report. +- `fix: (openclaw#) thanks @` -4. Fix issues from `.local/review.md` - -- Fix all BLOCKER and IMPORTANT items. -- NITs are optional. -- Keep scope tight. - -Keep a running log in `.local/prep.md`: - -- List which review items you fixed. -- List which files you touched. -- Note behavior changes. - -5. Update `CHANGELOG.md` if flagged in review - -Check `.local/review.md` section H for guidance. -If flagged and user-facing: - -- Check if `CHANGELOG.md` exists. +Use explicit file list: ```sh -ls CHANGELOG.md 2>/dev/null +source .local/pr-meta.env +scripts/committer "fix: (openclaw#$PR_NUMBER) thanks @$PR_AUTHOR" ... ``` -- Follow existing format. -- Add a concise entry with PR number and contributor. - -6. Update docs if flagged in review - -Check `.local/review.md` section G for guidance. -If flagged, update only docs related to the PR changes. - -7. Commit prep fixes - -Stage only specific files: +Validate commit subject: ```sh -git add ... +scripts/pr-prepare validate-commit ``` -Preferred commit tool: +5. Run gates ```sh -committer "fix: (#) (thanks @$contrib)" +scripts/pr-prepare gates ``` -If `committer` is not found: +6. Push safely to PR head ```sh -git commit -m "fix: (#) (thanks @$contrib)" +scripts/pr-prepare push ``` -8. Run full gates before pushing +This push step includes: + +- robust fork remote resolution from owner/name, +- pre-push remote SHA verification, +- one automatic rebase + gate rerun + retry if lease push fails, +- post-push PR-head propagation retry, +- idempotent behavior when local prep HEAD is already on the PR head, +- post-push SHA verification and `.local/prep.env` generation. + +7. Verify handoff artifacts ```sh -pnpm install -pnpm build -pnpm ui:build -pnpm check -pnpm test +ls -la .local/prep.md .local/prep.env ``` -Require all to pass. If something fails, fix, commit, and rerun. Allow at most 3 fix and rerun cycles. If gates still fail after 3 attempts, stop and report the failures. Do not loop indefinitely. +8. Output -9. Push updates back to the PR head branch - -```sh -# Ensure remote for PR head exists -git remote add prhead "$head_repo_url.git" 2>/dev/null || git remote set-url prhead "$head_repo_url.git" - -# Use force with lease after rebase -# Double check: $head must NOT be "main" or "master" -echo "Pushing to branch: $head" -if [ "$head" = "main" ] || [ "$head" = "master" ]; then - echo "ERROR: head branch is main/master. This is wrong. Stopping." - exit 1 -fi -git push --force-with-lease prhead HEAD:$head -``` - -10. Verify PR is not behind main (Mandatory) - -```sh -git fetch origin main -git fetch origin pull//head:pr--verify --force -git merge-base --is-ancestor origin/main pr--verify && echo "PR is up to date with main" || echo "ERROR: PR is still behind main, rebase again" -git branch -D pr--verify 2>/dev/null || true -``` - -If still behind main, repeat steps 2 through 9. - -11. Write prep summary artifacts (Mandatory) - -Update `.local/prep.md` with: - -- Current HEAD sha from `git rev-parse HEAD`. -- Short bullet list of changes. -- Gate results. -- Push confirmation. -- Rebase verification result. - -Create or overwrite `.local/prep.md` and verify it exists and is non-empty: - -```sh -git rev-parse HEAD -ls -la .local/prep.md -wc -l .local/prep.md -``` - -12. Output - -Include a diff stat summary: - -```sh -git diff --stat origin/main..HEAD -git diff --shortstat origin/main..HEAD -``` - -Report totals: X files changed, Y insertions(+), Z deletions(-). - -If gates passed and push succeeded, print exactly: - -``` -PR is ready for /mergepr -``` - -Otherwise, list remaining failures and stop. +- Summarize resolved findings and gate results. +- Print exactly: `PR is ready for /merge-pr`. ## Guardrails -- Worktree only. -- Do not delete the worktree on success. `/mergepr` may reuse it. -- Do not run `gh pr merge`. -- Never push to main. Only push to the PR head branch. -- Run and pass all gates before pushing. +- Do not run `gh pr merge` in this skill. +- Do not delete worktree. diff --git a/.agents/skills/review-pr/SKILL.md b/.agents/skills/review-pr/SKILL.md index 4bcd76333bc..7327b343334 100644 --- a/.agents/skills/review-pr/SKILL.md +++ b/.agents/skills/review-pr/SKILL.md @@ -1,228 +1,141 @@ --- name: review-pr -description: Review-only GitHub pull request analysis with the gh CLI. Use when asked to review a PR, provide structured feedback, or assess readiness to land. Do not merge, push, or make code changes you intend to keep. +description: Script-first review-only GitHub pull request analysis. Use for deterministic PR review with structured findings handoff to /prepare-pr. --- # Review PR ## Overview -Perform a thorough review-only PR assessment and return a structured recommendation on readiness for /preparepr. +Perform a read-only review and produce both human and machine-readable outputs. ## Inputs - Ask for PR number or URL. -- If missing, always ask. Never auto-detect from conversation. -- If ambiguous, ask. +- If missing, always ask. ## Safety -- Never push to `main` or `origin/main`, not during review, not ever. -- Do not run `git push` at all during review. Treat review as read only. -- Do not stop or kill the gateway. Do not run gateway stop commands. Do not kill processes on port 18792. +- Never push, merge, or modify code intended to keep. +- Work only in `.worktrees/pr-`. -## Execution Rule +## Execution Contract -- Execute the workflow. Do not stop after printing the TODO checklist. -- If delegating, require the delegate to run commands and capture outputs, not a plan. - -## Known Failure Modes - -- If you see "fatal: not a git repository", you are in the wrong directory. Use `~/dev/openclaw` if available; otherwise ask user. -- Do not stop after printing the checklist. That is not completion. - -## Writing Style for Output - -- Write casual and direct. -- Avoid em dashes and en dashes. Use commas or separate sentences. - -## Completion Criteria - -- Run the commands in the worktree and inspect the PR directly. -- Produce the structured review sections A through J. -- Save the full review to `.local/review.md` inside the worktree. - -## First: Create a TODO Checklist - -Create a checklist of all review steps, print it, then continue and execute the commands. - -## Setup: Use a Worktree - -Use an isolated worktree for all review work. +1. Run wrapper setup: ```sh -cd ~/dev/openclaw -# Sanity: confirm you are in the repo -git rev-parse --show-toplevel - -WORKTREE_DIR=".worktrees/pr-" -git fetch origin main - -# Reuse existing worktree if it exists, otherwise create new -if [ -d "$WORKTREE_DIR" ]; then - cd "$WORKTREE_DIR" - git checkout temp/pr- 2>/dev/null || git checkout -b temp/pr- - git fetch origin main - git reset --hard origin/main -else - git worktree add "$WORKTREE_DIR" -b temp/pr- origin/main - cd "$WORKTREE_DIR" -fi - -# Create local scratch space that persists across /reviewpr to /preparepr to /mergepr -mkdir -p .local +scripts/pr-review ``` -Run all commands inside the worktree directory. -Start on `origin/main` so you can check for existing implementations before looking at PR code. +2. Use explicit branch mode switches: + +- Main baseline mode: `scripts/pr review-checkout-main ` +- PR-head mode: `scripts/pr review-checkout-pr ` + +3. Before writing review outputs, run branch guard: + +```sh +scripts/pr review-guard +``` + +4. Write both outputs: + +- `.local/review.md` with sections A through J. +- `.local/review.json` with structured findings. + +5. Validate artifacts semantically: + +```sh +scripts/pr review-validate-artifacts +``` ## Steps -1. Identify PR meta and context +1. Setup and metadata ```sh -gh pr view --json number,title,state,isDraft,author,baseRefName,headRefName,headRepository,url,body,labels,assignees,reviewRequests,files,additions,deletions --jq '{number,title,url,state,isDraft,author:.author.login,base:.baseRefName,head:.headRefName,headRepo:.headRepository.nameWithOwner,additions,deletions,files:.files|length,body}' +scripts/pr-review +ls -la .local/pr-meta.json .local/pr-meta.env .local/review-context.env .local/review-mode.env ``` -2. Check if this already exists in main before looking at the PR branch - -- Identify the core feature or fix from the PR title and description. -- Search for existing implementations using keywords from the PR title, changed file paths, and function or component names from the diff. +2. Existing implementation check on main ```sh -# Use keywords from the PR title and changed files -rg -n "" -S src packages apps ui || true -rg -n "" -S src packages apps ui || true - -git log --oneline --all --grep="" | head -20 +scripts/pr review-checkout-main +rg -n "" -S src extensions apps || true +git log --oneline --all --grep "" | head -20 ``` -If it already exists, call it out as a BLOCKER or at least IMPORTANT. - -3. Claim the PR - -Assign yourself so others know someone is reviewing. Skip if the PR looks like spam or is a draft you plan to recommend closing. +3. Claim PR ```sh gh_user=$(gh api user --jq .login) -gh pr edit --add-assignee "$gh_user" +gh pr edit --add-assignee "$gh_user" || echo "Could not assign reviewer, continuing" ``` -4. Read the PR description carefully - -Use the body from step 1. Summarize goal, scope, and missing context. - -5. Read the diff thoroughly - -Minimum: +4. Read PR description and diff ```sh +scripts/pr review-checkout-pr gh pr diff + +source .local/review-context.env +git diff --stat "$MERGE_BASE"..pr- +git diff "$MERGE_BASE"..pr- ``` -If you need full code context locally, fetch the PR head to a local ref and diff it. Do not create a merge commit. +5. Optional local tests + +Use the wrapper for target validation and executed-test verification: ```sh -git fetch origin pull//head:pr- -# Show changes without modifying the working tree - -git diff --stat origin/main..pr- -git diff origin/main..pr- +scripts/pr review-tests [ ...] ``` -If you want to browse the PR version of files directly, temporarily check out `pr-` in the worktree. Do not commit or push. Return to `temp/pr-` and reset to `origin/main` afterward. +6. Initialize review artifact templates ```sh -# Use only if needed -# git checkout pr- -# ...inspect files... - -git checkout temp/pr- -git reset --hard origin/main +scripts/pr review-artifacts-init ``` -6. Validate the change is needed and valuable +7. Produce review outputs -Be honest. Call out low value AI slop. +- Fill `.local/review.md` sections A through J. +- Fill `.local/review.json`. -7. Evaluate implementation quality +Minimum JSON shape: -Review correctness, design, performance, and ergonomics. +```json +{ + "recommendation": "READY FOR /prepare-pr", + "findings": [ + { + "id": "F1", + "severity": "IMPORTANT", + "title": "...", + "area": "path/or/component", + "fix": "Actionable fix" + } + ], + "tests": { + "ran": [], + "gaps": [], + "result": "pass" + }, + "docs": "up_to_date|missing|not_applicable", + "changelog": "required|not_required" +} +``` -8. Perform a security review - -Assume OpenClaw subagents run with full disk access, including git, gh, and shell. Check auth, input validation, secrets, dependencies, tool safety, and privacy. - -9. Review tests and verification - -Identify what exists, what is missing, and what would be a minimal regression test. - -10. Check docs - -Check if the PR touches code with related documentation such as README, docs, inline API docs, or config examples. - -- If docs exist for the changed area and the PR does not update them, flag as IMPORTANT. -- If the PR adds a new feature or config option with no docs, flag as IMPORTANT. -- If the change is purely internal with no user-facing impact, skip this. - -11. Check changelog - -Check if `CHANGELOG.md` exists and whether the PR warrants an entry. - -- If the project has a changelog and the PR is user-facing, flag missing entry as IMPORTANT. -- Leave the change for /preparepr, only flag it here. - -12. Answer the key question - -Decide if /preparepr can fix issues or the contributor must update the PR. - -13. Save findings to the worktree - -Write the full structured review sections A through J to `.local/review.md`. -Create or overwrite the file and verify it exists and is non-empty. +8. Guard + validate before final output ```sh -ls -la .local/review.md -wc -l .local/review.md +scripts/pr review-guard +scripts/pr review-validate-artifacts ``` -14. Output the structured review - -Produce a review that matches what you saved to `.local/review.md`. - -A) TL;DR recommendation - -- One of: READY FOR /preparepr | NEEDS WORK | NEEDS DISCUSSION | NOT USEFUL (CLOSE) -- 1 to 3 sentences. - -B) What changed - -C) What is good - -D) Security findings - -E) Concerns or questions (actionable) - -- Numbered list. -- Mark each item as BLOCKER, IMPORTANT, or NIT. -- For each, point to file or area and propose a concrete fix. - -F) Tests - -G) Docs status - -- State if related docs are up to date, missing, or not applicable. - -H) Changelog - -- State if `CHANGELOG.md` needs an entry and which category. - -I) Follow ups (optional) - -J) Suggested PR comment (optional) - ## Guardrails -- Worktree only. -- Do not delete the worktree after review. -- Review only, do not merge, do not push. +- Keep review read-only. +- Do not delete worktree. +- Use merge-base scoped diff for local context to avoid stale branch drift. diff --git a/.cursor/plans/dench_filesystem_crm_integration.plan.md b/.cursor/plans/dench_filesystem_crm_integration.plan.md new file mode 100644 index 00000000000..d2f3698bd23 --- /dev/null +++ b/.cursor/plans/dench_filesystem_crm_integration.plan.md @@ -0,0 +1,665 @@ +--- +name: Dench Filesystem CRM Integration +overview: Replace Dench's slow tool-by-tool CRM agent with a filesystem-first architecture where OpenClaw manages a DuckDB database and markdown documents in a dedicated workspace folder, synced via S3, and surfaced in Dench's sidebar. +todos: + - id: inject-skill-infra + content: "Phase 1: Add `inject: true` skill metadata support to OpenClaw (types.ts, workspace.ts, system-prompt.ts, run.ts)" + status: pending + - id: dench-skill + content: "Phase 2: Create skills/dench/SKILL.md with DuckDB schema, SQL patterns, CRM patterns, document management instructions" + status: pending + - id: s3-sync-script + content: "Phase 3: Create S3 sync script and sandbox startup hook for dench workspace persistence" + status: pending + - id: dench-sidebar + content: "Phase 4 (DEFERRED): Add workspace data source to Dench sidebar (tRPC endpoints + buildKnowledgeTree merge)" + status: pending + - id: lambda-sync + content: "Phase 4b (DEFERRED): S3-to-PostgreSQL Lambda sync for fast sidebar queries (or simpler: direct DuckDB read from S3)" + status: pending +isProject: false +--- + +# Dench Filesystem-First CRM via OpenClaw + +## Architecture Overview + +The current Dench CRM agent uses 15+ individual tRPC-backed tools (createObjectTool, createFieldTool, createEntryTool, searchEntriesTool, etc.) that each make a Prisma call. This is slow -- the agent often needs 10+ tool calls for a single user request. + +The new architecture gives OpenClaw a `dench/` workspace folder with a DuckDB database and markdown files. The agent generates SQL directly via `exec` (duckdb CLI) and writes documents as `.md` files. S3 syncs this data between sandbox sessions, and Dench's sidebar reads from S3/PostgreSQL. + +**Why DuckDB over SQLite:** + +- Native PIVOT/UNPIVOT -- essential for the EAV (Entity-Attribute-Value) pattern used by custom fields; every "show me entries as a table" query needs pivot +- PostgreSQL-compatible SQL dialect -- matches Dench's Supabase/Postgres, so generated SQL is portable +- Native JSON type -- clean handling of `enum_values`, `enum_colors`, field mappings (no JSON1 extension needed) +- Built-in CSV/Parquet import/export with auto-detection -- bulk CRM operations ("import 500 leads from CSV") +- FTS extension -- full-text search across entry fields +- `generate_series` + macros -- nanoid 32 generation in pure SQL +- ~50-100ms startup overhead is acceptable given agent tool-call overhead is already 200-500ms + +```mermaid +flowchart TB + subgraph sandbox [OpenClaw Sandbox] + agent[OpenClaw Agent] + skill[Dench Skill - always in context] + ctx[dench/workspace_context.yaml] + db[dench/workspace.duckdb] + docs[dench/documents/*.md] + agent --> skill + agent -->|"read-only context"| ctx + agent -->|"exec: duckdb"| db + agent -->|"write/edit"| docs + end + + subgraph s3layer [Persistence Layer] + s3["S3: dench/{orgId}/"] + end + + subgraph denchApp [Dench Web App] + sidebar[App Sidebar] + api[tRPC API] + pg[PostgreSQL - fs_files sync] + end + + db -->|sync on save| s3 + docs -->|sync on save| s3 + s3 -->|Lambda trigger| pg + pg --> api --> sidebar + s3 -->|download on sandbox start| db + s3 -->|download on sandbox start| docs +``` + +--- + +## Phase 1: Always-In-Context Skill Infrastructure (OpenClaw) + +Currently, all skills are lazy-loaded: only name + description appear in the system prompt, and the agent must `read` the SKILL.md to get full instructions. For the Dench CRM skill, we need the full content injected automatically. + +**Approach:** Add an `inject: true` flag to skill metadata. When set, the skill's full content is included in the system prompt alongside bootstrap files (in the "Project Context" section), not in the lazy-loaded skills list. + +**Files to modify:** + +- [src/agents/skills/types.ts](src/agents/skills/types.ts) -- Add `inject?: boolean` to `OpenClawSkillMetadata` +- [src/agents/skills/workspace.ts](src/agents/skills/workspace.ts) -- In `buildWorkspaceSkillSnapshot()`, separate injected skills from lazy-loaded skills. Return injected skill contents alongside the prompt. +- [src/agents/system-prompt.ts](src/agents/system-prompt.ts) -- Accept injected skill content and include it in the "Project Context" / "Workspace Files" section (similar to how bootstrap files like AGENTS.md are included via `contextFiles`) +- [src/agents/pi-embedded-runner/run.ts](src/agents/pi-embedded-runner/run.ts) -- Pass injected skill content through to the system prompt builder + +**Key change in `buildWorkspaceSkillSnapshot`:** + +```typescript +// New return type addition +export type SkillSnapshot = { + prompt: string; // lazy-loaded skills prompt (XML) + injectedContent?: string; // always-in-context skill content (concatenated) + skills: Array<{ name: string; primaryEnv?: string }>; + // ... +}; +``` + +--- + +## Phase 2: Dench CRM Skill (OpenClaw) + +Create `skills/dench/SKILL.md` with: + +- Full DuckDB schema reference +- nanoid 32 macro for ID generation (matching Dench's Supabase nanoid IDs) +- SQL patterns for all CRUD operations including PIVOT for table views +- Document management instructions +- Workspace structure documentation +- CRM patterns (contact, lead, deal, etc.) ported from the [Dench CRM agent prompt](file:///Users/kumareth/Documents/projects/dench/src/lib/agents/crm-agent.ts) + +**New file:** `skills/dench/SKILL.md` + +**Skill frontmatter:** + +```yaml +--- +name: dench +description: Manage Dench CRM workspace - create objects, fields, entries via DuckDB and documents as markdown files +metadata: + openclaw: + inject: true + emoji: "📊" +--- +``` + +**Workspace directory structure managed by the skill:** + +``` +~/.openclaw/workspace/dench/ + workspace_context.yaml # READ-ONLY context (org, members, integrations, defaults) + workspace.duckdb # DuckDB database (CRM data) + documents/ # Markdown documents (nested by path) + getting-started.md + projects/ + project-alpha.md + exports/ # Generated CSV/Excel exports + WORKSPACE.md # Auto-generated schema documentation +``` + +**workspace_context.yaml** -- read-only context the agent consumes on startup (written by Dench, never by the agent): + +```yaml +# Dench Workspace Context (READ-ONLY) +# This file is generated by Dench and synced via S3. +# The agent reads this for organizational context but MUST NOT modify it. +# Changes flow from Dench UI -> S3 -> this file (on sandbox init). + +workspace: + version: 1 + +# Organization identity (synced from Dench on sandbox init) +organization: + id: "org_abc123" + name: "Acme Corp" + slug: "acme-corp" + business: + name: "Acme Corporation" + type: "saas" # saas, agency, ecommerce, services, etc. + industry: "Technology" + website: "https://acme.com" + +# Team members -- needed for "user" type fields (e.g. "Assigned To") +# Agent uses these IDs when creating entries with user-type fields. +members: + - id: "usr_abc123" + name: "John Doe" + email: "john@acme.com" + role: owner + - id: "usr_def456" + name: "Jane Smith" + email: "jane@acme.com" + role: admin + - id: "usr_ghi789" + name: "Bob Wilson" + email: "bob@acme.com" + role: member + +# Protected objects -- cannot be deleted/renamed by agent +# Mirrors Dench's base-objects.ts immutable list +protected_objects: + - name: "people" + description: "Contact records" + icon: "users" + - name: "companies" + description: "Company records" + icon: "building-2" + +# Connected integrations -- agent reads this for sync context +# Populated by Dench when sandbox initializes from S3 +integrations: + connections: [] + # Example when connected: + # - app_key: "salesforce" + # app_name: "Salesforce" + # connection_id: "conn_xyz" + # synced_objects: + # - external_resource: "Lead" + # local_object: "lead" + # sync_direction: bidirectional # import, export, bidirectional + # sync_frequency: hourly # realtime, hourly, daily, manual + # field_mappings: + # "FirstName": "Full Name" + # "Email": "Email Address" + +# Enrichment configuration +enrichment: + enabled: false + provider: "aviato" # aviato, apollo + auto_enrich: false # Auto-enrich new entries on creation + +# CRM defaults +defaults: + default_view: table # table, kanban + date_format: "YYYY-MM-DD" + naming_convention: singular_lowercase # Object names: "lead" not "Leads" + +# S3 persistence +sync: + s3_bucket: "dench-workspaces" + s3_prefix: "" # Set to org_id on init + frequency: on_write # on_write, manual + last_synced_at: null + +# Credit account (for enrichment, AI operations) +credits: + allowance_balance: 0 + topup_balance: 0 +``` + +**Why YAML for context (not DuckDB):** This is read-only context the agent consumes once at startup -- never writes. The agent can `cat workspace_context.yaml` to understand the full org context instantly. Members list means no separate query to resolve user-type field assignments. Integrations give awareness of sync relationships without querying external APIs. This follows the Fintool pattern where user state (preferences, watchlists) lives as YAML in S3 while dense queryable data lives in the database. The data flow is one-way: Dench UI -> S3 -> workspace_context.yaml (on sandbox init). The agent never writes back to this file. + +**DuckDB schema** (initialized by agent on first use via `duckdb dench/workspace.duckdb`): + +```sql +-- nanoid 32 macro: generates 32-char IDs matching Dench's Supabase nanoid format +CREATE OR REPLACE MACRO nanoid32() AS ( + SELECT string_agg( + substr('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_-', + (floor(random() * 64) + 1)::int, 1), '') + FROM generate_series(1, 32) +); + +CREATE TABLE IF NOT EXISTS objects ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + name VARCHAR NOT NULL, + description VARCHAR, + icon VARCHAR, + default_view VARCHAR DEFAULT 'table', -- 'table' or 'kanban' + parent_document_id VARCHAR, + sort_order INTEGER DEFAULT 0, + source_app VARCHAR, + immutable BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(name) +); + +CREATE TABLE IF NOT EXISTS fields ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + object_id VARCHAR NOT NULL REFERENCES objects(id) ON DELETE CASCADE, + name VARCHAR NOT NULL, + description VARCHAR, + type VARCHAR NOT NULL, -- text, number, email, phone, boolean, date, richtext, user, relation, enum + required BOOLEAN DEFAULT false, + default_value VARCHAR, + related_object_id VARCHAR REFERENCES objects(id) ON DELETE SET NULL, + relationship_type VARCHAR, -- one_to_one, one_to_many, many_to_one, many_to_many + enum_values JSON, -- ["New","In Progress","Done"] + enum_colors JSON, -- ["#ef4444","#f59e0b","#22c55e"] + enum_multiple BOOLEAN DEFAULT false, + sort_order INTEGER DEFAULT 0, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(object_id, name) +); + +CREATE TABLE IF NOT EXISTS entries ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + object_id VARCHAR NOT NULL REFERENCES objects(id) ON DELETE CASCADE, + sort_order INTEGER DEFAULT 0, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS entry_fields ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + entry_id VARCHAR NOT NULL REFERENCES entries(id) ON DELETE CASCADE, + field_id VARCHAR NOT NULL REFERENCES fields(id) ON DELETE CASCADE, + value VARCHAR, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(entry_id, field_id) +); + +CREATE TABLE IF NOT EXISTS statuses ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + object_id VARCHAR NOT NULL REFERENCES objects(id) ON DELETE CASCADE, + name VARCHAR NOT NULL, + color VARCHAR DEFAULT '#94a3b8', + sort_order INTEGER DEFAULT 0, + is_default BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(object_id, name) +); + +CREATE TABLE IF NOT EXISTS documents ( + id VARCHAR PRIMARY KEY DEFAULT (nanoid32()), + title VARCHAR DEFAULT 'Untitled', + icon VARCHAR, + cover_image VARCHAR, + file_path VARCHAR NOT NULL UNIQUE, -- relative path in documents/ dir + parent_id VARCHAR REFERENCES documents(id) ON DELETE CASCADE, + parent_object_id VARCHAR REFERENCES objects(id) ON DELETE CASCADE, + sort_order INTEGER DEFAULT 0, + is_published BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +-- Full-text search index (DuckDB FTS extension) +INSTALL fts; LOAD fts; +``` + +**Auto-generated views** -- after every object/field mutation, the agent regenerates a PIVOT view for each object. These are stored queries (zero data duplication) that make the EAV pattern invisible: + +```sql +-- Auto-generated after creating the "leads" object and its fields +CREATE OR REPLACE VIEW v_leads AS +PIVOT ( + SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = (SELECT id FROM objects WHERE name = 'leads') +) ON field_name USING first(value); + +-- Now query like a normal table: +SELECT * FROM v_leads WHERE "Status" = 'New' ORDER BY "Full Name" LIMIT 50; +SELECT "Status", COUNT(*) FROM v_leads GROUP BY "Status"; +SELECT * FROM v_leads WHERE "Email Address" LIKE '%@gmail.com'; +``` + +Views are regenerated (not data, just the query definition) whenever fields are added/removed/renamed. Naming convention: `v_{object_name}` (e.g., `v_leads`, `v_companies`, `v_people`). + +**Filesystem directory structure** -- auto-projected from DuckDB after schema mutations. Represents the sidebar's nested knowledge tree. NO entry data in the filesystem (DuckDB is sole source of truth for entries): + +``` +dench/ + workspace.duckdb # SOLE source of truth for all structured data + views + workspace_context.yaml # Read-only org context + knowledge/ # Root of knowledge tree (= sidebar root) + people/ # Object "people" (directory = object node in sidebar) + .object.yaml # Lightweight metadata projection (id, icon, view, field list) + onboarding-guide.md # Document nested UNDER the people object + companies/ # Object "companies" + .object.yaml + projects/ # Document "Projects" (directory with children) + projects.md # Document content + tasks/ # Object nested UNDER the projects document + .object.yaml + roadmap.md # Sibling document + sales/ + sales.md + leads/ # Object nested under sales document + .object.yaml + follow-up-playbook.md # Document nested under leads object + exports/ # CSV/Parquet exports (on-demand, not auto-generated) + WORKSPACE.md # Auto-generated schema summary +``` + +**Source of truth rules:** + +- **Entries (rows)**: DuckDB ONLY (queried via `v_{object}` views). Never duplicated to filesystem. +- **Fields (columns)**: DuckDB. Summary projected to `.object.yaml` (read-only). +- **Objects (tables)**: DuckDB. Projected as directories + `.object.yaml` (read-only). Queryable via auto-generated views. +- **Document metadata**: DuckDB. Projected as directory structure (read-only). +- **Document content**: Filesystem (`.md` files). DuckDB stores `file_path` reference only. +- **Nesting/ordering**: DuckDB. Projected as directory hierarchy (read-only). + +**Key DuckDB advantages leveraged:** + +- **PIVOT views**: Auto-generated `v_{object}` views make EAV invisible -- query like normal tables +- **Native JSON**: `enum_values` and `enum_colors` are native JSON columns, no string parsing needed +- **CSV import**: `COPY v_leads TO 'exports/leads.csv';` or import with `COPY ... FROM 'import.csv' (AUTO_DETECT true);` +- **PostgreSQL dialect**: Generated SQL is directly portable to Dench's Supabase/Postgres + +**Skill content structure** -- the full SKILL.md incorporates and adapts every section from Dench's existing CRM agent prompt ([src/lib/agents/crm-agent.ts](file:///Users/kumareth/Documents/projects/dench/src/lib/agents/crm-agent.ts)), rewritten for DuckDB/filesystem execution instead of tool calls: + +### Section 1: Role and Workspace Startup + +- Role: Dench CRM Management Agent operating via DuckDB and filesystem +- On every conversation: read `dench/workspace_context.yaml` (READ-ONLY) for org context, members, integrations, protected objects +- Initialize DuckDB if not exists: `duckdb dench/workspace.duckdb < schema.sql` +- Database path: `dench/workspace.duckdb`, documents path: `dench/documents/` + +### Section 2: Primary Responsibilities (adapted from ``) + +- **Request analysis**: Same as original -- extract intent, identify entities/objects/fields/relationships, transform vague requests into structured SQL +- **Object creation**: Instead of `createObjectTool`, generate `INSERT INTO objects` SQL. Naming convention: singular, lowercase (e.g., "lead", "customer"). Check existing with `SELECT` first. For kanban: auto-create Status (enum) and Assigned To (user) fields in same transaction +- **Field management**: Instead of `createFieldTool`, generate `INSERT INTO fields` SQL. Field types: text, number, email, phone, boolean, date, richtext, user, relation, enum. Use `INSERT ... ON CONFLICT (object_id, name) DO UPDATE` for idempotency +- **Entry creation**: Instead of `createEntryTool`, generate `INSERT INTO entries` + `INSERT INTO entry_fields` in a transaction. Resolve field names to field IDs via `SELECT id FROM fields WHERE object_id = ? AND name = ?` +- **Entry search**: Instead of `searchEntriesTool`, generate PIVOT queries with WHERE/LIKE/ORDER BY. Use DuckDB FTS for full-text search. Operators: `=` (equals), `LIKE '%...%'` (contains), `LIKE '...%'` (startsWith), `LIKE '%...'` (endsWith), `IS NULL` (isEmpty), `IS NOT NULL` (isNotEmpty) + +### Section 3: SQL Operation Guide (replaces ``) + +Each former tool maps to SQL patterns: + +- **createObjectTool -> INSERT object**: `INSERT INTO objects (name, description, icon, default_view) VALUES (...) ON CONFLICT (name) DO NOTHING RETURNING *;` +- **createFieldTool -> INSERT field**: `INSERT INTO fields (object_id, name, type, required, enum_values, enum_colors, related_object_id, relationship_type, sort_order) VALUES (...) ON CONFLICT (object_id, name) DO UPDATE SET ...;` +- **createEntryTool -> INSERT entry + entry_fields**: Transaction with `INSERT INTO entries` then `INSERT INTO entry_fields` for each field value +- **getObjectTool -> SELECT object + fields**: `SELECT o.*, json_group_array(json_object('id', f.id, 'name', f.name, 'type', f.type)) as fields FROM objects o LEFT JOIN fields f ON f.object_id = o.id WHERE o.id = ? GROUP BY o.id;` +- **getObjectsTool -> SELECT all objects**: `SELECT o.*, COUNT(e.id) as entry_count FROM objects o LEFT JOIN entries e ON e.object_id = o.id GROUP BY o.id ORDER BY o.sort_order;` +- **getOrganizationMembersTool -> READ workspace_context.yaml**: Members list is in `workspace_context.yaml` under `members:`. Read with `cat dench/workspace_context.yaml` and extract the members section. User fields store member IDs like `usr_abc123`. +- **searchEntriesTool -> query the auto-generated view**: + ```sql + -- Simple: query the pre-built PIVOT view like a normal table + SELECT * FROM v_leads WHERE "Status" = 'New' ORDER BY created_at DESC LIMIT 50; + SELECT * FROM v_leads WHERE "Email Address" LIKE '%@gmail.com'; + SELECT * FROM v_leads WHERE "Full Name" ILIKE '%john%'; + SELECT "Status", COUNT(*) FROM v_leads GROUP BY "Status"; + ``` + Views are auto-generated per object (`v_{object_name}`) so the agent never writes raw PIVOT queries for reads. +- **updateObjectTool -> UPDATE object**: `UPDATE objects SET name = ?, description = ?, updated_at = now() WHERE id = ?;` +- **updateFieldTool -> UPDATE field**: `UPDATE fields SET ... WHERE id = ?;` +- **updateEntryTool -> UPSERT entry_fields**: `INSERT INTO entry_fields (entry_id, field_id, value) VALUES (?, ?, ?) ON CONFLICT (entry_id, field_id) DO UPDATE SET value = excluded.value, updated_at = now();` +- **deleteObjectTool -> DELETE cascade**: `DELETE FROM objects WHERE id = ? AND immutable = false;` (cascades to fields, entries, entry_fields via FK) +- **deleteFieldTool -> DELETE field**: `DELETE FROM fields WHERE id = ?;` (cascades to entry_fields) +- **deleteEntryTool -> DELETE entry**: `DELETE FROM entries WHERE id = ?;` (cascades to entry_fields) +- **createManyEntriesTool -> Batch INSERT**: Wrap multiple entry+entry_fields inserts in `BEGIN TRANSACTION; ... COMMIT;` +- **Bulk import from CSV**: `COPY ... FROM 'import.csv' (AUTO_DETECT true);` with field mapping + +### Section 4: Execution Workflows (adapted from ``) + +Same workflow selection principle: choose the minimal workflow needed based on user intent. + +- **Create New CRM Structure**: `SELECT` to check existence -> `INSERT objects` -> `INSERT fields` (all in one `exec` call with multi-statement SQL in a transaction) +- **Search and Display**: Generate PIVOT query with appropriate WHERE/ORDER BY/LIMIT +- **Add New Entries**: `SELECT` object+fields -> `INSERT entries` + `INSERT entry_fields` in transaction +- **Update Existing Data**: PIVOT query to find -> `UPDATE` entry_fields +- **Quick Information**: `SELECT` with aggregate counts +- **Bulk Operations**: Multi-row INSERT in transaction, report counts +- **Data Cleanup**: PIVOT query with filters -> `DELETE` matching entries + +Key differences from original: + +1. **All steps happen in a single `exec` call** with multi-statement SQL, not 10+ separate tool calls +2. **After schema mutations**: regenerate the `v_{object}` view and project the filesystem directory structure +3. **Reads use views**: `SELECT * FROM v_leads` instead of raw PIVOT queries + +Example of creating a full CRM structure in one shot: + +```sql +BEGIN TRANSACTION; +INSERT INTO objects (name, description, icon, default_view) VALUES ('lead', 'Sales leads', 'user-plus', 'table') ON CONFLICT (name) DO NOTHING; +INSERT INTO fields (object_id, name, type, required, sort_order) VALUES + ((SELECT id FROM objects WHERE name = 'lead'), 'Full Name', 'text', true, 0), + ((SELECT id FROM objects WHERE name = 'lead'), 'Email Address', 'email', true, 1), + ((SELECT id FROM objects WHERE name = 'lead'), 'Phone Number', 'phone', false, 2) +ON CONFLICT (object_id, name) DO NOTHING; +INSERT INTO fields (object_id, name, type, enum_values, enum_colors, sort_order) VALUES + ((SELECT id FROM objects WHERE name = 'lead'), 'Status', 'enum', + '["New","Contacted","Qualified","Converted"]'::JSON, + '["#94a3b8","#3b82f6","#f59e0b","#22c55e"]'::JSON, 3) +ON CONFLICT (object_id, name) DO NOTHING; + +-- Auto-generate the PIVOT view for this object +CREATE OR REPLACE VIEW v_lead AS +PIVOT ( + SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = (SELECT id FROM objects WHERE name = 'lead') +) ON field_name USING first(value); + +COMMIT; +-- Then: project filesystem structure (mkdir knowledge/lead/, write .object.yaml) +``` + +### Section 5: CRM Patterns (adapted from ``) + +Identical patterns, but with SQL examples instead of tool call examples: + +- **Contact/Customer**: Full Name (text, required), Email Address (email, required), Phone Number (phone), Company (relation), Notes (richtext) +- **Lead/Prospect**: Full Name, Email, Phone, Status (enum: New/Contacted/Qualified/Converted), Source (enum), Score (number), Assigned To (user), Notes (richtext) +- **Company/Organization**: Company Name (text, required), Industry (enum), Website (text), Type (enum), Notes (richtext) +- **Deal/Opportunity**: Deal Name, Amount (number), Stage (enum: Discovery/Proposal/Negotiation/Closed Won/Closed Lost), Close Date (date), Probability (number), Primary Contact (relation), Assigned To (user), Notes (richtext) +- **Case/Project**: Case Number, Title, Client (relation), Status (enum), Priority (enum), Due Date (date), Assigned To (user), Notes (richtext) +- **Property/Asset**: Address, Property Type (enum), Price (number), Status (enum), Square Footage (number), Bedrooms (number), Notes (richtext) +- **Task/Activity** (kanban): Title, Description, Assigned To (user), Due Date, Status (enum: In Queue/In Progress/Done), Priority (enum), Notes (richtext). Use `default_view = 'kanban'` and auto-create Status + Assigned To fields. + +### Section 6: Field Type Reference (adapted from ``) + +- **text**: General text data, names, descriptions, addresses. Stored as VARCHAR. +- **email**: Email addresses. Stored as VARCHAR. Agent validates format. +- **phone**: Phone numbers. Stored as VARCHAR. Agent normalizes format. +- **number**: Numeric values (prices, quantities, scores). Stored as VARCHAR in entry_fields (EAV), cast with `::NUMERIC` in queries. +- **boolean**: Yes/no flags. Stored as "true"/"false" strings. +- **date**: Dates. Stored as ISO 8601 strings. Cast with `::DATE` in queries. +- **richtext**: Rich text for Notes fields. Content stored as entry_field value (plain text / markdown). Displayed in Notion-style editor in Dench UI. +- **user**: Person/assignee fields. Stores member ID (e.g., "usr_abc123") from `workspace_context.yaml` members list. ALWAYS resolve member name -> ID before inserting. +- **enum**: Dropdown/select fields. Field definition stores `enum_values` as JSON array. Entry stores the selected value string. `enum_colors` parallel array for styling. `enum_multiple = true` for multi-select (value stored as JSON array string). +- **relation**: Links to entries in another object. Field stores `related_object_id` and `relationship_type`. Entry stores the related entry ID(s). `many_to_one` for single select, `many_to_many` for multi-select (stored as JSON array). + +### Section 7: Naming Conventions and Data Handling (adapted from `` and ``) + +- Object names: singular, lowercase, one word ("lead" not "Leads") +- Field names: human-readable, proper capitalization ("Email Address" not "email") +- Be descriptive: "Phone Number" not "Phone" +- Be consistent within an object +- Validate email formats, normalize phone numbers, use ISO 8601 dates +- Trim whitespace from all values +- Check for duplicates before creating entries (SELECT before INSERT) + +### Section 8: Error Handling (adapted from ``) + +- `UNIQUE constraint` on INSERT -> item already exists, treat as SUCCESS (use `ON CONFLICT DO NOTHING` or `DO UPDATE`) +- Protected object deletion -> check `immutable` column and `protected_objects` in workspace_context.yaml +- Field type mismatch -> warn user before changing type on field with existing data +- Missing required fields -> validate before INSERT, report which fields are missing + +### Section 9: Document Management (new -- not in original CRM agent) + +- Create document: `write` tool to create `dench/documents/.md` + `INSERT INTO documents` with metadata +- Document content is the .md file; DuckDB `documents` table tracks metadata (title, icon, nesting, order) +- Cross-nesting: documents under objects (`parent_object_id`), objects under documents (`parent_document_id`) +- Document tree mirrors filesystem: `dench/documents/projects/alpha.md` -> `file_path = 'projects/alpha.md'` + +### Section 10: Protected Objects and Read-Only Context (new) + +- Read `protected_objects` from `workspace_context.yaml` on startup +- NEVER delete, rename, or modify immutable objects (People, Companies) +- NEVER modify `workspace_context.yaml` -- it is read-only context from Dench +- Members list is authoritative for user-type field resolution + +### Section 11: Post-Mutation Checklist (MANDATORY -- revised after agent testing) + +**Problem identified:** In testing, the agent correctly executed SQL (object + fields + entries) but skipped the filesystem projection (.object.yaml) and sometimes the PIVOT view. Root cause: the original skill mentioned these as afterthoughts ("Then project the filesystem...") with no concrete template or examples. The agent follows examples literally -- if examples only show SQL, it only does SQL. + +**Fix:** Every workflow example now uses an explicit 3-step structure. The post-mutation section is now a checklist, not a description. + +After creating/modifying an OBJECT or FIELDS: + +- `CREATE OR REPLACE VIEW v_{object_name}` -- regenerate PIVOT view +- `mkdir -p dench/knowledge/{object_name}/` -- create directory +- Write `.object.yaml` with id, name, description, icon, default_view, entry_count, and full field list +- Update WORKSPACE.md + +After adding ENTRIES: + +- Update `entry_count` in `.object.yaml` +- Verify: `SELECT * FROM v_{object} LIMIT 5` + +After deleting an OBJECT: + +- `DROP VIEW IF EXISTS v_{object_name}` +- `rm -rf dench/knowledge/{object_name}/` + +The skill now includes: + +- A concrete `.object.yaml` template with example content (previously missing entirely) +- Full bash commands for generating `.object.yaml` from DuckDB queries +- "Step 1 / Step 2 / Step 3" structure in every workflow example (SQL, Filesystem, Verify) +- Critical Reminders section leads with "NEVER SKIP FILESYSTEM PROJECTION" and "THREE STEPS, EVERY TIME" + +### Section 12: Critical Reminders (adapted from ``) + +- Handle the ENTIRE CRM operation from analysis to SQL execution **to filesystem projection** to summary +- **NEVER SKIP FILESYSTEM PROJECTION**: After any object mutation, create/update `.object.yaml` AND the `v_{object}` view. If missing, the object is invisible in the sidebar. +- **THREE STEPS, EVERY TIME**: (1) SQL transaction, (2) filesystem projection, (3) verify +- Always check existing data before creating (SELECT before INSERT, or ON CONFLICT) +- Search proactively to provide better UX (PIVOT with filters) +- Never assume field names -- always verify with `SELECT * FROM fields WHERE object_id = ?` +- Extract ALL data from user messages +- NOTES FIELDS: type "richtext", displayed in Notion editor +- USER FIELDS: Resolve member name to ID from workspace_context.yaml BEFORE inserting +- ENUM FIELDS: type "enum" with `enum_values` JSON array +- RELATION FIELDS: type "relation" with `related_object_id` +- KANBAN BOARDS: `default_view = 'kanban'`, auto-create Status and Assigned To fields +- PROTECTED OBJECTS: Never delete objects listed in `workspace_context.yaml` `protected_objects` +- ONE EXEC CALL: Batch related SQL in a single transaction whenever possible -- this is the entire point of the filesystem-first approach +- ENTRY COUNT: After adding entries, update `entry_count` in `.object.yaml` + +--- + +## Phase 3: S3 Sync Layer + +**Two sync directions:** + +1. **Sandbox -> S3:** After every database write or document save, sync changed files to S3 +2. **S3 -> Sandbox:** On sandbox startup, download the latest dench/ folder from S3 + +**Option A (simpler): Script-based sync** + +- Add a `dench/sync.sh` script that the agent calls after mutations +- Uses AWS CLI: `aws s3 sync dench/ s3://dench-workspaces/{orgId}/` +- Credentials injected via ABAC (short-lived, scoped to org prefix) +- Skill instructs the agent to run sync after writes + +**Option B (more robust): inotifywait + background sync** + +- Background process watches dench/ for changes +- Debounced sync to S3 (e.g., 5s after last change) +- More reliable but more infrastructure + +**Recommendation:** Start with Option A. The agent is already making exec calls; one more `aws s3 sync` is trivial. + +**New file:** `skills/dench/sync.sh` (bundled with the skill, referenced by SKILL.md) + +**Sandbox startup hook:** + +- Add to OpenClaw's sandbox initialization: download dench workspace from S3 if it exists +- Could be a `BOOTSTRAP.md` instruction or a sandbox pre-warm step + +--- + +## Phase 4: Dench Sidebar Integration + +The Dench sidebar currently reads from Prisma (`getAll` objects + `getAll` documents). It needs a new data source for workspace-managed data. + +**Approach:** Add a new tRPC endpoint that reads from S3 (or the synced PostgreSQL `fs_files` table) and returns the same tree structure the sidebar expects. + +**Files to modify in Dench:** + +- [src/lib/trpc/routers/objects.ts](file:///Users/kumareth/Documents/projects/dench/src/lib/trpc/routers/objects.ts) -- Add `getWorkspaceObjects` endpoint that queries the synced SQLite data (either by reading dench.db from S3, or from a PostgreSQL materialized view) +- [src/lib/trpc/routers/documents.ts](file:///Users/kumareth/Documents/projects/dench/src/lib/trpc/routers/documents.ts) -- Add `getWorkspaceDocuments` endpoint +- [src/components/app-sidebar.tsx](file:///Users/kumareth/Documents/projects/dench/src/components/app-sidebar.tsx) -- Merge workspace data into the `buildKnowledgeTree` function + +**S3 -> PostgreSQL sync (Lambda):** + +- When workspace.duckdb is uploaded to S3, a Lambda function: + 1. Downloads the DuckDB file + 2. Reads objects, fields, entries, documents tables + 3. Upserts into PostgreSQL tables (or a dedicated `workspace_objects`, `workspace_documents` table) +- This mirrors the Fintool pattern: S3 source of truth, Lambda sync, PG for fast queries + +**Alternative (simpler for v1):** Dench backend downloads workspace.duckdb from S3 on demand and queries it directly using `duckdb` Node.js bindings (`@duckdb/node-api`). No Lambda needed initially. + +--- + +## Phase 5: Real-Time Updates (Future) + +- WebSocket notifications when S3 objects change (via S3 Event Notifications -> SNS -> WebSocket) +- Dench sidebar auto-refreshes when workspace data changes +- Bidirectional: Dench UI edits write back to S3, agent picks up changes on next sandbox start + +--- + +## Key Design Decisions + +- **Three-layer storage** (following Fintool pattern): + - **YAML** (`workspace_context.yaml`): Read-only workspace identity, team members, integrations, defaults. Written by Dench, consumed by agent. Never modified by the agent -- data flows one-way from Dench UI -> S3 -> this file. + - **DuckDB** (`workspace.duckdb`): Dense, relational CRM data (objects, fields, entries, statuses). Queryable via SQL with PIVOT, JOIN, FTS. + - **Markdown** (`documents/*.md`): Rich document content. Agent uses `write`/`edit` tools directly. DuckDB tracks metadata (title, icon, nesting) while the file system holds the content. +- **DuckDB over SQLite**: Native PIVOT/UNPIVOT is essential for the EAV data model (rendering custom-field entries as tables). PostgreSQL-compatible SQL dialect means generated SQL is portable to Dench's Supabase/Postgres. Native JSON type eliminates string parsing for enum_values/colors. Built-in CSV/Parquet import with auto-detection enables bulk CRM operations. FTS extension provides full-text search. ~50-100ms startup overhead is acceptable. +- **nanoid 32 IDs**: Matches Dench's Supabase PostgreSQL nanoid format. Generated via a DuckDB macro using `generate_series` + `random()` over the standard nanoid alphabet (`0-9A-Za-z_-`). 32 chars provides 192 bits of entropy. +- **Members in YAML not DuckDB**: The agent needs member IDs for "user" type fields (like "Assigned To"). Putting members in workspace_context.yaml means the agent reads them once on startup without a separate SQL query. The list changes infrequently (team changes, not per-request). Agent reads only, never writes. +- **Integrations in YAML not DuckDB**: The agent needs to know what apps are connected and how fields map, but doesn't need to query this relationally. YAML gives the agent instant context about sync relationships. Agent reads only, never writes. +- **Skill inject vs bootstrap file**: Using skill metadata `inject: true` is cleaner than adding a new bootstrap file type. It keeps the Dench instructions modular and version-controlled in the skills directory. +- **One transaction per operation**: The skill instructs the agent to wrap multi-step operations in `BEGIN ... COMMIT` for atomicity. DuckDB supports full ACID transactions. diff --git a/.cursor/plans/file_chat_sidebar_368973cb.plan.md b/.cursor/plans/file_chat_sidebar_368973cb.plan.md new file mode 100644 index 00000000000..a5e14610732 --- /dev/null +++ b/.cursor/plans/file_chat_sidebar_368973cb.plan.md @@ -0,0 +1,138 @@ +--- +name: File chat sidebar +overview: Add a collapsible chat sidebar to the workspace file view that lets users chat with the agent about the currently open file, with file-scoped sessions stored separately from the main chat list. +todos: + - id: extract-chat-panel + content: Extract ChatPanel component from page.tsx with fileContext prop support + status: in_progress + - id: simplify-home-page + content: Simplify page.tsx to render Sidebar + ChatPanel + status: pending + - id: tag-file-sessions + content: Add filePath field to WebSessionMeta and filtering to GET /api/web-sessions + status: pending + - id: workspace-chat-sidebar + content: Add collapsible ChatPanel sidebar to workspace page with file context + status: pending + - id: live-reload + content: Re-fetch file content after agent finishes streaming to show edits live + status: pending +isProject: false +--- + +# File Chat Sidebar for Workspace + +## Architecture + +The workspace page layout changes from `[sidebar | content]` to `[sidebar | content | chat-panel]`. The chat panel reuses the same `useChat` + `ChatMessage` + session persistence logic from the main chat page. + +```mermaid +graph LR + subgraph workspace [Workspace Page Layout] + WS[WorkspaceSidebar_260px] --> MC[MainContent_flex1] + MC --> CP[ChatPanel_380px] + end + subgraph storage [Session Storage] + idx[index.json] -->|filePath field| fileScoped[File-scoped sessions] + idx -->|no filePath| globalSessions[Global sessions] + end + CP -->|POST /api/chat| agent[Agent Runner] + CP -->|file context in message| agent +``` + +## Step 1: Extract `ChatPanel` from `page.tsx` + +Extract the entire chat UI (messages list, input form, session management, `useChat`, streaming status) from [apps/web/app/page.tsx](apps/web/app/page.tsx) into a new reusable component: + +**New file:** `apps/web/app/components/chat-panel.tsx` + +```typescript +type ChatPanelProps = { + /** When set, scopes sessions to this file and prepends content as context */ + fileContext?: { path: string; content: string; filename: string }; + /** External session list (for sidebar session list) */ + sessions?: WebSessionMeta[]; + onSessionsChange?: () => void; + /** Compact mode for workspace sidebar (no lobster, smaller empty state) */ + compact?: boolean; +}; +``` + +- Internally uses `useChat` from `@ai-sdk/react` with `DefaultChatTransport` +- Manages its own `currentSessionId`, `savedMessageIdsRef`, `input`, etc. (same logic as `page.tsx`) +- When `fileContext` is provided: + - The first message in each session is prefixed with: `"[Context: file '{path}']\n\n{content}\n\n---\n\nUser question: {userText}"` -- subsequent messages just send `userText` as-is (the agent already has context from the conversation) + - Session creation passes `filePath` to the API +- Renders: header bar (with session title / status), scrollable message list using `ChatMessage`, error bar, input form + +Then **simplify `page.tsx**` to just: + +```tsx + + +``` + +## Step 2: Tag file sessions in web-sessions API + +Modify [apps/web/app/api/web-sessions/route.ts](apps/web/app/api/web-sessions/route.ts): + +- Add optional `filePath` to `WebSessionMeta` type +- `POST` accepts `filePath` in the body and stores it in the index +- `GET` accepts `?filePath=...` query param: + - If `filePath` is set: returns only sessions where `meta.filePath === filePath` + - If `filePath` is absent: returns only sessions where `meta.filePath` is falsy (excludes file-scoped sessions from the main list) + +This single change means the main chat sidebar automatically stops showing file sessions, with no other code changes needed. + +## Step 3: Add chat sidebar to workspace page + +Modify [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx): + +- Add a collapsible right panel (380px) that renders `ChatPanel` with `fileContext` +- The panel appears when a file or document is selected (content kinds: `"document"`, `"file"`) +- Add a toggle button in the breadcrumbs bar to show/hide the chat panel +- Pass the file's `content`, `path`, and `filename` as `fileContext` +- The panel shows a mini session list at top (file-scoped sessions only) and the chat below + +Layout becomes: + +``` ++------------------+------------------------+-------------------+ +| WorkspaceSidebar | Content (flex-1) | ChatPanel (380px) | +| (260px) | | [toggle] | ++------------------+------------------------+-------------------+ +``` + +## Step 4: File-scoped session list inside ChatPanel + +Inside `ChatPanel`, when `fileContext` is provided: + +- Fetch sessions filtered by `filePath` via `GET /api/web-sessions?filePath=...` +- Show a compact session list (just titles, clickable) above the messages area +- "New chat" button creates a new file-scoped session +- Selecting a session loads its messages (same logic as main page's `handleSessionSelect`) + +## Step 5: Live file reload after agent edits + +When the agent finishes streaming (status goes from `streaming` -> `ready`) and `fileContext` is provided: + +- Re-fetch the file content via `GET /api/workspace/file?path=...` +- Call a callback `onFileChanged?.(newContent)` so the workspace page can update `ContentState` without a full reload +- This makes edits appear live in the file viewer/document view next to the chat + +## Key files touched + +| File | Change | +| ---------------------------------------- | ------------------------------------ | +| `apps/web/app/components/chat-panel.tsx` | **New** -- extracted chat UI + logic | +| `apps/web/app/page.tsx` | Simplify to use `ChatPanel` | +| `apps/web/app/api/web-sessions/route.ts` | Add `filePath` field + filtering | +| `apps/web/app/workspace/page.tsx` | Add right chat sidebar with toggle | + +## What stays the same (reused as-is) + +- `ChatMessage` component -- no changes +- `ChainOfThought` component -- no changes +- `/api/chat` route + `agent-runner.ts` -- no changes (file context goes in the message text) +- `/api/web-sessions/[id]` and `/api/web-sessions/[id]/messages` routes -- no changes +- `useChat` from `@ai-sdk/react` -- same transport, same everything diff --git a/.cursor/plans/full_web_ui_redesign_9ad2e285.plan.md b/.cursor/plans/full_web_ui_redesign_9ad2e285.plan.md new file mode 100644 index 00000000000..afd0127c22f --- /dev/null +++ b/.cursor/plans/full_web_ui_redesign_9ad2e285.plan.md @@ -0,0 +1,324 @@ +--- +name: Full Web UI Redesign +overview: "Complete redesign of the OpenClaw web app (apps/web/) to match the Dench design system: switch from dark theme to light, adopt Instrument Serif + Inter fonts, port the Dench color palette and layout patterns, and rewrite every component and page from the ground up." +todos: + - id: foundation + content: "Phase 1: Rewrite globals.css (light theme, HSL tokens, font imports) and layout.tsx (next/font, remove dark mode)" + status: pending + - id: landing + content: "Phase 2: Rewrite app/page.tsx as Dench-style landing page (navbar, hero, demo sections, footer)" + status: pending + - id: layout-shell + content: "Phase 3: Create app-navbar.tsx, rewrite workspace/page.tsx layout with top navbar + sidebar grid" + status: pending + - id: sidebar + content: "Phase 4: Redesign workspace-sidebar.tsx and file-manager-tree.tsx to match Dench sidebar" + status: pending + - id: data-table + content: "Phase 5: Redesign object-table.tsx with Dench-style toolbar, sticky headers, pagination, enum badges" + status: pending + - id: kanban + content: "Phase 6: Redesign object-kanban.tsx with light cards, columns, board header" + status: pending + - id: entry-detail + content: "Phase 7: Redesign entry-detail-modal.tsx as right-panel slide-out with properties list" + status: pending + - id: dashboard-chat + content: "Phase 8a: Build dashboard view with greeting, centered chat input, suggestion chips, and animate-down-to-bottom Framer Motion layoutId transition" + status: pending + - id: chat + content: "Phase 8b: Restyle chat-panel.tsx, chat-message.tsx, chain-of-thought.tsx for light theme + bottom composer" + status: pending + - id: remaining + content: "Phase 9: Restyle all remaining components (breadcrumbs, document-view, file-viewer, database-viewer, empty-state, markdown, context-menu, slash-command, charts, etc.)" + status: pending + - id: deps + content: "Phase 10: Add framer-motion dependency, verify fonts work, test build" + status: pending +isProject: false +--- + +# Full Web UI Redesign — Dench Design System + +## Current State + +The OpenClaw web app is a **dark-themed** Next.js 15 app with: + +- Dark background (`#0a0a0a`), dark surfaces (`#141414`), orange accent (`#e85d3a`) +- Inter font only, no serif headings +- Minimal homepage (centered text + CTA) +- Workspace layout: left sidebar (260px) + content + optional chat panel +- Custom table/kanban/viewer components, all dark-styled +- Tailwind v4 (CSS-based config), no shadcn/ui + +## Target State (Dench Design) + +Per the screenshots and Dench source: + +- **Light theme** — `bg-neutral-50` layout, white cards, `bg-neutral-100` sidebar/navbar +- **Instrument Serif** for headings/titles, **Inter** for body text, **Lora** for branding +- **Top navbar** (grid 3-col, with Dashboard/Workflows/Integrations tabs, org logo, user menu) +- **Left sidebar** (260px, `bg-neutral-100`, collapsible knowledge tree with item counts) +- **Data tables** with: sticky header, column borders, search bar, filter/column controls, enum badges, relation chips, pagination +- **Kanban board** with rounded cards, priority badges, assignee avatars +- **Entry detail** right-panel slide-out with property list +- **Landing page** with hero section, demo sections, clean navigation bar +- **Dashboard chat UX** — centered greeting ("Good evening, Kumar?") in Instrument Serif + centered chat input with suggestion chips; on first message, the input animates down to a bottom-docked composer via Framer Motion shared `layoutId` spring transition +- HSL-based CSS variables (shadcn pattern), `--radius: 0.5rem`, neutral base color + +## Architecture Decision: Tailwind v4 + +The OpenClaw app uses **Tailwind v4** (CSS-based config via `@import "tailwindcss"`), while Dench uses Tailwind v3 (JS config). We will keep Tailwind v4 but port all design tokens into `globals.css` using `@theme` blocks and CSS custom properties. No downgrade needed. + +## Architecture Decision: Light + Dark Theme + +Dench is light-only. We will use Dench's light palette as the `:root` default AND create a custom dark palette under `.dark` (class-based toggle via ``). All components will use CSS variable references (e.g. `bg-background`, `text-foreground`, `border-border`) so they automatically adapt. No hardcoded hex/rgb in components. + +**Light palette** (from Dench): + +- `--background: 0 0% 96%` (neutral-50 feel) +- `--foreground: 0 0% 3.9%` +- `--card: 0 0% 100%` / `--card-foreground: 0 0% 3.9%` +- `--muted: 0 0% 96.1%` / `--muted-foreground: 0 0% 45.1%` +- `--border: 0 0% 89.8%` +- `--primary: 0 0% 9%` / `--primary-foreground: 0 0% 98%` +- `--accent: 0 0% 96.1%` / `--accent-foreground: 0 0% 9%` +- `--destructive: 0 84.2% 60.2%` + +**Dark palette** (custom, designed to complement Dench's light theme): + +- `--background: 0 0% 7%` (#121212 — rich near-black, not pure black) +- `--foreground: 0 0% 93%` (#ededed) +- `--card: 0 0% 10%` (#1a1a1a) / `--card-foreground: 0 0% 93%` +- `--muted: 0 0% 14%` (#242424) / `--muted-foreground: 0 0% 55%` (#8c8c8c) +- `--border: 0 0% 18%` (#2e2e2e) +- `--primary: 0 0% 93%` / `--primary-foreground: 0 0% 9%` +- `--accent: 0 0% 16%` (#292929) / `--accent-foreground: 0 0% 93%` +- `--destructive: 0 62% 55%` +- Sidebar: `--sidebar-bg: 0 0% 9%` (#171717) +- Navbar: similar to sidebar, subtle `border-b` at `--border` + +Sidebar/navbar in dark mode use a slightly elevated surface (`#171717`) rather than pure background, for depth. + +**Theme toggle:** add a sun/moon toggle button in the navbar (right side, near user avatar). Use `next-themes` or a simple `useEffect` + `localStorage` approach to persist preference and apply `.dark` class on ``. + +--- + +## Files to Change + +### Phase 1 — Foundation (Theme, Fonts, Layout Shell) + +**[app/globals.css](apps/web/app/globals.css)** — Complete rewrite: + +- `:root` block: Dench's light-theme HSL palette (background, foreground, card, primary, secondary, muted, accent, destructive, border, ring, sidebar, chart-1 through chart-5) +- `.dark` block: custom dark palette (see "Architecture Decision: Light + Dark Theme" above) — all same variable names, dark values +- Add `@theme` block for Tailwind v4 mapping CSS vars to utility classes (`bg-background`, `text-foreground`, `border-border`, `bg-card`, `text-muted-foreground`, etc.) +- Import Instrument Serif from Google Fonts +- Add `.font-instrument` utility class +- Port scrollbar, prose, editor, and slash-command styles using CSS variables (theme-aware, not hardcoded) +- Port workflow state colors (`--workflow-active`, `--workflow-processing`, `--workflow-idle`) + +**[app/layout.tsx](apps/web/app/layout.tsx)** — Rewrite: + +- Import Inter and Lora via `next/font/google` +- Set CSS variables `--font-corporate` and `--font-lora` +- Default to light: no `className="dark"` on `` (let theme provider handle it) +- Apply `font-corporate` to `` +- Add `suppressHydrationWarning` on `` for theme flash prevention +- Add inline script or `next-themes` `ThemeProvider` for class-based dark mode toggle with `localStorage` persistence +- Update metadata title/description to "Dench" branding + +**New: `app/hooks/use-theme.ts**` — Simple theme hook: + +- Read/write `localStorage` key `"theme"` (`"light"` | `"dark"` | `"system"`) +- Apply/remove `.dark` class on `document.documentElement` +- Expose `theme`, `setTheme`, `resolvedTheme` for components + +### Phase 2 — Landing Page + +**[app/page.tsx](apps/web/app/page.tsx)** — Full rewrite to match Dench landing: + +- Sticky navigation bar (logo "Dench" in `font-lora`, Login button in rounded-full blue pill) +- Hero section: "AI CRM" headline in `font-instrument font-bold`, subtext, "Get Started Free" CTA +- Full-width CRM demo area (window chrome with traffic-light dots, scaled mock table) +- Additional demo sections (workflow, kanban) — simplified versions +- Footer with copyright, links + +### Phase 3 — Workspace Layout Shell + +**[app/workspace/page.tsx](apps/web/app/workspace/page.tsx)** — Rewrite layout structure: + +- Add top `AppNavbar` component: `bg-neutral-100 border-b border-border shadow-[0_0_40px_rgba(0,0,0,0.05)]` + - Left: org logo + "Powered by Dench" + org name in `font-instrument` + - Center: tab navigation (Dashboard, Workflows, Integrations) with active state + - Right: credit display, notification bell, sun/moon theme toggle, user avatar dropdown +- Main area: `grid lg:grid-cols-[260px_1fr]` under navbar +- Full height: `h-[100dvh] flex flex-col bg-neutral-50` +- Content area: `overflow-y-auto overflow-x-hidden` +- Replace all inline `style={{}}` dark colors with Tailwind classes + +**New component: `app/components/workspace/app-navbar.tsx**` — Top navbar (extracted for reuse) + +### Phase 4 — Sidebar Redesign + +**[app/components/workspace/workspace-sidebar.tsx](apps/web/app/components/workspace/workspace-sidebar.tsx)** — Full rewrite: + +- Background: `bg-sidebar` with `border-r border-border` (light: neutral-100, dark: #171717 via CSS var) +- Shadow: theme-aware subtle shadow +- Header: "KNOWLEDGE" section label in uppercase `text-[11px] font-medium tracking-wider text-muted-foreground` +- Knowledge items: `text-[13px]`, hover `bg-accent`, `rounded-xl` +- Item badges showing entry counts in `bg-muted border border-border` pills +- Icons per item type (objects get custom icons, documents get doc icon) +- Collapsible sections: KNOWLEDGE, CHATS, TELEPHONY +- Bottom: "API Keys" link +- Remove all inline `style={{}}` dark colors + +**[app/components/workspace/file-manager-tree.tsx](apps/web/app/components/workspace/file-manager-tree.tsx)** — Restyle tree items: + +- Light-theme hover states, active states matching `bg-neutral-200` +- `text-[13px]` sizing, proper icon colors +- Drag-and-drop visual indicators in light theme + +### Phase 5 — Data Table Redesign + +**[app/components/workspace/object-table.tsx](apps/web/app/components/workspace/object-table.tsx)** — Complete rewrite to match Dench data-table: + +- Toolbar: object name in `font-instrument`, search input (`rounded-full shadow-[0_0_21px_0_rgba(0,0,0,0.07)]`), "Ask AI" button, Table/Board view toggle, refresh/import/filter/columns/+ Add buttons +- Table header: `sticky top-0 z-30 bg-card border-b-2 border-border/80`, sortable columns with sort arrows +- Table cells: `px-4 border-r border-border/30`, proper text truncation +- Enum badges: colored pill style matching Dench (translucent background + border) +- Relation chips: link icon + blue text +- Row hover: `hover:bg-muted/50` +- Pagination bar: "Showing 1 to N of N results", rows-per-page selector, page navigation +- "..." action menu per row (right column) + +### Phase 6 — Kanban Board Redesign + +**[app/components/workspace/object-kanban.tsx](apps/web/app/components/workspace/object-kanban.tsx)** — Rewrite: + +- Board header: view toggle (Board/Table), "Ask AI" button, search, "Group by" selector +- Columns: `bg-muted/50 rounded-2xl border border-border/60`, column title + count badge +- Cards: `bg-card rounded-xl border border-border/80 shadow-sm` +- Card content: title, field badges (objective, risk profile), date, assignee avatar +- "+ Add Item" at column bottom +- "Drop cards here" empty column placeholder + +### Phase 7 — Entry Detail Panel + +**[app/components/workspace/entry-detail-modal.tsx](apps/web/app/components/workspace/entry-detail-modal.tsx)** — Redesign as right-panel slide-out: + +- Takes ~40% of content width, pushes table left +- Header: icon + title in large font, "Created Jan 12, 2026 at 12:47 PM" subtitle +- "PROPERTIES" section label +- Property rows: label (uppercase text-xs text-muted-foreground) + value +- Relation fields show colored link chips +- Enum fields show colored badges (matching table) +- "Add a property" at bottom +- Close button (>> icon) top-right + +### Phase 8a — Dashboard Chat UX (Greeting + Animate-to-Bottom Input) + +This is the hero interaction on the workspace "Dashboard" tab — a centered greeting with a chat input that transitions into the bottom-docked composer after the first message. + +**How Dench implements it:** + +- `DashboardHeader`: time-based greeting ("Good morning/afternoon/evening, Name?") with staggered word-by-word Framer Motion entrance (`y:20 → 0`, `blur(8px) → blur(0)`) +- `DashboardChatbox`: centered TipTap input with placeholder "Build a workflow to automate your tasks", attach/voice/submit buttons, suggestion chips below (shuffled from a pool of ~27 templates, showing 7 in two rows) +- **Layout animation:** both the centered input and the bottom composer share a Framer Motion `layoutId="chat-thread-composer"`. When `showStartComposer` flips to false after the first message, Framer Motion automatically animates the input from center to bottom with `transition={{ type: "spring", stiffness: 260, damping: 30 }}` + +**New components to create:** + +`app/components/workspace/dashboard-view.tsx` — Dashboard home view: + +- Greeting in `font-instrument text-4xl` with time-based message + user name +- Word-by-word staggered Framer Motion entrance animation +- Centered chat input area below greeting + +`app/components/workspace/dashboard-chatbox.tsx` — Centered input + chips: + +- Rounded white card with subtle shadow, textarea/input with placeholder +- Attach (paperclip), voice (mic), submit (arrow) icon buttons +- Suggestion chip rows: 3 on first row, 4 on second row, each with icon + label + `rounded-xl` border +- Accepts `layoutId` prop for shared layout animation +- `mode` prop: `"dashboard"` (centered, with greeting) vs `"thread"` (same input but used within chat thread) +- Entry animation: `opacity: 0, y: 20` → `opacity: 1, y: 0`, duration 0.8s + +**Modify [app/workspace/page.tsx](apps/web/app/workspace/page.tsx):** + +- When no content selected (Dashboard tab active), render `DashboardView` +- On chat submit: transition to chat thread view +- Use `LayoutGroup` from Framer Motion to wrap the dashboard + chat area +- Track `showStartComposer` state: when true, show centered `DashboardChatbox`; when false, show messages + bottom `ChatComposer` — both sharing the same `layoutId` + +**Prompt templates** (simplified set for OpenClaw): + +- Follow-up Emails, Calendly Prep, Zoom Recap, Facebook Leads, Calendar Sync, Salesforce Sync, Intercom Chat (matching the Dench screenshot chips) + +### Phase 8b — Chat & Message Restyling + +**[app/components/chat-panel.tsx](apps/web/app/components/chat-panel.tsx)** — Restyle: + +- Theme-aware background (`bg-card`), card-colored input area +- Input: rounded border, subtle shadow, consistent with dashboard chatbox style +- Bottom-docked composer with `layoutId` for shared animation +- Session tabs in light theme +- Tool call indicators in light theme +- Send button styling (rounded, neutral) + +**[app/components/chat-message.tsx](apps/web/app/components/chat-message.tsx)** — Restyle: + +- Theme-aware message bubbles (user: `bg-muted`, assistant: `bg-card`) +- Code blocks with `bg-muted` +- Markdown rendering in light theme +- Chain-of-thought styling update + +**[app/components/chain-of-thought.tsx](apps/web/app/components/chain-of-thought.tsx)** — Light theme + +### Phase 9 — Remaining Components + +All components below: replace every hardcoded color (`style={{}}`, hex, rgb) with semantic Tailwind utilities (`bg-background`, `text-foreground`, `border-border`, `bg-card`, `text-muted-foreground`, `bg-muted`, etc.) so they work in both light and dark: + +- **[breadcrumbs.tsx](apps/web/app/components/workspace/breadcrumbs.tsx)** — `text-muted-foreground`, `hover:text-foreground` +- **[document-view.tsx](apps/web/app/components/workspace/document-view.tsx)** — `bg-card` background, `border-border` +- **[file-viewer.tsx](apps/web/app/components/workspace/file-viewer.tsx)** — `bg-muted` code blocks, `text-foreground` +- **[database-viewer.tsx](apps/web/app/components/workspace/database-viewer.tsx)** — `bg-card` tables, `bg-muted` query editor +- **[empty-state.tsx](apps/web/app/components/workspace/empty-state.tsx)** — `text-muted-foreground` illustration +- **[markdown-content.tsx](apps/web/app/components/workspace/markdown-content.tsx)** — Prose styles via CSS vars +- **[markdown-editor.tsx](apps/web/app/components/workspace/markdown-editor.tsx)** — `bg-card` editor chrome +- **[context-menu.tsx](apps/web/app/components/workspace/context-menu.tsx)** — `bg-card` dropdown, `border-border` +- **[slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx)** — `bg-card` command palette +- **[inline-rename.tsx](apps/web/app/components/workspace/inline-rename.tsx)** — `bg-card` input, `border-border` +- **[knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx)** — Theme-aware tree styles +- **[charts/](apps/web/app/components/charts/)** — All chart components: CSS var chart colors, `bg-card` panels +- **[sidebar.tsx](apps/web/app/components/sidebar.tsx)** — Theme-aware (if still used) + +### Phase 10 — Package Dependencies + +**[package.json](apps/web/package.json)** — Add if needed: + +- `framer-motion` (for landing page + dashboard chat animations) +- `next-themes` (for dark/light toggle with `localStorage` + class-based switching, SSR-safe) +- Verify `next/font/google` is available (bundled with Next.js) + +--- + +## Key Design Tokens + +- **Radius:** `0.5rem` base +- **Primary font:** Inter via `next/font/google` +- **Heading font:** Instrument Serif via Google Fonts import +- **Brand font:** Lora via `next/font/google` +- **Sidebar width:** 260px +- **Shadows (light):** `shadow-[0_0_40px_rgba(0,0,0,0.05)]` (sidebar/navbar), `shadow-[0_0_21px_0_rgba(0,0,0,0.07)]` (search) +- **Shadows (dark):** `shadow-[0_0_40px_rgba(0,0,0,0.2)]` (sidebar/navbar), `shadow-[0_0_21px_0_rgba(0,0,0,0.15)]` (search) + +## Component Styling Rules (Theme-Safe) + +All components MUST use semantic CSS variable-backed utilities — never hardcoded colors: + +- `bg-background` / `bg-card` / `bg-muted` / `bg-accent` — not `bg-white`, `bg-neutral-50`, `bg-[#1a1a1a]` +- `text-foreground` / `text-muted-foreground` / `text-card-foreground` — not `text-black`, `text-gray-500` +- `border-border` — not `border-neutral-200`, `border-[#2e2e2e]` +- `bg-sidebar` for sidebar/navbar backgrounds +- For shadows that differ between themes: use a CSS variable `--shadow-subtle` / `--shadow-elevated` or conditional `dark:shadow-*` utilities +- Exceptions: Dench-specific decorative elements (landing page traffic-light dots, brand colors) can use fixed values diff --git a/.cursor/plans/mention_search_and_entry_modals_8a47eefc.plan.md b/.cursor/plans/mention_search_and_entry_modals_8a47eefc.plan.md new file mode 100644 index 00000000000..0baa352a9d2 --- /dev/null +++ b/.cursor/plans/mention_search_and_entry_modals_8a47eefc.plan.md @@ -0,0 +1,245 @@ +--- +name: Mention Search and Entry Modals +overview: Overhaul the tiptap @ mention system to search files AND object entries with fast fuzzy matching, fix the broken link system with canonical internal URIs, and add query-param-based entry detail modals accessible from table rows and @ mention links. +todos: + - id: search-index-api + content: "Phase 1: Build GET /api/workspace/search-index endpoint -- returns flat JSON of all files + all entries from every DuckDB object with display fields" + status: completed + - id: fuse-search-hook + content: "Phase 2: Add fuse.js dep, create useSearchIndex() hook in lib/search-index.ts, build Fuse instance on fetch" + status: completed + - id: mention-upgrade + content: "Phase 2b: Rewrite createFileMention -> createWorkspaceMention in slash-command.tsx, wire up Fuse search, update CommandList for entry results" + status: completed + - id: link-utilities + content: "Phase 3: Create lib/workspace-links.ts with parseWorkspaceLink/buildEntryLink, define @entry/{object}/{id} format" + status: completed + - id: link-insert-fix + content: "Phase 3b: Update mention insert commands to use canonical link format; update markdown-editor.tsx link click handler to parse workspace links" + status: completed + - id: entry-api + content: "Phase 4: Build GET /api/workspace/objects/[name]/entries/[id] endpoint for single entry data" + status: completed + - id: entry-modal + content: "Phase 4b: Build EntryDetailModal component with full field display, relation links, close/escape handling" + status: completed + - id: query-param-routing + content: "Phase 4c: Wire ?entry= query param in workspace/page.tsx -- render modal, sync URL on open/close, handle initial load" + status: completed + - id: table-row-click + content: "Phase 4d: Make object-table.tsx rows clickable with onEntryClick prop, wire to entry param in parent" + status: completed + - id: fix-nav-bugs + content: "Phase 5: Fix path resolution in onNavigate (knowledge/ prefix handling), sync URL bar with activePath via router.replace()" + status: completed +isProject: false +--- + +# @ Mention Search, Link System, and Entry Detail Modals + +## Current Problems + +1. **@ mention only searches files** -- entries (rows) in objects (tables) are invisible to the search. Uses naive `String.includes()` with no fuzzy matching. +2. **File links are broken** -- `buildFileItems` in [slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx) sets `href: node.path` (e.g. `knowledge/leads`), but `onNavigate` in [workspace/page.tsx](apps/web/app/workspace/page.tsx) calls `findNode(tree, path)` which may not resolve correctly depending on whether the path includes `knowledge/` prefix. No canonical link format exists. +3. **No entry detail view** -- table rows in [object-table.tsx](apps/web/app/components/workspace/object-table.tsx) are not clickable. There is no modal, route, or UI to view a single entry. +4. **No URL-based navigation** -- navigating to content is purely callback-based (state updates). The URL only updates on initial load via `?path=`. Sharing a link to a specific entry is impossible. + +--- + +## Architecture + +```mermaid +flowchart TB + subgraph searchIndex [Search Index - API] + searchEndpoint["GET /api/workspace/search-index"] + duckdb["DuckDB: objects + entries + fields"] + tree["Filesystem: tree nodes"] + searchEndpoint --> duckdb + searchEndpoint --> tree + end + + subgraph clientSearch [Client-Side Fuzzy Search] + fuseIndex["Fuse.js index - built once on load"] + mentionPlugin["@ Mention Plugin"] + fuseIndex --> mentionPlugin + end + + subgraph routing [Query Param Routing] + pathParam["?path=knowledge/leads"] + entryParam["&entry=abc123"] + pathParam --> workspacePage["Workspace Page"] + entryParam --> entryModal["Entry Detail Modal"] + end + + searchEndpoint -->|"JSON: files + entries"| fuseIndex + mentionPlugin -->|"insert link"| internalLink["dench://entry/leads/abc123"] + internalLink -->|"onNavigate resolves"| routing +``` + +--- + +## Phase 1: Search Index API Endpoint + +**New file:** `apps/web/app/api/workspace/search-index/route.ts` + +Returns a flat JSON array of all searchable items -- both files and entry rows from every object. The client fetches this once on workspace load and rebuilds on tree changes (SSE watcher already triggers refreshes). + +```typescript +type SearchIndexItem = { + // Shared + id: string; // unique key (path for files, entryId for entries) + label: string; // display text (filename or display-field value) + sublabel?: string; // secondary text (path for files, object name for entries) + kind: "file" | "object" | "entry"; + icon?: string; + + // For entries + objectName?: string; + entryId?: string; + fields?: Record; // first 3-4 field key-value pairs for preview + + // For files/objects + path?: string; + nodeType?: "document" | "folder" | "file" | "report" | "database"; +}; +``` + +**Server implementation:** + +- Reuse existing `buildTree()` from [tree/route.ts](apps/web/app/api/workspace/tree/route.ts) for files +- For entries: query every object from DuckDB, resolve display fields, and return flattened entries with their first few field values as searchable text +- SQL: `SELECT * FROM v_{objectName} LIMIT 500` per object (capped for perf) +- Cache with short TTL or rely on client refetch on SSE tree-change events + +--- + +## Phase 2: Client-Side Fuzzy Search with Fuse.js + +**New file:** `apps/web/lib/search-index.ts` + +A React hook `useSearchIndex()` that: + +1. Fetches `/api/workspace/search-index` once on mount +2. Builds a `Fuse` instance over the items, keyed on `label`, `sublabel`, and `fields` values +3. Exposes a `search(query: string): SearchIndexItem[]` function +4. Rebuilds when tree changes (listen to same SSE watcher signal) + +**Fuse.js config:** + +- Keys: `["label", "sublabel", "objectName", "fields.*"]` with weighted scoring +- Threshold: ~0.4 (tolerant fuzzy) +- Max results: 20 + +**Update [slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx):** + +Replace `createFileMention(tree)` with `createWorkspaceMention(searchFn)`: + +- `items` callback uses the Fuse search function instead of `flattenTree` + `.includes()` +- Results are grouped: show files first, then entries grouped by object +- The `CommandList` component gets a minor update to show entry items differently (object badge, field preview) + +**Dep:** Add `fuse.js` to `apps/web/package.json` -- lightweight (~7KB gzipped), runs entirely client-side, no wasm. + +--- + +## Phase 3: Canonical Internal Link Format + +Define a URI scheme for internal workspace links, parsed in a single utility: + +**New file:** `apps/web/lib/workspace-links.ts` + +```typescript +// Canonical formats: +// Files: "knowledge/path/to/doc.md" (relative path, unchanged) +// Objects: "knowledge/leads" (relative path, unchanged) +// Entries: "@entry/leads/abc123" (new: @entry/{objectName}/{entryId}) + +type WorkspaceLink = + | { kind: "file"; path: string } + | { kind: "object"; objectName: string; path: string } + | { kind: "entry"; objectName: string; entryId: string }; + +function parseWorkspaceLink(href: string): WorkspaceLink; +function buildEntryLink(objectName: string, entryId: string): string; +function buildFileLink(path: string): string; +function workspaceLinkToUrl(link: WorkspaceLink): string; +// Returns: "/workspace?path=knowledge/leads" or "/workspace?path=knowledge/leads&entry=abc123" +``` + +**Update mention insert command** in `slash-command.tsx`: + +- File items: keep `href: node.path` (unchanged) +- Entry items: use `href: "@entry/{objectName}/{entryId}"`, display text = display field value + +**Update link click handler** in [markdown-editor.tsx](apps/web/app/components/workspace/markdown-editor.tsx) (lines 215-238): + +- Parse the href with `parseWorkspaceLink()` +- For file/object links: `onNavigate(link.path)` (as today, but path resolution fixed) +- For entry links: `onNavigate("@entry/leads/abc123")` -- parent resolves to modal + +**Update workspace page** `onNavigate` handler: + +- Parse the link, and if it's an `@entry/...` link, set query params to open the entry modal +- If it's a file/object link, do `findNode(tree, path)` as before but with proper path normalization + +--- + +## Phase 4: Entry Detail Modal with Query Param Routing + +**URL pattern:** `/workspace?path=knowledge/leads&entry=abc123` + +When `entry` query param is present, render an `EntryDetailModal` as an overlay on top of the current workspace content. + +**New file:** `apps/web/app/components/workspace/entry-detail-modal.tsx` + +- Full-screen overlay/side panel with entry data +- Fetches entry data from a new API endpoint or uses already-loaded object data +- Renders all fields with proper type-specific display (reuse `CellValue` from [object-table.tsx](apps/web/app/components/workspace/object-table.tsx)) +- Relation fields are clickable -- clicking opens the related entry's modal (updates `entry` param) +- Close button / Escape / clicking backdrop clears the `entry` param +- URL is shareable and bookmarkable + +**New API:** `GET /api/workspace/objects/[name]/entries/[id]` + +- Returns a single entry with all field values, resolved relation labels, and reverse relations +- Lightweight endpoint for the modal to fetch data without loading the full object + +**Update [workspace/page.tsx](apps/web/app/workspace/page.tsx):** + +- Read `entry` from `searchParams` +- When present, render `` on top of current content +- Update URL using `router.replace()` (shallow) when opening/closing modal -- no full page navigation +- On initial load, if both `path` and `entry` are set, load the object AND open the modal + +**Update [object-table.tsx](apps/web/app/components/workspace/object-table.tsx):** + +- Make table rows clickable +- `onClick` handler calls a new `onEntryClick(entryId)` prop +- Parent sets the `entry` query param, opening the modal + +--- + +## Phase 5: Fix Existing Link Navigation Bugs + +**Bug 1:** Path resolution mismatch -- `node.path` from tree includes `knowledge/` prefix but link resolution sometimes strips it. Fix: normalize paths in `findNode()` and `onNavigate()` using the new `parseWorkspaceLink()`. + +**Bug 2:** The `onNavigate` callback in `DocumentView` does `findNode(tree, path)` but the href from a markdown link may not exactly match a tree path (e.g., `leads` vs `knowledge/leads`). Fix: add fallback resolution that tries with/without `knowledge/` prefix, and tries matching the last segment against object names. + +**Bug 3:** No URL update when navigating -- selecting a sidebar item or clicking a link updates state but not the URL bar. Fix: use `router.replace()` to keep URL in sync with `activePath` so links are shareable. + +--- + +## Key Files to Modify + +- [apps/web/app/components/workspace/slash-command.tsx](apps/web/app/components/workspace/slash-command.tsx) -- Replace file-only search with unified fuzzy search over files + entries +- [apps/web/app/components/workspace/markdown-editor.tsx](apps/web/app/components/workspace/markdown-editor.tsx) -- Fix link click handler to support entry links +- [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx) -- Add entry modal rendering, fix URL sync, fix `onNavigate` resolution +- [apps/web/app/components/workspace/object-table.tsx](apps/web/app/components/workspace/object-table.tsx) -- Make rows clickable with `onEntryClick` + +## New Files + +- `apps/web/app/api/workspace/search-index/route.ts` -- Search index endpoint +- `apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts` -- Single entry endpoint +- `apps/web/lib/workspace-links.ts` -- Link parsing/building utilities +- `apps/web/lib/search-index.ts` -- Client-side Fuse.js hook +- `apps/web/app/components/workspace/entry-detail-modal.tsx` -- Entry modal component diff --git a/.cursor/plans/reports_analytics_layer_d6cf8500.plan.md b/.cursor/plans/reports_analytics_layer_d6cf8500.plan.md new file mode 100644 index 00000000000..c00ac4740c3 --- /dev/null +++ b/.cursor/plans/reports_analytics_layer_d6cf8500.plan.md @@ -0,0 +1,258 @@ +--- +name: Reports Analytics Layer +overview: "Add a generative-UI reports feature to the Dench web app: the agent creates JSON report definitions with SQL queries and chart configs, rendered live via Recharts in both the workspace view and inline in chat as artifacts." +todos: + - id: recharts-dep + content: Add recharts dependency to apps/web/package.json + status: completed + - id: chart-panel + content: Create ChartPanel component supporting bar/line/area/pie/donut/radar/scatter/funnel via Recharts with CSS variable theming + status: completed + - id: filter-bar + content: Create FilterBar component with dateRange, select, multiSelect filter types; fetches options via SQL + status: completed + - id: report-viewer + content: "Create ReportViewer component: loads report config, executes panel SQL with filter injection, renders ChartPanel grid" + status: completed + - id: report-execute-api + content: Create POST /api/workspace/reports/execute route for SQL execution with filter clause injection + status: completed + - id: workspace-report-type + content: Add report content type to workspace/page.tsx ContentState + ContentRenderer; detect .report.json in tree API + status: completed + - id: knowledge-tree-report + content: Add report node type + icon to knowledge-tree.tsx and tree/route.ts + status: completed + - id: chat-artifact + content: Modify chat-message.tsx to detect report-json fenced blocks and render inline ReportCard + status: completed + - id: report-card + content: Create ReportCard component for compact inline chart rendering in chat with Pin/Open actions + status: completed + - id: sidebar-reports + content: Add Reports section to sidebar.tsx listing .report.json files from workspace tree + status: completed + - id: skill-update + content: Add Section 13 (Report Generation) to skills/dench/SKILL.md with schema, examples, and agent instructions + status: completed +isProject: false +--- + +# Reports / Analytics Layer for Dench Workspace + +## Architecture + +Reports are **JSON config files** (`.report.json`) that declare SQL queries to run against `workspace.duckdb` and how to visualize the results. The web app executes SQL at render time (live data), renders charts via Recharts, and supports interactive filters. + +Reports surface in **three places**: + +1. **Workspace view** -- full-page report dashboard when clicking a report in the tree +2. **Chat** -- inline chart artifact when the agent generates a report in conversation +3. **Sidebar** -- reports listed under a new "Reports" section + +```mermaid +flowchart LR + subgraph agent [Agent] + skill[Dench Skill] + skill -->|"write .report.json"| fs[Filesystem] + skill -->|"emit report block in text"| chat[Chat Stream] + end + + subgraph web [Web App] + fs --> treeAPI["/api/workspace/tree"] + fs --> reportAPI["/api/workspace/reports"] + reportAPI --> execAPI["/api/workspace/query"] + execAPI --> duckdb["workspace.duckdb"] + + treeAPI --> sidebar[Sidebar + KnowledgeTree] + reportAPI --> workspaceView[ReportViewer in Workspace] + chat --> chatUI["ChatMessage with inline ReportCard"] + chatUI --> execAPI + workspaceView --> recharts[Recharts Charts] + chatUI --> recharts + end +``` + +--- + +## Report Definition Format + +Stored as `.report.json` files in `dench/reports/` (or nested under any knowledge path). Agent generates these via the `write` tool. + +```json +{ + "version": 1, + "title": "Deals Pipeline", + "description": "Revenue breakdown by stage and rep", + "panels": [ + { + "id": "deals-by-stage", + "title": "Deal Count by Stage", + "type": "bar", + "sql": "SELECT \"Stage\", COUNT(*) as count FROM v_deal GROUP BY \"Stage\" ORDER BY count DESC", + "mapping": { "xAxis": "Stage", "yAxis": ["count"] }, + "size": "half" + }, + { + "id": "revenue-trend", + "title": "Revenue Over Time", + "type": "area", + "sql": "SELECT DATE_TRUNC('month', created_at) as month, SUM(\"Amount\"::NUMERIC) as revenue FROM v_deal GROUP BY month ORDER BY month", + "mapping": { "xAxis": "month", "yAxis": ["revenue"] }, + "size": "half" + }, + { + "id": "stage-distribution", + "title": "Stage Distribution", + "type": "pie", + "sql": "SELECT \"Stage\", COUNT(*) as count FROM v_deal GROUP BY \"Stage\"", + "mapping": { "nameKey": "Stage", "valueKey": "count" }, + "size": "third" + } + ], + "filters": [ + { + "id": "date-range", + "type": "dateRange", + "label": "Date Range", + "column": "created_at" + }, + { + "id": "assigned-to", + "type": "select", + "label": "Assigned To", + "sql": "SELECT DISTINCT \"Assigned To\" as value FROM v_deal WHERE \"Assigned To\" IS NOT NULL", + "column": "Assigned To" + } + ] +} +``` + +**Chart types supported:** `bar`, `line`, `area`, `pie`, `donut`, `radar`, `radialBar`, `scatter`, `funnel`. +**Panel sizes:** `full`, `half`, `third` (CSS grid layout). +**Filter types:** `dateRange`, `select`, `multiSelect`, `number`. + +When filters are active, they inject `WHERE` clauses into each panel's SQL before execution. + +--- + +## Phase 1: Recharts + Report Viewer Component + +**Add Recharts dependency:** + +- [apps/web/package.json](apps/web/package.json) -- add `recharts` to dependencies + +**New components** (all in `apps/web/app/components/charts/`): + +- `**chart-panel.tsx**` -- Wrapper that takes a panel config + data rows, renders the correct Recharts chart (Bar, Line, Area, Pie, etc.). Uses the app's CSS variable palette for theming (`--color-accent`, `--color-text`, `--color-border`). One component, switch on `panel.type`. +- `**filter-bar.tsx**` -- Horizontal filter strip. Reads filter configs from the report, fetches options for `select` type filters via SQL, renders date pickers / dropdowns. Emits active filter state upward. +- `**report-viewer.tsx**` -- Full report dashboard. Fetches report config (from file or prop), iterates panels, executes each panel's SQL (with filter injection), renders `ChartPanel` components in a CSS grid (`size` controls column span). Includes a header with title/description and the filter bar. + +--- + +## Phase 2: Workspace Integration + +**Extend content types** in [apps/web/app/workspace/page.tsx](apps/web/app/workspace/page.tsx): + +- Add `report` to the `ContentState` union: `{ kind: "report"; reportPath: string; filename: string }` +- Add `report` case to `ContentRenderer` that renders `ReportViewer` +- In `loadContent`, detect `.report.json` files and load as `report` kind + +**Extend knowledge tree** in [apps/web/app/components/workspace/knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx): + +- Add `"report"` to the `TreeNode.type` union +- Add report icon (bar chart icon) to `NodeTypeIcon` + +**Extend tree API** in [apps/web/app/api/workspace/tree/route.ts](apps/web/app/api/workspace/tree/route.ts): + +- Detect `.report.json` files and assign them `type: "report"` in the tree + +**New API route** -- `apps/web/app/api/workspace/reports/execute/route.ts`: + +- POST `{ sql: string, filters?: FilterState }` -- injects filter WHERE clauses into SQL, executes via `duckdbQuery`, returns rows +- This is separate from the generic query endpoint because it handles filter injection safely + +--- + +## Phase 3: Chat Artifact (Inline Reports) + +**Report block convention in agent text:** +The agent emits a fenced code block with language `report-json` containing the report JSON. Example in the streamed text: + +```` +Here's your pipeline analysis: + +```report-json +{"version":1,"title":"Deals by Stage","panels":[...]} +```` + +The data shows most deals are in the Discovery stage. + +````` + +**Modify [apps/web/app/components/chat-message.tsx](apps/web/app/components/chat-message.tsx):** + +- In `groupParts`, detect text segments containing ````report-json ... ```` blocks +- Split text around report blocks into `text` and `report-artifact` segments +- New segment type: `{ type: "report-artifact"; config: ReportConfig }` + +**New component** -- `apps/web/app/components/charts/report-card.tsx`: + +- Compact inline report card rendered inside chat bubbles +- Shows report title + a subset of panels (auto-sized to fit chat width) +- "Open in Workspace" button that saves to filesystem + navigates +- "Pin" action to persist an ephemeral chat report as a `.report.json` file + +--- + +## Phase 4: Sidebar Reports Section + +**Modify [apps/web/app/components/sidebar.tsx](apps/web/app/components/sidebar.tsx):** + +- Add "Reports" as a new `SidebarSection` (between Workspace and Memories) +- Fetch report list from `/api/workspace/tree` (filter for `type: "report"` nodes) +- Each report links to `/workspace?path=reports/{name}.report.json` +- Show chart icon + report title + +--- + +## Phase 5: Dench Skill Updates + +**Modify [skills/dench/SKILL.md](skills/dench/SKILL.md):** + +- Add Section 13: Report Generation +- Document the `.report.json` format with full schema reference +- Provide example reports for common CRM analytics: + - Pipeline funnel (deals by stage) + - Revenue trend over time + - Lead source breakdown + - Activity/task completion rates + - Contact growth over time +- Instruct the agent to: + - Create reports in `reports/` directory + - Use the existing `v_{object}` PIVOT views in SQL queries + - Include relevant filters (date range, assignee, status) + - Emit `report-json` blocks in chat for inline rendering + - Choose appropriate chart types for the data shape +- Add a "Post-Report Checklist" matching the existing post-mutation pattern + +--- + +## File Summary + + +| Action | File | +| ------ | -------------------------------------------------------------------- | +| Modify | `apps/web/package.json` (add recharts) | +| Create | `apps/web/app/components/charts/chart-panel.tsx` | +| Create | `apps/web/app/components/charts/report-viewer.tsx` | +| Create | `apps/web/app/components/charts/report-card.tsx` | +| Create | `apps/web/app/components/charts/filter-bar.tsx` | +| Create | `apps/web/app/api/workspace/reports/execute/route.ts` | +| Modify | `apps/web/app/workspace/page.tsx` (add report content type) | +| Modify | `apps/web/app/components/chat-message.tsx` (detect report blocks) | +| Modify | `apps/web/app/components/sidebar.tsx` (Reports section) | +| Modify | `apps/web/app/components/workspace/knowledge-tree.tsx` (report node) | +| Modify | `apps/web/app/api/workspace/tree/route.ts` (detect .report.json) | +| Modify | `skills/dench/SKILL.md` (report generation instructions) | +````` diff --git a/.cursor/plans/sidebar_file_manager_02ed8b45.plan.md b/.cursor/plans/sidebar_file_manager_02ed8b45.plan.md new file mode 100644 index 00000000000..bc5d5719c90 --- /dev/null +++ b/.cursor/plans/sidebar_file_manager_02ed8b45.plan.md @@ -0,0 +1,282 @@ +--- +name: Sidebar File Manager +overview: Transform the workspace file tree sidebar (both on `/` home and `/workspace` pages) into a full-fledged file system manager with context menus, drag-and-drop file moves, create/delete/rename operations, system file locking, and live reactivity -- modeled after macOS Finder. +todos: + - id: api-system-files + content: Add isSystemFile() to lib/workspace.ts, extend safeResolvePath to support non-existent target paths + status: pending + - id: api-delete + content: Add DELETE handler to /api/workspace/file/route.ts with system file protection + status: pending + - id: api-rename + content: Create /api/workspace/rename/route.ts (POST) with validation and system file protection + status: pending + - id: api-move + content: Create /api/workspace/move/route.ts (POST) for drag-and-drop moves + status: pending + - id: api-mkdir + content: Create /api/workspace/mkdir/route.ts (POST) for creating new directories + status: pending + - id: api-copy + content: Create /api/workspace/copy/route.ts (POST) for duplicating files/folders + status: pending + - id: api-watch + content: Create /api/workspace/watch/route.ts SSE endpoint using chokidar for live file events + status: pending + - id: install-dndkit + content: Install @dnd-kit/core, @dnd-kit/sortable, @dnd-kit/utilities in apps/web + status: pending + - id: context-menu + content: "Build context-menu.tsx: portal-based right-click menu with Finder-like options, system file lock states" + status: pending + - id: inline-rename + content: "Build inline-rename.tsx: double-click/F2 to rename in-place with validation" + status: pending + - id: file-manager-tree + content: "Build file-manager-tree.tsx: unified DnD tree wrapping context menu, inline rename, drag-drop, keyboard shortcuts" + status: pending + - id: sse-hook + content: "Build useWorkspaceWatcher hook: SSE connection with debounced tree refetch and auto-reconnect" + status: pending + - id: integrate-workspace-sidebar + content: Replace KnowledgeTree with FileManagerTree in workspace-sidebar.tsx + status: pending + - id: integrate-home-sidebar + content: Replace WorkspaceTreeNode in sidebar.tsx with FileManagerTree (compact mode) + status: pending + - id: integrate-workspace-page + content: Wire workspace/page.tsx to useWorkspaceWatcher for live-reactive tree state + status: pending + - id: keyboard-shortcuts + content: Add keyboard navigation and file operation shortcuts to FileManagerTree + status: pending +isProject: false +--- + +# Full File System Manager Sidebar + +## Current State + +Two sidebar trees render workspace files read-only: + +- **Home sidebar** (`[apps/web/app/components/sidebar.tsx](apps/web/app/components/sidebar.tsx)`): `WorkspaceSection` / `WorkspaceTreeNode` -- compact tree inside collapsible section +- **Workspace sidebar** (`[apps/web/app/components/workspace/workspace-sidebar.tsx](apps/web/app/components/workspace/workspace-sidebar.tsx)`): wraps `KnowledgeTree` from `[knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx)` + +Both fetch from `GET /api/workspace/tree` (`[apps/web/app/api/workspace/tree/route.ts](apps/web/app/api/workspace/tree/route.ts)`). File read/write exists at `/api/workspace/file` (`[apps/web/app/api/workspace/file/route.ts](apps/web/app/api/workspace/file/route.ts)`). No delete, rename, move, mkdir, or copy endpoints exist. No context menus, drag-and-drop, or live refresh. + +## Architecture + +```mermaid +flowchart TD + subgraph frontend [Frontend Components] + ContextMenu[ContextMenu Component] + FileTree[FileTree - Unified Tree with DnD] + InlineRename[Inline Rename Input] + NewFileDialog[New File/Folder Prompt] + end + + subgraph api [API Routes - apps/web/app/api/workspace/] + TreeRoute[GET /tree] + FileRoute[GET+POST /file] + DeleteRoute[DELETE /file] + RenameRoute[POST /rename] + MoveRoute[POST /move] + MkdirRoute[POST /mkdir] + CopyRoute[POST /copy] + WatchRoute[GET /watch - SSE] + end + + subgraph fsLib [lib/workspace.ts] + SafeResolve[safeResolvePath] + IsSystemFile[isSystemFile] + end + + FileTree -->|right-click| ContextMenu + FileTree -->|drag-drop| MoveRoute + ContextMenu -->|"New File/Folder"| MkdirRoute + ContextMenu -->|Delete| DeleteRoute + ContextMenu -->|Rename| InlineRename + ContextMenu -->|Duplicate| CopyRoute + ContextMenu -->|"Move to..."| MoveRoute + InlineRename --> RenameRoute + NewFileDialog --> MkdirRoute + NewFileDialog --> FileRoute + + DeleteRoute --> SafeResolve + RenameRoute --> SafeResolve + MoveRoute --> SafeResolve + CopyRoute --> SafeResolve + DeleteRoute --> IsSystemFile + RenameRoute --> IsSystemFile + MoveRoute --> IsSystemFile + + WatchRoute -->|"SSE events"| FileTree +``` + +## 1. New Backend API Endpoints + +Add to `[apps/web/app/api/workspace/](apps/web/app/api/workspace/)`: + +`**DELETE /api/workspace/file**` - Delete a file or folder + +- Body: `{ path: string }` +- Reject if `isSystemFile(path)` returns true +- Use `fs.rmSync(absPath, { recursive: true })` for folders + +`**POST /api/workspace/rename**` - Rename a file or folder (new route file) + +- Body: `{ path: string, newName: string }` +- Reject system files; validate newName (no slashes, no `.` prefix for non-system) +- Use `fs.renameSync(oldAbs, newAbs)` where newAbs replaces only the basename + +`**POST /api/workspace/move**` - Move a file/folder to a new parent (new route file) + +- Body: `{ sourcePath: string, destinationDir: string }` +- Reject system files; validate destination exists and is a directory +- Use `fs.renameSync(srcAbs, join(destAbs, basename(srcAbs)))` + +`**POST /api/workspace/mkdir**` - Create a directory (new route file) + +- Body: `{ path: string }` +- Use `fs.mkdirSync(absPath, { recursive: true })` + +`**POST /api/workspace/copy**` - Duplicate a file or folder (new route file) + +- Body: `{ path: string, destinationPath?: string }` +- If no destination, auto-name as ` copy.` +- Use `fs.cpSync` for recursive copy + +`**GET /api/workspace/watch**` - SSE endpoint for live changes (new route file) + +- Uses `chokidar` to watch the dench workspace root +- Streams SSE events: `{ type: "add"|"change"|"unlink"|"addDir"|"unlinkDir", path: string }` +- Client reconnects on close; debounce events (200ms) + +### System File Protection + +Add to `[apps/web/lib/workspace.ts](apps/web/lib/workspace.ts)`: + +```typescript +const SYSTEM_FILE_PATTERNS = [ + /^\.object\.yaml$/, + /^workspace\.duckdb/, + /^workspace_context\.yaml$/, + /\.wal$/, + /\.tmp$/, +]; + +export function isSystemFile(relativePath: string): boolean { + const basename = relativePath.split("/").pop() ?? ""; + return SYSTEM_FILE_PATTERNS.some((p) => p.test(basename)); +} +``` + +Extend `safeResolvePath` to optionally accept a `mustExist: false` flag (currently returns null for non-existent paths, but mkdir/create need paths that don't exist yet). + +## 2. Context Menu Component + +Create `[apps/web/app/components/workspace/context-menu.tsx](apps/web/app/components/workspace/context-menu.tsx)`: + +- Pure CSS + React portal-based context menu (no library, matches the zero-dep approach) +- Positioned at cursor coordinates, clamped to viewport +- Closes on click-outside, Escape, or scroll +- Menu items with icons, keyboard shortcut hints, separators, and disabled states + +**Menu structure** (mirrors Finder): + +| For files | For folders | For empty area | +| ---------- | -------------------- | -------------- | +| Open | Open | New File | +| Rename | New File inside... | New Folder | +| Duplicate | New Folder inside... | Paste | +| Copy | Rename | --- | +| Move to... | Duplicate | --- | +| --- | Copy | --- | +| Get Info | Move to... | --- | +| --- | --- | --- | +| Delete | Delete | --- | + +System files (`.object.yaml`, `workspace.duckdb`) show the same menu but all mutating actions are **disabled** with a lock icon and "System file" tooltip. + +## 3. Drag-and-Drop for File Moves + +Install `@dnd-kit/core` + `@dnd-kit/sortable` + `@dnd-kit/utilities` (lightweight, ~15KB gzipped, React 19 compatible). + +Create `[apps/web/app/components/workspace/dnd-file-tree.tsx](apps/web/app/components/workspace/dnd-file-tree.tsx)`: + +- Each tree node is both a **draggable** and a **droppable** (folders accept drops) +- Drag overlay shows a ghost of the file/folder name with icon +- Drop targets highlight with accent border when hovered +- On drop: call `POST /api/workspace/move` with `{ sourcePath, destinationDir }` +- System files: draggable is **disabled** (visual lock indicator) +- Folder auto-expand on hover during drag (300ms delay) +- Drop validation: prevent dropping a folder into itself or its children + +## 4. Inline Rename + +- Double-click or press Enter/F2 on a selected node to enter rename mode +- Replace the label `` with a controlled `` pre-filled with the current name +- Commit on Enter or blur; cancel on Escape +- Call `POST /api/workspace/rename` on commit +- Shake animation + red border on validation error (empty name, invalid chars, name collision) + +## 5. Unified FileManagerTree Component + +Refactor `[knowledge-tree.tsx](apps/web/app/components/workspace/knowledge-tree.tsx)` into a new `FileManagerTree` that wraps DnD + context menu + inline rename: + +``` +FileManagerTree (DndContext provider + SSE watcher) + └─ FileManagerNode (draggable + droppable + onContextMenu + double-click rename) + ├─ ChevronIcon + ├─ NodeIcon (+ lock badge for system files) + ├─ InlineRenameInput | Label + └─ Children (recursive) +``` + +Both sidebars (`sidebar.tsx` WorkspaceSection and `workspace-sidebar.tsx`) will use this unified component, passing a `compact` prop to control spacing/features for the home sidebar (e.g., hide "Get Info", simpler context menu). + +## 6. Live Reactivity (SSE) + +Create a `useWorkspaceWatcher` hook in `[apps/web/app/hooks/use-workspace-watcher.ts](apps/web/app/hooks/use-workspace-watcher.ts)`: + +- Opens an `EventSource` to `GET /api/workspace/watch` +- On any file event, debounces and refetches the tree from `/api/workspace/tree` +- Provides `tree`, `loading`, and `refresh()` to consumers +- Auto-reconnects with exponential backoff (1s, 2s, 4s, max 30s) +- Falls back to polling every 5s if SSE fails + +Both sidebars and the workspace page will use this hook for shared, reactive tree state. + +## 7. Keyboard Shortcuts + +In the `FileManagerTree`, attach keyboard handlers on the focused tree container: + +- **Delete / Backspace**: Delete selected item (with confirmation dialog) +- **Enter / F2**: Start inline rename +- **Cmd+C**: Copy path to clipboard +- **Cmd+D**: Duplicate +- **Cmd+N**: New file in current folder +- **Cmd+Shift+N**: New folder in current folder +- **Arrow keys**: Navigate tree up/down, expand/collapse with left/right +- **Space**: Quick Look / preview toggle (future) + +## Files Changed + +| File | Action | +| --------------------------------------------------------- | -------------------------------------------------------------- | +| `apps/web/package.json` | Add `@dnd-kit/core`, `@dnd-kit/sortable`, `@dnd-kit/utilities` | +| `apps/web/lib/workspace.ts` | Add `isSystemFile()`, extend `safeResolvePath` | +| `apps/web/app/api/workspace/file/route.ts` | Add `DELETE` handler | +| `apps/web/app/api/workspace/rename/route.ts` | New -- rename endpoint | +| `apps/web/app/api/workspace/move/route.ts` | New -- move endpoint | +| `apps/web/app/api/workspace/mkdir/route.ts` | New -- mkdir endpoint | +| `apps/web/app/api/workspace/copy/route.ts` | New -- copy endpoint | +| `apps/web/app/api/workspace/watch/route.ts` | New -- SSE file watcher | +| `apps/web/app/components/workspace/context-menu.tsx` | New -- right-click menu | +| `apps/web/app/components/workspace/file-manager-tree.tsx` | New -- unified DnD + context menu tree | +| `apps/web/app/components/workspace/inline-rename.tsx` | New -- inline rename input | +| `apps/web/app/hooks/use-workspace-watcher.ts` | New -- SSE watcher hook | +| `apps/web/app/components/workspace/knowledge-tree.tsx` | Refactor into file-manager-tree | +| `apps/web/app/components/workspace/workspace-sidebar.tsx` | Use new FileManagerTree | +| `apps/web/app/components/sidebar.tsx` | Use new FileManagerTree (compact mode) | +| `apps/web/app/workspace/page.tsx` | Use `useWorkspaceWatcher` hook | diff --git a/.dockerignore b/.dockerignore index af1dfc73a35..73d00fff147 100644 --- a/.dockerignore +++ b/.dockerignore @@ -46,3 +46,15 @@ Swabble/ Core/ Users/ vendor/ + +# Needed for building the Canvas A2UI bundle during Docker image builds. +# Keep the rest of apps/ and vendor/ excluded to avoid a large build context. +!apps/shared/ +!apps/shared/OpenClawKit/ +!apps/shared/OpenClawKit/Tools/ +!apps/shared/OpenClawKit/Tools/CanvasA2UI/ +!apps/shared/OpenClawKit/Tools/CanvasA2UI/** +!vendor/a2ui/ +!vendor/a2ui/renderers/ +!vendor/a2ui/renderers/lit/ +!vendor/a2ui/renderers/lit/** diff --git a/.env.example b/.env.example index 29652fe4654..8bc4defd429 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,70 @@ -# Copy to .env and fill with your Twilio credentials -TWILIO_ACCOUNT_SID=ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -TWILIO_AUTH_TOKEN=your_auth_token_here -# Must be a WhatsApp-enabled Twilio number, prefixed with whatsapp: -TWILIO_WHATSAPP_FROM=whatsapp:+17343367101 +# OpenClaw .env example +# +# Quick start: +# 1) Copy this file to `.env` (for local runs from this repo), OR to `~/.openclaw/.env` (for launchd/systemd daemons). +# 2) Fill only the values you use. +# 3) Keep real secrets out of git. +# +# Env-source precedence for environment variables (highest -> lowest): +# process env, ./.env, ~/.openclaw/.env, then openclaw.json `env` block. +# Existing non-empty process env vars are not overridden by dotenv/config env loading. +# Note: direct config keys (for example `gateway.auth.token` or channel tokens in openclaw.json) +# are resolved separately from env loading and often take precedence over env fallbacks. + +# ----------------------------------------------------------------------------- +# Gateway auth + paths +# ----------------------------------------------------------------------------- +# Recommended if the gateway binds beyond loopback. +OPENCLAW_GATEWAY_TOKEN=change-me-to-a-long-random-token +# Example generator: openssl rand -hex 32 + +# Optional alternative auth mode (use token OR password). +# OPENCLAW_GATEWAY_PASSWORD=change-me-to-a-strong-password + +# Optional path overrides (defaults shown for reference). +# OPENCLAW_STATE_DIR=~/.openclaw +# OPENCLAW_CONFIG_PATH=~/.openclaw/openclaw.json +# OPENCLAW_HOME=~ + +# Optional: import missing keys from your login shell profile. +# OPENCLAW_LOAD_SHELL_ENV=1 +# OPENCLAW_SHELL_ENV_TIMEOUT_MS=15000 + +# ----------------------------------------------------------------------------- +# Model provider API keys (set at least one) +# ----------------------------------------------------------------------------- +# OPENAI_API_KEY=sk-... +# ANTHROPIC_API_KEY=sk-ant-... +# GEMINI_API_KEY=... +# OPENROUTER_API_KEY=sk-or-... + +# Optional additional providers +# ZAI_API_KEY=... +# AI_GATEWAY_API_KEY=... +# MINIMAX_API_KEY=... +# SYNTHETIC_API_KEY=... + +# ----------------------------------------------------------------------------- +# Channels (only set what you enable) +# ----------------------------------------------------------------------------- +# TELEGRAM_BOT_TOKEN=123456:ABCDEF... +# DISCORD_BOT_TOKEN=... +# SLACK_BOT_TOKEN=xoxb-... +# SLACK_APP_TOKEN=xapp-... + +# Optional channel env fallbacks +# MATTERMOST_BOT_TOKEN=... +# MATTERMOST_URL=https://chat.example.com +# ZALO_BOT_TOKEN=... +# OPENCLAW_TWITCH_ACCESS_TOKEN=oauth:... + +# ----------------------------------------------------------------------------- +# Tools + voice/media (optional) +# ----------------------------------------------------------------------------- +# BRAVE_API_KEY=... +# PERPLEXITY_API_KEY=pplx-... +# FIRECRAWL_API_KEY=... + +# ELEVENLABS_API_KEY=... +# XI_API_KEY=... # alias for ElevenLabs +# DEEPGRAM_API_KEY=... diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 7ba6bf4f77a..1b38a9ddf05 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,8 @@ -blank_issues_enabled: true +blank_issues_enabled: false contact_links: - name: Onboarding url: https://discord.gg/clawd - about: New to Clawdbot? Join Discord for setup guidance from Krill in \#help. + about: New to OpenClaw? Join Discord for setup guidance from Krill in \#help. - name: Support url: https://discord.gg/clawd about: Get help from Krill and the community on Discord in \#help. diff --git a/.github/actions/detect-docs-only/action.yml b/.github/actions/detect-docs-changes/action.yml similarity index 76% rename from .github/actions/detect-docs-only/action.yml rename to .github/actions/detect-docs-changes/action.yml index 5bdc5d7d89b..853442a7783 100644 --- a/.github/actions/detect-docs-only/action.yml +++ b/.github/actions/detect-docs-changes/action.yml @@ -8,6 +8,9 @@ outputs: docs_only: description: "'true' if all changes are docs/markdown, 'false' otherwise" value: ${{ steps.check.outputs.docs_only }} + docs_changed: + description: "'true' if any changed file is under docs/ or is markdown" + value: ${{ steps.check.outputs.docs_changed }} runs: using: composite @@ -28,9 +31,18 @@ runs: CHANGED=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo "UNKNOWN") if [ "$CHANGED" = "UNKNOWN" ] || [ -z "$CHANGED" ]; then echo "docs_only=false" >> "$GITHUB_OUTPUT" + echo "docs_changed=false" >> "$GITHUB_OUTPUT" exit 0 fi + # Check if any changed file is a doc + DOCS=$(echo "$CHANGED" | grep -E '^docs/|\.md$|\.mdx$' || true) + if [ -n "$DOCS" ]; then + echo "docs_changed=true" >> "$GITHUB_OUTPUT" + else + echo "docs_changed=false" >> "$GITHUB_OUTPUT" + fi + # Check if all changed files are docs or markdown NON_DOCS=$(echo "$CHANGED" | grep -vE '^docs/|\.md$|\.mdx$' || true) if [ -z "$NON_DOCS" ]; then diff --git a/.github/actions/setup-node-env/action.yml b/.github/actions/setup-node-env/action.yml new file mode 100644 index 00000000000..5fa4f6728bc --- /dev/null +++ b/.github/actions/setup-node-env/action.yml @@ -0,0 +1,83 @@ +name: Setup Node environment +description: > + Initialize submodules with retry, install Node 22, pnpm, optionally Bun, + and run pnpm install. Requires actions/checkout to run first. +inputs: + node-version: + description: Node.js version to install. + required: false + default: "22.x" + pnpm-version: + description: pnpm version for corepack. + required: false + default: "10.23.0" + install-bun: + description: Whether to install Bun alongside Node. + required: false + default: "true" + frozen-lockfile: + description: Whether to use --frozen-lockfile for install. + required: false + default: "true" +runs: + using: composite + steps: + - name: Checkout submodules (retry) + shell: bash + run: | + set -euo pipefail + git submodule sync --recursive + for attempt in 1 2 3 4 5; do + if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then + exit 0 + fi + echo "Submodule update failed (attempt $attempt/5). Retrying…" + sleep $((attempt * 10)) + done + exit 1 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + check-latest: true + + - name: Setup pnpm + cache store + uses: ./.github/actions/setup-pnpm-store-cache + with: + pnpm-version: ${{ inputs.pnpm-version }} + cache-key-suffix: "node22" + + - name: Setup Bun + if: inputs.install-bun == 'true' + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Runtime versions + shell: bash + run: | + node -v + npm -v + pnpm -v + if command -v bun &>/dev/null; then bun -v; fi + + - name: Capture node path + shell: bash + run: echo "NODE_BIN=$(dirname "$(node -p "process.execPath")")" >> "$GITHUB_ENV" + + - name: Install dependencies + shell: bash + env: + CI: "true" + run: | + export PATH="$NODE_BIN:$PATH" + which node + node -v + pnpm -v + LOCKFILE_FLAG="" + if [ "${{ inputs.frozen-lockfile }}" = "true" ]; then + LOCKFILE_FLAG="--frozen-lockfile" + fi + pnpm install $LOCKFILE_FLAG --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || \ + pnpm install $LOCKFILE_FLAG --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true diff --git a/.github/labeler.yml b/.github/labeler.yml index a1259f44aa4..78366fb2097 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -9,6 +9,11 @@ - "src/discord/**" - "extensions/discord/**" - "docs/channels/discord.md" +"channel: irc": + - changed-files: + - any-glob-to-any-file: + - "extensions/irc/**" + - "docs/channels/irc.md" "channel: feishu": - changed-files: - any-glob-to-any-file: @@ -79,6 +84,11 @@ - any-glob-to-any-file: - "extensions/tlon/**" - "docs/channels/tlon.md" +"channel: twitch": + - changed-files: + - any-glob-to-any-file: + - "extensions/twitch/**" + - "docs/channels/twitch.md" "channel: voice-call": - changed-files: - any-glob-to-any-file: @@ -226,3 +236,19 @@ - changed-files: - any-glob-to-any-file: - "extensions/qwen-portal-auth/**" +"extensions: device-pair": + - changed-files: + - any-glob-to-any-file: + - "extensions/device-pair/**" +"extensions: minimax-portal-auth": + - changed-files: + - any-glob-to-any-file: + - "extensions/minimax-portal-auth/**" +"extensions: phone-control": + - changed-files: + - any-glob-to-any-file: + - "extensions/phone-control/**" +"extensions: talk-voice": + - changed-files: + - any-glob-to-any-file: + - "extensions/talk-voice/**" diff --git a/.github/workflows/auto-response.yml b/.github/workflows/auto-response.yml index 6375111a62b..c979d120c48 100644 --- a/.github/workflows/auto-response.yml +++ b/.github/workflows/auto-response.yml @@ -39,6 +39,11 @@ jobs: message: "Please use [our support server](https://discord.gg/clawd) and ask in #help or #users-helping-users to resolve this, or follow the stuck FAQ at https://docs.openclaw.ai/help/faq#im-stuck-whats-the-fastest-way-to-get-unstuck.", }, + { + label: "r: testflight", + close: true, + message: "Not available, build from source.", + }, { label: "r: third-party-extension", close: true, @@ -60,10 +65,36 @@ jobs: const title = issue.title ?? ""; const body = issue.body ?? ""; const haystack = `${title}\n${body}`.toLowerCase(); - const hasLabel = (issue.labels ?? []).some((label) => + const hasMoltbookLabel = (issue.labels ?? []).some((label) => typeof label === "string" ? label === "r: moltbook" : label?.name === "r: moltbook", ); - if (haystack.includes("moltbook") && !hasLabel) { + const hasTestflightLabel = (issue.labels ?? []).some((label) => + typeof label === "string" + ? label === "r: testflight" + : label?.name === "r: testflight", + ); + const hasSecurityLabel = (issue.labels ?? []).some((label) => + typeof label === "string" ? label === "security" : label?.name === "security", + ); + if (title.toLowerCase().includes("security") && !hasSecurityLabel) { + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: ["security"], + }); + return; + } + if (title.toLowerCase().includes("testflight") && !hasTestflightLabel) { + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: ["r: testflight"], + }); + return; + } + if (haystack.includes("moltbook") && !hasMoltbookLabel) { await github.rest.issues.addLabels({ owner: context.repo.owner, repo: context.repo.repo, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81264929097..b84ca6da4b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ jobs: runs-on: ubuntu-latest outputs: docs_only: ${{ steps.check.outputs.docs_only }} + docs_changed: ${{ steps.check.outputs.docs_changed }} steps: - name: Checkout uses: actions/checkout@v4 @@ -25,7 +26,7 @@ jobs: - name: Detect docs-only changes id: check - uses: ./.github/actions/detect-docs-only + uses: ./.github/actions/detect-docs-changes # Detect which heavy areas are touched so PRs can skip unrelated expensive jobs. # Push to main keeps broad coverage. @@ -83,6 +84,10 @@ jobs: esac case "$path" in + # Generated protocol models are already covered by protocol:check and + # should not force the full native macOS lane. + apps/macos/Sources/OpenClawProtocol/*|apps/shared/OpenClawKit/Sources/OpenClawProtocol/*) + ;; apps/macos/*|apps/ios/*|apps/shared/*|Swabble/*) run_macos=true ;; @@ -120,7 +125,7 @@ jobs: # Build dist once for Node-relevant changes and share it with downstream jobs. build-artifacts: - needs: [docs-scope, changed-scope] + needs: [docs-scope, changed-scope, check] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -129,49 +134,10 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Runtime versions - run: | - node -v - npm -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies (frozen) - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + install-bun: "false" - name: Build dist run: pnpm build @@ -183,9 +149,10 @@ jobs: path: dist/ retention-days: 1 - install-check: - needs: [docs-scope, changed-scope] - if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') + # Validate npm pack contents after build (only on push to main, not PRs). + release-check: + needs: [docs-scope, build-artifacts] + if: github.event_name == 'push' && needs.docs-scope.outputs.docs_only != 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout @@ -193,61 +160,28 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env with: - node-version: 22.x - check-latest: true + install-bun: "false" - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache + - name: Download dist artifact + uses: actions/download-artifact@v4 with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" + name: dist-build + path: dist/ - - name: Runtime versions - run: | - node -v - npm -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies (frozen) - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + - name: Check release contents + run: pnpm release:check checks: - needs: [docs-scope, changed-scope] + needs: [docs-scope, changed-scope, check] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: fail-fast: false matrix: include: - - runtime: node - task: tsgo - command: pnpm tsgo - runtime: node task: test command: pnpm canvas:a2ui:bundle && pnpm test @@ -263,128 +197,46 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Setup Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Runtime versions - run: | - node -v - npm -v - bun -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + - name: Setup Node environment + uses: ./.github/actions/setup-node-env - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) run: ${{ matrix.command }} - # Lint and format always run, even on docs-only changes. - checks-lint: + # Types, lint, and format check. + check: + name: "check" + needs: [docs-scope] + if: needs.docs-scope.outputs.docs_only != 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 - strategy: - fail-fast: false - matrix: - include: - - task: lint - command: pnpm lint - - task: format - command: pnpm format steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Check types and lint and oxfmt + run: pnpm check + + # Validate docs (format, lint, broken links) only when docs files changed. + check-docs: + needs: [docs-scope] + if: needs.docs-scope.outputs.docs_changed == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@v4 with: - node-version: 22.x - check-latest: true + submodules: false - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" + - name: Setup Node environment + uses: ./.github/actions/setup-node-env - - name: Setup Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Runtime versions - run: | - node -v - npm -v - bun -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true - - - name: Run ${{ matrix.task }} - run: ${{ matrix.command }} + - name: Check docs + run: pnpm check:docs secrets: runs-on: blacksmith-4vcpu-ubuntu-2404 @@ -412,7 +264,7 @@ jobs: fi checks-windows: - needs: [docs-scope, changed-scope, build-artifacts] + needs: [docs-scope, changed-scope, build-artifacts, check] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') runs-on: blacksmith-4vcpu-windows-2025 env: @@ -461,19 +313,6 @@ jobs: Write-Warning "Failed to apply Defender exclusions, continuing. $($_.Exception.Message)" } - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - name: Download dist artifact (lint lane) if: matrix.task == 'lint' uses: actions/download-artifact@v4 @@ -533,7 +372,7 @@ jobs: # running 4 separate jobs per PR (as before) starved the queue. One job # per PR allows 5 PRs to run macOS checks simultaneously. macos: - needs: [docs-scope, changed-scope] + needs: [docs-scope, changed-scope, check] if: github.event_name == 'pull_request' && needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_macos == 'true' runs-on: macos-latest steps: @@ -542,50 +381,10 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - # --- Node/pnpm setup (for TS tests) --- - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Runtime versions - run: | - node -v - npm -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + install-bun: "false" # --- Run all checks sequentially (fast gates first) --- - name: TS tests (macOS) @@ -613,6 +412,14 @@ jobs: swiftlint --config .swiftlint.yml swiftformat --lint apps/macos/Sources --config .swiftformat + - name: Cache SwiftPM + uses: actions/cache@v4 + with: + path: ~/Library/Caches/org.swift.swiftpm + key: ${{ runner.os }}-swiftpm-${{ hashFiles('apps/macos/Package.resolved') }} + restore-keys: | + ${{ runner.os }}-swiftpm- + - name: Swift build (release) run: | set -euo pipefail @@ -646,19 +453,6 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - name: Select Xcode 26.1 run: | sudo xcode-select -s /Applications/Xcode_26.1.app @@ -811,7 +605,7 @@ jobs: PY android: - needs: [docs-scope, changed-scope] + needs: [docs-scope, changed-scope, check] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_android == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: @@ -828,19 +622,6 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - name: Setup Java uses: actions/setup-java@v4 with: diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index aa175961df2..a286026ae32 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -6,6 +6,12 @@ on: - main tags: - "v*" + paths-ignore: + - "docs/**" + - "**/*.md" + - "**/*.mdx" + - ".agents/**" + - "skills/**" env: REGISTRY: ghcr.io @@ -56,8 +62,8 @@ jobs: platforms: linux/amd64 labels: ${{ steps.meta.outputs.labels }} tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:amd64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:amd64,mode=max provenance: false push: true @@ -105,8 +111,8 @@ jobs: platforms: linux/arm64 labels: ${{ steps.meta.outputs.labels }} tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:arm64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache:arm64,mode=max provenance: false push: true diff --git a/.github/workflows/install-smoke.yml b/.github/workflows/install-smoke.yml index 1f42d8f4039..e6c0914f018 100644 --- a/.github/workflows/install-smoke.yml +++ b/.github/workflows/install-smoke.yml @@ -23,7 +23,7 @@ jobs: - name: Detect docs-only changes id: check - uses: ./.github/actions/detect-docs-only + uses: ./.github/actions/detect-docs-changes install-smoke: needs: [docs-scope] diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index f403c1030c0..cdb200a946e 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -25,19 +25,120 @@ jobs: configuration-path: .github/labeler.yml repo-token: ${{ steps.app-token.outputs.token }} sync-labels: true + - name: Apply PR size label + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + script: | + const pullRequest = context.payload.pull_request; + if (!pullRequest) { + return; + } + + const sizeLabels = ["size: XS", "size: S", "size: M", "size: L", "size: XL"]; + const labelColor = "fbca04"; + + for (const label of sizeLabels) { + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + }); + } catch (error) { + if (error?.status !== 404) { + throw error; + } + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + color: labelColor, + }); + } + } + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequest.number, + per_page: 100, + }); + + const excludedLockfiles = new Set(["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lockb"]); + const totalChangedLines = files.reduce((total, file) => { + const path = file.filename ?? ""; + if (path === "docs.acp.md" || path.startsWith("docs/") || excludedLockfiles.has(path)) { + return total; + } + return total + (file.additions ?? 0) + (file.deletions ?? 0); + }, 0); + + let targetSizeLabel = "size: XL"; + if (totalChangedLines < 50) { + targetSizeLabel = "size: XS"; + } else if (totalChangedLines < 200) { + targetSizeLabel = "size: S"; + } else if (totalChangedLines < 500) { + targetSizeLabel = "size: M"; + } else if (totalChangedLines < 1000) { + targetSizeLabel = "size: L"; + } + + const currentLabels = await github.paginate(github.rest.issues.listLabelsOnIssue, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + per_page: 100, + }); + + for (const label of currentLabels) { + const name = label.name ?? ""; + if (!sizeLabels.includes(name)) { + continue; + } + if (name === targetSizeLabel) { + continue; + } + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + name, + }); + } + + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + labels: [targetSizeLabel], + }); - name: Apply maintainer label for org members uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 with: github-token: ${{ steps.app-token.outputs.token }} script: | - const association = context.payload.pull_request?.author_association; - if (!association) { + const login = context.payload.pull_request?.user?.login; + if (!login) { return; } - if (![ - "MEMBER", - "OWNER", - ].includes(association)) { + + let isMaintainer = false; + try { + const membership = await github.rest.teams.getMembershipForUserInOrg({ + org: context.repo.owner, + team_slug: "maintainer", + username: login, + }); + isMaintainer = membership?.data?.state === "active"; + } catch (error) { + if (error?.status !== 404) { + throw error; + } + } + + if (!isMaintainer) { return; } @@ -62,14 +163,26 @@ jobs: with: github-token: ${{ steps.app-token.outputs.token }} script: | - const association = context.payload.issue?.author_association; - if (!association) { + const login = context.payload.issue?.user?.login; + if (!login) { return; } - if (![ - "MEMBER", - "OWNER", - ].includes(association)) { + + let isMaintainer = false; + try { + const membership = await github.rest.teams.getMembershipForUserInOrg({ + org: context.repo.owner, + team_slug: "maintainer", + username: login, + }); + isMaintainer = membership?.data?.state === "active"; + } catch (error) { + if (error?.status !== 404) { + throw error; + } + } + + if (!isMaintainer) { return; } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000000..ccafcf01a18 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,51 @@ +name: Stale + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: {} + +jobs: + stale: + permissions: + issues: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 + id: app-token + with: + app-id: "2729701" + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + - name: Mark stale issues and pull requests + uses: actions/stale@v9 + with: + repo-token: ${{ steps.app-token.outputs.token }} + days-before-issue-stale: 7 + days-before-issue-close: 5 + days-before-pr-stale: 5 + days-before-pr-close: 3 + stale-issue-label: stale + stale-pr-label: stale + exempt-issue-labels: enhancement,maintainer,pinned,security,no-stale + exempt-pr-labels: maintainer,no-stale + operations-per-run: 500 + exempt-all-assignees: true + remove-stale-when-updated: true + stale-issue-message: | + This issue has been automatically marked as stale due to inactivity. + Please add updates or it will be closed. + stale-pr-message: | + This pull request has been automatically marked as stale due to inactivity. + Please add updates or it will be closed. + close-issue-message: | + Closing due to inactivity. + If this is still an issue, please retry on the latest OpenClaw release and share updated details. + If you are absolutely sure it still happens on the latest release, open a new issue with fresh repro steps. + close-issue-reason: not_planned + close-pr-message: | + Closing due to inactivity. + If you believe this PR should be revived, post in #pr-thunderdome-dangerzone on Discord to talk to a maintainer. + That channel is the escape hatch for high-quality PRs that get auto-closed. diff --git a/.gitignore b/.gitignore index 770623d64bf..f54c8905056 100644 --- a/.gitignore +++ b/.gitignore @@ -3,11 +3,13 @@ node_modules .env docker-compose.extra.yml dist -*.bun-build pnpm-lock.yaml bun.lock bun.lockb coverage +__pycache__/ +*.pyc +.tsbuildinfo .pnpm-store .worktrees/ .DS_Store @@ -16,6 +18,11 @@ ui/src/ui/__screenshots__/ ui/playwright-report/ ui/test-results/ +# Android build artifacts +apps/android/.gradle/ +apps/android/app/build/ +apps/android/.cxx/ + # Bun build artifacts *.bun-build apps/macos/.build/ @@ -52,7 +59,6 @@ apps/ios/fastlane/screenshots/ apps/ios/fastlane/test_output/ apps/ios/fastlane/logs/ apps/ios/fastlane/.env -apps/ios/fastlane/report.xml # fastlane build artifacts (local) apps/ios/*.ipa @@ -60,10 +66,10 @@ apps/ios/*.dSYM.zip # provisioning profiles (local) apps/ios/*.mobileprovision -.env # Local untracked files .local/ +docs/.local/ IDENTITY.md USER.md .tgz diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc index 0b6b8f0fb71..94035711053 100644 --- a/.markdownlint-cli2.jsonc +++ b/.markdownlint-cli2.jsonc @@ -1,6 +1,6 @@ { "globs": ["docs/**/*.md", "docs/**/*.mdx", "README.md"], - "ignores": ["docs/zh-CN/**", "docs/.i18n/**", "docs/reference/templates/**"], + "ignores": ["docs/zh-CN/**", "docs/.i18n/**", "docs/reference/templates/**", "**/.local/**"], "config": { "default": true, diff --git a/.pi/prompts/landpr.md b/.pi/prompts/landpr.md index c36820839c5..1b150c05e0d 100644 --- a/.pi/prompts/landpr.md +++ b/.pi/prompts/landpr.md @@ -11,8 +11,10 @@ Input Do (end-to-end) Goal: PR must end in GitHub state = MERGED (never CLOSED). Use `gh pr merge` with `--rebase` or `--squash`. -1. Repo clean: `git status`. -2. Identify PR meta (author + head branch): +1. Assign PR to self: + - `gh pr edit --add-assignee @me` +2. Repo clean: `git status`. +3. Identify PR meta (author + head branch): ```sh gh pr view --json number,title,author,headRefName,baseRefName,headRepository --jq '{number,title,author:.author.login,head:.headRefName,base:.baseRefName,headRepo:.headRepository.nameWithOwner}' @@ -21,50 +23,50 @@ Goal: PR must end in GitHub state = MERGED (never CLOSED). Use `gh pr merge` wit head_repo_url=$(gh pr view --json headRepository --jq .headRepository.url) ``` -3. Fast-forward base: +4. Fast-forward base: - `git checkout main` - `git pull --ff-only` -4. Create temp base branch from main: +5. Create temp base branch from main: - `git checkout -b temp/landpr-` -5. Check out PR branch locally: +6. Check out PR branch locally: - `gh pr checkout ` -6. Rebase PR branch onto temp base: +7. Rebase PR branch onto temp base: - `git rebase temp/landpr-` - Fix conflicts; keep history tidy. -7. Fix + tests + changelog: +8. Fix + tests + changelog: - Implement fixes + add/adjust tests - Update `CHANGELOG.md` and mention `#` + `@$contrib` -8. Decide merge strategy: +9. Decide merge strategy: - Rebase if we want to preserve commit history - Squash if we want a single clean commit - If unclear, ask -9. Full gate (BEFORE commit): - - `pnpm lint && pnpm build && pnpm test` -10. Commit via committer (include # + contributor in commit message): +10. Full gate (BEFORE commit): + - `pnpm lint && pnpm build && pnpm test` +11. Commit via committer (include # + contributor in commit message): - `committer "fix: (#) (thanks @$contrib)" CHANGELOG.md ` - `land_sha=$(git rev-parse HEAD)` -11. Push updated PR branch (rebase => usually needs force): +12. Push updated PR branch (rebase => usually needs force): ```sh git remote add prhead "$head_repo_url.git" 2>/dev/null || git remote set-url prhead "$head_repo_url.git" git push --force-with-lease prhead HEAD:$head ``` -12. Merge PR (must show MERGED on GitHub): +13. Merge PR (must show MERGED on GitHub): - Rebase: `gh pr merge --rebase` - Squash: `gh pr merge --squash` - Never `gh pr close` (closing is wrong) -13. Sync main: +14. Sync main: - `git checkout main` - `git pull --ff-only` -14. Comment on PR with what we did + SHAs + thanks: +15. Comment on PR with what we did + SHAs + thanks: ```sh merge_sha=$(gh pr view --json mergeCommit --jq '.mergeCommit.oid') gh pr comment --body "Landed via temp rebase onto main.\n\n- Gate: pnpm lint && pnpm build && pnpm test\n- Land commit: $land_sha\n- Merge commit: $merge_sha\n\nThanks @$contrib!" ``` -15. Verify PR state == MERGED: +16. Verify PR state == MERGED: - `gh pr view --json state --jq .state` -16. Delete temp branch: +17. Delete temp branch: - `git branch -D temp/landpr-` diff --git a/AGENTS.md b/AGENTS.md index 482c8fe523d..a791f55b094 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,12 +15,13 @@ - Core channel docs: `docs/channels/` - Core channel code: `src/telegram`, `src/discord`, `src/slack`, `src/signal`, `src/imessage`, `src/web` (WhatsApp web), `src/channels`, `src/routing` - Extensions (channel plugins): `extensions/*` (e.g. `extensions/msteams`, `extensions/matrix`, `extensions/zalo`, `extensions/zalouser`, `extensions/voice-call`) -- When adding channels/extensions/apps/docs, review `.github/labeler.yml` for label coverage. +- When adding channels/extensions/apps/docs, update `.github/labeler.yml` and create matching GitHub labels (use existing channel/extension label colors). ## Docs Linking (Mintlify) - Docs are hosted on Mintlify (docs.openclaw.ai). - Internal doc links in `docs/**/*.md`: root-relative, no `.md`/`.mdx` (example: `[Config](/configuration)`). +- When working with documentation, read the mintlify skill. - Section cross-references: use anchors on root-relative paths (example: `[Hooks](/configuration#hooks)`). - Doc headings and anchors: avoid em dashes and apostrophes in headings because they break Mintlify anchor links. - When Peter asks for links, reply with full `https://docs.openclaw.ai/...` URLs (not root-relative). @@ -60,6 +61,8 @@ - Type-check/build: `pnpm build` - TypeScript checks: `pnpm tsgo` - Lint/format: `pnpm check` +- Format check: `pnpm format` (oxfmt --check) +- Format fix: `pnpm format:fix` (oxfmt --write) - Tests: `pnpm test` (vitest); coverage: `pnpm test:coverage` ## Coding Style & Naming Conventions @@ -90,34 +93,18 @@ ## Commit & Pull Request Guidelines +**Full maintainer PR workflow:** `.agents/skills/PR_WORKFLOW.md` -- triage order, quality bar, rebase rules, commit/changelog conventions, co-contributor policy, and the 3-step skill pipeline (`review-pr` > `prepare-pr` > `merge-pr`). + - Create commits with `scripts/committer "" `; avoid manual `git add`/`git commit` so staging stays scoped. - Follow concise, action-oriented commit messages (e.g., `CLI: add verbose flag to send`). - Group related changes; avoid bundling unrelated refactors. -- Changelog workflow: keep latest released version at top (no `Unreleased`); after publishing, bump version and start a new top section. -- PRs should summarize scope, note testing performed, and mention any user-facing changes or new flags. - Read this when submitting a PR: `docs/help/submitting-a-pr.md` ([Submitting a PR](https://docs.openclaw.ai/help/submitting-a-pr)) - Read this when submitting an issue: `docs/help/submitting-an-issue.md` ([Submitting an Issue](https://docs.openclaw.ai/help/submitting-an-issue)) -- PR review flow: when given a PR link, review via `gh pr view`/`gh pr diff` and do **not** change branches. -- PR review calls: prefer a single `gh pr view --json ...` to batch metadata/comments; run `gh pr diff` only when needed. -- Before starting a review when a GH Issue/PR is pasted: run `git pull`; if there are local changes or unpushed commits, stop and alert the user before reviewing. -- Goal: merge PRs. Prefer **rebase** when commits are clean; **squash** when history is messy. -- PR merge flow: create a temp branch from `main`, merge the PR branch into it (prefer squash unless commit history is important; use rebase/merge when it is). Always try to merge the PR unless it’s truly difficult, then use another approach. If we squash, add the PR author as a co-contributor. Apply fixes, add changelog entry (include PR # + thanks), run full gate before the final commit, commit, merge back to `main`, delete the temp branch, and end on `main`. -- If you review a PR and later do work on it, land via merge/squash (no direct-main commits) and always add the PR author as a co-contributor. -- When working on a PR: add a changelog entry with the PR number and thank the contributor. -- When working on an issue: reference the issue in the changelog entry. -- When merging a PR: leave a PR comment that explains exactly what we did and include the SHA hashes. -- When merging a PR from a new contributor: add their avatar to the README “Thanks to all clawtributors” thumbnail list. -- After merging a PR: run `bun scripts/update-clawtributors.ts` if the contributor is missing, then commit the regenerated README. ## Shorthand Commands - `sync`: if working tree is dirty, commit all changes (pick a sensible Conventional Commit message), then `git pull --rebase`; if rebase conflicts and cannot resolve, stop; otherwise `git push`. -### PR Workflow (Review vs Land) - -- **Review mode (PR link only):** read `gh pr view/diff`; **do not** switch branches; **do not** change code. -- **Landing mode:** create an integration branch from `main`, bring in PR commits (**prefer rebase** for linear history; **merge allowed** when complexity/conflicts make it safer), apply fixes, add changelog (+ thanks + PR #), run full gate **locally before committing** (`pnpm build && pnpm check && pnpm test`), commit, merge back to `main`, then `git switch main` (never stay on a topic branch after landing). Important: contributor needs to be in git graph after this! - ## Security & Configuration Tips - Web provider stores creds at `~/.openclaw/credentials/`; rerun `openclaw login` if logged out. @@ -134,6 +121,7 @@ - Vocabulary: "makeup" = "mac app". - Never edit `node_modules` (global/Homebrew/npm/git installs too). Updates overwrite. Skill notes go in `tools.md` or `AGENTS.md`. +- When adding a new `AGENTS.md` anywhere in the repo, also add a `CLAUDE.md` symlink pointing to it (example: `ln -s AGENTS.md CLAUDE.md`). - Signal: "update fly" => `fly ssh console -a flawd-bot -C "bash -lc 'cd /data/clawd/openclaw && git pull --rebase origin main'"` then `fly machines restart e825232f34d058 -a flawd-bot`. - When working on a GitHub Issue or PR, print the full URL at the end of the task. - When answering questions, respond with high-confidence answers only: verify in code; do not guess. @@ -148,6 +136,7 @@ - SwiftUI state management (iOS/macOS): prefer the `Observation` framework (`@Observable`, `@Bindable`) over `ObservableObject`/`@StateObject`; don’t introduce new `ObservableObject` unless required for compatibility, and migrate existing usages when touching related code. - Connection providers: when adding a new connection, update every UI surface and docs (macOS app, web UI, mobile if applicable, onboarding/overview docs) and add matching status + configuration forms so provider lists and settings stay in sync. - Version locations: `package.json` (CLI), `apps/android/app/build.gradle.kts` (versionName/versionCode), `apps/ios/Sources/Info.plist` + `apps/ios/Tests/Info.plist` (CFBundleShortVersionString/CFBundleVersion), `apps/macos/Sources/OpenClaw/Resources/Info.plist` (CFBundleShortVersionString/CFBundleVersion), `docs/install/updating.md` (pinned npm version), `docs/platforms/mac/release.md` (APP_VERSION/APP_BUILD examples), Peekaboo Xcode projects/Info.plists (MARKETING_VERSION/CURRENT_PROJECT_VERSION). +- "Bump version everywhere" means all version locations above **except** `appcast.xml` (only touch appcast when cutting a new macOS Sparkle release). - **Restart apps:** “restart iOS/Android apps” means rebuild (recompile/install) and relaunch, not just kill/launch. - **Device checks:** before testing, verify connected real devices (iOS/Android) before reaching for simulators/emulators. - iOS Team ID lookup: `security find-identity -p codesigning -v` → use Apple Development (…) TEAMID. Fallback: `defaults read com.apple.dt.Xcode IDEProvisioningTeamIdentifiers`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ce4fa2698e..bedf9ebaa41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,53 +2,124 @@ Docs: https://docs.openclaw.ai -## 2026.2.6-4 +## 2026.2.10 -### Added +### Changes -- Gateway: add `agents.create`, `agents.update`, `agents.delete` RPC methods for web UI agent management. (#11045) Thanks @advaitpaliwal. -- Gateway: add node command allowlists (default-deny unknown node commands; configurable via `gateway.nodes.allowCommands` / `gateway.nodes.denyCommands`). (#11755) Thanks @mbelinky. -- Plugins: add `device-pair` (Telegram `/pair` flow) and `phone-control` (iOS/Android node controls). (#11755) Thanks @mbelinky. -- iOS: add alpha iOS node app (Telegram setup-code pairing + Talk/Chat surfaces). (#11756) Thanks @mbelinky. -- Docs: seed initial ja-JP translations (POC) and make docs-i18n prompts language-pluggable for Japanese. (#11988) Thanks @joshp123. -- Paths: add `OPENCLAW_HOME` environment variable for overriding the home directory used by all internal path resolution. (#12091) Thanks @sebslight. +- Version alignment: bump manifests and package versions to `2026.2.10`; keep `appcast.xml` unchanged until the next macOS release cut. +- CLI: add `openclaw logs --local-time` to display log timestamps in local timezone. (#13818) Thanks @xialonglee. +- Config: avoid redacting `maxTokens`-like fields during config snapshot redaction, preventing round-trip validation failures in `/config`. (#14006) Thanks @constansino. ### Fixes -- Exec approvals: format forwarded command text as inline/fenced monospace for safer approval scanning across channels. (#11937) -- Config: clamp `maxTokens` to `contextWindow` to prevent invalid model configs. (#5516) Thanks @lailoo. -- Docs: fix language switcher ordering and Japanese locale flag in Mintlify nav. (#12023) Thanks @joshp123. -- Paths: make internal path resolution respect `HOME`/`USERPROFILE` before `os.homedir()` across config, agents, sessions, pairing, cron, and CLI profiles. (#12091) Thanks @sebslight. -- Paths: structurally resolve `OPENCLAW_HOME`-derived home paths and fix Windows drive-letter handling in tool meta shortening. (#12125) Thanks @mcaxtr. -- Thinking: allow xhigh for `github-copilot/gpt-5.2-codex` and `github-copilot/gpt-5.2`. (#11646) Thanks @seans-openclawbot. -- Discord: support forum/media `thread create` starter messages, wire `message thread create --message`, and harden thread-create routing. (#10062) Thanks @jarvis89757. -- Gateway: stabilize chat routing by canonicalizing node session keys for node-originated chat methods. (#11755) Thanks @mbelinky. -- Web UI: make chat refresh smoothly scroll to the latest messages and suppress new-messages badge flash during manual refresh. -- Cron: route text-only isolated agent announces through the shared subagent announce flow; add exponential backoff for repeated errors; preserve future `nextRunAtMs` on restart; include current-boundary schedule matches; prevent stale threadId reuse across targets; and add per-job execution timeout. (#11641) Thanks @tyler6204. -- Subagents: stabilize announce timing, preserve compaction metrics across retries, clamp overflow-prone long timeouts, and cap impossible context usage token totals. (#11551) Thanks @tyler6204. -- Agents: recover from context overflow caused by oversized tool results (pre-emptive capping + fallback truncation). (#11579) Thanks @tyler6204. -- Gateway: no more post-compaction amnesia; injected transcript writes now preserve Pi session `parentId` chain so agents can remember again. Thanks @Takhoffman 🦞. +- Cron: use requested `agentId` for isolated job auth resolution. (#13983) Thanks @0xRaini. +- Cron: prevent cron jobs from skipping execution when `nextRunAtMs` advances. (#14068) Thanks @WalterSumbon. +- Cron: pass `agentId` to `runHeartbeatOnce` for main-session jobs. (#14140) Thanks @ishikawa-pro. +- Cron: re-arm timers when `onTimer` fires while a job is still executing. (#14233) Thanks @tomron87. +- Cron: prevent duplicate fires when multiple jobs trigger simultaneously. (#14256) Thanks @xinhuagu. +- Cron: isolate scheduler errors so one bad job does not break all jobs. (#14385) Thanks @MarvinDontPanic. +- Cron: prevent one-shot `at` jobs from re-firing on restart after skipped/errored runs. (#13878) Thanks @lailoo. +- WhatsApp: convert Markdown bold/strikethrough to WhatsApp formatting. (#14285) Thanks @Raikan10. +- WhatsApp: allow media-only sends and normalize leading blank payloads. (#14408) Thanks @karimnaguib. +- WhatsApp: default MIME type for voice messages when Baileys omits it. (#14444) Thanks @mcaxtr. +- Ollama: use configured `models.providers.ollama.baseUrl` for model discovery and normalize `/v1` endpoints to the native Ollama API root. (#14131) Thanks @shtse8. +- Slack: detect control commands when channel messages start with bot mention prefixes (for example, `@Bot /new`). (#14142) Thanks @beefiker. +- Discord tests: use a partial @buape/carbon mock in slash command coverage. (#13262) Thanks @arosstale. +- CLI/Wizard: exit with code 1 when `configure`, `agents add`, or interactive `onboard` wizards are canceled, so `set -e` automation stops correctly. (#14156) Thanks @0xRaini. +- Feishu: pass `Buffer` directly to the Feishu SDK upload APIs instead of `Readable.from(...)` to avoid form-data upload failures. (#10345) Thanks @youngerstyle. +- Feishu: trigger mention-gated group handling only when the bot itself is mentioned (not just any mention). (#11088) Thanks @openperf. +- Feishu: probe status uses the resolved account context for multi-account credential checks. (#11233) Thanks @onevcat. +- Feishu DocX: preserve top-level converted block order using `firstLevelBlockIds` when writing/appending documents. (#13994) Thanks @Cynosure159. +- Feishu plugin packaging: remove `workspace:*` `openclaw` dependency from `extensions/feishu` and sync lockfile for install compatibility. (#14423) Thanks @jackcooper2015. +- Telegram: handle no-text message in model picker editMessageText. (#14397) Thanks @0xRaini. +- Slack: change default replyToMode from "off" to "all". (#14364) Thanks @nm-de. +- Tests: update thread ID handling in Slack message collection tests. (#14108) Thanks @swizzmagik. +- Telegram: surface REACTION_INVALID as non-fatal warning. (#14340) Thanks @0xRaini. + +## 2026.2.9 + +### Added + +- Commands: add `commands.allowFrom` config for separate command authorization, allowing operators to restrict slash commands to specific users while keeping chat open to others. (#12430) Thanks @thewilloftheshadow. +- Docker: add ClawDock shell helpers for Docker workflows. (#12817) Thanks @Olshansk. +- iOS: alpha node app + setup-code onboarding. (#11756) Thanks @mbelinky. +- Channels: comprehensive BlueBubbles and channel cleanup. (#11093) Thanks @tyler6204. +- Channels: IRC first-class channel support. (#11482) Thanks @vignesh07. +- Plugins: device pairing + phone control plugins (Telegram `/pair`, iOS/Android node controls). (#11755) Thanks @mbelinky. +- Tools: add Grok (xAI) as a `web_search` provider. (#12419) Thanks @tmchow. +- Gateway: add agent management RPC methods for the web UI (`agents.create`, `agents.update`, `agents.delete`). (#11045) Thanks @advaitpaliwal. +- Gateway: stream thinking events to WS clients and broadcast tool events independent of verbose level. (#10568) Thanks @nk1tz. +- Web UI: show a Compaction divider in chat history. (#11341) Thanks @Takhoffman. +- Agents: include runtime shell in agent envelopes. (#1835) Thanks @Takhoffman. +- Agents: auto-select `zai/glm-4.6v` for image understanding when ZAI is primary provider. (#10267) Thanks @liuy. +- Paths: add `OPENCLAW_HOME` for overriding the home directory used by internal path resolution. (#12091) Thanks @sebslight. +- Onboarding: add Custom Provider flow for OpenAI and Anthropic-compatible endpoints. (#11106) Thanks @MackDing. +- Hooks: route webhook agent runs to specific `agentId`s, add `hooks.allowedAgentIds` controls, and fall back to default agent when unknown IDs are provided. (#13672) Thanks @BillChirico. + +### Fixes + +- Cron: prevent one-shot `at` jobs from re-firing on gateway restart when previously skipped or errored. (#13845) +- Discord: add exec approval cleanup option to delete DMs after approval/denial/timeout. (#13205) Thanks @thewilloftheshadow. +- Sessions: prune stale entries, cap session store size, rotate large stores, accept duration/size thresholds, default to warn-only maintenance, and prune cron run sessions after retention windows. (#13083) Thanks @skyfallsin, @Glucksberg, @gumadeiras. +- CI: Implement pipeline and workflow order. Thanks @quotentiroler. +- WhatsApp: preserve original filenames for inbound documents. (#12691) Thanks @akramcodez. +- Telegram: harden quote parsing; preserve quote context; avoid QUOTE_TEXT_INVALID; avoid nested reply quote misclassification. (#12156) Thanks @rybnikov. +- Telegram: recover proactive sends when stale topic thread IDs are used by retrying without `message_thread_id`. (#11620) +- Discord: auto-create forum/media thread posts on send, with chunked follow-up replies and media handling for forum sends. (#12380) Thanks @magendary, @thewilloftheshadow. +- Discord: cap gateway reconnect attempts to avoid infinite retry loops. (#12230) Thanks @Yida-Dev. - Telegram: render markdown spoilers with `` HTML tags. (#11543) Thanks @ezhikkk. -- Telegram: recover proactive sends when stale topic thread IDs are used by retrying without `message_thread_id`, and clear explicit no-thread route updates instead of inheriting stale thread state. (#11620) +- Telegram: truncate command registration to 100 entries to avoid `BOT_COMMANDS_TOO_MUCH` failures on startup. (#12356) Thanks @arosstale. +- Telegram: match DM `allowFrom` against sender user id (fallback to chat id) and clarify pairing logs. (#12779) Thanks @liuxiaopai-ai. +- Pairing/Telegram: include the actual pairing code in approve commands, route Telegram pairing replies through the shared pairing message builder, and add regression checks to prevent `` placeholder drift. +- Onboarding: QuickStart now auto-installs shell completion (prompt only in Manual). +- Onboarding/Providers: add LiteLLM provider onboarding and preserve custom LiteLLM proxy base URLs while enforcing API-key auth mode. (#12823) Thanks @ryan-crabbe. +- Docker: make `docker-setup.sh` compatible with macOS Bash 3.2 and empty extra mounts. (#9441) Thanks @mateusz-michalik. +- Auth: strip embedded line breaks from pasted API keys and tokens before storing/resolving credentials. +- Agents: strip reasoning tags and downgraded tool markers from messaging tool and streaming output to prevent leakage. (#11053, #13453) Thanks @liebertar, @meaadore1221-afk, @gumadeiras. +- Browser: prevent stuck `act:evaluate` from wedging the browser tool, and make cancellation stop waiting promptly. (#13498) Thanks @onutc. +- Security/Gateway: default-deny missing connect `scopes` (no implicit `operator.admin`). +- Web UI: make chat refresh smoothly scroll to the latest messages and suppress new-messages badge flash during manual refresh. +- Web UI: coerce Form Editor values to schema types before `config.set` and `config.apply`, preventing numeric and boolean fields from being serialized as strings. (#13468) Thanks @mcaxtr. +- Tools/web_search: include provider-specific settings in the web search cache key, and pass `inlineCitations` for Grok. (#12419) Thanks @tmchow. +- Tools/web_search: fix Grok response parsing for xAI Responses API output blocks. (#13049) Thanks @ereid7. +- Tools/web_search: normalize direct Perplexity model IDs while keeping OpenRouter model IDs unchanged. (#12795) Thanks @cdorsey. +- Model failover: treat HTTP 400 errors as failover-eligible, enabling automatic model fallback. (#1879) Thanks @orenyomtov. +- Errors: prevent false positive context overflow detection when conversation mentions "context overflow" topic. (#2078) Thanks @sbking. +- Errors: avoid rewriting/swallowing normal assistant replies that mention error keywords by scoping `sanitizeUserFacingText` rewrites to error-context. (#12988) Thanks @Takhoffman. +- Config: re-hydrate state-dir `.env` during runtime config loads so `${VAR}` substitutions remain resolvable. (#12748) Thanks @rodrigouroz. +- Gateway: no more post-compaction amnesia; injected transcript writes now preserve Pi session `parentId` chain so agents can remember again. (#12283) Thanks @Takhoffman. +- Gateway: fix multi-agent sessions.usage discovery. (#11523) Thanks @Takhoffman. +- Agents: recover from context overflow caused by oversized tool results (pre-emptive capping + fallback truncation). (#11579) Thanks @tyler6204. +- Subagents/compaction: stabilize announce timing and preserve compaction metrics across retries. (#11664) Thanks @tyler6204. +- Subagents: report timeout-aborted runs as timed out instead of completed successfully in parent-session announcements. (#13996) Thanks @dario-github. +- Cron: share isolated announce flow and harden scheduling/delivery reliability. (#11641) Thanks @tyler6204. +- Cron tool: recover flat params when LLM omits the `job` wrapper for add requests. (#12124) Thanks @tyler6204. - Gateway/CLI: when `gateway.bind=lan`, use a LAN IP for probe URLs and Control UI links. (#11448) Thanks @AnonO6. +- CLI: make `openclaw plugins list` output scannable by hoisting source roots and shortening bundled/global/workspace plugin paths. +- Hooks: fix bundled hooks broken since 2026.2.2 (tsdown migration). (#9295) Thanks @patrickshao. +- Security/Plugins: install plugin and hook dependencies with `--ignore-scripts` to prevent lifecycle script execution. +- Routing: refresh bindings per message by loading config at route resolution so binding changes apply without restart. (#11372) Thanks @juanpablodlc. +- Exec approvals: render forwarded commands in monospace for safer approval scanning. (#11937) Thanks @sebslight. +- Config: clamp `maxTokens` to `contextWindow` to prevent invalid model configs. (#5516) Thanks @lailoo. +- Thinking: allow xhigh for `github-copilot/gpt-5.2-codex` and `github-copilot/gpt-5.2`. (#11646) Thanks @LatencyTDH. +- Thinking: honor `/think off` for reasoning-capable models. (#9564) Thanks @liuy. +- Discord: support forum/media thread-create starter messages, wire `message thread create --message`, and harden routing. (#10062) Thanks @jarvis89757. +- Paths: structurally resolve `OPENCLAW_HOME`-derived home paths and fix Windows drive-letter handling in tool meta shortening. (#12125) Thanks @mcaxtr. - Memory: set Voyage embeddings `input_type` for improved retrieval. (#10818) Thanks @mcinteerj. -- Memory/QMD: run boot refresh in background by default, add configurable QMD maintenance timeouts, and retry QMD after fallback failures. (#9690, #9705) -- Memory/QMD: log explicit warnings when `memory.qmd.scope` blocks a search request. (#10191) +- Memory: disable async batch embeddings by default for memory indexing (opt-in via `agents.defaults.memorySearch.remote.batch.enabled`). (#13069) Thanks @mcinteerj. +- Memory/QMD: reuse default model cache across agents instead of re-downloading per agent. (#12114) Thanks @tyler6204. +- Memory/QMD: run boot refresh in background by default, add configurable QMD maintenance timeouts, retry QMD after fallback failures, and scope QMD queries to OpenClaw-managed collections. (#9690, #9705, #10042) Thanks @vignesh07. +- Memory/QMD: initialize QMD backend on gateway startup so background update timers restart after process reloads. (#10797) Thanks @vignesh07. +- Config/Memory: auto-migrate legacy top-level `memorySearch` settings into `agents.defaults.memorySearch`. (#11278, #9143) Thanks @vignesh07. +- Memory/QMD: treat plain-text `No results found` output from QMD as an empty result instead of throwing invalid JSON errors. (#9824) +- Memory/QMD: add `memory.qmd.searchMode` to choose `query`, `search`, or `vsearch` recall mode. (#9967, #10084) - Media understanding: recognize `.caf` audio attachments for transcription. (#10982) Thanks @succ985. - State dir: honor `OPENCLAW_STATE_DIR` for default device identity and canvas storage paths. (#4824) Thanks @kossoy. -- Doctor/State dir: suppress repeated legacy migration warnings only for valid symlink mirrors, while keeping warnings for empty or invalid legacy trees. (#11709) Thanks @gumadeiras. -- Tests: harden flaky hotspots by removing timer sleeps, consolidating onboarding provider-auth coverage, and improving memory test realism. (#11598) Thanks @gumadeiras. -- macOS: honor Nix-managed defaults suite (`ai.openclaw.mac`) for nixMode to prevent onboarding from reappearing after bundle-id churn. (#12205) Thanks @joshp123. ## 2026.2.6 ### Changes -- Hygiene: remove `workspace:*` from `dependencies` in msteams, nostr, zalo extensions (breaks external `npm install`; keep in `devDependencies` only). -- Hygiene: add non-root `sandbox` user to `Dockerfile.sandbox` and `Dockerfile.sandbox-browser`. -- Hygiene: remove dead `vitest` key from `package.json` (superseded by `vitest.config.ts`). -- Hygiene: remove redundant top-level `overrides` from `package.json` (pnpm uses `pnpm.overrides`). -- Hygiene: sync `onlyBuiltDependencies` between `pnpm-workspace.yaml` and `package.json` (add missing `node-llama-cpp`, sort alphabetically). - Cron: default `wakeMode` is now `"now"` for new jobs (was `"next-heartbeat"`). (#10776) Thanks @tyler6204. - Cron: `cron run` defaults to force execution; use `--due` to restrict to due-only. (#10776) Thanks @tyler6204. - Models: support Anthropic Opus 4.6 and OpenAI Codex gpt-5.3-codex (forward-compat fallbacks). (#9853, #10720, #9995) Thanks @TinyTb, @calvin-hpnet, @tyler6204. @@ -71,6 +142,9 @@ Docs: https://docs.openclaw.ai - Cron: scheduler reliability (timer drift, restart catch-up, lock contention, stale running markers). (#10776) Thanks @tyler6204. - Cron: store migration hardening (legacy field migration, parse error handling, explicit delivery mode persistence). (#10776) Thanks @tyler6204. +- Memory: set Voyage embeddings `input_type` for improved retrieval. (#10818) Thanks @mcinteerj. +- Memory/QMD: run boot refresh in background by default, add configurable QMD maintenance timeouts, retry QMD after fallback failures, and scope QMD queries to OpenClaw-managed collections. (#9690, #9705, #10042) Thanks @vignesh07. +- Media understanding: recognize `.caf` audio attachments for transcription. (#10982) Thanks @succ985. - Telegram: auto-inject DM topic threadId in message tool + subagent announce. (#7235) Thanks @Lukavyi. - Security: require auth for Gateway canvas host and A2UI assets. (#9518) Thanks @coygeek. - Cron: fix scheduling and reminder delivery regressions; harden next-run recompute + timer re-arming + legacy schedule fields. (#9733, #9823, #9948, #9932) Thanks @tyler6204, @pycckuu, @j2h4u, @fujiwara-tofu-shop. @@ -121,6 +195,7 @@ Docs: https://docs.openclaw.ai - Cron: deliver announce runs directly, honor delivery mode, and respect wakeMode for summaries. (#8540) Thanks @tyler6204. - Telegram: include forward_from_chat metadata in forwarded messages and harden cron delivery target checks. (#8392) Thanks @Glucksberg. - macOS: fix cron payload summary rendering and ISO 8601 formatter concurrency safety. +- Discord: enforce DM allowlists for agent components (buttons/select menus), honoring pairing store approvals and tag matches. (#11254) Thanks @thedudeabidesai. ## 2026.2.2-3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 169e0dcb9c6..a5e9164a94d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,6 +16,9 @@ Welcome to the lobster tank! 🦞 - **Shadow** - Discord + Slack subsystem - GitHub: [@thewilloftheshadow](https://github.com/thewilloftheshadow) · X: [@4shad0wed](https://x.com/4shad0wed) +- **Vignesh** - Memory (QMD), formal modeling, TUI, and Lobster + - GitHub: [@vignesh07](https://github.com/vignesh07) · X: [@\_vgnsh](https://x.com/_vgnsh) + - **Jos** - Telegram, API, Nix mode - GitHub: [@joshp123](https://github.com/joshp123) · X: [@jjpcodes](https://x.com/jjpcodes) @@ -25,6 +28,9 @@ Welcome to the lobster tank! 🦞 - **Gustavo Madeira Santana** - Multi-agents, CLI, web UI - GitHub: [@gumadeiras](https://github.com/gumadeiras) · X: [@gumadeiras](https://x.com/gumadeiras) +- **Maximilian Nussbaumer** - DevOps, CI, Code Sanity + - GitHub: [@quotentiroler](https://github.com/quotentiroler) · X: [@quotentiroler](https://x.com/quotentiroler) + ## How to Contribute 1. **Bugs & small fixes** → Open a PR! @@ -35,6 +41,7 @@ Welcome to the lobster tank! 🦞 - Test locally with your OpenClaw instance - Run tests: `pnpm build && pnpm check && pnpm test` +- Ensure CI checks pass - Keep PRs focused (one thing per PR) - Describe what & why @@ -72,7 +79,33 @@ We are currently prioritizing: - **Stability**: Fixing edge cases in channel connections (WhatsApp/Telegram). - **UX**: Improving the onboarding wizard and error messages. -- **Skills**: Expanding the library of bundled skills and improving the Skill Creation developer experience. +- **Skills**: For skill contributions, head to [ClawHub](https://clawhub.ai/) — the community hub for OpenClaw skills. - **Performance**: Optimizing token usage and compaction logic. Check the [GitHub Issues](https://github.com/openclaw/openclaw/issues) for "good first issue" labels! + +## Report a Vulnerability + +We take security reports seriously. Report vulnerabilities directly to the repository where the issue lives: + +- **Core CLI and gateway** — [openclaw/openclaw](https://github.com/openclaw/openclaw) +- **macOS desktop app** — [openclaw/openclaw](https://github.com/openclaw/openclaw) (apps/macos) +- **iOS app** — [openclaw/openclaw](https://github.com/openclaw/openclaw) (apps/ios) +- **Android app** — [openclaw/openclaw](https://github.com/openclaw/openclaw) (apps/android) +- **ClawHub** — [openclaw/clawhub](https://github.com/openclaw/clawhub) +- **Trust and threat model** — [openclaw/trust](https://github.com/openclaw/trust) + +For issues that don't fit a specific repo, or if you're unsure, email **security@openclaw.ai** and we'll route it. + +### Required in Reports + +1. **Title** +2. **Severity Assessment** +3. **Impact** +4. **Affected Component** +5. **Technical Reproduction** +6. **Demonstrated Impact** +7. **Environment** +8. **Remediation Advice** + +Reports without reproduction steps, demonstrated impact, and remediation advice will be deprioritized. Given the volume of AI-generated scanner findings, we must ensure we're receiving vetted reports from researchers who understand the issues. diff --git a/Dockerfile b/Dockerfile index 237a6a238ac..716ab2099f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY scripts ./scripts RUN pnpm install --frozen-lockfile COPY . . -RUN OPENCLAW_A2UI_SKIP_MISSING=1 pnpm build +RUN pnpm build # Force pnpm for UI build (Bun may fail on ARM/Synology architectures) ENV OPENCLAW_PREFER_PNPM=1 RUN pnpm ui:build diff --git a/README-header.png b/README-header.png deleted file mode 100644 index 243ff29c184..00000000000 Binary files a/README-header.png and /dev/null differ diff --git a/README.md b/README.md index e7455515b8a..b979d00ef12 100644 --- a/README.md +++ b/README.md @@ -1,396 +1,89 @@ -# 🦞 OpenClaw AI SDK +# Ironclaw + +**AI-powered CRM platform with multi-channel agent gateway, DuckDB workspace, and knowledge management.**

- OpenClaw + Vercel AI SDK v6 -

- -

- Clawdbot's future-compatible fork using Vercel's AI SDK by default -

- -

- CI status - GitHub release - Discord + npm version MIT License

-> **This is a fork of [OpenClaw](https://github.com/openclaw/openclaw)** that uses **Vercel's AI SDK v6** by default instead of the pi-mono agent. Dual SDK support makes this useful for developers in the Vercel ecosystem using `useChat()` and AI SDK v5/6 primitives. +--- -**OpenClaw** is a _personal AI assistant_ you run on your own devices. -It answers you on the channels you already use (WhatsApp, Telegram, Slack, Discord, Google Chat, Signal, iMessage, Microsoft Teams, WebChat), plus extension channels like BlueBubbles, Matrix, Zalo, and Zalo Personal. It can speak and listen on macOS/iOS/Android, and can render a live Canvas you control. The Gateway is just the control plane — the product is the assistant. +Ironclaw is a personal AI assistant and CRM toolkit that runs on your own devices. It connects to your existing messaging channels (WhatsApp, Telegram, Slack, Discord, Google Chat, Signal, iMessage, Microsoft Teams, and more), manages structured data through a DuckDB-powered workspace, and provides a rich web interface for knowledge management and reporting. -If you want a personal, single-user assistant that feels local, fast, and always-on, this is it. +Built on [OpenClaw](https://github.com/openclaw/openclaw) with **Vercel AI SDK v6** as the default LLM orchestration layer. -<<<<<<< HEAD +## Features -## Why This Fork? +- **Multi-channel inbox** -- WhatsApp, Telegram, Slack, Discord, Google Chat, Signal, iMessage, Microsoft Teams, Matrix, WebChat, and more. +- **DuckDB workspace** -- Structured data objects, file management, full-text search, and bulk operations through a local DuckDB-backed store. +- **Web UI (Dench)** -- Modern chat interface with chain-of-thought reasoning, report cards, media viewer, and a database explorer. Supports light and dark themes. +- **Agent gateway** -- Local-first WebSocket control plane for sessions, channels, tools, and events. Routes agent execution through lane-based concurrency. +- **Vercel AI SDK v6** -- Default LLM engine with support for Anthropic, OpenAI, Google, Groq, Mistral, xAI, OpenRouter, and Azure. Full extended thinking/reasoning support. +- **Knowledge management** -- File tree, search index, workspace objects with custom fields, and entry-level detail views. +- **TanStack data tables** -- Sortable, filterable, bulk-selectable tables for workspace objects powered by `@tanstack/react-table`. +- **Companion apps** -- macOS menu bar app, iOS/Android nodes with voice, camera, and canvas capabilities. +- **Skills platform** -- Bundled, managed, and workspace-scoped skills with install gating. -| Feature | Original OpenClaw | This Fork (AI SDK) | -| ---------------------- | ------------------ | ------------------------------------------------ | -| LLM Orchestration | pi-mono agent only | **Vercel AI SDK v6** (default) and pi-mono agent | -| Dual Engine Support | No | **Yes** - switch via config | -| `useChat()` Compatible | No | **Yes** | +## Install -# [Website](https://openclaw.ai) · [Docs](https://docs.openclaw.ai) · [DeepWiki](https://deepwiki.com/openclaw/openclaw) · [Getting Started](https://docs.openclaw.ai/start/getting-started) · [Updating](https://docs.openclaw.ai/install/updating) · [Showcase](https://docs.openclaw.ai/start/showcase) · [FAQ](https://docs.openclaw.ai/start/faq) · [Wizard](https://docs.openclaw.ai/start/wizard) · [Nix](https://github.com/openclaw/nix-clawdbot) · [Docker](https://docs.openclaw.ai/install/docker) · [Discord](https://discord.gg/clawd) +**Runtime: Node 22+** -[Website](https://openclaw.ai) · [Docs](https://docs.openclaw.ai) · [DeepWiki](https://deepwiki.com/openclaw/openclaw) · [Getting Started](https://docs.openclaw.ai/start/getting-started) · [Updating](https://docs.openclaw.ai/install/updating) · [Showcase](https://docs.openclaw.ai/start/showcase) · [FAQ](https://docs.openclaw.ai/start/faq) · [Wizard](https://docs.openclaw.ai/start/wizard) · [Nix](https://github.com/openclaw/nix-openclaw) · [Docker](https://docs.openclaw.ai/install/docker) · [Discord](https://discord.gg/clawd) - -> > > > > > > 0cf93b8fa74566258131f9e8ca30f313aac89d26 - -Preferred setup: run the onboarding wizard (`openclaw onboard`) in your terminal. -The wizard guides you step by step through setting up the gateway, workspace, channels, and skills. The CLI wizard is the recommended path and works on **macOS, Linux, and Windows (via WSL2; strongly recommended)**. -Works with npm, pnpm, or bun. -New install? Start here: [Getting started](https://docs.openclaw.ai/start/getting-started) - -**Subscriptions (OAuth):** - -- **[Anthropic](https://www.anthropic.com/)** (Claude Pro/Max) -- **[OpenAI](https://openai.com/)** (ChatGPT/Codex) - -Model note: while any model is supported, I strongly recommend **Anthropic Pro/Max (100/200) + Opus 4.6** for long‑context strength and better prompt‑injection resistance. See [Onboarding](https://docs.openclaw.ai/start/onboarding). - -## Models (selection + auth) - -- Models config + CLI: [Models](https://docs.openclaw.ai/concepts/models) -- Auth profile rotation (OAuth vs API keys) + fallbacks: [Model failover](https://docs.openclaw.ai/concepts/model-failover) - -## Install (recommended) - -Runtime: **Node ≥22**. - -### From this fork (AI SDK version) +### From npm + +```bash +npm install -g ironclaw@latest + +ironclaw onboard --install-daemon +``` + +### From source ```bash -# Clone this fork git clone https://github.com/kumarabhirup/openclaw-ai-sdk.git cd openclaw-ai-sdk pnpm install pnpm build -pnpm openclaw onboard --install-daemon +pnpm dev onboard --install-daemon ``` -### From npm (original OpenClaw) +## Quick start ```bash -npm install -g openclaw@latest -# or: pnpm add -g openclaw@latest - -openclaw onboard --install-daemon -``` - -The wizard installs the Gateway daemon (launchd/systemd user service) so it stays running. - -## AI SDK Configuration - -This fork defaults to the **AI SDK engine**. You can switch between engines: - -```bash -# During setup, choose your preferred engine -openclaw configure - -# Or set directly in config -openclaw config set agents.engine aisdk # Use AI SDK (default) -openclaw config set agents.engine pi-agent # Use original pi-agent -``` - -### Supported Providers (AI SDK) - -| Provider | Environment Variable | Models | -| ---------- | ------------------------------ | --------------------------------- | -| Anthropic | `ANTHROPIC_API_KEY` | Claude 4/3.x, Opus, Sonnet, Haiku | -| OpenAI | `OPENAI_API_KEY` | GPT-4o, GPT-4, o1, o3 | -| Google | `GOOGLE_GENERATIVE_AI_API_KEY` | Gemini 2.x, 1.5 Pro | -| AI Gateway | `AI_GATEWAY_API_KEY` | All providers via Vercel | -| OpenRouter | `OPENROUTER_API_KEY` | 100+ models | -| Azure | `AZURE_OPENAI_API_KEY` | Azure OpenAI models | -| Groq | `GROQ_API_KEY` | Llama, Mixtral | -| Mistral | `MISTRAL_API_KEY` | Mistral models | -| xAI | `XAI_API_KEY` | Grok models | - -### Anthropic Thinking/Reasoning - -This fork fully supports [Anthropic's extended thinking](https://ai-sdk.dev/providers/ai-sdk-providers/anthropic#reasoning): - -```bash -# Set thinking level (maps to AI SDK budgetTokens) -openclaw agent --message "Complex task" --thinking high - -# Thinking levels: off, minimal, low, medium, high, xhigh -# xhigh uses 32K budget tokens + effort: high for Opus 4.5 -``` - -## Quick start (TL;DR) - -Runtime: **Node ≥22**. - -Full beginner guide (auth, pairing, channels): [Getting started](https://docs.openclaw.ai/start/getting-started) - -```bash -openclaw onboard --install-daemon - -openclaw gateway --port 18789 --verbose +# Start the gateway +ironclaw gateway --port 18789 --verbose # Send a message -openclaw message send --to +1234567890 --message "Hello from OpenClaw" +ironclaw message send --to +1234567890 --message "Hello from Ironclaw" -# Talk to the assistant (optionally deliver back to any connected channel: WhatsApp/Telegram/Slack/Discord/Google Chat/Signal/iMessage/BlueBubbles/Microsoft Teams/Matrix/Zalo/Zalo Personal/WebChat) -openclaw agent --message "Ship checklist" --thinking high +# Talk to the agent +ironclaw agent --message "Summarize today's tasks" --thinking high ``` -Upgrading? [Updating guide](https://docs.openclaw.ai/install/updating) (and run `openclaw doctor`). +## Web UI -## Development channels +The web application lives in `apps/web/` and is built with Next.js. It provides: -- **stable**: tagged releases (`vYYYY.M.D` or `vYYYY.M.D-`), npm dist-tag `latest`. -- **beta**: prerelease tags (`vYYYY.M.D-beta.N`), npm dist-tag `beta` (macOS app may be missing). -- **dev**: moving head of `main`, npm dist-tag `dev` (when published). +- **Chat panel** with streaming responses, chain-of-thought display, and markdown rendering (via `react-markdown` + `remark-gfm`). +- **Workspace sidebar** with a file manager tree, knowledge tree, and database viewer. +- **Object tables** with sorting, filtering, row selection, and bulk delete. +- **Entry detail modals** with field editing and media previews. +- **Report cards** with chart panels and filter bars. +- **Media viewer** supporting images, video, audio, and PDFs. -Switch channels (git + npm): `openclaw update --channel stable|beta|dev`. -Details: [Development channels](https://docs.openclaw.ai/install/development-channels). - -## From source (development) - -Prefer `pnpm` for builds from source. Bun is optional for running TypeScript directly. +To run the web UI in development: ```bash -git clone https://github.com/kumarabhirup/openclaw-ai-sdk.git -cd openclaw-ai-sdk - +cd apps/web pnpm install -pnpm ui:build # auto-installs UI deps on first run -pnpm build - -pnpm openclaw onboard --install-daemon - -# Dev loop (auto-reload on TS changes) -pnpm gateway:watch +pnpm dev ``` -Note: `pnpm openclaw ...` runs TypeScript directly (via `tsx`). `pnpm build` produces `dist/` for running via Node / the packaged `openclaw` binary. - -### Syncing with upstream OpenClaw - -This fork is designed to minimize merge conflicts with upstream: - -```bash -# Add upstream remote (one-time) -git remote add upstream https://github.com/openclaw/openclaw.git - -# Pull latest upstream changes -git fetch upstream -git merge upstream/main - -# AI SDK code lives in src/agents/aisdk/ - isolated from pi-agent changes -``` - -## Security defaults (DM access) - -OpenClaw connects to real messaging surfaces. Treat inbound DMs as **untrusted input**. - -Full security guide: [Security](https://docs.openclaw.ai/gateway/security) - -Default behavior on Telegram/WhatsApp/Signal/iMessage/Microsoft Teams/Discord/Google Chat/Slack: - -- **DM pairing** (`dmPolicy="pairing"` / `channels.discord.dm.policy="pairing"` / `channels.slack.dm.policy="pairing"`): unknown senders receive a short pairing code and the bot does not process their message. -- Approve with: `openclaw pairing approve ` (then the sender is added to a local allowlist store). -- Public inbound DMs require an explicit opt-in: set `dmPolicy="open"` and include `"*"` in the channel allowlist (`allowFrom` / `channels.discord.dm.allowFrom` / `channels.slack.dm.allowFrom`). - -Run `openclaw doctor` to surface risky/misconfigured DM policies. - -## Highlights - -- **[Local-first Gateway](https://docs.openclaw.ai/gateway)** — single control plane for sessions, channels, tools, and events. -- **[Multi-channel inbox](https://docs.openclaw.ai/channels)** — WhatsApp, Telegram, Slack, Discord, Google Chat, Signal, BlueBubbles (iMessage), iMessage (legacy), Microsoft Teams, Matrix, Zalo, Zalo Personal, WebChat, macOS, iOS/Android. -- **[Multi-agent routing](https://docs.openclaw.ai/gateway/configuration)** — route inbound channels/accounts/peers to isolated agents (workspaces + per-agent sessions). -- **[Voice Wake](https://docs.openclaw.ai/nodes/voicewake) + [Talk Mode](https://docs.openclaw.ai/nodes/talk)** — always-on speech for macOS/iOS/Android with ElevenLabs. -- **[Live Canvas](https://docs.openclaw.ai/platforms/mac/canvas)** — agent-driven visual workspace with [A2UI](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui). -- **[First-class tools](https://docs.openclaw.ai/tools)** — browser, canvas, nodes, cron, sessions, and Discord/Slack actions. -- **[Companion apps](https://docs.openclaw.ai/platforms/macos)** — macOS menu bar app + iOS/Android [nodes](https://docs.openclaw.ai/nodes). -- **[Onboarding](https://docs.openclaw.ai/start/wizard) + [skills](https://docs.openclaw.ai/tools/skills)** — wizard-driven setup with bundled/managed/workspace skills. - -### Core platform - -- [Gateway WS control plane](https://docs.openclaw.ai/gateway) with sessions, presence, config, cron, webhooks, [Control UI](https://docs.openclaw.ai/web), and [Canvas host](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui). -- [CLI surface](https://docs.openclaw.ai/tools/agent-send): gateway, agent, send, [wizard](https://docs.openclaw.ai/start/wizard), and [doctor](https://docs.openclaw.ai/gateway/doctor). -- **Dual LLM Engine** (this fork): AI SDK v6 (default) or Pi agent runtime, switchable via config. -- **[Vercel AI SDK v6](https://ai-sdk.dev/)** integration: `streamText()`, provider-specific options (thinking, reasoning, effort), AI Gateway support. -- [Pi agent runtime](https://docs.openclaw.ai/concepts/agent) in RPC mode with tool streaming and block streaming (fallback engine). -- [Session model](https://docs.openclaw.ai/concepts/session): `main` for direct chats, group isolation, activation modes, queue modes, reply-back. Group rules: [Groups](https://docs.openclaw.ai/concepts/groups). -- [Media pipeline](https://docs.openclaw.ai/nodes/images): images/audio/video, transcription hooks, size caps, temp file lifecycle. Audio details: [Audio](https://docs.openclaw.ai/nodes/audio). - -### Channels - -- [Channels](https://docs.openclaw.ai/channels): [WhatsApp](https://docs.openclaw.ai/channels/whatsapp) (Baileys), [Telegram](https://docs.openclaw.ai/channels/telegram) (grammY), [Slack](https://docs.openclaw.ai/channels/slack) (Bolt), [Discord](https://docs.openclaw.ai/channels/discord) (discord.js), [Google Chat](https://docs.openclaw.ai/channels/googlechat) (Chat API), [Signal](https://docs.openclaw.ai/channels/signal) (signal-cli), [BlueBubbles](https://docs.openclaw.ai/channels/bluebubbles) (iMessage, recommended), [iMessage](https://docs.openclaw.ai/channels/imessage) (legacy imsg), [Microsoft Teams](https://docs.openclaw.ai/channels/msteams) (extension), [Matrix](https://docs.openclaw.ai/channels/matrix) (extension), [Zalo](https://docs.openclaw.ai/channels/zalo) (extension), [Zalo Personal](https://docs.openclaw.ai/channels/zalouser) (extension), [WebChat](https://docs.openclaw.ai/web/webchat). -- [Group routing](https://docs.openclaw.ai/concepts/group-messages): mention gating, reply tags, per-channel chunking and routing. Channel rules: [Channels](https://docs.openclaw.ai/channels). - -### Apps + nodes - -- [macOS app](https://docs.openclaw.ai/platforms/macos): menu bar control plane, [Voice Wake](https://docs.openclaw.ai/nodes/voicewake)/PTT, [Talk Mode](https://docs.openclaw.ai/nodes/talk) overlay, [WebChat](https://docs.openclaw.ai/web/webchat), debug tools, [remote gateway](https://docs.openclaw.ai/gateway/remote) control. -- [iOS node](https://docs.openclaw.ai/platforms/ios): [Canvas](https://docs.openclaw.ai/platforms/mac/canvas), [Voice Wake](https://docs.openclaw.ai/nodes/voicewake), [Talk Mode](https://docs.openclaw.ai/nodes/talk), camera, screen recording, Bonjour pairing. -- [Android node](https://docs.openclaw.ai/platforms/android): [Canvas](https://docs.openclaw.ai/platforms/mac/canvas), [Talk Mode](https://docs.openclaw.ai/nodes/talk), camera, screen recording, optional SMS. -- [macOS node mode](https://docs.openclaw.ai/nodes): system.run/notify + canvas/camera exposure. - -### Tools + automation - -- [Browser control](https://docs.openclaw.ai/tools/browser): dedicated openclaw Chrome/Chromium, snapshots, actions, uploads, profiles. -- [Canvas](https://docs.openclaw.ai/platforms/mac/canvas): [A2UI](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui) push/reset, eval, snapshot. -- [Nodes](https://docs.openclaw.ai/nodes): camera snap/clip, screen record, [location.get](https://docs.openclaw.ai/nodes/location-command), notifications. -- [Cron + wakeups](https://docs.openclaw.ai/automation/cron-jobs); [webhooks](https://docs.openclaw.ai/automation/webhook); [Gmail Pub/Sub](https://docs.openclaw.ai/automation/gmail-pubsub). -- [Skills platform](https://docs.openclaw.ai/tools/skills): bundled, managed, and workspace skills with install gating + UI. - -### Runtime + safety - -- [Channel routing](https://docs.openclaw.ai/concepts/channel-routing), [retry policy](https://docs.openclaw.ai/concepts/retry), and [streaming/chunking](https://docs.openclaw.ai/concepts/streaming). -- [Presence](https://docs.openclaw.ai/concepts/presence), [typing indicators](https://docs.openclaw.ai/concepts/typing-indicators), and [usage tracking](https://docs.openclaw.ai/concepts/usage-tracking). -- [Models](https://docs.openclaw.ai/concepts/models), [model failover](https://docs.openclaw.ai/concepts/model-failover), and [session pruning](https://docs.openclaw.ai/concepts/session-pruning). -- [Security](https://docs.openclaw.ai/gateway/security) and [troubleshooting](https://docs.openclaw.ai/channels/troubleshooting). - -### Ops + packaging - -- [Control UI](https://docs.openclaw.ai/web) + [WebChat](https://docs.openclaw.ai/web/webchat) served directly from the Gateway. -- [Tailscale Serve/Funnel](https://docs.openclaw.ai/gateway/tailscale) or [SSH tunnels](https://docs.openclaw.ai/gateway/remote) with token/password auth. -- [Nix mode](https://docs.openclaw.ai/install/nix) for declarative config; [Docker](https://docs.openclaw.ai/install/docker)-based installs. -- [Doctor](https://docs.openclaw.ai/gateway/doctor) migrations, [logging](https://docs.openclaw.ai/logging). - -## How it works (short) - -``` -WhatsApp / Telegram / Slack / Discord / Google Chat / Signal / iMessage / BlueBubbles / Microsoft Teams / Matrix / Zalo / Zalo Personal / WebChat - │ - ▼ -┌───────────────────────────────┐ -│ Gateway │ -│ (control plane) │ -│ ws://127.0.0.1:18789 │ -└──────────────┬────────────────┘ - │ - ├─ Engine Router // AI SDK v6 & Pi agent - ├─ CLI (openclaw …) - ├─ WebChat UI - ├─ macOS app - └─ iOS / Android nodes -``` - -## Key subsystems - -- **[Gateway WebSocket network](https://docs.openclaw.ai/concepts/architecture)** — single WS control plane for clients, tools, and events (plus ops: [Gateway runbook](https://docs.openclaw.ai/gateway)). -- **[Tailscale exposure](https://docs.openclaw.ai/gateway/tailscale)** — Serve/Funnel for the Gateway dashboard + WS (remote access: [Remote](https://docs.openclaw.ai/gateway/remote)). -- **[Browser control](https://docs.openclaw.ai/tools/browser)** — openclaw‑managed Chrome/Chromium with CDP control. -- **[Canvas + A2UI](https://docs.openclaw.ai/platforms/mac/canvas)** — agent‑driven visual workspace (A2UI host: [Canvas/A2UI](https://docs.openclaw.ai/platforms/mac/canvas#canvas-a2ui)). -- **[Voice Wake](https://docs.openclaw.ai/nodes/voicewake) + [Talk Mode](https://docs.openclaw.ai/nodes/talk)** — always‑on speech and continuous conversation. -- **[Nodes](https://docs.openclaw.ai/nodes)** — Canvas, camera snap/clip, screen record, `location.get`, notifications, plus macOS‑only `system.run`/`system.notify`. - -## Tailscale access (Gateway dashboard) - -OpenClaw can auto-configure Tailscale **Serve** (tailnet-only) or **Funnel** (public) while the Gateway stays bound to loopback. Configure `gateway.tailscale.mode`: - -- `off`: no Tailscale automation (default). -- `serve`: tailnet-only HTTPS via `tailscale serve` (uses Tailscale identity headers by default). -- `funnel`: public HTTPS via `tailscale funnel` (requires shared password auth). - -Notes: - -- `gateway.bind` must stay `loopback` when Serve/Funnel is enabled (OpenClaw enforces this). -- Serve can be forced to require a password by setting `gateway.auth.mode: "password"` or `gateway.auth.allowTailscale: false`. -- Funnel refuses to start unless `gateway.auth.mode: "password"` is set. -- Optional: `gateway.tailscale.resetOnExit` to undo Serve/Funnel on shutdown. - -Details: [Tailscale guide](https://docs.openclaw.ai/gateway/tailscale) · [Web surfaces](https://docs.openclaw.ai/web) - -## Remote Gateway (Linux is great) - -It’s perfectly fine to run the Gateway on a small Linux instance. Clients (macOS app, CLI, WebChat) can connect over **Tailscale Serve/Funnel** or **SSH tunnels**, and you can still pair device nodes (macOS/iOS/Android) to execute device‑local actions when needed. - -- **Gateway host** runs the exec tool and channel connections by default. -- **Device nodes** run device‑local actions (`system.run`, camera, screen recording, notifications) via `node.invoke`. - In short: exec runs where the Gateway lives; device actions run where the device lives. - -Details: [Remote access](https://docs.openclaw.ai/gateway/remote) · [Nodes](https://docs.openclaw.ai/nodes) · [Security](https://docs.openclaw.ai/gateway/security) - -## macOS permissions via the Gateway protocol - -The macOS app can run in **node mode** and advertises its capabilities + permission map over the Gateway WebSocket (`node.list` / `node.describe`). Clients can then execute local actions via `node.invoke`: - -- `system.run` runs a local command and returns stdout/stderr/exit code; set `needsScreenRecording: true` to require screen-recording permission (otherwise you’ll get `PERMISSION_MISSING`). -- `system.notify` posts a user notification and fails if notifications are denied. -- `canvas.*`, `camera.*`, `screen.record`, and `location.get` are also routed via `node.invoke` and follow TCC permission status. - -Elevated bash (host permissions) is separate from macOS TCC: - -- Use `/elevated on|off` to toggle per‑session elevated access when enabled + allowlisted. -- Gateway persists the per‑session toggle via `sessions.patch` (WS method) alongside `thinkingLevel`, `verboseLevel`, `model`, `sendPolicy`, and `groupActivation`. - -Details: [Nodes](https://docs.openclaw.ai/nodes) · [macOS app](https://docs.openclaw.ai/platforms/macos) · [Gateway protocol](https://docs.openclaw.ai/concepts/architecture) - -## Agent to Agent (sessions\_\* tools) - -- Use these to coordinate work across sessions without jumping between chat surfaces. -- `sessions_list` — discover active sessions (agents) and their metadata. -- `sessions_history` — fetch transcript logs for a session. -- `sessions_send` — message another session; optional reply‑back ping‑pong + announce step (`REPLY_SKIP`, `ANNOUNCE_SKIP`). - -Details: [Session tools](https://docs.openclaw.ai/concepts/session-tool) - -## Skills registry (ClawHub) - -ClawHub is a minimal skill registry. With ClawHub enabled, the agent can search for skills automatically and pull in new ones as needed. - -[ClawHub](https://clawhub.com) - -## Chat commands - -Send these in WhatsApp/Telegram/Slack/Google Chat/Microsoft Teams/WebChat (group commands are owner-only): - -- `/status` — compact session status (model + tokens, cost when available) -- `/new` or `/reset` — reset the session -- `/compact` — compact session context (summary) -- `/think ` — off|minimal|low|medium|high|xhigh (GPT-5.2 + Codex models only) -- `/verbose on|off` -- `/usage off|tokens|full` — per-response usage footer -- `/restart` — restart the gateway (owner-only in groups) -- `/activation mention|always` — group activation toggle (groups only) - -## Apps (optional) - -The Gateway alone delivers a great experience. All apps are optional and add extra features. - -If you plan to build/run companion apps, follow the platform runbooks below. - -### macOS (OpenClaw.app) (optional) - -- Menu bar control for the Gateway and health. -- Voice Wake + push-to-talk overlay. -- WebChat + debug tools. -- Remote gateway control over SSH. - -Note: signed builds required for macOS permissions to stick across rebuilds (see `docs/mac/permissions.md`). - -### iOS node (optional) - -- Pairs as a node via the Bridge. -- Voice trigger forwarding + Canvas surface. -- Controlled via `openclaw nodes …`. - -Runbook: [iOS connect](https://docs.openclaw.ai/platforms/ios). - -### Android node (optional) - -- Pairs via the same Bridge + pairing flow as iOS. -- Exposes Canvas, Camera, and Screen capture commands. -- Runbook: [Android connect](https://docs.openclaw.ai/platforms/android). - -## Agent workspace + skills - -- Workspace root: `~/.openclaw/workspace` (configurable via `agents.defaults.workspace`). -- Injected prompt files: `AGENTS.md`, `SOUL.md`, `TOOLS.md`. -- Skills: `~/.openclaw/workspace/skills//SKILL.md`. - ## Configuration -Minimal `~/.openclaw/openclaw.json` (model + defaults): +Ironclaw stores its config at `~/.openclaw/openclaw.json`. Minimal example: ```json5 { @@ -400,237 +93,117 @@ Minimal `~/.openclaw/openclaw.json` (model + defaults): } ``` -[Full configuration reference (all keys + examples).](https://docs.openclaw.ai/gateway/configuration) +### Supported providers -## Security model (important) +| Provider | Environment Variable | Models | +| ---------- | ------------------------------ | --------------------------------- | +| Anthropic | `ANTHROPIC_API_KEY` | Claude 4/3.x, Opus, Sonnet, Haiku | +| OpenAI | `OPENAI_API_KEY` | GPT-4o, GPT-4, o1, o3 | +| Google | `GOOGLE_GENERATIVE_AI_API_KEY` | Gemini 2.x, 1.5 Pro | +| OpenRouter | `OPENROUTER_API_KEY` | 100+ models | +| Groq | `GROQ_API_KEY` | Llama, Mixtral | +| Mistral | `MISTRAL_API_KEY` | Mistral models | +| xAI | `XAI_API_KEY` | Grok models | +| Azure | `AZURE_OPENAI_API_KEY` | Azure OpenAI models | -- **Default:** tools run on the host for the **main** session, so the agent has full access when it’s just you. -- **Group/channel safety:** set `agents.defaults.sandbox.mode: "non-main"` to run **non‑main sessions** (groups/channels) inside per‑session Docker sandboxes; bash then runs in Docker for those sessions. -- **Sandbox defaults:** allowlist `bash`, `process`, `read`, `write`, `edit`, `sessions_list`, `sessions_history`, `sessions_send`, `sessions_spawn`; denylist `browser`, `canvas`, `nodes`, `cron`, `discord`, `gateway`. +### Thinking / reasoning -Details: [Security guide](https://docs.openclaw.ai/gateway/security) · [Docker + sandboxing](https://docs.openclaw.ai/install/docker) · [Sandbox config](https://docs.openclaw.ai/gateway/configuration) +```bash +# Set thinking level (maps to AI SDK budgetTokens) +ironclaw agent --message "Complex analysis" --thinking high -### [WhatsApp](https://docs.openclaw.ai/channels/whatsapp) - -- Link the device: `pnpm openclaw channels login` (stores creds in `~/.openclaw/credentials`). -- Allowlist who can talk to the assistant via `channels.whatsapp.allowFrom`. -- If `channels.whatsapp.groups` is set, it becomes a group allowlist; include `"*"` to allow all. - -### [Telegram](https://docs.openclaw.ai/channels/telegram) - -- Set `TELEGRAM_BOT_TOKEN` or `channels.telegram.botToken` (env wins). -- Optional: set `channels.telegram.groups` (with `channels.telegram.groups."*".requireMention`); when set, it is a group allowlist (include `"*"` to allow all). Also `channels.telegram.allowFrom` or `channels.telegram.webhookUrl` + `channels.telegram.webhookSecret` as needed. - -```json5 -{ - channels: { - telegram: { - botToken: "123456:ABCDEF", - }, - }, -} +# Levels: off, minimal, low, medium, high, xhigh ``` -### [Slack](https://docs.openclaw.ai/channels/slack) +## Channel setup -- Set `SLACK_BOT_TOKEN` + `SLACK_APP_TOKEN` (or `channels.slack.botToken` + `channels.slack.appToken`). +Each channel is configured in `~/.openclaw/openclaw.json` under `channels.*`: -### [Discord](https://docs.openclaw.ai/channels/discord) +- **WhatsApp** -- Link via `ironclaw channels login`. Set `channels.whatsapp.allowFrom`. +- **Telegram** -- Set `TELEGRAM_BOT_TOKEN` or `channels.telegram.botToken`. +- **Slack** -- Set `SLACK_BOT_TOKEN` + `SLACK_APP_TOKEN`. +- **Discord** -- Set `DISCORD_BOT_TOKEN` or `channels.discord.token`. +- **Signal** -- Requires `signal-cli` + `channels.signal` config. +- **iMessage** -- Via BlueBubbles (recommended) or legacy macOS-only integration. +- **Microsoft Teams** -- Configure a Teams app + Bot Framework. +- **WebChat** -- Uses the Gateway WebSocket directly, no extra config. -- Set `DISCORD_BOT_TOKEN` or `channels.discord.token` (env wins). -- Optional: set `commands.native`, `commands.text`, or `commands.useAccessGroups`, plus `channels.discord.dm.allowFrom`, `channels.discord.guilds`, or `channels.discord.mediaMaxMb` as needed. +## Chat commands -```json5 -{ - channels: { - discord: { - token: "1234abcd", - }, - }, -} +Send these in any connected channel: + +| Command | Description | +| ----------------------------- | ------------------------------- | +| `/status` | Session status (model + tokens) | +| `/new` or `/reset` | Reset the session | +| `/compact` | Compact session context | +| `/think ` | Set thinking level | +| `/verbose on\|off` | Toggle verbose output | +| `/usage off\|tokens\|full` | Per-response usage footer | +| `/restart` | Restart the gateway | +| `/activation mention\|always` | Group activation toggle | + +## Architecture + +``` +WhatsApp / Telegram / Slack / Discord / Signal / iMessage / Teams / WebChat + | + v + +----------------------------+ + | Gateway | + | (control plane) | + | ws://127.0.0.1:18789 | + +-------------+--------------+ + | + +-- Vercel AI SDK v6 engine + +-- CLI (ironclaw ...) + +-- Web UI (Dench) + +-- macOS app + +-- iOS / Android nodes ``` -### [Signal](https://docs.openclaw.ai/channels/signal) +## Project structure -- Requires `signal-cli` and a `channels.signal` config section. - -### [BlueBubbles (iMessage)](https://docs.openclaw.ai/channels/bluebubbles) - -- **Recommended** iMessage integration. -- Configure `channels.bluebubbles.serverUrl` + `channels.bluebubbles.password` and a webhook (`channels.bluebubbles.webhookPath`). -- The BlueBubbles server runs on macOS; the Gateway can run on macOS or elsewhere. - -### [iMessage (legacy)](https://docs.openclaw.ai/channels/imessage) - -- Legacy macOS-only integration via `imsg` (Messages must be signed in). -- If `channels.imessage.groups` is set, it becomes a group allowlist; include `"*"` to allow all. - -### [Microsoft Teams](https://docs.openclaw.ai/channels/msteams) - -- Configure a Teams app + Bot Framework, then add a `msteams` config section. -- Allowlist who can talk via `msteams.allowFrom`; group access via `msteams.groupAllowFrom` or `msteams.groupPolicy: "open"`. - -### [WebChat](https://docs.openclaw.ai/web/webchat) - -- Uses the Gateway WebSocket; no separate WebChat port/config. - -Browser control (optional): - -```json5 -{ - browser: { - enabled: true, - color: "#FF4500", - }, -} +``` +src/ Core CLI, commands, gateway, agent, media pipeline +apps/web/ Next.js web UI (Dench) +apps/ios/ iOS companion node +apps/android/ Android companion node +apps/macos/ macOS menu bar app +extensions/ Channel plugins (MS Teams, Matrix, Zalo, voice-call) +docs/ Documentation +scripts/ Build, deploy, and utility scripts +skills/ Workspace skills ``` -## Docs +## Development -Use these when you’re past the onboarding flow and want the deeper reference. +```bash +pnpm install # Install deps +pnpm build # Type-check + build +pnpm check # Lint + format check +pnpm test # Run tests (vitest) +pnpm test:coverage # Tests with coverage +pnpm dev # Dev mode (auto-reload) +``` -- [Start with the docs index for navigation and “what’s where.”](https://docs.openclaw.ai) -- [Read the architecture overview for the gateway + protocol model.](https://docs.openclaw.ai/concepts/architecture) -- [Use the full configuration reference when you need every key and example.](https://docs.openclaw.ai/gateway/configuration) -- [Run the Gateway by the book with the operational runbook.](https://docs.openclaw.ai/gateway) -- [Learn how the Control UI/Web surfaces work and how to expose them safely.](https://docs.openclaw.ai/web) -- [Understand remote access over SSH tunnels or tailnets.](https://docs.openclaw.ai/gateway/remote) -- [Follow the onboarding wizard flow for a guided setup.](https://docs.openclaw.ai/start/wizard) -- [Wire external triggers via the webhook surface.](https://docs.openclaw.ai/automation/webhook) -- [Set up Gmail Pub/Sub triggers.](https://docs.openclaw.ai/automation/gmail-pubsub) -- [Learn the macOS menu bar companion details.](https://docs.openclaw.ai/platforms/mac/menu-bar) -- [Platform guides: Windows (WSL2)](https://docs.openclaw.ai/platforms/windows), [Linux](https://docs.openclaw.ai/platforms/linux), [macOS](https://docs.openclaw.ai/platforms/macos), [iOS](https://docs.openclaw.ai/platforms/ios), [Android](https://docs.openclaw.ai/platforms/android) -- [Debug common failures with the troubleshooting guide.](https://docs.openclaw.ai/channels/troubleshooting) -- [Review security guidance before exposing anything.](https://docs.openclaw.ai/gateway/security) +## Security -## Advanced docs (discovery + control) +- DM pairing is enabled by default -- unknown senders receive a pairing code. +- Approve senders with `ironclaw pairing approve `. +- Non-main sessions can be sandboxed in Docker (`agents.defaults.sandbox.mode: "non-main"`). +- Run `ironclaw doctor` to surface risky or misconfigured DM policies. -- [Discovery + transports](https://docs.openclaw.ai/gateway/discovery) -- [Bonjour/mDNS](https://docs.openclaw.ai/gateway/bonjour) -- [Gateway pairing](https://docs.openclaw.ai/gateway/pairing) -- [Remote gateway README](https://docs.openclaw.ai/gateway/remote-gateway-readme) -- [Control UI](https://docs.openclaw.ai/web/control-ui) -- [Dashboard](https://docs.openclaw.ai/web/dashboard) +## Upstream -## Operations & troubleshooting +Ironclaw is a fork of [OpenClaw](https://github.com/openclaw/openclaw). To sync with upstream: -- [Health checks](https://docs.openclaw.ai/gateway/health) -- [Gateway lock](https://docs.openclaw.ai/gateway/gateway-lock) -- [Background process](https://docs.openclaw.ai/gateway/background-process) -- [Browser troubleshooting (Linux)](https://docs.openclaw.ai/tools/browser-linux-troubleshooting) -- [Logging](https://docs.openclaw.ai/logging) +```bash +git remote add upstream https://github.com/openclaw/openclaw.git +git fetch upstream +git merge upstream/main +``` -## Deep dives +## License -- [Agent loop](https://docs.openclaw.ai/concepts/agent-loop) -- [Presence](https://docs.openclaw.ai/concepts/presence) -- [TypeBox schemas](https://docs.openclaw.ai/concepts/typebox) -- [RPC adapters](https://docs.openclaw.ai/reference/rpc) -- [Queue](https://docs.openclaw.ai/concepts/queue) - -## Workspace & skills - -- [Skills config](https://docs.openclaw.ai/tools/skills-config) -- [Default AGENTS](https://docs.openclaw.ai/reference/AGENTS.default) -- [Templates: AGENTS](https://docs.openclaw.ai/reference/templates/AGENTS) -- [Templates: BOOTSTRAP](https://docs.openclaw.ai/reference/templates/BOOTSTRAP) -- [Templates: IDENTITY](https://docs.openclaw.ai/reference/templates/IDENTITY) -- [Templates: SOUL](https://docs.openclaw.ai/reference/templates/SOUL) -- [Templates: TOOLS](https://docs.openclaw.ai/reference/templates/TOOLS) -- [Templates: USER](https://docs.openclaw.ai/reference/templates/USER) - -## Platform internals - -- [macOS dev setup](https://docs.openclaw.ai/platforms/mac/dev-setup) -- [macOS menu bar](https://docs.openclaw.ai/platforms/mac/menu-bar) -- [macOS voice wake](https://docs.openclaw.ai/platforms/mac/voicewake) -- [iOS node](https://docs.openclaw.ai/platforms/ios) -- [Android node](https://docs.openclaw.ai/platforms/android) -- [Windows (WSL2)](https://docs.openclaw.ai/platforms/windows) -- [Linux app](https://docs.openclaw.ai/platforms/linux) - -## Email hooks (Gmail) - -- [docs.openclaw.ai/gmail-pubsub](https://docs.openclaw.ai/automation/gmail-pubsub) - -## About This Fork - -This AI SDK fork was created by [Kumar Abhirup](https://github.com/kumarabhirup) to bring Vercel AI SDK compatibility to OpenClaw, making it easier for developers in the Vercel/Next.js ecosystem to integrate and extend. - -**Fork features:** - -- Vercel AI SDK v6 as the default LLM orchestration layer -- Full Anthropic thinking/reasoning support via provider options -- AI Gateway support for unified provider access -- Fork-friendly architecture that minimizes upstream merge conflicts -- Dual engine support (AI SDK + pi-agent) for flexibility - -## Molty - -OpenClaw was built for **Molty**, a space lobster AI assistant. 🦞 -by Peter Steinberger and the community. - -- [openclaw.ai](https://openclaw.ai) -- [soul.md](https://soul.md) -- [steipete.me](https://steipete.me) -- [@openclaw](https://x.com/openclaw) - -## Community - -See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines, maintainers, and how to submit PRs. -AI/vibe-coded PRs welcome! 🤖 - -Special thanks to [Mario Zechner](https://mariozechner.at/) for his support and for -[pi-mono](https://github.com/badlogic/pi-mono). -Special thanks to Adam Doppelt for lobster.bot. -Special thanks to [Vercel](https://vercel.com) for the [AI SDK](https://ai-sdk.dev/). - -Thanks to all clawtributors: - -

- steipete joshp123 cpojer Mariano Belinky plum-dawg bohdanpodvirnyi sebslight iHildy jaydenfyi joaohlisboa - mneves75 MatthieuBizien Glucksberg MaudeBot gumadeiras tyler6204 rahthakor vrknetha vignesh07 radek-paclt - abdelsfane Tobias Bischoff christianklotz czekaj ethanpalm mukhtharcm maxsumrall xadenryan VACInc rodrigouroz - juanpablodlc conroywhitney hsrvc magimetal zerone0x Takhoffman meaningfool mudrii patelhiren NicholasSpisak - jonisjongithub abhisekbasu1 jamesgroat BunsDev claude JustYannicc Hyaxia dantelex SocialNerd42069 daveonkels - google-labs-jules[bot] lc0rp adam91holt mousberg hougangdev shakkernerd coygeek mteam88 hirefrank M00N7682 - joeynyc orlyjamie dbhurley Eng. Juan Combetto TSavo aerolalit julianengel bradleypriest benithors lsh411 - gut-puncture rohannagpal timolins f-trycua benostein elliotsecops nachx639 pvoo sreekaransrinath gupsammy - cristip73 stefangalescu nachoiacovino Vasanth Rao Naik Sabavat petter-b thewilloftheshadow leszekszpunar scald pycckuu andranik-sahakyan - davidguttman sleontenko denysvitali clawdinator[bot] TinyTb sircrumpet peschee nicolasstanley davidiach nonggialiang - ironbyte-rgb rafaelreis-r dominicnunez lploc94 ratulsarna sfo2001 lutr0 kiranjd danielz1z Iranb - AdeboyeDN Alg0rix obviyus papago2355 emanuelst evanotero KristijanJovanovski jlowin rdev rhuanssauro - joshrad-dev osolmaz adityashaw2 CashWilliams sheeek ryancontent jasonsschin artuskg onutc pauloportella - HirokiKobayashi-R ThanhNguyxn 18-RAJAT kimitaka yuting0624 neooriginal manuelhettich minghinmatthewlam unisone baccula - manikv12 myfunc travisirby fujiwara-tofu-shop buddyh connorshea bjesuiter kyleok slonce70 mcinteerj - badlogic dependabot[bot] amitbiswal007 John-Rood timkrase uos-status gerardward2007 roshanasingh4 tosh-hamburg azade-c - dlauer grp06 JonUleis shivamraut101 cheeeee robbyczgw-cla YuriNachos Josh Phillips Wangnov kaizen403 - pookNast Whoaa512 chriseidhof ngutman therealZpoint-bot wangai-studio ysqander Yurii Chukhlib aj47 kennyklee - superman32432432 Hisleren shatner antons austinm911 blacksmith-sh[bot] damoahdominic dan-dr GHesericsu HeimdallStrategy - imfing jalehman jarvis-medmatic kkarimi Lukavyi mahmoudashraf93 pkrmf RandyVentures robhparker Ryan Lisse - Yeom-JinHo doodlewind dougvk erikpr1994 fal3 Ghost hyf0-agent jonasjancarik Keith the Silly Goose L36 Server - Marc mitschabaude-bot mkbehr neist sibbl zats abhijeet117 chrisrodz Friederike Seiler gabriel-trigo - iamadig itsjling Jonathan D. Rhyne (DJ-D) Joshua Mitchell kelvinCB Kit koala73 manmal mattqdev mitsuhiko - ogulcancelik pasogott petradonka rubyrunsstuff siddhantjain spiceoogway suminhthanh svkozak wes-davis 24601 - ameno- bonald bravostation Chris Taylor dguido Django Navarro evalexpr henrino3 humanwritten j2h4u - larlyssa odysseus0 oswalpalash pcty-nextgen-service-account pi0 rmorse Roopak Nijhara Syhids Ubuntu xiaose - Aaron Konyer aaronveklabs aldoeliacim andreabadesso Andrii BinaryMuse bqcfjwhz85-arch cash-echo-bot Clawd ClawdFx - damaozi danballance Elarwei001 EnzeD erik-agens Evizero fcatuhe gildo hclsys itsjaydesu - ivancasco ivanrvpereira Jarvis jayhickey jeffersonwarrior jeffersonwarrior jverdi lailoo longmaba Marco Marandiz - MarvinCui mattezell mjrussell odnxe optimikelabs p6l-richard philipp-spiess Pocket Clawd robaxelsen Sash Catanzarite - Suksham-sharma T5-AndyML tewatia thejhinvirtuoso travisp VAC william arzt yudshj zknicker 0oAstro - abhaymundhara aduk059 aisling404 akramcodez alejandro maza Alex-Alaniz alexanderatallah alexstyl AlexZhangji andrewting19 - anpoirier araa47 arthyn Asleep123 Ayush Ojha Ayush10 bguidolim bolismauro caelum0x championswimmer - chenyuan99 Chloe-VP Clawdbot Maintainers conhecendoia dasilva333 David-Marsh-Photo deepsoumya617 Developer Dimitrios Ploutarchos Drake Thomsen - dvrshil dxd5001 dylanneve1 Felix Krause foeken frankekn fredheir ganghyun kim grrowl gtsifrikas - HassanFleyah HazAT hrdwdmrbl hugobarauna iamEvanYT ichbinlucaskim Jamie Openshaw Jane Jarvis Deploy Jefferson Nunn - jogi47 kentaro Kevin Lin kira-ariaki kitze Kiwitwitter levifig Lloyd loganaden longjos - loukotal louzhixian mac mimi martinpucik Matt mini mcaxtr mertcicekci0 Miles mrdbstn MSch - Mustafa Tag Eldeen mylukin nathanbosse ndraiman nexty5870 Noctivoro Omar-Khaleel ozgur-polat ppamment prathamdby - ptn1411 rafelbev reeltimeapps RLTCmpe Rony Kelner ryancnelson Samrat Jha senoldogann Seredeep sergical - shiv19 shiyuanhai Shrinija17 siraht snopoke stephenchen2025 techboss testingabc321 The Admiral thesash - Vibe Kanban vincentkoc voidserf Vultr-Clawd Admin Wimmie wolfred wstock wytheme YangHuang2280 yazinsai - yevhen YiWang24 ymat19 Zach Knickerbocker zackerthescar 0xJonHoldsCrypto aaronn Alphonse-arianee atalovesyou Azade - carlulsoe ddyo Erik jiulingyun latitudeki5223 Manuel Maly Mourad Boustani odrobnik pcty-nextgen-ios-builder Quentin - Randy Torres rhjoh Rolf Fredheim ronak-guliani William Stock -

+[MIT](LICENSE) diff --git a/SECURITY.md b/SECURITY.md index ec0ff9f30cf..c3db26fa650 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,9 +4,31 @@ If you believe you've found a security issue in OpenClaw, please report it priva ## Reporting -For full reporting instructions - including which repo to report to and how - see our [Trust page](https://trust.openclaw.ai). +Report vulnerabilities directly to the repository where the issue lives: -Include: reproduction steps, impact assessment, and (if possible) a minimal PoC. +- **Core CLI and gateway** — [openclaw/openclaw](https://github.com/openclaw/openclaw) +- **macOS desktop app** — [openclaw/openclaw](https://github.com/openclaw/openclaw) (apps/macos) +- **iOS app** — [openclaw/openclaw](https://github.com/openclaw/openclaw) (apps/ios) +- **Android app** — [openclaw/openclaw](https://github.com/openclaw/openclaw) (apps/android) +- **ClawHub** — [openclaw/clawhub](https://github.com/openclaw/clawhub) +- **Trust and threat model** — [openclaw/trust](https://github.com/openclaw/trust) + +For issues that don't fit a specific repo, or if you're unsure, email **security@openclaw.ai** and we'll route it. + +For full reporting instructions see our [Trust page](https://trust.openclaw.ai). + +### Required in Reports + +1. **Title** +2. **Severity Assessment** +3. **Impact** +4. **Affected Component** +5. **Technical Reproduction** +6. **Demonstrated Impact** +7. **Environment** +8. **Remediation Advice** + +Reports without reproduction steps, demonstrated impact, and remediation advice will be deprioritized. Given the volume of AI-generated scanner findings, we must ensure we're receiving vetted reports from researchers who understand the issues. ## Security & Trust diff --git a/appcast.xml b/appcast.xml index fc08573d4f9..cacb573c21c 100644 --- a/appcast.xml +++ b/appcast.xml @@ -2,6 +2,62 @@ OpenClaw + + 2026.2.9 + Mon, 09 Feb 2026 13:23:25 -0600 + https://raw.githubusercontent.com/openclaw/openclaw/main/appcast.xml + 9194 + 2026.2.9 + 15.0 + OpenClaw 2026.2.9 +

Added

+
    +
  • iOS: alpha node app + setup-code onboarding. (#11756) Thanks @mbelinky.
  • +
  • Channels: comprehensive BlueBubbles and channel cleanup. (#11093) Thanks @tyler6204.
  • +
  • Plugins: device pairing + phone control plugins (Telegram /pair, iOS/Android node controls). (#11755) Thanks @mbelinky.
  • +
  • Tools: add Grok (xAI) as a web_search provider. (#12419) Thanks @tmchow.
  • +
  • Gateway: add agent management RPC methods for the web UI (agents.create, agents.update, agents.delete). (#11045) Thanks @advaitpaliwal.
  • +
  • Web UI: show a Compaction divider in chat history. (#11341) Thanks @Takhoffman.
  • +
  • Agents: include runtime shell in agent envelopes. (#1835) Thanks @Takhoffman.
  • +
  • Paths: add OPENCLAW_HOME for overriding the home directory used by internal path resolution. (#12091) Thanks @sebslight.
  • +
+

Fixes

+
    +
  • Telegram: harden quote parsing; preserve quote context; avoid QUOTE_TEXT_INVALID; avoid nested reply quote misclassification. (#12156) Thanks @rybnikov.
  • +
  • Telegram: recover proactive sends when stale topic thread IDs are used by retrying without message_thread_id. (#11620)
  • +
  • Telegram: render markdown spoilers with HTML tags. (#11543) Thanks @ezhikkk.
  • +
  • Telegram: truncate command registration to 100 entries to avoid BOT_COMMANDS_TOO_MUCH failures on startup. (#12356) Thanks @arosstale.
  • +
  • Telegram: match DM allowFrom against sender user id (fallback to chat id) and clarify pairing logs. (#12779) Thanks @liuxiaopai-ai.
  • +
  • Onboarding: QuickStart now auto-installs shell completion (prompt only in Manual).
  • +
  • Auth: strip embedded line breaks from pasted API keys and tokens before storing/resolving credentials.
  • +
  • Web UI: make chat refresh smoothly scroll to the latest messages and suppress new-messages badge flash during manual refresh.
  • +
  • Tools/web_search: include provider-specific settings in the web search cache key, and pass inlineCitations for Grok. (#12419) Thanks @tmchow.
  • +
  • Tools/web_search: normalize direct Perplexity model IDs while keeping OpenRouter model IDs unchanged. (#12795) Thanks @cdorsey.
  • +
  • Model failover: treat HTTP 400 errors as failover-eligible, enabling automatic model fallback. (#1879) Thanks @orenyomtov.
  • +
  • Errors: prevent false positive context overflow detection when conversation mentions "context overflow" topic. (#2078) Thanks @sbking.
  • +
  • Gateway: no more post-compaction amnesia; injected transcript writes now preserve Pi session parentId chain so agents can remember again. (#12283) Thanks @Takhoffman.
  • +
  • Gateway: fix multi-agent sessions.usage discovery. (#11523) Thanks @Takhoffman.
  • +
  • Agents: recover from context overflow caused by oversized tool results (pre-emptive capping + fallback truncation). (#11579) Thanks @tyler6204.
  • +
  • Subagents/compaction: stabilize announce timing and preserve compaction metrics across retries. (#11664) Thanks @tyler6204.
  • +
  • Cron: share isolated announce flow and harden scheduling/delivery reliability. (#11641) Thanks @tyler6204.
  • +
  • Cron tool: recover flat params when LLM omits the job wrapper for add requests. (#12124) Thanks @tyler6204.
  • +
  • Gateway/CLI: when gateway.bind=lan, use a LAN IP for probe URLs and Control UI links. (#11448) Thanks @AnonO6.
  • +
  • Hooks: fix bundled hooks broken since 2026.2.2 (tsdown migration). (#9295) Thanks @patrickshao.
  • +
  • Routing: refresh bindings per message by loading config at route resolution so binding changes apply without restart. (#11372) Thanks @juanpablodlc.
  • +
  • Exec approvals: render forwarded commands in monospace for safer approval scanning. (#11937) Thanks @sebslight.
  • +
  • Config: clamp maxTokens to contextWindow to prevent invalid model configs. (#5516) Thanks @lailoo.
  • +
  • Thinking: allow xhigh for github-copilot/gpt-5.2-codex and github-copilot/gpt-5.2. (#11646) Thanks @LatencyTDH.
  • +
  • Discord: support forum/media thread-create starter messages, wire message thread create --message, and harden routing. (#10062) Thanks @jarvis89757.
  • +
  • Paths: structurally resolve OPENCLAW_HOME-derived home paths and fix Windows drive-letter handling in tool meta shortening. (#12125) Thanks @mcaxtr.
  • +
  • Memory: set Voyage embeddings input_type for improved retrieval. (#10818) Thanks @mcinteerj.
  • +
  • Memory/QMD: reuse default model cache across agents instead of re-downloading per agent. (#12114) Thanks @tyler6204.
  • +
  • Media understanding: recognize .caf audio attachments for transcription. (#10982) Thanks @succ985.
  • +
  • State dir: honor OPENCLAW_STATE_DIR for default device identity and canvas storage paths. (#4824) Thanks @kossoy.
  • +
+

View full changelog

+]]>
+ +
2026.2.3 Wed, 04 Feb 2026 17:47:10 -0800 @@ -96,71 +152,5 @@ ]]> - - 2026.2.1 - Mon, 02 Feb 2026 03:53:03 -0800 - https://raw.githubusercontent.com/openclaw/openclaw/main/appcast.xml - 8650 - 2026.2.1 - 15.0 - OpenClaw 2026.2.1 -

Changes

-
    -
  • Docs: onboarding/install/i18n/exec-approvals/Control UI/exe.dev/cacheRetention updates + misc nav/typos. (#3050, #3461, #4064, #4675, #4729, #4763, #5003, #5402, #5446, #5474, #5663, #5689, #5694, #5967, #6270, #6300, #6311, #6416, #6487, #6550, #6789)
  • -
  • Telegram: use shared pairing store. (#6127) Thanks @obviyus.
  • -
  • Agents: add OpenRouter app attribution headers. Thanks @alexanderatallah.
  • -
  • Agents: add system prompt safety guardrails. (#5445) Thanks @joshp123.
  • -
  • Agents: update pi-ai to 0.50.9 and rename cacheControlTtl -> cacheRetention (with back-compat mapping).
  • -
  • Agents: extend CreateAgentSessionOptions with systemPrompt/skills/contextFiles.
  • -
  • Agents: add tool policy conformance snapshot (no runtime behavior change). (#6011)
  • -
  • Auth: update MiniMax OAuth hint + portal auth note copy.
  • -
  • Discord: inherit thread parent bindings for routing. (#3892) Thanks @aerolalit.
  • -
  • Gateway: inject timestamps into agent and chat.send messages. (#3705) Thanks @conroywhitney, @CashWilliams.
  • -
  • Gateway: require TLS 1.3 minimum for TLS listeners. (#5970) Thanks @loganaden.
  • -
  • Web UI: refine chat layout + extend session active duration.
  • -
  • CI: add formal conformance + alias consistency checks. (#5723, #5807)
  • -
-

Fixes

-
    -
  • Plugins: validate plugin/hook install paths and reject traversal-like names.
  • -
  • Telegram: add download timeouts for file fetches. (#6914) Thanks @hclsys.
  • -
  • Telegram: enforce thread specs for DM vs forum sends. (#6833) Thanks @obviyus.
  • -
  • Streaming: flush block streaming on paragraph boundaries for newline chunking. (#7014)
  • -
  • Streaming: stabilize partial streaming filters.
  • -
  • Auto-reply: avoid referencing workspace files in /new greeting prompt. (#5706) Thanks @bravostation.
  • -
  • Tools: align tool execute adapters/signatures (legacy + parameter order + arg normalization).
  • -
  • Tools: treat "*" tool allowlist entries as valid to avoid spurious unknown-entry warnings.
  • -
  • Skills: update session-logs paths from .clawdbot to .openclaw. (#4502)
  • -
  • Slack: harden media fetch limits and Slack file URL validation. (#6639) Thanks @davidiach.
  • -
  • Lint: satisfy curly rule after import sorting. (#6310)
  • -
  • Process: resolve Windows spawn() failures for npm-family CLIs by appending .cmd when needed. (#5815) Thanks @thejhinvirtuoso.
  • -
  • Discord: resolve PluralKit proxied senders for allowlists and labels. (#5838) Thanks @thewilloftheshadow.
  • -
  • Tlon: add timeout to SSE client fetch calls (CWE-400). (#5926)
  • -
  • Memory search: L2-normalize local embedding vectors to fix semantic search. (#5332)
  • -
  • Agents: align embedded runner + typings with pi-coding-agent API updates (pi 0.51.0).
  • -
  • Agents: ensure OpenRouter attribution headers apply in the embedded runner.
  • -
  • Agents: cap context window resolution for compaction safeguard. (#6187) Thanks @iamEvanYT.
  • -
  • System prompt: resolve overrides and hint using session_status for current date/time. (#1897, #1928, #2108, #3677)
  • -
  • Agents: fix Pi prompt template argument syntax. (#6543)
  • -
  • Subagents: fix announce failover race (always emit lifecycle end; timeout=0 means no-timeout). (#6621)
  • -
  • Teams: gate media auth retries.
  • -
  • Telegram: restore draft streaming partials. (#5543) Thanks @obviyus.
  • -
  • Onboarding: friendlier Windows onboarding message. (#6242) Thanks @shanselman.
  • -
  • TUI: prevent crash when searching with digits in the model selector.
  • -
  • Agents: wire before_tool_call plugin hook into tool execution. (#6570, #6660) Thanks @ryancnelson.
  • -
  • Browser: secure Chrome extension relay CDP sessions.
  • -
  • Docker: use container port for gateway command instead of host port. (#5110) Thanks @mise42.
  • -
  • fix(lobster): block arbitrary exec via lobsterPath/cwd injection (GHSA-4mhr-g7xj-cg8j). (#5335) Thanks @vignesh07.
  • -
  • Security: sanitize WhatsApp accountId to prevent path traversal. (#4610)
  • -
  • Security: restrict MEDIA path extraction to prevent LFI. (#4930)
  • -
  • Security: validate message-tool filePath/path against sandbox root. (#6398)
  • -
  • Security: block LD*/DYLD* env overrides for host exec. (#4896) Thanks @HassanFleyah.
  • -
  • Security: harden web tool content wrapping + file parsing safeguards. (#4058) Thanks @VACInc.
  • -
  • Security: enforce Twitch allowFrom allowlist gating (deny non-allowlisted senders). Thanks @MegaManSec.
  • -
-

View full changelog

-]]>
- -
-
+ \ No newline at end of file diff --git a/apps/android/app/build.gradle.kts b/apps/android/app/build.gradle.kts index 47056143f57..60cd8961129 100644 --- a/apps/android/app/build.gradle.kts +++ b/apps/android/app/build.gradle.kts @@ -22,7 +22,7 @@ android { minSdk = 31 targetSdk = 36 versionCode = 202602030 - versionName = "2026.2.6" + versionName = "2026.2.10" } buildTypes { diff --git a/apps/ios/Sources/Info.plist b/apps/ios/Sources/Info.plist index 66c06c0dcac..4a6bc68ba71 100644 --- a/apps/ios/Sources/Info.plist +++ b/apps/ios/Sources/Info.plist @@ -17,13 +17,13 @@ CFBundleName $(PRODUCT_NAME) CFBundlePackageType - APPL - CFBundleShortVersionString - 2026.2.6 - CFBundleVersion - 20260202 - NSAppTransportSecurity - + APPL + CFBundleShortVersionString + 2026.2.10 + CFBundleVersion + 20260202 + NSAppTransportSecurity + NSAllowsArbitraryLoadsInWebContent diff --git a/apps/ios/Tests/Info.plist b/apps/ios/Tests/Info.plist index 3d40318166f..7e0ecde3697 100644 --- a/apps/ios/Tests/Info.plist +++ b/apps/ios/Tests/Info.plist @@ -15,10 +15,10 @@ CFBundleName $(PRODUCT_NAME) CFBundlePackageType - BNDL - CFBundleShortVersionString - 2026.2.6 - CFBundleVersion - 20260202 - + BNDL + CFBundleShortVersionString + 2026.2.10 + CFBundleVersion + 20260202 + diff --git a/apps/ios/project.yml b/apps/ios/project.yml index 6189ca639ce..2ff2bbfdbc3 100644 --- a/apps/ios/project.yml +++ b/apps/ios/project.yml @@ -81,7 +81,7 @@ targets: properties: CFBundleDisplayName: OpenClaw CFBundleIconName: AppIcon - CFBundleShortVersionString: "2026.2.6" + CFBundleShortVersionString: "2026.2.10" CFBundleVersion: "20260202" UILaunchScreen: {} UIApplicationSceneManifest: @@ -130,5 +130,5 @@ targets: path: Tests/Info.plist properties: CFBundleDisplayName: OpenClawTests - CFBundleShortVersionString: "2026.2.6" + CFBundleShortVersionString: "2026.2.10" CFBundleVersion: "20260202" diff --git a/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift b/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift index 7ab9a64ca62..9b6bb099341 100644 --- a/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift +++ b/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift @@ -585,34 +585,38 @@ extension MenuSessionsInjector { let item = NSMenuItem() item.tag = self.tag item.isEnabled = false - let view = AnyView(SessionMenuPreviewView( - width: width, - maxLines: maxLines, - title: title, - items: [], - status: .loading)) - let hosting = NSHostingView(rootView: view) - hosting.frame.size.width = max(1, width) - let size = hosting.fittingSize - hosting.frame = NSRect(origin: .zero, size: NSSize(width: width, height: size.height)) - item.view = hosting + let view = AnyView( + SessionMenuPreviewView( + width: width, + maxLines: maxLines, + title: title, + items: [], + status: .loading) + .environment(\.isEnabled, true)) + let hosted = HighlightedMenuItemHostView(rootView: view, width: width) + item.view = hosted - let task = Task { [weak hosting] in + let task = Task { [weak hosted, weak item] in let snapshot = await SessionMenuPreviewLoader.load(sessionKey: sessionKey, maxItems: 10) guard !Task.isCancelled else { return } + await MainActor.run { - guard let hosting else { return } - let nextView = AnyView(SessionMenuPreviewView( - width: width, - maxLines: maxLines, - title: title, - items: snapshot.items, - status: snapshot.status)) - hosting.rootView = nextView - hosting.invalidateIntrinsicContentSize() - hosting.frame.size.width = max(1, width) - let size = hosting.fittingSize - hosting.frame.size.height = size.height + let nextView = AnyView( + SessionMenuPreviewView( + width: width, + maxLines: maxLines, + title: title, + items: snapshot.items, + status: snapshot.status) + .environment(\.isEnabled, true)) + + if let item { + item.view = HighlightedMenuItemHostView(rootView: nextView, width: width) + return + } + + guard let hosted else { return } + hosted.update(rootView: nextView, width: width) } } self.previewTasks.append(task) diff --git a/apps/macos/Sources/OpenClaw/Resources/Info.plist b/apps/macos/Sources/OpenClaw/Resources/Info.plist index 067035d87e0..e933214b8af 100644 --- a/apps/macos/Sources/OpenClaw/Resources/Info.plist +++ b/apps/macos/Sources/OpenClaw/Resources/Info.plist @@ -15,7 +15,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 2026.2.6 + 2026.2.10 CFBundleVersion 202602020 CFBundleIconFile diff --git a/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift b/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift index 9932b4a15bb..898a8a31cfa 100644 --- a/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift +++ b/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift @@ -250,7 +250,8 @@ actor GatewayWizardClient { let clientId = "openclaw-macos" let clientMode = "ui" let role = "operator" - let scopes: [String] = [] + // Explicit scopes; gateway no longer defaults empty scopes to admin. + let scopes: [String] = ["operator.admin", "operator.approvals", "operator.pairing"] let client: [String: ProtoAnyCodable] = [ "id": ProtoAnyCodable(clientId), "displayName": ProtoAnyCodable(Host.current().localizedName ?? "OpenClaw macOS Wizard CLI"), diff --git a/apps/macos/Sources/OpenClawProtocol/GatewayModels.swift b/apps/macos/Sources/OpenClawProtocol/GatewayModels.swift index 9e88442266e..c82e218c641 100644 --- a/apps/macos/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/macos/Sources/OpenClawProtocol/GatewayModels.swift @@ -1,4 +1,5 @@ // Generated by scripts/protocol-gen-swift.ts — do not edit by hand +// swiftlint:disable file_length import Foundation public let GATEWAY_PROTOCOL_VERSION = 3 @@ -383,7 +384,7 @@ public struct AgentEvent: Codable, Sendable { public struct SendParams: Codable, Sendable { public let to: String - public let message: String + public let message: String? public let mediaurl: String? public let mediaurls: [String]? public let gifplayback: Bool? @@ -394,7 +395,7 @@ public struct SendParams: Codable, Sendable { public init( to: String, - message: String, + message: String?, mediaurl: String?, mediaurls: [String]?, gifplayback: Bool?, diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 9e88442266e..c82e218c641 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -1,4 +1,5 @@ // Generated by scripts/protocol-gen-swift.ts — do not edit by hand +// swiftlint:disable file_length import Foundation public let GATEWAY_PROTOCOL_VERSION = 3 @@ -383,7 +384,7 @@ public struct AgentEvent: Codable, Sendable { public struct SendParams: Codable, Sendable { public let to: String - public let message: String + public let message: String? public let mediaurl: String? public let mediaurls: [String]? public let gifplayback: Bool? @@ -394,7 +395,7 @@ public struct SendParams: Codable, Sendable { public init( to: String, - message: String, + message: String?, mediaurl: String?, mediaurls: [String]?, gifplayback: Bool?, diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index 99bc172a4cb..a1a142d9f43 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -1,5 +1,6 @@ import type { UIMessage } from "ai"; import { runAgent, type ToolResult } from "@/lib/agent-runner"; +import { resolveAgentWorkspacePrefix } from "@/lib/workspace"; // Force Node.js runtime (required for child_process) export const runtime = "nodejs"; @@ -14,9 +15,9 @@ export const maxDuration = 600; function buildToolOutput( result?: ToolResult, ): Record { - if (!result) return {}; + if (!result) {return {};} const out: Record = {}; - if (result.text) out.text = result.text; + if (result.text) {out.text = result.text;} if (result.details) { // Forward useful details (exit code, duration, status, cwd) for (const key of [ @@ -28,14 +29,15 @@ function buildToolOutput( "reason", ]) { if (result.details[key] !== undefined) - out[key] = result.details[key]; + {out[key] = result.details[key];} } } return out; } export async function POST(req: Request) { - const { messages }: { messages: UIMessage[] } = await req.json(); + const { messages, sessionId }: { messages: UIMessage[]; sessionId?: string } = + await req.json(); // Extract the latest user message text const lastUserMessage = messages.filter((m) => m.role === "user").pop(); @@ -51,6 +53,18 @@ export async function POST(req: Request) { return new Response("No message provided", { status: 400 }); } + // Resolve workspace file paths to be agent-cwd-relative. + // Tree paths are workspace-root-relative (e.g. "knowledge/leads/foo.md"), + // but the agent runs from the repo root and needs "dench/knowledge/leads/foo.md". + let agentMessage = userText; + const wsPrefix = resolveAgentWorkspacePrefix(); + if (wsPrefix) { + agentMessage = userText.replace( + /\[Context: workspace file '([^']+)'\]/, + `[Context: workspace file '${wsPrefix}/$1']`, + ); + } + // Create a custom SSE stream using the AI SDK v6 data stream wire format. // DefaultChatTransport parses these events into UIMessage parts automatically. const encoder = new TextEncoder(); @@ -72,10 +86,13 @@ export async function POST(req: Request) { // onLifecycleEnd closes the text part (textStarted→false), so // onClose can't rely on textStarted alone to detect "no output". let everSentText = false; + // Track whether the status reasoning block is the one currently open + // so we can close it cleanly when real content arrives. + let statusReasoningActive = false; /** Write an SSE event; silently no-ops if the stream was already cancelled. */ const writeEvent = (data: unknown) => { - if (closed) return; + if (closed) {return;} const json = JSON.stringify(data); controller.enqueue(encoder.encode(`data: ${json}\n\n`)); }; @@ -88,6 +105,7 @@ export async function POST(req: Request) { id: currentReasoningId, }); reasoningStarted = false; + statusReasoningActive = false; } }; @@ -99,9 +117,38 @@ export async function POST(req: Request) { } }; + /** Open a status reasoning block (auto-closes any existing one). */ + const openStatusReasoning = (label: string) => { + closeReasoning(); + closeText(); + currentReasoningId = nextId("status"); + writeEvent({ + type: "reasoning-start", + id: currentReasoningId, + }); + writeEvent({ + type: "reasoning-delta", + id: currentReasoningId, + delta: label, + }); + reasoningStarted = true; + statusReasoningActive = true; + }; + try { - await runAgent(userText, abortController.signal, { + await runAgent(agentMessage, abortController.signal, { + onLifecycleStart: () => { + // Show immediate feedback — the agent has started working. + // This eliminates the "Streaming... (silence)" gap. + openStatusReasoning("Preparing response..."); + }, + onThinkingDelta: (delta) => { + // Close the status block if it's still the active one; + // real reasoning content is now arriving. + if (statusReasoningActive) { + closeReasoning(); + } if (!reasoningStarted) { currentReasoningId = nextId("reasoning"); writeEvent({ @@ -185,26 +232,75 @@ export async function POST(req: Request) { } }, + onCompactionStart: () => { + // Show compaction status while the gateway is + // optimizing the session context (can take 10-30s). + openStatusReasoning("Optimizing session context..."); + }, + + onCompactionEnd: (willRetry) => { + // Close the compaction status block. If the gateway + // will retry the prompt, leave the reasoning area open + // so the next status/thinking block follows smoothly. + if (statusReasoningActive) { + if (willRetry) { + // Append a note, keep block open for retry + writeEvent({ + type: "reasoning-delta", + id: currentReasoningId, + delta: "\nRetrying with compacted context...", + }); + } else { + closeReasoning(); + } + } + }, + onLifecycleEnd: () => { closeReasoning(); closeText(); }, - onError: (err) => { - console.error("[chat] Agent error:", err); + onAgentError: (message) => { + // Surface agent-level errors (API 402, rate limits, etc.) + // as visible text in the chat so the user sees what happened. closeReasoning(); - if (!textStarted) { - currentTextId = nextId("text"); - writeEvent({ - type: "text-start", - id: currentTextId, - }); - textStarted = true; - } + closeText(); + + currentTextId = nextId("text"); + writeEvent({ + type: "text-start", + id: currentTextId, + }); writeEvent({ type: "text-delta", id: currentTextId, - delta: `Error starting agent: ${err.message}`, + delta: `[error] ${message}`, + }); + writeEvent({ + type: "text-end", + id: currentTextId, + }); + textStarted = false; + everSentText = true; + }, + + onError: (err) => { + console.error("[chat] Agent error:", err); + closeReasoning(); + closeText(); + + currentTextId = nextId("text"); + writeEvent({ + type: "text-start", + id: currentTextId, + }); + textStarted = true; + everSentText = true; + writeEvent({ + type: "text-delta", + id: currentTextId, + delta: `[error] Failed to start agent: ${err.message}`, }); writeEvent({ type: "text-end", id: currentTextId }); textStarted = false; @@ -219,10 +315,14 @@ export async function POST(req: Request) { type: "text-start", id: currentTextId, }); + const msg = + _code !== null && _code !== 0 + ? `[error] Agent exited with code ${_code}. Check server logs for details.` + : "[error] No response from agent."; writeEvent({ type: "text-delta", id: currentTextId, - delta: "(No response from agent)", + delta: msg, }); writeEvent({ type: "text-end", @@ -233,7 +333,7 @@ export async function POST(req: Request) { closeText(); } }, - }); + }, sessionId ? { sessionId } : undefined); } catch (error) { console.error("[chat] Stream error:", error); writeEvent({ diff --git a/apps/web/app/api/web-sessions/route.ts b/apps/web/app/api/web-sessions/route.ts index a07455e6e46..3327e74a81c 100644 --- a/apps/web/app/api/web-sessions/route.ts +++ b/apps/web/app/api/web-sessions/route.ts @@ -14,6 +14,8 @@ export type WebSessionMeta = { createdAt: number; updatedAt: number; messageCount: number; + /** When set, this session is scoped to a specific workspace file. */ + filePath?: string; }; function ensureDir() { @@ -24,7 +26,7 @@ function ensureDir() { function readIndex(): WebSessionMeta[] { ensureDir(); - if (!existsSync(INDEX_FILE)) return []; + if (!existsSync(INDEX_FILE)) {return [];} try { return JSON.parse(readFileSync(INDEX_FILE, "utf-8")); } catch { @@ -37,9 +39,18 @@ function writeIndex(sessions: WebSessionMeta[]) { writeFileSync(INDEX_FILE, JSON.stringify(sessions, null, 2)); } -/** GET /api/web-sessions — list all web chat sessions */ -export async function GET() { - const sessions = readIndex(); +/** GET /api/web-sessions — list web chat sessions. + * ?filePath=... → returns only sessions scoped to that file. + * No filePath → returns only global (non-file) sessions. */ +export async function GET(req: Request) { + const url = new URL(req.url); + const filePath = url.searchParams.get("filePath"); + + const all = readIndex(); + const sessions = filePath + ? all.filter((s) => s.filePath === filePath) + : all.filter((s) => !s.filePath); + return Response.json({ sessions }); } @@ -53,6 +64,7 @@ export async function POST(req: Request) { createdAt: Date.now(), updatedAt: Date.now(), messageCount: 0, + ...(body.filePath ? { filePath: body.filePath } : {}), }; const sessions = readIndex(); diff --git a/apps/web/app/api/workspace/assets/[...path]/route.ts b/apps/web/app/api/workspace/assets/[...path]/route.ts new file mode 100644 index 00000000000..1c7869b0ef5 --- /dev/null +++ b/apps/web/app/api/workspace/assets/[...path]/route.ts @@ -0,0 +1,57 @@ +import { readFileSync, existsSync } from "node:fs"; +import { extname } from "node:path"; +import { safeResolvePath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const MIME_MAP: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".svg": "image/svg+xml", + ".bmp": "image/bmp", + ".ico": "image/x-icon", +}; + +/** + * GET /api/workspace/assets/ + * Serves an image file from the workspace's assets/ directory. + */ +export async function GET( + _req: Request, + { params }: { params: Promise<{ path: string[] }> }, +) { + const segments = (await params).path; + if (!segments || segments.length === 0) { + return new Response("Not found", { status: 404 }); + } + + const relPath = "assets/" + segments.join("/"); + const ext = extname(relPath).toLowerCase(); + + // Only serve known image types + const mime = MIME_MAP[ext]; + if (!mime) { + return new Response("Unsupported file type", { status: 400 }); + } + + const absPath = safeResolvePath(relPath); + if (!absPath || !existsSync(absPath)) { + return new Response("Not found", { status: 404 }); + } + + try { + const buffer = readFileSync(absPath); + return new Response(buffer, { + headers: { + "Content-Type": mime, + "Cache-Control": "public, max-age=31536000, immutable", + }, + }); + } catch { + return new Response("Read error", { status: 500 }); + } +} diff --git a/apps/web/app/api/workspace/context/route.ts b/apps/web/app/api/workspace/context/route.ts new file mode 100644 index 00000000000..37e0e5f31ae --- /dev/null +++ b/apps/web/app/api/workspace/context/route.ts @@ -0,0 +1,116 @@ +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveDenchRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export type WorkspaceContext = { + exists: boolean; + organization?: { + id?: string; + name?: string; + slug?: string; + }; + members?: Array<{ + id: string; + name: string; + email: string; + role: string; + }>; + defaults?: { + default_view?: string; + date_format?: string; + naming_convention?: string; + }; +}; + +/** + * Parse workspace_context.yaml with basic YAML extraction. + * Handles the specific structure defined by the Dench skill. + */ +function parseWorkspaceContext(content: string): WorkspaceContext { + const ctx: WorkspaceContext = { exists: true }; + + // Extract organization block + const orgMatch = content.match( + /organization:\s*\n((?:\s{2,}.+\n)*)/, + ); + if (orgMatch) { + const orgBlock = orgMatch[1]; + const org: Record = {}; + for (const line of orgBlock.split("\n")) { + const kv = line.match(/^\s+(\w+)\s*:\s*"?([^"\n]+)"?/); + if (kv) {org[kv[1]] = kv[2].trim();} + } + ctx.organization = { + id: org.id, + name: org.name, + slug: org.slug, + }; + } + + // Extract members list + const membersMatch = content.match( + /members:\s*\n((?:\s{2,}.+\n)*)/, + ); + if (membersMatch) { + const membersBlock = membersMatch[1]; + const members: WorkspaceContext["members"] = []; + let current: Record = {}; + + for (const line of membersBlock.split("\n")) { + const itemStart = line.match(/^\s+-\s+(\w+)\s*:\s*"?([^"\n]+)"?/); + const propLine = line.match(/^\s+(\w+)\s*:\s*"?([^"\n]+)"?/); + + if (itemStart) { + if (current.id) {members.push(current as never);} + current = { [itemStart[1]]: itemStart[2].trim() }; + } else if (propLine && !line.trim().startsWith("-")) { + current[propLine[1]] = propLine[2].trim(); + } + } + if (current.id) {members.push(current as never);} + ctx.members = members; + } + + // Extract defaults block + const defaultsMatch = content.match( + /defaults:\s*\n((?:\s{2,}.+\n)*)/, + ); + if (defaultsMatch) { + const defaultsBlock = defaultsMatch[1]; + const defaults: Record = {}; + for (const line of defaultsBlock.split("\n")) { + const kv = line.match(/^\s+(\w[\w_]*)\s*:\s*(.+)/); + if (kv) {defaults[kv[1]] = kv[2].trim();} + } + ctx.defaults = { + default_view: defaults.default_view, + date_format: defaults.date_format, + naming_convention: defaults.naming_convention, + }; + } + + return ctx; +} + +export async function GET() { + const root = resolveDenchRoot(); + if (!root) { + return Response.json({ exists: false } satisfies WorkspaceContext); + } + + const ctxPath = join(root, "workspace_context.yaml"); + if (!existsSync(ctxPath)) { + return Response.json({ exists: true } satisfies WorkspaceContext); + } + + try { + const content = readFileSync(ctxPath, "utf-8"); + const parsed = parseWorkspaceContext(content); + return Response.json(parsed); + } catch { + return Response.json({ exists: true } satisfies WorkspaceContext); + } +} diff --git a/apps/web/app/api/workspace/copy/route.ts b/apps/web/app/api/workspace/copy/route.ts new file mode 100644 index 00000000000..9623f8f2b2c --- /dev/null +++ b/apps/web/app/api/workspace/copy/route.ts @@ -0,0 +1,77 @@ +import { cpSync, existsSync, statSync } from "node:fs"; +import { dirname, basename, extname, join } from "node:path"; +import { safeResolvePath, safeResolveNewPath, isSystemFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/copy + * Body: { path: string, destinationPath?: string } + * + * Duplicates a file or folder. If no destinationPath is provided, + * creates a copy next to the original with " copy" appended. + */ +export async function POST(req: Request) { + let body: { path?: string; destinationPath?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, destinationPath } = body; + if (!relPath || typeof relPath !== "string") { + return Response.json( + { error: "Missing 'path' field" }, + { status: 400 }, + ); + } + + const srcAbs = safeResolvePath(relPath); + if (!srcAbs) { + return Response.json( + { error: "Source not found or path traversal rejected" }, + { status: 404 }, + ); + } + + let destRelPath: string; + if (destinationPath && typeof destinationPath === "string") { + destRelPath = destinationPath; + } else { + // Auto-generate "name copy.ext" or "name copy" for folders + const name = basename(relPath); + const dir = dirname(relPath); + const ext = extname(name); + const stem = ext ? name.slice(0, -ext.length) : name; + const copyName = ext ? `${stem} copy${ext}` : `${stem} copy`; + destRelPath = dir === "." ? copyName : `${dir}/${copyName}`; + } + + const destAbs = safeResolveNewPath(destRelPath); + if (!destAbs) { + return Response.json( + { error: "Invalid destination path" }, + { status: 400 }, + ); + } + + if (existsSync(destAbs)) { + return Response.json( + { error: "Destination already exists" }, + { status: 409 }, + ); + } + + try { + const isDir = statSync(srcAbs).isDirectory(); + cpSync(srcAbs, destAbs, { recursive: isDir }); + return Response.json({ ok: true, sourcePath: relPath, newPath: destRelPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Copy failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/db/introspect/route.ts b/apps/web/app/api/workspace/db/introspect/route.ts new file mode 100644 index 00000000000..3c78a50cea0 --- /dev/null +++ b/apps/web/app/api/workspace/db/introspect/route.ts @@ -0,0 +1,93 @@ +import { safeResolvePath, duckdbQueryOnFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type TableInfo = { + table_name: string; + column_count: number; + estimated_row_count: number; + columns: Array<{ + name: string; + type: string; + is_nullable: boolean; + }>; +}; + +/** + * GET /api/workspace/db/introspect?path= + * + * Introspects a DuckDB / SQLite / generic DB file using the duckdb CLI. + * Returns the list of tables with their columns and approximate row counts. + */ +export async function GET(request: Request) { + const { searchParams } = new URL(request.url); + const relPath = searchParams.get("path"); + + if (!relPath) { + return Response.json( + { error: "Missing required `path` query parameter" }, + { status: 400 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + // Get all user tables (skip internal DuckDB catalogs) + const rawTables = duckdbQueryOnFile<{ + table_name: string; + table_type: string; + }>( + absPath, + "SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = 'main' ORDER BY table_name", + ); + + if (rawTables.length === 0) { + return Response.json({ tables: [], path: relPath }); + } + + const tables: TableInfo[] = []; + + for (const t of rawTables) { + // Fetch columns for this table + const cols = duckdbQueryOnFile<{ + column_name: string; + data_type: string; + is_nullable: string; + }>( + absPath, + `SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = 'main' AND table_name = '${t.table_name.replace(/'/g, "''")}' ORDER BY ordinal_position`, + ); + + // Get approximate row count + let rowCount = 0; + try { + const countResult = duckdbQueryOnFile<{ cnt: number }>( + absPath, + `SELECT count(*) as cnt FROM "${t.table_name.replace(/"/g, '""')}"`, + ); + rowCount = countResult[0]?.cnt ?? 0; + } catch { + // skip if we can't count + } + + tables.push({ + table_name: t.table_name, + column_count: cols.length, + estimated_row_count: rowCount, + columns: cols.map((c) => ({ + name: c.column_name, + type: c.data_type, + is_nullable: c.is_nullable === "YES", + })), + }); + } + + return Response.json({ tables, path: relPath }); +} diff --git a/apps/web/app/api/workspace/db/query/route.ts b/apps/web/app/api/workspace/db/query/route.ts new file mode 100644 index 00000000000..69c7011af79 --- /dev/null +++ b/apps/web/app/api/workspace/db/query/route.ts @@ -0,0 +1,56 @@ +import { safeResolvePath, duckdbQueryOnFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/db/query + * Body: { path: string, sql: string } + * + * Executes a read-only SQL query against a database file and returns JSON rows. + * Only SELECT statements are allowed for safety. + */ +export async function POST(request: Request) { + let body: { path?: string; sql?: string }; + try { + body = await request.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, sql } = body; + + if (!relPath || !sql) { + return Response.json( + { error: "Missing required `path` and `sql` fields" }, + { status: 400 }, + ); + } + + // Basic safety: only allow SELECT-like statements + const trimmedSql = sql.trim().toUpperCase(); + if ( + !trimmedSql.startsWith("SELECT") && + !trimmedSql.startsWith("PRAGMA") && + !trimmedSql.startsWith("DESCRIBE") && + !trimmedSql.startsWith("SHOW") && + !trimmedSql.startsWith("EXPLAIN") && + !trimmedSql.startsWith("WITH") + ) { + return Response.json( + { error: "Only read-only queries (SELECT, DESCRIBE, SHOW, EXPLAIN, WITH) are allowed" }, + { status: 403 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + const rows = duckdbQueryOnFile(absPath, sql); + return Response.json({ rows, sql }); +} diff --git a/apps/web/app/api/workspace/file/route.ts b/apps/web/app/api/workspace/file/route.ts new file mode 100644 index 00000000000..6f51a6be274 --- /dev/null +++ b/apps/web/app/api/workspace/file/route.ts @@ -0,0 +1,121 @@ +import { writeFileSync, mkdirSync, rmSync, statSync } from "node:fs"; +import { dirname } from "node:path"; +import { readWorkspaceFile, safeResolvePath, safeResolveNewPath, isSystemFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function GET(req: Request) { + const url = new URL(req.url); + const path = url.searchParams.get("path"); + + if (!path) { + return Response.json( + { error: "Missing 'path' query parameter" }, + { status: 400 }, + ); + } + + const file = readWorkspaceFile(path); + if (!file) { + return Response.json( + { error: "File not found or access denied" }, + { status: 404 }, + ); + } + + return Response.json(file); +} + +/** + * POST /api/workspace/file + * Body: { path: string, content: string } + * + * Writes a file to the dench workspace. Creates parent directories as needed. + */ +export async function POST(req: Request) { + let body: { path?: string; content?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, content } = body; + if (!relPath || typeof relPath !== "string" || typeof content !== "string") { + return Response.json( + { error: "Missing 'path' and 'content' fields" }, + { status: 400 }, + ); + } + + // Use safeResolveNewPath (not safeResolvePath) because the file may not exist yet + const absPath = safeResolveNewPath(relPath); + if (!absPath) { + return Response.json( + { error: "Invalid path or path traversal rejected" }, + { status: 400 }, + ); + } + + try { + mkdirSync(dirname(absPath), { recursive: true }); + writeFileSync(absPath, content, "utf-8"); + return Response.json({ ok: true, path: relPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Write failed" }, + { status: 500 }, + ); + } +} + +/** + * DELETE /api/workspace/file + * Body: { path: string } + * + * Deletes a file or folder from the dench workspace. + * System files (.object.yaml, workspace.duckdb, etc.) are protected. + */ +export async function DELETE(req: Request) { + let body: { path?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath } = body; + if (!relPath || typeof relPath !== "string") { + return Response.json( + { error: "Missing 'path' field" }, + { status: 400 }, + ); + } + + if (isSystemFile(relPath)) { + return Response.json( + { error: "Cannot delete system file" }, + { status: 403 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "File not found or path traversal rejected" }, + { status: 404 }, + ); + } + + try { + const stat = statSync(absPath); + rmSync(absPath, { recursive: stat.isDirectory() }); + return Response.json({ ok: true, path: relPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Delete failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/mkdir/route.ts b/apps/web/app/api/workspace/mkdir/route.ts new file mode 100644 index 00000000000..2414acdf594 --- /dev/null +++ b/apps/web/app/api/workspace/mkdir/route.ts @@ -0,0 +1,53 @@ +import { mkdirSync, existsSync } from "node:fs"; +import { safeResolveNewPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/mkdir + * Body: { path: string } + * + * Creates a new directory in the dench workspace. + */ +export async function POST(req: Request) { + let body: { path?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath } = body; + if (!relPath || typeof relPath !== "string") { + return Response.json( + { error: "Missing 'path' field" }, + { status: 400 }, + ); + } + + const absPath = safeResolveNewPath(relPath); + if (!absPath) { + return Response.json( + { error: "Invalid path or path traversal rejected" }, + { status: 400 }, + ); + } + + if (existsSync(absPath)) { + return Response.json( + { error: "Directory already exists" }, + { status: 409 }, + ); + } + + try { + mkdirSync(absPath, { recursive: true }); + return Response.json({ ok: true, path: relPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "mkdir failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/move/route.ts b/apps/web/app/api/workspace/move/route.ts new file mode 100644 index 00000000000..4223b260990 --- /dev/null +++ b/apps/web/app/api/workspace/move/route.ts @@ -0,0 +1,93 @@ +import { renameSync, existsSync, statSync } from "node:fs"; +import { join, basename } from "node:path"; +import { safeResolvePath, safeResolveNewPath, isSystemFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/move + * Body: { sourcePath: string, destinationDir: string } + * + * Moves a file or folder into a different directory. + * System files are protected from moving. + */ +export async function POST(req: Request) { + let body: { sourcePath?: string; destinationDir?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { sourcePath, destinationDir } = body; + if (!sourcePath || typeof sourcePath !== "string" || !destinationDir || typeof destinationDir !== "string") { + return Response.json( + { error: "Missing 'sourcePath' and 'destinationDir' fields" }, + { status: 400 }, + ); + } + + if (isSystemFile(sourcePath)) { + return Response.json( + { error: "Cannot move system file" }, + { status: 403 }, + ); + } + + const srcAbs = safeResolvePath(sourcePath); + if (!srcAbs) { + return Response.json( + { error: "Source not found or path traversal rejected" }, + { status: 404 }, + ); + } + + const destDirAbs = safeResolvePath(destinationDir); + if (!destDirAbs) { + return Response.json( + { error: "Destination not found or path traversal rejected" }, + { status: 404 }, + ); + } + + // Destination must be a directory + if (!statSync(destDirAbs).isDirectory()) { + return Response.json( + { error: "Destination is not a directory" }, + { status: 400 }, + ); + } + + // Prevent moving a folder into itself or its children + const srcAbsNorm = srcAbs + "/"; + if (destDirAbs.startsWith(srcAbsNorm) || destDirAbs === srcAbs) { + return Response.json( + { error: "Cannot move a folder into itself" }, + { status: 400 }, + ); + } + + const itemName = basename(srcAbs); + const destAbs = join(destDirAbs, itemName); + + if (existsSync(destAbs)) { + return Response.json( + { error: `'${itemName}' already exists in destination` }, + { status: 409 }, + ); + } + + // Build new relative path + const newRelPath = destinationDir === "." ? itemName : `${destinationDir}/${itemName}`; + + try { + renameSync(srcAbs, destAbs); + return Response.json({ ok: true, oldPath: sourcePath, newPath: newRelPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Move failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/objects/[name]/display-field/route.ts b/apps/web/app/api/workspace/objects/[name]/display-field/route.ts new file mode 100644 index 00000000000..2f34545a0cb --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/display-field/route.ts @@ -0,0 +1,82 @@ +import { duckdbQuery, duckdbPath, duckdbExec } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * PATCH /api/workspace/objects/[name]/display-field + * Set which field is used as the display label for entries of this object. + * Body: { displayField: string } + */ +export async function PATCH( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB database not found" }, + { status: 404 }, + ); + } + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const body = await req.json(); + const { displayField } = body; + + if (typeof displayField !== "string" || !displayField.trim()) { + return Response.json( + { error: "displayField must be a non-empty string" }, + { status: 400 }, + ); + } + + // Ensure display_field column exists + duckdbExec( + "ALTER TABLE objects ADD COLUMN IF NOT EXISTS display_field VARCHAR", + ); + + // Verify the object exists + const objects = duckdbQuery<{ id: string }>( + `SELECT id FROM objects WHERE name = '${name}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + + // Verify the field exists on this object + const escapedField = displayField.replace(/'/g, "''"); + const fieldCheck = duckdbQuery<{ id: string }>( + `SELECT id FROM fields WHERE object_id = '${objects[0].id}' AND name = '${escapedField}' LIMIT 1`, + ); + if (fieldCheck.length === 0) { + return Response.json( + { error: `Field '${displayField}' not found on object '${name}'` }, + { status: 400 }, + ); + } + + // Update the display_field + const success = duckdbExec( + `UPDATE objects SET display_field = '${escapedField}', updated_at = now() WHERE name = '${name}'`, + ); + + if (!success) { + return Response.json( + { error: "Failed to update display field" }, + { status: 500 }, + ); + } + + return Response.json({ ok: true, displayField }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts b/apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts new file mode 100644 index 00000000000..83c6c75eb80 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/entries/[id]/route.ts @@ -0,0 +1,511 @@ +import { + duckdbQuery, + duckdbExec, + duckdbPath, + parseRelationValue, +} from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +// --- Types --- + +type ObjectRow = { + id: string; + name: string; + description?: string; + icon?: string; + default_view?: string; + display_field?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; + description?: string; + required?: boolean; + enum_values?: string; + enum_colors?: string; + enum_multiple?: boolean; + related_object_id?: string; + relationship_type?: string; + sort_order?: number; +}; + +// --- Helpers --- + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +function tryParseJson(value: unknown): unknown { + if (typeof value !== "string") { + return value; + } + try { + return JSON.parse(value); + } catch { + return value; + } +} + +function resolveDisplayField( + obj: ObjectRow, + fields: FieldRow[], +): string { + if (obj.display_field) { + return obj.display_field; + } + const nameField = fields.find( + (f) => + /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) { + return nameField.name; + } + const textField = fields.find((f) => f.type === "text"); + if (textField) { + return textField.name; + } + return fields[0]?.name ?? "id"; +} + +// --- Route handlers --- + +/** + * GET /api/workspace/objects/[name]/entries/[id] + * Returns a single entry with all field values, relation labels, and reverse relations. + */ +export async function GET( + _req: Request, + { params }: { params: Promise<{ name: string; id: string }> }, +) { + const { name, id } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + if (!id || id.length > 64) { + return Response.json( + { error: "Invalid entry ID" }, + { status: 400 }, + ); + } + + // Fetch object + const objects = duckdbQuery( + `SELECT * FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const obj = objects[0]; + + // Fetch fields + const fields = duckdbQuery( + `SELECT * FROM fields WHERE object_id = '${sqlEscape(obj.id)}' ORDER BY sort_order`, + ); + + // Fetch entry field values + const entryRows = duckdbQuery<{ + entry_id: string; + created_at: string; + updated_at: string; + field_name: string; + value: string | null; + }>( + `SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.id = '${sqlEscape(id)}' + AND e.object_id = '${sqlEscape(obj.id)}'`, + ); + + if (entryRows.length === 0) { + const exists = duckdbQuery<{ cnt: number }>( + `SELECT COUNT(*) as cnt FROM entries WHERE id = '${sqlEscape(id)}' AND object_id = '${sqlEscape(obj.id)}'`, + ); + if (!exists[0] || exists[0].cnt === 0) { + return Response.json( + { error: "Entry not found" }, + { status: 404 }, + ); + } + } + + // Pivot into a single record + const entry: Record = { entry_id: id }; + for (const row of entryRows) { + entry.created_at ??= row.created_at; + entry.updated_at ??= row.updated_at; + if (row.field_name) { + entry[row.field_name] = row.value; + } + } + + // Parse enum JSON strings in fields + const parsedFields = fields.map((f) => ({ + ...f, + enum_values: f.enum_values + ? tryParseJson(f.enum_values) + : undefined, + enum_colors: f.enum_colors + ? tryParseJson(f.enum_colors) + : undefined, + })); + + // Resolve relation labels for this entry + const relationLabels: Record> = + {}; + const relatedObjectNames: Record = {}; + + const relationFields = fields.filter( + (f) => f.type === "relation" && f.related_object_id, + ); + + for (const rf of relationFields) { + const relatedObjs = duckdbQuery( + `SELECT * FROM objects WHERE id = '${sqlEscape(rf.related_object_id!)}' LIMIT 1`, + ); + if (relatedObjs.length === 0) { + continue; + } + const relObj = relatedObjs[0]; + relatedObjectNames[rf.name] = relObj.name; + + const val = entry[rf.name]; + if (val == null || val === "") { + relationLabels[rf.name] = {}; + continue; + } + + const ids = parseRelationValue(String(val)); + if (ids.length === 0) { + relationLabels[rf.name] = {}; + continue; + } + + const relFields = duckdbQuery( + `SELECT * FROM fields WHERE object_id = '${sqlEscape(relObj.id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField( + relObj, + relFields, + ); + + const idList = ids + .map((i) => `'${sqlEscape(i)}'`) + .join(","); + const displayRows = duckdbQuery<{ + entry_id: string; + value: string; + }>( + `SELECT e.id as entry_id, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.id IN (${idList}) + AND f.object_id = '${sqlEscape(relObj.id)}' + AND f.name = '${sqlEscape(displayFieldName)}'`, + ); + + const labelMap: Record = {}; + for (const row of displayRows) { + labelMap[row.entry_id] = row.value || row.entry_id; + } + for (const i of ids) { + if (!labelMap[i]) { + labelMap[i] = i; + } + } + relationLabels[rf.name] = labelMap; + } + + // Enrich fields with related object names + const enrichedFields = parsedFields.map((f) => ({ + ...f, + related_object_name: + f.type === "relation" + ? relatedObjectNames[f.name] + : undefined, + })); + + // Find reverse relations for this entry + const reverseRelations = findReverseRelationsForEntry( + obj.id, + id, + ); + + const effectiveDisplayField = resolveDisplayField(obj, fields); + + return Response.json({ + object: obj, + fields: enrichedFields, + entry, + relationLabels, + reverseRelations, + effectiveDisplayField, + }); +} + +/** + * PATCH /api/workspace/objects/[name]/entries/[id] + * Update field values for an entry. + * Body: { fields: { [fieldName]: newValue } } + */ +export async function PATCH( + req: Request, + { params }: { params: Promise<{ name: string; id: string }> }, +) { + const { name, id } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + // Find object + const objects = duckdbQuery<{ id: string }>( + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Verify entry exists + const exists = duckdbQuery<{ cnt: number }>( + `SELECT COUNT(*) as cnt FROM entries WHERE id = '${sqlEscape(id)}' AND object_id = '${sqlEscape(objectId)}'`, + ); + if (!exists[0] || exists[0].cnt === 0) { + return Response.json( + { error: "Entry not found" }, + { status: 404 }, + ); + } + + const body = await req.json(); + const fieldUpdates: Record = + body.fields ?? {}; + + // Get field IDs by name + const dbFields = duckdbQuery<{ id: string; name: string }>( + `SELECT id, name FROM fields WHERE object_id = '${sqlEscape(objectId)}'`, + ); + const fieldMap = new Map(dbFields.map((f) => [f.name, f.id])); + + let updatedCount = 0; + for (const [fieldName, value] of Object.entries(fieldUpdates)) { + const fieldId = fieldMap.get(fieldName); + if (!fieldId) {continue;} + + const escapedValue = + value == null ? "NULL" : `'${sqlEscape(String(value))}'`; + + // Try update first, then insert if no rows affected + const existingRows = duckdbQuery<{ cnt: number }>( + `SELECT COUNT(*) as cnt FROM entry_fields WHERE entry_id = '${sqlEscape(id)}' AND field_id = '${sqlEscape(fieldId)}'`, + ); + + if (existingRows[0]?.cnt > 0) { + duckdbExec( + `UPDATE entry_fields SET value = ${escapedValue} WHERE entry_id = '${sqlEscape(id)}' AND field_id = '${sqlEscape(fieldId)}'`, + ); + } else { + duckdbExec( + `INSERT INTO entry_fields (entry_id, field_id, value) VALUES ('${sqlEscape(id)}', '${sqlEscape(fieldId)}', ${escapedValue})`, + ); + } + updatedCount++; + } + + // Touch updated_at on the entry + const now = new Date().toISOString(); + duckdbExec( + `UPDATE entries SET updated_at = '${now}' WHERE id = '${sqlEscape(id)}'`, + ); + + return Response.json({ ok: true, updatedCount }); +} + +/** + * DELETE /api/workspace/objects/[name]/entries/[id] + * Delete a single entry and its field values. + */ +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ name: string; id: string }> }, +) { + const { name, id } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + // Find object + const objects = duckdbQuery<{ id: string }>( + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Delete field values first, then entry + duckdbExec( + `DELETE FROM entry_fields WHERE entry_id = '${sqlEscape(id)}'`, + ); + duckdbExec( + `DELETE FROM entries WHERE id = '${sqlEscape(id)}' AND object_id = '${sqlEscape(objectId)}'`, + ); + + return Response.json({ ok: true }); +} + +// --- Reverse relations for a single entry --- + +type ReverseRelation = { + fieldName: string; + sourceObjectName: string; + sourceObjectId: string; + displayField: string; + links: Array<{ id: string; label: string }>; +}; + +function findReverseRelationsForEntry( + objectId: string, + entryId: string, +): ReverseRelation[] { + const reverseFields = duckdbQuery<{ + id: string; + name: string; + object_id: string; + source_object_name: string; + }>( + `SELECT f.id, f.name, f.object_id, o.name as source_object_name + FROM fields f + JOIN objects o ON o.id = f.object_id + WHERE f.type = 'relation' + AND f.related_object_id = '${sqlEscape(objectId)}'`, + ); + + if (reverseFields.length === 0) { + return []; + } + + const result: ReverseRelation[] = []; + + for (const rrf of reverseFields) { + const refRows = duckdbQuery<{ + source_entry_id: string; + target_value: string; + }>( + `SELECT ef.entry_id as source_entry_id, ef.value as target_value + FROM entry_fields ef + WHERE ef.field_id = '${sqlEscape(rrf.id)}' + AND ef.value IS NOT NULL + AND ef.value != ''`, + ); + + const matchingSourceIds: string[] = []; + for (const row of refRows) { + const targetIds = parseRelationValue(row.target_value); + if (targetIds.includes(entryId)) { + matchingSourceIds.push(row.source_entry_id); + } + } + + if (matchingSourceIds.length === 0) { + continue; + } + + const sourceObj = duckdbQuery( + `SELECT * FROM objects WHERE id = '${sqlEscape(rrf.object_id)}' LIMIT 1`, + ); + if (sourceObj.length === 0) { + continue; + } + + const sourceFields = duckdbQuery( + `SELECT * FROM fields WHERE object_id = '${sqlEscape(rrf.object_id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField( + sourceObj[0], + sourceFields, + ); + + const idList = matchingSourceIds + .map((i) => `'${sqlEscape(i)}'`) + .join(","); + const displayRows = duckdbQuery<{ + entry_id: string; + value: string; + }>( + `SELECT ef.entry_id, ef.value + FROM entry_fields ef + JOIN fields f ON f.id = ef.field_id + WHERE ef.entry_id IN (${idList}) + AND f.name = '${sqlEscape(displayFieldName)}' + AND f.object_id = '${sqlEscape(rrf.object_id)}'`, + ); + + const displayMap: Record = {}; + for (const row of displayRows) { + displayMap[row.entry_id] = row.value || row.entry_id; + } + + const links = matchingSourceIds.map((sid) => ({ + id: sid, + label: displayMap[sid] || sid, + })); + + result.push({ + fieldName: rrf.name, + sourceObjectName: rrf.source_object_name, + sourceObjectId: rrf.object_id, + displayField: displayFieldName, + links, + }); + } + + return result; +} diff --git a/apps/web/app/api/workspace/objects/[name]/entries/bulk-delete/route.ts b/apps/web/app/api/workspace/objects/[name]/entries/bulk-delete/route.ts new file mode 100644 index 00000000000..2af3b11b223 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/entries/bulk-delete/route.ts @@ -0,0 +1,72 @@ +import { duckdbExec, duckdbQuery, duckdbPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * POST /api/workspace/objects/[name]/entries/bulk-delete + * Delete multiple entries at once. + * Body: { entryIds: string[] } + */ +export async function POST( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const body = await req.json(); + const entryIds: string[] = body.entryIds; + + if (!Array.isArray(entryIds) || entryIds.length === 0) { + return Response.json( + { error: "entryIds must be a non-empty array" }, + { status: 400 }, + ); + } + + // Validate object exists + const objects = duckdbQuery<{ id: string }>( + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + const idList = entryIds + .map((id) => `'${sqlEscape(id)}'`) + .join(","); + + // Delete field values first, then entries + duckdbExec( + `DELETE FROM entry_fields WHERE entry_id IN (${idList})`, + ); + duckdbExec( + `DELETE FROM entries WHERE id IN (${idList}) AND object_id = '${sqlEscape(objectId)}'`, + ); + + return Response.json({ + ok: true, + deletedCount: entryIds.length, + }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/entries/route.ts b/apps/web/app/api/workspace/objects/[name]/entries/route.ts new file mode 100644 index 00000000000..49bff713e2b --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/entries/route.ts @@ -0,0 +1,95 @@ +import { duckdbExec, duckdbQuery, duckdbPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * POST /api/workspace/objects/[name]/entries + * Create a new entry with optional field values. + * Body: { fields?: Record } + */ +export async function POST( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + // Find object + const objects = duckdbQuery<{ id: string }>( + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Generate UUID for the new entry + const idRows = duckdbQuery<{ id: string }>( + "SELECT uuid()::VARCHAR as id", + ); + const entryId = idRows[0]?.id; + if (!entryId) { + return Response.json( + { error: "Failed to generate UUID" }, + { status: 500 }, + ); + } + + // Create entry + const now = new Date().toISOString(); + const ok = duckdbExec( + `INSERT INTO entries (id, object_id, created_at, updated_at) VALUES ('${sqlEscape(entryId)}', '${sqlEscape(objectId)}', '${now}', '${now}')`, + ); + if (!ok) { + return Response.json( + { error: "Failed to create entry" }, + { status: 500 }, + ); + } + + // Insert field values if provided + let body: { fields?: Record } = {}; + try { + body = await req.json(); + } catch { + // no body is fine + } + + if (body.fields && typeof body.fields === "object") { + // Get field IDs by name + const dbFields = duckdbQuery<{ id: string; name: string }>( + `SELECT id, name FROM fields WHERE object_id = '${sqlEscape(objectId)}'`, + ); + const fieldMap = new Map(dbFields.map((f) => [f.name, f.id])); + + for (const [fieldName, value] of Object.entries(body.fields)) { + const fieldId = fieldMap.get(fieldName); + if (!fieldId || value == null) {continue;} + duckdbExec( + `INSERT INTO entry_fields (entry_id, field_id, value) VALUES ('${sqlEscape(entryId)}', '${sqlEscape(fieldId)}', '${sqlEscape(String(value))}')`, + ); + } + } + + return Response.json({ entryId, ok: true }, { status: 201 }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/route.ts b/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/route.ts new file mode 100644 index 00000000000..7ca84c1fa93 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/fields/[fieldId]/route.ts @@ -0,0 +1,96 @@ +import { duckdbExec, duckdbQuery, duckdbPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * PATCH /api/workspace/objects/[name]/fields/[fieldId] + * Rename a field. + * Body: { name: string } + */ +export async function PATCH( + req: Request, + { + params, + }: { params: Promise<{ name: string; fieldId: string }> }, +) { + const { name, fieldId } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const body = await req.json(); + const newName: string = body.name; + + if ( + !newName || + typeof newName !== "string" || + newName.trim().length === 0 + ) { + return Response.json( + { error: "Name is required" }, + { status: 400 }, + ); + } + + // Validate object exists + const objects = duckdbQuery<{ id: string }>( + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Validate field exists and belongs to this object + const fieldExists = duckdbQuery<{ cnt: number }>( + `SELECT COUNT(*) as cnt FROM fields WHERE id = '${sqlEscape(fieldId)}' AND object_id = '${sqlEscape(objectId)}'`, + ); + if (!fieldExists[0] || fieldExists[0].cnt === 0) { + return Response.json( + { error: "Field not found" }, + { status: 404 }, + ); + } + + // Check for duplicate name + const duplicateCheck = duckdbQuery<{ cnt: number }>( + `SELECT COUNT(*) as cnt FROM fields WHERE object_id = '${sqlEscape(objectId)}' AND name = '${sqlEscape(newName.trim())}' AND id != '${sqlEscape(fieldId)}'`, + ); + if (duplicateCheck[0]?.cnt > 0) { + return Response.json( + { error: "A field with that name already exists" }, + { status: 409 }, + ); + } + + const ok = duckdbExec( + `UPDATE fields SET name = '${sqlEscape(newName.trim())}' WHERE id = '${sqlEscape(fieldId)}'`, + ); + + if (!ok) { + return Response.json( + { error: "Failed to rename field" }, + { status: 500 }, + ); + } + + return Response.json({ ok: true }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/fields/reorder/route.ts b/apps/web/app/api/workspace/objects/[name]/fields/reorder/route.ts new file mode 100644 index 00000000000..158e1f2ff22 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/fields/reorder/route.ts @@ -0,0 +1,64 @@ +import { duckdbExec, duckdbQuery, duckdbPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * PATCH /api/workspace/objects/[name]/fields/reorder + * Reorder fields by updating sort_order. + * Body: { fieldOrder: string[] } — array of field IDs in desired order + */ +export async function PATCH( + req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB not found" }, + { status: 404 }, + ); + } + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + const body = await req.json(); + const fieldOrder: string[] = body.fieldOrder; + + if (!Array.isArray(fieldOrder) || fieldOrder.length === 0) { + return Response.json( + { error: "fieldOrder must be a non-empty array" }, + { status: 400 }, + ); + } + + // Validate object exists + const objects = duckdbQuery<{ id: string }>( + `SELECT id FROM objects WHERE name = '${sqlEscape(name)}' LIMIT 1`, + ); + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + const objectId = objects[0].id; + + // Update sort_order for each field + for (let i = 0; i < fieldOrder.length; i++) { + duckdbExec( + `UPDATE fields SET sort_order = ${i} WHERE id = '${sqlEscape(fieldOrder[i])}' AND object_id = '${sqlEscape(objectId)}'`, + ); + } + + return Response.json({ ok: true }); +} diff --git a/apps/web/app/api/workspace/objects/[name]/route.ts b/apps/web/app/api/workspace/objects/[name]/route.ts new file mode 100644 index 00000000000..086abf6f4c6 --- /dev/null +++ b/apps/web/app/api/workspace/objects/[name]/route.ts @@ -0,0 +1,417 @@ +import { duckdbQuery, duckdbPath, duckdbExec, parseRelationValue } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +type ObjectRow = { + id: string; + name: string; + description?: string; + icon?: string; + default_view?: string; + display_field?: string; + immutable?: boolean; + created_at?: string; + updated_at?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; + description?: string; + required?: boolean; + enum_values?: string; + enum_colors?: string; + enum_multiple?: boolean; + related_object_id?: string; + relationship_type?: string; + sort_order?: number; +}; + +type StatusRow = { + id: string; + name: string; + color?: string; + sort_order?: number; + is_default?: boolean; +}; + +type EavRow = { + entry_id: string; + created_at: string; + updated_at: string; + field_name: string; + value: string | null; +}; + +// --- Schema migration (idempotent, runs once per process) --- + +let schemaMigrated = false; + +function ensureDisplayFieldColumn() { + if (schemaMigrated) {return;} + duckdbExec( + "ALTER TABLE objects ADD COLUMN IF NOT EXISTS display_field VARCHAR", + ); + schemaMigrated = true; +} + +// --- Helpers --- + +/** + * Pivot raw EAV rows into one object per entry with field names as keys. + */ +function pivotEavRows(rows: EavRow[]): Record[] { + const grouped = new Map>(); + + for (const row of rows) { + let entry = grouped.get(row.entry_id); + if (!entry) { + entry = { + entry_id: row.entry_id, + created_at: row.created_at, + updated_at: row.updated_at, + }; + grouped.set(row.entry_id, entry); + } + if (row.field_name) { + entry[row.field_name] = row.value; + } + } + + return Array.from(grouped.values()); +} + +function tryParseJson(value: unknown): unknown { + if (typeof value !== "string") {return value;} + try { + return JSON.parse(value); + } catch { + return value; + } +} + +/** SQL-escape a string (double single-quotes). */ +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** + * Determine the display field for an object. + * Priority: explicit display_field > heuristic (name/title) > first text field > first field. + */ +function resolveDisplayField( + obj: ObjectRow, + objFields: FieldRow[], +): string { + if (obj.display_field) {return obj.display_field;} + + // Heuristic: look for name/title fields + const nameField = objFields.find( + (f) => + /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) {return nameField.name;} + + // Fallback: first text field + const textField = objFields.find((f) => f.type === "text"); + if (textField) {return textField.name;} + + // Ultimate fallback: first field + return objFields[0]?.name ?? "id"; +} + +/** + * Resolve relation field values to human-readable display labels. + * Returns: { fieldName: { entryId: displayLabel } } + */ +function resolveRelationLabels( + fields: FieldRow[], + entries: Record[], +): { + labels: Record>; + relatedObjectNames: Record; +} { + const labels: Record> = {}; + const relatedObjectNames: Record = {}; + + const relationFields = fields.filter( + (f) => f.type === "relation" && f.related_object_id, + ); + + for (const rf of relationFields) { + const relatedObjs = duckdbQuery( + `SELECT * FROM objects WHERE id = '${sqlEscape(rf.related_object_id!)}' LIMIT 1`, + ); + if (relatedObjs.length === 0) {continue;} + const relObj = relatedObjs[0]; + relatedObjectNames[rf.name] = relObj.name; + + // Get related object's fields for display field resolution + const relFields = duckdbQuery( + `SELECT * FROM fields WHERE object_id = '${sqlEscape(relObj.id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField(relObj, relFields); + + // Collect all referenced entry IDs from our entries + const entryIds = new Set(); + for (const entry of entries) { + const val = entry[rf.name]; + if (val == null || val === "") {continue;} + for (const id of parseRelationValue(String(val))) { + entryIds.add(id); + } + } + + if (entryIds.size === 0) { + labels[rf.name] = {}; + continue; + } + + // Query display values for the referenced entries + const idList = Array.from(entryIds) + .map((id) => `'${sqlEscape(id)}'`) + .join(","); + const displayRows = duckdbQuery<{ + entry_id: string; + value: string; + }>( + `SELECT e.id as entry_id, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.id IN (${idList}) + AND f.object_id = '${sqlEscape(relObj.id)}' + AND f.name = '${sqlEscape(displayFieldName)}'`, + ); + + const labelMap: Record = {}; + for (const row of displayRows) { + labelMap[row.entry_id] = row.value || row.entry_id; + } + // Fill in any IDs that didn't get a display label + for (const id of entryIds) { + if (!labelMap[id]) {labelMap[id] = id;} + } + + labels[rf.name] = labelMap; + } + + return { labels, relatedObjectNames }; +} + +type ReverseRelation = { + fieldName: string; + sourceObjectName: string; + sourceObjectId: string; + displayField: string; + entries: Record>; +}; + +/** + * Find reverse relations: other objects with relation fields pointing TO this object. + * For each, resolve the display labels and group by target entry ID. + */ +function findReverseRelations(objectId: string): ReverseRelation[] { + // Find all relation fields in other objects that reference this object + const reverseFields = duckdbQuery< + FieldRow & { source_object_id: string; source_object_name: string } + >( + `SELECT f.*, f.object_id as source_object_id, o.name as source_object_name + FROM fields f + JOIN objects o ON o.id = f.object_id + WHERE f.type = 'relation' + AND f.related_object_id = '${sqlEscape(objectId)}'`, + ); + + if (reverseFields.length === 0) {return [];} + + const result: ReverseRelation[] = []; + + for (const rrf of reverseFields) { + // Get source object and its fields + const sourceObjs = duckdbQuery( + `SELECT * FROM objects WHERE id = '${sqlEscape(rrf.source_object_id)}' LIMIT 1`, + ); + if (sourceObjs.length === 0) {continue;} + + const sourceFields = duckdbQuery( + `SELECT * FROM fields WHERE object_id = '${sqlEscape(rrf.source_object_id)}' ORDER BY sort_order`, + ); + const displayFieldName = resolveDisplayField(sourceObjs[0], sourceFields); + + // Fetch all source entries that have this relation field set + const refRows = duckdbQuery<{ + source_entry_id: string; + target_value: string; + }>( + `SELECT ef.entry_id as source_entry_id, ef.value as target_value + FROM entry_fields ef + WHERE ef.field_id = '${sqlEscape(rrf.id)}' + AND ef.value IS NOT NULL + AND ef.value != ''`, + ); + + if (refRows.length === 0) {continue;} + + // Get display labels for the source entries + const sourceEntryIds = [ + ...new Set(refRows.map((r) => r.source_entry_id)), + ]; + const idList = sourceEntryIds + .map((id) => `'${sqlEscape(id)}'`) + .join(","); + const displayRows = duckdbQuery<{ + entry_id: string; + value: string; + }>( + `SELECT ef.entry_id, ef.value + FROM entry_fields ef + JOIN fields f ON f.id = ef.field_id + WHERE ef.entry_id IN (${idList}) + AND f.name = '${sqlEscape(displayFieldName)}' + AND f.object_id = '${sqlEscape(rrf.source_object_id)}'`, + ); + + const displayMap: Record = {}; + for (const row of displayRows) { + displayMap[row.entry_id] = row.value || row.entry_id; + } + + // Build: target_entry_id -> [{id, label}] + const entriesMap: Record< + string, + Array<{ id: string; label: string }> + > = {}; + for (const row of refRows) { + const targetIds = parseRelationValue(row.target_value); + for (const targetId of targetIds) { + if (!entriesMap[targetId]) {entriesMap[targetId] = [];} + entriesMap[targetId].push({ + id: row.source_entry_id, + label: displayMap[row.source_entry_id] || row.source_entry_id, + }); + } + } + + result.push({ + fieldName: rrf.name, + sourceObjectName: rrf.source_object_name, + sourceObjectId: rrf.source_object_id, + displayField: displayFieldName, + entries: entriesMap, + }); + } + + return result; +} + +// --- Route handler --- + +export async function GET( + _req: Request, + { params }: { params: Promise<{ name: string }> }, +) { + const { name } = await params; + + if (!duckdbPath()) { + return Response.json( + { error: "DuckDB database not found" }, + { status: 404 }, + ); + } + + // Sanitize name to prevent injection (only allow alphanumeric + underscore) + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + return Response.json( + { error: "Invalid object name" }, + { status: 400 }, + ); + } + + // Ensure display_field column exists (idempotent migration) + ensureDisplayFieldColumn(); + + // Fetch object metadata + const objects = duckdbQuery( + `SELECT * FROM objects WHERE name = '${name}' LIMIT 1`, + ); + + if (objects.length === 0) { + return Response.json( + { error: `Object '${name}' not found` }, + { status: 404 }, + ); + } + + const obj = objects[0]; + + // Fetch fields for this object + const fields = duckdbQuery( + `SELECT * FROM fields WHERE object_id = '${obj.id}' ORDER BY sort_order`, + ); + + // Fetch statuses for this object + const statuses = duckdbQuery( + `SELECT * FROM statuses WHERE object_id = '${obj.id}' ORDER BY sort_order`, + ); + + // Try the PIVOT view first, then fall back to raw EAV query + client-side pivot + let entries: Record[] = []; + + const pivotEntries = duckdbQuery( + `SELECT * FROM v_${name} ORDER BY created_at DESC LIMIT 200`, + ); + + if (pivotEntries.length > 0) { + entries = pivotEntries; + } else { + const rawRows = duckdbQuery( + `SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = '${obj.id}' + ORDER BY e.created_at DESC + LIMIT 5000`, + ); + + entries = pivotEavRows(rawRows); + } + + // Parse enum JSON strings in fields + const parsedFields = fields.map((f) => ({ + ...f, + enum_values: f.enum_values ? tryParseJson(f.enum_values) : undefined, + enum_colors: f.enum_colors ? tryParseJson(f.enum_colors) : undefined, + })); + + // Resolve relation field values to human-readable display labels + const { labels: relationLabels, relatedObjectNames } = + resolveRelationLabels(fields, entries); + + // Enrich fields with related object names for frontend display + const enrichedFields = parsedFields.map((f) => ({ + ...f, + related_object_name: + f.type === "relation" ? relatedObjectNames[f.name] : undefined, + })); + + // Find reverse relations (other objects linking TO this one) + const reverseRelations = findReverseRelations(obj.id); + + // Compute the effective display field for this object + const effectiveDisplayField = resolveDisplayField(obj, fields); + + return Response.json({ + object: obj, + fields: enrichedFields, + statuses, + entries, + relationLabels, + reverseRelations, + effectiveDisplayField, + }); +} diff --git a/apps/web/app/api/workspace/query/route.ts b/apps/web/app/api/workspace/query/route.ts new file mode 100644 index 00000000000..2df1dcc2962 --- /dev/null +++ b/apps/web/app/api/workspace/query/route.ts @@ -0,0 +1,43 @@ +import { duckdbQuery } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST(req: Request) { + let body: { sql?: string }; + try { + body = await req.json(); + } catch { + return Response.json( + { error: "Invalid JSON body" }, + { status: 400 }, + ); + } + + const { sql } = body; + if (!sql || typeof sql !== "string") { + return Response.json( + { error: "Missing 'sql' field in request body" }, + { status: 400 }, + ); + } + + // Basic SQL safety: reject obviously dangerous statements + const upper = sql.toUpperCase().trim(); + if ( + upper.startsWith("DROP") || + upper.startsWith("DELETE") || + upper.startsWith("INSERT") || + upper.startsWith("UPDATE") || + upper.startsWith("ALTER") || + upper.startsWith("CREATE") + ) { + return Response.json( + { error: "Only SELECT queries are allowed" }, + { status: 403 }, + ); + } + + const rows = duckdbQuery(sql); + return Response.json({ rows }); +} diff --git a/apps/web/app/api/workspace/raw-file/route.ts b/apps/web/app/api/workspace/raw-file/route.ts new file mode 100644 index 00000000000..69b859e5c3a --- /dev/null +++ b/apps/web/app/api/workspace/raw-file/route.ts @@ -0,0 +1,122 @@ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { safeResolvePath, resolveDenchRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const MIME_MAP: Record = { + // Images + jpg: "image/jpeg", + jpeg: "image/jpeg", + png: "image/png", + gif: "image/gif", + webp: "image/webp", + svg: "image/svg+xml", + ico: "image/x-icon", + bmp: "image/bmp", + tiff: "image/tiff", + tif: "image/tiff", + avif: "image/avif", + heic: "image/heic", + heif: "image/heif", + // Video + mp4: "video/mp4", + webm: "video/webm", + mov: "video/quicktime", + avi: "video/x-msvideo", + mkv: "video/x-matroska", + // Audio + mp3: "audio/mpeg", + wav: "audio/wav", + ogg: "audio/ogg", + m4a: "audio/mp4", + // Documents + pdf: "application/pdf", +}; + +/** + * Resolve a file path, trying multiple strategies: + * 1. Absolute path — the agent may read files from anywhere on the local machine + * (Photos library, Downloads, etc.), so we serve any readable absolute path. + * 2. Workspace-relative via safeResolvePath + * 3. Bare filename — search common workspace subdirectories + * + * Security note: this is a local-only dev server; it never runs in production. + */ +function resolveFile(path: string): string | null { + // 1. Absolute path — serve directly if it exists on disk + if (path.startsWith("/")) { + const abs = resolve(path); + if (existsSync(abs)) {return abs;} + // Fall through to workspace-relative in case the leading / is accidental + } + + // 2. Standard workspace-relative resolution + const resolved = safeResolvePath(path); + if (resolved) {return resolved;} + + // 3. Try common subdirectories in case the path is a bare filename + const root = resolveDenchRoot(); + if (!root) {return null;} + const rootAbs = resolve(root); + const basename = path.split("/").pop() ?? path; + if (basename === path) { + const subdirs = [ + "assets", + "knowledge", + "manufacturing", + "uploads", + "files", + "images", + "media", + "reports", + "exports", + ]; + for (const sub of subdirs) { + const candidate = resolve(root, sub, basename); + if ( + candidate.startsWith(rootAbs) && + existsSync(candidate) + ) { + return candidate; + } + } + } + + return null; +} + +/** + * GET /api/workspace/raw-file?path=... + * Serves a workspace file with the correct Content-Type for inline display. + * Used by the chain-of-thought component to render images, videos, and PDFs. + */ +export async function GET(req: Request) { + const url = new URL(req.url); + const path = url.searchParams.get("path"); + + if (!path) { + return new Response("Missing path", { status: 400 }); + } + + const absolute = resolveFile(path); + if (!absolute) { + return new Response("Not found", { status: 404 }); + } + + const ext = path.split(".").pop()?.toLowerCase() ?? ""; + const contentType = MIME_MAP[ext] ?? "application/octet-stream"; + + try { + const buffer = readFileSync(absolute); + return new Response(buffer, { + headers: { + "Content-Type": contentType, + "Cache-Control": "public, max-age=3600", + }, + }); + } catch { + return new Response("Read error", { status: 500 }); + } +} diff --git a/apps/web/app/api/workspace/rename/route.ts b/apps/web/app/api/workspace/rename/route.ts new file mode 100644 index 00000000000..e97f7f8ac8d --- /dev/null +++ b/apps/web/app/api/workspace/rename/route.ts @@ -0,0 +1,84 @@ +import { renameSync, existsSync } from "node:fs"; +import { join, dirname, basename } from "node:path"; +import { safeResolvePath, safeResolveNewPath, isSystemFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/rename + * Body: { path: string, newName: string } + * + * Renames a file or folder within the same directory. + * System files are protected from renaming. + */ +export async function POST(req: Request) { + let body: { path?: string; newName?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: relPath, newName } = body; + if (!relPath || typeof relPath !== "string" || !newName || typeof newName !== "string") { + return Response.json( + { error: "Missing 'path' and 'newName' fields" }, + { status: 400 }, + ); + } + + if (isSystemFile(relPath)) { + return Response.json( + { error: "Cannot rename system file" }, + { status: 403 }, + ); + } + + // Validate newName: no slashes, no empty, no traversal + if (newName.includes("/") || newName.includes("\\") || newName.trim() === "") { + return Response.json( + { error: "Invalid file name" }, + { status: 400 }, + ); + } + + const absPath = safeResolvePath(relPath); + if (!absPath) { + return Response.json( + { error: "Source not found or path traversal rejected" }, + { status: 404 }, + ); + } + + const parentDir = dirname(absPath); + const newAbsPath = join(parentDir, newName); + + // Ensure the new path stays within workspace + const parentRel = dirname(relPath); + const newRelPath = parentRel === "." ? newName : `${parentRel}/${newName}`; + const validated = safeResolveNewPath(newRelPath); + if (!validated) { + return Response.json( + { error: "Invalid destination path" }, + { status: 400 }, + ); + } + + if (existsSync(newAbsPath)) { + return Response.json( + { error: `A file named '${newName}' already exists` }, + { status: 409 }, + ); + } + + try { + renameSync(absPath, newAbsPath); + return Response.json({ ok: true, oldPath: relPath, newPath: newRelPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Rename failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/reports/execute/route.ts b/apps/web/app/api/workspace/reports/execute/route.ts new file mode 100644 index 00000000000..fb6b8ed23a7 --- /dev/null +++ b/apps/web/app/api/workspace/reports/execute/route.ts @@ -0,0 +1,54 @@ +import { duckdbQuery } from "@/lib/workspace"; +import { buildFilterClauses, injectFilters, checkSqlSafety } from "@/lib/report-filters"; +import type { FilterEntry } from "@/lib/report-filters"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/workspace/reports/execute + * + * Body: { sql: string, filters?: FilterEntry[] } + * + * Executes a report panel's SQL query with optional filter injection. + * Only SELECT-compatible queries are allowed. + */ +export async function POST(req: Request) { + let body: { + sql?: string; + filters?: FilterEntry[]; + }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { sql, filters } = body; + if (!sql || typeof sql !== "string") { + return Response.json( + { error: "Missing 'sql' field in request body" }, + { status: 400 }, + ); + } + + // Basic SQL safety: reject mutation statements + const safetyError = checkSqlSafety(sql); + if (safetyError) { + return Response.json({ error: safetyError }, { status: 403 }); + } + + // Build filter clauses and inject into SQL + const filterClauses = buildFilterClauses(filters); + const finalSql = injectFilters(sql, filterClauses); + + try { + const rows = duckdbQuery(finalSql); + return Response.json({ rows, sql: finalSql }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Query execution failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/search-index/route.ts b/apps/web/app/api/workspace/search-index/route.ts new file mode 100644 index 00000000000..342cd44108e --- /dev/null +++ b/apps/web/app/api/workspace/search-index/route.ts @@ -0,0 +1,266 @@ +import { readdirSync, readFileSync, existsSync, type Dirent } from "node:fs"; +import { join } from "node:path"; +import { + resolveDenchRoot, + parseSimpleYaml, + duckdbQuery, + duckdbPath, + isDatabaseFile, +} from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +// --- Types --- + +export type SearchIndexItem = { + /** Unique key: relative path for files, entryId for entries */ + id: string; + /** Primary display text (filename or display-field value) */ + label: string; + /** Secondary text (path for files, object name for entries) */ + sublabel?: string; + /** Item kind for grouping and icons */ + kind: "file" | "object" | "entry"; + /** Icon hint */ + icon?: string; + + // Entry-specific + objectName?: string; + entryId?: string; + /** First few field key-value pairs for search and preview */ + fields?: Record; + + // File/object-specific + path?: string; + nodeType?: "document" | "folder" | "file" | "report" | "database"; +}; + +// --- DB types --- + +type ObjectRow = { + id: string; + name: string; + description?: string; + icon?: string; + default_view?: string; + display_field?: string; +}; + +type FieldRow = { + id: string; + name: string; + type: string; + sort_order?: number; +}; + +type EavRow = { + entry_id: string; + created_at: string; + updated_at: string; + field_name: string; + value: string | null; +}; + +// --- Helpers --- + +function sqlEscape(s: string): string { + return s.replace(/'/g, "''"); +} + +/** Determine the display field (same heuristic as the objects route). */ +function resolveDisplayField(obj: ObjectRow, fields: FieldRow[]): string { + if (obj.display_field) {return obj.display_field;} + + const nameField = fields.find( + (f) => /\bname\b/i.test(f.name) || /\btitle\b/i.test(f.name), + ); + if (nameField) {return nameField.name;} + + const textField = fields.find((f) => f.type === "text"); + if (textField) {return textField.name;} + + return fields[0]?.name ?? "id"; +} + +/** Flatten a tree recursively to produce file/object search items. */ +function flattenTree( + absDir: string, + relBase: string, + dbObjects: Map, + items: SearchIndexItem[], +) { + let entries: Dirent[]; + try { + entries = readdirSync(absDir, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (entry.name.startsWith(".")) {continue;} + + const absPath = join(absDir, entry.name); + const relPath = relBase ? `${relBase}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + const dbObj = dbObjects.get(entry.name); + // Check for .object.yaml + const yamlPath = join(absPath, ".object.yaml"); + const hasYaml = existsSync(yamlPath); + + if (dbObj || hasYaml) { + let icon: string | undefined; + if (hasYaml) { + try { + const parsed = parseSimpleYaml( + readFileSync(yamlPath, "utf-8"), + ); + icon = parsed.icon as string | undefined; + } catch { + // ignore + } + } + + items.push({ + id: relPath, + label: entry.name, + sublabel: relPath, + kind: "object", + icon: icon ?? dbObj?.icon, + path: relPath, + nodeType: undefined, + }); + } else { + // Regular folder -- don't add as item, but recurse + } + + flattenTree(absPath, relPath, dbObjects, items); + } else if (entry.isFile()) { + const isReport = entry.name.endsWith(".report.json"); + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = isDatabaseFile(entry.name); + + items.push({ + id: relPath, + label: entry.name.replace(/\.md$/, ""), + sublabel: relPath, + kind: "file", + path: relPath, + nodeType: isReport + ? "report" + : isDatabase + ? "database" + : isDocument + ? "document" + : "file", + }); + } + } +} + +/** Fetch all entries from all objects and produce search items. */ +function buildEntryItems(): SearchIndexItem[] { + const items: SearchIndexItem[] = []; + + const objects = duckdbQuery( + "SELECT * FROM objects ORDER BY name", + ); + + for (const obj of objects) { + const fields = duckdbQuery( + `SELECT * FROM fields WHERE object_id = '${sqlEscape(obj.id)}' ORDER BY sort_order`, + ); + const displayField = resolveDisplayField(obj, fields); + // Pick the first few text-like fields for searchable preview (max 4) + const previewFields = fields + .filter((f) => !["relation", "richtext"].includes(f.type)) + .slice(0, 4); + + // Try PIVOT view first, then raw EAV + let entries: Record[] = duckdbQuery( + `SELECT * FROM v_${obj.name} ORDER BY created_at DESC LIMIT 500`, + ); + + if (entries.length === 0) { + const rawRows = duckdbQuery( + `SELECT e.id as entry_id, e.created_at, e.updated_at, + f.name as field_name, ef.value + FROM entries e + JOIN entry_fields ef ON ef.entry_id = e.id + JOIN fields f ON f.id = ef.field_id + WHERE e.object_id = '${sqlEscape(obj.id)}' + ORDER BY e.created_at DESC + LIMIT 2500`, + ); + + // Pivot manually + const grouped = new Map>(); + for (const row of rawRows) { + let entry = grouped.get(row.entry_id); + if (!entry) { + entry = { entry_id: row.entry_id }; + grouped.set(row.entry_id, entry); + } + if (row.field_name) {entry[row.field_name] = row.value;} + } + entries = Array.from(grouped.values()); + } + + for (const entry of entries) { + const entryId = String(entry.entry_id ?? ""); + if (!entryId) {continue;} + + const displayValue = String(entry[displayField] ?? ""); + const fieldPreview: Record = {}; + for (const f of previewFields) { + const val = entry[f.name]; + if (val != null && val !== "") { + fieldPreview[f.name] = String(val); + } + } + + items.push({ + id: `entry:${obj.name}:${entryId}`, + label: displayValue || `(${obj.name} entry)`, + sublabel: obj.name, + kind: "entry", + icon: obj.icon, + objectName: obj.name, + entryId, + fields: Object.keys(fieldPreview).length > 0 ? fieldPreview : undefined, + }); + } + } + + return items; +} + +// --- Route handler --- + +export async function GET() { + const items: SearchIndexItem[] = []; + + // 1. Files + objects from tree + const root = resolveDenchRoot(); + if (root) { + const dbObjects = new Map(); + if (duckdbPath()) { + const objs = duckdbQuery( + "SELECT * FROM objects", + ); + for (const o of objs) {dbObjects.set(o.name, o);} + } + + // Scan entire dench root (the dench folder IS the knowledge base) + flattenTree(root, "", dbObjects, items); + } + + // 2. Entries from all objects + if (duckdbPath()) { + items.push(...buildEntryItems()); + } + + return Response.json({ items }); +} diff --git a/apps/web/app/api/workspace/tree/route.ts b/apps/web/app/api/workspace/tree/route.ts new file mode 100644 index 00000000000..43c6971d342 --- /dev/null +++ b/apps/web/app/api/workspace/tree/route.ts @@ -0,0 +1,336 @@ +import { readdirSync, readFileSync, existsSync, type Dirent } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; +import { resolveDenchRoot, parseSimpleYaml, duckdbQuery, isDatabaseFile } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export type TreeNode = { + name: string; + path: string; // relative to dench/ (or ~skills/, ~memories/, ~workspace/ for virtual nodes) + type: "object" | "document" | "folder" | "file" | "database" | "report"; + icon?: string; + defaultView?: "table" | "kanban"; + children?: TreeNode[]; + /** Virtual nodes live outside the dench workspace (e.g. Skills, Memories). */ + virtual?: boolean; +}; + +type DbObject = { + name: string; + icon?: string; + default_view?: string; +}; + +/** Read .object.yaml metadata from a directory if it exists. */ +function readObjectMeta( + dirPath: string, +): { icon?: string; defaultView?: string } | null { + const yamlPath = join(dirPath, ".object.yaml"); + if (!existsSync(yamlPath)) {return null;} + + try { + const content = readFileSync(yamlPath, "utf-8"); + const parsed = parseSimpleYaml(content); + return { + icon: parsed.icon as string | undefined, + defaultView: parsed.default_view as string | undefined, + }; + } catch { + return null; + } +} + +/** + * Query DuckDB for all objects so we can identify object directories + * even when .object.yaml files are missing. + */ +function loadDbObjects(): Map { + const map = new Map(); + const rows = duckdbQuery( + "SELECT name, icon, default_view FROM objects", + ); + for (const row of rows) { + map.set(row.name, row); + } + return map; +} + +/** Recursively build a tree from a workspace directory. */ +function buildTree( + absDir: string, + relativeBase: string, + dbObjects: Map, +): TreeNode[] { + const nodes: TreeNode[] = []; + + let entries: Dirent[]; + try { + entries = readdirSync(absDir, { withFileTypes: true }); + } catch { + return nodes; + } + + // Sort: directories first, then files, alphabetical within each group + const sorted = entries + .filter((e) => !e.name.startsWith(".") || e.name === ".object.yaml") + .toSorted((a, b) => { + if (a.isDirectory() && !b.isDirectory()) {return -1;} + if (!a.isDirectory() && b.isDirectory()) {return 1;} + return a.name.localeCompare(b.name); + }); + + for (const entry of sorted) { + // Skip hidden files except .object.yaml (but don't list it as a node) + if (entry.name === ".object.yaml") {continue;} + if (entry.name.startsWith(".")) {continue;} + + const absPath = join(absDir, entry.name); + const relPath = relativeBase + ? `${relativeBase}/${entry.name}` + : entry.name; + + if (entry.isDirectory()) { + const objectMeta = readObjectMeta(absPath); + const dbObject = dbObjects.get(entry.name); + const children = buildTree(absPath, relPath, dbObjects); + + if (objectMeta || dbObject) { + // This directory represents a CRM object (from .object.yaml OR DuckDB) + nodes.push({ + name: entry.name, + path: relPath, + type: "object", + icon: objectMeta?.icon ?? dbObject?.icon, + defaultView: + ((objectMeta?.defaultView ?? dbObject?.default_view) as + | "table" + | "kanban") ?? "table", + children: children.length > 0 ? children : undefined, + }); + } else { + // Regular folder + nodes.push({ + name: entry.name, + path: relPath, + type: "folder", + children: children.length > 0 ? children : undefined, + }); + } + } else if (entry.isFile()) { + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isReport = entry.name.endsWith(".report.json"); + const isDocument = ext === "md" || ext === "mdx"; + const isDatabase = isDatabaseFile(entry.name); + + nodes.push({ + name: entry.name, + path: relPath, + type: isReport ? "report" : isDatabase ? "database" : isDocument ? "document" : "file", + }); + } + } + + return nodes; +} + +// --- Virtual folder builders --- + +/** Parse YAML frontmatter from a SKILL.md file (lightweight). */ +function parseSkillFrontmatter(content: string): { name?: string; emoji?: string } { + const match = content.match(/^---\s*\n([\s\S]*?)\n---/); + if (!match) {return {};} + const yaml = match[1]; + const result: Record = {}; + for (const line of yaml.split("\n")) { + const kv = line.match(/^(\w+)\s*:\s*(.+)/); + if (kv) {result[kv[1]] = kv[2].replace(/^["']|["']$/g, "").trim();} + } + return { name: result.name, emoji: result.emoji }; +} + +/** Build a virtual "Skills" folder from ~/.openclaw/skills/ and ~/.openclaw/workspace/skills/. */ +function buildSkillsVirtualFolder(): TreeNode | null { + const home = homedir(); + const dirs = [ + join(home, ".openclaw", "skills"), + join(home, ".openclaw", "workspace", "skills"), + ]; + + const children: TreeNode[] = []; + const seen = new Set(); + + for (const dir of dirs) { + if (!existsSync(dir)) {continue;} + try { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory() || seen.has(entry.name)) {continue;} + const skillMdPath = join(dir, entry.name, "SKILL.md"); + if (!existsSync(skillMdPath)) {continue;} + + seen.add(entry.name); + let displayName = entry.name; + try { + const content = readFileSync(skillMdPath, "utf-8"); + const meta = parseSkillFrontmatter(content); + if (meta.name) {displayName = meta.name;} + if (meta.emoji) {displayName = `${meta.emoji} ${displayName}`;} + } catch { + // skip + } + + children.push({ + name: displayName, + path: `~skills/${entry.name}/SKILL.md`, + type: "document", + virtual: true, + }); + } + } catch { + // dir unreadable + } + } + + if (children.length === 0) {return null;} + children.sort((a, b) => a.name.localeCompare(b.name)); + + return { + name: "Skills", + path: "~skills", + type: "folder", + virtual: true, + children, + }; +} + +/** + * Build top-level workspace root file nodes (USER.md, SOUL.md, TOOLS.md, etc.). + * These live directly in ~/.openclaw/workspace/ but outside the dench/ subdirectory. + * They are virtual (not movable/renamable/deletable) but editable. + */ +function buildWorkspaceRootFiles(): TreeNode[] { + const workspaceDir = join(homedir(), ".openclaw", "workspace"); + if (!existsSync(workspaceDir)) {return [];} + + // Files already handled by the Memories virtual folder + const SKIP_FILES = new Set(["MEMORY.md", "memory.md"]); + + const nodes: TreeNode[] = []; + + try { + const entries = readdirSync(workspaceDir, { withFileTypes: true }); + for (const entry of entries) { + // Skip subdirectories (handled elsewhere) and hidden files + if (entry.isDirectory()) {continue;} + if (entry.name.startsWith(".")) {continue;} + if (SKIP_FILES.has(entry.name)) {continue;} + + const ext = entry.name.split(".").pop()?.toLowerCase(); + const isDocument = ext === "md" || ext === "mdx"; + + nodes.push({ + name: entry.name, + path: `~workspace/${entry.name}`, + type: isDocument ? "document" : "file", + virtual: true, + }); + } + } catch { + // dir unreadable + } + + // Sort alphabetically + nodes.sort((a, b) => a.name.localeCompare(b.name)); + return nodes; +} + +/** Build a virtual "Memories" folder from ~/.openclaw/workspace/. */ +function buildMemoriesVirtualFolder(): TreeNode | null { + const workspaceDir = join(homedir(), ".openclaw", "workspace"); + const children: TreeNode[] = []; + + // MEMORY.md + for (const filename of ["MEMORY.md", "memory.md"]) { + const memPath = join(workspaceDir, filename); + if (existsSync(memPath)) { + children.push({ + name: "MEMORY.md", + path: `~memories/MEMORY.md`, + type: "document", + virtual: true, + }); + break; + } + } + + // Daily logs from memory/ + const memoryDir = join(workspaceDir, "memory"); + if (existsSync(memoryDir)) { + try { + const entries = readdirSync(memoryDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".md")) {continue;} + children.push({ + name: entry.name, + path: `~memories/${entry.name}`, + type: "document", + virtual: true, + }); + } + } catch { + // dir unreadable + } + } + + if (children.length === 0) {return null;} + // Sort: MEMORY.md first, then reverse chronological for daily logs + children.sort((a, b) => { + if (a.name === "MEMORY.md") {return -1;} + if (b.name === "MEMORY.md") {return 1;} + return b.name.localeCompare(a.name); + }); + + return { + name: "Memories", + path: "~memories", + type: "folder", + virtual: true, + children, + }; +} + +export async function GET() { + const root = resolveDenchRoot(); + if (!root) { + // Even without a dench workspace, return virtual folders if they exist + const tree: TreeNode[] = []; + tree.push(...buildWorkspaceRootFiles()); + const skillsFolder = buildSkillsVirtualFolder(); + if (skillsFolder) {tree.push(skillsFolder);} + const memoriesFolder = buildMemoriesVirtualFolder(); + if (memoriesFolder) {tree.push(memoriesFolder);} + return Response.json({ tree, exists: false }); + } + + // Load objects from DuckDB for smart directory detection + const dbObjects = loadDbObjects(); + + // Scan the entire dench root -- the dench folder IS the knowledge base. + // All top-level directories (manufacturing, knowledge, reports, etc.) + // and files are visible in the sidebar. + const tree = buildTree(root, "", dbObjects); + + // Workspace root files (USER.md, SOUL.md, etc.) -- editable but reserved + const workspaceRootFiles = buildWorkspaceRootFiles(); + if (workspaceRootFiles.length > 0) {tree.push(...workspaceRootFiles);} + + // Virtual folders go after all real files/folders + const skillsFolder = buildSkillsVirtualFolder(); + if (skillsFolder) {tree.push(skillsFolder);} + const memoriesFolder = buildMemoriesVirtualFolder(); + if (memoriesFolder) {tree.push(memoriesFolder);} + + return Response.json({ tree, exists: true }); +} diff --git a/apps/web/app/api/workspace/upload/route.ts b/apps/web/app/api/workspace/upload/route.ts new file mode 100644 index 00000000000..84398db08c1 --- /dev/null +++ b/apps/web/app/api/workspace/upload/route.ts @@ -0,0 +1,86 @@ +import { writeFileSync, mkdirSync } from "node:fs"; +import { join, dirname, extname } from "node:path"; +import { resolveDenchRoot, safeResolveNewPath } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const ALLOWED_EXTENSIONS = new Set([ + ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", ".ico", +]); + +const MAX_SIZE = 10 * 1024 * 1024; // 10 MB + +/** + * POST /api/workspace/upload + * Accepts multipart form data with a "file" field. + * Saves to assets/- inside the workspace. + * Returns { ok, path } where path is workspace-relative. + */ +export async function POST(req: Request) { + const root = resolveDenchRoot(); + if (!root) { + return Response.json( + { error: "Workspace not found" }, + { status: 500 }, + ); + } + + let formData: FormData; + try { + formData = await req.formData(); + } catch { + return Response.json({ error: "Invalid form data" }, { status: 400 }); + } + + const file = formData.get("file"); + if (!file || !(file instanceof File)) { + return Response.json( + { error: "Missing 'file' field" }, + { status: 400 }, + ); + } + + // Validate extension + const ext = extname(file.name).toLowerCase(); + if (!ALLOWED_EXTENSIONS.has(ext)) { + return Response.json( + { error: `File type ${ext} is not allowed` }, + { status: 400 }, + ); + } + + // Validate size + if (file.size > MAX_SIZE) { + return Response.json( + { error: "File is too large (max 10 MB)" }, + { status: 400 }, + ); + } + + // Build a safe filename: timestamp + sanitized original name + const safeName = file.name + .replace(/[^a-zA-Z0-9._-]/g, "_") + .replace(/_{2,}/g, "_"); + const relPath = join("assets", `${Date.now()}-${safeName}`); + + const absPath = safeResolveNewPath(relPath); + if (!absPath) { + return Response.json( + { error: "Invalid path" }, + { status: 400 }, + ); + } + + try { + mkdirSync(dirname(absPath), { recursive: true }); + const buffer = Buffer.from(await file.arrayBuffer()); + writeFileSync(absPath, buffer); + return Response.json({ ok: true, path: relPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Upload failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/virtual-file/route.ts b/apps/web/app/api/workspace/virtual-file/route.ts new file mode 100644 index 00000000000..a5f840e9327 --- /dev/null +++ b/apps/web/app/api/workspace/virtual-file/route.ts @@ -0,0 +1,160 @@ +import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { join, dirname, resolve, normalize } from "node:path"; +import { homedir } from "node:os"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * Resolve a virtual path (~skills/... or ~memories/...) to an absolute filesystem path. + * Returns null if the path is invalid or tries to escape. + */ +function resolveVirtualPath(virtualPath: string): string | null { + const home = homedir(); + + if (virtualPath.startsWith("~skills/")) { + // ~skills//SKILL.md + const rest = virtualPath.slice("~skills/".length); + // Validate: must be /SKILL.md + const parts = rest.split("/"); + if (parts.length !== 2 || parts[1] !== "SKILL.md" || !parts[0]) { + return null; + } + const skillName = parts[0]; + // Prevent path traversal + if (skillName.includes("..") || skillName.includes("/")) { + return null; + } + + // Check workspace skills first, then managed skills + const candidates = [ + join(home, ".openclaw", "workspace", "skills", skillName, "SKILL.md"), + join(home, ".openclaw", "skills", skillName, "SKILL.md"), + ]; + for (const candidate of candidates) { + if (existsSync(candidate)) { + return candidate; + } + } + // Default to workspace skills dir for new files + return candidates[0]; + } + + if (virtualPath.startsWith("~memories/")) { + const rest = virtualPath.slice("~memories/".length); + // Prevent path traversal + if (rest.includes("..") || rest.includes("/")) { + return null; + } + + const workspaceDir = join(home, ".openclaw", "workspace"); + + if (rest === "MEMORY.md") { + // Check both casing + for (const filename of ["MEMORY.md", "memory.md"]) { + const candidate = join(workspaceDir, filename); + if (existsSync(candidate)) { + return candidate; + } + } + // Default to MEMORY.md for new files + return join(workspaceDir, "MEMORY.md"); + } + + // Daily log: must be a .md file in the memory/ subdirectory + if (!rest.endsWith(".md")) { + return null; + } + return join(workspaceDir, "memory", rest); + } + + if (virtualPath.startsWith("~workspace/")) { + const rest = virtualPath.slice("~workspace/".length); + // Only allow direct filenames (no subdirectories, no traversal) + if (!rest || rest.includes("..") || rest.includes("/")) { + return null; + } + return join(home, ".openclaw", "workspace", rest); + } + + return null; +} + +/** + * Double-check that the resolved path stays within expected directories. + */ +function isSafePath(absPath: string): boolean { + const home = homedir(); + const normalized = normalize(resolve(absPath)); + const allowed = [ + normalize(join(home, ".openclaw", "skills")), + normalize(join(home, ".openclaw", "workspace", "skills")), + normalize(join(home, ".openclaw", "workspace")), + ]; + return allowed.some((dir) => normalized.startsWith(dir)); +} + +export async function GET(req: Request) { + const url = new URL(req.url); + const path = url.searchParams.get("path"); + + if (!path) { + return Response.json({ error: "Missing 'path' query parameter" }, { status: 400 }); + } + + const absPath = resolveVirtualPath(path); + if (!absPath || !isSafePath(absPath)) { + return Response.json({ error: "Invalid virtual path" }, { status: 400 }); + } + + if (!existsSync(absPath)) { + return Response.json({ error: "File not found" }, { status: 404 }); + } + + try { + const content = readFileSync(absPath, "utf-8"); + const ext = absPath.split(".").pop()?.toLowerCase(); + let type: "markdown" | "yaml" | "text" = "text"; + if (ext === "md" || ext === "mdx") {type = "markdown";} + else if (ext === "yaml" || ext === "yml") {type = "yaml";} + return Response.json({ content, type }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Read failed" }, + { status: 500 }, + ); + } +} + +export async function POST(req: Request) { + let body: { path?: string; content?: string }; + try { + body = await req.json(); + } catch { + return Response.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { path: virtualPath, content } = body; + if (!virtualPath || typeof virtualPath !== "string" || typeof content !== "string") { + return Response.json( + { error: "Missing 'path' and 'content' fields" }, + { status: 400 }, + ); + } + + const absPath = resolveVirtualPath(virtualPath); + if (!absPath || !isSafePath(absPath)) { + return Response.json({ error: "Invalid virtual path" }, { status: 400 }); + } + + try { + mkdirSync(dirname(absPath), { recursive: true }); + writeFileSync(absPath, content, "utf-8"); + return Response.json({ ok: true, path: virtualPath }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "Write failed" }, + { status: 500 }, + ); + } +} diff --git a/apps/web/app/api/workspace/watch/route.ts b/apps/web/app/api/workspace/watch/route.ts new file mode 100644 index 00000000000..72e0e2e6710 --- /dev/null +++ b/apps/web/app/api/workspace/watch/route.ts @@ -0,0 +1,106 @@ +import { resolveDenchRoot } from "@/lib/workspace"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * GET /api/workspace/watch + * + * Server-Sent Events endpoint that watches the dench workspace for file changes. + * Sends events: { type: "add"|"change"|"unlink"|"addDir"|"unlinkDir", path: string } + * Falls back gracefully if chokidar is unavailable. + */ +export async function GET() { + const root = resolveDenchRoot(); + if (!root) { + return new Response("Workspace not found", { status: 404 }); + } + + const encoder = new TextEncoder(); + + const stream = new ReadableStream({ + async start(controller) { + // Send initial heartbeat so the client knows the connection is alive + controller.enqueue(encoder.encode("event: connected\ndata: {}\n\n")); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let watcher: any = null; + let closed = false; + + // Debounce: batch rapid events into a single "refresh" signal + let debounceTimer: ReturnType | null = null; + + function sendEvent(type: string, filePath: string) { + if (closed) {return;} + if (debounceTimer) {clearTimeout(debounceTimer);} + debounceTimer = setTimeout(() => { + if (closed) {return;} + try { + const data = JSON.stringify({ type, path: filePath }); + controller.enqueue(encoder.encode(`event: change\ndata: ${data}\n\n`)); + } catch { + // Stream may have been closed + } + }, 200); + } + + // Keep-alive heartbeat every 30s to prevent proxy/timeout disconnects + const heartbeat = setInterval(() => { + if (closed) {return;} + try { + controller.enqueue(encoder.encode(": heartbeat\n\n")); + } catch { + // Ignore if closed + } + }, 30_000); + + try { + // Dynamic import so the route still compiles if chokidar is missing + const chokidar = await import("chokidar"); + watcher = chokidar.watch(root, { + ignoreInitial: true, + awaitWriteFinish: { stabilityThreshold: 150, pollInterval: 50 }, + ignored: [ + /(^|[\\/])node_modules([\\/]|$)/, + /\.duckdb\.wal$/, + /\.duckdb\.tmp$/, + ], + depth: 10, + }); + + watcher.on("all", (eventType: string, filePath: string) => { + // Make path relative to workspace root + const rel = filePath.startsWith(root) + ? filePath.slice(root.length + 1) + : filePath; + sendEvent(eventType, rel); + }); + } catch { + // chokidar not available, send a fallback event and close + controller.enqueue( + encoder.encode("event: error\ndata: {\"error\":\"File watching unavailable\"}\n\n"), + ); + } + + // Cleanup when the client disconnects + // The cancel callback is invoked by the runtime when the response is aborted + const originalCancel = stream.cancel?.bind(stream); + stream.cancel = async (reason) => { + closed = true; + clearInterval(heartbeat); + if (debounceTimer) {clearTimeout(debounceTimer);} + if (watcher) {await watcher.close();} + if (originalCancel) {return originalCancel(reason);} + }; + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }, + }); +} diff --git a/apps/web/app/components/chain-of-thought.tsx b/apps/web/app/components/chain-of-thought.tsx index 9670aa43392..c7fc3c13552 100644 --- a/apps/web/app/components/chain-of-thought.tsx +++ b/apps/web/app/components/chain-of-thought.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; /* ─── Public types ─── */ @@ -14,8 +14,395 @@ export type ChainPart = args?: Record; output?: Record; errorText?: string; + } + | { + kind: "status"; + label: string; + isActive: boolean; }; +/* ─── Media / file type helpers ─── */ + +const IMAGE_EXTS = new Set([ + "jpg", + "jpeg", + "png", + "gif", + "webp", + "svg", + "bmp", + "avif", + "heic", + "heif", + "tiff", + "tif", + "ico", +]); +const VIDEO_EXTS = new Set([ + "mp4", + "webm", + "mov", + "avi", + "mkv", +]); +const PDF_EXTS = new Set(["pdf"]); +const AUDIO_EXTS = new Set(["mp3", "wav", "ogg", "m4a"]); + +type MediaKind = "image" | "video" | "pdf" | "audio" | null; + +function getFileExt(path: string): string { + return (path.split(".").pop() ?? "").toLowerCase(); +} + +function detectMedia(path: string): MediaKind { + const ext = getFileExt(path); + if (IMAGE_EXTS.has(ext)) {return "image";} + if (VIDEO_EXTS.has(ext)) {return "video";} + if (PDF_EXTS.has(ext)) {return "pdf";} + if (AUDIO_EXTS.has(ext)) {return "audio";} + return null; +} + +function rawFileUrl(path: string): string { + return `/api/workspace/raw-file?path=${encodeURIComponent(path)}`; +} + +/** Resolve a media URL — use raw URL directly if it's already HTTP */ +function resolveMediaUrl(path: string): string { + if (path.startsWith("http://") || path.startsWith("https://")) { + return path; + } + return rawFileUrl(path); +} + +/** Regex to find file paths with media extensions in free text */ +const MEDIA_FILE_RE = + /(?:^|[\s"'(=])(((?:\/|\.\/)?[\w.\-/\\]+)\.(?:jpe?g|png|gif|webp|svg|bmp|avif|heic|heif|tiff?|ico|mp4|webm|mov|avi|mkv|mp3|wav|ogg|m4a|pdf))\b/i; + +const PATH_KEYS = [ + "path", + "file", + "file_path", + "filePath", + "filename", + "url", + "src", + "name", + "target", +]; + +/** + * Extract the file path from tool args and/or output. + * Searches standard keys, then all string values, then output text. + */ +function getFilePath( + args?: Record, + output?: Record, +): string | null { + // 1. Check standard keys in args + if (args) { + for (const key of PATH_KEYS) { + const v = args[key]; + if (typeof v === "string" && v.length > 0) {return v;} + } + } + + // 2. Check standard keys in output + if (output) { + for (const key of PATH_KEYS) { + const v = output[key]; + if (typeof v === "string" && v.length > 0 && looksLikePath(v)) + {return v;} + } + } + + // 3. Scan all string values in args for file-like paths + if (args) { + const found = findPathInValues(args); + if (found) {return found;} + } + + // 4. Extract from output text + if (output?.text && typeof output.text === "string") { + const m = output.text.match(MEDIA_FILE_RE); + if (m) {return m[1];} + } + + // 5. Scan output values too + if (output) { + const found = findPathInValues(output); + if (found) {return found;} + } + + return null; +} + +/** Check if a string looks like a file path (has an extension, no spaces) */ +function looksLikePath(s: string): boolean { + return ( + s.length > 2 && + s.length < 500 && + /\.\w{1,5}$/.test(s) && + !s.includes(" ") + ); +} + +/** Search all string values in an object for a path-like string */ +function findPathInValues(obj: Record): string | null { + for (const val of Object.values(obj)) { + if (typeof val === "string" && looksLikePath(val)) { + return val; + } + } + return null; +} + +/* ─── Domain / URL extraction helpers ─── */ + +const URL_RE = /https?:\/\/[^\s"'<>,;)}\]]+/gi; + +function extractDomains(text: string): string[] { + const urls = text.match(URL_RE) ?? []; + const domains = new Set(); + for (const url of urls) { + try { + const hostname = new URL(url).hostname; + if (hostname && !hostname.includes("localhost")) { + domains.add(hostname); + } + } catch { + /* skip */ + } + } + return [...domains].slice(0, 8); +} + +function faviconUrl(domain: string): string { + return `https://www.google.com/s2/favicons?domain=${encodeURIComponent(domain)}&sz=32`; +} + +/* ─── Classify tool steps ─── */ + +type StepKind = + | "search" + | "fetch" + | "read" + | "exec" + | "write" + | "image" + | "generic"; + +function classifyTool(name: string): StepKind { + const n = name.toLowerCase().replace(/[_-]/g, ""); + if ( + [ + "websearch", + "search", + "googlesearch", + "bingsearch", + "browsersearch", + "tavily", + ].some((k) => n.includes(k)) + ) + {return "search";} + if ( + ["fetchurl", "fetch", "browse", "browseurl", "webfetch"].some( + (k) => n.includes(k), + ) + ) + {return "fetch";} + if ( + ["read", "file", "readfile", "getfile"].some( + (k) => n.includes(k), + ) + ) + {return "read";} + if ( + [ + "bash", + "shell", + "execute", + "exec", + "terminal", + "command", + "run", + ].some((k) => n.includes(k)) + ) + {return "exec";} + if ( + [ + "write", + "create", + "edit", + "str_replace", + "save", + "patch", + ].some((k) => n.includes(k)) + ) + {return "write";} + if ( + [ + "image", + "screenshot", + "photo", + "picture", + "dalle", + "generateimage", + ].some((k) => n.includes(k)) + ) + {return "image";} + return "generic"; +} + +function buildStepLabel( + kind: StepKind, + toolName: string, + args?: Record, + output?: Record, +): string { + const strVal = (key: string) => { + const v = args?.[key]; + return typeof v === "string" && v.length > 0 ? v : null; + }; + + switch (kind) { + case "search": { + const q = + strVal("query") ?? + strVal("search_query") ?? + strVal("search") ?? + strVal("q"); + return q + ? `Searching for ${q.length > 60 ? q.slice(0, 60) + "..." : q}` + : "Searching..."; + } + case "fetch": { + const u = + strVal("url") ?? strVal("path") ?? strVal("src"); + if (u) { + try { + return `Fetching ${new URL(u).hostname}`; + } catch { + return `Fetching ${u.length > 50 ? u.slice(0, 50) + "..." : u}`; + } + } + return "Fetching page"; + } + case "read": { + const p = getFilePath(args, output); + if (p) { + const short = p.split("/").pop() ?? p; + return short.startsWith("http") + ? `Fetching ${short.slice(0, 50)}` + : `Reading ${short}`; + } + return "Reading file"; + } + case "exec": { + const cmd = strVal("command") ?? strVal("cmd"); + if (cmd) { + const short = + cmd.length > 60 ? cmd.slice(0, 60) + "..." : cmd; + return `Running: ${short}`; + } + return "Running command"; + } + case "write": { + const p = strVal("path") ?? strVal("file") ?? strVal("file_path"); + if (p) { + const short = p.split("/").pop() ?? p; + return `Editing ${short}`; + } + return "Editing file"; + } + case "image": + return strVal("description") + ? `Generating image: ${strVal("description")!.slice(0, 50)}` + : "Generating image"; + default: + return toolName + .replace(/_/g, " ") + .replace(/\b\w/g, (c) => c.toUpperCase()) + .trim(); + } +} + +/** Extract domains from tool output for search steps */ +function getSearchDomains( + output?: Record, +): string[] { + if (!output) {return [];} + const text = typeof output.text === "string" ? output.text : ""; + const results = output.results; + let combined = text; + if (Array.isArray(results)) { + for (const r of results) { + if (typeof r === "object" && r !== null) { + const obj = r as Record; + if (typeof obj.url === "string") + {combined += ` ${obj.url}`;} + if (typeof obj.link === "string") + {combined += ` ${obj.link}`;} + } + } + } + return extractDomains(combined); +} + +/* ─── Group consecutive media reads ─── */ + +type ToolPart = Extract; + +type VisualItem = + | { type: "tool"; tool: ToolPart } + | { + type: "media-group"; + mediaKind: "image" | "video" | "pdf" | "audio"; + items: Array<{ path: string; tool: ToolPart }>; + }; + +function groupToolSteps(tools: ToolPart[]): VisualItem[] { + const result: VisualItem[] = []; + let i = 0; + while (i < tools.length) { + const tool = tools[i]; + const kind = classifyTool(tool.toolName); + // Check both args AND output for the file path + const filePath = getFilePath(tool.args, tool.output); + const media = filePath ? detectMedia(filePath) : null; + + // If this is a media read, look for consecutive media reads of the same kind + if (kind === "read" && media && filePath) { + const group: Array<{ path: string; tool: ToolPart }> = [ + { path: filePath, tool }, + ]; + let j = i + 1; + while (j < tools.length) { + const next = tools[j]; + const nextKind = classifyTool(next.toolName); + const nextPath = getFilePath(next.args, next.output); + const nextMedia = nextPath ? detectMedia(nextPath) : null; + if (nextKind === "read" && nextMedia === media && nextPath) { + group.push({ path: nextPath, tool: next }); + j++; + } else { + break; + } + } + result.push({ + type: "media-group", + mediaKind: media, + items: group, + }); + i = j; + } else { + result.push({ type: "tool", tool }); + i++; + } + } + return result; +} + /* ─── Main component ─── */ export function ChainOfThought({ parts }: { parts: ChainPart[] }) { @@ -25,10 +412,39 @@ export function ChainOfThought({ parts }: { parts: ChainPart[] }) { const isActive = parts.some( (p) => (p.kind === "reasoning" && p.isStreaming) || - (p.kind === "tool" && p.status === "running"), + (p.kind === "tool" && p.status === "running") || + (p.kind === "status" && p.isActive), ); - // Auto-collapse once all steps finish (active → inactive transition) + /* ─── Live elapsed-time tracking ─── */ + const startRef = useRef(null); + const [elapsed, setElapsed] = useState(0); + + useEffect(() => { + if (isActive && startRef.current === null) { + startRef.current = Date.now(); + } + }, [isActive]); + + useEffect(() => { + if (!isActive) {return;} + const tick = () => { + if (startRef.current !== null) { + setElapsed(Math.floor((Date.now() - startRef.current) / 1000)); + } + }; + tick(); + const id = setInterval(tick, 1000); + return () => clearInterval(id); + }, [isActive]); + + const formatDuration = useCallback((s: number) => { + if (s < 60) {return `${s}s`;} + const m = Math.floor(s / 60); + const rem = s % 60; + return rem > 0 ? `${m}m ${rem}s` : `${m}m`; + }, []); + useEffect(() => { if (prevActiveRef.current && !isActive && parts.length > 0) { setIsOpen(false); @@ -36,7 +452,10 @@ export function ChainOfThought({ parts }: { parts: ChainPart[] }) { prevActiveRef.current = isActive; }, [isActive, parts.length]); - // Aggregate reasoning text from all reasoning parts + const statusParts = parts.filter( + (p): p is Extract => + p.kind === "status", + ); const reasoningText = parts .filter( (p): p is Extract => @@ -48,78 +467,126 @@ export function ChainOfThought({ parts }: { parts: ChainPart[] }) { (p) => p.kind === "reasoning" && p.isStreaming, ); - // Tool steps const tools = parts.filter( - (p): p is Extract => p.kind === "tool", + (p): p is ToolPart => p.kind === "tool", ); - const completedTools = tools.filter((t) => t.status === "done").length; - const activeTool = tools.find((t) => t.status === "running"); + const visualItems = groupToolSteps(tools); - // Header label summarizes current/completed activity - let headerLabel: string; - if (isActive) { - if (activeTool) { - // Show what the active tool is doing - const summary = getToolSummary( - activeTool.toolName, - activeTool.args, - ); - headerLabel = summary || formatToolName(activeTool.toolName); - } else { - headerLabel = "Thinking"; - } - } else if (tools.length > 0) { - headerLabel = `Reasoned with ${completedTools} tool${completedTools !== 1 ? "s" : ""}`; - } else { - headerLabel = "Reasoned"; - } + // Derive a more descriptive header from status parts + const activeStatus = statusParts.find((s) => s.isActive); + const headerLabel = isActive + ? activeStatus + ? elapsed > 0 + ? `${activeStatus.label} ${formatDuration(elapsed)}` + : activeStatus.label + : elapsed > 0 + ? `Thinking... ${formatDuration(elapsed)}` + : "Thinking..." + : elapsed > 0 + ? `Thought for ${formatDuration(elapsed)}` + : "Thought"; return ( -
- {/* Trigger */} +
+ {/* Header trigger */} - {/* Collapsible content (smooth CSS grid animation) */} + {/* Collapsible content */}
-
- {/* Reasoning text block */} - {reasoningText && ( - + {/* Timeline connector line */} +
+ {statusParts.map((sp, idx) => ( + - )} - - {/* Tool step timeline */} - {tools.length > 0 && ( -
- {tools.map((tool) => ( - - ))} + ))} + {reasoningText && ( +
+
+ + + + +
+
+ +
)} + {visualItems.map((item, idx) => { + if (item.type === "media-group") { + return ( + + ); + } + return ( + + ); + })}
@@ -127,10 +594,9 @@ export function ChainOfThought({ parts }: { parts: ChainPart[] }) { ); } -/* ─── Sub-components ─── */ +/* ─── Reasoning block ─── */ -/** Expandable reasoning text display */ -function ReasoningText({ +function ReasoningBlock({ text, isStreaming, }: { @@ -138,27 +604,34 @@ function ReasoningText({ isStreaming: boolean; }) { const [expanded, setExpanded] = useState(false); - const isLong = text.length > 300; + const isLong = text.length > 400; return ( -
+
{text} {isStreaming && ( - + )}
{isLong && !expanded && ( @@ -167,7 +640,324 @@ function ReasoningText({ ); } -/** Rich tool step with args display and collapsible output */ +/* ─── Status step (lifecycle / compaction indicators) ─── */ + +function StatusStep({ + label, + isActive, +}: { + label: string; + isActive: boolean; +}) { + return ( +
+
+ {isActive ? ( + + ) : ( + + + + )} +
+ + {label} + +
+ ); +} + +/* ─── Media group (images, videos, PDFs, audio) ─── */ + +function MediaGroup({ + mediaKind, + items, +}: { + mediaKind: "image" | "video" | "pdf" | "audio"; + items: Array<{ path: string; tool: ToolPart }>; +}) { + const [expanded, setExpanded] = useState(false); + const anyRunning = items.some( + (i) => i.tool.status === "running", + ); + + // Show completed items progressively — don't wait for allDone + const completedItems = items.filter( + (i) => i.tool.status === "done", + ); + const doneCount = completedItems.length; + + const label = anyRunning + ? `Reading ${items.length} ${mediaKind}${items.length > 1 ? "s" : ""}...` + : mediaKind === "image" + ? items.length === 1 + ? `Read 1 image` + : `Read ${items.length} images` + : mediaKind === "video" + ? items.length === 1 + ? `Read 1 video` + : `Read ${items.length} videos` + : mediaKind === "pdf" + ? items.length === 1 + ? `Read 1 PDF` + : `Read ${items.length} PDFs` + : items.length === 1 + ? `Read 1 audio file` + : `Read ${items.length} audio files`; + + // Show up to 6 thumbnails by default, expandable + const PREVIEW_COUNT = 6; + const displayItems = expanded + ? completedItems + : completedItems.slice(0, PREVIEW_COUNT); + const hasMore = + completedItems.length > PREVIEW_COUNT && !expanded; + + return ( +
+
+ {anyRunning ? ( + + ) : ( + + )} +
+
+
+ {label} +
+ + {/* Image thumbnail grid — show progressively as items complete */} + {doneCount > 0 && mediaKind === "image" && ( +
+ {displayItems.map((item) => ( + + ))} + {anyRunning && ( +
+ +
+ )} + {hasMore && ( + + )} +
+ )} + + {/* Video inline */} + {doneCount > 0 && mediaKind === "video" && ( +
+ {displayItems.map((item) => ( +
+ )} + + {/* PDF links */} + {doneCount > 0 && mediaKind === "pdf" && ( +
+ {displayItems.map((item) => { + const filename = + item.path.split("/").pop() ?? + item.path; + return ( + + + + {filename} + + + ); + })} +
+ )} + + {/* Audio inline */} + {doneCount > 0 && mediaKind === "audio" && ( +
+ {displayItems.map((item) => ( +
+ )} +
+
+ ); +} + +/** Image thumbnail with error fallback */ +function MediaThumb({ + path, + single, +}: { + path: string; + single: boolean; +}) { + const [error, setError] = useState(false); + const filename = path.split("/").pop() ?? path; + const url = resolveMediaUrl(path); + const w = single ? 200 : 80; + const h = single ? 150 : 80; + + if (error) { + return ( +
+ {filename} +
+ ); + } + + return ( + + {/* eslint-disable-next-line @next/next/no-img-element */} + {filename} setError(true)} + /> + + ); +} + +/* ─── Tool step (non-media) ─── */ + function ToolStep({ toolName, status, @@ -182,251 +972,407 @@ function ToolStep({ errorText?: string; }) { const [showOutput, setShowOutput] = useState(false); - const displayType = getToolDisplayType(toolName); - const primaryArg = getPrimaryArg(toolName, args); + const kind = classifyTool(toolName); + const label = buildStepLabel(kind, toolName, args, output); + const domains = + (kind === "search" || kind === "fetch") && status === "done" + ? getSearchDomains(output) + : []; const outputText = typeof output?.text === "string" ? output.text : undefined; - const exitCode = - output?.exitCode !== undefined ? Number(output.exitCode) : undefined; + + // For single-file reads that are media, render inline preview + const filePath = getFilePath(args, output); + const media = filePath ? detectMedia(filePath) : null; + const isSingleMedia = kind === "read" && media && status === "done"; return ( -
- {/* Tool name + status */} -
- {status === "running" && ( - - )} - {status === "done" && ( - - )} - {status === "error" && ( - - )} - - - {formatToolName(toolName)} - - - {/* Exit code badge for bash/exec tools */} - {exitCode !== undefined && exitCode !== 0 && ( - - exit {exitCode} - +
+
+ {status === "running" ? ( + + ) : status === "error" ? ( + + ) : ( + )}
- {/* Primary argument: command, path, query, code, etc. */} - {primaryArg && ( -
- {displayType === "bash" ? ( - - ) : displayType === "code" ? ( - - ) : ( -
- {primaryArg} +
+
+ {label} +
+ + {/* Single media inline preview (when not grouped) */} + {isSingleMedia && filePath && media === "image" && ( +
+ +
+ )} + + {isSingleMedia && filePath && media === "video" && ( +
- )} - {/* Error message */} - {status === "error" && errorText && ( -
- {errorText} -
- )} - - {/* Tool output */} - {outputText && status === "done" && ( -
- - {showOutput && ( - + {errorText} +
+ )} + + {/* Output toggle — skip for media files and search */} + {outputText && + status === "done" && + kind !== "search" && + !isSingleMedia && ( +
+ + {showOutput && ( +
+									{outputText.length > 2000
+										? outputText.slice(0, 2000) +
+											"\n..."
+										: outputText}
+								
+ )} +
)} -
- )} +
); } -/** Monospace code block with optional line limit */ -function CodeBlock({ - content, - maxLines = 10, -}: { - content: string; - maxLines?: number; -}) { - const [expanded, setExpanded] = useState(false); - const lines = content.split("\n"); - const isLong = lines.length > maxLines; - const displayContent = - !expanded && isLong - ? lines.slice(0, maxLines).join("\n") + "\n..." - : content; +/* ─── Domain badge with favicon ─── */ +function DomainBadge({ domain }: { domain: string }) { + const short = domain.replace(/^www\./, ""); return ( -
-
-				{displayContent}
-			
- {isLong && !expanded && ( - - )} -
+ + {/* eslint-disable-next-line @next/next/no-img-element */} + + {short} + ); } -/* ─── Tool classification helpers ─── */ +/* ─── Step icons ─── */ -type ToolDisplayType = "bash" | "code" | "file" | "search" | "generic"; +function StepIcon({ kind }: { kind: StepKind }) { + const color = "var(--color-text-muted)"; + const size = 16; -function getToolDisplayType(toolName: string): ToolDisplayType { - const name = toolName.toLowerCase().replace(/[_-]/g, ""); - if ( - ["bash", "shell", "execute", "exec", "terminal", "command"].some((k) => - name.includes(k), - ) - ) - return "bash"; - if ( - ["runcode", "python", "javascript", "typescript", "notebook"].some( - (k) => name.includes(k), - ) - ) - return "code"; - if ( - ["file", "read", "write", "create", "edit", "str_replace"].some((k) => - name.includes(k), - ) - ) - return "file"; - if ( - ["search", "web", "grep", "find", "glob"].some((k) => - name.includes(k), - ) - ) - return "search"; - return "generic"; -} - -function getPrimaryArg( - toolName: string, - args?: Record, -): string | undefined { - if (!args) return undefined; - const type = getToolDisplayType(toolName); - switch (type) { - case "bash": - return strArg(args, "command") ?? strArg(args, "cmd"); - case "code": - return strArg(args, "code") ?? strArg(args, "script"); - case "file": - return ( - strArg(args, "path") ?? - strArg(args, "file") ?? - strArg(args, "file_path") - ); + switch (kind) { case "search": return ( - strArg(args, "query") ?? - strArg(args, "search") ?? - strArg(args, "pattern") ?? - strArg(args, "q") + + + + + ); + case "fetch": + return ( + + + + + + ); + case "read": + return ( + + + + + ); + case "exec": + return ( + + + + + ); + case "write": + return ( + + + + + ); + case "image": + return ( + + + + + ); - default: { - // Return first short string arg - for (const val of Object.values(args)) { - if (typeof val === "string" && val.length > 0 && val.length < 200) return val; - } - return undefined; - } - } -} - -/** Safely extract a string value from an args object */ -function strArg( - args: Record, - key: string, -): string | undefined { - const val = args[key]; - return typeof val === "string" && val.length > 0 ? val : undefined; -} - -/** Build a short summary for the active tool (shown in collapsed header) */ -function getToolSummary( - toolName: string, - args?: Record, -): string | undefined { - if (!args) return undefined; - const type = getToolDisplayType(toolName); - const primary = getPrimaryArg(toolName, args); - if (!primary) return undefined; - - switch (type) { - case "bash": { - // Show first 40 chars of command - const short = - primary.length > 40 ? primary.slice(0, 40) + "..." : primary; - return `Running: ${short}`; - } - case "file": { - return `Reading ${primary.split("/").pop()}`; - } - case "search": { - return `Searching: ${primary}`; - } default: - return undefined; + return ( + + + + ); } } -/* ─── Helpers ─── */ - -/** Convert tool_name_like_this → Tool Name Like This */ -function formatToolName(name: string): string { - return name - .replace(/_/g, " ") - .replace(/\b\w/g, (c) => c.toUpperCase()) - .trim(); +function ErrorCircleIcon() { + return ( + + + + + + ); } -/* ─── Inline SVG icons (avoids adding lucide-react dep) ─── */ +function PdfIcon() { + return ( + + + + + + + + ); +} -function SparkleIcon({ className }: { className?: string }) { +/* ─── Header icons ─── */ + +function ThinkingIcon({ className }: { className?: string }) { return ( - + + + ); } @@ -446,35 +1392,3 @@ function ChevronIcon({ className }: { className?: string }) { ); } - -function CheckIcon({ className }: { className?: string }) { - return ( - - - - ); -} - -function XIcon({ className }: { className?: string }) { - return ( - - - - ); -} diff --git a/apps/web/app/components/charts/chart-panel.tsx b/apps/web/app/components/charts/chart-panel.tsx new file mode 100644 index 00000000000..3c38ad49768 --- /dev/null +++ b/apps/web/app/components/charts/chart-panel.tsx @@ -0,0 +1,414 @@ +"use client"; + +import { useMemo } from "react"; +import { + BarChart, + Bar, + LineChart, + Line, + AreaChart, + Area, + PieChart, + Pie, + Cell, + RadarChart, + Radar, + PolarGrid, + PolarAngleAxis, + PolarRadiusAxis, + ScatterChart, + Scatter, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + Legend, + ResponsiveContainer, + FunnelChart, + Funnel, + LabelList, +} from "recharts"; +import type { PanelConfig } from "./types"; + +// --- Color palette derived from CSS variables + accessible defaults --- + +const CHART_PALETTE = [ + "#2563eb", // accent + "#60a5fa", // blue + "#22c55e", // green + "#f59e0b", // amber + "#c084fc", // purple + "#fb923c", // orange + "#14b8a6", // teal + "#f43f5e", // rose + "#a78bfa", // violet + "#38bdf8", // sky +]; + +type ChartPanelProps = { + config: PanelConfig; + data: Record[]; + /** Compact mode for inline chat cards */ + compact?: boolean; +}; + +// --- Shared tooltip/axis styles --- + +const axisStyle = { + fontSize: 11, + fill: "var(--color-text-muted)", +}; + +const gridStyle = { + stroke: "var(--color-border-strong)", + strokeDasharray: "3 3", +}; + +function tooltipStyle() { + return { + contentStyle: { + background: "var(--color-surface)", + border: "1px solid var(--color-border)", + borderRadius: 8, + fontSize: 12, + color: "var(--color-text)", + }, + itemStyle: { color: "var(--color-text)" }, + labelStyle: { color: "var(--color-text-muted)", marginBottom: 4 }, + }; +} + +// --- Formatters --- + +function formatValue(val: unknown): string { + if (val === null || val === undefined) {return "";} + if (typeof val === "number") { + if (Math.abs(val) >= 1_000_000) {return `${(val / 1_000_000).toFixed(1)}M`;} + if (Math.abs(val) >= 1_000) {return `${(val / 1_000).toFixed(1)}K`;} + return Number.isInteger(val) ? String(val) : val.toFixed(2); + } + return String(val); +} + +function formatLabel(val: unknown): string { + if (val === null || val === undefined) {return "";} + const str = String(val); + // Truncate long date strings + if (str.length > 16 && !isNaN(Date.parse(str))) { + return str.slice(0, 10); + } + // Truncate long labels + if (str.length > 20) {return str.slice(0, 18) + "...";} + return str; +} + +// --- Chart renderers --- + +function CartesianChart({ + config, + data, + compact, + ChartComponent, + SeriesComponent, + areaProps, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; + ChartComponent: typeof BarChart ; + SeriesComponent: typeof Bar | typeof Line | typeof Area; + areaProps?: Record; +}) { + const { mapping } = config; + const xKey = mapping.xAxis ?? Object.keys(data[0] ?? {})[0] ?? "x"; + const yKeys = mapping.yAxis ?? Object.keys(data[0] ?? {}).filter((k) => k !== xKey); + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + return ( + + + + + + + {yKeys.length > 1 && !compact && } + {yKeys.map((key, i) => { + const color = colors[i % colors.length]; + const props: Record = { + key, + dataKey: key, + fill: color, + stroke: color, + name: key, + ...areaProps, + }; + if (SeriesComponent === Bar) { + props.radius = [4, 4, 0, 0]; + props.maxBarSize = 48; + } + if (SeriesComponent === Line) { + props.strokeWidth = 2; + props.dot = { r: 3, fill: color }; + props.activeDot = { r: 5 }; + } + if (SeriesComponent === Area) { + props.fillOpacity = 0.15; + props.strokeWidth = 2; + } + // @ts-expect-error - dynamic component props + return ; + })} + + + ); +} + +function PieDonutChart({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping, type } = config; + const nameKey = mapping.nameKey ?? Object.keys(data[0] ?? {})[0] ?? "name"; + const valueKey = mapping.valueKey ?? Object.keys(data[0] ?? {})[1] ?? "value"; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + const innerRadius = type === "donut" ? "50%" : 0; + + return ( + + + { + const p = props as Record; + const name = p.name; + const percent = typeof p.percent === "number" ? p.percent : 0; + return `${formatLabel(name)} ${(percent * 100).toFixed(0)}%`; + }) as never} + labelLine={!compact} + style={{ fontSize: 11 }} + > + {data.map((_, i) => ( + + ))} + + + {!compact && } + + + ); +} + +function RadarChartPanel({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping } = config; + const nameKey = mapping.xAxis ?? mapping.nameKey ?? Object.keys(data[0] ?? {})[0] ?? "name"; + const valueKeys = mapping.yAxis ?? [Object.keys(data[0] ?? {})[1] ?? "value"]; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + return ( + + + + + + {valueKeys.map((key, i) => ( + + ))} + + {!compact && valueKeys.length > 1 && } + + + ); +} + +function ScatterChartPanel({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping } = config; + const xKey = mapping.xAxis ?? Object.keys(data[0] ?? {})[0] ?? "x"; + const yKeys = mapping.yAxis ?? [Object.keys(data[0] ?? {})[1] ?? "y"]; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + return ( + + + + + + + {yKeys.map((key, i) => ( + + ))} + {!compact && yKeys.length > 1 && } + + + ); +} + +function FunnelChartPanel({ + config, + data, + compact, +}: { + config: PanelConfig; + data: Record[]; + compact?: boolean; +}) { + const { mapping } = config; + const nameKey = mapping.nameKey ?? Object.keys(data[0] ?? {})[0] ?? "name"; + const valueKey = mapping.valueKey ?? Object.keys(data[0] ?? {})[1] ?? "value"; + const colors = mapping.colors ?? CHART_PALETTE; + const height = compact ? 200 : 320; + const ttStyle = tooltipStyle(); + + // Funnel expects data with fill colors + const funnelData = data.map((row, i) => ({ + ...row, + fill: colors[i % colors.length], + })); + + return ( + + + + + + + + + ); +} + +// --- Main ChartPanel component --- + +export function ChartPanel({ config, data, compact }: ChartPanelProps) { + // Coerce numeric values for Recharts + const processedData = useMemo(() => { + if (!data || data.length === 0) {return [];} + const { mapping } = config; + const numericKeys = new Set([ + ...(mapping.yAxis ?? []), + ...(mapping.valueKey ? [mapping.valueKey] : []), + ]); + + return data.map((row) => { + const out: Record = { ...row }; + for (const key of numericKeys) { + if (key in out) { + const v = out[key]; + if (typeof v === "string" && v !== "" && !isNaN(Number(v))) { + out[key] = Number(v); + } + } + } + return out; + }); + }, [data, config]); + + if (processedData.length === 0) { + return ( +
+ No data +
+ ); + } + + switch (config.type) { + case "bar": + return ; + case "line": + return ; + case "area": + return ; + case "pie": + return ; + case "donut": + return ; + case "radar": + case "radialBar": + return ; + case "scatter": + return ; + case "funnel": + return ; + default: + return ; + } +} diff --git a/apps/web/app/components/charts/filter-bar.tsx b/apps/web/app/components/charts/filter-bar.tsx new file mode 100644 index 00000000000..7113387c346 --- /dev/null +++ b/apps/web/app/components/charts/filter-bar.tsx @@ -0,0 +1,345 @@ +"use client"; + +import { useEffect, useState, useCallback } from "react"; +import type { FilterConfig, FilterState, FilterValue } from "./types"; + +type FilterBarProps = { + filters: FilterConfig[]; + value: FilterState; + onChange: (state: FilterState) => void; +}; + +// --- Icons --- + +function FilterIcon() { + return ( + + + + ); +} + +function XIcon() { + return ( + + + + ); +} + +// --- Individual filter components --- + +function DateRangeFilter({ + filter, + value, + onChange, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; +}) { + const current = value?.type === "dateRange" ? value : { type: "dateRange" as const }; + + return ( +
+ + onChange({ ...current, from: e.target.value || undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + colorScheme: "dark", + }} + /> + to + onChange({ ...current, to: e.target.value || undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + colorScheme: "dark", + }} + /> +
+ ); +} + +function SelectFilter({ + filter, + value, + onChange, + options, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; + options: string[]; +}) { + const current = value?.type === "select" ? value.value : undefined; + + return ( +
+ + +
+ ); +} + +function MultiSelectFilter({ + filter, + value, + onChange, + options, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; + options: string[]; +}) { + const current = value?.type === "multiSelect" ? (value.values ?? []) : []; + + const toggleOption = (opt: string) => { + const next = current.includes(opt) + ? current.filter((v) => v !== opt) + : [...current, opt]; + onChange({ type: "multiSelect", values: next.length > 0 ? next : undefined }); + }; + + return ( +
+ +
+ {options.map((opt) => { + const selected = current.includes(opt); + return ( + + ); + })} +
+
+ ); +} + +function NumberFilter({ + filter, + value, + onChange, +}: { + filter: FilterConfig; + value: FilterValue | undefined; + onChange: (v: FilterValue) => void; +}) { + const current = value?.type === "number" ? value : { type: "number" as const }; + + return ( +
+ + onChange({ ...current, min: e.target.value ? Number(e.target.value) : undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none w-20" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + }} + /> + to + onChange({ ...current, max: e.target.value ? Number(e.target.value) : undefined })} + className="px-2 py-1 rounded-md text-[11px] outline-none w-20" + style={{ + background: "var(--color-bg)", + border: "1px solid var(--color-border)", + color: "var(--color-text)", + }} + /> +
+ ); +} + +// --- Main FilterBar --- + +export function FilterBar({ filters, value, onChange }: FilterBarProps) { + // Fetch options for select/multiSelect filters + const [optionsMap, setOptionsMap] = useState>({}); + + const fetchOptions = useCallback(async () => { + const toFetch = filters.filter( + (f) => (f.type === "select" || f.type === "multiSelect") && f.sql, + ); + if (toFetch.length === 0) {return;} + + const results: Record = {}; + await Promise.all( + toFetch.map(async (f) => { + try { + const res = await fetch("/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: f.sql }), + }); + if (!res.ok) {return;} + const data = await res.json(); + const rows: Record[] = data.rows ?? []; + // Extract the first column's values as options + const opts = rows + .map((r) => { + const vals = Object.values(r); + return vals[0] != null ? String(vals[0]) : null; + }) + .filter((v): v is string => v !== null); + results[f.id] = opts; + } catch { + // skip failed option fetches + } + }), + ); + setOptionsMap(results); + }, [filters]); + + useEffect(() => { + fetchOptions(); + }, [fetchOptions]); + + const handleFilterChange = useCallback( + (filterId: string, v: FilterValue) => { + onChange({ ...value, [filterId]: v }); + }, + [value, onChange], + ); + + const hasActiveFilters = Object.values(value).some((v) => { + if (!v) {return false;} + if (v.type === "dateRange") {return v.from || v.to;} + if (v.type === "select") {return v.value;} + if (v.type === "multiSelect") {return v.values && v.values.length > 0;} + if (v.type === "number") {return v.min !== undefined || v.max !== undefined;} + return false; + }); + + const clearFilters = () => onChange({}); + + if (filters.length === 0) {return null;} + + return ( +
+ + + Filters + + + {filters.map((filter) => { + const fv = value[filter.id]; + switch (filter.type) { + case "dateRange": + return ( + handleFilterChange(filter.id, v)} + /> + ); + case "select": + return ( + handleFilterChange(filter.id, v)} + options={optionsMap[filter.id] ?? []} + /> + ); + case "multiSelect": + return ( + handleFilterChange(filter.id, v)} + options={optionsMap[filter.id] ?? []} + /> + ); + case "number": + return ( + handleFilterChange(filter.id, v)} + /> + ); + default: + return null; + } + })} + + {hasActiveFilters && ( + + )} +
+ ); +} diff --git a/apps/web/app/components/charts/report-card.tsx b/apps/web/app/components/charts/report-card.tsx new file mode 100644 index 00000000000..18ba1f3d9d9 --- /dev/null +++ b/apps/web/app/components/charts/report-card.tsx @@ -0,0 +1,289 @@ +"use client"; + +import { useEffect, useState, useCallback } from "react"; +import { ChartPanel } from "./chart-panel"; +import type { ReportConfig, PanelConfig } from "./types"; + +type ReportCardProps = { + config: ReportConfig; +}; + +// --- Icons --- + +function ChartBarIcon() { + return ( + + + + + + ); +} + +function ExternalLinkIcon() { + return ( + + + + + + ); +} + +function PinIcon() { + return ( + + + + + ); +} + +// --- Panel data state --- + +type PanelData = { + rows: Record[]; + loading: boolean; + error?: string; +}; + +// --- Main ReportCard --- + +export function ReportCard({ config }: ReportCardProps) { + const [panelData, setPanelData] = useState>({}); + const [pinning, setPinning] = useState(false); + const [pinned, setPinned] = useState(false); + + // Show at most 2 panels inline + const visiblePanels = config.panels.slice(0, 2); + + // Execute panel SQL queries + const executePanels = useCallback(async () => { + const initial: Record = {}; + for (const panel of visiblePanels) { + initial[panel.id] = { rows: [], loading: true }; + } + setPanelData(initial); + + await Promise.all( + visiblePanels.map(async (panel) => { + try { + const res = await fetch("/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sql: panel.sql }), + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { rows: [], loading: false, error: data.error || `HTTP ${res.status}` }, + })); + return; + } + const data = await res.json(); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { rows: data.rows ?? [], loading: false }, + })); + } catch (err) { + setPanelData((prev) => ({ + ...prev, + [panel.id]: { rows: [], loading: false, error: err instanceof Error ? err.message : "Failed" }, + })); + } + }), + ); + }, [visiblePanels]); + + useEffect(() => { + executePanels(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // Pin report to workspace filesystem + const handlePin = async () => { + setPinning(true); + try { + const slug = config.title + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40); + const filename = `${slug}.report.json`; + + await fetch("/api/workspace/file", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: `reports/${filename}`, + content: JSON.stringify(config, null, 2), + }), + }); + setPinned(true); + } catch { + // silently fail + } finally { + setPinning(false); + } + }; + + return ( +
+ {/* Header */} +
+
+ + + + + {config.title} + + + {config.panels.length} chart{config.panels.length !== 1 ? "s" : ""} + +
+ +
+ {!pinned && ( + + )} + {pinned && ( + + Saved + + )} + + + Open + +
+
+ + {/* Description */} + {config.description && ( +
+

+ {config.description} +

+
+ )} + + {/* Panels (compact mode) */} +
1 ? "grid-cols-2" : "grid-cols-1"}`}> + {visiblePanels.map((panel) => ( + + ))} +
+ + {/* More panels indicator */} + {config.panels.length > 2 && ( +
+ + +{config.panels.length - 2} more chart{config.panels.length - 2 !== 1 ? "s" : ""} + +
+ )} +
+ ); +} + +// --- Compact panel card for inline rendering --- + +function CompactPanelCard({ + panel, + data, +}: { + panel: PanelConfig; + data?: PanelData; +}) { + return ( +
+
+

+ {panel.title} +

+
+
+ {data?.loading ? ( +
+
+
+ ) : data?.error ? ( +
+

+ {data.error} +

+
+ ) : ( + + )} +
+
+ ); +} diff --git a/apps/web/app/components/charts/report-viewer.tsx b/apps/web/app/components/charts/report-viewer.tsx new file mode 100644 index 00000000000..cd251b3cb14 --- /dev/null +++ b/apps/web/app/components/charts/report-viewer.tsx @@ -0,0 +1,407 @@ +"use client"; + +import { useEffect, useState, useCallback, useMemo } from "react"; +import { ChartPanel } from "./chart-panel"; +import { FilterBar } from "./filter-bar"; +import type { ReportConfig, FilterState, PanelConfig, FilterConfig } from "./types"; + +type ReportViewerProps = { + /** Report config object (inline or loaded) */ + config?: ReportConfig; + /** Path to load report config from filesystem */ + reportPath?: string; +}; + +// --- Icons --- + +function ChartBarIcon({ size = 20 }: { size?: number }) { + return ( + + + + + + ); +} + +function RefreshIcon() { + return ( + + + + + + + ); +} + +// --- Helpers --- + +type PanelData = { + panelId: string; + rows: Record[]; + loading: boolean; + error?: string; +}; + +/** Build filter entries for the API from active filter state + filter configs. */ +function buildFilterEntries( + filterState: FilterState, + filterConfigs: FilterConfig[], +): Array<{ id: string; column: string; value: FilterState[string] }> { + const entries: Array<{ id: string; column: string; value: FilterState[string] }> = []; + for (const fc of filterConfigs) { + const v = filterState[fc.id]; + if (!v) {continue;} + // Only include if the filter has an active value + const hasValue = + (v.type === "dateRange" && (v.from || v.to)) || + (v.type === "select" && v.value) || + (v.type === "multiSelect" && v.values && v.values.length > 0) || + (v.type === "number" && (v.min !== undefined || v.max !== undefined)); + if (hasValue) { + entries.push({ id: fc.id, column: fc.column, value: v }); + } + } + return entries; +} + +// --- Grid size helpers --- + +function panelColSpan(size?: string): string { + switch (size) { + case "full": + return "col-span-6"; + case "third": + return "col-span-2"; + case "half": + default: + return "col-span-3"; + } +} + +// --- Main ReportViewer --- + +export function ReportViewer({ config: propConfig, reportPath }: ReportViewerProps) { + const [config, setConfig] = useState(propConfig ?? null); + const [configLoading, setConfigLoading] = useState(!propConfig && !!reportPath); + const [configError, setConfigError] = useState(null); + const [panelData, setPanelData] = useState>({}); + const [filterState, setFilterState] = useState({}); + const [refreshKey, setRefreshKey] = useState(0); + + // Load report config from filesystem if path provided + useEffect(() => { + if (propConfig) { + setConfig(propConfig); + return; + } + if (!reportPath) {return;} + + let cancelled = false; + setConfigLoading(true); + setConfigError(null); + + fetch(`/api/workspace/file?path=${encodeURIComponent(reportPath)}`) + .then(async (res) => { + if (!res.ok) {throw new Error(`Failed to load report: HTTP ${res.status}`);} + const data = await res.json(); + if (cancelled) {return;} + try { + const parsed = JSON.parse(data.content) as ReportConfig; + setConfig(parsed); + } catch { + throw new Error("Invalid report JSON"); + } + }) + .catch((err) => { + if (!cancelled) { + setConfigError(err instanceof Error ? err.message : "Failed to load report"); + } + }) + .finally(() => { + if (!cancelled) {setConfigLoading(false);} + }); + + return () => { cancelled = true; }; + }, [propConfig, reportPath]); + + // Execute all panel SQL queries when config or filters change + const executeAllPanels = useCallback(async () => { + if (!config) {return;} + + const filterEntries = buildFilterEntries(filterState, config.filters ?? []); + + // Mark all panels as loading + const initialState: Record = {}; + for (const panel of config.panels) { + initialState[panel.id] = { panelId: panel.id, rows: [], loading: true }; + } + setPanelData(initialState); + + // Execute all panels in parallel + await Promise.all( + config.panels.map(async (panel) => { + try { + const res = await fetch("/api/workspace/reports/execute", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sql: panel.sql, + filters: filterEntries.length > 0 ? filterEntries : undefined, + }), + }); + + if (!res.ok) { + const data = await res.json().catch(() => ({})); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { + panelId: panel.id, + rows: [], + loading: false, + error: data.error || `HTTP ${res.status}`, + }, + })); + return; + } + + const data = await res.json(); + setPanelData((prev) => ({ + ...prev, + [panel.id]: { + panelId: panel.id, + rows: data.rows ?? [], + loading: false, + }, + })); + } catch (err) { + setPanelData((prev) => ({ + ...prev, + [panel.id]: { + panelId: panel.id, + rows: [], + loading: false, + error: err instanceof Error ? err.message : "Query failed", + }, + })); + } + }), + ); + }, [config, filterState]); + + // Re-execute when config, filters, or refresh key changes + useEffect(() => { + executeAllPanels(); + }, [executeAllPanels, refreshKey]); + + const totalRows = useMemo(() => { + return Object.values(panelData).reduce((sum, pd) => sum + pd.rows.length, 0); + }, [panelData]); + + // --- Loading state --- + if (configLoading) { + return ( +
+
+ + Loading report... + +
+ ); + } + + // --- Error state --- + if (configError) { + return ( +
+ +

+ Failed to load report +

+

+ {configError} +

+
+ ); + } + + if (!config) { + return ( +
+

+ No report configuration found +

+
+ ); + } + + return ( +
+ {/* Report header */} +
+
+
+
+ + + +

+ {config.title} +

+
+ {config.description && ( +

+ {config.description} +

+ )} +
+ +
+ + {config.panels.length} panel{config.panels.length !== 1 ? "s" : ""} + + + {totalRows} rows + + +
+
+
+ + {/* Filters */} + {config.filters && config.filters.length > 0 && ( + + )} + + {/* Panel grid */} +
+
+ {config.panels.map((panel) => ( + + ))} +
+
+
+ ); +} + +// --- Individual panel card --- + +function PanelCard({ + panel, + data, +}: { + panel: PanelConfig; + data?: PanelData; +}) { + const colSpan = panelColSpan(panel.size); + + return ( +
+ {/* Panel header */} +
+

+ {panel.title} +

+ {data && !data.loading && !data.error && ( + + {data.rows.length} rows + + )} +
+ + {/* Chart area */} +
+ {data?.loading ? ( +
+
+
+ ) : data?.error ? ( +
+

+ Query error +

+

+ {data.error} +

+
+ ) : ( + + )} +
+
+ ); +} diff --git a/apps/web/app/components/charts/types.ts b/apps/web/app/components/charts/types.ts new file mode 100644 index 00000000000..150dfdc3d36 --- /dev/null +++ b/apps/web/app/components/charts/types.ts @@ -0,0 +1,64 @@ +/** Shared types for the report/analytics system. */ + +export type ChartType = + | "bar" + | "line" + | "area" + | "pie" + | "donut" + | "radar" + | "radialBar" + | "scatter" + | "funnel"; + +export type PanelSize = "full" | "half" | "third"; + +export type PanelMapping = { + /** Key for x-axis or category axis */ + xAxis?: string; + /** One or more keys for y-axis values (supports stacked/multi-series) */ + yAxis?: string[]; + /** Key used as label for pie/donut/funnel */ + nameKey?: string; + /** Key used as value for pie/donut/funnel */ + valueKey?: string; + /** Custom colors for series (hex). Falls back to palette. */ + colors?: string[]; +}; + +export type PanelConfig = { + id: string; + title: string; + type: ChartType; + sql: string; + mapping: PanelMapping; + size?: PanelSize; +}; + +export type FilterType = "dateRange" | "select" | "multiSelect" | "number"; + +export type FilterConfig = { + id: string; + type: FilterType; + label: string; + column: string; + /** SQL to fetch available options (for select/multiSelect) */ + sql?: string; +}; + +export type ReportConfig = { + version: number; + title: string; + description?: string; + panels: PanelConfig[]; + filters?: FilterConfig[]; +}; + +/** Active filter values keyed by filter ID */ +export type FilterState = Record; + +export type FilterValue = + | { type: "dateRange"; from?: string; to?: string } + | { type: "select"; value?: string } + | { type: "multiSelect"; values?: string[] } + | { type: "number"; min?: number; max?: number }; diff --git a/apps/web/app/components/chat-message.tsx b/apps/web/app/components/chat-message.tsx index bdacb5ef005..b7b63acd779 100644 --- a/apps/web/app/components/chat-message.tsx +++ b/apps/web/app/components/chat-message.tsx @@ -1,18 +1,46 @@ "use client"; +import dynamic from "next/dynamic"; import type { UIMessage } from "ai"; +import type { Components } from "react-markdown"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; import { ChainOfThought, type ChainPart } from "./chain-of-thought"; +import { splitReportBlocks, hasReportBlocks } from "@/lib/report-blocks"; +import type { ReportConfig } from "./charts/types"; + +// Lazy-load ReportCard (uses Recharts which is heavy) +const ReportCard = dynamic( + () => + import("./charts/report-card").then((m) => ({ + default: m.ReportCard, + })), + { + ssr: false, + loading: () => ( +
+ ), + }, +); /* ─── Part grouping ─── */ type MessageSegment = | { type: "text"; text: string } - | { type: "chain"; parts: ChainPart[] }; + | { type: "chain"; parts: ChainPart[] } + | { type: "report-artifact"; config: ReportConfig }; /** Map AI SDK tool state string to a simplified status */ function toolStatus(state: string): "running" | "done" | "error" { - if (state === "output-available") return "done"; - if (state === "error") return "error"; + if (state === "output-available") { + return "done"; + } + if (state === "error") { + return "error"; + } return "running"; } @@ -34,21 +62,42 @@ function groupParts(parts: UIMessage["parts"]): MessageSegment[] { for (const part of parts) { if (part.type === "text") { flush(); - segments.push({ - type: "text", - text: (part as { type: "text"; text: string }).text, - }); + const text = (part as { type: "text"; text: string }).text; + if (hasReportBlocks(text)) { + segments.push( + ...(splitReportBlocks(text) as MessageSegment[]), + ); + } else { + segments.push({ type: "text", text }); + } } else if (part.type === "reasoning") { const rp = part as { type: "reasoning"; text: string; state?: string; }; - chain.push({ - kind: "reasoning", - text: rp.text, - isStreaming: rp.state === "streaming", - }); + // Detect status reasoning blocks emitted by lifecycle/compaction events. + // These have short, specific labels — render as status indicators instead. + const statusLabels = [ + "Preparing response...", + "Optimizing session context...", + ]; + const isStatus = statusLabels.some((l) => + rp.text.startsWith(l), + ); + if (isStatus) { + chain.push({ + kind: "status", + label: rp.text.split("\n")[0], + isActive: rp.state === "streaming", + }); + } else { + chain.push({ + kind: "reasoning", + text: rp.text, + isStreaming: rp.state === "streaming", + }); + } } else if (part.type === "dynamic-tool") { const tp = part as { type: "dynamic-tool"; @@ -98,56 +147,158 @@ function groupParts(parts: UIMessage["parts"]): MessageSegment[] { function asRecord( val: unknown, ): Record | undefined { - if (val && typeof val === "object" && !Array.isArray(val)) + if (val && typeof val === "object" && !Array.isArray(val)) { return val as Record; + } return undefined; } -/* ─── Chat message ─── */ +/* ─── Markdown component overrides for chat ─── */ + +const mdComponents: Components = { + // Open external links in new tab + a: ({ href, children, ...props }) => { + const isExternal = + href && (href.startsWith("http") || href.startsWith("//")); + return ( + + {children} + + ); + }, + // Render images with loading=lazy + img: ({ src, alt, ...props }) => ( + // eslint-disable-next-line @next/next/no-img-element + {alt + ), +}; + +/* ─── Chat message (Dench-inspired free-flowing text) ─── */ export function ChatMessage({ message }: { message: UIMessage }) { const isUser = message.role === "user"; const segments = groupParts(message.parts); - return ( -
- {!isUser && ( -
- O -
- )} + if (isUser) { + // User: right-aligned subtle pill (like Dench) + const textContent = segments + .filter( + (s): s is { type: "text"; text: string } => + s.type === "text", + ) + .map((s) => s.text) + .join("\n"); -
- {segments.map((segment, index) => { - if (segment.type === "text") { + return ( +
+
+

{textContent}

+
+
+ ); + } + + // Assistant: free-flowing text, left-aligned, NO bubble + return ( +
+ {segments.map((segment, index) => { + if (segment.type === "text") { + // Detect agent error messages + const errorMatch = segment.text.match( + /^\[error\]\s*([\s\S]*)$/, + ); + if (errorMatch) { return (
- {segment.text} + + + {errorMatch[1].trim()} +
); } - return ( - - ); - })} + return ( +
+ + {segment.text} +
- - {isUser && ( -
- U -
- )} + ); + } + if (segment.type === "report-artifact") { + return ( + + ); + } + return ( + + ); + })}
); } diff --git a/apps/web/app/components/chat-panel.tsx b/apps/web/app/components/chat-panel.tsx new file mode 100644 index 00000000000..6d8e7ed3e24 --- /dev/null +++ b/apps/web/app/components/chat-panel.tsx @@ -0,0 +1,863 @@ +"use client"; + +import { useChat } from "@ai-sdk/react"; +import { DefaultChatTransport, type UIMessage } from "ai"; +import { + forwardRef, + useCallback, + useEffect, + useImperativeHandle, + useMemo, + useRef, + useState, +} from "react"; +import { ChatMessage } from "./chat-message"; + +/** Imperative handle for parent-driven session control (main page). */ +export type ChatPanelHandle = { + loadSession: (sessionId: string) => Promise; + newSession: () => Promise; +}; + +export type FileContext = { + path: string; + filename: string; +}; + +type FileScopedSession = { + id: string; + title: string; + createdAt: number; + updatedAt: number; + messageCount: number; +}; + +type ChatPanelProps = { + /** When set, scopes sessions to this file and prepends content as context. */ + fileContext?: FileContext; + /** Compact mode for workspace sidebar (smaller UI, built-in session tabs). */ + compact?: boolean; + /** Called when file content may have changed after agent edits. */ + onFileChanged?: (newContent: string) => void; + /** Called when active session changes (for external sidebar highlighting). */ + onActiveSessionChange?: (sessionId: string | null) => void; + /** Called when session list needs refresh (for external sidebar). */ + onSessionsChange?: () => void; +}; + +export const ChatPanel = forwardRef( + function ChatPanel( + { + fileContext, + compact, + onFileChanged, + onActiveSessionChange, + onSessionsChange, + }, + ref, + ) { + const [input, setInput] = useState(""); + const [currentSessionId, setCurrentSessionId] = useState< + string | null + >(null); + const [loadingSession, setLoadingSession] = useState(false); + const [startingNewSession, setStartingNewSession] = useState(false); + const messagesEndRef = useRef(null); + + // Track persisted messages to avoid double-saves + const savedMessageIdsRef = useRef>(new Set()); + // Set when /new or + triggers a new session + const newSessionPendingRef = useRef(false); + // Whether the next message should include file context + const isFirstFileMessageRef = useRef(true); + + // File-scoped session list (compact mode only) + const [fileSessions, setFileSessions] = useState< + FileScopedSession[] + >([]); + + const filePath = fileContext?.path ?? null; + + // ── Ref-based session ID for transport ── + const sessionIdRef = useRef(null); + useEffect(() => { + sessionIdRef.current = currentSessionId; + }, [currentSessionId]); + + // ── Transport (per-instance) ── + const transport = useMemo( + () => + new DefaultChatTransport({ + api: "/api/chat", + body: () => { + const sid = sessionIdRef.current; + return sid ? { sessionId: sid } : {}; + }, + }), + [], + ); + + const { messages, sendMessage, status, stop, error, setMessages } = + useChat({ transport }); + + const isStreaming = + status === "streaming" || status === "submitted"; + + // Auto-scroll to bottom on new messages + useEffect(() => { + messagesEndRef.current?.scrollIntoView({ + behavior: "smooth", + }); + }, [messages]); + + // ── Session persistence helpers ── + + const createSession = useCallback( + async (title: string): Promise => { + const body: Record = { title }; + if (filePath) { + body.filePath = filePath; + } + const res = await fetch("/api/web-sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + const data = await res.json(); + return data.session.id; + }, + [filePath], + ); + + const saveMessages = useCallback( + async ( + sessionId: string, + msgs: Array<{ + id: string; + role: string; + content: string; + parts?: unknown[]; + }>, + title?: string, + ) => { + const toSave = msgs.map((m) => ({ + id: m.id, + role: m.role, + content: m.content, + ...(m.parts ? { parts: m.parts } : {}), + timestamp: new Date().toISOString(), + })); + try { + await fetch( + `/api/web-sessions/${sessionId}/messages`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + messages: toSave, + title, + }), + }, + ); + for (const m of msgs) { + savedMessageIdsRef.current.add(m.id); + } + onSessionsChange?.(); + } catch (err) { + console.error("Failed to save messages:", err); + } + }, + [onSessionsChange], + ); + + /** Extract plain text from a UIMessage */ + const getMessageText = useCallback( + (msg: (typeof messages)[number]): string => { + return ( + msg.parts + ?.filter( + ( + p, + ): p is { + type: "text"; + text: string; + } => p.type === "text", + ) + .map((p) => p.text) + .join("\n") ?? "" + ); + }, + [], + ); + + // ── File-scoped session initialization ── + const fetchFileSessionsRef = useRef< + (() => Promise) | null + >(null); + + fetchFileSessionsRef.current = async () => { + if (!filePath) { + return []; + } + try { + const res = await fetch( + `/api/web-sessions?filePath=${encodeURIComponent(filePath)}`, + ); + const data = await res.json(); + return (data.sessions || []) as FileScopedSession[]; + } catch { + return []; + } + }; + + useEffect(() => { + if (!filePath) { + return; + } + let cancelled = false; + + sessionIdRef.current = null; + setCurrentSessionId(null); + onActiveSessionChange?.(null); + setMessages([]); + savedMessageIdsRef.current.clear(); + isFirstFileMessageRef.current = true; + + (async () => { + const sessions = + (await fetchFileSessionsRef.current?.()) ?? []; + if (cancelled) { + return; + } + setFileSessions(sessions); + + if (sessions.length > 0) { + const latest = sessions[0]; + setCurrentSessionId(latest.id); + sessionIdRef.current = latest.id; + onActiveSessionChange?.(latest.id); + isFirstFileMessageRef.current = false; + + try { + const msgRes = await fetch( + `/api/web-sessions/${latest.id}`, + ); + if (cancelled) { + return; + } + const msgData = await msgRes.json(); + const sessionMessages: Array<{ + id: string; + role: "user" | "assistant"; + content: string; + parts?: Array>; + }> = msgData.messages || []; + + const uiMessages = sessionMessages.map( + (msg) => { + savedMessageIdsRef.current.add(msg.id); + return { + id: msg.id, + role: msg.role, + parts: (msg.parts ?? [ + { + type: "text" as const, + text: msg.content, + }, + ]) as UIMessage["parts"], + }; + }, + ); + if (!cancelled) { + setMessages(uiMessages); + } + } catch { + // ignore + } + } + })(); + + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps -- stable setters + }, [filePath]); + + // ── Persist unsaved messages + live-reload after streaming ── + const prevStatusRef = useRef(status); + useEffect(() => { + const wasStreaming = + prevStatusRef.current === "streaming" || + prevStatusRef.current === "submitted"; + const isNowReady = status === "ready"; + + if (wasStreaming && isNowReady && currentSessionId) { + const unsaved = messages.filter( + (m) => !savedMessageIdsRef.current.has(m.id), + ); + if (unsaved.length > 0) { + const toSave = unsaved.map((m) => ({ + id: m.id, + role: m.role, + content: getMessageText(m), + parts: m.parts, + })); + saveMessages(currentSessionId, toSave); + } + + if (filePath) { + fetchFileSessionsRef.current?.().then( + (sessions) => { + setFileSessions(sessions); + }, + ); + } + + if (filePath && onFileChanged) { + fetch( + `/api/workspace/file?path=${encodeURIComponent(filePath)}`, + ) + .then((r) => r.json()) + .then((data) => { + if (data.content) { + onFileChanged(data.content); + } + }) + .catch(() => {}); + } + } + prevStatusRef.current = status; + }, [ + status, + messages, + currentSessionId, + saveMessages, + getMessageText, + filePath, + onFileChanged, + ]); + + // ── Actions ── + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + if (!input.trim() || isStreaming) { + return; + } + + const userText = input.trim(); + setInput(""); + + if (userText.toLowerCase() === "/new") { + handleNewSession(); + return; + } + + let sessionId = currentSessionId; + if (!sessionId) { + const title = + userText.length > 60 + ? userText.slice(0, 60) + "..." + : userText; + sessionId = await createSession(title); + setCurrentSessionId(sessionId); + sessionIdRef.current = sessionId; + onActiveSessionChange?.(sessionId); + onSessionsChange?.(); + + if (filePath) { + fetchFileSessionsRef.current?.().then( + (sessions) => { + setFileSessions(sessions); + }, + ); + } + } + + let messageText = userText; + if (fileContext && isFirstFileMessageRef.current) { + messageText = `[Context: workspace file '${fileContext.path}']\n\n${userText}`; + isFirstFileMessageRef.current = false; + } + + sendMessage({ text: messageText }); + }; + + const handleSessionSelect = useCallback( + async (sessionId: string) => { + if (sessionId === currentSessionId) { + return; + } + + stop(); + setLoadingSession(true); + setCurrentSessionId(sessionId); + sessionIdRef.current = sessionId; + onActiveSessionChange?.(sessionId); + savedMessageIdsRef.current.clear(); + isFirstFileMessageRef.current = false; + + try { + const response = await fetch( + `/api/web-sessions/${sessionId}`, + ); + if (!response.ok) { + throw new Error("Failed to load session"); + } + + const data = await response.json(); + const sessionMessages: Array<{ + id: string; + role: "user" | "assistant"; + content: string; + parts?: Array>; + }> = data.messages || []; + + const uiMessages = sessionMessages.map((msg) => { + savedMessageIdsRef.current.add(msg.id); + return { + id: msg.id, + role: msg.role, + parts: (msg.parts ?? [ + { + type: "text" as const, + text: msg.content, + }, + ]) as UIMessage["parts"], + }; + }); + + setMessages(uiMessages); + } catch (err) { + console.error("Error loading session:", err); + } finally { + setLoadingSession(false); + } + }, + [ + currentSessionId, + setMessages, + onActiveSessionChange, + stop, + ], + ); + + const handleNewSession = useCallback(async () => { + stop(); + setCurrentSessionId(null); + sessionIdRef.current = null; + onActiveSessionChange?.(null); + setMessages([]); + savedMessageIdsRef.current.clear(); + isFirstFileMessageRef.current = true; + newSessionPendingRef.current = false; + + if (!filePath) { + setStartingNewSession(true); + try { + await fetch("/api/new-session", { + method: "POST", + }); + } catch (err) { + console.error("Failed to send /new:", err); + } finally { + setStartingNewSession(false); + } + } + }, [setMessages, onActiveSessionChange, filePath, stop]); + + useImperativeHandle( + ref, + () => ({ + loadSession: handleSessionSelect, + newSession: handleNewSession, + }), + [handleSessionSelect, handleNewSession], + ); + + // ── Status label ── + + const statusLabel = startingNewSession + ? "Starting new session..." + : loadingSession + ? "Loading session..." + : status === "ready" + ? "Ready" + : status === "submitted" + ? "Thinking..." + : status === "streaming" + ? "Streaming..." + : status === "error" + ? "Error" + : status; + + // ── Render ── + + return ( +
+ {/* Header */} +
+
+ {compact && fileContext ? ( + <> +

+ Chat: {fileContext.filename} +

+

+ {statusLabel} +

+ + ) : ( + <> +

+ {currentSessionId + ? "Chat Session" + : "New Chat"} +

+

+ {statusLabel} +

+ + )} +
+
+ {compact && ( + + )} + {isStreaming && ( + + )} +
+
+ + {/* File-scoped session tabs (compact mode) */} + {compact && fileContext && fileSessions.length > 0 && ( +
+ {fileSessions.slice(0, 10).map((s) => ( + + ))} +
+ )} + + {/* Messages */} +
+ {loadingSession ? ( +
+
+
+

+ Loading session... +

+
+
+ ) : messages.length === 0 ? ( +
+
+ {compact ? ( +

+ Ask about this file +

+ ) : ( + <> +

+ What can I help with? +

+

+ Send a message to start a + conversation with your + agent. +

+ + )} +
+
+ ) : ( +
+ {messages.map((message) => ( + + ))} +
+
+ )} +
+ + {/* Transport-level error display */} + {error && ( +
+ + + + + +

{error.message}

+
+ )} + + {/* Input — Dench-style rounded area with toolbar */} +
+
+
+
+ + setInput(e.target.value) + } + placeholder={ + compact && fileContext + ? `Ask about ${fileContext.filename}...` + : "Ask anything..." + } + disabled={ + isStreaming || + loadingSession || + startingNewSession + } + className={`w-full ${compact ? "px-3 py-2.5 text-xs" : "px-4 py-3.5 text-sm"} bg-transparent outline-none placeholder:text-[var(--color-text-muted)] disabled:opacity-50`} + style={{ + color: "var(--color-text)", + }} + /> +
+ {/* Toolbar row */} +
+
+ {/* Placeholder toolbar icons */} + +
+ {/* Send button */} + +
+
+
+
+
+ ); + }, +); diff --git a/apps/web/app/components/sidebar.tsx b/apps/web/app/components/sidebar.tsx index ca48a85ace9..5016af5d66c 100644 --- a/apps/web/app/components/sidebar.tsx +++ b/apps/web/app/components/sidebar.tsx @@ -1,6 +1,7 @@ "use client"; -import { useEffect, useState } from "react"; +import { useEffect, useState, useCallback } from "react"; +import { FileManagerTree } from "./workspace/file-manager-tree"; // --- Types --- @@ -24,7 +25,16 @@ type MemoryFile = { sizeBytes: number; }; -type SidebarSection = "chats" | "skills" | "memories"; +type TreeNode = { + name: string; + path: string; + type: "object" | "document" | "folder" | "file" | "database" | "report"; + icon?: string; + defaultView?: "table" | "kanban"; + children?: TreeNode[]; +}; + +type SidebarSection = "chats" | "skills" | "memories" | "workspace" | "reports"; type SidebarProps = { onSessionSelect?: (sessionId: string) => void; @@ -38,11 +48,11 @@ type SidebarProps = { function timeAgo(ts: number): string { const diff = Date.now() - ts; const seconds = Math.floor(diff / 1000); - if (seconds < 60) return `${seconds}s ago`; + if (seconds < 60) {return `${seconds}s ago`;} const minutes = Math.floor(seconds / 60); - if (minutes < 60) return `${minutes}m ago`; + if (minutes < 60) {return `${minutes}m ago`;} const hours = Math.floor(minutes / 60); - if (hours < 24) return `${hours}h ago`; + if (hours < 24) {return `${hours}h ago`;} const days = Math.floor(hours / 24); return `${days}d ago`; } @@ -200,6 +210,95 @@ function MemoriesSection({ ); } +// --- Workspace Section (uses FileManagerTree in compact mode) --- + +function WorkspaceSection({ tree, onRefresh }: { tree: TreeNode[]; onRefresh: () => void }) { + const handleSelect = useCallback((node: TreeNode) => { + // Navigate to workspace page for actionable items + if (node.type === "object" || node.type === "document" || node.type === "file" || node.type === "database" || node.type === "report") { + window.location.href = `/workspace?path=${encodeURIComponent(node.path)}`; + } + }, []); + + if (tree.length === 0) { + return ( +

+ No workspace data yet. +

+ ); + } + + return ( +
+ + + {/* Full workspace link */} + + + + + Open full workspace + +
+ ); +} + +// --- Reports Section --- + +function ReportsSection({ tree }: { tree: TreeNode[] }) { + // Collect all report nodes from the tree (recursive) + const reports: TreeNode[] = []; + function collect(nodes: TreeNode[]) { + for (const n of nodes) { + if (n.type === "report") {reports.push(n);} + if (n.children) {collect(n.children);} + } + } + collect(tree); + + if (reports.length === 0) { + return ( +

+ No reports yet. Ask the agent to create one. +

+ ); + } + + return ( + + ); +} + // --- Collapsible Header --- function SectionHeader({ @@ -246,18 +345,19 @@ export function Sidebar({ activeSessionId, refreshKey, }: SidebarProps) { - const [openSections, setOpenSections] = useState>(new Set(["chats"])); + const [openSections, setOpenSections] = useState>(new Set(["chats", "workspace"])); const [webSessions, setWebSessions] = useState([]); const [skills, setSkills] = useState([]); const [mainMemory, setMainMemory] = useState(null); const [dailyLogs, setDailyLogs] = useState([]); + const [workspaceTree, setWorkspaceTree] = useState([]); const [loading, setLoading] = useState(true); const toggleSection = (section: SidebarSection) => { setOpenSections((prev) => { const next = new Set(prev); - if (next.has(section)) next.delete(section); - else next.add(section); + if (next.has(section)) {next.delete(section);} + else {next.add(section);} return next; }); }; @@ -267,15 +367,17 @@ export function Sidebar({ async function load() { setLoading(true); try { - const [webSessionsRes, skillsRes, memoriesRes] = await Promise.all([ + const [webSessionsRes, skillsRes, memoriesRes, workspaceRes] = await Promise.all([ fetch("/api/web-sessions").then((r) => r.json()), fetch("/api/skills").then((r) => r.json()), fetch("/api/memories").then((r) => r.json()), + fetch("/api/workspace/tree").then((r) => r.json()).catch(() => ({ tree: [] })), ]); setWebSessions(webSessionsRes.sessions ?? []); setSkills(skillsRes.skills ?? []); setMainMemory(memoriesRes.mainMemory ?? null); setDailyLogs(memoriesRes.dailyLogs ?? []); + setWorkspaceTree(workspaceRes.tree ?? []); } catch (err) { console.error("Failed to load sidebar data:", err); } finally { @@ -285,13 +387,22 @@ export function Sidebar({ load(); }, [refreshKey]); + const refreshWorkspace = useCallback(async () => { + try { + const res = await fetch("/api/workspace/tree"); + const data = await res.json(); + setWorkspaceTree(data.tree ?? []); + } catch { + // ignore + } + }, []); + return (