2026-02-01 10:03:47 +09:00
import crypto from "node:crypto" ;
2026-02-22 21:35:11 +00:00
import type { AgentToolResult } from "@mariozechner/pi-agent-core" ;
2026-02-01 10:03:47 +09:00
import {
browserAct ,
browserArmDialog ,
browserArmFileChooser ,
browserConsoleMessages ,
browserNavigate ,
browserPdfSave ,
browserScreenshotAction ,
} from "../../browser/client-actions.js" ;
2026-01-04 05:07:37 +01:00
import {
browserCloseTab ,
browserFocusTab ,
browserOpenTab ,
2026-01-15 04:50:11 +00:00
browserProfiles ,
2026-01-04 05:07:37 +01:00
browserSnapshot ,
browserStart ,
browserStatus ,
browserStop ,
browserTabs ,
} from "../../browser/client.js" ;
import { resolveBrowserConfig } from "../../browser/config.js" ;
2026-01-12 18:05:10 +00:00
import { DEFAULT_AI_SNAPSHOT_MAX_CHARS } from "../../browser/constants.js" ;
2026-02-20 16:36:25 +00:00
import { DEFAULT_UPLOAD_DIR , resolveExistingPathsWithinRoot } from "../../browser/paths.js" ;
2026-02-14 13:35:11 +00:00
import { applyBrowserProxyPaths , persistBrowserProxyFiles } from "../../browser/proxy-files.js" ;
2026-01-04 05:07:37 +01:00
import { loadConfig } from "../../config/config.js" ;
2026-02-13 00:46:11 +01:00
import { wrapExternalContent } from "../../security/external-content.js" ;
2026-01-14 01:08:15 +00:00
import { BrowserToolSchema } from "./browser-tool.schema.js" ;
2026-01-14 14:31:43 +00:00
import { type AnyAgentTool , imageResultFromFile , jsonResult , readStringParam } from "./common.js" ;
2026-01-24 04:19:43 +00:00
import { callGatewayTool } from "./gateway.js" ;
2026-02-01 10:03:47 +09:00
import { listNodes , resolveNodeIdFromList , type NodeListNode } from "./nodes-utils.js" ;
2026-01-24 04:19:43 +00:00
2026-02-13 00:46:11 +01:00
function wrapBrowserExternalJson ( params : {
kind : "snapshot" | "console" | "tabs" ;
payload : unknown ;
includeWarning? : boolean ;
} ) : { wrappedText : string ; safeDetails : Record < string , unknown > } {
const extractedText = JSON . stringify ( params . payload , null , 2 ) ;
const wrappedText = wrapExternalContent ( extractedText , {
source : "browser" ,
includeWarning : params.includeWarning ? ? true ,
} ) ;
return {
wrappedText ,
safeDetails : {
ok : true ,
externalContent : {
untrusted : true ,
source : "browser" ,
kind : params.kind ,
wrapped : true ,
} ,
} ,
} ;
}
2026-02-22 21:35:11 +00:00
function formatTabsToolResult ( tabs : unknown [ ] ) : AgentToolResult < unknown > {
2026-02-22 21:18:02 +00:00
const wrapped = wrapBrowserExternalJson ( {
kind : "tabs" ,
payload : { tabs } ,
includeWarning : false ,
} ) ;
2026-02-22 21:35:11 +00:00
const content : AgentToolResult < unknown > [ "content" ] = [
{ type : "text" , text : wrapped.wrappedText } ,
] ;
2026-02-22 21:18:02 +00:00
return {
2026-02-22 21:35:11 +00:00
content ,
2026-02-22 21:18:02 +00:00
details : { . . . wrapped . safeDetails , tabCount : tabs.length } ,
} ;
}
function readOptionalTargetAndTimeout ( params : Record < string , unknown > ) {
const targetId = typeof params . targetId === "string" ? params . targetId . trim ( ) : undefined ;
const timeoutMs =
typeof params . timeoutMs === "number" && Number . isFinite ( params . timeoutMs )
? params . timeoutMs
: undefined ;
return { targetId , timeoutMs } ;
}
2026-01-24 04:19:43 +00:00
type BrowserProxyFile = {
path : string ;
base64 : string ;
mimeType? : string ;
} ;
type BrowserProxyResult = {
result : unknown ;
files? : BrowserProxyFile [ ] ;
} ;
const DEFAULT_BROWSER_PROXY_TIMEOUT_MS = 20 _000 ;
type BrowserNodeTarget = {
nodeId : string ;
label? : string ;
} ;
function isBrowserNode ( node : NodeListNode ) {
const caps = Array . isArray ( node . caps ) ? node . caps : [ ] ;
const commands = Array . isArray ( node . commands ) ? node . commands : [ ] ;
return caps . includes ( "browser" ) || commands . includes ( "browser.proxy" ) ;
}
async function resolveBrowserNodeTarget ( params : {
requestedNode? : string ;
2026-01-27 03:23:42 +00:00
target ? : "sandbox" | "host" | "node" ;
sandboxBridgeUrl? : string ;
2026-01-24 04:19:43 +00:00
} ) : Promise < BrowserNodeTarget | null > {
const cfg = loadConfig ( ) ;
const policy = cfg . gateway ? . nodes ? . browser ;
const mode = policy ? . mode ? ? "auto" ;
if ( mode === "off" ) {
if ( params . target === "node" || params . requestedNode ) {
throw new Error ( "Node browser proxy is disabled (gateway.nodes.browser.mode=off)." ) ;
}
return null ;
}
2026-01-27 03:23:42 +00:00
if ( params . sandboxBridgeUrl ? . trim ( ) && params . target !== "node" && ! params . requestedNode ) {
2026-01-24 04:19:43 +00:00
return null ;
}
2026-01-31 16:19:20 +09:00
if ( params . target && params . target !== "node" ) {
return null ;
}
2026-01-24 04:19:43 +00:00
if ( mode === "manual" && params . target !== "node" && ! params . requestedNode ) {
return null ;
}
const nodes = await listNodes ( { } ) ;
const browserNodes = nodes . filter ( ( node ) = > node . connected && isBrowserNode ( node ) ) ;
if ( browserNodes . length === 0 ) {
if ( params . target === "node" || params . requestedNode ) {
throw new Error ( "No connected browser-capable nodes." ) ;
}
return null ;
}
const requested = params . requestedNode ? . trim ( ) || policy ? . node ? . trim ( ) ;
if ( requested ) {
const nodeId = resolveNodeIdFromList ( browserNodes , requested , false ) ;
const node = browserNodes . find ( ( entry ) = > entry . nodeId === nodeId ) ;
return { nodeId , label : node?.displayName ? ? node ? . remoteIp ? ? nodeId } ;
}
if ( params . target === "node" ) {
if ( browserNodes . length === 1 ) {
2026-01-31 16:03:28 +09:00
const node = browserNodes [ 0 ] ;
2026-01-24 04:19:43 +00:00
return { nodeId : node.nodeId , label : node.displayName ? ? node . remoteIp ? ? node . nodeId } ;
}
throw new Error (
` Multiple browser-capable nodes connected ( ${ browserNodes . length } ). Set gateway.nodes.browser.node or pass node=<id>. ` ,
) ;
}
2026-01-31 16:19:20 +09:00
if ( mode === "manual" ) {
return null ;
}
2026-01-24 04:19:43 +00:00
if ( browserNodes . length === 1 ) {
2026-01-31 16:03:28 +09:00
const node = browserNodes [ 0 ] ;
2026-01-24 04:19:43 +00:00
return { nodeId : node.nodeId , label : node.displayName ? ? node . remoteIp ? ? node . nodeId } ;
}
return null ;
}
async function callBrowserProxy ( params : {
nodeId : string ;
method : string ;
path : string ;
query? : Record < string , string | number | boolean | undefined > ;
body? : unknown ;
timeoutMs? : number ;
profile? : string ;
} ) : Promise < BrowserProxyResult > {
const gatewayTimeoutMs =
typeof params . timeoutMs === "number" && Number . isFinite ( params . timeoutMs )
? Math . max ( 1 , Math . floor ( params . timeoutMs ) )
: DEFAULT_BROWSER_PROXY_TIMEOUT_MS ;
2026-01-31 16:46:45 +09:00
const payload = await callGatewayTool < { payloadJSON? : string ; payload? : string } > (
2026-01-24 04:19:43 +00:00
"node.invoke" ,
{ timeoutMs : gatewayTimeoutMs } ,
{
nodeId : params.nodeId ,
command : "browser.proxy" ,
params : {
method : params.method ,
path : params.path ,
query : params.query ,
body : params.body ,
timeoutMs : params.timeoutMs ,
profile : params.profile ,
} ,
idempotencyKey : crypto.randomUUID ( ) ,
} ,
2026-01-31 16:03:28 +09:00
) ;
2026-01-24 04:19:43 +00:00
const parsed =
payload ? . payload ? ?
( typeof payload ? . payloadJSON === "string" && payload . payloadJSON
? ( JSON . parse ( payload . payloadJSON ) as BrowserProxyResult )
: null ) ;
2026-01-31 07:51:26 +00:00
if ( ! parsed || typeof parsed !== "object" || ! ( "result" in parsed ) ) {
2026-01-24 04:19:43 +00:00
throw new Error ( "browser proxy failed" ) ;
}
2026-01-31 07:59:01 +00:00
return parsed ;
2026-01-24 04:19:43 +00:00
}
async function persistProxyFiles ( files : BrowserProxyFile [ ] | undefined ) {
2026-02-14 13:35:11 +00:00
return await persistBrowserProxyFiles ( files ) ;
2026-01-24 04:19:43 +00:00
}
function applyProxyPaths ( result : unknown , mapping : Map < string , string > ) {
2026-02-14 13:35:11 +00:00
applyBrowserProxyPaths ( result , mapping ) ;
2026-01-24 04:19:43 +00:00
}
2026-01-04 05:07:37 +01:00
2026-01-11 01:24:02 +01:00
function resolveBrowserBaseUrl ( params : {
2026-01-27 03:23:42 +00:00
target ? : "sandbox" | "host" ;
sandboxBridgeUrl? : string ;
2026-01-11 01:24:02 +01:00
allowHostControl? : boolean ;
2026-01-27 03:23:42 +00:00
} ) : string | undefined {
2026-01-04 05:07:37 +01:00
const cfg = loadConfig ( ) ;
2026-01-27 03:23:42 +00:00
const resolved = resolveBrowserConfig ( cfg . browser , cfg ) ;
const normalizedSandbox = params . sandboxBridgeUrl ? . trim ( ) ? ? "" ;
const target = params . target ? ? ( normalizedSandbox ? "sandbox" : "host" ) ;
2026-01-11 01:24:02 +01:00
if ( target === "sandbox" ) {
2026-01-27 03:23:42 +00:00
if ( ! normalizedSandbox ) {
2026-01-11 01:24:02 +01:00
throw new Error (
'Sandbox browser is unavailable. Enable agents.defaults.sandbox.browser.enabled or use target="host" if allowed.' ,
) ;
}
2026-01-27 03:23:42 +00:00
return normalizedSandbox . replace ( /\/$/ , "" ) ;
2026-01-11 01:24:02 +01:00
}
if ( params . allowHostControl === false ) {
throw new Error ( "Host browser control is disabled by sandbox policy." ) ;
}
if ( ! resolved . enabled ) {
2026-01-04 05:07:37 +01:00
throw new Error (
2026-01-30 03:15:10 +01:00
"Browser control is disabled. Set browser.enabled=true in ~/.openclaw/openclaw.json." ,
2026-01-04 05:07:37 +01:00
) ;
}
2026-01-27 03:23:42 +00:00
return undefined ;
2026-01-04 05:07:37 +01:00
}
export function createBrowserTool ( opts ? : {
2026-01-27 03:23:42 +00:00
sandboxBridgeUrl? : string ;
2026-01-11 01:24:02 +01:00
allowHostControl? : boolean ;
2026-01-04 05:07:37 +01:00
} ) : AnyAgentTool {
2026-01-27 03:23:42 +00:00
const targetDefault = opts ? . sandboxBridgeUrl ? "sandbox" : "host" ;
2026-01-11 01:52:23 +01:00
const hostHint =
2026-01-14 14:31:43 +00:00
opts ? . allowHostControl === false ? "Host target blocked by policy." : "Host target allowed." ;
2026-01-04 05:07:37 +01:00
return {
label : "Browser" ,
name : "browser" ,
2026-01-11 01:52:23 +01:00
description : [
2026-01-30 03:15:10 +01:00
"Control the browser via OpenClaw's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/actions)." ,
'Profiles: use profile="chrome" for Chrome extension relay takeover (your existing Chrome tabs). Use profile="openclaw" for the isolated openclaw-managed browser.' ,
2026-01-15 10:18:57 +00:00
'If the user mentions the Chrome extension / Browser Relay / toolbar button / “attach tab”, ALWAYS use profile="chrome" (do not ask which profile).' ,
2026-01-24 04:19:43 +00:00
'When a node-hosted browser proxy is available, the tool may auto-route to it. Pin a node with node=<id|name> or target="node".' ,
2026-01-30 03:15:10 +01:00
"Chrome extension relay needs an attached tab: user must click the OpenClaw Browser Relay toolbar icon on the tab (badge ON). If no tab is connected, ask them to attach it." ,
2026-01-15 09:36:48 +00:00
"When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc)." ,
2026-01-15 10:16:33 +00:00
'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.' ,
2026-01-11 01:52:23 +01:00
"Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists." ,
2026-01-27 03:23:42 +00:00
` target selects browser location (sandbox|host|node). Default: ${ targetDefault } . ` ,
2026-01-11 01:52:23 +01:00
hostHint ,
] . join ( " " ) ,
2026-01-04 05:07:37 +01:00
parameters : BrowserToolSchema ,
execute : async ( _toolCallId , args ) = > {
const params = args as Record < string , unknown > ;
const action = readStringParam ( params , "action" , { required : true } ) ;
2026-01-06 09:54:31 -07:00
const profile = readStringParam ( params , "profile" ) ;
2026-01-24 04:19:43 +00:00
const requestedNode = readStringParam ( params , "node" ) ;
2026-01-27 03:23:42 +00:00
let target = readStringParam ( params , "target" ) as "sandbox" | "host" | "node" | undefined ;
2026-01-24 04:19:43 +00:00
2026-01-27 03:23:42 +00:00
if ( requestedNode && target && target !== "node" ) {
throw new Error ( 'node is only supported with target="node".' ) ;
2026-01-24 04:19:43 +00:00
}
2026-01-27 03:23:42 +00:00
if ( ! target && ! requestedNode && profile === "chrome" ) {
2026-01-24 04:19:43 +00:00
// Chrome extension relay takeover is a host Chrome feature; prefer host unless explicitly targeting a node.
2026-01-15 10:34:57 +00:00
target = "host" ;
}
2026-01-24 04:19:43 +00:00
const nodeTarget = await resolveBrowserNodeTarget ( {
requestedNode : requestedNode ? ? undefined ,
2026-01-11 01:24:02 +01:00
target ,
2026-01-27 03:23:42 +00:00
sandboxBridgeUrl : opts?.sandboxBridgeUrl ,
2026-01-11 01:24:02 +01:00
} ) ;
2026-01-04 05:07:37 +01:00
2026-01-24 04:19:43 +00:00
const resolvedTarget = target === "node" ? undefined : target ;
const baseUrl = nodeTarget
2026-01-27 03:23:42 +00:00
? undefined
2026-01-24 04:19:43 +00:00
: resolveBrowserBaseUrl ( {
target : resolvedTarget ,
2026-01-27 03:23:42 +00:00
sandboxBridgeUrl : opts?.sandboxBridgeUrl ,
2026-01-24 04:19:43 +00:00
allowHostControl : opts?.allowHostControl ,
} ) ;
const proxyRequest = nodeTarget
? async ( opts : {
method : string ;
path : string ;
query? : Record < string , string | number | boolean | undefined > ;
body? : unknown ;
timeoutMs? : number ;
profile? : string ;
} ) = > {
const proxy = await callBrowserProxy ( {
nodeId : nodeTarget.nodeId ,
method : opts.method ,
path : opts.path ,
query : opts.query ,
body : opts.body ,
timeoutMs : opts.timeoutMs ,
profile : opts.profile ,
} ) ;
const mapping = await persistProxyFiles ( proxy . files ) ;
applyProxyPaths ( proxy . result , mapping ) ;
return proxy . result ;
}
: null ;
2026-01-04 05:07:37 +01:00
switch ( action ) {
case "status" :
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
return jsonResult (
await proxyRequest ( {
method : "GET" ,
path : "/" ,
profile ,
} ) ,
) ;
}
2026-01-06 09:54:31 -07:00
return jsonResult ( await browserStatus ( baseUrl , { profile } ) ) ;
2026-01-04 05:07:37 +01:00
case "start" :
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
await proxyRequest ( {
method : "POST" ,
path : "/start" ,
profile ,
} ) ;
return jsonResult (
await proxyRequest ( {
method : "GET" ,
path : "/" ,
profile ,
} ) ,
) ;
}
2026-01-06 09:54:31 -07:00
await browserStart ( baseUrl , { profile } ) ;
return jsonResult ( await browserStatus ( baseUrl , { profile } ) ) ;
2026-01-04 05:07:37 +01:00
case "stop" :
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
await proxyRequest ( {
method : "POST" ,
path : "/stop" ,
profile ,
} ) ;
return jsonResult (
await proxyRequest ( {
method : "GET" ,
path : "/" ,
profile ,
} ) ,
) ;
}
2026-01-06 09:54:31 -07:00
await browserStop ( baseUrl , { profile } ) ;
return jsonResult ( await browserStatus ( baseUrl , { profile } ) ) ;
2026-01-15 04:50:11 +00:00
case "profiles" :
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = await proxyRequest ( {
method : "GET" ,
path : "/profiles" ,
} ) ;
return jsonResult ( result ) ;
}
2026-01-15 04:50:11 +00:00
return jsonResult ( { profiles : await browserProfiles ( baseUrl ) } ) ;
2026-01-04 05:07:37 +01:00
case "tabs" :
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = await proxyRequest ( {
method : "GET" ,
path : "/tabs" ,
profile ,
} ) ;
const tabs = ( result as { tabs? : unknown [ ] } ) . tabs ? ? [ ] ;
2026-02-22 21:18:02 +00:00
return formatTabsToolResult ( tabs ) ;
2026-02-13 00:46:11 +01:00
}
{
const tabs = await browserTabs ( baseUrl , { profile } ) ;
2026-02-22 21:18:02 +00:00
return formatTabsToolResult ( tabs ) ;
2026-01-24 04:19:43 +00:00
}
2026-01-04 05:07:37 +01:00
case "open" : {
const targetUrl = readStringParam ( params , "targetUrl" , {
required : true ,
} ) ;
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = await proxyRequest ( {
method : "POST" ,
path : "/tabs/open" ,
profile ,
body : { url : targetUrl } ,
} ) ;
return jsonResult ( result ) ;
}
2026-01-14 14:31:43 +00:00
return jsonResult ( await browserOpenTab ( baseUrl , targetUrl , { profile } ) ) ;
2026-01-04 05:07:37 +01:00
}
case "focus" : {
const targetId = readStringParam ( params , "targetId" , {
required : true ,
} ) ;
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = await proxyRequest ( {
method : "POST" ,
path : "/tabs/focus" ,
profile ,
body : { targetId } ,
} ) ;
return jsonResult ( result ) ;
}
2026-01-06 09:54:31 -07:00
await browserFocusTab ( baseUrl , targetId , { profile } ) ;
2026-01-04 05:07:37 +01:00
return jsonResult ( { ok : true } ) ;
}
case "close" : {
const targetId = readStringParam ( params , "targetId" ) ;
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = targetId
? await proxyRequest ( {
method : "DELETE" ,
path : ` /tabs/ ${ encodeURIComponent ( targetId ) } ` ,
profile ,
} )
: await proxyRequest ( {
method : "POST" ,
path : "/act" ,
profile ,
body : { kind : "close" } ,
} ) ;
return jsonResult ( result ) ;
}
2026-01-31 16:19:20 +09:00
if ( targetId ) {
await browserCloseTab ( baseUrl , targetId , { profile } ) ;
} else {
await browserAct ( baseUrl , { kind : "close" } , { profile } ) ;
}
2026-01-04 05:07:37 +01:00
return jsonResult ( { ok : true } ) ;
}
case "snapshot" : {
2026-01-21 03:02:15 +00:00
const snapshotDefaults = loadConfig ( ) . browser ? . snapshotDefaults ;
2026-01-04 05:07:37 +01:00
const format =
2026-01-16 15:50:02 +01:00
params . snapshotFormat === "ai" || params . snapshotFormat === "aria"
2026-01-31 16:03:28 +09:00
? params . snapshotFormat
2026-01-04 05:07:37 +01:00
: "ai" ;
2026-01-21 03:02:15 +00:00
const mode =
params . mode === "efficient"
? "efficient"
: format === "ai" && snapshotDefaults ? . mode === "efficient"
? "efficient"
: undefined ;
2026-01-15 03:50:48 +00:00
const labels = typeof params . labels === "boolean" ? params.labels : undefined ;
2026-01-15 10:16:33 +00:00
const refs = params . refs === "aria" || params . refs === "role" ? params.refs : undefined ;
2026-01-13 02:29:48 +00:00
const hasMaxChars = Object . hasOwn ( params , "maxChars" ) ;
2026-01-14 14:31:43 +00:00
const targetId = typeof params . targetId === "string" ? params . targetId . trim ( ) : undefined ;
2026-01-04 05:07:37 +01:00
const limit =
typeof params . limit === "number" && Number . isFinite ( params . limit )
? params . limit
: undefined ;
2026-01-11 21:44:08 -08:00
const maxChars =
typeof params . maxChars === "number" &&
2026-01-12 07:37:14 +00:00
Number . isFinite ( params . maxChars ) &&
params . maxChars > 0
? Math . floor ( params . maxChars )
: undefined ;
const resolvedMaxChars =
2026-01-15 03:50:48 +00:00
format === "ai"
? hasMaxChars
? maxChars
: mode === "efficient"
? undefined
: DEFAULT_AI_SNAPSHOT_MAX_CHARS
: undefined ;
2026-01-12 08:36:20 +00:00
const interactive =
2026-01-14 14:31:43 +00:00
typeof params . interactive === "boolean" ? params.interactive : undefined ;
const compact = typeof params . compact === "boolean" ? params.compact : undefined ;
2026-01-12 08:36:20 +00:00
const depth =
typeof params . depth === "number" && Number . isFinite ( params . depth )
? params . depth
: undefined ;
2026-01-14 14:31:43 +00:00
const selector = typeof params . selector === "string" ? params . selector . trim ( ) : undefined ;
const frame = typeof params . frame === "string" ? params . frame . trim ( ) : undefined ;
2026-01-24 04:19:43 +00:00
const snapshot = proxyRequest
? ( ( await proxyRequest ( {
method : "GET" ,
path : "/snapshot" ,
profile ,
query : {
format ,
targetId ,
limit ,
. . . ( typeof resolvedMaxChars === "number" ? { maxChars : resolvedMaxChars } : { } ) ,
refs ,
interactive ,
compact ,
depth ,
selector ,
frame ,
labels ,
mode ,
} ,
} ) ) as Awaited < ReturnType < typeof browserSnapshot > > )
: await browserSnapshot ( baseUrl , {
format ,
targetId ,
limit ,
. . . ( typeof resolvedMaxChars === "number" ? { maxChars : resolvedMaxChars } : { } ) ,
refs ,
interactive ,
compact ,
depth ,
selector ,
frame ,
labels ,
mode ,
profile ,
} ) ;
2026-01-04 05:07:37 +01:00
if ( snapshot . format === "ai" ) {
2026-02-13 00:46:11 +01:00
const extractedText = snapshot . snapshot ? ? "" ;
const wrappedSnapshot = wrapExternalContent ( extractedText , {
source : "browser" ,
includeWarning : true ,
} ) ;
const safeDetails = {
ok : true ,
format : snapshot.format ,
targetId : snapshot.targetId ,
url : snapshot.url ,
truncated : snapshot.truncated ,
stats : snapshot.stats ,
refs : snapshot.refs ? Object . keys ( snapshot . refs ) . length : undefined ,
labels : snapshot.labels ,
labelsCount : snapshot.labelsCount ,
labelsSkipped : snapshot.labelsSkipped ,
imagePath : snapshot.imagePath ,
imageType : snapshot.imageType ,
externalContent : {
untrusted : true ,
source : "browser" ,
kind : "snapshot" ,
format : "ai" ,
wrapped : true ,
} ,
} ;
2026-01-15 03:50:48 +00:00
if ( labels && snapshot . imagePath ) {
return await imageResultFromFile ( {
label : "browser:snapshot" ,
path : snapshot.imagePath ,
2026-02-13 00:46:11 +01:00
extraText : wrappedSnapshot ,
details : safeDetails ,
2026-01-15 03:50:48 +00:00
} ) ;
}
2026-01-04 05:07:37 +01:00
return {
2026-02-13 00:46:11 +01:00
content : [ { type : "text" , text : wrappedSnapshot } ] ,
details : safeDetails ,
} ;
}
{
const wrapped = wrapBrowserExternalJson ( {
kind : "snapshot" ,
payload : snapshot ,
} ) ;
return {
content : [ { type : "text" , text : wrapped.wrappedText } ] ,
details : {
. . . wrapped . safeDetails ,
format : "aria" ,
targetId : snapshot.targetId ,
url : snapshot.url ,
nodeCount : snapshot.nodes.length ,
externalContent : {
untrusted : true ,
source : "browser" ,
kind : "snapshot" ,
format : "aria" ,
wrapped : true ,
} ,
} ,
2026-01-04 05:07:37 +01:00
} ;
}
}
case "screenshot" : {
const targetId = readStringParam ( params , "targetId" ) ;
const fullPage = Boolean ( params . fullPage ) ;
const ref = readStringParam ( params , "ref" ) ;
const element = readStringParam ( params , "element" ) ;
const type = params . type === "jpeg" ? "jpeg" : "png" ;
2026-01-24 04:19:43 +00:00
const result = proxyRequest
? ( ( await proxyRequest ( {
method : "POST" ,
path : "/screenshot" ,
profile ,
body : {
targetId ,
fullPage ,
ref ,
element ,
type ,
} ,
} ) ) as Awaited < ReturnType < typeof browserScreenshotAction > > )
: await browserScreenshotAction ( baseUrl , {
targetId ,
fullPage ,
ref ,
element ,
type ,
profile ,
} ) ;
2026-01-04 05:07:37 +01:00
return await imageResultFromFile ( {
label : "browser:screenshot" ,
path : result.path ,
details : result ,
} ) ;
}
case "navigate" : {
const targetUrl = readStringParam ( params , "targetUrl" , {
required : true ,
} ) ;
const targetId = readStringParam ( params , "targetId" ) ;
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = await proxyRequest ( {
method : "POST" ,
path : "/navigate" ,
profile ,
body : {
url : targetUrl ,
targetId ,
} ,
} ) ;
return jsonResult ( result ) ;
}
2026-01-04 05:07:37 +01:00
return jsonResult (
2026-01-06 11:04:33 -07:00
await browserNavigate ( baseUrl , {
url : targetUrl ,
targetId ,
profile ,
} ) ,
2026-01-04 05:07:37 +01:00
) ;
}
case "console" : {
2026-01-14 14:31:43 +00:00
const level = typeof params . level === "string" ? params . level . trim ( ) : undefined ;
const targetId = typeof params . targetId === "string" ? params . targetId . trim ( ) : undefined ;
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
2026-02-13 00:46:11 +01:00
const result = ( await proxyRequest ( {
2026-01-24 04:19:43 +00:00
method : "GET" ,
path : "/console" ,
profile ,
query : {
level ,
targetId ,
} ,
2026-02-13 00:46:11 +01:00
} ) ) as { ok? : boolean ; targetId? : string ; messages? : unknown [ ] } ;
const wrapped = wrapBrowserExternalJson ( {
kind : "console" ,
payload : result ,
includeWarning : false ,
2026-01-24 04:19:43 +00:00
} ) ;
2026-02-13 00:46:11 +01:00
return {
content : [ { type : "text" , text : wrapped.wrappedText } ] ,
details : {
. . . wrapped . safeDetails ,
targetId : typeof result . targetId === "string" ? result.targetId : undefined ,
messageCount : Array.isArray ( result . messages ) ? result.messages.length : undefined ,
} ,
} ;
}
{
const result = await browserConsoleMessages ( baseUrl , { level , targetId , profile } ) ;
const wrapped = wrapBrowserExternalJson ( {
kind : "console" ,
payload : result ,
includeWarning : false ,
} ) ;
return {
content : [ { type : "text" , text : wrapped.wrappedText } ] ,
details : {
. . . wrapped . safeDetails ,
targetId : result.targetId ,
messageCount : result.messages.length ,
} ,
} ;
2026-01-24 04:19:43 +00:00
}
2026-01-04 05:07:37 +01:00
}
case "pdf" : {
2026-01-14 14:31:43 +00:00
const targetId = typeof params . targetId === "string" ? params . targetId . trim ( ) : undefined ;
2026-01-24 04:19:43 +00:00
const result = proxyRequest
? ( ( await proxyRequest ( {
method : "POST" ,
path : "/pdf" ,
profile ,
body : { targetId } ,
} ) ) as Awaited < ReturnType < typeof browserPdfSave > > )
: await browserPdfSave ( baseUrl , { targetId , profile } ) ;
2026-01-04 05:07:37 +01:00
return {
content : [ { type : "text" , text : ` FILE: ${ result . path } ` } ] ,
details : result ,
} ;
}
case "upload" : {
2026-01-14 14:31:43 +00:00
const paths = Array . isArray ( params . paths ) ? params . paths . map ( ( p ) = > String ( p ) ) : [ ] ;
2026-01-31 16:19:20 +09:00
if ( paths . length === 0 ) {
throw new Error ( "paths required" ) ;
}
2026-02-20 16:36:25 +00:00
const uploadPathsResult = await resolveExistingPathsWithinRoot ( {
2026-02-14 14:42:08 +01:00
rootDir : DEFAULT_UPLOAD_DIR ,
requestedPaths : paths ,
scopeLabel : ` uploads directory ( ${ DEFAULT_UPLOAD_DIR } ) ` ,
} ) ;
if ( ! uploadPathsResult . ok ) {
throw new Error ( uploadPathsResult . error ) ;
}
const normalizedPaths = uploadPathsResult . paths ;
2026-01-04 05:07:37 +01:00
const ref = readStringParam ( params , "ref" ) ;
const inputRef = readStringParam ( params , "inputRef" ) ;
const element = readStringParam ( params , "element" ) ;
2026-02-22 21:18:02 +00:00
const { targetId , timeoutMs } = readOptionalTargetAndTimeout ( params ) ;
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = await proxyRequest ( {
method : "POST" ,
path : "/hooks/file-chooser" ,
profile ,
body : {
2026-02-14 14:42:08 +01:00
paths : normalizedPaths ,
2026-01-24 04:19:43 +00:00
ref ,
inputRef ,
element ,
targetId ,
timeoutMs ,
} ,
} ) ;
return jsonResult ( result ) ;
}
2026-01-04 05:07:37 +01:00
return jsonResult (
await browserArmFileChooser ( baseUrl , {
2026-02-14 14:42:08 +01:00
paths : normalizedPaths ,
2026-01-04 05:07:37 +01:00
ref ,
inputRef ,
element ,
targetId ,
timeoutMs ,
2026-01-06 09:54:31 -07:00
profile ,
2026-01-04 05:07:37 +01:00
} ) ,
) ;
}
case "dialog" : {
const accept = Boolean ( params . accept ) ;
2026-01-14 14:31:43 +00:00
const promptText = typeof params . promptText === "string" ? params.promptText : undefined ;
2026-02-22 21:18:02 +00:00
const { targetId , timeoutMs } = readOptionalTargetAndTimeout ( params ) ;
2026-01-24 04:19:43 +00:00
if ( proxyRequest ) {
const result = await proxyRequest ( {
method : "POST" ,
path : "/hooks/dialog" ,
profile ,
body : {
accept ,
promptText ,
targetId ,
timeoutMs ,
} ,
} ) ;
return jsonResult ( result ) ;
}
2026-01-04 05:07:37 +01:00
return jsonResult (
await browserArmDialog ( baseUrl , {
accept ,
promptText ,
targetId ,
timeoutMs ,
2026-01-06 09:54:31 -07:00
profile ,
2026-01-04 05:07:37 +01:00
} ) ,
) ;
}
case "act" : {
const request = params . request as Record < string , unknown > | undefined ;
if ( ! request || typeof request !== "object" ) {
throw new Error ( "request required" ) ;
}
2026-01-15 10:34:57 +00:00
try {
2026-01-24 04:19:43 +00:00
const result = proxyRequest
? await proxyRequest ( {
method : "POST" ,
path : "/act" ,
profile ,
body : request ,
} )
: await browserAct ( baseUrl , request as Parameters < typeof browserAct > [ 1 ] , {
profile ,
} ) ;
2026-01-15 10:34:57 +00:00
return jsonResult ( result ) ;
} catch ( err ) {
const msg = String ( err ) ;
if ( msg . includes ( "404:" ) && msg . includes ( "tab not found" ) && profile === "chrome" ) {
2026-01-24 04:19:43 +00:00
const tabs = proxyRequest
? ( (
( await proxyRequest ( {
method : "GET" ,
path : "/tabs" ,
profile ,
} ) ) as { tabs? : unknown [ ] }
) . tabs ? ? [ ] )
: await browserTabs ( baseUrl , { profile } ) . catch ( ( ) = > [ ] ) ;
2026-01-15 10:34:57 +00:00
if ( ! tabs . length ) {
throw new Error (
2026-01-30 03:15:10 +01:00
"No Chrome tabs are attached via the OpenClaw Browser Relay extension. Click the toolbar icon on the tab you want to control (badge ON), then retry." ,
2026-01-31 16:03:28 +09:00
{ cause : err } ,
2026-01-15 10:34:57 +00:00
) ;
}
throw new Error (
` Chrome tab not found (stale targetId?). Run action=tabs profile="chrome" and use one of the returned targetIds. ` ,
2026-01-31 16:03:28 +09:00
{ cause : err } ,
2026-01-15 10:34:57 +00:00
) ;
}
throw err ;
}
2026-01-04 05:07:37 +01:00
}
default :
throw new Error ( ` Unknown action: ${ action } ` ) ;
}
} ,
} ;
}