feat(ui): add image view mode and migrate vnc proxy to gateway

- Created claw-image-panel component for static image preview

- Refactored app-render and app.ts to support tool switching between vnc and images

- Migrated VNC WebSocket proxy from Vite to the Gateway HTTP server (/api/debug-vnc)

- Added static image serving route (/api/debug-image) to Gateway HTTP server

- Updated storage defaults to align with the new gateway-hosted VNC WebSocket path
This commit is contained in:
dapangyu-fish 2026-03-20 20:44:28 +08:00
parent 5e9463630c
commit c1bfa4a054
10 changed files with 1043 additions and 32 deletions

21
pnpm-lock.yaml generated
View File

@ -633,17 +633,18 @@ importers:
ui:
dependencies:
'@lit-labs/signals':
specifier: ^0.2.0
version: 0.2.0
'@lit/context':
specifier: ^1.1.6
version: 1.1.6
'@noble/ed25519':
<<<<<<< HEAD
specifier: 3.0.0
version: 3.0.0
specifier: 3.0.1
version: 3.0.1
'@novnc/novnc':
specifier: ^1.6.0
version: 1.6.0
=======
specifier: 3.0.1
version: 3.0.1
>>>>>>> main
dompurify:
specifier: ^3.3.3
version: 3.3.3
@ -671,14 +672,10 @@ importers:
version: 8.0.0(@types/node@25.5.0)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2)
vitest:
specifier: 4.1.0
<<<<<<< HEAD
version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/browser-playwright@4.1.0)(jsdom@28.1.0(@noble/hashes@2.0.1))(vite@8.0.0(@types/node@25.5.0)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2))
version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/browser-playwright@4.1.0)(jsdom@29.0.0(@noble/hashes@2.0.1))(vite@8.0.0(@types/node@25.5.0)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2))
ws:
specifier: ^8.19.0
version: 8.19.0
=======
version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/browser-playwright@4.1.0)(jsdom@29.0.0(@noble/hashes@2.0.1))(vite@8.0.0(@types/node@25.5.0)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2))
>>>>>>> main
packages:

View File

@ -800,6 +800,28 @@ export function createGatewayHttpServer(opts: {
rateLimiter,
}),
},
{
name: "debug-image",
run: async () => {
if (requestPath === "/api/debug-image") {
try {
const fs = await import("node:fs");
const img = fs.readFileSync(
"/home/fish/.openclaw/workspace/linux-desktop-control/images/desktop_screeshot_20260318_181311_annotated.png",
);
res.statusCode = 200;
res.setHeader("Content-Type", "image/png");
res.setHeader("Access-Control-Allow-Origin", "*");
res.end(img);
} catch {
res.statusCode = 404;
res.end("Not Found");
}
return true;
}
return false;
},
},
{
name: "slack",
run: () => handleSlackHttpRequest(req, res),
@ -955,6 +977,58 @@ export function attachGatewayUpgradeHandler(opts: {
if (scopedCanvas.rewrittenUrl) {
req.url = scopedCanvas.rewrittenUrl;
}
const url = new URL(req.url ?? "/", "http://localhost");
if (url.pathname === "/api/debug-vnc") {
const target = url.searchParams.get("target");
let host = process.env.OPENCLAW_VNC_HOST || "localhost";
let port = parseInt(process.env.OPENCLAW_VNC_PORT || "5900", 10);
if (target) {
if (target.includes(":")) {
const parts = target.split(":");
host = parts[0] || host;
const p = parseInt(parts[1] || "", 10);
if (!isNaN(p)) {
port = p;
}
} else {
host = target;
}
}
const net = await import("node:net");
// Reuse the same wss for upgrading, but handle connection directly
// Or better yet, we can create an ad-hoc WebSocketServer just for this upgrade
// to avoid mixing with gateway clients.
const { WebSocketServer: VncWss } = await import("ws");
const vncWss = new VncWss({ noServer: true, perMessageDeflate: false });
vncWss.handleUpgrade(req, socket, head, (ws) => {
const tcpSocket = net.connect(port, host);
tcpSocket.on("data", (data) => {
if (ws.readyState === ws.OPEN) {
ws.send(data);
}
});
ws.on("message", (data) => {
if (!tcpSocket.writable) {
return;
}
if (Buffer.isBuffer(data)) {
tcpSocket.write(data);
} else if (Array.isArray(data)) {
tcpSocket.write(Buffer.concat(data));
} else {
tcpSocket.write(Buffer.from(data));
}
});
ws.on("close", () => tcpSocket.end());
tcpSocket.on("close", () => ws.close());
tcpSocket.on("error", () => ws.close());
ws.on("error", () => tcpSocket.end());
});
return;
}
if (canvasHost) {
const url = new URL(req.url ?? "/", "http://localhost");
if (url.pathname === CANVAS_WS_PATH) {

View File

@ -380,6 +380,11 @@ export function renderApp(state: AppViewState) {
? rawDeliveryToSuggestions.filter((value) => isHttpUrl(value))
: rawDeliveryToSuggestions;
const gatewayHttpUrl = state.settings.gatewayUrl
.replace(/^ws:\/\//i, "http://")
.replace(/^wss:\/\//i, "https://");
const debugImageUrl = `${gatewayHttpUrl}/api/debug-image`;
return html`
${renderCommandPalette({
open: state.paletteOpen,
@ -1524,16 +1529,35 @@ export function renderApp(state: AppViewState) {
flex-direction: column;
"
>
<claw-computer-panel
.enabled=${state.showClawComputer}
.vncUrl=${state.settings.vncWsUrl}
.vncTarget=${state.settings.vncTarget}
.password=${state.settings.vncPassword}
@close=${() => state.toggleClawComputer()}
@float=${() => state.setClawComputerWidth(0)}
@dock=${() => state.setClawComputerWidth(600)}
style="flex: 1; min-height: 0;"
></claw-computer-panel>
${
state.activeClawTool === "images"
? html`
<claw-image-panel
.enabled=${state.showClawComputer}
.activeTool=${state.activeClawTool}
.imageUrl=${debugImageUrl}
@tool-change=${(e: CustomEvent) => state.setActiveClawTool(e.detail.tool)}
@close=${() => state.toggleClawComputer()}
@float=${() => state.setClawComputerWidth(0)}
@dock=${() => state.setClawComputerWidth(600)}
style="flex: 1; min-height: 0;"
></claw-image-panel>
`
: html`
<claw-computer-panel
.enabled=${state.showClawComputer}
.activeTool=${state.activeClawTool}
.vncUrl=${state.settings.vncWsUrl}
.vncTarget=${state.settings.vncTarget}
.password=${state.settings.vncPassword}
@tool-change=${(e: CustomEvent) => state.setActiveClawTool(e.detail.tool)}
@close=${() => state.toggleClawComputer()}
@float=${() => state.setClawComputerWidth(0)}
@dock=${() => state.setClawComputerWidth(600)}
style="flex: 1; min-height: 0;"
></claw-computer-panel>
`
}
</div>
</div>
`

View File

@ -373,5 +373,7 @@ export type AppViewState = {
showClawComputer: boolean;
toggleClawComputer: () => void;
clawComputerWidth: number;
activeClawTool: string;
setClawComputerWidth: (width: number) => void;
setActiveClawTool: (tool: string) => void;
};

View File

@ -89,6 +89,7 @@ import { type ChatAttachment, type ChatQueueItem, type CronFormState } from "./u
import { generateUUID } from "./uuid.ts";
import type { NostrProfileFormState } from "./views/channels.nostr-profile-form.ts";
import "./components/claw-computer-panel.ts";
import "./components/claw-image-panel.ts";
declare global {
interface Window {
@ -178,6 +179,7 @@ export class OpenClawApp extends LitElement {
@state() showClawComputer = false;
@state() clawComputerWidth = 600;
@state() activeClawTool = "vnc"; // vnc, browser, images
@state() nodesLoading = false;
@state() nodes: Array<Record<string, unknown>> = [];
@ -736,6 +738,10 @@ export class OpenClawApp extends LitElement {
this.clawComputerWidth = width;
}
setActiveClawTool(tool: string) {
this.activeClawTool = tool;
}
render() {
return renderApp(this as unknown as AppViewState);
}

View File

@ -62,7 +62,7 @@ export class ClawComputerPanel extends LitElement {
private resizeObserver: ResizeObserver | null = null;
private readonly TOOLBAR_SPACE = 50;
@state() private activeTool = "vnc"; // vnc, browser, images
@property({ type: String }) activeTool = "vnc"; // vnc, browser, images
@property({ type: Boolean }) enabled = false;
@ -536,8 +536,9 @@ export class ClawComputerPanel extends LitElement {
};
private setActiveTool(tool: string) {
this.activeTool = tool;
console.log("Active tool changed to:", tool);
this.dispatchEvent(
new CustomEvent("tool-change", { detail: { tool }, bubbles: true, composed: true }),
);
}
private handleDragStart = (e: MouseEvent) => {
@ -910,7 +911,8 @@ export class ClawComputerPanel extends LitElement {
private connect = async () => {
let url =
this.vncUrl || `${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/vnc`;
this.vncUrl ||
`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/api/debug-vnc`;
if (this.vncTarget) {
try {

View File

@ -0,0 +1,905 @@
import { LitElement, html, css } from "lit";
import { customElement, state, property } from "lit/decorators.js";
import { createRef, ref, Ref } from "lit/directives/ref.js";
@customElement("claw-image-panel")
export class ClawImagePanel extends LitElement {
// Floating window state
@state() private isFloating = false;
@state() private dockedOffsetY = 0;
@state() private dockedOffsetX = 0;
@state() private floatingRect = { x: 0, y: 0, width: 600, height: 400 };
private screenRef: Ref<HTMLDivElement> = createRef<HTMLDivElement>();
private dragStart = { x: 0, y: 0 };
private initialRect = {
x: 0,
y: 0,
width: 0,
height: 0,
offsetY: 0,
offsetX: 0,
containerHeight: 0,
containerTop: 0,
containerLeft: 0,
containerWidth: 0,
};
private isDragging = false;
private isResizing = false;
private resizeEdge = "";
private aspectRatio = 1;
// 记录拖拽开始时鼠标相对于拖拽句柄的位置(点 A
private dragAnchor = { x: 0, y: 0 };
// 拖拽过程中的临时悬浮状态
private tempIsFloating = false;
// canvas 的比例(用于悬浮模式缩放时保持)
private canvasRatio = 16 / 9;
private resizeObserver: ResizeObserver | null = null;
private readonly TOOLBAR_SPACE = 50;
@property({ type: String }) activeTool = "images"; // vnc, browser, images
@property({ type: String }) imageUrl = "";
@property({ type: Boolean }) enabled = false;
private setupResizeObserver() {
if (this.resizeObserver) {
return;
}
this.resizeObserver = new ResizeObserver(() => {
this.clampDockedOffset();
});
this.resizeObserver.observe(this);
}
private cleanupResizeObserver() {
if (this.resizeObserver) {
this.resizeObserver.disconnect();
this.resizeObserver = null;
}
}
private clampDockedOffset() {
if (this.isFloating || !this.shadowRoot) {
return;
}
const screenElement = this.shadowRoot.querySelector(".screen");
if (!screenElement) {
return;
}
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
const screen = screenElement as HTMLElement;
const hostHeight = this.offsetHeight;
const screenHeight = screen.offsetHeight;
// 计算允许的最大垂直偏移量(上下边界)
// 默认居中时screenTop = (hostHeight - screenHeight) / 2
// maxUp = -screenTop (移动到最顶部)
// maxDown = screenTop (移动到最底部)
// 注意:这里的 hostHeight 实际上是 screen-container 的 content-box 高度(因为有 padding
// 但 offsetHeight 返回的是包含 padding 的高度。
// 不过,我们这里的逻辑主要是为了限制偏移量。
// 由于我们给容器加了 padding-top: 50pxFlex 居中是相对于剩余空间Content Box的。
// 只要我们限制 screen 不超出 Content Box就自然避开了工具栏。
// 获取容器的实际可用高度(减去 padding
const style = getComputedStyle(this.shadowRoot.querySelector(".screen-container")!);
const paddingTop = parseFloat(style.paddingTop) || 0;
const contentHeight = hostHeight - paddingTop;
// 计算在 Content Box 内的居中位置的顶部距离
const centerTop = (contentHeight - screenHeight) / 2;
const maxOffset = Math.max(0, centerTop);
// 如果当前偏移量超出了允许范围,进行钳制
if (Math.abs(this.dockedOffsetY) > maxOffset) {
this.dockedOffsetY = Math.max(-maxOffset, Math.min(maxOffset, this.dockedOffsetY));
}
}
updated(_changedProperties: Map<string, unknown>) {
// No connection logic needed
}
static styles = css`
:host {
display: block;
height: 100%;
background: var(--bg-accent);
color: var(--text);
font-family: system-ui, sans-serif;
--vnc-border-color: var(--border);
--vnc-window-bg: color-mix(in srgb, var(--bg-accent), black 10%);
}
@media (prefers-color-scheme: light) {
:host {
--vnc-border-color: color-mix(in srgb, var(--border), black 15%);
--vnc-window-bg: color-mix(in srgb, var(--bg-accent), black 10%);
}
}
@media (prefers-color-scheme: dark) {
:host {
--vnc-border-color: color-mix(in srgb, var(--border), white 15%);
--vnc-window-bg: color-mix(in srgb, var(--bg-accent), white 5%);
}
}
:host([theme="light"]) {
--vnc-border-color: color-mix(in srgb, var(--border), black 15%);
--vnc-window-bg: color-mix(in srgb, var(--bg-accent), black 10%);
}
:host([theme="dark"]) {
--vnc-border-color: color-mix(in srgb, var(--border), white 15%);
--vnc-window-bg: color-mix(in srgb, var(--bg-accent), white 5%);
}
.container {
height: 100%;
display: flex;
flex-direction: column;
position: relative;
}
.screen-container {
flex: 1;
width: 100%;
height: 100%;
background: var(--bg-accent);
overflow: hidden;
position: relative;
display: flex;
align-items: center;
justify-content: center;
/* Reserve space for top toolbar */
padding-top: 50px;
box-sizing: border-box;
}
.screen {
width: auto;
height: auto;
max-width: 100%;
max-height: 100%;
display: flex;
align-items: center;
justify-content: center;
padding: 5.4px;
padding-top: 32.4px;
background: var(--vnc-window-bg);
border-radius: 6px;
box-shadow:
0 0 0 1px var(--vnc-border-color, var(--border)),
0 20px 50px rgba(0, 0, 0, 0.4);
box-sizing: border-box;
position: relative;
pointer-events: none;
}
.screen > *:not(.drag-handle):not(.window-controls):not(.resize-handle) {
pointer-events: auto;
}
.screen.dragging > *:not(.drag-handle):not(.window-controls):not(.resize-handle) {
pointer-events: none !important;
}
.screen.dragging canvas {
pointer-events: none !important;
}
.window-controls {
position: absolute;
top: 10.8px;
left: 10.8px;
display: flex;
gap: 7.2px;
z-index: 25;
pointer-events: auto;
}
.window-control {
width: 10.8px;
height: 10.8px;
border-radius: 50%;
cursor: pointer;
border: 1px solid rgba(0, 0, 0, 0.1);
transition:
transform 0.1s,
opacity 0.2s;
}
.window-control:hover {
opacity: 0.8;
transform: scale(1.1);
}
.window-control.maximize {
background-color: #27c93f;
border-color: #1aab29;
}
.window-control.close {
background-color: #ff5f56;
border-color: #e0443e;
}
.window-control.minimize {
background-color: #ffbd2e;
}
.screen canvas {
max-width: 100% !important;
max-height: 100% !important;
width: auto !important;
height: auto !important;
outline: none;
display: block;
margin: auto !important;
border-radius: 0;
box-shadow: none;
pointer-events: auto !important;
}
.status-overlay {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
background: var(--card);
padding: 16px 24px;
border-radius: 8px;
color: var(--text);
font-weight: 500;
pointer-events: none;
z-index: 100;
border: 1px solid var(--border);
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
}
.screen.floating {
position: fixed;
z-index: 9999;
top: 0;
left: 0;
max-width: none;
max-height: none;
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.5);
}
/* Top Toolbar */
.top-toolbar {
position: absolute;
top: 0;
left: 50%;
transform: translateX(-50%);
display: flex;
gap: 8px;
padding: 6px;
background: var(--card);
border: 1px solid var(--border);
border-radius: 999px;
z-index: 100;
pointer-events: auto;
}
.toolbar-btn {
width: 32px;
height: 32px;
border-radius: 50%;
border: 1px solid transparent;
background: transparent;
color: var(--muted);
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
transition: all 0.15s ease;
}
.toolbar-btn:hover {
background: var(--bg-hover);
color: var(--text);
}
.toolbar-btn.active {
background: var(--accent);
color: white;
border-color: var(--accent);
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
}
.toolbar-btn svg {
width: 18px;
height: 18px;
stroke: currentColor;
fill: none;
stroke-width: 2px;
stroke-linecap: round;
stroke-linejoin: round;
}
.drag-handle {
position: absolute;
top: 0;
left: 0;
right: 0;
height: 32.4px;
cursor: grab;
z-index: 20;
pointer-events: auto;
}
.drag-handle:active {
cursor: grabbing;
}
.resize-handle {
position: absolute;
background: transparent;
z-index: 30;
pointer-events: auto;
}
.resize-handle.top {
top: -5px;
left: 0;
right: 0;
height: 10px;
cursor: ns-resize;
}
.resize-handle.bottom {
bottom: -5px;
left: 0;
right: 0;
height: 10px;
cursor: ns-resize;
}
.resize-handle.left {
left: -5px;
top: 0;
bottom: 0;
width: 10px;
cursor: ew-resize;
}
.resize-handle.right {
right: -5px;
top: 0;
bottom: 0;
width: 10px;
cursor: ew-resize;
}
.resize-handle.top-left {
top: -5px;
left: -5px;
width: 15px;
height: 15px;
cursor: nwse-resize;
z-index: 35;
}
.resize-handle.top-right {
top: -5px;
right: -5px;
width: 15px;
height: 15px;
cursor: nesw-resize;
z-index: 35;
}
.resize-handle.bottom-left {
bottom: -5px;
left: -5px;
width: 15px;
height: 15px;
cursor: nesw-resize;
z-index: 35;
}
.resize-handle.bottom-right {
bottom: -5px;
right: -5px;
width: 15px;
height: 15px;
cursor: nwse-resize;
z-index: 35;
}
`;
render() {
const displayFloating = this.isDragging ? this.tempIsFloating : this.isFloating;
const screenStyle = displayFloating
? `transform: translate(${this.floatingRect.x}px, ${this.floatingRect.y}px); width: ${this.floatingRect.width}px; height: ${this.floatingRect.height}px;`
: `transform: translate(${this.dockedOffsetX}px, ${this.dockedOffsetY}px);`;
return html`
<div class="container">
<div class="screen-container">
<!-- Top Toolbar (Dock Mode Icons) - Fixed at top of container -->
${
!displayFloating
? html`
<div class="top-toolbar">
<button
class="toolbar-btn ${this.activeTool === "vnc" ? "active" : ""}"
@click=${() => this.setActiveTool("vnc")}
title="Remote Desktop"
>
<svg viewBox="0 0 24 24">
<rect width="20" height="14" x="2" y="3" rx="2" />
<line x1="8" x2="16" y1="21" y2="21" />
<line x1="12" x2="12" y1="17" y2="21" />
</svg>
</button>
<button
class="toolbar-btn ${this.activeTool === "browser" ? "active" : ""}"
@click=${() => this.setActiveTool("browser")}
title="Browser"
>
<svg viewBox="0 0 24 24">
<circle cx="12" cy="12" r="10" />
<line x1="2" x2="22" y1="12" y2="12" />
<path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z" />
</svg>
</button>
<button
class="toolbar-btn ${this.activeTool === "images" ? "active" : ""}"
@click=${() => this.setActiveTool("images")}
title="Images"
>
<svg viewBox="0 0 24 24">
<rect width="18" height="18" x="3" y="3" rx="2" ry="2" />
<circle cx="9" cy="9" r="2" />
<path d="m21 15-3.086-3.086a2 2 0 0 0-2.828 0L6 21" />
</svg>
</button>
</div>
`
: null
}
<div
${ref(this.screenRef)}
class="screen ${displayFloating ? "floating" : ""}"
style="${screenStyle}"
>
<img src="${this.imageUrl}" style="max-width: 100%; max-height: 100%; object-fit: contain; pointer-events: auto;" />
<div class="drag-handle" @mousedown=${this.handleDragStart}></div>
${
displayFloating
? html`
<div class="resize-handle top" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "top")}></div>
<div class="resize-handle bottom" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "bottom")}></div>
<div class="resize-handle left" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "left")}></div>
<div class="resize-handle right" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "right")}></div>
<div class="resize-handle top-left" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "top-left")}></div>
<div class="resize-handle top-right" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "top-right")}></div>
<div class="resize-handle bottom-left" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "bottom-left")}></div>
<div class="resize-handle bottom-right" @mousedown=${(e: MouseEvent) => this.handleResizeStart(e, "bottom-right")}></div>
`
: null
}
<div class="window-controls">
<div class="window-control close" @click=${this.handleClose}></div>
<div class="window-control minimize"></div>
<div class="window-control maximize" @click=${this.toggleFullscreen}></div>
</div>
</div>
</div>
</div>
`;
}
private handleClose = () => {
this.dispatchEvent(new CustomEvent("close", { bubbles: true, composed: true }));
};
private setActiveTool(tool: string) {
this.dispatchEvent(
new CustomEvent("tool-change", { detail: { tool }, bubbles: true, composed: true }),
);
}
private handleDragStart = (e: MouseEvent) => {
if (this.isResizing) {
return;
}
e.preventDefault();
e.stopPropagation();
// 先清理旧的监听器,防止重复添加
this.cleanupDragListeners();
this.isDragging = true;
this.dragStart = { x: e.clientX, y: e.clientY };
this.tempIsFloating = this.isFloating; // 初始化临时状态
const screen = this.shadowRoot?.querySelector(".screen") as HTMLDivElement | null;
const dragHandle = this.shadowRoot?.querySelector(".drag-handle") as HTMLDivElement | null;
if (screen && dragHandle) {
// 拖拽过程中,暂时禁用 screen 内部所有元素的 pointer-events
// 这样即使鼠标移动到 VNC 窗口内部,也不会被 canvas 捕获
screen.classList.add("dragging");
// 记录鼠标相对于拖拽句柄的位置(点 A
const dragHandleRect = dragHandle.getBoundingClientRect();
this.dragAnchor = {
x: e.clientX - dragHandleRect.left,
y: e.clientY - dragHandleRect.top,
};
const rect = screen.getBoundingClientRect();
// 获取 claw-computer-panel 本身的尺寸
const hostRect = this.getBoundingClientRect();
// 使用 offsetWidth/offsetHeight 获取实际尺寸
const screenWidth = screen.offsetWidth;
const screenHeight = screen.offsetHeight;
const hostHeight = this.offsetHeight;
this.initialRect = {
x: this.isFloating ? this.floatingRect.x : rect.left,
y: this.isFloating ? this.floatingRect.y : rect.top,
width: screenWidth,
height: screenHeight,
offsetY: this.dockedOffsetY,
offsetX: this.dockedOffsetX,
containerHeight: hostHeight,
containerTop: hostRect.top,
containerLeft: hostRect.left,
containerWidth: this.offsetWidth,
};
if (!this.isFloating) {
this.floatingRect = {
x: rect.left,
y: rect.top,
width: screenWidth,
height: screenHeight,
};
// 打印尺寸信息,用于调试
const canvas = this.shadowRoot?.querySelector(".screen canvas") as HTMLCanvasElement | null;
if (canvas) {
console.log("=== 切换到悬浮模式时的尺寸信息 ===");
console.log("screen 宽度:", screenWidth);
console.log("screen 高度:", screenHeight);
console.log("screen 比例:", screenWidth / screenHeight);
console.log("canvas 宽度:", canvas.clientWidth);
console.log("canvas 高度:", canvas.clientHeight);
console.log("canvas 比例:", canvas.clientWidth / canvas.clientHeight);
}
}
}
window.addEventListener("mousemove", this.handleDragMove, { capture: true, passive: false });
window.addEventListener("mouseup", this.handleDragEnd, { capture: true });
};
private handleDragMove = (e: MouseEvent) => {
if (!this.isDragging) {
return;
}
// 安全检查:如果鼠标按钮不再按下,就停止拖拽
if (e.buttons !== 1) {
this.cleanupDragListeners();
return;
}
e.preventDefault();
e.stopPropagation();
const dx = e.clientX - this.dragStart.x;
const dy = e.clientY - this.dragStart.y;
// 获取我们自己的左边界
const hostRect = this.getBoundingClientRect();
// 根据鼠标位置更新临时悬浮状态
if (e.clientX < hostRect.left) {
// 鼠标在左侧,临时切换到悬浮模式
if (!this.tempIsFloating) {
this.tempIsFloating = true;
// 计算新的悬浮窗口位置,保持鼠标相对于拖拽句柄的位置(点 A
const newX = e.clientX - this.dragAnchor.x;
const newY = e.clientY - this.dragAnchor.y;
// 更新 floatingRect
this.floatingRect = {
x: newX,
y: newY,
width: this.initialRect.width,
height: this.initialRect.height,
};
// 更新 initialRect使其基于新的悬浮位置
this.initialRect = {
...this.initialRect,
x: newX,
y: newY,
};
// 更新 dragStart使其从当前位置继续拖拽
this.dragStart = { x: e.clientX, y: e.clientY };
}
} else {
// 鼠标在右侧,临时切换回停靠模式
if (this.tempIsFloating) {
this.tempIsFloating = false;
// 重置停靠模式的偏移量
this.dockedOffsetY = this.initialRect.offsetY;
this.dockedOffsetX = 0;
// 更新 dragStart使其从当前位置继续拖拽
this.dragStart = { x: e.clientX, y: e.clientY };
}
}
if (this.tempIsFloating) {
let newX = this.initialRect.x + dx;
let newY = this.initialRect.y + dy;
const windowWidth = window.innerWidth;
const windowHeight = window.innerHeight;
// 限制悬浮窗口不超出屏幕边界
newX = Math.max(0, Math.min(newX, windowWidth - this.initialRect.width));
newY = Math.max(0, Math.min(newY, windowHeight - this.initialRect.height));
this.floatingRect = {
x: newX,
y: newY,
width: this.initialRect.width,
height: this.initialRect.height,
};
} else {
// 如果没有超过分割线,继续停靠模式的拖拽
// 每次都直接获取当前尺寸
const hostHeight = this.offsetHeight;
const screen = this.shadowRoot?.querySelector(".screen") as HTMLDivElement | null;
const screenHeight = screen?.offsetHeight || 0;
// 计算最高和最低不能超过多少
// 获取容器的实际可用高度(减去 padding
const container = this.shadowRoot?.querySelector(".screen-container");
const style = container ? getComputedStyle(container) : null;
const paddingTop = style ? parseFloat(style.paddingTop) || 0 : 50;
const contentHeight = hostHeight - paddingTop;
// 当 screen 居中时,初始顶部位置是 (contentHeight - screenHeight) / 2
const centerTop = (contentHeight - screenHeight) / 2;
// 允许的最上偏移量 (offset = -centerTop)
// 目标位置是 PADDING_TOP
const maxUpOffset = -centerTop;
// 允许的最下偏移量 (offset = centerTop)
// 保持原来的逻辑,底部贴合容器底部
const maxDownOffset = centerTop;
let newOffsetY = this.initialRect.offsetY + dy;
// 使用计算出来的边界限制
newOffsetY = Math.max(maxUpOffset, Math.min(newOffsetY, maxDownOffset));
this.dockedOffsetY = newOffsetY;
// 禁止左右移动
this.dockedOffsetX = 0;
}
};
private cleanupDragListeners = () => {
this.isDragging = false;
// 移除拖拽过程中添加的类,恢复 pointer-events
const screen = this.shadowRoot?.querySelector(".screen") as HTMLDivElement | null;
if (screen) {
screen.classList.remove("dragging");
}
try {
window.removeEventListener("mousemove", this.handleDragMove, { capture: true });
window.removeEventListener("mouseup", this.handleDragEnd, { capture: true });
} catch (e) {
console.error("Error removing drag event listeners:", e);
}
};
private handleDragEnd = (e: MouseEvent) => {
// 根据松手位置决定是否永久切换到悬浮模式
const hostRect = this.getBoundingClientRect();
const isDroppingInDockArea = e.clientX >= hostRect.left;
if (!isDroppingInDockArea) {
// 在左侧(悬浮区域)松手
if (!this.isFloating) {
// 原本是停靠,现在变悬浮 -> 触发 float
this.isFloating = true;
this.dispatchEvent(new CustomEvent("float", { bubbles: true, composed: true }));
}
} else {
// 在右侧(停靠区域)松手
if (this.isFloating) {
// 原本是悬浮,现在变停靠 -> 触发 dock
this.isFloating = false;
this.dockedOffsetX = 0;
this.dispatchEvent(new CustomEvent("dock", { bubbles: true, composed: true }));
} else {
// 原本是停靠,现在还是停靠 -> 只是调整了垂直位置,或者是点击
// 不需要触发 dock 事件,以免重置宽度
this.isFloating = false;
this.dockedOffsetX = 0;
}
}
this.cleanupDragListeners();
};
private cleanupResizeListeners = () => {
this.isResizing = false;
try {
window.removeEventListener("mousemove", this.handleResizeMove);
window.removeEventListener("mouseup", this.handleResizeEnd);
} catch (e) {
console.error("Error removing resize event listeners:", e);
}
};
private handleResizeStart = (e: MouseEvent, edge: string) => {
e.preventDefault();
e.stopPropagation();
// 先清理旧的监听器,防止重复添加
this.cleanupResizeListeners();
this.isResizing = true;
this.resizeEdge = edge;
this.dragStart = { x: e.clientX, y: e.clientY };
// 每次开始缩放时,获取最新的 canvas 比例
const canvas = this.shadowRoot?.querySelector(".screen canvas") as HTMLCanvasElement | null;
if (canvas && canvas.clientWidth > 0 && canvas.clientHeight > 0) {
this.canvasRatio = canvas.clientWidth / canvas.clientHeight;
}
this.initialRect = {
x: this.floatingRect.x,
y: this.floatingRect.y,
width: this.floatingRect.width,
height: this.floatingRect.height,
offsetY: 0,
offsetX: 0,
containerHeight: 0,
containerTop: 0,
containerLeft: 0,
containerWidth: 0,
};
this.aspectRatio = this.floatingRect.width / this.floatingRect.height;
window.addEventListener("mousemove", this.handleResizeMove);
window.addEventListener("mouseup", this.handleResizeEnd);
};
private handleResizeMove = (e: MouseEvent) => {
if (!this.isResizing) {
return;
}
// 安全检查:如果鼠标按钮不再按下,就停止调整大小
if (e.buttons !== 1) {
this.cleanupResizeListeners();
return;
}
e.preventDefault();
const dx = e.clientX - this.dragStart.x;
const dy = e.clientY - this.dragStart.y;
// 考虑到窗口有边框和标题栏,实际的 canvas 内容区域要减去这些 padding
// 左右 padding: 5.4px * 2 = 10.8px
// 上下 padding: 32.4px (标题栏) + 5.4px (底边) = 37.8px
const PADDING_X = 10.8;
const PADDING_Y = 37.8;
let { x, y, width, height } = this.initialRect;
// 1. 先计算出用户拖拽后的预期宽高
if (this.resizeEdge.includes("right")) {
width += dx;
}
if (this.resizeEdge.includes("left")) {
x += dx;
width -= dx;
}
if (this.resizeEdge.includes("bottom")) {
height += dy;
}
if (this.resizeEdge.includes("top")) {
y += dy;
height -= dy;
}
// 2. 根据 canvas 的比例强制修正窗口尺寸,确保窗口能仅仅贴合 VNC
let canvasW = width - PADDING_X;
let canvasH = height - PADDING_Y;
if (this.resizeEdge === "top" || this.resizeEdge === "bottom") {
// 如果只拖拽上下边,以高度为基准计算宽度
canvasW = canvasH * this.canvasRatio;
width = canvasW + PADDING_X;
if (this.resizeEdge.includes("left")) {
x = this.initialRect.x + (this.initialRect.width - width);
}
} else {
// 左右边或者对角线,统一以宽度为基准计算高度
canvasH = canvasW / this.canvasRatio;
height = canvasH + PADDING_Y;
if (this.resizeEdge.includes("top")) {
y = this.initialRect.y + (this.initialRect.height - height);
}
}
// 3. 最小尺寸限制
if (width < 200) {
width = 200;
canvasW = width - PADDING_X;
canvasH = canvasW / this.canvasRatio;
height = canvasH + PADDING_Y;
if (this.resizeEdge.includes("left")) {
x = this.initialRect.x + (this.initialRect.width - width);
}
if (this.resizeEdge.includes("top")) {
y = this.initialRect.y + (this.initialRect.height - height);
}
}
// 注意:这里不再单独限制 height因为 height 已经和 width 绑定了
this.floatingRect = { x, y, width, height };
};
private handleResizeEnd = () => {
this.cleanupResizeListeners();
};
private handleResize = () => {
// No-op for image
};
private toggleFullscreen = () => {
const container = this.shadowRoot?.querySelector(".screen-container");
if (container) {
void (container as HTMLElement).requestFullscreen?.();
}
};
private handleWindowBlur = () => {
this.cleanupDragListeners();
this.cleanupResizeListeners();
};
firstUpdated() {
window.addEventListener("resize", this.handleResize);
window.addEventListener("blur", this.handleWindowBlur);
this.setupResizeObserver();
}
disconnectedCallback() {
super.disconnectedCallback();
window.removeEventListener("resize", this.handleResize);
window.removeEventListener("blur", this.handleWindowBlur);
this.cleanupDragListeners();
this.cleanupResizeListeners();
this.cleanupResizeObserver();
}
}

View File

@ -134,6 +134,8 @@ describe("loadSettings default gateway URL derivation", () => {
navWidth: 220,
navGroupsCollapsed: {},
borderRadius: 50,
vncTarget: "localhost:5900",
vncWsUrl: "wss://gateway.example:8443/api/debug-vnc",
sessionsByGateway: {
"wss://gateway.example:8443/openclaw": {
sessionKey: "agent",
@ -250,6 +252,8 @@ describe("loadSettings default gateway URL derivation", () => {
navWidth: 220,
navGroupsCollapsed: {},
borderRadius: 50,
vncTarget: "localhost:5900",
vncWsUrl: "wss://gateway.example:8443/api/debug-vnc",
});
expect(loadSettings()).toMatchObject({
gatewayUrl: gwUrl,
@ -269,6 +273,8 @@ describe("loadSettings default gateway URL derivation", () => {
navWidth: 220,
navGroupsCollapsed: {},
borderRadius: 50,
vncTarget: "localhost:5900",
vncWsUrl: "wss://gateway.example:8443/api/debug-vnc",
sessionsByGateway: {
[gwUrl]: {
sessionKey: "main",

View File

@ -194,10 +194,7 @@ export function loadSettings(): UiSettings {
navCollapsed: false,
navWidth: 220,
navGroupsCollapsed: {},
vncWsUrl:
typeof location !== "undefined"
? `${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/vnc`
: "ws://localhost:18789/vnc",
vncWsUrl: `${defaultUrl.replace(/\/$/, "")}/api/debug-vnc`,
vncTarget: "localhost:5900",
borderRadius: 50,
};

View File

@ -1,7 +1,6 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import { defineConfig } from "vite";
import { vncProxyPlugin } from "./vnc-proxy-plugin.ts";
const here = path.dirname(fileURLToPath(import.meta.url));
@ -41,7 +40,6 @@ export default defineConfig(() => {
strictPort: true,
},
plugins: [
vncProxyPlugin(),
{
name: "control-ui-dev-stubs",
configureServer(server) {