From a60d986020e5db7a48d3e61cdeecd782bf32cc20 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 13 Jan 2026 12:34:00 -0600 Subject: [PATCH] screencast --- package.json | 2 + pnpm-lock.yaml | 27 ++++ src/actions.ts | 109 ++++++++++++- src/browser.ts | 208 +++++++++++++++++++++++++ src/daemon.ts | 52 ++++++- src/protocol.ts | 54 +++++++ src/stream-server.ts | 358 +++++++++++++++++++++++++++++++++++++++++++ src/types.ts | 64 +++++++- 8 files changed, 870 insertions(+), 4 deletions(-) create mode 100644 src/stream-server.ts diff --git a/package.json b/package.json index 0453749..545f39d 100644 --- a/package.json +++ b/package.json @@ -53,10 +53,12 @@ "homepage": "https://github.com/vercel-labs/agent-browser#readme", "dependencies": { "playwright-core": "^1.57.0", + "ws": "^8.19.0", "zod": "^3.22.4" }, "devDependencies": { "@types/node": "^20.10.0", + "@types/ws": "^8.18.1", "husky": "^9.1.7", "lint-staged": "^15.2.11", "playwright": "^1.57.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 434e3ff..77dc072 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: playwright-core: specifier: ^1.57.0 version: 1.57.0 + ws: + specifier: ^8.19.0 + version: 8.19.0 zod: specifier: ^3.22.4 version: 3.25.76 @@ -18,6 +21,9 @@ importers: '@types/node': specifier: ^20.10.0 version: 20.19.28 + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 husky: specifier: ^9.1.7 version: 9.1.7 @@ -341,6 +347,9 @@ packages: '@types/node@20.19.28': resolution: {integrity: sha512-VyKBr25BuFDzBFCK5sUM6ZXiWfqgCTwTAOK8qzGV/m9FCirXYDlmczJ+d5dXBAQALGCdRRdbteKYfJ84NGEusw==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@vitest/expect@4.0.16': resolution: {integrity: sha512-eshqULT2It7McaJkQGLkPjPjNph+uevROGuIMJdG3V+0BSR2w9u6J9Lwu+E8cK5TETlfou8GRijhafIMhXsimA==} @@ -805,6 +814,18 @@ packages: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} + ws@8.19.0: + resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + yaml@2.8.2: resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} engines: {node: '>= 14.6'} @@ -985,6 +1006,10 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/ws@8.18.1': + dependencies: + '@types/node': 20.19.28 + '@vitest/expect@4.0.16': dependencies: '@standard-schema/spec': 1.1.0 @@ -1427,6 +1452,8 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.1.2 + ws@8.19.0: {} + yaml@2.8.2: {} zod@3.25.76: {} diff --git a/src/actions.ts b/src/actions.ts index 6c168d9..747ca1b 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -1,5 +1,5 @@ import type { Page, Frame } from 'playwright-core'; -import type { BrowserManager } from './browser.js'; +import type { BrowserManager, ScreencastFrame } from './browser.js'; import type { Command, Response, @@ -94,6 +94,11 @@ import type { MultiSelectCommand, WaitForDownloadCommand, ResponseBodyCommand, + ScreencastStartCommand, + ScreencastStopCommand, + InputMouseCommand, + InputKeyboardCommand, + InputTouchCommand, NavigateData, ScreenshotData, EvaluateData, @@ -102,9 +107,25 @@ import type { TabNewData, TabSwitchData, TabCloseData, + ScreencastStartData, + ScreencastStopData, + InputEventData, } from './types.js'; import { successResponse, errorResponse } from './protocol.js'; +// Callback for screencast frames - will be set by the daemon when streaming is active +let screencastFrameCallback: ((frame: ScreencastFrame) => void) | null = null; + +/** + * Set the callback for screencast frames + * This is called by the daemon to set up frame streaming + */ +export function setScreencastFrameCallback( + callback: ((frame: ScreencastFrame) => void) | null +): void { + screencastFrameCallback = callback; +} + // Snapshot response type interface SnapshotData { snapshot: string; @@ -386,6 +407,16 @@ export async function executeCommand(command: Command, browser: BrowserManager): return await handleWaitForDownload(command, browser); case 'responsebody': return await handleResponseBody(command, browser); + case 'screencast_start': + return await handleScreencastStart(command, browser); + case 'screencast_stop': + return await handleScreencastStop(command, browser); + case 'input_mouse': + return await handleInputMouse(command, browser); + case 'input_keyboard': + return await handleInputKeyboard(command, browser); + case 'input_touch': + return await handleInputTouch(command, browser); default: { // TypeScript narrows to never here, but we handle it for safety const unknownCommand = command as { id: string; action: string }; @@ -1769,3 +1800,79 @@ async function handleResponseBody( body: parsed, }); } + +// Screencast and input injection handlers + +async function handleScreencastStart( + command: ScreencastStartCommand, + browser: BrowserManager +): Promise> { + if (!screencastFrameCallback) { + throw new Error('Screencast frame callback not set. Start the streaming server first.'); + } + + await browser.startScreencast(screencastFrameCallback, { + format: command.format, + quality: command.quality, + maxWidth: command.maxWidth, + maxHeight: command.maxHeight, + everyNthFrame: command.everyNthFrame, + }); + + return successResponse(command.id, { + started: true, + format: command.format ?? 'jpeg', + quality: command.quality ?? 80, + }); +} + +async function handleScreencastStop( + command: ScreencastStopCommand, + browser: BrowserManager +): Promise> { + await browser.stopScreencast(); + return successResponse(command.id, { stopped: true }); +} + +async function handleInputMouse( + command: InputMouseCommand, + browser: BrowserManager +): Promise> { + await browser.injectMouseEvent({ + type: command.type, + x: command.x, + y: command.y, + button: command.button, + clickCount: command.clickCount, + deltaX: command.deltaX, + deltaY: command.deltaY, + modifiers: command.modifiers, + }); + return successResponse(command.id, { injected: true }); +} + +async function handleInputKeyboard( + command: InputKeyboardCommand, + browser: BrowserManager +): Promise> { + await browser.injectKeyboardEvent({ + type: command.type, + key: command.key, + code: command.code, + text: command.text, + modifiers: command.modifiers, + }); + return successResponse(command.id, { injected: true }); +} + +async function handleInputTouch( + command: InputTouchCommand, + browser: BrowserManager +): Promise> { + await browser.injectTouchEvent({ + type: command.type, + touchPoints: command.touchPoints, + modifiers: command.modifiers, + }); + return successResponse(command.id, { injected: true }); +} diff --git a/src/browser.ts b/src/browser.ts index c948273..284783c 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -11,10 +11,35 @@ import { type Request, type Route, type Locator, + type CDPSession, } from 'playwright-core'; import type { LaunchCommand } from './types.js'; import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js'; +// Screencast frame data from CDP +export interface ScreencastFrame { + data: string; // base64 encoded image + metadata: { + offsetTop: number; + pageScaleFactor: number; + deviceWidth: number; + deviceHeight: number; + scrollOffsetX: number; + scrollOffsetY: number; + timestamp?: number; + }; + sessionId: number; +} + +// Screencast options +export interface ScreencastOptions { + format?: 'jpeg' | 'png'; + quality?: number; // 0-100, only for jpeg + maxWidth?: number; + maxHeight?: number; + everyNthFrame?: number; +} + interface TrackedRequest { url: string; method: string; @@ -54,6 +79,12 @@ export class BrowserManager { private lastSnapshot: string = ''; private scopedHeaderRoutes: Map Promise> = new Map(); + // CDP session for screencast and input injection + private cdpSession: CDPSession | null = null; + private screencastActive: boolean = false; + private screencastSessionId: number = 0; + private frameCallback: ((frame: ScreencastFrame) => void) | null = null; + /** * Check if browser is launched */ @@ -850,10 +881,186 @@ export class BrowserManager { return tabs; } + /** + * Get or create a CDP session for the current page + * Only works with Chromium-based browsers + */ + async getCDPSession(): Promise { + if (this.cdpSession) { + return this.cdpSession; + } + + const page = this.getPage(); + const context = page.context(); + + // Create a new CDP session attached to the page + this.cdpSession = await context.newCDPSession(page); + return this.cdpSession; + } + + /** + * Check if screencast is currently active + */ + isScreencasting(): boolean { + return this.screencastActive; + } + + /** + * Start screencast - streams viewport frames via CDP + * @param callback Function called for each frame + * @param options Screencast options + */ + async startScreencast( + callback: (frame: ScreencastFrame) => void, + options?: ScreencastOptions + ): Promise { + if (this.screencastActive) { + throw new Error('Screencast already active'); + } + + const cdp = await this.getCDPSession(); + this.frameCallback = callback; + this.screencastActive = true; + + // Listen for screencast frames + cdp.on('Page.screencastFrame', async (params: any) => { + const frame: ScreencastFrame = { + data: params.data, + metadata: params.metadata, + sessionId: params.sessionId, + }; + + // Acknowledge the frame to receive the next one + await cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId }); + + // Call the callback with the frame + if (this.frameCallback) { + this.frameCallback(frame); + } + }); + + // Start the screencast + await cdp.send('Page.startScreencast', { + format: options?.format ?? 'jpeg', + quality: options?.quality ?? 80, + maxWidth: options?.maxWidth ?? 1280, + maxHeight: options?.maxHeight ?? 720, + everyNthFrame: options?.everyNthFrame ?? 1, + }); + } + + /** + * Stop screencast + */ + async stopScreencast(): Promise { + if (!this.screencastActive) { + return; + } + + try { + const cdp = await this.getCDPSession(); + await cdp.send('Page.stopScreencast'); + } catch { + // Ignore errors when stopping + } + + this.screencastActive = false; + this.frameCallback = null; + } + + /** + * Inject a mouse event via CDP + */ + async injectMouseEvent(params: { + type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel'; + x: number; + y: number; + button?: 'left' | 'right' | 'middle' | 'none'; + clickCount?: number; + deltaX?: number; + deltaY?: number; + modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift + }): Promise { + const cdp = await this.getCDPSession(); + + const cdpButton = + params.button === 'left' + ? 'left' + : params.button === 'right' + ? 'right' + : params.button === 'middle' + ? 'middle' + : 'none'; + + await cdp.send('Input.dispatchMouseEvent', { + type: params.type, + x: params.x, + y: params.y, + button: cdpButton, + clickCount: params.clickCount ?? 1, + deltaX: params.deltaX ?? 0, + deltaY: params.deltaY ?? 0, + modifiers: params.modifiers ?? 0, + }); + } + + /** + * Inject a keyboard event via CDP + */ + async injectKeyboardEvent(params: { + type: 'keyDown' | 'keyUp' | 'char'; + key?: string; + code?: string; + text?: string; + modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift + }): Promise { + const cdp = await this.getCDPSession(); + + await cdp.send('Input.dispatchKeyEvent', { + type: params.type, + key: params.key, + code: params.code, + text: params.text, + modifiers: params.modifiers ?? 0, + }); + } + + /** + * Inject touch event via CDP (for mobile emulation) + */ + async injectTouchEvent(params: { + type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel'; + touchPoints: Array<{ x: number; y: number; id?: number }>; + modifiers?: number; + }): Promise { + const cdp = await this.getCDPSession(); + + await cdp.send('Input.dispatchTouchEvent', { + type: params.type, + touchPoints: params.touchPoints.map((tp, i) => ({ + x: tp.x, + y: tp.y, + id: tp.id ?? i, + })), + modifiers: params.modifiers ?? 0, + }); + } + /** * Close the browser and clean up */ async close(): Promise { + // Stop screencast if active + if (this.screencastActive) { + await this.stopScreencast(); + } + + // Clean up CDP session + if (this.cdpSession) { + await this.cdpSession.detach().catch(() => {}); + this.cdpSession = null; + } + // CDP: only disconnect, don't close external app's pages if (this.cdpPort !== null) { if (this.browser) { @@ -880,5 +1087,6 @@ export class BrowserManager { this.activePageIndex = 0; this.refMap = {}; this.lastSnapshot = ''; + this.frameCallback = null; } } diff --git a/src/daemon.ts b/src/daemon.ts index 61ecc07..f894318 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -5,6 +5,7 @@ import * as os from 'os'; import { BrowserManager } from './browser.js'; import { parseCommand, serializeResponse, errorResponse } from './protocol.js'; import { executeCommand } from './actions.js'; +import { StreamServer } from './stream-server.js'; // Platform detection const isWindows = process.platform === 'win32'; @@ -12,6 +13,12 @@ const isWindows = process.platform === 'win32'; // Session support - each session gets its own socket/pid let currentSession = process.env.AGENT_BROWSER_SESSION || 'default'; +// Stream server for browser preview +let streamServer: StreamServer | null = null; + +// Default stream port (can be overridden with AGENT_BROWSER_STREAM_PORT) +const DEFAULT_STREAM_PORT = 9223; + /** * Set the current session */ @@ -105,8 +112,10 @@ export function getConnectionInfo( */ export function cleanupSocket(session?: string): void { const pidFile = getPidFile(session); + const streamPortFile = getStreamPortFile(session); try { if (fs.existsSync(pidFile)) fs.unlinkSync(pidFile); + if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile); if (isWindows) { const portFile = getPortFile(session); if (fs.existsSync(portFile)) fs.unlinkSync(portFile); @@ -120,15 +129,40 @@ export function cleanupSocket(session?: string): void { } /** - * Start the daemon server + * Get the stream port file path */ -export async function startDaemon(): Promise { +export function getStreamPortFile(session?: string): string { + const sess = session ?? currentSession; + return path.join(os.tmpdir(), `agent-browser-${sess}.stream`); +} + +/** + * Start the daemon server + * @param options.streamPort Port for WebSocket stream server (0 to disable) + */ +export async function startDaemon(options?: { streamPort?: number }): Promise { // Clean up any stale socket cleanupSocket(); const browser = new BrowserManager(); let shuttingDown = false; + // Start stream server if port is specified (or use default if env var is set) + const streamPort = + options?.streamPort ?? + (process.env.AGENT_BROWSER_STREAM_PORT + ? parseInt(process.env.AGENT_BROWSER_STREAM_PORT, 10) + : 0); + + if (streamPort > 0) { + streamServer = new StreamServer(browser, streamPort); + await streamServer.start(); + + // Write stream port to file for clients to discover + const streamPortFile = getStreamPortFile(); + fs.writeFileSync(streamPortFile, streamPort.toString()); + } + const server = net.createServer((socket) => { let buffer = ''; @@ -227,6 +261,20 @@ export async function startDaemon(): Promise { const shutdown = async () => { if (shuttingDown) return; shuttingDown = true; + + // Stop stream server if running + if (streamServer) { + await streamServer.stop(); + streamServer = null; + // Clean up stream port file + const streamPortFile = getStreamPortFile(); + try { + if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile); + } catch { + // Ignore cleanup errors + } + } + await browser.close(); server.close(); cleanupSocket(); diff --git a/src/protocol.ts b/src/protocol.ts index bafdf53..d4a36c3 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -585,6 +585,55 @@ const responseBodySchema = baseCommandSchema.extend({ timeout: z.number().positive().optional(), }); +// Screencast schemas for streaming browser viewport +const screencastStartSchema = baseCommandSchema.extend({ + action: z.literal('screencast_start'), + format: z.enum(['jpeg', 'png']).optional(), + quality: z.number().min(0).max(100).optional(), + maxWidth: z.number().positive().optional(), + maxHeight: z.number().positive().optional(), + everyNthFrame: z.number().positive().optional(), +}); + +const screencastStopSchema = baseCommandSchema.extend({ + action: z.literal('screencast_stop'), +}); + +// Input injection schemas for pair browsing +const inputMouseSchema = baseCommandSchema.extend({ + action: z.literal('input_mouse'), + type: z.enum(['mousePressed', 'mouseReleased', 'mouseMoved', 'mouseWheel']), + x: z.number(), + y: z.number(), + button: z.enum(['left', 'right', 'middle', 'none']).optional(), + clickCount: z.number().positive().optional(), + deltaX: z.number().optional(), + deltaY: z.number().optional(), + modifiers: z.number().optional(), +}); + +const inputKeyboardSchema = baseCommandSchema.extend({ + action: z.literal('input_keyboard'), + type: z.enum(['keyDown', 'keyUp', 'char']), + key: z.string().optional(), + code: z.string().optional(), + text: z.string().optional(), + modifiers: z.number().optional(), +}); + +const inputTouchSchema = baseCommandSchema.extend({ + action: z.literal('input_touch'), + type: z.enum(['touchStart', 'touchEnd', 'touchMove', 'touchCancel']), + touchPoints: z.array( + z.object({ + x: z.number(), + y: z.number(), + id: z.number().optional(), + }) + ), + modifiers: z.number().optional(), +}); + const pressSchema = baseCommandSchema.extend({ action: z.literal('press'), key: z.string().min(1), @@ -795,6 +844,11 @@ const commandSchema = z.discriminatedUnion('action', [ multiSelectSchema, waitForDownloadSchema, responseBodySchema, + screencastStartSchema, + screencastStopSchema, + inputMouseSchema, + inputKeyboardSchema, + inputTouchSchema, ]); // Parse result type diff --git a/src/stream-server.ts b/src/stream-server.ts new file mode 100644 index 0000000..cf10628 --- /dev/null +++ b/src/stream-server.ts @@ -0,0 +1,358 @@ +import { WebSocketServer, WebSocket } from 'ws'; +import type { BrowserManager, ScreencastFrame } from './browser.js'; +import { setScreencastFrameCallback } from './actions.js'; + +// Message types for WebSocket communication +export interface FrameMessage { + type: 'frame'; + data: string; // base64 encoded image + metadata: { + offsetTop: number; + pageScaleFactor: number; + deviceWidth: number; + deviceHeight: number; + scrollOffsetX: number; + scrollOffsetY: number; + timestamp?: number; + }; +} + +export interface InputMouseMessage { + type: 'input_mouse'; + eventType: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel'; + x: number; + y: number; + button?: 'left' | 'right' | 'middle' | 'none'; + clickCount?: number; + deltaX?: number; + deltaY?: number; + modifiers?: number; +} + +export interface InputKeyboardMessage { + type: 'input_keyboard'; + eventType: 'keyDown' | 'keyUp' | 'char'; + key?: string; + code?: string; + text?: string; + modifiers?: number; +} + +export interface InputTouchMessage { + type: 'input_touch'; + eventType: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel'; + touchPoints: Array<{ x: number; y: number; id?: number }>; + modifiers?: number; +} + +export interface StatusMessage { + type: 'status'; + connected: boolean; + screencasting: boolean; + viewportWidth?: number; + viewportHeight?: number; +} + +export interface ErrorMessage { + type: 'error'; + message: string; +} + +export type StreamMessage = + | FrameMessage + | InputMouseMessage + | InputKeyboardMessage + | InputTouchMessage + | StatusMessage + | ErrorMessage; + +/** + * WebSocket server for streaming browser viewport and receiving input + */ +export class StreamServer { + private wss: WebSocketServer | null = null; + private clients: Set = new Set(); + private browser: BrowserManager; + private port: number; + private isScreencasting: boolean = false; + + constructor(browser: BrowserManager, port: number = 9223) { + this.browser = browser; + this.port = port; + } + + /** + * Start the WebSocket server + */ + start(): Promise { + return new Promise((resolve, reject) => { + try { + this.wss = new WebSocketServer({ port: this.port }); + + this.wss.on('connection', (ws) => { + this.handleConnection(ws); + }); + + this.wss.on('error', (error) => { + console.error('[StreamServer] WebSocket error:', error); + reject(error); + }); + + this.wss.on('listening', () => { + console.log(`[StreamServer] Listening on port ${this.port}`); + + // Set up the screencast frame callback + setScreencastFrameCallback((frame) => { + this.broadcastFrame(frame); + }); + + resolve(); + }); + } catch (error) { + reject(error); + } + }); + } + + /** + * Stop the WebSocket server + */ + async stop(): Promise { + // Stop screencasting + if (this.isScreencasting) { + await this.stopScreencast(); + } + + // Clear the callback + setScreencastFrameCallback(null); + + // Close all clients + for (const client of this.clients) { + client.close(); + } + this.clients.clear(); + + // Close the server + if (this.wss) { + return new Promise((resolve) => { + this.wss!.close(() => { + this.wss = null; + resolve(); + }); + }); + } + } + + /** + * Handle a new WebSocket connection + */ + private handleConnection(ws: WebSocket): void { + console.log('[StreamServer] Client connected'); + this.clients.add(ws); + + // Send initial status + this.sendStatus(ws); + + // Start screencasting if this is the first client + if (this.clients.size === 1 && !this.isScreencasting) { + this.startScreencast().catch((error) => { + console.error('[StreamServer] Failed to start screencast:', error); + this.sendError(ws, error.message); + }); + } + + // Handle messages from client + ws.on('message', (data) => { + try { + const message = JSON.parse(data.toString()) as StreamMessage; + this.handleMessage(message, ws); + } catch (error) { + console.error('[StreamServer] Failed to parse message:', error); + } + }); + + // Handle client disconnect + ws.on('close', () => { + console.log('[StreamServer] Client disconnected'); + this.clients.delete(ws); + + // Stop screencasting if no more clients + if (this.clients.size === 0 && this.isScreencasting) { + this.stopScreencast().catch((error) => { + console.error('[StreamServer] Failed to stop screencast:', error); + }); + } + }); + + ws.on('error', (error) => { + console.error('[StreamServer] Client error:', error); + this.clients.delete(ws); + }); + } + + /** + * Handle incoming messages from clients + */ + private async handleMessage(message: StreamMessage, ws: WebSocket): Promise { + try { + switch (message.type) { + case 'input_mouse': + await this.browser.injectMouseEvent({ + type: message.eventType, + x: message.x, + y: message.y, + button: message.button, + clickCount: message.clickCount, + deltaX: message.deltaX, + deltaY: message.deltaY, + modifiers: message.modifiers, + }); + break; + + case 'input_keyboard': + await this.browser.injectKeyboardEvent({ + type: message.eventType, + key: message.key, + code: message.code, + text: message.text, + modifiers: message.modifiers, + }); + break; + + case 'input_touch': + await this.browser.injectTouchEvent({ + type: message.eventType, + touchPoints: message.touchPoints, + modifiers: message.modifiers, + }); + break; + + case 'status': + // Client is requesting status + this.sendStatus(ws); + break; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.sendError(ws, errorMessage); + } + } + + /** + * Broadcast a frame to all connected clients + */ + private broadcastFrame(frame: ScreencastFrame): void { + const message: FrameMessage = { + type: 'frame', + data: frame.data, + metadata: frame.metadata, + }; + + const payload = JSON.stringify(message); + + for (const client of this.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(payload); + } + } + } + + /** + * Send status to a client + */ + private sendStatus(ws: WebSocket): void { + let viewportWidth: number | undefined; + let viewportHeight: number | undefined; + + try { + const page = this.browser.getPage(); + const viewport = page.viewportSize(); + viewportWidth = viewport?.width; + viewportHeight = viewport?.height; + } catch { + // Browser not launched yet + } + + const message: StatusMessage = { + type: 'status', + connected: true, + screencasting: this.isScreencasting, + viewportWidth, + viewportHeight, + }; + + if (ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify(message)); + } + } + + /** + * Send an error to a client + */ + private sendError(ws: WebSocket, errorMessage: string): void { + const message: ErrorMessage = { + type: 'error', + message: errorMessage, + }; + + if (ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify(message)); + } + } + + /** + * Start screencasting + */ + private async startScreencast(): Promise { + if (this.isScreencasting) return; + + // Check if browser is launched + if (!this.browser.isLaunched()) { + throw new Error('Browser not launched'); + } + + await this.browser.startScreencast((frame) => this.broadcastFrame(frame), { + format: 'jpeg', + quality: 80, + maxWidth: 1280, + maxHeight: 720, + everyNthFrame: 1, + }); + + this.isScreencasting = true; + + // Notify all clients + for (const client of this.clients) { + this.sendStatus(client); + } + } + + /** + * Stop screencasting + */ + private async stopScreencast(): Promise { + if (!this.isScreencasting) return; + + await this.browser.stopScreencast(); + this.isScreencasting = false; + + // Notify all clients + for (const client of this.clients) { + this.sendStatus(client); + } + } + + /** + * Get the port the server is running on + */ + getPort(): number { + return this.port; + } + + /** + * Get the number of connected clients + */ + getClientCount(): number { + return this.clients.size; + } +} diff --git a/src/types.ts b/src/types.ts index b46f728..4615e67 100644 --- a/src/types.ts +++ b/src/types.ts @@ -458,6 +458,49 @@ export interface ResponseBodyCommand extends BaseCommand { timeout?: number; } +// Screencast commands for streaming browser viewport +export interface ScreencastStartCommand extends BaseCommand { + action: 'screencast_start'; + format?: 'jpeg' | 'png'; + quality?: number; // 0-100, jpeg only + maxWidth?: number; + maxHeight?: number; + everyNthFrame?: number; +} + +export interface ScreencastStopCommand extends BaseCommand { + action: 'screencast_stop'; +} + +// Input injection commands for pair browsing +export interface InputMouseCommand extends BaseCommand { + action: 'input_mouse'; + type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel'; + x: number; + y: number; + button?: 'left' | 'right' | 'middle' | 'none'; + clickCount?: number; + deltaX?: number; + deltaY?: number; + modifiers?: number; +} + +export interface InputKeyboardCommand extends BaseCommand { + action: 'input_keyboard'; + type: 'keyDown' | 'keyUp' | 'char'; + key?: string; + code?: string; + text?: string; + modifiers?: number; +} + +export interface InputTouchCommand extends BaseCommand { + action: 'input_touch'; + type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel'; + touchPoints: Array<{ x: number; y: number; id?: number }>; + modifiers?: number; +} + // Video recording export interface VideoStartCommand extends BaseCommand { action: 'video_start'; @@ -841,7 +884,12 @@ export type Command = | InsertTextCommand | MultiSelectCommand | WaitForDownloadCommand - | ResponseBodyCommand; + | ResponseBodyCommand + | ScreencastStartCommand + | ScreencastStopCommand + | InputMouseCommand + | InputKeyboardCommand + | InputTouchCommand; // Response types export interface SuccessResponse { @@ -909,6 +957,20 @@ export interface TabCloseData { remaining: number; } +export interface ScreencastStartData { + started: boolean; + format: string; + quality: number; +} + +export interface ScreencastStopData { + stopped: boolean; +} + +export interface InputEventData { + injected: boolean; +} + // Browser state export interface BrowserState { browser: Browser | null;