From 2dc093cd62bc52577ad627a2c53e1d77a0e5f2d8 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 13 Jan 2026 14:53:27 -0600 Subject: [PATCH] add screencast (#67) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs * updates * Fix: The handleCopy function fails to handle errors from navigator.clipboard.writeText(), causing unhandled exceptions and misleading UI feedback when clipboard operations fail. Co-authored-by: ctate * Fix: The benchmark file uses emojis (📊, 🚀, 🔨, 📈, 📋, ✅, ⏱️, ⚠) in console output, violating repository guidelines that forbid emojis in code and output. Co-authored-by: ctate * Remove benchmark/run.ts from PR * screencast * update docs * address comments --------- Co-authored-by: Vercel --- README.md | 107 +++++++++ cli/src/output.rs | 5 + docs/src/app/streaming/page.tsx | 217 ++++++++++++++++++ docs/src/components/sidebar.tsx | 1 + package.json | 2 + pnpm-lock.yaml | 27 +++ src/actions.ts | 111 ++++++++- src/browser.test.ts | 252 +++++++++++++++++++++ src/browser.ts | 250 ++++++++++++++++++++- src/daemon.ts | 52 ++++- src/protocol.test.ts | 385 ++++++++++++++++++++++++++++++++ src/protocol.ts | 54 +++++ src/stream-server.ts | 364 ++++++++++++++++++++++++++++++ src/types.ts | 64 +++++- 14 files changed, 1885 insertions(+), 6 deletions(-) create mode 100644 docs/src/app/streaming/page.tsx create mode 100644 src/stream-server.ts diff --git a/README.md b/README.md index bed0b09..8c742b0 100644 --- a/README.md +++ b/README.md @@ -479,6 +479,113 @@ This enables control of: - WebView2 applications - Any browser exposing a CDP endpoint +## Streaming (Browser Preview) + +Stream the browser viewport via WebSocket for live preview or "pair browsing" where a human can watch and interact alongside an AI agent. + +### Enable Streaming + +Set the `AGENT_BROWSER_STREAM_PORT` environment variable: + +```bash +AGENT_BROWSER_STREAM_PORT=9223 agent-browser open example.com +``` + +This starts a WebSocket server on the specified port that streams the browser viewport and accepts input events. + +### WebSocket Protocol + +Connect to `ws://localhost:9223` to receive frames and send input: + +**Receive frames:** +```json +{ + "type": "frame", + "data": "", + "metadata": { + "deviceWidth": 1280, + "deviceHeight": 720, + "pageScaleFactor": 1, + "offsetTop": 0, + "scrollOffsetX": 0, + "scrollOffsetY": 0 + } +} +``` + +**Send mouse events:** +```json +{ + "type": "input_mouse", + "eventType": "mousePressed", + "x": 100, + "y": 200, + "button": "left", + "clickCount": 1 +} +``` + +**Send keyboard events:** +```json +{ + "type": "input_keyboard", + "eventType": "keyDown", + "key": "Enter", + "code": "Enter" +} +``` + +**Send touch events:** +```json +{ + "type": "input_touch", + "eventType": "touchStart", + "touchPoints": [{ "x": 100, "y": 200 }] +} +``` + +### Programmatic API + +For advanced use, control streaming directly via the protocol: + +```typescript +import { BrowserManager } from 'agent-browser'; + +const browser = new BrowserManager(); +await browser.launch({ headless: true }); +await browser.navigate('https://example.com'); + +// Start screencast +await browser.startScreencast((frame) => { + // frame.data is base64-encoded image + // frame.metadata contains viewport info + console.log('Frame received:', frame.metadata.deviceWidth, 'x', frame.metadata.deviceHeight); +}, { + format: 'jpeg', + quality: 80, + maxWidth: 1280, + maxHeight: 720, +}); + +// Inject mouse events +await browser.injectMouseEvent({ + type: 'mousePressed', + x: 100, + y: 200, + button: 'left', +}); + +// Inject keyboard events +await browser.injectKeyboardEvent({ + type: 'keyDown', + key: 'Enter', + code: 'Enter', +}); + +// Stop when done +await browser.stopScreencast(); +``` + ## Architecture agent-browser uses a client-daemon architecture: diff --git a/cli/src/output.rs b/cli/src/output.rs index 3ed8751..c0939c8 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -1198,6 +1198,11 @@ Options: --cdp Connect via CDP (Chrome DevTools Protocol) --debug Debug output +Environment: + AGENT_BROWSER_SESSION Session name (default: "default") + AGENT_BROWSER_EXECUTABLE_PATH Custom browser executable path + AGENT_BROWSER_STREAM_PORT Enable WebSocket streaming on port (e.g., 9223) + Examples: agent-browser open example.com agent-browser snapshot -i # Interactive elements only diff --git a/docs/src/app/streaming/page.tsx b/docs/src/app/streaming/page.tsx new file mode 100644 index 0000000..aa93037 --- /dev/null +++ b/docs/src/app/streaming/page.tsx @@ -0,0 +1,217 @@ +import { CodeBlock } from "@/components/code-block"; + +export default function Streaming() { + return ( +
+
+

Streaming

+

+ Stream the browser viewport via WebSocket for live preview or "pair browsing" + where a human can watch and interact alongside an AI agent. +

+ +

Enable streaming

+

+ Set the AGENT_BROWSER_STREAM_PORT environment variable to start + a WebSocket server: +

+ + +

+ The server streams viewport frames and accepts input events (mouse, keyboard, touch). +

+ +

WebSocket protocol

+

Connect to ws://localhost:9223 to receive frames and send input.

+ +

Frame messages

+

The server sends frame messages with base64-encoded images:

+ ", + "metadata": { + "deviceWidth": 1280, + "deviceHeight": 720, + "pageScaleFactor": 1, + "offsetTop": 0, + "scrollOffsetX": 0, + "scrollOffsetY": 0 + } +}`} /> + +

Status messages

+

Connection and screencast status:

+ + +

Input injection

+

Send input events to control the browser remotely.

+ +

Mouse events

+ + +

Keyboard events

+ + +

Touch events

+ + +

Programmatic API

+

For advanced use, control streaming directly via the TypeScript API:

+ { + console.log('Frame:', frame.metadata.deviceWidth, 'x', frame.metadata.deviceHeight); + // frame.data is base64-encoded image +}, { + format: 'jpeg', // or 'png' + quality: 80, // 0-100, jpeg only + maxWidth: 1280, + maxHeight: 720, + everyNthFrame: 1 +}); + +// Inject mouse event +await browser.injectMouseEvent({ + type: 'mousePressed', + x: 100, + y: 200, + button: 'left', + clickCount: 1 +}); + +// Inject keyboard event +await browser.injectKeyboardEvent({ + type: 'keyDown', + key: 'Enter', + code: 'Enter' +}); + +// Inject touch event +await browser.injectTouchEvent({ + type: 'touchStart', + touchPoints: [{ x: 100, y: 200 }] +}); + +// Check if screencasting +console.log('Active:', browser.isScreencasting()); + +// Stop screencast +await browser.stopScreencast();`} /> + +

Use cases

+
    +
  • Pair browsing - Human watches and assists AI agent in real-time
    • +
  • Remote preview - View browser output in a separate UI
    • +
  • Recording - Capture frames for video generation
    • +
  • Mobile testing - Inject touch events for mobile emulation
    • +
  • Accessibility testing - Manual interaction during automated tests
    • +
+
+
+ ); +} diff --git a/docs/src/components/sidebar.tsx b/docs/src/components/sidebar.tsx index 1d40f51..55a8585 100644 --- a/docs/src/components/sidebar.tsx +++ b/docs/src/components/sidebar.tsx @@ -12,6 +12,7 @@ const navigation = [ { name: "Selectors", href: "/selectors" }, { name: "Sessions", href: "/sessions" }, { name: "Snapshots", href: "/snapshots" }, + { name: "Streaming", href: "/streaming" }, { name: "Agent Mode", href: "/agent-mode" }, { name: "CDP Mode", href: "/cdp-mode" }, ]; diff --git a/package.json b/package.json index 0453749..545f39d 100644 --- a/package.json +++ b/package.json @@ -53,10 +53,12 @@ "homepage": "https://github.com/vercel-labs/agent-browser#readme", "dependencies": { "playwright-core": "^1.57.0", + "ws": "^8.19.0", "zod": "^3.22.4" }, "devDependencies": { "@types/node": "^20.10.0", + "@types/ws": "^8.18.1", "husky": "^9.1.7", "lint-staged": "^15.2.11", "playwright": "^1.57.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 434e3ff..77dc072 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: playwright-core: specifier: ^1.57.0 version: 1.57.0 + ws: + specifier: ^8.19.0 + version: 8.19.0 zod: specifier: ^3.22.4 version: 3.25.76 @@ -18,6 +21,9 @@ importers: '@types/node': specifier: ^20.10.0 version: 20.19.28 + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 husky: specifier: ^9.1.7 version: 9.1.7 @@ -341,6 +347,9 @@ packages: '@types/node@20.19.28': resolution: {integrity: sha512-VyKBr25BuFDzBFCK5sUM6ZXiWfqgCTwTAOK8qzGV/m9FCirXYDlmczJ+d5dXBAQALGCdRRdbteKYfJ84NGEusw==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@vitest/expect@4.0.16': resolution: {integrity: sha512-eshqULT2It7McaJkQGLkPjPjNph+uevROGuIMJdG3V+0BSR2w9u6J9Lwu+E8cK5TETlfou8GRijhafIMhXsimA==} @@ -805,6 +814,18 @@ packages: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} + ws@8.19.0: + resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + yaml@2.8.2: resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} engines: {node: '>= 14.6'} @@ -985,6 +1006,10 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/ws@8.18.1': + dependencies: + '@types/node': 20.19.28 + '@vitest/expect@4.0.16': dependencies: '@standard-schema/spec': 1.1.0 @@ -1427,6 +1452,8 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.1.2 + ws@8.19.0: {} + yaml@2.8.2: {} zod@3.25.76: {} diff --git a/src/actions.ts b/src/actions.ts index 6c168d9..e9af532 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -1,5 +1,5 @@ import type { Page, Frame } from 'playwright-core'; -import type { BrowserManager } from './browser.js'; +import type { BrowserManager, ScreencastFrame } from './browser.js'; import type { Command, Response, @@ -94,6 +94,11 @@ import type { MultiSelectCommand, WaitForDownloadCommand, ResponseBodyCommand, + ScreencastStartCommand, + ScreencastStopCommand, + InputMouseCommand, + InputKeyboardCommand, + InputTouchCommand, NavigateData, ScreenshotData, EvaluateData, @@ -102,9 +107,25 @@ import type { TabNewData, TabSwitchData, TabCloseData, + ScreencastStartData, + ScreencastStopData, + InputEventData, } from './types.js'; import { successResponse, errorResponse } from './protocol.js'; +// Callback for screencast frames - will be set by the daemon when streaming is active +let screencastFrameCallback: ((frame: ScreencastFrame) => void) | null = null; + +/** + * Set the callback for screencast frames + * This is called by the daemon to set up frame streaming + */ +export function setScreencastFrameCallback( + callback: ((frame: ScreencastFrame) => void) | null +): void { + screencastFrameCallback = callback; +} + // Snapshot response type interface SnapshotData { snapshot: string; @@ -386,6 +407,16 @@ export async function executeCommand(command: Command, browser: BrowserManager): return await handleWaitForDownload(command, browser); case 'responsebody': return await handleResponseBody(command, browser); + case 'screencast_start': + return await handleScreencastStart(command, browser); + case 'screencast_stop': + return await handleScreencastStop(command, browser); + case 'input_mouse': + return await handleInputMouse(command, browser); + case 'input_keyboard': + return await handleInputKeyboard(command, browser); + case 'input_touch': + return await handleInputTouch(command, browser); default: { // TypeScript narrows to never here, but we handle it for safety const unknownCommand = command as { id: string; action: string }; @@ -678,7 +709,7 @@ async function handleTabSwitch( command: TabSwitchCommand, browser: BrowserManager ): Promise> { - const result = browser.switchTo(command.index); + const result = await browser.switchTo(command.index); const page = browser.getPage(); return successResponse(command.id, { ...result, @@ -1769,3 +1800,79 @@ async function handleResponseBody( body: parsed, }); } + +// Screencast and input injection handlers + +async function handleScreencastStart( + command: ScreencastStartCommand, + browser: BrowserManager +): Promise> { + if (!screencastFrameCallback) { + throw new Error('Screencast frame callback not set. Start the streaming server first.'); + } + + await browser.startScreencast(screencastFrameCallback, { + format: command.format, + quality: command.quality, + maxWidth: command.maxWidth, + maxHeight: command.maxHeight, + everyNthFrame: command.everyNthFrame, + }); + + return successResponse(command.id, { + started: true, + format: command.format ?? 'jpeg', + quality: command.quality ?? 80, + }); +} + +async function handleScreencastStop( + command: ScreencastStopCommand, + browser: BrowserManager +): Promise> { + await browser.stopScreencast(); + return successResponse(command.id, { stopped: true }); +} + +async function handleInputMouse( + command: InputMouseCommand, + browser: BrowserManager +): Promise> { + await browser.injectMouseEvent({ + type: command.type, + x: command.x, + y: command.y, + button: command.button, + clickCount: command.clickCount, + deltaX: command.deltaX, + deltaY: command.deltaY, + modifiers: command.modifiers, + }); + return successResponse(command.id, { injected: true }); +} + +async function handleInputKeyboard( + command: InputKeyboardCommand, + browser: BrowserManager +): Promise> { + await browser.injectKeyboardEvent({ + type: command.type, + key: command.key, + code: command.code, + text: command.text, + modifiers: command.modifiers, + }); + return successResponse(command.id, { injected: true }); +} + +async function handleInputTouch( + command: InputTouchCommand, + browser: BrowserManager +): Promise> { + await browser.injectTouchEvent({ + type: command.type, + touchPoints: command.touchPoints, + modifiers: command.modifiers, + }); + return successResponse(command.id, { injected: true }); +} diff --git a/src/browser.test.ts b/src/browser.test.ts index ec61581..80e0737 100644 --- a/src/browser.test.ts +++ b/src/browser.test.ts @@ -378,4 +378,256 @@ describe('BrowserManager', () => { await expect(browser.clearScopedHeaders('https://never-set.com')).resolves.not.toThrow(); }); }); + + describe('CDP session', () => { + it('should create CDP session on demand', async () => { + const cdp = await browser.getCDPSession(); + expect(cdp).toBeDefined(); + }); + + it('should reuse existing CDP session', async () => { + const cdp1 = await browser.getCDPSession(); + const cdp2 = await browser.getCDPSession(); + expect(cdp1).toBe(cdp2); + }); + }); + + describe('screencast', () => { + it('should report screencasting state correctly', () => { + expect(browser.isScreencasting()).toBe(false); + }); + + it('should start screencast', async () => { + const frames: Array<{ data: string }> = []; + await browser.startScreencast((frame) => { + frames.push(frame); + }); + expect(browser.isScreencasting()).toBe(true); + + // Wait a bit for at least one frame + await new Promise((resolve) => setTimeout(resolve, 200)); + + await browser.stopScreencast(); + expect(browser.isScreencasting()).toBe(false); + expect(frames.length).toBeGreaterThan(0); + }); + + it('should start screencast with custom options', async () => { + const frames: Array<{ data: string }> = []; + await browser.startScreencast( + (frame) => { + frames.push(frame); + }, + { + format: 'png', + quality: 100, + maxWidth: 800, + maxHeight: 600, + everyNthFrame: 1, + } + ); + expect(browser.isScreencasting()).toBe(true); + + // Wait for a frame + await new Promise((resolve) => setTimeout(resolve, 200)); + + await browser.stopScreencast(); + expect(frames.length).toBeGreaterThan(0); + }); + + it('should throw when starting screencast twice', async () => { + await browser.startScreencast(() => {}); + await expect(browser.startScreencast(() => {})).rejects.toThrow('Screencast already active'); + await browser.stopScreencast(); + }); + + it('should handle stop when not screencasting', async () => { + // Should not throw + await expect(browser.stopScreencast()).resolves.not.toThrow(); + }); + }); + + describe('tab switch invalidates CDP session', () => { + // Clean up any extra tabs before each test + beforeEach(async () => { + // Close all tabs except the first one + const tabs = await browser.listTabs(); + for (let i = tabs.length - 1; i > 0; i--) { + await browser.closeTab(i); + } + // Ensure we're on tab 0 + await browser.switchTo(0); + // Stop any active screencast + if (browser.isScreencasting()) { + await browser.stopScreencast(); + } + }); + + it('should not invalidate CDP when switching to same tab', async () => { + // Get CDP session for current tab + const cdp1 = await browser.getCDPSession(); + + // Switch to same tab - should NOT invalidate + await browser.switchTo(0); + + // Should be the same session + const cdp2 = await browser.getCDPSession(); + expect(cdp2).toBe(cdp1); + }); + + it('should invalidate CDP session on tab switch', async () => { + // Get CDP session for tab 0 + const cdp1 = await browser.getCDPSession(); + expect(cdp1).toBeDefined(); + + // Create new tab - this switches to the new tab automatically + await browser.newTab(); + + // Get CDP session - should be different since we're on a new page + const cdp2 = await browser.getCDPSession(); + expect(cdp2).toBeDefined(); + + // Sessions should be different objects (different pages have different CDP sessions) + expect(cdp2).not.toBe(cdp1); + }); + + it('should stop screencast on tab switch', async () => { + // Start screencast on tab 0 + await browser.startScreencast(() => {}); + expect(browser.isScreencasting()).toBe(true); + + // Create new tab and switch + await browser.newTab(); + await browser.switchTo(1); + + // Screencast should be stopped (it's page-specific) + expect(browser.isScreencasting()).toBe(false); + }); + }); + + describe('input injection', () => { + it('should inject mouse move event', async () => { + await expect( + browser.injectMouseEvent({ + type: 'mouseMoved', + x: 100, + y: 100, + }) + ).resolves.not.toThrow(); + }); + + it('should inject mouse click events', async () => { + await expect( + browser.injectMouseEvent({ + type: 'mousePressed', + x: 100, + y: 100, + button: 'left', + clickCount: 1, + }) + ).resolves.not.toThrow(); + + await expect( + browser.injectMouseEvent({ + type: 'mouseReleased', + x: 100, + y: 100, + button: 'left', + }) + ).resolves.not.toThrow(); + }); + + it('should inject mouse wheel event', async () => { + await expect( + browser.injectMouseEvent({ + type: 'mouseWheel', + x: 100, + y: 100, + deltaX: 0, + deltaY: 100, + }) + ).resolves.not.toThrow(); + }); + + it('should inject keyboard events', async () => { + await expect( + browser.injectKeyboardEvent({ + type: 'keyDown', + key: 'a', + code: 'KeyA', + }) + ).resolves.not.toThrow(); + + await expect( + browser.injectKeyboardEvent({ + type: 'keyUp', + key: 'a', + code: 'KeyA', + }) + ).resolves.not.toThrow(); + }); + + it('should inject char event', async () => { + // CDP char events only accept single characters + await expect( + browser.injectKeyboardEvent({ + type: 'char', + text: 'h', + }) + ).resolves.not.toThrow(); + }); + + it('should inject keyboard with modifiers', async () => { + await expect( + browser.injectKeyboardEvent({ + type: 'keyDown', + key: 'c', + code: 'KeyC', + modifiers: 2, // Ctrl + }) + ).resolves.not.toThrow(); + }); + + it('should inject touch events', async () => { + await expect( + browser.injectTouchEvent({ + type: 'touchStart', + touchPoints: [{ x: 100, y: 100 }], + }) + ).resolves.not.toThrow(); + + await expect( + browser.injectTouchEvent({ + type: 'touchMove', + touchPoints: [{ x: 150, y: 150 }], + }) + ).resolves.not.toThrow(); + + await expect( + browser.injectTouchEvent({ + type: 'touchEnd', + touchPoints: [], + }) + ).resolves.not.toThrow(); + }); + + it('should inject multi-touch events', async () => { + await expect( + browser.injectTouchEvent({ + type: 'touchStart', + touchPoints: [ + { x: 100, y: 100, id: 0 }, + { x: 200, y: 200, id: 1 }, + ], + }) + ).resolves.not.toThrow(); + + await expect( + browser.injectTouchEvent({ + type: 'touchEnd', + touchPoints: [], + }) + ).resolves.not.toThrow(); + }); + }); }); diff --git a/src/browser.ts b/src/browser.ts index f649bd8..c6c27f9 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -11,12 +11,37 @@ import { type Request, type Route, type Locator, + type CDPSession, } from 'playwright-core'; import path from 'node:path'; import os from 'node:os'; import type { LaunchCommand } from './types.js'; import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js'; +// Screencast frame data from CDP +export interface ScreencastFrame { + data: string; // base64 encoded image + metadata: { + offsetTop: number; + pageScaleFactor: number; + deviceWidth: number; + deviceHeight: number; + scrollOffsetX: number; + scrollOffsetY: number; + timestamp?: number; + }; + sessionId: number; +} + +// Screencast options +export interface ScreencastOptions { + format?: 'jpeg' | 'png'; + quality?: number; // 0-100, only for jpeg + maxWidth?: number; + maxHeight?: number; + everyNthFrame?: number; +} + interface TrackedRequest { url: string; method: string; @@ -57,6 +82,13 @@ export class BrowserManager { private lastSnapshot: string = ''; private scopedHeaderRoutes: Map Promise> = new Map(); + // CDP session for screencast and input injection + private cdpSession: CDPSession | null = null; + private screencastActive: boolean = false; + private screencastSessionId: number = 0; + private frameCallback: ((frame: ScreencastFrame) => void) | null = null; + private screencastFrameHandler: ((params: any) => void) | null = null; + /** * Check if browser is launched */ @@ -768,6 +800,9 @@ export class BrowserManager { throw new Error('Browser not launched'); } + // Invalidate CDP session since we're switching to a new page + await this.invalidateCDPSession(); + const context = this.contexts[0]; // Use first context for tabs const page = await context.newPage(); this.pages.push(page); @@ -806,14 +841,36 @@ export class BrowserManager { return { index: this.activePageIndex, total: this.pages.length }; } + /** + * Invalidate the current CDP session (must be called before switching pages) + * This ensures screencast and input injection work correctly after tab switch + */ + private async invalidateCDPSession(): Promise { + // Stop screencast if active (it's tied to the current page's CDP session) + if (this.screencastActive) { + await this.stopScreencast(); + } + + // Detach and clear the CDP session + if (this.cdpSession) { + await this.cdpSession.detach().catch(() => {}); + this.cdpSession = null; + } + } + /** * Switch to a specific tab/page by index */ - switchTo(index: number): { index: number; url: string; title: string } { + async switchTo(index: number): Promise<{ index: number; url: string; title: string }> { if (index < 0 || index >= this.pages.length) { throw new Error(`Invalid tab index: ${index}. Available: 0-${this.pages.length - 1}`); } + // Invalidate CDP session before switching (it's page-specific) + if (index !== this.activePageIndex) { + await this.invalidateCDPSession(); + } + this.activePageIndex = index; const page = this.pages[index]; @@ -838,6 +895,11 @@ export class BrowserManager { throw new Error('Cannot close the last tab. Use "close" to close the browser.'); } + // If closing the active tab, invalidate CDP session first + if (targetIndex === this.activePageIndex) { + await this.invalidateCDPSession(); + } + const page = this.pages[targetIndex]; await page.close(); this.pages.splice(targetIndex, 1); @@ -867,10 +929,195 @@ export class BrowserManager { return tabs; } + /** + * Get or create a CDP session for the current page + * Only works with Chromium-based browsers + */ + async getCDPSession(): Promise { + if (this.cdpSession) { + return this.cdpSession; + } + + const page = this.getPage(); + const context = page.context(); + + // Create a new CDP session attached to the page + this.cdpSession = await context.newCDPSession(page); + return this.cdpSession; + } + + /** + * Check if screencast is currently active + */ + isScreencasting(): boolean { + return this.screencastActive; + } + + /** + * Start screencast - streams viewport frames via CDP + * @param callback Function called for each frame + * @param options Screencast options + */ + async startScreencast( + callback: (frame: ScreencastFrame) => void, + options?: ScreencastOptions + ): Promise { + if (this.screencastActive) { + throw new Error('Screencast already active'); + } + + const cdp = await this.getCDPSession(); + this.frameCallback = callback; + this.screencastActive = true; + + // Create and store the frame handler so we can remove it later + this.screencastFrameHandler = async (params: any) => { + const frame: ScreencastFrame = { + data: params.data, + metadata: params.metadata, + sessionId: params.sessionId, + }; + + // Acknowledge the frame to receive the next one + await cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId }); + + // Call the callback with the frame + if (this.frameCallback) { + this.frameCallback(frame); + } + }; + + // Listen for screencast frames + cdp.on('Page.screencastFrame', this.screencastFrameHandler); + + // Start the screencast + await cdp.send('Page.startScreencast', { + format: options?.format ?? 'jpeg', + quality: options?.quality ?? 80, + maxWidth: options?.maxWidth ?? 1280, + maxHeight: options?.maxHeight ?? 720, + everyNthFrame: options?.everyNthFrame ?? 1, + }); + } + + /** + * Stop screencast + */ + async stopScreencast(): Promise { + if (!this.screencastActive) { + return; + } + + try { + const cdp = await this.getCDPSession(); + await cdp.send('Page.stopScreencast'); + + // Remove the event listener to prevent accumulation + if (this.screencastFrameHandler) { + cdp.off('Page.screencastFrame', this.screencastFrameHandler); + } + } catch { + // Ignore errors when stopping + } + + this.screencastActive = false; + this.frameCallback = null; + this.screencastFrameHandler = null; + } + + /** + * Inject a mouse event via CDP + */ + async injectMouseEvent(params: { + type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel'; + x: number; + y: number; + button?: 'left' | 'right' | 'middle' | 'none'; + clickCount?: number; + deltaX?: number; + deltaY?: number; + modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift + }): Promise { + const cdp = await this.getCDPSession(); + + const cdpButton = + params.button === 'left' + ? 'left' + : params.button === 'right' + ? 'right' + : params.button === 'middle' + ? 'middle' + : 'none'; + + await cdp.send('Input.dispatchMouseEvent', { + type: params.type, + x: params.x, + y: params.y, + button: cdpButton, + clickCount: params.clickCount ?? 1, + deltaX: params.deltaX ?? 0, + deltaY: params.deltaY ?? 0, + modifiers: params.modifiers ?? 0, + }); + } + + /** + * Inject a keyboard event via CDP + */ + async injectKeyboardEvent(params: { + type: 'keyDown' | 'keyUp' | 'char'; + key?: string; + code?: string; + text?: string; + modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift + }): Promise { + const cdp = await this.getCDPSession(); + + await cdp.send('Input.dispatchKeyEvent', { + type: params.type, + key: params.key, + code: params.code, + text: params.text, + modifiers: params.modifiers ?? 0, + }); + } + + /** + * Inject touch event via CDP (for mobile emulation) + */ + async injectTouchEvent(params: { + type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel'; + touchPoints: Array<{ x: number; y: number; id?: number }>; + modifiers?: number; + }): Promise { + const cdp = await this.getCDPSession(); + + await cdp.send('Input.dispatchTouchEvent', { + type: params.type, + touchPoints: params.touchPoints.map((tp, i) => ({ + x: tp.x, + y: tp.y, + id: tp.id ?? i, + })), + modifiers: params.modifiers ?? 0, + }); + } + /** * Close the browser and clean up */ async close(): Promise { + // Stop screencast if active + if (this.screencastActive) { + await this.stopScreencast(); + } + + // Clean up CDP session + if (this.cdpSession) { + await this.cdpSession.detach().catch(() => {}); + this.cdpSession = null; + } + // CDP: only disconnect, don't close external app's pages if (this.cdpPort !== null) { if (this.browser) { @@ -898,5 +1145,6 @@ export class BrowserManager { this.activePageIndex = 0; this.refMap = {}; this.lastSnapshot = ''; + this.frameCallback = null; } } diff --git a/src/daemon.ts b/src/daemon.ts index 81cad2b..2d3a19a 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -5,6 +5,7 @@ import * as os from 'os'; import { BrowserManager } from './browser.js'; import { parseCommand, serializeResponse, errorResponse } from './protocol.js'; import { executeCommand } from './actions.js'; +import { StreamServer } from './stream-server.js'; // Platform detection const isWindows = process.platform === 'win32'; @@ -12,6 +13,12 @@ const isWindows = process.platform === 'win32'; // Session support - each session gets its own socket/pid let currentSession = process.env.AGENT_BROWSER_SESSION || 'default'; +// Stream server for browser preview +let streamServer: StreamServer | null = null; + +// Default stream port (can be overridden with AGENT_BROWSER_STREAM_PORT) +const DEFAULT_STREAM_PORT = 9223; + /** * Set the current session */ @@ -105,8 +112,10 @@ export function getConnectionInfo( */ export function cleanupSocket(session?: string): void { const pidFile = getPidFile(session); + const streamPortFile = getStreamPortFile(session); try { if (fs.existsSync(pidFile)) fs.unlinkSync(pidFile); + if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile); if (isWindows) { const portFile = getPortFile(session); if (fs.existsSync(portFile)) fs.unlinkSync(portFile); @@ -120,15 +129,40 @@ export function cleanupSocket(session?: string): void { } /** - * Start the daemon server + * Get the stream port file path */ -export async function startDaemon(): Promise { +export function getStreamPortFile(session?: string): string { + const sess = session ?? currentSession; + return path.join(os.tmpdir(), `agent-browser-${sess}.stream`); +} + +/** + * Start the daemon server + * @param options.streamPort Port for WebSocket stream server (0 to disable) + */ +export async function startDaemon(options?: { streamPort?: number }): Promise { // Clean up any stale socket cleanupSocket(); const browser = new BrowserManager(); let shuttingDown = false; + // Start stream server if port is specified (or use default if env var is set) + const streamPort = + options?.streamPort ?? + (process.env.AGENT_BROWSER_STREAM_PORT + ? parseInt(process.env.AGENT_BROWSER_STREAM_PORT, 10) + : 0); + + if (streamPort > 0) { + streamServer = new StreamServer(browser, streamPort); + await streamServer.start(); + + // Write stream port to file for clients to discover + const streamPortFile = getStreamPortFile(); + fs.writeFileSync(streamPortFile, streamPort.toString()); + } + const server = net.createServer((socket) => { let buffer = ''; @@ -233,6 +267,20 @@ export async function startDaemon(): Promise { const shutdown = async () => { if (shuttingDown) return; shuttingDown = true; + + // Stop stream server if running + if (streamServer) { + await streamServer.stop(); + streamServer = null; + // Clean up stream port file + const streamPortFile = getStreamPortFile(); + try { + if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile); + } catch { + // Ignore cleanup errors + } + } + await browser.close(); server.close(); cleanupSocket(); diff --git a/src/protocol.test.ts b/src/protocol.test.ts index c7a0556..720306f 100644 --- a/src/protocol.test.ts +++ b/src/protocol.test.ts @@ -620,6 +620,391 @@ describe('parseCommand', () => { }); }); + describe('screencast', () => { + it('should parse screencast_start with defaults', () => { + const result = parseCommand(cmd({ id: '1', action: 'screencast_start' })); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.action).toBe('screencast_start'); + } + }); + + it('should parse screencast_start with all options', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'screencast_start', + format: 'png', + quality: 90, + maxWidth: 1920, + maxHeight: 1080, + everyNthFrame: 2, + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.format).toBe('png'); + expect(result.command.quality).toBe(90); + expect(result.command.maxWidth).toBe(1920); + expect(result.command.maxHeight).toBe(1080); + expect(result.command.everyNthFrame).toBe(2); + } + }); + + it('should reject screencast_start with invalid format', () => { + const result = parseCommand(cmd({ id: '1', action: 'screencast_start', format: 'gif' })); + expect(result.success).toBe(false); + }); + + it('should reject screencast_start with quality out of range', () => { + const result = parseCommand(cmd({ id: '1', action: 'screencast_start', quality: 150 })); + expect(result.success).toBe(false); + }); + + it('should reject screencast_start with negative maxWidth', () => { + const result = parseCommand(cmd({ id: '1', action: 'screencast_start', maxWidth: -100 })); + expect(result.success).toBe(false); + }); + + it('should parse screencast_stop', () => { + const result = parseCommand(cmd({ id: '1', action: 'screencast_stop' })); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.action).toBe('screencast_stop'); + } + }); + }); + + describe('input injection', () => { + describe('input_mouse', () => { + it('should parse mousePressed event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mousePressed', + x: 100, + y: 200, + button: 'left', + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.action).toBe('input_mouse'); + expect(result.command.type).toBe('mousePressed'); + expect(result.command.x).toBe(100); + expect(result.command.y).toBe(200); + expect(result.command.button).toBe('left'); + } + }); + + it('should parse mouseReleased event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mouseReleased', + x: 100, + y: 200, + }) + ); + expect(result.success).toBe(true); + }); + + it('should parse mouseMoved event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mouseMoved', + x: 150, + y: 250, + }) + ); + expect(result.success).toBe(true); + }); + + it('should parse mouseWheel event with deltas', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mouseWheel', + x: 100, + y: 200, + deltaX: 0, + deltaY: 100, + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.deltaX).toBe(0); + expect(result.command.deltaY).toBe(100); + } + }); + + it('should parse mouse event with modifiers', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mousePressed', + x: 100, + y: 200, + modifiers: 6, // Ctrl + Meta + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.modifiers).toBe(6); + } + }); + + it('should parse mouse event with clickCount', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mousePressed', + x: 100, + y: 200, + clickCount: 2, + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.clickCount).toBe(2); + } + }); + + it('should reject input_mouse with invalid type', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'invalid', + x: 100, + y: 200, + }) + ); + expect(result.success).toBe(false); + }); + + it('should reject input_mouse without x coordinate', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mousePressed', + y: 200, + }) + ); + expect(result.success).toBe(false); + }); + + it('should reject input_mouse without y coordinate', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_mouse', + type: 'mousePressed', + x: 100, + }) + ); + expect(result.success).toBe(false); + }); + }); + + describe('input_keyboard', () => { + it('should parse keyDown event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_keyboard', + type: 'keyDown', + key: 'Enter', + code: 'Enter', + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.action).toBe('input_keyboard'); + expect(result.command.type).toBe('keyDown'); + expect(result.command.key).toBe('Enter'); + expect(result.command.code).toBe('Enter'); + } + }); + + it('should parse keyUp event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_keyboard', + type: 'keyUp', + key: 'a', + }) + ); + expect(result.success).toBe(true); + }); + + it('should parse char event with text', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_keyboard', + type: 'char', + text: 'hello', + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.text).toBe('hello'); + } + }); + + it('should parse keyboard event with modifiers', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_keyboard', + type: 'keyDown', + key: 'c', + modifiers: 2, // Ctrl + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.modifiers).toBe(2); + } + }); + + it('should reject input_keyboard with invalid type', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_keyboard', + type: 'invalid', + }) + ); + expect(result.success).toBe(false); + }); + }); + + describe('input_touch', () => { + it('should parse touchStart event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'touchStart', + touchPoints: [{ x: 100, y: 200 }], + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.action).toBe('input_touch'); + expect(result.command.type).toBe('touchStart'); + expect(result.command.touchPoints).toHaveLength(1); + expect(result.command.touchPoints[0].x).toBe(100); + expect(result.command.touchPoints[0].y).toBe(200); + } + }); + + it('should parse touchEnd event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'touchEnd', + touchPoints: [], + }) + ); + expect(result.success).toBe(true); + }); + + it('should parse touchMove event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'touchMove', + touchPoints: [{ x: 150, y: 250 }], + }) + ); + expect(result.success).toBe(true); + }); + + it('should parse touchCancel event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'touchCancel', + touchPoints: [], + }) + ); + expect(result.success).toBe(true); + }); + + it('should parse multi-touch event', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'touchStart', + touchPoints: [ + { x: 100, y: 200, id: 0 }, + { x: 300, y: 400, id: 1 }, + ], + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.touchPoints).toHaveLength(2); + } + }); + + it('should parse touch event with modifiers', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'touchStart', + touchPoints: [{ x: 100, y: 200 }], + modifiers: 8, // Shift + }) + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.command.modifiers).toBe(8); + } + }); + + it('should reject input_touch with invalid type', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'invalid', + touchPoints: [], + }) + ); + expect(result.success).toBe(false); + }); + + it('should reject input_touch without touchPoints', () => { + const result = parseCommand( + cmd({ + id: '1', + action: 'input_touch', + type: 'touchStart', + }) + ); + expect(result.success).toBe(false); + }); + }); + }); + describe('invalid commands', () => { it('should reject unknown action', () => { const result = parseCommand(cmd({ id: '1', action: 'unknown' })); diff --git a/src/protocol.ts b/src/protocol.ts index bafdf53..d4a36c3 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -585,6 +585,55 @@ const responseBodySchema = baseCommandSchema.extend({ timeout: z.number().positive().optional(), }); +// Screencast schemas for streaming browser viewport +const screencastStartSchema = baseCommandSchema.extend({ + action: z.literal('screencast_start'), + format: z.enum(['jpeg', 'png']).optional(), + quality: z.number().min(0).max(100).optional(), + maxWidth: z.number().positive().optional(), + maxHeight: z.number().positive().optional(), + everyNthFrame: z.number().positive().optional(), +}); + +const screencastStopSchema = baseCommandSchema.extend({ + action: z.literal('screencast_stop'), +}); + +// Input injection schemas for pair browsing +const inputMouseSchema = baseCommandSchema.extend({ + action: z.literal('input_mouse'), + type: z.enum(['mousePressed', 'mouseReleased', 'mouseMoved', 'mouseWheel']), + x: z.number(), + y: z.number(), + button: z.enum(['left', 'right', 'middle', 'none']).optional(), + clickCount: z.number().positive().optional(), + deltaX: z.number().optional(), + deltaY: z.number().optional(), + modifiers: z.number().optional(), +}); + +const inputKeyboardSchema = baseCommandSchema.extend({ + action: z.literal('input_keyboard'), + type: z.enum(['keyDown', 'keyUp', 'char']), + key: z.string().optional(), + code: z.string().optional(), + text: z.string().optional(), + modifiers: z.number().optional(), +}); + +const inputTouchSchema = baseCommandSchema.extend({ + action: z.literal('input_touch'), + type: z.enum(['touchStart', 'touchEnd', 'touchMove', 'touchCancel']), + touchPoints: z.array( + z.object({ + x: z.number(), + y: z.number(), + id: z.number().optional(), + }) + ), + modifiers: z.number().optional(), +}); + const pressSchema = baseCommandSchema.extend({ action: z.literal('press'), key: z.string().min(1), @@ -795,6 +844,11 @@ const commandSchema = z.discriminatedUnion('action', [ multiSelectSchema, waitForDownloadSchema, responseBodySchema, + screencastStartSchema, + screencastStopSchema, + inputMouseSchema, + inputKeyboardSchema, + inputTouchSchema, ]); // Parse result type diff --git a/src/stream-server.ts b/src/stream-server.ts new file mode 100644 index 0000000..002d3cc --- /dev/null +++ b/src/stream-server.ts @@ -0,0 +1,364 @@ +import { WebSocketServer, WebSocket } from 'ws'; +import type { BrowserManager, ScreencastFrame } from './browser.js'; +import { setScreencastFrameCallback } from './actions.js'; + +// Message types for WebSocket communication +export interface FrameMessage { + type: 'frame'; + data: string; // base64 encoded image + metadata: { + offsetTop: number; + pageScaleFactor: number; + deviceWidth: number; + deviceHeight: number; + scrollOffsetX: number; + scrollOffsetY: number; + timestamp?: number; + }; +} + +export interface InputMouseMessage { + type: 'input_mouse'; + eventType: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel'; + x: number; + y: number; + button?: 'left' | 'right' | 'middle' | 'none'; + clickCount?: number; + deltaX?: number; + deltaY?: number; + modifiers?: number; +} + +export interface InputKeyboardMessage { + type: 'input_keyboard'; + eventType: 'keyDown' | 'keyUp' | 'char'; + key?: string; + code?: string; + text?: string; + modifiers?: number; +} + +export interface InputTouchMessage { + type: 'input_touch'; + eventType: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel'; + touchPoints: Array<{ x: number; y: number; id?: number }>; + modifiers?: number; +} + +export interface StatusMessage { + type: 'status'; + connected: boolean; + screencasting: boolean; + viewportWidth?: number; + viewportHeight?: number; +} + +export interface ErrorMessage { + type: 'error'; + message: string; +} + +export type StreamMessage = + | FrameMessage + | InputMouseMessage + | InputKeyboardMessage + | InputTouchMessage + | StatusMessage + | ErrorMessage; + +/** + * WebSocket server for streaming browser viewport and receiving input + */ +export class StreamServer { + private wss: WebSocketServer | null = null; + private clients: Set = new Set(); + private browser: BrowserManager; + private port: number; + private isScreencasting: boolean = false; + + constructor(browser: BrowserManager, port: number = 9223) { + this.browser = browser; + this.port = port; + } + + /** + * Start the WebSocket server + */ + start(): Promise { + return new Promise((resolve, reject) => { + try { + this.wss = new WebSocketServer({ port: this.port }); + + this.wss.on('connection', (ws) => { + this.handleConnection(ws); + }); + + this.wss.on('error', (error) => { + console.error('[StreamServer] WebSocket error:', error); + reject(error); + }); + + this.wss.on('listening', () => { + console.log(`[StreamServer] Listening on port ${this.port}`); + + // Set up the screencast frame callback + setScreencastFrameCallback((frame) => { + this.broadcastFrame(frame); + }); + + resolve(); + }); + } catch (error) { + reject(error); + } + }); + } + + /** + * Stop the WebSocket server + */ + async stop(): Promise { + // Stop screencasting + if (this.isScreencasting) { + await this.stopScreencast(); + } + + // Clear the callback + setScreencastFrameCallback(null); + + // Close all clients + for (const client of this.clients) { + client.close(); + } + this.clients.clear(); + + // Close the server + if (this.wss) { + return new Promise((resolve) => { + this.wss!.close(() => { + this.wss = null; + resolve(); + }); + }); + } + } + + /** + * Handle a new WebSocket connection + */ + private handleConnection(ws: WebSocket): void { + console.log('[StreamServer] Client connected'); + this.clients.add(ws); + + // Send initial status + this.sendStatus(ws); + + // Start screencasting if this is the first client + if (this.clients.size === 1 && !this.isScreencasting) { + this.startScreencast().catch((error) => { + console.error('[StreamServer] Failed to start screencast:', error); + this.sendError(ws, error.message); + }); + } + + // Handle messages from client + ws.on('message', (data) => { + try { + const message = JSON.parse(data.toString()) as StreamMessage; + this.handleMessage(message, ws); + } catch (error) { + console.error('[StreamServer] Failed to parse message:', error); + } + }); + + // Handle client disconnect + ws.on('close', () => { + console.log('[StreamServer] Client disconnected'); + this.clients.delete(ws); + + // Stop screencasting if no more clients + if (this.clients.size === 0 && this.isScreencasting) { + this.stopScreencast().catch((error) => { + console.error('[StreamServer] Failed to stop screencast:', error); + }); + } + }); + + ws.on('error', (error) => { + console.error('[StreamServer] Client error:', error); + this.clients.delete(ws); + }); + } + + /** + * Handle incoming messages from clients + */ + private async handleMessage(message: StreamMessage, ws: WebSocket): Promise { + try { + switch (message.type) { + case 'input_mouse': + await this.browser.injectMouseEvent({ + type: message.eventType, + x: message.x, + y: message.y, + button: message.button, + clickCount: message.clickCount, + deltaX: message.deltaX, + deltaY: message.deltaY, + modifiers: message.modifiers, + }); + break; + + case 'input_keyboard': + await this.browser.injectKeyboardEvent({ + type: message.eventType, + key: message.key, + code: message.code, + text: message.text, + modifiers: message.modifiers, + }); + break; + + case 'input_touch': + await this.browser.injectTouchEvent({ + type: message.eventType, + touchPoints: message.touchPoints, + modifiers: message.modifiers, + }); + break; + + case 'status': + // Client is requesting status + this.sendStatus(ws); + break; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.sendError(ws, errorMessage); + } + } + + /** + * Broadcast a frame to all connected clients + */ + private broadcastFrame(frame: ScreencastFrame): void { + const message: FrameMessage = { + type: 'frame', + data: frame.data, + metadata: frame.metadata, + }; + + const payload = JSON.stringify(message); + + for (const client of this.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(payload); + } + } + } + + /** + * Send status to a client + */ + private sendStatus(ws: WebSocket): void { + let viewportWidth: number | undefined; + let viewportHeight: number | undefined; + + try { + const page = this.browser.getPage(); + const viewport = page.viewportSize(); + viewportWidth = viewport?.width; + viewportHeight = viewport?.height; + } catch { + // Browser not launched yet + } + + const message: StatusMessage = { + type: 'status', + connected: true, + screencasting: this.isScreencasting, + viewportWidth, + viewportHeight, + }; + + if (ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify(message)); + } + } + + /** + * Send an error to a client + */ + private sendError(ws: WebSocket, errorMessage: string): void { + const message: ErrorMessage = { + type: 'error', + message: errorMessage, + }; + + if (ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify(message)); + } + } + + /** + * Start screencasting + */ + private async startScreencast(): Promise { + // Set flag immediately to prevent race conditions with concurrent calls + if (this.isScreencasting) return; + this.isScreencasting = true; + + try { + // Check if browser is launched + if (!this.browser.isLaunched()) { + throw new Error('Browser not launched'); + } + + await this.browser.startScreencast((frame) => this.broadcastFrame(frame), { + format: 'jpeg', + quality: 80, + maxWidth: 1280, + maxHeight: 720, + everyNthFrame: 1, + }); + + // Notify all clients + for (const client of this.clients) { + this.sendStatus(client); + } + } catch (error) { + // Reset flag on failure so caller can retry + this.isScreencasting = false; + throw error; + } + } + + /** + * Stop screencasting + */ + private async stopScreencast(): Promise { + if (!this.isScreencasting) return; + + await this.browser.stopScreencast(); + this.isScreencasting = false; + + // Notify all clients + for (const client of this.clients) { + this.sendStatus(client); + } + } + + /** + * Get the port the server is running on + */ + getPort(): number { + return this.port; + } + + /** + * Get the number of connected clients + */ + getClientCount(): number { + return this.clients.size; + } +} diff --git a/src/types.ts b/src/types.ts index 8290394..30aa155 100644 --- a/src/types.ts +++ b/src/types.ts @@ -459,6 +459,49 @@ export interface ResponseBodyCommand extends BaseCommand { timeout?: number; } +// Screencast commands for streaming browser viewport +export interface ScreencastStartCommand extends BaseCommand { + action: 'screencast_start'; + format?: 'jpeg' | 'png'; + quality?: number; // 0-100, jpeg only + maxWidth?: number; + maxHeight?: number; + everyNthFrame?: number; +} + +export interface ScreencastStopCommand extends BaseCommand { + action: 'screencast_stop'; +} + +// Input injection commands for pair browsing +export interface InputMouseCommand extends BaseCommand { + action: 'input_mouse'; + type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel'; + x: number; + y: number; + button?: 'left' | 'right' | 'middle' | 'none'; + clickCount?: number; + deltaX?: number; + deltaY?: number; + modifiers?: number; +} + +export interface InputKeyboardCommand extends BaseCommand { + action: 'input_keyboard'; + type: 'keyDown' | 'keyUp' | 'char'; + key?: string; + code?: string; + text?: string; + modifiers?: number; +} + +export interface InputTouchCommand extends BaseCommand { + action: 'input_touch'; + type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel'; + touchPoints: Array<{ x: number; y: number; id?: number }>; + modifiers?: number; +} + // Video recording export interface VideoStartCommand extends BaseCommand { action: 'video_start'; @@ -842,7 +885,12 @@ export type Command = | InsertTextCommand | MultiSelectCommand | WaitForDownloadCommand - | ResponseBodyCommand; + | ResponseBodyCommand + | ScreencastStartCommand + | ScreencastStopCommand + | InputMouseCommand + | InputKeyboardCommand + | InputTouchCommand; // Response types export interface SuccessResponse { @@ -910,6 +958,20 @@ export interface TabCloseData { remaining: number; } +export interface ScreencastStartData { + started: boolean; + format: string; + quality: number; +} + +export interface ScreencastStopData { + stopped: boolean; +} + +export interface InputEventData { + injected: boolean; +} + // Browser state export interface BrowserState { browser: Browser | null;