address comments

update docs
Merge remote-tracking branch 'origin/main' into ctate/screencast
2026-01-13 14:43:10 -06:00 · 2026-01-13 12:50:19 -06:00 · 2026-01-13 12:34:51 -06:00 · 2026-01-13 12:34:00 -06:00 · 2026-01-13 02:54:56 -06:00 · 2026-01-13 02:41:39 -06:00
15 changed files with 1889 additions and 9 deletions
@@ -18,9 +18,9 @@ git clone https://github.com/vercel-labs/agent-browser
 cd agent-browser
 pnpm install
 pnpm build
-pnpm build:native
-./bin/agent-browser install
-pnpm link --global
+pnpm build:native   # Requires Rust (https://rustup.rs)
+pnpm link --global  # Makes agent-browser available globally
+agent-browser install
 ```

 ### Linux Dependencies
@@ -479,6 +479,113 @@ This enables control of:
 - WebView2 applications
 - Any browser exposing a CDP endpoint

+## Streaming (Browser Preview)
+
+Stream the browser viewport via WebSocket for live preview or "pair browsing" where a human can watch and interact alongside an AI agent.
+
+### Enable Streaming
+
+Set the `AGENT_BROWSER_STREAM_PORT` environment variable:
+
+```bash
+AGENT_BROWSER_STREAM_PORT=9223 agent-browser open example.com
+```
+
+This starts a WebSocket server on the specified port that streams the browser viewport and accepts input events.
+
+### WebSocket Protocol
+
+Connect to `ws://localhost:9223` to receive frames and send input:
+
+**Receive frames:**
+```json
+{
+  "type": "frame",
+  "data": "<base64-encoded-jpeg>",
+  "metadata": {
+    "deviceWidth": 1280,
+    "deviceHeight": 720,
+    "pageScaleFactor": 1,
+    "offsetTop": 0,
+    "scrollOffsetX": 0,
+    "scrollOffsetY": 0
+  }
+}
+```
+
+**Send mouse events:**
+```json
+{
+  "type": "input_mouse",
+  "eventType": "mousePressed",
+  "x": 100,
+  "y": 200,
+  "button": "left",
+  "clickCount": 1
+}
+```
+
+**Send keyboard events:**
+```json
+{
+  "type": "input_keyboard",
+  "eventType": "keyDown",
+  "key": "Enter",
+  "code": "Enter"
+}
+```
+
+**Send touch events:**
+```json
+{
+  "type": "input_touch",
+  "eventType": "touchStart",
+  "touchPoints": [{ "x": 100, "y": 200 }]
+}
+```
+
+### Programmatic API
+
+For advanced use, control streaming directly via the protocol:
+
+```typescript
+import { BrowserManager } from 'agent-browser';
+
+const browser = new BrowserManager();
+await browser.launch({ headless: true });
+await browser.navigate('https://example.com');
+
+// Start screencast
+await browser.startScreencast((frame) => {
+  // frame.data is base64-encoded image
+  // frame.metadata contains viewport info
+  console.log('Frame received:', frame.metadata.deviceWidth, 'x', frame.metadata.deviceHeight);
+}, {
+  format: 'jpeg',
+  quality: 80,
+  maxWidth: 1280,
+  maxHeight: 720,
+});
+
+// Inject mouse events
+await browser.injectMouseEvent({
+  type: 'mousePressed',
+  x: 100,
+  y: 200,
+  button: 'left',
+});
+
+// Inject keyboard events
+await browser.injectKeyboardEvent({
+  type: 'keyDown',
+  key: 'Enter',
+  code: 'Enter',
+});
+
+// Stop when done
+await browser.stopScreencast();
+```
+
 ## Architecture

 agent-browser uses a client-daemon architecture:
@@ -901,6 +901,7 @@ mod tests {
            debug: false,
            headers: None,
            executable_path: None,
+            cdp: None,
        }
    }

@@ -1197,6 +1197,11 @@ Options:
  --cdp <port>               Connect via CDP (Chrome DevTools Protocol)
  --debug                    Debug output

+Environment:
+  AGENT_BROWSER_SESSION          Session name (default: "default")
+  AGENT_BROWSER_EXECUTABLE_PATH  Custom browser executable path
+  AGENT_BROWSER_STREAM_PORT      Enable WebSocket streaming on port (e.g., 9223)
+
 Examples:
  agent-browser open example.com
  agent-browser snapshot -i              # Interactive elements only
@@ -0,0 +1,217 @@
+import { CodeBlock } from "@/components/code-block";
+
+export default function Streaming() {
+  return (
+    <div className="max-w-2xl mx-auto px-4 sm:px-6 py-8 sm:py-12">
+      <div className="prose">
+        <h1>Streaming</h1>
+        <p>
+          Stream the browser viewport via WebSocket for live preview or &quot;pair browsing&quot;
+          where a human can watch and interact alongside an AI agent.
+        </p>
+
+        <h2>Enable streaming</h2>
+        <p>
+          Set the <code>AGENT_BROWSER_STREAM_PORT</code> environment variable to start
+          a WebSocket server:
+        </p>
+        <CodeBlock code={`AGENT_BROWSER_STREAM_PORT=9223 agent-browser open example.com`} />
+
+        <p>
+          The server streams viewport frames and accepts input events (mouse, keyboard, touch).
+        </p>
+
+        <h2>WebSocket protocol</h2>
+        <p>Connect to <code>ws://localhost:9223</code> to receive frames and send input.</p>
+
+        <h3>Frame messages</h3>
+        <p>The server sends frame messages with base64-encoded images:</p>
+        <CodeBlock code={`{
+  "type": "frame",
+  "data": "<base64-encoded-jpeg>",
+  "metadata": {
+    "deviceWidth": 1280,
+    "deviceHeight": 720,
+    "pageScaleFactor": 1,
+    "offsetTop": 0,
+    "scrollOffsetX": 0,
+    "scrollOffsetY": 0
+  }
+}`} />
+
+        <h3>Status messages</h3>
+        <p>Connection and screencast status:</p>
+        <CodeBlock code={`{
+  "type": "status",
+  "connected": true,
+  "screencasting": true,
+  "viewportWidth": 1280,
+  "viewportHeight": 720
+}`} />
+
+        <h2>Input injection</h2>
+        <p>Send input events to control the browser remotely.</p>
+
+        <h3>Mouse events</h3>
+        <CodeBlock code={`// Click
+{
+  "type": "input_mouse",
+  "eventType": "mousePressed",
+  "x": 100,
+  "y": 200,
+  "button": "left",
+  "clickCount": 1
+}
+
+// Release
+{
+  "type": "input_mouse",
+  "eventType": "mouseReleased",
+  "x": 100,
+  "y": 200,
+  "button": "left"
+}
+
+// Move
+{
+  "type": "input_mouse",
+  "eventType": "mouseMoved",
+  "x": 150,
+  "y": 250
+}
+
+// Scroll
+{
+  "type": "input_mouse",
+  "eventType": "mouseWheel",
+  "x": 100,
+  "y": 200,
+  "deltaX": 0,
+  "deltaY": 100
+}`} />
+
+        <h3>Keyboard events</h3>
+        <CodeBlock code={`// Key down
+{
+  "type": "input_keyboard",
+  "eventType": "keyDown",
+  "key": "Enter",
+  "code": "Enter"
+}
+
+// Key up
+{
+  "type": "input_keyboard",
+  "eventType": "keyUp",
+  "key": "Enter",
+  "code": "Enter"
+}
+
+// Type character
+{
+  "type": "input_keyboard",
+  "eventType": "char",
+  "text": "a"
+}
+
+// With modifiers (1=Alt, 2=Ctrl, 4=Meta, 8=Shift)
+{
+  "type": "input_keyboard",
+  "eventType": "keyDown",
+  "key": "c",
+  "code": "KeyC",
+  "modifiers": 2
+}`} />
+
+        <h3>Touch events</h3>
+        <CodeBlock code={`// Touch start
+{
+  "type": "input_touch",
+  "eventType": "touchStart",
+  "touchPoints": [{ "x": 100, "y": 200 }]
+}
+
+// Touch move
+{
+  "type": "input_touch",
+  "eventType": "touchMove",
+  "touchPoints": [{ "x": 150, "y": 250 }]
+}
+
+// Touch end
+{
+  "type": "input_touch",
+  "eventType": "touchEnd",
+  "touchPoints": []
+}
+
+// Multi-touch (pinch zoom)
+{
+  "type": "input_touch",
+  "eventType": "touchStart",
+  "touchPoints": [
+    { "x": 100, "y": 200, "id": 0 },
+    { "x": 200, "y": 200, "id": 1 }
+  ]
+}`} />
+
+        <h2>Programmatic API</h2>
+        <p>For advanced use, control streaming directly via the TypeScript API:</p>
+        <CodeBlock code={`import { BrowserManager } from 'agent-browser';
+
+const browser = new BrowserManager();
+await browser.launch({ headless: true });
+await browser.navigate('https://example.com');
+
+// Start screencast with callback
+await browser.startScreencast((frame) => {
+  console.log('Frame:', frame.metadata.deviceWidth, 'x', frame.metadata.deviceHeight);
+  // frame.data is base64-encoded image
+}, {
+  format: 'jpeg',  // or 'png'
+  quality: 80,     // 0-100, jpeg only
+  maxWidth: 1280,
+  maxHeight: 720,
+  everyNthFrame: 1
+});
+
+// Inject mouse event
+await browser.injectMouseEvent({
+  type: 'mousePressed',
+  x: 100,
+  y: 200,
+  button: 'left',
+  clickCount: 1
+});
+
+// Inject keyboard event
+await browser.injectKeyboardEvent({
+  type: 'keyDown',
+  key: 'Enter',
+  code: 'Enter'
+});
+
+// Inject touch event
+await browser.injectTouchEvent({
+  type: 'touchStart',
+  touchPoints: [{ x: 100, y: 200 }]
+});
+
+// Check if screencasting
+console.log('Active:', browser.isScreencasting());
+
+// Stop screencast
+await browser.stopScreencast();`} />
+
+        <h2>Use cases</h2>
+        <ul>
+          <li><strong>Pair browsing</strong> - Human watches and assists AI agent in real-time</li>
+          <li><strong>Remote preview</strong> - View browser output in a separate UI</li>
+          <li><strong>Recording</strong> - Capture frames for video generation</li>
+          <li><strong>Mobile testing</strong> - Inject touch events for mobile emulation</li>
+          <li><strong>Accessibility testing</strong> - Manual interaction during automated tests</li>
+        </ul>
+      </div>
+    </div>
+  );
+}
@@ -12,6 +12,7 @@ const navigation = [
  { name: "Selectors", href: "/selectors" },
  { name: "Sessions", href: "/sessions" },
  { name: "Snapshots", href: "/snapshots" },
+  { name: "Streaming", href: "/streaming" },
  { name: "Agent Mode", href: "/agent-mode" },
  { name: "CDP Mode", href: "/cdp-mode" },
 ];
@@ -53,10 +53,12 @@
  "homepage": "https://github.com/vercel-labs/agent-browser#readme",
  "dependencies": {
    "playwright-core": "^1.57.0",
+    "ws": "^8.19.0",
    "zod": "^3.22.4"
  },
  "devDependencies": {
    "@types/node": "^20.10.0",
+    "@types/ws": "^8.18.1",
    "husky": "^9.1.7",
    "lint-staged": "^15.2.11",
    "playwright": "^1.57.0",
@@ -11,6 +11,9 @@ importers:
      playwright-core:
        specifier: ^1.57.0
        version: 1.57.0
+      ws:
+        specifier: ^8.19.0
+        version: 8.19.0
      zod:
        specifier: ^3.22.4
        version: 3.25.76
@@ -18,6 +21,9 @@ importers:
      '@types/node':
        specifier: ^20.10.0
        version: 20.19.28
+      '@types/ws':
+        specifier: ^8.18.1
+        version: 8.18.1
      husky:
        specifier: ^9.1.7
        version: 9.1.7
@@ -341,6 +347,9 @@ packages:
  '@types/node@20.19.28':
    resolution: {integrity: sha512-VyKBr25BuFDzBFCK5sUM6ZXiWfqgCTwTAOK8qzGV/m9FCirXYDlmczJ+d5dXBAQALGCdRRdbteKYfJ84NGEusw==}

+  '@types/ws@8.18.1':
+    resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
+
  '@vitest/expect@4.0.16':
    resolution: {integrity: sha512-eshqULT2It7McaJkQGLkPjPjNph+uevROGuIMJdG3V+0BSR2w9u6J9Lwu+E8cK5TETlfou8GRijhafIMhXsimA==}

@@ -805,6 +814,18 @@ packages:
    resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==}
    engines: {node: '>=18'}

+  ws@8.19.0:
+    resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==}
+    engines: {node: '>=10.0.0'}
+    peerDependencies:
+      bufferutil: ^4.0.1
+      utf-8-validate: '>=5.0.2'
+    peerDependenciesMeta:
+      bufferutil:
+        optional: true
+      utf-8-validate:
+        optional: true
+
  yaml@2.8.2:
    resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==}
    engines: {node: '>= 14.6'}
@@ -985,6 +1006,10 @@ snapshots:
    dependencies:
      undici-types: 6.21.0

+  '@types/ws@8.18.1':
+    dependencies:
+      '@types/node': 20.19.28
+
  '@vitest/expect@4.0.16':
    dependencies:
      '@standard-schema/spec': 1.1.0
@@ -1427,6 +1452,8 @@ snapshots:
      string-width: 7.2.0
      strip-ansi: 7.1.2

+  ws@8.19.0: {}
+
  yaml@2.8.2: {}

  zod@3.25.76: {}
@@ -1,5 +1,5 @@
 import type { Page, Frame } from 'playwright-core';
-import type { BrowserManager } from './browser.js';
+import type { BrowserManager, ScreencastFrame } from './browser.js';
 import type {
  Command,
  Response,
@@ -94,6 +94,11 @@ import type {
  MultiSelectCommand,
  WaitForDownloadCommand,
  ResponseBodyCommand,
+  ScreencastStartCommand,
+  ScreencastStopCommand,
+  InputMouseCommand,
+  InputKeyboardCommand,
+  InputTouchCommand,
  NavigateData,
  ScreenshotData,
  EvaluateData,
@@ -102,9 +107,25 @@ import type {
  TabNewData,
  TabSwitchData,
  TabCloseData,
+  ScreencastStartData,
+  ScreencastStopData,
+  InputEventData,
 } from './types.js';
 import { successResponse, errorResponse } from './protocol.js';

+// Callback for screencast frames - will be set by the daemon when streaming is active
+let screencastFrameCallback: ((frame: ScreencastFrame) => void) | null = null;
+
+/**
+ * Set the callback for screencast frames
+ * This is called by the daemon to set up frame streaming
+ */
+export function setScreencastFrameCallback(
+  callback: ((frame: ScreencastFrame) => void) | null
+): void {
+  screencastFrameCallback = callback;
+}
+
 // Snapshot response type
 interface SnapshotData {
  snapshot: string;
@@ -386,6 +407,16 @@ export async function executeCommand(command: Command, browser: BrowserManager):
        return await handleWaitForDownload(command, browser);
      case 'responsebody':
        return await handleResponseBody(command, browser);
+      case 'screencast_start':
+        return await handleScreencastStart(command, browser);
+      case 'screencast_stop':
+        return await handleScreencastStop(command, browser);
+      case 'input_mouse':
+        return await handleInputMouse(command, browser);
+      case 'input_keyboard':
+        return await handleInputKeyboard(command, browser);
+      case 'input_touch':
+        return await handleInputTouch(command, browser);
      default: {
        // TypeScript narrows to never here, but we handle it for safety
        const unknownCommand = command as { id: string; action: string };
@@ -678,7 +709,7 @@ async function handleTabSwitch(
  command: TabSwitchCommand,
  browser: BrowserManager
 ): Promise<Response<TabSwitchData>> {
-  const result = browser.switchTo(command.index);
+  const result = await browser.switchTo(command.index);
  const page = browser.getPage();
  return successResponse(command.id, {
    ...result,
@@ -1769,3 +1800,79 @@ async function handleResponseBody(
    body: parsed,
  });
 }
+
+// Screencast and input injection handlers
+
+async function handleScreencastStart(
+  command: ScreencastStartCommand,
+  browser: BrowserManager
+): Promise<Response<ScreencastStartData>> {
+  if (!screencastFrameCallback) {
+    throw new Error('Screencast frame callback not set. Start the streaming server first.');
+  }
+
+  await browser.startScreencast(screencastFrameCallback, {
+    format: command.format,
+    quality: command.quality,
+    maxWidth: command.maxWidth,
+    maxHeight: command.maxHeight,
+    everyNthFrame: command.everyNthFrame,
+  });
+
+  return successResponse(command.id, {
+    started: true,
+    format: command.format ?? 'jpeg',
+    quality: command.quality ?? 80,
+  });
+}
+
+async function handleScreencastStop(
+  command: ScreencastStopCommand,
+  browser: BrowserManager
+): Promise<Response<ScreencastStopData>> {
+  await browser.stopScreencast();
+  return successResponse(command.id, { stopped: true });
+}
+
+async function handleInputMouse(
+  command: InputMouseCommand,
+  browser: BrowserManager
+): Promise<Response<InputEventData>> {
+  await browser.injectMouseEvent({
+    type: command.type,
+    x: command.x,
+    y: command.y,
+    button: command.button,
+    clickCount: command.clickCount,
+    deltaX: command.deltaX,
+    deltaY: command.deltaY,
+    modifiers: command.modifiers,
+  });
+  return successResponse(command.id, { injected: true });
+}
+
+async function handleInputKeyboard(
+  command: InputKeyboardCommand,
+  browser: BrowserManager
+): Promise<Response<InputEventData>> {
+  await browser.injectKeyboardEvent({
+    type: command.type,
+    key: command.key,
+    code: command.code,
+    text: command.text,
+    modifiers: command.modifiers,
+  });
+  return successResponse(command.id, { injected: true });
+}
+
+async function handleInputTouch(
+  command: InputTouchCommand,
+  browser: BrowserManager
+): Promise<Response<InputEventData>> {
+  await browser.injectTouchEvent({
+    type: command.type,
+    touchPoints: command.touchPoints,
+    modifiers: command.modifiers,
+  });
+  return successResponse(command.id, { injected: true });
+}
@@ -378,4 +378,256 @@ describe('BrowserManager', () => {
      await expect(browser.clearScopedHeaders('https://never-set.com')).resolves.not.toThrow();
    });
  });
+
+  describe('CDP session', () => {
+    it('should create CDP session on demand', async () => {
+      const cdp = await browser.getCDPSession();
+      expect(cdp).toBeDefined();
+    });
+
+    it('should reuse existing CDP session', async () => {
+      const cdp1 = await browser.getCDPSession();
+      const cdp2 = await browser.getCDPSession();
+      expect(cdp1).toBe(cdp2);
+    });
+  });
+
+  describe('screencast', () => {
+    it('should report screencasting state correctly', () => {
+      expect(browser.isScreencasting()).toBe(false);
+    });
+
+    it('should start screencast', async () => {
+      const frames: Array<{ data: string }> = [];
+      await browser.startScreencast((frame) => {
+        frames.push(frame);
+      });
+      expect(browser.isScreencasting()).toBe(true);
+
+      // Wait a bit for at least one frame
+      await new Promise((resolve) => setTimeout(resolve, 200));
+
+      await browser.stopScreencast();
+      expect(browser.isScreencasting()).toBe(false);
+      expect(frames.length).toBeGreaterThan(0);
+    });
+
+    it('should start screencast with custom options', async () => {
+      const frames: Array<{ data: string }> = [];
+      await browser.startScreencast(
+        (frame) => {
+          frames.push(frame);
+        },
+        {
+          format: 'png',
+          quality: 100,
+          maxWidth: 800,
+          maxHeight: 600,
+          everyNthFrame: 1,
+        }
+      );
+      expect(browser.isScreencasting()).toBe(true);
+
+      // Wait for a frame
+      await new Promise((resolve) => setTimeout(resolve, 200));
+
+      await browser.stopScreencast();
+      expect(frames.length).toBeGreaterThan(0);
+    });
+
+    it('should throw when starting screencast twice', async () => {
+      await browser.startScreencast(() => {});
+      await expect(browser.startScreencast(() => {})).rejects.toThrow('Screencast already active');
+      await browser.stopScreencast();
+    });
+
+    it('should handle stop when not screencasting', async () => {
+      // Should not throw
+      await expect(browser.stopScreencast()).resolves.not.toThrow();
+    });
+  });
+
+  describe('tab switch invalidates CDP session', () => {
+    // Clean up any extra tabs before each test
+    beforeEach(async () => {
+      // Close all tabs except the first one
+      const tabs = await browser.listTabs();
+      for (let i = tabs.length - 1; i > 0; i--) {
+        await browser.closeTab(i);
+      }
+      // Ensure we're on tab 0
+      await browser.switchTo(0);
+      // Stop any active screencast
+      if (browser.isScreencasting()) {
+        await browser.stopScreencast();
+      }
+    });
+
+    it('should not invalidate CDP when switching to same tab', async () => {
+      // Get CDP session for current tab
+      const cdp1 = await browser.getCDPSession();
+
+      // Switch to same tab - should NOT invalidate
+      await browser.switchTo(0);
+
+      // Should be the same session
+      const cdp2 = await browser.getCDPSession();
+      expect(cdp2).toBe(cdp1);
+    });
+
+    it('should invalidate CDP session on tab switch', async () => {
+      // Get CDP session for tab 0
+      const cdp1 = await browser.getCDPSession();
+      expect(cdp1).toBeDefined();
+
+      // Create new tab - this switches to the new tab automatically
+      await browser.newTab();
+
+      // Get CDP session - should be different since we're on a new page
+      const cdp2 = await browser.getCDPSession();
+      expect(cdp2).toBeDefined();
+
+      // Sessions should be different objects (different pages have different CDP sessions)
+      expect(cdp2).not.toBe(cdp1);
+    });
+
+    it('should stop screencast on tab switch', async () => {
+      // Start screencast on tab 0
+      await browser.startScreencast(() => {});
+      expect(browser.isScreencasting()).toBe(true);
+
+      // Create new tab and switch
+      await browser.newTab();
+      await browser.switchTo(1);
+
+      // Screencast should be stopped (it's page-specific)
+      expect(browser.isScreencasting()).toBe(false);
+    });
+  });
+
+  describe('input injection', () => {
+    it('should inject mouse move event', async () => {
+      await expect(
+        browser.injectMouseEvent({
+          type: 'mouseMoved',
+          x: 100,
+          y: 100,
+        })
+      ).resolves.not.toThrow();
+    });
+
+    it('should inject mouse click events', async () => {
+      await expect(
+        browser.injectMouseEvent({
+          type: 'mousePressed',
+          x: 100,
+          y: 100,
+          button: 'left',
+          clickCount: 1,
+        })
+      ).resolves.not.toThrow();
+
+      await expect(
+        browser.injectMouseEvent({
+          type: 'mouseReleased',
+          x: 100,
+          y: 100,
+          button: 'left',
+        })
+      ).resolves.not.toThrow();
+    });
+
+    it('should inject mouse wheel event', async () => {
+      await expect(
+        browser.injectMouseEvent({
+          type: 'mouseWheel',
+          x: 100,
+          y: 100,
+          deltaX: 0,
+          deltaY: 100,
+        })
+      ).resolves.not.toThrow();
+    });
+
+    it('should inject keyboard events', async () => {
+      await expect(
+        browser.injectKeyboardEvent({
+          type: 'keyDown',
+          key: 'a',
+          code: 'KeyA',
+        })
+      ).resolves.not.toThrow();
+
+      await expect(
+        browser.injectKeyboardEvent({
+          type: 'keyUp',
+          key: 'a',
+          code: 'KeyA',
+        })
+      ).resolves.not.toThrow();
+    });
+
+    it('should inject char event', async () => {
+      // CDP char events only accept single characters
+      await expect(
+        browser.injectKeyboardEvent({
+          type: 'char',
+          text: 'h',
+        })
+      ).resolves.not.toThrow();
+    });
+
+    it('should inject keyboard with modifiers', async () => {
+      await expect(
+        browser.injectKeyboardEvent({
+          type: 'keyDown',
+          key: 'c',
+          code: 'KeyC',
+          modifiers: 2, // Ctrl
+        })
+      ).resolves.not.toThrow();
+    });
+
+    it('should inject touch events', async () => {
+      await expect(
+        browser.injectTouchEvent({
+          type: 'touchStart',
+          touchPoints: [{ x: 100, y: 100 }],
+        })
+      ).resolves.not.toThrow();
+
+      await expect(
+        browser.injectTouchEvent({
+          type: 'touchMove',
+          touchPoints: [{ x: 150, y: 150 }],
+        })
+      ).resolves.not.toThrow();
+
+      await expect(
+        browser.injectTouchEvent({
+          type: 'touchEnd',
+          touchPoints: [],
+        })
+      ).resolves.not.toThrow();
+    });
+
+    it('should inject multi-touch events', async () => {
+      await expect(
+        browser.injectTouchEvent({
+          type: 'touchStart',
+          touchPoints: [
+            { x: 100, y: 100, id: 0 },
+            { x: 200, y: 200, id: 1 },
+          ],
+        })
+      ).resolves.not.toThrow();
+
+      await expect(
+        browser.injectTouchEvent({
+          type: 'touchEnd',
+          touchPoints: [],
+        })
+      ).resolves.not.toThrow();
+    });
+  });
 });
@@ -11,10 +11,35 @@ import {
  type Request,
  type Route,
  type Locator,
+  type CDPSession,
 } from 'playwright-core';
 import type { LaunchCommand } from './types.js';
 import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js';

+// Screencast frame data from CDP
+export interface ScreencastFrame {
+  data: string; // base64 encoded image
+  metadata: {
+    offsetTop: number;
+    pageScaleFactor: number;
+    deviceWidth: number;
+    deviceHeight: number;
+    scrollOffsetX: number;
+    scrollOffsetY: number;
+    timestamp?: number;
+  };
+  sessionId: number;
+}
+
+// Screencast options
+export interface ScreencastOptions {
+  format?: 'jpeg' | 'png';
+  quality?: number; // 0-100, only for jpeg
+  maxWidth?: number;
+  maxHeight?: number;
+  everyNthFrame?: number;
+}
+
 interface TrackedRequest {
  url: string;
  method: string;
@@ -54,6 +79,13 @@ export class BrowserManager {
  private lastSnapshot: string = '';
  private scopedHeaderRoutes: Map<string, (route: Route) => Promise<void>> = new Map();

+  // CDP session for screencast and input injection
+  private cdpSession: CDPSession | null = null;
+  private screencastActive: boolean = false;
+  private screencastSessionId: number = 0;
+  private frameCallback: ((frame: ScreencastFrame) => void) | null = null;
+  private screencastFrameHandler: ((params: any) => void) | null = null;
+
  /**
   * Check if browser is launched
   */
@@ -751,6 +783,9 @@ export class BrowserManager {
      throw new Error('Browser not launched');
    }

+    // Invalidate CDP session since we're switching to a new page
+    await this.invalidateCDPSession();
+
    const context = this.contexts[0]; // Use first context for tabs
    const page = await context.newPage();
    this.pages.push(page);
@@ -789,14 +824,36 @@ export class BrowserManager {
    return { index: this.activePageIndex, total: this.pages.length };
  }

+  /**
+   * Invalidate the current CDP session (must be called before switching pages)
+   * This ensures screencast and input injection work correctly after tab switch
+   */
+  private async invalidateCDPSession(): Promise<void> {
+    // Stop screencast if active (it's tied to the current page's CDP session)
+    if (this.screencastActive) {
+      await this.stopScreencast();
+    }
+
+    // Detach and clear the CDP session
+    if (this.cdpSession) {
+      await this.cdpSession.detach().catch(() => {});
+      this.cdpSession = null;
+    }
+  }
+
  /**
   * Switch to a specific tab/page by index
   */
-  switchTo(index: number): { index: number; url: string; title: string } {
+  async switchTo(index: number): Promise<{ index: number; url: string; title: string }> {
    if (index < 0 || index >= this.pages.length) {
      throw new Error(`Invalid tab index: ${index}. Available: 0-${this.pages.length - 1}`);
    }

+    // Invalidate CDP session before switching (it's page-specific)
+    if (index !== this.activePageIndex) {
+      await this.invalidateCDPSession();
+    }
+
    this.activePageIndex = index;
    const page = this.pages[index];

@@ -821,6 +878,11 @@ export class BrowserManager {
      throw new Error('Cannot close the last tab. Use "close" to close the browser.');
    }

+    // If closing the active tab, invalidate CDP session first
+    if (targetIndex === this.activePageIndex) {
+      await this.invalidateCDPSession();
+    }
+
    const page = this.pages[targetIndex];
    await page.close();
    this.pages.splice(targetIndex, 1);
@@ -850,10 +912,195 @@ export class BrowserManager {
    return tabs;
  }

+  /**
+   * Get or create a CDP session for the current page
+   * Only works with Chromium-based browsers
+   */
+  async getCDPSession(): Promise<CDPSession> {
+    if (this.cdpSession) {
+      return this.cdpSession;
+    }
+
+    const page = this.getPage();
+    const context = page.context();
+
+    // Create a new CDP session attached to the page
+    this.cdpSession = await context.newCDPSession(page);
+    return this.cdpSession;
+  }
+
+  /**
+   * Check if screencast is currently active
+   */
+  isScreencasting(): boolean {
+    return this.screencastActive;
+  }
+
+  /**
+   * Start screencast - streams viewport frames via CDP
+   * @param callback Function called for each frame
+   * @param options Screencast options
+   */
+  async startScreencast(
+    callback: (frame: ScreencastFrame) => void,
+    options?: ScreencastOptions
+  ): Promise<void> {
+    if (this.screencastActive) {
+      throw new Error('Screencast already active');
+    }
+
+    const cdp = await this.getCDPSession();
+    this.frameCallback = callback;
+    this.screencastActive = true;
+
+    // Create and store the frame handler so we can remove it later
+    this.screencastFrameHandler = async (params: any) => {
+      const frame: ScreencastFrame = {
+        data: params.data,
+        metadata: params.metadata,
+        sessionId: params.sessionId,
+      };
+
+      // Acknowledge the frame to receive the next one
+      await cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId });
+
+      // Call the callback with the frame
+      if (this.frameCallback) {
+        this.frameCallback(frame);
+      }
+    };
+
+    // Listen for screencast frames
+    cdp.on('Page.screencastFrame', this.screencastFrameHandler);
+
+    // Start the screencast
+    await cdp.send('Page.startScreencast', {
+      format: options?.format ?? 'jpeg',
+      quality: options?.quality ?? 80,
+      maxWidth: options?.maxWidth ?? 1280,
+      maxHeight: options?.maxHeight ?? 720,
+      everyNthFrame: options?.everyNthFrame ?? 1,
+    });
+  }
+
+  /**
+   * Stop screencast
+   */
+  async stopScreencast(): Promise<void> {
+    if (!this.screencastActive) {
+      return;
+    }
+
+    try {
+      const cdp = await this.getCDPSession();
+      await cdp.send('Page.stopScreencast');
+
+      // Remove the event listener to prevent accumulation
+      if (this.screencastFrameHandler) {
+        cdp.off('Page.screencastFrame', this.screencastFrameHandler);
+      }
+    } catch {
+      // Ignore errors when stopping
+    }
+
+    this.screencastActive = false;
+    this.frameCallback = null;
+    this.screencastFrameHandler = null;
+  }
+
+  /**
+   * Inject a mouse event via CDP
+   */
+  async injectMouseEvent(params: {
+    type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel';
+    x: number;
+    y: number;
+    button?: 'left' | 'right' | 'middle' | 'none';
+    clickCount?: number;
+    deltaX?: number;
+    deltaY?: number;
+    modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift
+  }): Promise<void> {
+    const cdp = await this.getCDPSession();
+
+    const cdpButton =
+      params.button === 'left'
+        ? 'left'
+        : params.button === 'right'
+          ? 'right'
+          : params.button === 'middle'
+            ? 'middle'
+            : 'none';
+
+    await cdp.send('Input.dispatchMouseEvent', {
+      type: params.type,
+      x: params.x,
+      y: params.y,
+      button: cdpButton,
+      clickCount: params.clickCount ?? 1,
+      deltaX: params.deltaX ?? 0,
+      deltaY: params.deltaY ?? 0,
+      modifiers: params.modifiers ?? 0,
+    });
+  }
+
+  /**
+   * Inject a keyboard event via CDP
+   */
+  async injectKeyboardEvent(params: {
+    type: 'keyDown' | 'keyUp' | 'char';
+    key?: string;
+    code?: string;
+    text?: string;
+    modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift
+  }): Promise<void> {
+    const cdp = await this.getCDPSession();
+
+    await cdp.send('Input.dispatchKeyEvent', {
+      type: params.type,
+      key: params.key,
+      code: params.code,
+      text: params.text,
+      modifiers: params.modifiers ?? 0,
+    });
+  }
+
+  /**
+   * Inject touch event via CDP (for mobile emulation)
+   */
+  async injectTouchEvent(params: {
+    type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel';
+    touchPoints: Array<{ x: number; y: number; id?: number }>;
+    modifiers?: number;
+  }): Promise<void> {
+    const cdp = await this.getCDPSession();
+
+    await cdp.send('Input.dispatchTouchEvent', {
+      type: params.type,
+      touchPoints: params.touchPoints.map((tp, i) => ({
+        x: tp.x,
+        y: tp.y,
+        id: tp.id ?? i,
+      })),
+      modifiers: params.modifiers ?? 0,
+    });
+  }
+
  /**
   * Close the browser and clean up
   */
  async close(): Promise<void> {
+    // Stop screencast if active
+    if (this.screencastActive) {
+      await this.stopScreencast();
+    }
+
+    // Clean up CDP session
+    if (this.cdpSession) {
+      await this.cdpSession.detach().catch(() => {});
+      this.cdpSession = null;
+    }
+
    // CDP: only disconnect, don't close external app's pages
    if (this.cdpPort !== null) {
      if (this.browser) {
@@ -880,5 +1127,6 @@ export class BrowserManager {
    this.activePageIndex = 0;
    this.refMap = {};
    this.lastSnapshot = '';
+    this.frameCallback = null;
  }
 }
@@ -5,6 +5,7 @@ import * as os from 'os';
 import { BrowserManager } from './browser.js';
 import { parseCommand, serializeResponse, errorResponse } from './protocol.js';
 import { executeCommand } from './actions.js';
+import { StreamServer } from './stream-server.js';

 // Platform detection
 const isWindows = process.platform === 'win32';
@@ -12,6 +13,12 @@ const isWindows = process.platform === 'win32';
 // Session support - each session gets its own socket/pid
 let currentSession = process.env.AGENT_BROWSER_SESSION || 'default';

+// Stream server for browser preview
+let streamServer: StreamServer | null = null;
+
+// Default stream port (can be overridden with AGENT_BROWSER_STREAM_PORT)
+const DEFAULT_STREAM_PORT = 9223;
+
 /**
 * Set the current session
 */
@@ -105,8 +112,10 @@ export function getConnectionInfo(
 */
 export function cleanupSocket(session?: string): void {
  const pidFile = getPidFile(session);
+  const streamPortFile = getStreamPortFile(session);
  try {
    if (fs.existsSync(pidFile)) fs.unlinkSync(pidFile);
+    if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile);
    if (isWindows) {
      const portFile = getPortFile(session);
      if (fs.existsSync(portFile)) fs.unlinkSync(portFile);
@@ -120,15 +129,40 @@ export function cleanupSocket(session?: string): void {
 }

 /**
- * Start the daemon server
+ * Get the stream port file path
 */
-export async function startDaemon(): Promise<void> {
+export function getStreamPortFile(session?: string): string {
+  const sess = session ?? currentSession;
+  return path.join(os.tmpdir(), `agent-browser-${sess}.stream`);
+}
+
+/**
+ * Start the daemon server
+ * @param options.streamPort Port for WebSocket stream server (0 to disable)
+ */
+export async function startDaemon(options?: { streamPort?: number }): Promise<void> {
  // Clean up any stale socket
  cleanupSocket();

  const browser = new BrowserManager();
  let shuttingDown = false;

+  // Start stream server if port is specified (or use default if env var is set)
+  const streamPort =
+    options?.streamPort ??
+    (process.env.AGENT_BROWSER_STREAM_PORT
+      ? parseInt(process.env.AGENT_BROWSER_STREAM_PORT, 10)
+      : 0);
+
+  if (streamPort > 0) {
+    streamServer = new StreamServer(browser, streamPort);
+    await streamServer.start();
+
+    // Write stream port to file for clients to discover
+    const streamPortFile = getStreamPortFile();
+    fs.writeFileSync(streamPortFile, streamPort.toString());
+  }
+
  const server = net.createServer((socket) => {
    let buffer = '';

@@ -227,6 +261,20 @@ export async function startDaemon(): Promise<void> {
  const shutdown = async () => {
    if (shuttingDown) return;
    shuttingDown = true;
+
+    // Stop stream server if running
+    if (streamServer) {
+      await streamServer.stop();
+      streamServer = null;
+      // Clean up stream port file
+      const streamPortFile = getStreamPortFile();
+      try {
+        if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile);
+      } catch {
+        // Ignore cleanup errors
+      }
+    }
+
    await browser.close();
    server.close();
    cleanupSocket();
@@ -620,6 +620,391 @@ describe('parseCommand', () => {
    });
  });

+  describe('screencast', () => {
+    it('should parse screencast_start with defaults', () => {
+      const result = parseCommand(cmd({ id: '1', action: 'screencast_start' }));
+      expect(result.success).toBe(true);
+      if (result.success) {
+        expect(result.command.action).toBe('screencast_start');
+      }
+    });
+
+    it('should parse screencast_start with all options', () => {
+      const result = parseCommand(
+        cmd({
+          id: '1',
+          action: 'screencast_start',
+          format: 'png',
+          quality: 90,
+          maxWidth: 1920,
+          maxHeight: 1080,
+          everyNthFrame: 2,
+        })
+      );
+      expect(result.success).toBe(true);
+      if (result.success) {
+        expect(result.command.format).toBe('png');
+        expect(result.command.quality).toBe(90);
+        expect(result.command.maxWidth).toBe(1920);
+        expect(result.command.maxHeight).toBe(1080);
+        expect(result.command.everyNthFrame).toBe(2);
+      }
+    });
+
+    it('should reject screencast_start with invalid format', () => {
+      const result = parseCommand(cmd({ id: '1', action: 'screencast_start', format: 'gif' }));
+      expect(result.success).toBe(false);
+    });
+
+    it('should reject screencast_start with quality out of range', () => {
+      const result = parseCommand(cmd({ id: '1', action: 'screencast_start', quality: 150 }));
+      expect(result.success).toBe(false);
+    });
+
+    it('should reject screencast_start with negative maxWidth', () => {
+      const result = parseCommand(cmd({ id: '1', action: 'screencast_start', maxWidth: -100 }));
+      expect(result.success).toBe(false);
+    });
+
+    it('should parse screencast_stop', () => {
+      const result = parseCommand(cmd({ id: '1', action: 'screencast_stop' }));
+      expect(result.success).toBe(true);
+      if (result.success) {
+        expect(result.command.action).toBe('screencast_stop');
+      }
+    });
+  });
+
+  describe('input injection', () => {
+    describe('input_mouse', () => {
+      it('should parse mousePressed event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mousePressed',
+            x: 100,
+            y: 200,
+            button: 'left',
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.action).toBe('input_mouse');
+          expect(result.command.type).toBe('mousePressed');
+          expect(result.command.x).toBe(100);
+          expect(result.command.y).toBe(200);
+          expect(result.command.button).toBe('left');
+        }
+      });
+
+      it('should parse mouseReleased event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mouseReleased',
+            x: 100,
+            y: 200,
+          })
+        );
+        expect(result.success).toBe(true);
+      });
+
+      it('should parse mouseMoved event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mouseMoved',
+            x: 150,
+            y: 250,
+          })
+        );
+        expect(result.success).toBe(true);
+      });
+
+      it('should parse mouseWheel event with deltas', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mouseWheel',
+            x: 100,
+            y: 200,
+            deltaX: 0,
+            deltaY: 100,
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.deltaX).toBe(0);
+          expect(result.command.deltaY).toBe(100);
+        }
+      });
+
+      it('should parse mouse event with modifiers', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mousePressed',
+            x: 100,
+            y: 200,
+            modifiers: 6, // Ctrl + Meta
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.modifiers).toBe(6);
+        }
+      });
+
+      it('should parse mouse event with clickCount', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mousePressed',
+            x: 100,
+            y: 200,
+            clickCount: 2,
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.clickCount).toBe(2);
+        }
+      });
+
+      it('should reject input_mouse with invalid type', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'invalid',
+            x: 100,
+            y: 200,
+          })
+        );
+        expect(result.success).toBe(false);
+      });
+
+      it('should reject input_mouse without x coordinate', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mousePressed',
+            y: 200,
+          })
+        );
+        expect(result.success).toBe(false);
+      });
+
+      it('should reject input_mouse without y coordinate', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_mouse',
+            type: 'mousePressed',
+            x: 100,
+          })
+        );
+        expect(result.success).toBe(false);
+      });
+    });
+
+    describe('input_keyboard', () => {
+      it('should parse keyDown event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_keyboard',
+            type: 'keyDown',
+            key: 'Enter',
+            code: 'Enter',
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.action).toBe('input_keyboard');
+          expect(result.command.type).toBe('keyDown');
+          expect(result.command.key).toBe('Enter');
+          expect(result.command.code).toBe('Enter');
+        }
+      });
+
+      it('should parse keyUp event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_keyboard',
+            type: 'keyUp',
+            key: 'a',
+          })
+        );
+        expect(result.success).toBe(true);
+      });
+
+      it('should parse char event with text', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_keyboard',
+            type: 'char',
+            text: 'hello',
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.text).toBe('hello');
+        }
+      });
+
+      it('should parse keyboard event with modifiers', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_keyboard',
+            type: 'keyDown',
+            key: 'c',
+            modifiers: 2, // Ctrl
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.modifiers).toBe(2);
+        }
+      });
+
+      it('should reject input_keyboard with invalid type', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_keyboard',
+            type: 'invalid',
+          })
+        );
+        expect(result.success).toBe(false);
+      });
+    });
+
+    describe('input_touch', () => {
+      it('should parse touchStart event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'touchStart',
+            touchPoints: [{ x: 100, y: 200 }],
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.action).toBe('input_touch');
+          expect(result.command.type).toBe('touchStart');
+          expect(result.command.touchPoints).toHaveLength(1);
+          expect(result.command.touchPoints[0].x).toBe(100);
+          expect(result.command.touchPoints[0].y).toBe(200);
+        }
+      });
+
+      it('should parse touchEnd event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'touchEnd',
+            touchPoints: [],
+          })
+        );
+        expect(result.success).toBe(true);
+      });
+
+      it('should parse touchMove event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'touchMove',
+            touchPoints: [{ x: 150, y: 250 }],
+          })
+        );
+        expect(result.success).toBe(true);
+      });
+
+      it('should parse touchCancel event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'touchCancel',
+            touchPoints: [],
+          })
+        );
+        expect(result.success).toBe(true);
+      });
+
+      it('should parse multi-touch event', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'touchStart',
+            touchPoints: [
+              { x: 100, y: 200, id: 0 },
+              { x: 300, y: 400, id: 1 },
+            ],
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.touchPoints).toHaveLength(2);
+        }
+      });
+
+      it('should parse touch event with modifiers', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'touchStart',
+            touchPoints: [{ x: 100, y: 200 }],
+            modifiers: 8, // Shift
+          })
+        );
+        expect(result.success).toBe(true);
+        if (result.success) {
+          expect(result.command.modifiers).toBe(8);
+        }
+      });
+
+      it('should reject input_touch with invalid type', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'invalid',
+            touchPoints: [],
+          })
+        );
+        expect(result.success).toBe(false);
+      });
+
+      it('should reject input_touch without touchPoints', () => {
+        const result = parseCommand(
+          cmd({
+            id: '1',
+            action: 'input_touch',
+            type: 'touchStart',
+          })
+        );
+        expect(result.success).toBe(false);
+      });
+    });
+  });
+
  describe('invalid commands', () => {
    it('should reject unknown action', () => {
      const result = parseCommand(cmd({ id: '1', action: 'unknown' }));
@@ -585,6 +585,55 @@ const responseBodySchema = baseCommandSchema.extend({
  timeout: z.number().positive().optional(),
 });

+// Screencast schemas for streaming browser viewport
+const screencastStartSchema = baseCommandSchema.extend({
+  action: z.literal('screencast_start'),
+  format: z.enum(['jpeg', 'png']).optional(),
+  quality: z.number().min(0).max(100).optional(),
+  maxWidth: z.number().positive().optional(),
+  maxHeight: z.number().positive().optional(),
+  everyNthFrame: z.number().positive().optional(),
+});
+
+const screencastStopSchema = baseCommandSchema.extend({
+  action: z.literal('screencast_stop'),
+});
+
+// Input injection schemas for pair browsing
+const inputMouseSchema = baseCommandSchema.extend({
+  action: z.literal('input_mouse'),
+  type: z.enum(['mousePressed', 'mouseReleased', 'mouseMoved', 'mouseWheel']),
+  x: z.number(),
+  y: z.number(),
+  button: z.enum(['left', 'right', 'middle', 'none']).optional(),
+  clickCount: z.number().positive().optional(),
+  deltaX: z.number().optional(),
+  deltaY: z.number().optional(),
+  modifiers: z.number().optional(),
+});
+
+const inputKeyboardSchema = baseCommandSchema.extend({
+  action: z.literal('input_keyboard'),
+  type: z.enum(['keyDown', 'keyUp', 'char']),
+  key: z.string().optional(),
+  code: z.string().optional(),
+  text: z.string().optional(),
+  modifiers: z.number().optional(),
+});
+
+const inputTouchSchema = baseCommandSchema.extend({
+  action: z.literal('input_touch'),
+  type: z.enum(['touchStart', 'touchEnd', 'touchMove', 'touchCancel']),
+  touchPoints: z.array(
+    z.object({
+      x: z.number(),
+      y: z.number(),
+      id: z.number().optional(),
+    })
+  ),
+  modifiers: z.number().optional(),
+});
+
 const pressSchema = baseCommandSchema.extend({
  action: z.literal('press'),
  key: z.string().min(1),
@@ -795,6 +844,11 @@ const commandSchema = z.discriminatedUnion('action', [
  multiSelectSchema,
  waitForDownloadSchema,
  responseBodySchema,
+  screencastStartSchema,
+  screencastStopSchema,
+  inputMouseSchema,
+  inputKeyboardSchema,
+  inputTouchSchema,
 ]);

 // Parse result type
@@ -0,0 +1,364 @@
+import { WebSocketServer, WebSocket } from 'ws';
+import type { BrowserManager, ScreencastFrame } from './browser.js';
+import { setScreencastFrameCallback } from './actions.js';
+
+// Message types for WebSocket communication
+export interface FrameMessage {
+  type: 'frame';
+  data: string; // base64 encoded image
+  metadata: {
+    offsetTop: number;
+    pageScaleFactor: number;
+    deviceWidth: number;
+    deviceHeight: number;
+    scrollOffsetX: number;
+    scrollOffsetY: number;
+    timestamp?: number;
+  };
+}
+
+export interface InputMouseMessage {
+  type: 'input_mouse';
+  eventType: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel';
+  x: number;
+  y: number;
+  button?: 'left' | 'right' | 'middle' | 'none';
+  clickCount?: number;
+  deltaX?: number;
+  deltaY?: number;
+  modifiers?: number;
+}
+
+export interface InputKeyboardMessage {
+  type: 'input_keyboard';
+  eventType: 'keyDown' | 'keyUp' | 'char';
+  key?: string;
+  code?: string;
+  text?: string;
+  modifiers?: number;
+}
+
+export interface InputTouchMessage {
+  type: 'input_touch';
+  eventType: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel';
+  touchPoints: Array<{ x: number; y: number; id?: number }>;
+  modifiers?: number;
+}
+
+export interface StatusMessage {
+  type: 'status';
+  connected: boolean;
+  screencasting: boolean;
+  viewportWidth?: number;
+  viewportHeight?: number;
+}
+
+export interface ErrorMessage {
+  type: 'error';
+  message: string;
+}
+
+export type StreamMessage =
+  | FrameMessage
+  | InputMouseMessage
+  | InputKeyboardMessage
+  | InputTouchMessage
+  | StatusMessage
+  | ErrorMessage;
+
+/**
+ * WebSocket server for streaming browser viewport and receiving input
+ */
+export class StreamServer {
+  private wss: WebSocketServer | null = null;
+  private clients: Set<WebSocket> = new Set();
+  private browser: BrowserManager;
+  private port: number;
+  private isScreencasting: boolean = false;
+
+  constructor(browser: BrowserManager, port: number = 9223) {
+    this.browser = browser;
+    this.port = port;
+  }
+
+  /**
+   * Start the WebSocket server
+   */
+  start(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      try {
+        this.wss = new WebSocketServer({ port: this.port });
+
+        this.wss.on('connection', (ws) => {
+          this.handleConnection(ws);
+        });
+
+        this.wss.on('error', (error) => {
+          console.error('[StreamServer] WebSocket error:', error);
+          reject(error);
+        });
+
+        this.wss.on('listening', () => {
+          console.log(`[StreamServer] Listening on port ${this.port}`);
+
+          // Set up the screencast frame callback
+          setScreencastFrameCallback((frame) => {
+            this.broadcastFrame(frame);
+          });
+
+          resolve();
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+
+  /**
+   * Stop the WebSocket server
+   */
+  async stop(): Promise<void> {
+    // Stop screencasting
+    if (this.isScreencasting) {
+      await this.stopScreencast();
+    }
+
+    // Clear the callback
+    setScreencastFrameCallback(null);
+
+    // Close all clients
+    for (const client of this.clients) {
+      client.close();
+    }
+    this.clients.clear();
+
+    // Close the server
+    if (this.wss) {
+      return new Promise((resolve) => {
+        this.wss!.close(() => {
+          this.wss = null;
+          resolve();
+        });
+      });
+    }
+  }
+
+  /**
+   * Handle a new WebSocket connection
+   */
+  private handleConnection(ws: WebSocket): void {
+    console.log('[StreamServer] Client connected');
+    this.clients.add(ws);
+
+    // Send initial status
+    this.sendStatus(ws);
+
+    // Start screencasting if this is the first client
+    if (this.clients.size === 1 && !this.isScreencasting) {
+      this.startScreencast().catch((error) => {
+        console.error('[StreamServer] Failed to start screencast:', error);
+        this.sendError(ws, error.message);
+      });
+    }
+
+    // Handle messages from client
+    ws.on('message', (data) => {
+      try {
+        const message = JSON.parse(data.toString()) as StreamMessage;
+        this.handleMessage(message, ws);
+      } catch (error) {
+        console.error('[StreamServer] Failed to parse message:', error);
+      }
+    });
+
+    // Handle client disconnect
+    ws.on('close', () => {
+      console.log('[StreamServer] Client disconnected');
+      this.clients.delete(ws);
+
+      // Stop screencasting if no more clients
+      if (this.clients.size === 0 && this.isScreencasting) {
+        this.stopScreencast().catch((error) => {
+          console.error('[StreamServer] Failed to stop screencast:', error);
+        });
+      }
+    });
+
+    ws.on('error', (error) => {
+      console.error('[StreamServer] Client error:', error);
+      this.clients.delete(ws);
+    });
+  }
+
+  /**
+   * Handle incoming messages from clients
+   */
+  private async handleMessage(message: StreamMessage, ws: WebSocket): Promise<void> {
+    try {
+      switch (message.type) {
+        case 'input_mouse':
+          await this.browser.injectMouseEvent({
+            type: message.eventType,
+            x: message.x,
+            y: message.y,
+            button: message.button,
+            clickCount: message.clickCount,
+            deltaX: message.deltaX,
+            deltaY: message.deltaY,
+            modifiers: message.modifiers,
+          });
+          break;
+
+        case 'input_keyboard':
+          await this.browser.injectKeyboardEvent({
+            type: message.eventType,
+            key: message.key,
+            code: message.code,
+            text: message.text,
+            modifiers: message.modifiers,
+          });
+          break;
+
+        case 'input_touch':
+          await this.browser.injectTouchEvent({
+            type: message.eventType,
+            touchPoints: message.touchPoints,
+            modifiers: message.modifiers,
+          });
+          break;
+
+        case 'status':
+          // Client is requesting status
+          this.sendStatus(ws);
+          break;
+      }
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      this.sendError(ws, errorMessage);
+    }
+  }
+
+  /**
+   * Broadcast a frame to all connected clients
+   */
+  private broadcastFrame(frame: ScreencastFrame): void {
+    const message: FrameMessage = {
+      type: 'frame',
+      data: frame.data,
+      metadata: frame.metadata,
+    };
+
+    const payload = JSON.stringify(message);
+
+    for (const client of this.clients) {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(payload);
+      }
+    }
+  }
+
+  /**
+   * Send status to a client
+   */
+  private sendStatus(ws: WebSocket): void {
+    let viewportWidth: number | undefined;
+    let viewportHeight: number | undefined;
+
+    try {
+      const page = this.browser.getPage();
+      const viewport = page.viewportSize();
+      viewportWidth = viewport?.width;
+      viewportHeight = viewport?.height;
+    } catch {
+      // Browser not launched yet
+    }
+
+    const message: StatusMessage = {
+      type: 'status',
+      connected: true,
+      screencasting: this.isScreencasting,
+      viewportWidth,
+      viewportHeight,
+    };
+
+    if (ws.readyState === WebSocket.OPEN) {
+      ws.send(JSON.stringify(message));
+    }
+  }
+
+  /**
+   * Send an error to a client
+   */
+  private sendError(ws: WebSocket, errorMessage: string): void {
+    const message: ErrorMessage = {
+      type: 'error',
+      message: errorMessage,
+    };
+
+    if (ws.readyState === WebSocket.OPEN) {
+      ws.send(JSON.stringify(message));
+    }
+  }
+
+  /**
+   * Start screencasting
+   */
+  private async startScreencast(): Promise<void> {
+    // Set flag immediately to prevent race conditions with concurrent calls
+    if (this.isScreencasting) return;
+    this.isScreencasting = true;
+
+    try {
+      // Check if browser is launched
+      if (!this.browser.isLaunched()) {
+        throw new Error('Browser not launched');
+      }
+
+      await this.browser.startScreencast((frame) => this.broadcastFrame(frame), {
+        format: 'jpeg',
+        quality: 80,
+        maxWidth: 1280,
+        maxHeight: 720,
+        everyNthFrame: 1,
+      });
+
+      // Notify all clients
+      for (const client of this.clients) {
+        this.sendStatus(client);
+      }
+    } catch (error) {
+      // Reset flag on failure so caller can retry
+      this.isScreencasting = false;
+      throw error;
+    }
+  }
+
+  /**
+   * Stop screencasting
+   */
+  private async stopScreencast(): Promise<void> {
+    if (!this.isScreencasting) return;
+
+    await this.browser.stopScreencast();
+    this.isScreencasting = false;
+
+    // Notify all clients
+    for (const client of this.clients) {
+      this.sendStatus(client);
+    }
+  }
+
+  /**
+   * Get the port the server is running on
+   */
+  getPort(): number {
+    return this.port;
+  }
+
+  /**
+   * Get the number of connected clients
+   */
+  getClientCount(): number {
+    return this.clients.size;
+  }
+}
@@ -458,6 +458,49 @@ export interface ResponseBodyCommand extends BaseCommand {
  timeout?: number;
 }

+// Screencast commands for streaming browser viewport
+export interface ScreencastStartCommand extends BaseCommand {
+  action: 'screencast_start';
+  format?: 'jpeg' | 'png';
+  quality?: number; // 0-100, jpeg only
+  maxWidth?: number;
+  maxHeight?: number;
+  everyNthFrame?: number;
+}
+
+export interface ScreencastStopCommand extends BaseCommand {
+  action: 'screencast_stop';
+}
+
+// Input injection commands for pair browsing
+export interface InputMouseCommand extends BaseCommand {
+  action: 'input_mouse';
+  type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel';
+  x: number;
+  y: number;
+  button?: 'left' | 'right' | 'middle' | 'none';
+  clickCount?: number;
+  deltaX?: number;
+  deltaY?: number;
+  modifiers?: number;
+}
+
+export interface InputKeyboardCommand extends BaseCommand {
+  action: 'input_keyboard';
+  type: 'keyDown' | 'keyUp' | 'char';
+  key?: string;
+  code?: string;
+  text?: string;
+  modifiers?: number;
+}
+
+export interface InputTouchCommand extends BaseCommand {
+  action: 'input_touch';
+  type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel';
+  touchPoints: Array<{ x: number; y: number; id?: number }>;
+  modifiers?: number;
+}
+
 // Video recording
 export interface VideoStartCommand extends BaseCommand {
  action: 'video_start';
@@ -841,7 +884,12 @@ export type Command =
  | InsertTextCommand
  | MultiSelectCommand
  | WaitForDownloadCommand
-  | ResponseBodyCommand;
+  | ResponseBodyCommand
+  | ScreencastStartCommand
+  | ScreencastStopCommand
+  | InputMouseCommand
+  | InputKeyboardCommand
+  | InputTouchCommand;

 // Response types
 export interface SuccessResponse<T = unknown> {
@@ -909,6 +957,20 @@ export interface TabCloseData {
  remaining: number;
 }

+export interface ScreencastStartData {
+  started: boolean;
+  format: string;
+  quality: number;
+}
+
+export interface ScreencastStopData {
+  stopped: boolean;
+}
+
+export interface InputEventData {
+  injected: boolean;
+}
+
 // Browser state
 export interface BrowserState {
  browser: Browser | null;
Author	SHA1	Message	Date
Chris Tate	1522b3b3a8	address comments	2026-01-13 14:43:10 -06:00
Chris Tate	7e127c6e73	update docs	2026-01-13 12:50:19 -06:00
Chris Tate	55dca4f38a	Merge remote-tracking branch 'origin/main' into ctate/screencast	2026-01-13 12:34:51 -06:00
Chris Tate	a60d986020	screencast	2026-01-13 12:34:00 -06:00
Chris TateandVercel <vercel[bot]@users.noreply.github.com>	4713c8b520	add docs (#54 ) * docs * updates * Fix: The handleCopy function fails to handle errors from navigator.clipboard.writeText(), causing unhandled exceptions and misleading UI feedback when clipboard operations fail. Co-authored-by: ctate <chris@ctate.dev> * Fix: The benchmark file uses emojis (📊, 🚀, 🔨, 📈, 📋, ✅, ⏱️, ⚠) in console output, violating repository guidelines that forbid emojis in code and output. Co-authored-by: ctate <chris@ctate.dev> * Remove benchmark/run.ts from PR --------- Co-authored-by: Vercel <vercel[bot]@users.noreply.github.com>	2026-01-13 02:54:56 -06:00
Chris Tate	b4bc761168	fix builds (#55 )	2026-01-13 02:41:39 -06:00
Bryan LeeandChris Tate	6eafe50952	fix incomplete build-from-source instructions (#40 ) (#41 ) Co-authored-by: Chris Tate <chris@ctate.dev>	2026-01-13 02:21:26 -06:00