screencast

src/browser.ts

@@ -11,10 +11,35 @@ import {
   type Request,
   type Route,
   type Locator,
+  type CDPSession,
 } from 'playwright-core';
 import type { LaunchCommand } from './types.js';
 import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js';
 
+// Screencast frame data from CDP
+export interface ScreencastFrame {
+  data: string; // base64 encoded image
+  metadata: {
+    offsetTop: number;
+    pageScaleFactor: number;
+    deviceWidth: number;
+    deviceHeight: number;
+    scrollOffsetX: number;
+    scrollOffsetY: number;
+    timestamp?: number;
+  };
+  sessionId: number;
+}
+
+// Screencast options
+export interface ScreencastOptions {
+  format?: 'jpeg' | 'png';
+  quality?: number; // 0-100, only for jpeg
+  maxWidth?: number;
+  maxHeight?: number;
+  everyNthFrame?: number;
+}
+
 interface TrackedRequest {
   url: string;
   method: string;
@@ -54,6 +79,12 @@ export class BrowserManager {
   private lastSnapshot: string = '';
   private scopedHeaderRoutes: Map<string, (route: Route) => Promise<void>> = new Map();
 
+  // CDP session for screencast and input injection
+  private cdpSession: CDPSession | null = null;
+  private screencastActive: boolean = false;
+  private screencastSessionId: number = 0;
+  private frameCallback: ((frame: ScreencastFrame) => void) | null = null;
+
   /**
    * Check if browser is launched
    */
@@ -850,10 +881,186 @@ export class BrowserManager {
     return tabs;
   }
 
+  /**
+   * Get or create a CDP session for the current page
+   * Only works with Chromium-based browsers
+   */
+  async getCDPSession(): Promise<CDPSession> {
+    if (this.cdpSession) {
+      return this.cdpSession;
+    }
+
+    const page = this.getPage();
+    const context = page.context();
+
+    // Create a new CDP session attached to the page
+    this.cdpSession = await context.newCDPSession(page);
+    return this.cdpSession;
+  }
+
+  /**
+   * Check if screencast is currently active
+   */
+  isScreencasting(): boolean {
+    return this.screencastActive;
+  }
+
+  /**
+   * Start screencast - streams viewport frames via CDP
+   * @param callback Function called for each frame
+   * @param options Screencast options
+   */
+  async startScreencast(
+    callback: (frame: ScreencastFrame) => void,
+    options?: ScreencastOptions
+  ): Promise<void> {
+    if (this.screencastActive) {
+      throw new Error('Screencast already active');
+    }
+
+    const cdp = await this.getCDPSession();
+    this.frameCallback = callback;
+    this.screencastActive = true;
+
+    // Listen for screencast frames
+    cdp.on('Page.screencastFrame', async (params: any) => {
+      const frame: ScreencastFrame = {
+        data: params.data,
+        metadata: params.metadata,
+        sessionId: params.sessionId,
+      };
+
+      // Acknowledge the frame to receive the next one
+      await cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId });
+
+      // Call the callback with the frame
+      if (this.frameCallback) {
+        this.frameCallback(frame);
+      }
+    });
+
+    // Start the screencast
+    await cdp.send('Page.startScreencast', {
+      format: options?.format ?? 'jpeg',
+      quality: options?.quality ?? 80,
+      maxWidth: options?.maxWidth ?? 1280,
+      maxHeight: options?.maxHeight ?? 720,
+      everyNthFrame: options?.everyNthFrame ?? 1,
+    });
+  }
+
+  /**
+   * Stop screencast
+   */
+  async stopScreencast(): Promise<void> {
+    if (!this.screencastActive) {
+      return;
+    }
+
+    try {
+      const cdp = await this.getCDPSession();
+      await cdp.send('Page.stopScreencast');
+    } catch {
+      // Ignore errors when stopping
+    }
+
+    this.screencastActive = false;
+    this.frameCallback = null;
+  }
+
+  /**
+   * Inject a mouse event via CDP
+   */
+  async injectMouseEvent(params: {
+    type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel';
+    x: number;
+    y: number;
+    button?: 'left' | 'right' | 'middle' | 'none';
+    clickCount?: number;
+    deltaX?: number;
+    deltaY?: number;
+    modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift
+  }): Promise<void> {
+    const cdp = await this.getCDPSession();
+
+    const cdpButton =
+      params.button === 'left'
+        ? 'left'
+        : params.button === 'right'
+          ? 'right'
+          : params.button === 'middle'
+            ? 'middle'
+            : 'none';
+
+    await cdp.send('Input.dispatchMouseEvent', {
+      type: params.type,
+      x: params.x,
+      y: params.y,
+      button: cdpButton,
+      clickCount: params.clickCount ?? 1,
+      deltaX: params.deltaX ?? 0,
+      deltaY: params.deltaY ?? 0,
+      modifiers: params.modifiers ?? 0,
+    });
+  }
+
+  /**
+   * Inject a keyboard event via CDP
+   */
+  async injectKeyboardEvent(params: {
+    type: 'keyDown' | 'keyUp' | 'char';
+    key?: string;
+    code?: string;
+    text?: string;
+    modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift
+  }): Promise<void> {
+    const cdp = await this.getCDPSession();
+
+    await cdp.send('Input.dispatchKeyEvent', {
+      type: params.type,
+      key: params.key,
+      code: params.code,
+      text: params.text,
+      modifiers: params.modifiers ?? 0,
+    });
+  }
+
+  /**
+   * Inject touch event via CDP (for mobile emulation)
+   */
+  async injectTouchEvent(params: {
+    type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel';
+    touchPoints: Array<{ x: number; y: number; id?: number }>;
+    modifiers?: number;
+  }): Promise<void> {
+    const cdp = await this.getCDPSession();
+
+    await cdp.send('Input.dispatchTouchEvent', {
+      type: params.type,
+      touchPoints: params.touchPoints.map((tp, i) => ({
+        x: tp.x,
+        y: tp.y,
+        id: tp.id ?? i,
+      })),
+      modifiers: params.modifiers ?? 0,
+    });
+  }
+
   /**
    * Close the browser and clean up
    */
   async close(): Promise<void> {
+    // Stop screencast if active
+    if (this.screencastActive) {
+      await this.stopScreencast();
+    }
+
+    // Clean up CDP session
+    if (this.cdpSession) {
+      await this.cdpSession.detach().catch(() => {});
+      this.cdpSession = null;
+    }
+
     // CDP: only disconnect, don't close external app's pages
     if (this.cdpPort !== null) {
       if (this.browser) {
@@ -880,5 +1087,6 @@ export class BrowserManager {
     this.activePageIndex = 0;
     this.refMap = {};
     this.lastSnapshot = '';
+    this.frameCallback = null;
   }
 }