import { chromium, firefox, webkit, devices, type Browser, type BrowserContext, type Page, type Frame, type Dialog, type Request, type Route, type Locator, type CDPSession, type Video, } from 'playwright-core'; import path from 'node:path'; import os from 'node:os'; import { existsSync, mkdirSync, rmSync } from 'node:fs'; import type { LaunchCommand } from './types.js'; import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js'; // Screencast frame data from CDP export interface ScreencastFrame { data: string; // base64 encoded image metadata: { offsetTop: number; pageScaleFactor: number; deviceWidth: number; deviceHeight: number; scrollOffsetX: number; scrollOffsetY: number; timestamp?: number; }; sessionId: number; } // Screencast options export interface ScreencastOptions { format?: 'jpeg' | 'png'; quality?: number; // 0-100, only for jpeg maxWidth?: number; maxHeight?: number; everyNthFrame?: number; } interface TrackedRequest { url: string; method: string; headers: Record; timestamp: number; resourceType: string; } interface ConsoleMessage { type: string; text: string; timestamp: number; } interface PageError { message: string; timestamp: number; } /** * Manages the Playwright browser lifecycle with multiple tabs/windows */ export class BrowserManager { private browser: Browser | null = null; private cdpEndpoint: string | null = null; // stores port number or full URL private isPersistentContext: boolean = false; private browserbaseSessionId: string | null = null; private browserbaseApiKey: string | null = null; private browserUseSessionId: string | null = null; private browserUseApiKey: string | null = null; private kernelSessionId: string | null = null; private kernelApiKey: string | null = null; private contexts: BrowserContext[] = []; private pages: Page[] = []; private activePageIndex: number = 0; private activeFrame: Frame | null = null; private dialogHandler: ((dialog: Dialog) => Promise) | null = null; private trackedRequests: TrackedRequest[] = []; private routes: Map Promise> = new Map(); private consoleMessages: ConsoleMessage[] = []; private pageErrors: PageError[] = []; private isRecordingHar: boolean = false; private refMap: RefMap = {}; private lastSnapshot: string = ''; private scopedHeaderRoutes: Map Promise> = new Map(); // CDP session for screencast and input injection private cdpSession: CDPSession | null = null; private screencastActive: boolean = false; private screencastSessionId: number = 0; private frameCallback: ((frame: ScreencastFrame) => void) | null = null; private screencastFrameHandler: ((params: any) => void) | null = null; // Video recording (Playwright native) private recordingContext: BrowserContext | null = null; private recordingPage: Page | null = null; private recordingOutputPath: string = ''; private recordingTempDir: string = ''; /** * Check if browser is launched */ isLaunched(): boolean { return this.browser !== null || this.isPersistentContext; } /** * Get enhanced snapshot with refs and cache the ref map */ async getSnapshot(options?: { interactive?: boolean; maxDepth?: number; compact?: boolean; selector?: string; }): Promise { const page = this.getPage(); const snapshot = await getEnhancedSnapshot(page, options); this.refMap = snapshot.refs; this.lastSnapshot = snapshot.tree; return snapshot; } /** * Get the cached ref map from last snapshot */ getRefMap(): RefMap { return this.refMap; } /** * Get a locator from a ref (e.g., "e1", "@e1", "ref=e1") * Returns null if ref doesn't exist or is invalid */ getLocatorFromRef(refArg: string): Locator | null { const ref = parseRef(refArg); if (!ref) return null; const refData = this.refMap[ref]; if (!refData) return null; const page = this.getPage(); // Build locator with exact: true to avoid substring matches let locator: Locator; if (refData.name) { locator = page.getByRole(refData.role as any, { name: refData.name, exact: true }); } else { locator = page.getByRole(refData.role as any); } // If an nth index is stored (for disambiguation), use it if (refData.nth !== undefined) { locator = locator.nth(refData.nth); } return locator; } /** * Check if a selector looks like a ref */ isRef(selector: string): boolean { return parseRef(selector) !== null; } /** * Get locator - supports both refs and regular selectors */ getLocator(selectorOrRef: string): Locator { // Check if it's a ref first const locator = this.getLocatorFromRef(selectorOrRef); if (locator) return locator; // Otherwise treat as regular selector const page = this.getPage(); return page.locator(selectorOrRef); } /** * Get the current active page, throws if not launched */ getPage(): Page { if (this.pages.length === 0) { throw new Error('Browser not launched. Call launch first.'); } return this.pages[this.activePageIndex]; } /** * Get the current frame (or page's main frame if no frame is selected) */ getFrame(): Frame { if (this.activeFrame) { return this.activeFrame; } return this.getPage().mainFrame(); } /** * Switch to a frame by selector, name, or URL */ async switchToFrame(options: { selector?: string; name?: string; url?: string }): Promise { const page = this.getPage(); if (options.selector) { const frameElement = await page.$(options.selector); if (!frameElement) { throw new Error(`Frame not found: ${options.selector}`); } const frame = await frameElement.contentFrame(); if (!frame) { throw new Error(`Element is not a frame: ${options.selector}`); } this.activeFrame = frame; } else if (options.name) { const frame = page.frame({ name: options.name }); if (!frame) { throw new Error(`Frame not found with name: ${options.name}`); } this.activeFrame = frame; } else if (options.url) { const frame = page.frame({ url: options.url }); if (!frame) { throw new Error(`Frame not found with URL: ${options.url}`); } this.activeFrame = frame; } } /** * Switch back to main frame */ switchToMainFrame(): void { this.activeFrame = null; } /** * Set up dialog handler */ setDialogHandler(response: 'accept' | 'dismiss', promptText?: string): void { const page = this.getPage(); // Remove existing handler if any if (this.dialogHandler) { page.removeListener('dialog', this.dialogHandler); } this.dialogHandler = async (dialog: Dialog) => { if (response === 'accept') { await dialog.accept(promptText); } else { await dialog.dismiss(); } }; page.on('dialog', this.dialogHandler); } /** * Clear dialog handler */ clearDialogHandler(): void { if (this.dialogHandler) { const page = this.getPage(); page.removeListener('dialog', this.dialogHandler); this.dialogHandler = null; } } /** * Start tracking requests */ startRequestTracking(): void { const page = this.getPage(); page.on('request', (request: Request) => { this.trackedRequests.push({ url: request.url(), method: request.method(), headers: request.headers(), timestamp: Date.now(), resourceType: request.resourceType(), }); }); } /** * Get tracked requests */ getRequests(filter?: string): TrackedRequest[] { if (filter) { return this.trackedRequests.filter((r) => r.url.includes(filter)); } return this.trackedRequests; } /** * Clear tracked requests */ clearRequests(): void { this.trackedRequests = []; } /** * Add a route to intercept requests */ async addRoute( url: string, options: { response?: { status?: number; body?: string; contentType?: string; headers?: Record; }; abort?: boolean; } ): Promise { const page = this.getPage(); const handler = async (route: Route) => { if (options.abort) { await route.abort(); } else if (options.response) { await route.fulfill({ status: options.response.status ?? 200, body: options.response.body ?? '', contentType: options.response.contentType ?? 'text/plain', headers: options.response.headers, }); } else { await route.continue(); } }; this.routes.set(url, handler); await page.route(url, handler); } /** * Remove a route */ async removeRoute(url?: string): Promise { const page = this.getPage(); if (url) { const handler = this.routes.get(url); if (handler) { await page.unroute(url, handler); this.routes.delete(url); } } else { // Remove all routes for (const [routeUrl, handler] of this.routes) { await page.unroute(routeUrl, handler); } this.routes.clear(); } } /** * Set geolocation */ async setGeolocation(latitude: number, longitude: number, accuracy?: number): Promise { const context = this.contexts[0]; if (context) { await context.setGeolocation({ latitude, longitude, accuracy }); } } /** * Set permissions */ async setPermissions(permissions: string[], grant: boolean): Promise { const context = this.contexts[0]; if (context) { if (grant) { await context.grantPermissions(permissions); } else { await context.clearPermissions(); } } } /** * Set viewport */ async setViewport(width: number, height: number): Promise { const page = this.getPage(); await page.setViewportSize({ width, height }); } /** * Set device scale factor (devicePixelRatio) via CDP * This sets window.devicePixelRatio which affects how the page renders and responds to media queries * * Note: When using CDP to set deviceScaleFactor, screenshots will be at logical pixel dimensions * (viewport size), not physical pixel dimensions (viewport × scale). This is a Playwright limitation * when using CDP emulation on existing contexts. For true HiDPI screenshots with physical pixels, * deviceScaleFactor must be set at context creation time. * * Must be called after setViewport to work correctly */ async setDeviceScaleFactor( deviceScaleFactor: number, width: number, height: number, mobile: boolean = false ): Promise { const cdp = await this.getCDPSession(); await cdp.send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor, mobile, }); } /** * Clear device metrics override to restore default devicePixelRatio */ async clearDeviceMetricsOverride(): Promise { const cdp = await this.getCDPSession(); await cdp.send('Emulation.clearDeviceMetricsOverride'); } /** * Get device descriptor */ getDevice(deviceName: string): (typeof devices)[keyof typeof devices] | undefined { return devices[deviceName as keyof typeof devices]; } /** * List available devices */ listDevices(): string[] { return Object.keys(devices); } /** * Start console message tracking */ startConsoleTracking(): void { const page = this.getPage(); page.on('console', (msg) => { this.consoleMessages.push({ type: msg.type(), text: msg.text(), timestamp: Date.now(), }); }); } /** * Get console messages */ getConsoleMessages(): ConsoleMessage[] { return this.consoleMessages; } /** * Clear console messages */ clearConsoleMessages(): void { this.consoleMessages = []; } /** * Start error tracking */ startErrorTracking(): void { const page = this.getPage(); page.on('pageerror', (error) => { this.pageErrors.push({ message: error.message, timestamp: Date.now(), }); }); } /** * Get page errors */ getPageErrors(): PageError[] { return this.pageErrors; } /** * Clear page errors */ clearPageErrors(): void { this.pageErrors = []; } /** * Start HAR recording */ async startHarRecording(): Promise { // HAR is started at context level, flag for tracking this.isRecordingHar = true; } /** * Check if HAR recording */ isHarRecording(): boolean { return this.isRecordingHar; } /** * Set offline mode */ async setOffline(offline: boolean): Promise { const context = this.contexts[0]; if (context) { await context.setOffline(offline); } } /** * Set extra HTTP headers (global - all requests) */ async setExtraHeaders(headers: Record): Promise { const context = this.contexts[0]; if (context) { await context.setExtraHTTPHeaders(headers); } } /** * Set scoped HTTP headers (only for requests matching the origin) * Uses route interception to add headers only to matching requests */ async setScopedHeaders(origin: string, headers: Record): Promise { const page = this.getPage(); // Build URL pattern from origin (e.g., "api.example.com" -> "**://api.example.com/**") // Handle both full URLs and just hostnames let urlPattern: string; try { const url = new URL(origin.startsWith('http') ? origin : `https://${origin}`); // Match any protocol, the host, and any path urlPattern = `**://${url.host}/**`; } catch { // If parsing fails, treat as hostname pattern urlPattern = `**://${origin}/**`; } // Remove existing route for this origin if any const existingHandler = this.scopedHeaderRoutes.get(urlPattern); if (existingHandler) { await page.unroute(urlPattern, existingHandler); } // Create handler that adds headers to matching requests const handler = async (route: Route) => { const requestHeaders = route.request().headers(); await route.continue({ headers: { ...requestHeaders, ...headers, }, }); }; // Store and register the route this.scopedHeaderRoutes.set(urlPattern, handler); await page.route(urlPattern, handler); } /** * Clear scoped headers for an origin (or all if no origin specified) */ async clearScopedHeaders(origin?: string): Promise { const page = this.getPage(); if (origin) { let urlPattern: string; try { const url = new URL(origin.startsWith('http') ? origin : `https://${origin}`); urlPattern = `**://${url.host}/**`; } catch { urlPattern = `**://${origin}/**`; } const handler = this.scopedHeaderRoutes.get(urlPattern); if (handler) { await page.unroute(urlPattern, handler); this.scopedHeaderRoutes.delete(urlPattern); } } else { // Clear all scoped header routes for (const [pattern, handler] of this.scopedHeaderRoutes) { await page.unroute(pattern, handler); } this.scopedHeaderRoutes.clear(); } } /** * Start tracing */ async startTracing(options: { screenshots?: boolean; snapshots?: boolean }): Promise { const context = this.contexts[0]; if (context) { await context.tracing.start({ screenshots: options.screenshots ?? true, snapshots: options.snapshots ?? true, }); } } /** * Stop tracing and save */ async stopTracing(path: string): Promise { const context = this.contexts[0]; if (context) { await context.tracing.stop({ path }); } } /** * Save storage state (cookies, localStorage, etc.) */ async saveStorageState(path: string): Promise { const context = this.contexts[0]; if (context) { await context.storageState({ path }); } } /** * Get all pages */ getPages(): Page[] { return this.pages; } /** * Get current page index */ getActiveIndex(): number { return this.activePageIndex; } /** * Get the current browser instance */ getBrowser(): Browser | null { return this.browser; } /** * Check if an existing CDP connection is still alive * by verifying we can access browser contexts and that at least one has pages */ private isCdpConnectionAlive(): boolean { if (!this.browser) return false; try { const contexts = this.browser.contexts(); if (contexts.length === 0) return false; return contexts.some((context) => context.pages().length > 0); } catch { return false; } } /** * Check if CDP connection needs to be re-established */ private needsCdpReconnect(cdpEndpoint: string): boolean { if (!this.browser?.isConnected()) return true; if (this.cdpEndpoint !== cdpEndpoint) return true; if (!this.isCdpConnectionAlive()) return true; return false; } /** * Close a Browserbase session via API */ private async closeBrowserbaseSession(sessionId: string, apiKey: string): Promise { await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}`, { method: 'DELETE', headers: { 'X-BB-API-Key': apiKey, }, }); } /** * Close a Browser Use session via API */ private async closeBrowserUseSession(sessionId: string, apiKey: string): Promise { const response = await fetch(`https://api.browser-use.com/api/v2/browsers/${sessionId}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json', 'X-Browser-Use-API-Key': apiKey, }, body: JSON.stringify({ action: 'stop' }), }); if (!response.ok) { throw new Error(`Failed to close Browser Use session: ${response.statusText}`); } } /** * Close a Kernel session via API */ private async closeKernelSession(sessionId: string, apiKey: string): Promise { const response = await fetch(`https://api.onkernel.com/browsers/${sessionId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, }, }); if (!response.ok) { throw new Error(`Failed to close Kernel session: ${response.statusText}`); } } /** * Connect to Browserbase remote browser via CDP. * Requires BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID environment variables. */ private async connectToBrowserbase(): Promise { const browserbaseApiKey = process.env.BROWSERBASE_API_KEY; const browserbaseProjectId = process.env.BROWSERBASE_PROJECT_ID; if (!browserbaseApiKey || !browserbaseProjectId) { throw new Error( 'BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID are required when using browserbase as a provider' ); } const response = await fetch('https://api.browserbase.com/v1/sessions', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-BB-API-Key': browserbaseApiKey, }, body: JSON.stringify({ projectId: browserbaseProjectId, }), }); if (!response.ok) { throw new Error(`Failed to create Browserbase session: ${response.statusText}`); } const session = (await response.json()) as { id: string; connectUrl: string }; const browser = await chromium.connectOverCDP(session.connectUrl).catch(() => { throw new Error('Failed to connect to Browserbase session via CDP'); }); try { const contexts = browser.contexts(); if (contexts.length === 0) { throw new Error('No browser context found in Browserbase session'); } const context = contexts[0]; const pages = context.pages(); const page = pages[0] ?? (await context.newPage()); this.browserbaseSessionId = session.id; this.browserbaseApiKey = browserbaseApiKey; this.browser = browser; context.setDefaultTimeout(10000); this.contexts.push(context); this.setupContextTracking(context); this.pages.push(page); this.activePageIndex = 0; this.setupPageTracking(page); } catch (error) { await this.closeBrowserbaseSession(session.id, browserbaseApiKey).catch((sessionError) => { console.error('Failed to close Browserbase session during cleanup:', sessionError); }); throw error; } } /** * Find or create a Kernel profile by name. * Returns the profile object if successful. */ private async findOrCreateKernelProfile( profileName: string, apiKey: string ): Promise<{ name: string }> { // First, try to get the existing profile const getResponse = await fetch( `https://api.onkernel.com/profiles/${encodeURIComponent(profileName)}`, { method: 'GET', headers: { Authorization: `Bearer ${apiKey}`, }, } ); if (getResponse.ok) { // Profile exists, return it return { name: profileName }; } if (getResponse.status !== 404) { throw new Error(`Failed to check Kernel profile: ${getResponse.statusText}`); } // Profile doesn't exist, create it const createResponse = await fetch('https://api.onkernel.com/profiles', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify({ name: profileName }), }); if (!createResponse.ok) { throw new Error(`Failed to create Kernel profile: ${createResponse.statusText}`); } return { name: profileName }; } /** * Connect to Kernel remote browser via CDP. * Requires KERNEL_API_KEY environment variable. */ private async connectToKernel(): Promise { const kernelApiKey = process.env.KERNEL_API_KEY; if (!kernelApiKey) { throw new Error('KERNEL_API_KEY is required when using kernel as a provider'); } // Find or create profile if KERNEL_PROFILE_NAME is set const profileName = process.env.KERNEL_PROFILE_NAME; let profileConfig: { profile: { name: string; save_changes: boolean } } | undefined; if (profileName) { await this.findOrCreateKernelProfile(profileName, kernelApiKey); profileConfig = { profile: { name: profileName, save_changes: true, // Save cookies/state back to the profile when session ends }, }; } const response = await fetch('https://api.onkernel.com/browsers', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${kernelApiKey}`, }, body: JSON.stringify({ // Kernel browsers are headful by default with stealth mode available // The user can configure these via environment variables if needed headless: process.env.KERNEL_HEADLESS?.toLowerCase() === 'true', stealth: process.env.KERNEL_STEALTH?.toLowerCase() !== 'false', // Default to stealth mode timeout_seconds: parseInt(process.env.KERNEL_TIMEOUT_SECONDS || '300', 10), // Load and save to a profile if specified ...profileConfig, }), }); if (!response.ok) { throw new Error(`Failed to create Kernel session: ${response.statusText}`); } let session: { session_id: string; cdp_ws_url: string }; try { session = (await response.json()) as { session_id: string; cdp_ws_url: string }; } catch (error) { throw new Error( `Failed to parse Kernel session response: ${error instanceof Error ? error.message : String(error)}` ); } if (!session.session_id || !session.cdp_ws_url) { throw new Error( `Invalid Kernel session response: missing ${!session.session_id ? 'session_id' : 'cdp_ws_url'}` ); } const browser = await chromium.connectOverCDP(session.cdp_ws_url).catch(() => { throw new Error('Failed to connect to Kernel session via CDP'); }); try { const contexts = browser.contexts(); let context: BrowserContext; let page: Page; // Kernel browsers launch with a default context and page if (contexts.length === 0) { context = await browser.newContext(); page = await context.newPage(); } else { context = contexts[0]; const pages = context.pages(); page = pages[0] ?? (await context.newPage()); } this.kernelSessionId = session.session_id; this.kernelApiKey = kernelApiKey; this.browser = browser; context.setDefaultTimeout(60000); this.contexts.push(context); this.pages.push(page); this.activePageIndex = 0; this.setupPageTracking(page); this.setupContextTracking(context); } catch (error) { await this.closeKernelSession(session.session_id, kernelApiKey).catch((sessionError) => { console.error('Failed to close Kernel session during cleanup:', sessionError); }); throw error; } } /** * Connect to Browser Use remote browser via CDP. * Requires BROWSER_USE_API_KEY environment variable. */ private async connectToBrowserUse(): Promise { const browserUseApiKey = process.env.BROWSER_USE_API_KEY; if (!browserUseApiKey) { throw new Error('BROWSER_USE_API_KEY is required when using browseruse as a provider'); } const response = await fetch('https://api.browser-use.com/api/v2/browsers', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Browser-Use-API-Key': browserUseApiKey, }, body: JSON.stringify({}), }); if (!response.ok) { throw new Error(`Failed to create Browser Use session: ${response.statusText}`); } let session: { id: string; cdpUrl: string }; try { session = (await response.json()) as { id: string; cdpUrl: string }; } catch (error) { throw new Error( `Failed to parse Browser Use session response: ${error instanceof Error ? error.message : String(error)}` ); } if (!session.id || !session.cdpUrl) { throw new Error( `Invalid Browser Use session response: missing ${!session.id ? 'id' : 'cdpUrl'}` ); } const browser = await chromium.connectOverCDP(session.cdpUrl).catch(() => { throw new Error('Failed to connect to Browser Use session via CDP'); }); try { const contexts = browser.contexts(); let context: BrowserContext; let page: Page; if (contexts.length === 0) { context = await browser.newContext(); page = await context.newPage(); } else { context = contexts[0]; const pages = context.pages(); page = pages[0] ?? (await context.newPage()); } this.browserUseSessionId = session.id; this.browserUseApiKey = browserUseApiKey; this.browser = browser; context.setDefaultTimeout(60000); this.contexts.push(context); this.pages.push(page); this.activePageIndex = 0; this.setupPageTracking(page); this.setupContextTracking(context); } catch (error) { await this.closeBrowserUseSession(session.id, browserUseApiKey).catch((sessionError) => { console.error('Failed to close Browser Use session during cleanup:', sessionError); }); throw error; } } /** * Launch the browser with the specified options * If already launched, this is a no-op (browser stays open) */ async launch(options: LaunchCommand): Promise { // Determine CDP endpoint: prefer cdpUrl over cdpPort for flexibility const cdpEndpoint = options.cdpUrl ?? (options.cdpPort ? String(options.cdpPort) : undefined); const hasExtensions = !!options.extensions?.length; const hasProfile = !!options.profile; const hasStorageState = !!options.storageState; if (hasExtensions && cdpEndpoint) { throw new Error('Extensions cannot be used with CDP connection'); } if (hasProfile && cdpEndpoint) { throw new Error('Profile cannot be used with CDP connection'); } if (hasStorageState && hasProfile) { throw new Error( 'Storage state cannot be used with profile (profile is already persistent storage)' ); } if (hasStorageState && hasExtensions) { throw new Error( 'Storage state cannot be used with extensions (extensions require persistent context)' ); } if (this.isLaunched()) { const needsRelaunch = (!cdpEndpoint && this.cdpEndpoint !== null) || (!!cdpEndpoint && this.needsCdpReconnect(cdpEndpoint)); if (needsRelaunch) { await this.close(); } else { return; } } if (cdpEndpoint) { await this.connectViaCDP(cdpEndpoint); return; } // Cloud browser providers require explicit opt-in via -p flag or AGENT_BROWSER_PROVIDER env var // -p flag takes precedence over env var const provider = options.provider ?? process.env.AGENT_BROWSER_PROVIDER; if (provider === 'browserbase') { await this.connectToBrowserbase(); return; } if (provider === 'browseruse') { await this.connectToBrowserUse(); return; } // Kernel: requires explicit opt-in via -p kernel flag or AGENT_BROWSER_PROVIDER=kernel if (provider === 'kernel') { await this.connectToKernel(); return; } const browserType = options.browser ?? 'chromium'; if (hasExtensions && browserType !== 'chromium') { throw new Error('Extensions are only supported in Chromium'); } const launcher = browserType === 'firefox' ? firefox : browserType === 'webkit' ? webkit : chromium; const viewport = options.viewport ?? { width: 1280, height: 720 }; let context: BrowserContext; if (hasExtensions) { // Extensions require persistent context in a temp directory const extPaths = options.extensions!.join(','); const session = process.env.AGENT_BROWSER_SESSION || 'default'; // Combine extension args with custom args const extArgs = [`--disable-extensions-except=${extPaths}`, `--load-extension=${extPaths}`]; const allArgs = options.args ? [...extArgs, ...options.args] : extArgs; context = await launcher.launchPersistentContext( path.join(os.tmpdir(), `agent-browser-ext-${session}`), { headless: false, executablePath: options.executablePath, args: allArgs, viewport, extraHTTPHeaders: options.headers, userAgent: options.userAgent, ...(options.proxy && { proxy: options.proxy }), ignoreHTTPSErrors: options.ignoreHTTPSErrors ?? false, } ); this.isPersistentContext = true; } else if (hasProfile) { // Profile uses persistent context for durable cookies/storage // Expand ~ to home directory since it won't be shell-expanded const profilePath = options.profile!.replace(/^~\//, os.homedir() + '/'); context = await launcher.launchPersistentContext(profilePath, { headless: options.headless ?? true, executablePath: options.executablePath, viewport, extraHTTPHeaders: options.headers, }); this.isPersistentContext = true; } else { // Regular ephemeral browser this.browser = await launcher.launch({ headless: options.headless ?? true, executablePath: options.executablePath, args: options.args, }); this.cdpEndpoint = null; context = await this.browser.newContext({ viewport, extraHTTPHeaders: options.headers, userAgent: options.userAgent, ...(options.proxy && { proxy: options.proxy }), ignoreHTTPSErrors: options.ignoreHTTPSErrors ?? false, ...(options.storageState && { storageState: options.storageState }), }); } context.setDefaultTimeout(60000); this.contexts.push(context); this.setupContextTracking(context); const page = context.pages()[0] ?? (await context.newPage()); // Only add if not already tracked (setupContextTracking may have already added it via 'page' event) if (!this.pages.includes(page)) { this.pages.push(page); this.setupPageTracking(page); } this.activePageIndex = this.pages.length > 0 ? this.pages.length - 1 : 0; } /** * Connect to a running browser via CDP (Chrome DevTools Protocol) * @param cdpEndpoint Either a port number (as string) or a full WebSocket URL (ws:// or wss://) */ private async connectViaCDP(cdpEndpoint: string | undefined): Promise { if (!cdpEndpoint) { throw new Error('CDP endpoint is required for CDP connection'); } // Determine the connection URL: // - If it starts with ws://, wss://, http://, or https://, use it directly // - If it's a numeric string (e.g., "9222"), treat as port for localhost // - Otherwise, treat it as a port number for localhost let cdpUrl: string; if ( cdpEndpoint.startsWith('ws://') || cdpEndpoint.startsWith('wss://') || cdpEndpoint.startsWith('http://') || cdpEndpoint.startsWith('https://') ) { cdpUrl = cdpEndpoint; } else if (/^\d+$/.test(cdpEndpoint)) { // Numeric string - treat as port number (handles JSON serialization quirks) cdpUrl = `http://localhost:${cdpEndpoint}`; } else { // Unknown format - still try as port for backward compatibility cdpUrl = `http://localhost:${cdpEndpoint}`; } const browser = await chromium.connectOverCDP(cdpUrl).catch(() => { throw new Error( `Failed to connect via CDP to ${cdpUrl}. ` + (cdpUrl.includes('localhost') ? `Make sure the app is running with --remote-debugging-port=${cdpEndpoint}` : 'Make sure the remote browser is accessible and the URL is correct.') ); }); // Validate and set up state, cleaning up browser connection if anything fails try { const contexts = browser.contexts(); if (contexts.length === 0) { throw new Error('No browser context found. Make sure the app has an open window.'); } // Filter out pages with empty URLs, which can cause Playwright to hang const allPages = contexts.flatMap((context) => context.pages()).filter((page) => page.url()); if (allPages.length === 0) { throw new Error('No page found. Make sure the app has loaded content.'); } // All validation passed - commit state this.browser = browser; this.cdpEndpoint = cdpEndpoint; for (const context of contexts) { context.setDefaultTimeout(10000); this.contexts.push(context); this.setupContextTracking(context); } for (const page of allPages) { this.pages.push(page); this.setupPageTracking(page); } this.activePageIndex = 0; } catch (error) { // Clean up browser connection if validation or setup failed await browser.close().catch(() => {}); throw error; } } /** * Set up console, error, and close tracking for a page */ private setupPageTracking(page: Page): void { page.on('console', (msg) => { this.consoleMessages.push({ type: msg.type(), text: msg.text(), timestamp: Date.now(), }); }); page.on('pageerror', (error) => { this.pageErrors.push({ message: error.message, timestamp: Date.now(), }); }); page.on('close', () => { const index = this.pages.indexOf(page); if (index !== -1) { this.pages.splice(index, 1); if (this.activePageIndex >= this.pages.length) { this.activePageIndex = Math.max(0, this.pages.length - 1); } } }); } /** * Set up tracking for new pages in a context (for CDP connections and popups/new tabs) * This handles pages created externally (e.g., via target="_blank" links) */ private setupContextTracking(context: BrowserContext): void { context.on('page', (page) => { // Only add if not already tracked (avoids duplicates when newTab() creates pages) if (!this.pages.includes(page)) { this.pages.push(page); this.setupPageTracking(page); } }); } /** * Create a new tab in the current context */ async newTab(): Promise<{ index: number; total: number }> { if (!this.browser || this.contexts.length === 0) { throw new Error('Browser not launched'); } // Invalidate CDP session since we're switching to a new page await this.invalidateCDPSession(); const context = this.contexts[0]; // Use first context for tabs const page = await context.newPage(); // Only add if not already tracked (setupContextTracking may have already added it via 'page' event) if (!this.pages.includes(page)) { this.pages.push(page); this.setupPageTracking(page); } this.activePageIndex = this.pages.length - 1; return { index: this.activePageIndex, total: this.pages.length }; } /** * Create a new window (new context) */ async newWindow(viewport?: { width: number; height: number; }): Promise<{ index: number; total: number }> { if (!this.browser) { throw new Error('Browser not launched'); } const context = await this.browser.newContext({ viewport: viewport ?? { width: 1280, height: 720 }, }); context.setDefaultTimeout(60000); this.contexts.push(context); this.setupContextTracking(context); const page = await context.newPage(); // Only add if not already tracked (setupContextTracking may have already added it via 'page' event) if (!this.pages.includes(page)) { this.pages.push(page); this.setupPageTracking(page); } this.activePageIndex = this.pages.length - 1; return { index: this.activePageIndex, total: this.pages.length }; } /** * Invalidate the current CDP session (must be called before switching pages) * This ensures screencast and input injection work correctly after tab switch */ private async invalidateCDPSession(): Promise { // Stop screencast if active (it's tied to the current page's CDP session) if (this.screencastActive) { await this.stopScreencast(); } // Detach and clear the CDP session if (this.cdpSession) { await this.cdpSession.detach().catch(() => {}); this.cdpSession = null; } } /** * Switch to a specific tab/page by index */ async switchTo(index: number): Promise<{ index: number; url: string; title: string }> { if (index < 0 || index >= this.pages.length) { throw new Error(`Invalid tab index: ${index}. Available: 0-${this.pages.length - 1}`); } // Invalidate CDP session before switching (it's page-specific) if (index !== this.activePageIndex) { await this.invalidateCDPSession(); } this.activePageIndex = index; const page = this.pages[index]; return { index: this.activePageIndex, url: page.url(), title: '', // Title requires async, will be fetched separately }; } /** * Close a specific tab/page */ async closeTab(index?: number): Promise<{ closed: number; remaining: number }> { const targetIndex = index ?? this.activePageIndex; if (targetIndex < 0 || targetIndex >= this.pages.length) { throw new Error(`Invalid tab index: ${targetIndex}`); } if (this.pages.length === 1) { throw new Error('Cannot close the last tab. Use "close" to close the browser.'); } // If closing the active tab, invalidate CDP session first if (targetIndex === this.activePageIndex) { await this.invalidateCDPSession(); } const page = this.pages[targetIndex]; await page.close(); this.pages.splice(targetIndex, 1); // Adjust active index if needed if (this.activePageIndex >= this.pages.length) { this.activePageIndex = this.pages.length - 1; } else if (this.activePageIndex > targetIndex) { this.activePageIndex--; } return { closed: targetIndex, remaining: this.pages.length }; } /** * List all tabs with their info */ async listTabs(): Promise> { const tabs = await Promise.all( this.pages.map(async (page, index) => ({ index, url: page.url(), title: await page.title().catch(() => ''), active: index === this.activePageIndex, })) ); return tabs; } /** * Get or create a CDP session for the current page * Only works with Chromium-based browsers */ async getCDPSession(): Promise { if (this.cdpSession) { return this.cdpSession; } const page = this.getPage(); const context = page.context(); // Create a new CDP session attached to the page this.cdpSession = await context.newCDPSession(page); return this.cdpSession; } /** * Check if screencast is currently active */ isScreencasting(): boolean { return this.screencastActive; } /** * Start screencast - streams viewport frames via CDP * @param callback Function called for each frame * @param options Screencast options */ async startScreencast( callback: (frame: ScreencastFrame) => void, options?: ScreencastOptions ): Promise { if (this.screencastActive) { throw new Error('Screencast already active'); } const cdp = await this.getCDPSession(); this.frameCallback = callback; this.screencastActive = true; // Create and store the frame handler so we can remove it later this.screencastFrameHandler = async (params: any) => { const frame: ScreencastFrame = { data: params.data, metadata: params.metadata, sessionId: params.sessionId, }; // Acknowledge the frame to receive the next one await cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId }); // Call the callback with the frame if (this.frameCallback) { this.frameCallback(frame); } }; // Listen for screencast frames cdp.on('Page.screencastFrame', this.screencastFrameHandler); // Start the screencast await cdp.send('Page.startScreencast', { format: options?.format ?? 'jpeg', quality: options?.quality ?? 80, maxWidth: options?.maxWidth ?? 1280, maxHeight: options?.maxHeight ?? 720, everyNthFrame: options?.everyNthFrame ?? 1, }); } /** * Stop screencast */ async stopScreencast(): Promise { if (!this.screencastActive) { return; } try { const cdp = await this.getCDPSession(); await cdp.send('Page.stopScreencast'); // Remove the event listener to prevent accumulation if (this.screencastFrameHandler) { cdp.off('Page.screencastFrame', this.screencastFrameHandler); } } catch { // Ignore errors when stopping } this.screencastActive = false; this.frameCallback = null; this.screencastFrameHandler = null; } /** * Inject a mouse event via CDP */ async injectMouseEvent(params: { type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel'; x: number; y: number; button?: 'left' | 'right' | 'middle' | 'none'; clickCount?: number; deltaX?: number; deltaY?: number; modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift }): Promise { const cdp = await this.getCDPSession(); const cdpButton = params.button === 'left' ? 'left' : params.button === 'right' ? 'right' : params.button === 'middle' ? 'middle' : 'none'; await cdp.send('Input.dispatchMouseEvent', { type: params.type, x: params.x, y: params.y, button: cdpButton, clickCount: params.clickCount ?? 1, deltaX: params.deltaX ?? 0, deltaY: params.deltaY ?? 0, modifiers: params.modifiers ?? 0, }); } /** * Inject a keyboard event via CDP */ async injectKeyboardEvent(params: { type: 'keyDown' | 'keyUp' | 'char'; key?: string; code?: string; text?: string; modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift }): Promise { const cdp = await this.getCDPSession(); await cdp.send('Input.dispatchKeyEvent', { type: params.type, key: params.key, code: params.code, text: params.text, modifiers: params.modifiers ?? 0, }); } /** * Inject touch event via CDP (for mobile emulation) */ async injectTouchEvent(params: { type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel'; touchPoints: Array<{ x: number; y: number; id?: number }>; modifiers?: number; }): Promise { const cdp = await this.getCDPSession(); await cdp.send('Input.dispatchTouchEvent', { type: params.type, touchPoints: params.touchPoints.map((tp, i) => ({ x: tp.x, y: tp.y, id: tp.id ?? i, })), modifiers: params.modifiers ?? 0, }); } /** * Check if video recording is currently active */ isRecording(): boolean { return this.recordingContext !== null; } /** * Start recording to a video file using Playwright's native video recording. * Creates a fresh browser context with video recording enabled. * Automatically captures current URL and transfers cookies/storage if no URL provided. * * @param outputPath - Path to the output video file (will be .webm) * @param url - Optional URL to navigate to (defaults to current page URL) */ async startRecording(outputPath: string, url?: string): Promise { if (this.recordingContext) { throw new Error( "Recording already in progress. Run 'record stop' first, or use 'record restart' to stop and start a new recording." ); } if (!this.browser) { throw new Error('Browser not launched. Call launch first.'); } // Check if output file already exists if (existsSync(outputPath)) { throw new Error(`Output file already exists: ${outputPath}`); } // Validate output path is .webm (Playwright native format) if (!outputPath.endsWith('.webm')) { throw new Error( 'Playwright native recording only supports WebM format. Please use a .webm extension.' ); } // Auto-capture current URL if none provided const currentPage = this.pages.length > 0 ? this.pages[this.activePageIndex] : null; const currentContext = this.contexts.length > 0 ? this.contexts[0] : null; if (!url && currentPage) { const currentUrl = currentPage.url(); if (currentUrl && currentUrl !== 'about:blank') { url = currentUrl; } } // Capture state from current context (cookies + storage) let storageState: | { cookies: Array<{ name: string; value: string; domain: string; path: string; expires: number; httpOnly: boolean; secure: boolean; sameSite: 'Strict' | 'Lax' | 'None'; }>; origins: Array<{ origin: string; localStorage: Array<{ name: string; value: string }>; }>; } | undefined; if (currentContext) { try { storageState = await currentContext.storageState(); } catch { // Ignore errors - context might be closed or invalid } } // Create a temp directory for video recording const session = process.env.AGENT_BROWSER_SESSION || 'default'; this.recordingTempDir = path.join( os.tmpdir(), `agent-browser-recording-${session}-${Date.now()}` ); mkdirSync(this.recordingTempDir, { recursive: true }); this.recordingOutputPath = outputPath; // Create a new context with video recording enabled and restored state const viewport = { width: 1280, height: 720 }; this.recordingContext = await this.browser.newContext({ viewport, recordVideo: { dir: this.recordingTempDir, size: viewport, }, storageState, }); this.recordingContext.setDefaultTimeout(10000); // Create a page in the recording context this.recordingPage = await this.recordingContext.newPage(); // Add the recording context and page to our managed lists this.contexts.push(this.recordingContext); this.pages.push(this.recordingPage); this.activePageIndex = this.pages.length - 1; // Set up page tracking this.setupPageTracking(this.recordingPage); // Invalidate CDP session since we switched pages await this.invalidateCDPSession(); // Navigate to URL if provided or captured if (url) { await this.recordingPage.goto(url, { waitUntil: 'load' }); } } /** * Stop recording and save the video file * @returns Recording result with path */ async stopRecording(): Promise<{ path: string; frames: number; error?: string }> { if (!this.recordingContext || !this.recordingPage) { return { path: '', frames: 0, error: 'No recording in progress' }; } const outputPath = this.recordingOutputPath; try { // Get the video object before closing the page const video = this.recordingPage.video(); // Remove recording page/context from our managed lists before closing const pageIndex = this.pages.indexOf(this.recordingPage); if (pageIndex !== -1) { this.pages.splice(pageIndex, 1); } const contextIndex = this.contexts.indexOf(this.recordingContext); if (contextIndex !== -1) { this.contexts.splice(contextIndex, 1); } // Close the page to finalize the video await this.recordingPage.close(); // Save the video to the desired output path if (video) { await video.saveAs(outputPath); } // Clean up temp directory if (this.recordingTempDir) { rmSync(this.recordingTempDir, { recursive: true, force: true }); } // Close the recording context await this.recordingContext.close(); // Reset recording state this.recordingContext = null; this.recordingPage = null; this.recordingOutputPath = ''; this.recordingTempDir = ''; // Adjust active page index if (this.pages.length > 0) { this.activePageIndex = Math.min(this.activePageIndex, this.pages.length - 1); } else { this.activePageIndex = 0; } // Invalidate CDP session since we may have switched pages await this.invalidateCDPSession(); return { path: outputPath, frames: 0 }; // Playwright doesn't expose frame count } catch (error) { // Clean up temp directory on error if (this.recordingTempDir) { rmSync(this.recordingTempDir, { recursive: true, force: true }); } // Reset state on error this.recordingContext = null; this.recordingPage = null; this.recordingOutputPath = ''; this.recordingTempDir = ''; const message = error instanceof Error ? error.message : String(error); return { path: outputPath, frames: 0, error: message }; } } /** * Restart recording - stops current recording (if any) and starts a new one. * Convenience method that combines stopRecording and startRecording. * * @param outputPath - Path to the output video file (must be .webm) * @param url - Optional URL to navigate to (defaults to current page URL) * @returns Result from stopping the previous recording (if any) */ async restartRecording( outputPath: string, url?: string ): Promise<{ previousPath?: string; stopped: boolean }> { let previousPath: string | undefined; let stopped = false; // Stop current recording if active if (this.recordingContext) { const result = await this.stopRecording(); previousPath = result.path; stopped = true; } // Start new recording await this.startRecording(outputPath, url); return { previousPath, stopped }; } /** * Close the browser and clean up */ async close(): Promise { // Stop recording if active (saves video) if (this.recordingContext) { await this.stopRecording(); } // Stop screencast if active if (this.screencastActive) { await this.stopScreencast(); } // Clean up CDP session if (this.cdpSession) { await this.cdpSession.detach().catch(() => {}); this.cdpSession = null; } if (this.browserbaseSessionId && this.browserbaseApiKey) { await this.closeBrowserbaseSession(this.browserbaseSessionId, this.browserbaseApiKey).catch( (error) => { console.error('Failed to close Browserbase session:', error); } ); this.browser = null; } else if (this.browserUseSessionId && this.browserUseApiKey) { await this.closeBrowserUseSession(this.browserUseSessionId, this.browserUseApiKey).catch( (error) => { console.error('Failed to close Browser Use session:', error); } ); this.browser = null; } else if (this.kernelSessionId && this.kernelApiKey) { await this.closeKernelSession(this.kernelSessionId, this.kernelApiKey).catch((error) => { console.error('Failed to close Kernel session:', error); }); this.browser = null; } else if (this.cdpEndpoint !== null) { // CDP: only disconnect, don't close external app's pages if (this.browser) { await this.browser.close().catch(() => {}); this.browser = null; } } else { // Regular browser: close everything for (const page of this.pages) { await page.close().catch(() => {}); } for (const context of this.contexts) { await context.close().catch(() => {}); } if (this.browser) { await this.browser.close().catch(() => {}); this.browser = null; } } this.pages = []; this.contexts = []; this.cdpEndpoint = null; this.browserbaseSessionId = null; this.browserbaseApiKey = null; this.browserUseSessionId = null; this.browserUseApiKey = null; this.kernelSessionId = null; this.kernelApiKey = null; this.isPersistentContext = false; this.activePageIndex = 0; this.refMap = {}; this.lastSnapshot = ''; this.frameCallback = null; } }