diff --git a/src/actions.ts b/src/actions.ts index 1aed45a..0b1fe8c 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -449,11 +449,16 @@ async function handleScreenshot( } async function handleSnapshot( - command: Command & { action: 'snapshot' }, + command: Command & { action: 'snapshot'; interactive?: boolean; maxDepth?: number; compact?: boolean; selector?: string }, browser: BrowserManager ): Promise> { - // Use enhanced snapshot with refs - const { tree, refs } = await browser.getSnapshot(); + // Use enhanced snapshot with refs and optional filtering + const { tree, refs } = await browser.getSnapshot({ + interactive: command.interactive, + maxDepth: command.maxDepth, + compact: command.compact, + selector: command.selector, + }); // Simplify refs for output (just role and name) const simpleRefs: Record = {}; diff --git a/src/browser.ts b/src/browser.ts index a7ff4d9..03b17e4 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -62,9 +62,14 @@ export class BrowserManager { /** * Get enhanced snapshot with refs and cache the ref map */ - async getSnapshot(): Promise { + async getSnapshot(options?: { + interactive?: boolean; + maxDepth?: number; + compact?: boolean; + selector?: string; + }): Promise { const page = this.getPage(); - const snapshot = await getEnhancedSnapshot(page); + const snapshot = await getEnhancedSnapshot(page, options); this.refMap = snapshot.refs; this.lastSnapshot = snapshot.tree; return snapshot; diff --git a/src/cli-light.ts b/src/cli-light.ts index 7e0067a..c1ce29d 100644 --- a/src/cli-light.ts +++ b/src/cli-light.ts @@ -140,118 +140,149 @@ async function sendCommand(cmd: Record): Promise { cleanup(); reject(new Error('Timeout')); } - }, 15000); + }, 30000); }); } // ============================================================================ -// CLI Parsing +// Command Parsing // ============================================================================ -function parseArgs(args: string[]): { cmd: Record | null; json: boolean } { - const json = args.includes('--json'); - const cleanArgs = args.filter(a => !a.startsWith('--')); +function parseCommand(parts: string[]): Record | null { + if (parts.length === 0) return null; - if (cleanArgs.length === 0) return { cmd: null, json }; - - const command = cleanArgs[0]; - const rest = cleanArgs.slice(1); + const command = parts[0]; + const rest = parts.slice(1); const id = Math.random().toString(36).slice(2, 10); switch (command) { case 'open': case 'goto': case 'navigate': - return { cmd: { id, action: 'navigate', url: rest[0]?.startsWith('http') ? rest[0] : `https://${rest[0]}` }, json }; + return { id, action: 'navigate', url: rest[0]?.startsWith('http') ? rest[0] : `https://${rest[0]}` }; case 'click': - return { cmd: { id, action: 'click', selector: rest[0] }, json }; + return { id, action: 'click', selector: rest[0] }; case 'fill': - return { cmd: { id, action: 'fill', selector: rest[0], value: rest.slice(1).join(' ') }, json }; + return { id, action: 'fill', selector: rest[0], value: rest.slice(1).join(' ') }; case 'type': - return { cmd: { id, action: 'type', selector: rest[0], text: rest.slice(1).join(' ') }, json }; + return { id, action: 'type', selector: rest[0], text: rest.slice(1).join(' ') }; case 'hover': - return { cmd: { id, action: 'hover', selector: rest[0] }, json }; + return { id, action: 'hover', selector: rest[0] }; - case 'snapshot': - return { cmd: { id, action: 'snapshot' }, json }; + case 'snapshot': { + const opts: Record = { id, action: 'snapshot' }; + // Parse snapshot options from rest args + for (let i = 0; i < rest.length; i++) { + const arg = rest[i]; + if (arg === '-i' || arg === '--interactive') { + opts.interactive = true; + } else if (arg === '-c' || arg === '--compact') { + opts.compact = true; + } else if (arg === '--depth' || arg === '-d') { + opts.maxDepth = parseInt(rest[++i], 10); + } else if (arg === '--selector' || arg === '-s') { + opts.selector = rest[++i]; + } + } + return opts; + } case 'screenshot': - return { cmd: { id, action: 'screenshot', path: rest[0] }, json }; + return { id, action: 'screenshot', path: rest[0] }; case 'close': case 'quit': - return { cmd: { id, action: 'close' }, json }; + return { id, action: 'close' }; case 'get': - if (rest[0] === 'text') return { cmd: { id, action: 'gettext', selector: rest[1] }, json }; - if (rest[0] === 'url') return { cmd: { id, action: 'url' }, json }; - if (rest[0] === 'title') return { cmd: { id, action: 'title' }, json }; - return { cmd: null, json }; + if (rest[0] === 'text') return { id, action: 'gettext', selector: rest[1] }; + if (rest[0] === 'url') return { id, action: 'url' }; + if (rest[0] === 'title') return { id, action: 'title' }; + return null; case 'press': - return { cmd: { id, action: 'press', key: rest[0] }, json }; + return { id, action: 'press', key: rest[0] }; case 'wait': if (/^\d+$/.test(rest[0])) { - return { cmd: { id, action: 'wait', timeout: parseInt(rest[0], 10) }, json }; + return { id, action: 'wait', timeout: parseInt(rest[0], 10) }; } - return { cmd: { id, action: 'wait', selector: rest[0] }, json }; + return { id, action: 'wait', selector: rest[0] }; case 'back': - return { cmd: { id, action: 'back' }, json }; + return { id, action: 'back' }; case 'forward': - return { cmd: { id, action: 'forward' }, json }; + return { id, action: 'forward' }; case 'reload': - return { cmd: { id, action: 'reload' }, json }; + return { id, action: 'reload' }; case 'eval': - return { cmd: { id, action: 'evaluate', script: rest.join(' ') }, json }; + return { id, action: 'evaluate', script: rest.join(' ') }; default: - return { cmd: null, json }; + return null; } } +function parseBatchCommands(args: string[]): Record[] { + const commands: Record[] = []; + + // Each argument after 'batch' is a command string + for (const arg of args) { + // Split the command string into parts + const parts = arg.match(/(?:[^\s"]+|"[^"]*")+/g) || []; + const cleanParts = parts.map(p => p.replace(/^"|"$/g, '')); + + const cmd = parseCommand(cleanParts); + if (cmd) { + commands.push(cmd); + } + } + + return commands; +} + // ============================================================================ // Output Formatting // ============================================================================ -function printResponse(response: Response, json: boolean): void { - if (json) { - console.log(JSON.stringify(response)); - return; - } - +function formatResponse(response: Response): string { if (!response.success) { - console.error('\x1b[31m✗ Error:\x1b[0m', response.error); - process.exit(1); + return `\x1b[31m✗ Error:\x1b[0m ${response.error}`; } const data = response.data as Record; if (data?.url && data?.title) { - console.log('\x1b[32m✓\x1b[0m', '\x1b[1m' + data.title + '\x1b[0m'); - console.log('\x1b[2m ' + data.url + '\x1b[0m'); + return `\x1b[32m✓\x1b[0m \x1b[1m${data.title}\x1b[0m\n\x1b[2m ${data.url}\x1b[0m`; } else if (data?.snapshot) { - console.log(data.snapshot); + return String(data.snapshot); } else if (data?.text !== undefined) { - console.log(data.text); + return String(data.text); } else if (data?.url) { - console.log(data.url); + return String(data.url); } else if (data?.title) { - console.log(data.title); + return String(data.title); } else if (data?.result !== undefined) { - console.log(typeof data.result === 'object' ? JSON.stringify(data.result, null, 2) : data.result); + return typeof data.result === 'object' ? JSON.stringify(data.result, null, 2) : String(data.result); } else if (data?.closed) { - console.log('\x1b[32m✓\x1b[0m Browser closed'); + return '\x1b[32m✓\x1b[0m Browser closed'; } else { - console.log('\x1b[32m✓\x1b[0m Done'); + return '\x1b[32m✓\x1b[0m Done'; + } +} + +function printResponse(response: Response, json: boolean): void { + if (json) { + console.log(JSON.stringify(response)); + } else { + console.log(formatResponse(response)); } } @@ -259,14 +290,12 @@ function printResponse(response: Response, json: boolean): void { // Main // ============================================================================ -async function main(): Promise { - const args = process.argv.slice(2); - - if (args.length === 0 || args.includes('--help') || args.includes('-h')) { - console.log(` +const HELP = ` agent-browser - fast browser automation CLI -Usage: agent-browser [args] [--json] +Usage: + agent-browser [args] [--json] + agent-browser batch ... [--json] Commands: open Navigate to URL @@ -274,7 +303,7 @@ Commands: fill Fill input type Type text hover Hover element - snapshot Get accessibility tree with refs + snapshot [options] Get accessibility tree with refs screenshot [path] Take screenshot get text Get text content get url Get current URL @@ -284,6 +313,16 @@ Commands: eval Evaluate JavaScript close Close browser +Snapshot Options: + -i, --interactive Only show interactive elements (buttons, links, inputs) + -c, --compact Remove empty structural elements + -d, --depth Limit tree depth (e.g., --depth 3) + -s, --selector Scope snapshot to CSS selector + +Batch Mode: + batch ... Execute multiple commands in sequence + Each command is a quoted string + Options: --json Output JSON (for AI agents) @@ -292,14 +331,110 @@ Examples: agent-browser snapshot agent-browser click @e2 agent-browser fill @e3 "hello" -`); + + # Batch mode - execute multiple commands efficiently + agent-browser batch "open example.com" "snapshot" "click a" + agent-browser batch "open google.com" "snapshot" "get title" --json +`; + +async function runBatch(commands: Record[], json: boolean): Promise { + const results: Response[] = []; + let hasError = false; + + for (const cmd of commands) { + try { + const response = await sendCommand(cmd); + results.push(response); + + if (!json) { + // Print each result as we go for non-JSON mode + console.log(`\x1b[36m[${cmd.action}]\x1b[0m`); + console.log(formatResponse(response)); + console.log(); + } + + if (!response.success) { + hasError = true; + break; // Stop on first error + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const errorResponse: Response = { + id: String(cmd.id), + success: false, + error: message, + }; + results.push(errorResponse); + hasError = true; + + if (!json) { + console.log(`\x1b[36m[${cmd.action}]\x1b[0m`); + console.log(`\x1b[31m✗ Error:\x1b[0m ${message}`); + } + break; + } + } + + if (json) { + console.log(JSON.stringify({ + success: !hasError, + results, + completed: results.length, + total: commands.length, + })); + } else { + console.log(`\x1b[2m─────────────────────────────────────\x1b[0m`); + console.log(`Completed ${results.length}/${commands.length} commands`); + } + + process.exit(hasError ? 1 : 0); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const json = args.includes('--json'); + const cleanArgs = args.filter(a => !a.startsWith('--')); + + if (cleanArgs.length === 0 || args.includes('--help') || args.includes('-h')) { + console.log(HELP); process.exit(0); } - const { cmd, json } = parseArgs(args); + // Check for batch mode + if (cleanArgs[0] === 'batch') { + const batchArgs = cleanArgs.slice(1); + if (batchArgs.length === 0) { + console.error('\x1b[31mBatch mode requires at least one command\x1b[0m'); + console.log('\nExample: agent-browser batch "open example.com" "snapshot"'); + process.exit(1); + } + + const commands = parseBatchCommands(batchArgs); + if (commands.length === 0) { + console.error('\x1b[31mNo valid commands found\x1b[0m'); + process.exit(1); + } + + try { + await ensureDaemon(); + await runBatch(commands, json); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (json) { + console.log(JSON.stringify({ success: false, error: message })); + } else { + console.error('\x1b[31m✗ Error:\x1b[0m', message); + } + process.exit(1); + } + return; + } + + // Single command mode + const cmd = parseCommand(cleanArgs); if (!cmd) { - console.error('\x1b[31mUnknown command\x1b[0m'); + console.error('\x1b[31mUnknown command:\x1b[0m', cleanArgs[0]); process.exit(1); } @@ -307,7 +442,7 @@ Examples: await ensureDaemon(); const response = await sendCommand(cmd); printResponse(response, json); - process.exit(0); + process.exit(response.success ? 0 : 1); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (json) { diff --git a/src/protocol.ts b/src/protocol.ts index 5ee6323..a2be79f 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -601,6 +601,10 @@ const screenshotSchema = baseCommandSchema.extend({ const snapshotSchema = baseCommandSchema.extend({ action: z.literal('snapshot'), + interactive: z.boolean().optional(), + maxDepth: z.number().nonnegative().optional(), + compact: z.boolean().optional(), + selector: z.string().optional(), }); const evaluateSchema = baseCommandSchema.extend({ diff --git a/src/snapshot.ts b/src/snapshot.ts index 3f0a4f1..3f517b8 100644 --- a/src/snapshot.ts +++ b/src/snapshot.ts @@ -11,9 +11,10 @@ * - textbox "Email" [ref=e3] * * Usage: - * agent-browser snapshot # Get snapshot with refs - * agent-browser click @e2 # Click element by ref - * agent-browser fill @e3 "test" # Fill element by ref + * agent-browser snapshot # Full snapshot + * agent-browser snapshot -i # Interactive elements only + * agent-browser snapshot --depth 3 # Limit depth + * agent-browser click @e2 # Click element by ref */ import type { Page, Locator } from 'playwright-core'; @@ -31,6 +32,17 @@ export interface EnhancedSnapshot { refs: RefMap; } +export interface SnapshotOptions { + /** Only include interactive elements (buttons, links, inputs, etc.) */ + interactive?: boolean; + /** Maximum depth of tree to include (0 = root only) */ + maxDepth?: number; + /** Remove structural elements without meaningful content */ + compact?: boolean; + /** CSS selector to scope the snapshot */ + selector?: string; +} + // Counter for generating refs let refCounter = 0; @@ -87,6 +99,30 @@ const CONTENT_ROLES = new Set([ 'navigation', ]); +/** + * Roles that are purely structural (can be filtered in compact mode) + */ +const STRUCTURAL_ROLES = new Set([ + 'generic', + 'group', + 'list', + 'table', + 'row', + 'rowgroup', + 'grid', + 'treegrid', + 'menu', + 'menubar', + 'toolbar', + 'tablist', + 'tree', + 'directory', + 'document', + 'application', + 'presentation', + 'none', +]); + /** * Build a selector string for storing in ref map */ @@ -99,95 +135,209 @@ function buildSelector(role: string, name?: string): string { } /** - * Get enhanced snapshot with refs - * - * Uses ariaSnapshot() which returns ARIA tree, then parses and adds refs + * Get enhanced snapshot with refs and optional filtering */ -export async function getEnhancedSnapshot(page: Page): Promise { +export async function getEnhancedSnapshot( + page: Page, + options: SnapshotOptions = {} +): Promise { resetRefs(); const refs: RefMap = {}; // Get ARIA snapshot from Playwright - const ariaTree = await page.locator(':root').ariaSnapshot(); + const locator = options.selector ? page.locator(options.selector) : page.locator(':root'); + const ariaTree = await locator.ariaSnapshot(); if (!ariaTree) { return { - tree: '(empty page)', + tree: '(empty)', refs: {}, }; } - // Parse the ARIA tree and add refs to interactive elements - const enhancedTree = addRefsToAriaTree(ariaTree, refs); + // Parse and enhance the ARIA tree + const enhancedTree = processAriaTree(ariaTree, refs, options); return { tree: enhancedTree, refs }; } /** - * Parse ARIA snapshot and add refs to interactive elements - * - * Input format from ariaSnapshot(): - * - document: - * - heading "Example Domain" [level=1] - * - paragraph: This is text - * - link "More info": - * - /url: https://... + * Process ARIA snapshot: add refs and apply filters */ -function addRefsToAriaTree(ariaTree: string, refs: RefMap): string { +function processAriaTree(ariaTree: string, refs: RefMap, options: SnapshotOptions): string { const lines = ariaTree.split('\n'); - const enhancedLines: string[] = []; + const result: string[] = []; - for (const line of lines) { - // Match lines like: - // - button "Submit" - // - heading "Title" [level=1] - // - link "Click me": - // - textbox "Email" - const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/); - - if (match) { - const [, prefix, role, name, suffix] = match; + // For interactive-only mode, we collect just interactive elements + if (options.interactive) { + for (const line of lines) { + const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/); + if (!match) continue; + + const [, , role, name, suffix] = match; const roleLower = role.toLowerCase(); - - // Skip metadata lines (like /url:) - if (role.startsWith('/')) { - enhancedLines.push(line); - continue; - } - - // Add ref for interactive or named content elements - const isInteractive = INTERACTIVE_ROLES.has(roleLower); - const isNamedContent = CONTENT_ROLES.has(roleLower) && name; - - if (isInteractive || isNamedContent) { + + if (INTERACTIVE_ROLES.has(roleLower)) { const ref = nextRef(); - - // Store ref data for later locator creation refs[ref] = { selector: buildSelector(roleLower, name), role: roleLower, name, }; - - // Insert ref tag before any trailing content (like [level=1] or :) - const refTag = `[ref=${ref}]`; - - // Build the enhanced line - let enhanced = `${prefix}${role}`; + + let enhanced = `- ${role}`; if (name) enhanced += ` "${name}"`; - enhanced += ` ${refTag}`; - if (suffix) enhanced += suffix; - - enhancedLines.push(enhanced); - } else { - enhancedLines.push(line); + enhanced += ` [ref=${ref}]`; + if (suffix && suffix.includes('[')) enhanced += suffix; + + result.push(enhanced); } - } else { - enhancedLines.push(line); + } + return result.join('\n') || '(no interactive elements)'; + } + + // Normal processing with depth/compact filters + for (const line of lines) { + const processed = processLine(line, refs, options); + if (processed !== null) { + result.push(processed); } } - return enhancedLines.join('\n'); + // If compact mode, remove empty structural elements + if (options.compact) { + return compactTree(result.join('\n')); + } + + return result.join('\n'); +} + +/** + * Get indentation level (number of spaces / 2) + */ +function getIndentLevel(line: string): number { + const match = line.match(/^(\s*)/); + return match ? Math.floor(match[1].length / 2) : 0; +} + +/** + * Process a single line: add ref if needed, filter if requested + */ +function processLine( + line: string, + refs: RefMap, + options: SnapshotOptions +): string | null { + const depth = getIndentLevel(line); + + // Check max depth + if (options.maxDepth !== undefined && depth > options.maxDepth) { + return null; + } + + // Match lines like: + // - button "Submit" + // - heading "Title" [level=1] + // - link "Click me": + const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/); + + if (!match) { + // Metadata lines (like /url:) or text content + if (options.interactive) { + // In interactive mode, only keep metadata under interactive elements + return null; + } + return line; + } + + const [, prefix, role, name, suffix] = match; + const roleLower = role.toLowerCase(); + + // Skip metadata lines (like /url:) + if (role.startsWith('/')) { + return line; + } + + const isInteractive = INTERACTIVE_ROLES.has(roleLower); + const isContent = CONTENT_ROLES.has(roleLower); + const isStructural = STRUCTURAL_ROLES.has(roleLower); + + // In interactive-only mode, filter non-interactive elements + if (options.interactive && !isInteractive) { + return null; + } + + // In compact mode, skip unnamed structural elements + if (options.compact && isStructural && !name) { + return null; + } + + // Add ref for interactive or named content elements + const shouldHaveRef = isInteractive || (isContent && name); + + if (shouldHaveRef) { + const ref = nextRef(); + + refs[ref] = { + selector: buildSelector(roleLower, name), + role: roleLower, + name, + }; + + // Build enhanced line with ref + let enhanced = `${prefix}${role}`; + if (name) enhanced += ` "${name}"`; + enhanced += ` [ref=${ref}]`; + if (suffix) enhanced += suffix; + + return enhanced; + } + + return line; +} + +/** + * Remove empty structural branches in compact mode + */ +function compactTree(tree: string): string { + const lines = tree.split('\n'); + const result: string[] = []; + + // Simple pass: keep lines that have content or refs + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Always keep lines with refs + if (line.includes('[ref=')) { + result.push(line); + continue; + } + + // Keep lines with text content (after :) + if (line.includes(':') && !line.endsWith(':')) { + result.push(line); + continue; + } + + // Check if this structural element has children with refs + const currentIndent = getIndentLevel(line); + let hasRelevantChildren = false; + + for (let j = i + 1; j < lines.length; j++) { + const childIndent = getIndentLevel(lines[j]); + if (childIndent <= currentIndent) break; + if (lines[j].includes('[ref=')) { + hasRelevantChildren = true; + break; + } + } + + if (hasRelevantChildren) { + result.push(line); + } + } + + return result.join('\n'); } /** @@ -205,3 +355,26 @@ export function parseRef(arg: string): string | null { } return null; } + +/** + * Get snapshot statistics + */ +export function getSnapshotStats(tree: string, refs: RefMap): { + lines: number; + chars: number; + tokens: number; + refs: number; + interactive: number; +} { + const interactive = Object.values(refs).filter(r => + INTERACTIVE_ROLES.has(r.role) + ).length; + + return { + lines: tree.split('\n').length, + chars: tree.length, + tokens: Math.ceil(tree.length / 4), + refs: Object.keys(refs).length, + interactive, + }; +}