smart snapshots

This commit is contained in:
Chris Tate
2026-01-11 02:44:27 -06:00
parent a144735451
commit 7443a4a4c5
5 changed files with 445 additions and 123 deletions
+8 -3
View File
@@ -449,11 +449,16 @@ async function handleScreenshot(
} }
async function handleSnapshot( async function handleSnapshot(
command: Command & { action: 'snapshot' }, command: Command & { action: 'snapshot'; interactive?: boolean; maxDepth?: number; compact?: boolean; selector?: string },
browser: BrowserManager browser: BrowserManager
): Promise<Response<SnapshotData>> { ): Promise<Response<SnapshotData>> {
// Use enhanced snapshot with refs // Use enhanced snapshot with refs and optional filtering
const { tree, refs } = await browser.getSnapshot(); const { tree, refs } = await browser.getSnapshot({
interactive: command.interactive,
maxDepth: command.maxDepth,
compact: command.compact,
selector: command.selector,
});
// Simplify refs for output (just role and name) // Simplify refs for output (just role and name)
const simpleRefs: Record<string, { role: string; name?: string }> = {}; const simpleRefs: Record<string, { role: string; name?: string }> = {};
+7 -2
View File
@@ -62,9 +62,14 @@ export class BrowserManager {
/** /**
* Get enhanced snapshot with refs and cache the ref map * Get enhanced snapshot with refs and cache the ref map
*/ */
async getSnapshot(): Promise<EnhancedSnapshot> { async getSnapshot(options?: {
interactive?: boolean;
maxDepth?: number;
compact?: boolean;
selector?: string;
}): Promise<EnhancedSnapshot> {
const page = this.getPage(); const page = this.getPage();
const snapshot = await getEnhancedSnapshot(page); const snapshot = await getEnhancedSnapshot(page, options);
this.refMap = snapshot.refs; this.refMap = snapshot.refs;
this.lastSnapshot = snapshot.tree; this.lastSnapshot = snapshot.tree;
return snapshot; return snapshot;
+193 -58
View File
@@ -140,118 +140,149 @@ async function sendCommand(cmd: Record<string, unknown>): Promise<Response> {
cleanup(); cleanup();
reject(new Error('Timeout')); reject(new Error('Timeout'));
} }
}, 15000); }, 30000);
}); });
} }
// ============================================================================ // ============================================================================
// CLI Parsing // Command Parsing
// ============================================================================ // ============================================================================
function parseArgs(args: string[]): { cmd: Record<string, unknown> | null; json: boolean } { function parseCommand(parts: string[]): Record<string, unknown> | null {
const json = args.includes('--json'); if (parts.length === 0) return null;
const cleanArgs = args.filter(a => !a.startsWith('--'));
if (cleanArgs.length === 0) return { cmd: null, json }; const command = parts[0];
const rest = parts.slice(1);
const command = cleanArgs[0];
const rest = cleanArgs.slice(1);
const id = Math.random().toString(36).slice(2, 10); const id = Math.random().toString(36).slice(2, 10);
switch (command) { switch (command) {
case 'open': case 'open':
case 'goto': case 'goto':
case 'navigate': case 'navigate':
return { cmd: { id, action: 'navigate', url: rest[0]?.startsWith('http') ? rest[0] : `https://${rest[0]}` }, json }; return { id, action: 'navigate', url: rest[0]?.startsWith('http') ? rest[0] : `https://${rest[0]}` };
case 'click': case 'click':
return { cmd: { id, action: 'click', selector: rest[0] }, json }; return { id, action: 'click', selector: rest[0] };
case 'fill': case 'fill':
return { cmd: { id, action: 'fill', selector: rest[0], value: rest.slice(1).join(' ') }, json }; return { id, action: 'fill', selector: rest[0], value: rest.slice(1).join(' ') };
case 'type': case 'type':
return { cmd: { id, action: 'type', selector: rest[0], text: rest.slice(1).join(' ') }, json }; return { id, action: 'type', selector: rest[0], text: rest.slice(1).join(' ') };
case 'hover': case 'hover':
return { cmd: { id, action: 'hover', selector: rest[0] }, json }; return { id, action: 'hover', selector: rest[0] };
case 'snapshot': case 'snapshot': {
return { cmd: { id, action: 'snapshot' }, json }; const opts: Record<string, unknown> = { id, action: 'snapshot' };
// Parse snapshot options from rest args
for (let i = 0; i < rest.length; i++) {
const arg = rest[i];
if (arg === '-i' || arg === '--interactive') {
opts.interactive = true;
} else if (arg === '-c' || arg === '--compact') {
opts.compact = true;
} else if (arg === '--depth' || arg === '-d') {
opts.maxDepth = parseInt(rest[++i], 10);
} else if (arg === '--selector' || arg === '-s') {
opts.selector = rest[++i];
}
}
return opts;
}
case 'screenshot': case 'screenshot':
return { cmd: { id, action: 'screenshot', path: rest[0] }, json }; return { id, action: 'screenshot', path: rest[0] };
case 'close': case 'close':
case 'quit': case 'quit':
return { cmd: { id, action: 'close' }, json }; return { id, action: 'close' };
case 'get': case 'get':
if (rest[0] === 'text') return { cmd: { id, action: 'gettext', selector: rest[1] }, json }; if (rest[0] === 'text') return { id, action: 'gettext', selector: rest[1] };
if (rest[0] === 'url') return { cmd: { id, action: 'url' }, json }; if (rest[0] === 'url') return { id, action: 'url' };
if (rest[0] === 'title') return { cmd: { id, action: 'title' }, json }; if (rest[0] === 'title') return { id, action: 'title' };
return { cmd: null, json }; return null;
case 'press': case 'press':
return { cmd: { id, action: 'press', key: rest[0] }, json }; return { id, action: 'press', key: rest[0] };
case 'wait': case 'wait':
if (/^\d+$/.test(rest[0])) { if (/^\d+$/.test(rest[0])) {
return { cmd: { id, action: 'wait', timeout: parseInt(rest[0], 10) }, json }; return { id, action: 'wait', timeout: parseInt(rest[0], 10) };
} }
return { cmd: { id, action: 'wait', selector: rest[0] }, json }; return { id, action: 'wait', selector: rest[0] };
case 'back': case 'back':
return { cmd: { id, action: 'back' }, json }; return { id, action: 'back' };
case 'forward': case 'forward':
return { cmd: { id, action: 'forward' }, json }; return { id, action: 'forward' };
case 'reload': case 'reload':
return { cmd: { id, action: 'reload' }, json }; return { id, action: 'reload' };
case 'eval': case 'eval':
return { cmd: { id, action: 'evaluate', script: rest.join(' ') }, json }; return { id, action: 'evaluate', script: rest.join(' ') };
default: default:
return { cmd: null, json }; return null;
} }
} }
function parseBatchCommands(args: string[]): Record<string, unknown>[] {
const commands: Record<string, unknown>[] = [];
// Each argument after 'batch' is a command string
for (const arg of args) {
// Split the command string into parts
const parts = arg.match(/(?:[^\s"]+|"[^"]*")+/g) || [];
const cleanParts = parts.map(p => p.replace(/^"|"$/g, ''));
const cmd = parseCommand(cleanParts);
if (cmd) {
commands.push(cmd);
}
}
return commands;
}
// ============================================================================ // ============================================================================
// Output Formatting // Output Formatting
// ============================================================================ // ============================================================================
function printResponse(response: Response, json: boolean): void { function formatResponse(response: Response): string {
if (json) {
console.log(JSON.stringify(response));
return;
}
if (!response.success) { if (!response.success) {
console.error('\x1b[31m✗ Error:\x1b[0m', response.error); return `\x1b[31m✗ Error:\x1b[0m ${response.error}`;
process.exit(1);
} }
const data = response.data as Record<string, unknown>; const data = response.data as Record<string, unknown>;
if (data?.url && data?.title) { if (data?.url && data?.title) {
console.log('\x1b[32m✓\x1b[0m', '\x1b[1m' + data.title + '\x1b[0m'); return `\x1b[32m✓\x1b[0m \x1b[1m${data.title}\x1b[0m\n\x1b[2m ${data.url}\x1b[0m`;
console.log('\x1b[2m ' + data.url + '\x1b[0m');
} else if (data?.snapshot) { } else if (data?.snapshot) {
console.log(data.snapshot); return String(data.snapshot);
} else if (data?.text !== undefined) { } else if (data?.text !== undefined) {
console.log(data.text); return String(data.text);
} else if (data?.url) { } else if (data?.url) {
console.log(data.url); return String(data.url);
} else if (data?.title) { } else if (data?.title) {
console.log(data.title); return String(data.title);
} else if (data?.result !== undefined) { } else if (data?.result !== undefined) {
console.log(typeof data.result === 'object' ? JSON.stringify(data.result, null, 2) : data.result); return typeof data.result === 'object' ? JSON.stringify(data.result, null, 2) : String(data.result);
} else if (data?.closed) { } else if (data?.closed) {
console.log('\x1b[32m✓\x1b[0m Browser closed'); return '\x1b[32m✓\x1b[0m Browser closed';
} else { } else {
console.log('\x1b[32m✓\x1b[0m Done'); return '\x1b[32m✓\x1b[0m Done';
}
}
function printResponse(response: Response, json: boolean): void {
if (json) {
console.log(JSON.stringify(response));
} else {
console.log(formatResponse(response));
} }
} }
@@ -259,14 +290,12 @@ function printResponse(response: Response, json: boolean): void {
// Main // Main
// ============================================================================ // ============================================================================
async function main(): Promise<void> { const HELP = `
const args = process.argv.slice(2);
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
console.log(`
agent-browser - fast browser automation CLI agent-browser - fast browser automation CLI
Usage: agent-browser <command> [args] [--json] Usage:
agent-browser <command> [args] [--json]
agent-browser batch <cmd1> <cmd2> ... [--json]
Commands: Commands:
open <url> Navigate to URL open <url> Navigate to URL
@@ -274,7 +303,7 @@ Commands:
fill <sel> <text> Fill input fill <sel> <text> Fill input
type <sel> <text> Type text type <sel> <text> Type text
hover <sel> Hover element hover <sel> Hover element
snapshot Get accessibility tree with refs snapshot [options] Get accessibility tree with refs
screenshot [path] Take screenshot screenshot [path] Take screenshot
get text <sel> Get text content get text <sel> Get text content
get url Get current URL get url Get current URL
@@ -284,6 +313,16 @@ Commands:
eval <js> Evaluate JavaScript eval <js> Evaluate JavaScript
close Close browser close Close browser
Snapshot Options:
-i, --interactive Only show interactive elements (buttons, links, inputs)
-c, --compact Remove empty structural elements
-d, --depth <n> Limit tree depth (e.g., --depth 3)
-s, --selector <sel> Scope snapshot to CSS selector
Batch Mode:
batch <cmd1> <cmd2> ... Execute multiple commands in sequence
Each command is a quoted string
Options: Options:
--json Output JSON (for AI agents) --json Output JSON (for AI agents)
@@ -292,14 +331,110 @@ Examples:
agent-browser snapshot agent-browser snapshot
agent-browser click @e2 agent-browser click @e2
agent-browser fill @e3 "hello" agent-browser fill @e3 "hello"
`);
# Batch mode - execute multiple commands efficiently
agent-browser batch "open example.com" "snapshot" "click a"
agent-browser batch "open google.com" "snapshot" "get title" --json
`;
async function runBatch(commands: Record<string, unknown>[], json: boolean): Promise<void> {
const results: Response[] = [];
let hasError = false;
for (const cmd of commands) {
try {
const response = await sendCommand(cmd);
results.push(response);
if (!json) {
// Print each result as we go for non-JSON mode
console.log(`\x1b[36m[${cmd.action}]\x1b[0m`);
console.log(formatResponse(response));
console.log();
}
if (!response.success) {
hasError = true;
break; // Stop on first error
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
const errorResponse: Response = {
id: String(cmd.id),
success: false,
error: message,
};
results.push(errorResponse);
hasError = true;
if (!json) {
console.log(`\x1b[36m[${cmd.action}]\x1b[0m`);
console.log(`\x1b[31m✗ Error:\x1b[0m ${message}`);
}
break;
}
}
if (json) {
console.log(JSON.stringify({
success: !hasError,
results,
completed: results.length,
total: commands.length,
}));
} else {
console.log(`\x1b[2m─────────────────────────────────────\x1b[0m`);
console.log(`Completed ${results.length}/${commands.length} commands`);
}
process.exit(hasError ? 1 : 0);
}
async function main(): Promise<void> {
const args = process.argv.slice(2);
const json = args.includes('--json');
const cleanArgs = args.filter(a => !a.startsWith('--'));
if (cleanArgs.length === 0 || args.includes('--help') || args.includes('-h')) {
console.log(HELP);
process.exit(0); process.exit(0);
} }
const { cmd, json } = parseArgs(args); // Check for batch mode
if (cleanArgs[0] === 'batch') {
const batchArgs = cleanArgs.slice(1);
if (batchArgs.length === 0) {
console.error('\x1b[31mBatch mode requires at least one command\x1b[0m');
console.log('\nExample: agent-browser batch "open example.com" "snapshot"');
process.exit(1);
}
const commands = parseBatchCommands(batchArgs);
if (commands.length === 0) {
console.error('\x1b[31mNo valid commands found\x1b[0m');
process.exit(1);
}
try {
await ensureDaemon();
await runBatch(commands, json);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
if (json) {
console.log(JSON.stringify({ success: false, error: message }));
} else {
console.error('\x1b[31m✗ Error:\x1b[0m', message);
}
process.exit(1);
}
return;
}
// Single command mode
const cmd = parseCommand(cleanArgs);
if (!cmd) { if (!cmd) {
console.error('\x1b[31mUnknown command\x1b[0m'); console.error('\x1b[31mUnknown command:\x1b[0m', cleanArgs[0]);
process.exit(1); process.exit(1);
} }
@@ -307,7 +442,7 @@ Examples:
await ensureDaemon(); await ensureDaemon();
const response = await sendCommand(cmd); const response = await sendCommand(cmd);
printResponse(response, json); printResponse(response, json);
process.exit(0); process.exit(response.success ? 0 : 1);
} catch (err) { } catch (err) {
const message = err instanceof Error ? err.message : String(err); const message = err instanceof Error ? err.message : String(err);
if (json) { if (json) {
+4
View File
@@ -601,6 +601,10 @@ const screenshotSchema = baseCommandSchema.extend({
const snapshotSchema = baseCommandSchema.extend({ const snapshotSchema = baseCommandSchema.extend({
action: z.literal('snapshot'), action: z.literal('snapshot'),
interactive: z.boolean().optional(),
maxDepth: z.number().nonnegative().optional(),
compact: z.boolean().optional(),
selector: z.string().optional(),
}); });
const evaluateSchema = baseCommandSchema.extend({ const evaluateSchema = baseCommandSchema.extend({
+229 -56
View File
@@ -11,9 +11,10 @@
* - textbox "Email" [ref=e3] * - textbox "Email" [ref=e3]
* *
* Usage: * Usage:
* agent-browser snapshot # Get snapshot with refs * agent-browser snapshot # Full snapshot
* agent-browser click @e2 # Click element by ref * agent-browser snapshot -i # Interactive elements only
* agent-browser fill @e3 "test" # Fill element by ref * agent-browser snapshot --depth 3 # Limit depth
* agent-browser click @e2 # Click element by ref
*/ */
import type { Page, Locator } from 'playwright-core'; import type { Page, Locator } from 'playwright-core';
@@ -31,6 +32,17 @@ export interface EnhancedSnapshot {
refs: RefMap; refs: RefMap;
} }
export interface SnapshotOptions {
/** Only include interactive elements (buttons, links, inputs, etc.) */
interactive?: boolean;
/** Maximum depth of tree to include (0 = root only) */
maxDepth?: number;
/** Remove structural elements without meaningful content */
compact?: boolean;
/** CSS selector to scope the snapshot */
selector?: string;
}
// Counter for generating refs // Counter for generating refs
let refCounter = 0; let refCounter = 0;
@@ -87,6 +99,30 @@ const CONTENT_ROLES = new Set([
'navigation', 'navigation',
]); ]);
/**
* Roles that are purely structural (can be filtered in compact mode)
*/
const STRUCTURAL_ROLES = new Set([
'generic',
'group',
'list',
'table',
'row',
'rowgroup',
'grid',
'treegrid',
'menu',
'menubar',
'toolbar',
'tablist',
'tree',
'directory',
'document',
'application',
'presentation',
'none',
]);
/** /**
* Build a selector string for storing in ref map * Build a selector string for storing in ref map
*/ */
@@ -99,95 +135,209 @@ function buildSelector(role: string, name?: string): string {
} }
/** /**
* Get enhanced snapshot with refs * Get enhanced snapshot with refs and optional filtering
*
* Uses ariaSnapshot() which returns ARIA tree, then parses and adds refs
*/ */
export async function getEnhancedSnapshot(page: Page): Promise<EnhancedSnapshot> { export async function getEnhancedSnapshot(
page: Page,
options: SnapshotOptions = {}
): Promise<EnhancedSnapshot> {
resetRefs(); resetRefs();
const refs: RefMap = {}; const refs: RefMap = {};
// Get ARIA snapshot from Playwright // Get ARIA snapshot from Playwright
const ariaTree = await page.locator(':root').ariaSnapshot(); const locator = options.selector ? page.locator(options.selector) : page.locator(':root');
const ariaTree = await locator.ariaSnapshot();
if (!ariaTree) { if (!ariaTree) {
return { return {
tree: '(empty page)', tree: '(empty)',
refs: {}, refs: {},
}; };
} }
// Parse the ARIA tree and add refs to interactive elements // Parse and enhance the ARIA tree
const enhancedTree = addRefsToAriaTree(ariaTree, refs); const enhancedTree = processAriaTree(ariaTree, refs, options);
return { tree: enhancedTree, refs }; return { tree: enhancedTree, refs };
} }
/** /**
* Parse ARIA snapshot and add refs to interactive elements * Process ARIA snapshot: add refs and apply filters
*
* Input format from ariaSnapshot():
* - document:
* - heading "Example Domain" [level=1]
* - paragraph: This is text
* - link "More info":
* - /url: https://...
*/ */
function addRefsToAriaTree(ariaTree: string, refs: RefMap): string { function processAriaTree(ariaTree: string, refs: RefMap, options: SnapshotOptions): string {
const lines = ariaTree.split('\n'); const lines = ariaTree.split('\n');
const enhancedLines: string[] = []; const result: string[] = [];
for (const line of lines) { // For interactive-only mode, we collect just interactive elements
// Match lines like: if (options.interactive) {
// - button "Submit" for (const line of lines) {
// - heading "Title" [level=1] const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
// - link "Click me": if (!match) continue;
// - textbox "Email"
const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
if (match) { const [, , role, name, suffix] = match;
const [, prefix, role, name, suffix] = match;
const roleLower = role.toLowerCase(); const roleLower = role.toLowerCase();
// Skip metadata lines (like /url:) if (INTERACTIVE_ROLES.has(roleLower)) {
if (role.startsWith('/')) {
enhancedLines.push(line);
continue;
}
// Add ref for interactive or named content elements
const isInteractive = INTERACTIVE_ROLES.has(roleLower);
const isNamedContent = CONTENT_ROLES.has(roleLower) && name;
if (isInteractive || isNamedContent) {
const ref = nextRef(); const ref = nextRef();
// Store ref data for later locator creation
refs[ref] = { refs[ref] = {
selector: buildSelector(roleLower, name), selector: buildSelector(roleLower, name),
role: roleLower, role: roleLower,
name, name,
}; };
// Insert ref tag before any trailing content (like [level=1] or :) let enhanced = `- ${role}`;
const refTag = `[ref=${ref}]`;
// Build the enhanced line
let enhanced = `${prefix}${role}`;
if (name) enhanced += ` "${name}"`; if (name) enhanced += ` "${name}"`;
enhanced += ` ${refTag}`; enhanced += ` [ref=${ref}]`;
if (suffix) enhanced += suffix; if (suffix && suffix.includes('[')) enhanced += suffix;
enhancedLines.push(enhanced); result.push(enhanced);
} else {
enhancedLines.push(line);
} }
} else { }
enhancedLines.push(line); return result.join('\n') || '(no interactive elements)';
}
// Normal processing with depth/compact filters
for (const line of lines) {
const processed = processLine(line, refs, options);
if (processed !== null) {
result.push(processed);
} }
} }
return enhancedLines.join('\n'); // If compact mode, remove empty structural elements
if (options.compact) {
return compactTree(result.join('\n'));
}
return result.join('\n');
}
/**
* Get indentation level (number of spaces / 2)
*/
function getIndentLevel(line: string): number {
const match = line.match(/^(\s*)/);
return match ? Math.floor(match[1].length / 2) : 0;
}
/**
* Process a single line: add ref if needed, filter if requested
*/
function processLine(
line: string,
refs: RefMap,
options: SnapshotOptions
): string | null {
const depth = getIndentLevel(line);
// Check max depth
if (options.maxDepth !== undefined && depth > options.maxDepth) {
return null;
}
// Match lines like:
// - button "Submit"
// - heading "Title" [level=1]
// - link "Click me":
const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
if (!match) {
// Metadata lines (like /url:) or text content
if (options.interactive) {
// In interactive mode, only keep metadata under interactive elements
return null;
}
return line;
}
const [, prefix, role, name, suffix] = match;
const roleLower = role.toLowerCase();
// Skip metadata lines (like /url:)
if (role.startsWith('/')) {
return line;
}
const isInteractive = INTERACTIVE_ROLES.has(roleLower);
const isContent = CONTENT_ROLES.has(roleLower);
const isStructural = STRUCTURAL_ROLES.has(roleLower);
// In interactive-only mode, filter non-interactive elements
if (options.interactive && !isInteractive) {
return null;
}
// In compact mode, skip unnamed structural elements
if (options.compact && isStructural && !name) {
return null;
}
// Add ref for interactive or named content elements
const shouldHaveRef = isInteractive || (isContent && name);
if (shouldHaveRef) {
const ref = nextRef();
refs[ref] = {
selector: buildSelector(roleLower, name),
role: roleLower,
name,
};
// Build enhanced line with ref
let enhanced = `${prefix}${role}`;
if (name) enhanced += ` "${name}"`;
enhanced += ` [ref=${ref}]`;
if (suffix) enhanced += suffix;
return enhanced;
}
return line;
}
/**
* Remove empty structural branches in compact mode
*/
function compactTree(tree: string): string {
const lines = tree.split('\n');
const result: string[] = [];
// Simple pass: keep lines that have content or refs
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
// Always keep lines with refs
if (line.includes('[ref=')) {
result.push(line);
continue;
}
// Keep lines with text content (after :)
if (line.includes(':') && !line.endsWith(':')) {
result.push(line);
continue;
}
// Check if this structural element has children with refs
const currentIndent = getIndentLevel(line);
let hasRelevantChildren = false;
for (let j = i + 1; j < lines.length; j++) {
const childIndent = getIndentLevel(lines[j]);
if (childIndent <= currentIndent) break;
if (lines[j].includes('[ref=')) {
hasRelevantChildren = true;
break;
}
}
if (hasRelevantChildren) {
result.push(line);
}
}
return result.join('\n');
} }
/** /**
@@ -205,3 +355,26 @@ export function parseRef(arg: string): string | null {
} }
return null; return null;
} }
/**
* Get snapshot statistics
*/
export function getSnapshotStats(tree: string, refs: RefMap): {
lines: number;
chars: number;
tokens: number;
refs: number;
interactive: number;
} {
const interactive = Object.values(refs).filter(r =>
INTERACTIVE_ROLES.has(r.role)
).length;
return {
lines: tree.split('\n').length,
chars: tree.length,
tokens: Math.ceil(tree.length / 4),
refs: Object.keys(refs).length,
interactive,
};
}