diff --git a/README.md b/README.md index 844a843..cb670f9 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,22 @@ pnpm build ```bash agent-browser open example.com -agent-browser click "#submit" -agent-browser fill "#email" "test@example.com" -agent-browser get text "h1" +agent-browser snapshot # Get accessibility tree with refs +agent-browser click @e2 # Click by ref from snapshot +agent-browser fill @e3 "test@example.com" # Fill by ref +agent-browser get text @e1 # Get text by ref agent-browser screenshot page.png agent-browser close ``` +### Traditional Selectors (also supported) + +```bash +agent-browser click "#submit" +agent-browser fill "#email" "test@example.com" +agent-browser find role button click --name "Submit" +``` + ## Commands ### Core Commands @@ -48,7 +57,7 @@ agent-browser upload # Upload files agent-browser download [path] # Wait for download agent-browser screenshot [path] # Take screenshot (--full for full page) agent-browser pdf # Save as PDF -agent-browser snapshot # Accessibility tree (best for AI) +agent-browser snapshot # Accessibility tree with refs (best for AI) agent-browser eval # Run JavaScript agent-browser close # Close browser ``` @@ -244,19 +253,49 @@ agent-browser session list ## Selectors +### Refs (Recommended for AI) + +Refs provide deterministic element selection from snapshots: + +```bash +# 1. Get snapshot with refs +agent-browser snapshot +# Output: +# - heading "Example Domain" [ref=e1] [level=1] +# - button "Submit" [ref=e2] +# - textbox "Email" [ref=e3] +# - link "Learn more" [ref=e4] + +# 2. Use refs to interact +agent-browser click @e2 # Click the button +agent-browser fill @e3 "test@example.com" # Fill the textbox +agent-browser get text @e1 # Get heading text +agent-browser hover @e4 # Hover the link +``` + +**Why use refs?** +- **Deterministic**: Ref points to exact element from snapshot +- **Fast**: No DOM re-query needed +- **AI-friendly**: Snapshot + ref workflow is optimal for LLMs + +### CSS Selectors + ```bash -# CSS agent-browser click "#id" agent-browser click ".class" agent-browser click "div > button" +``` -# Text +### Text & XPath + +```bash agent-browser click "text=Submit" - -# XPath agent-browser click "xpath=//button" +``` -# Semantic (recommended) +### Semantic Locators + +```bash agent-browser find role button click --name "Submit" agent-browser find label "Email" fill "test@test.com" ``` @@ -267,8 +306,26 @@ Use `--json` for machine-readable output: ```bash agent-browser snapshot --json -agent-browser get text "h1" --json -agent-browser is visible ".modal" --json +# Returns: {"success":true,"data":{"snapshot":"...","refs":{"e1":{"role":"heading","name":"Title"},...}}} + +agent-browser get text @e1 --json +agent-browser is visible @e2 --json +``` + +### Optimal AI Workflow + +```bash +# 1. Navigate and get snapshot +agent-browser open example.com +agent-browser snapshot --json # AI parses tree and refs + +# 2. AI identifies target refs from snapshot +# 3. Execute actions using refs +agent-browser click @e2 +agent-browser fill @e3 "input text" + +# 4. Get new snapshot if page changed +agent-browser snapshot --json ``` ## License diff --git a/package.json b/package.json index f08c757..f7c58ae 100644 --- a/package.json +++ b/package.json @@ -29,12 +29,12 @@ "author": "", "license": "Apache-2.0", "dependencies": { - "playwright-core": "^1.40.0", + "playwright-core": "^1.57.0", "zod": "^3.22.4" }, "devDependencies": { "@types/node": "^20.10.0", - "playwright": "^1.40.0", + "playwright": "^1.57.0", "prettier": "^3.7.4", "tsx": "^4.6.0", "typescript": "^5.3.0", diff --git a/src/actions.ts b/src/actions.ts index 34a4915..1aed45a 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -108,6 +108,7 @@ import { successResponse, errorResponse } from './protocol.js'; // Snapshot response type interface SnapshotData { snapshot: string; + refs?: Record; } /** @@ -380,8 +381,10 @@ async function handleNavigate( } async function handleClick(command: ClickCommand, browser: BrowserManager): Promise { - const page = browser.getPage(); - await page.click(command.selector, { + // Support both refs (@e1) and regular selectors + const locator = browser.getLocator(command.selector); + + await locator.click({ button: command.button, clickCount: command.clickCount, delay: command.delay, @@ -391,13 +394,13 @@ async function handleClick(command: ClickCommand, browser: BrowserManager): Prom } async function handleType(command: TypeCommand, browser: BrowserManager): Promise { - const page = browser.getPage(); + const locator = browser.getLocator(command.selector); if (command.clear) { - await page.fill(command.selector, ''); + await locator.fill(''); } - await page.type(command.selector, command.text, { + await locator.pressSequentially(command.text, { delay: command.delay, }); @@ -449,12 +452,18 @@ async function handleSnapshot( command: Command & { action: 'snapshot' }, browser: BrowserManager ): Promise> { - const page = browser.getPage(); - // Use ariaSnapshot which returns a string representation of the accessibility tree - const snapshot = await page.locator(':root').ariaSnapshot(); + // Use enhanced snapshot with refs + const { tree, refs } = await browser.getSnapshot(); + + // Simplify refs for output (just role and name) + const simpleRefs: Record = {}; + for (const [ref, data] of Object.entries(refs)) { + simpleRefs[ref] = { role: data.role, name: data.name }; + } return successResponse(command.id, { - snapshot: snapshot ?? 'Empty page', + snapshot: tree || 'Empty page', + refs: Object.keys(simpleRefs).length > 0 ? simpleRefs : undefined, }); } @@ -533,17 +542,17 @@ async function handleScroll(command: ScrollCommand, browser: BrowserManager): Pr } async function handleSelect(command: SelectCommand, browser: BrowserManager): Promise { - const page = browser.getPage(); + const locator = browser.getLocator(command.selector); const values = Array.isArray(command.values) ? command.values : [command.values]; - await page.selectOption(command.selector, values); + await locator.selectOption(values); return successResponse(command.id, { selected: values }); } async function handleHover(command: HoverCommand, browser: BrowserManager): Promise { - const page = browser.getPage(); - await page.hover(command.selector); + const locator = browser.getLocator(command.selector); + await locator.hover(); return successResponse(command.id, { hovered: true }); } @@ -622,27 +631,27 @@ async function handleWindowNew( // New handlers for enhanced Playwright parity async function handleFill(command: FillCommand, browser: BrowserManager): Promise { - const frame = browser.getFrame(); - await frame.fill(command.selector, command.value); + const locator = browser.getLocator(command.selector); + await locator.fill(command.value); return successResponse(command.id, { filled: true }); } async function handleCheck(command: CheckCommand, browser: BrowserManager): Promise { - const frame = browser.getFrame(); - await frame.check(command.selector); + const locator = browser.getLocator(command.selector); + await locator.check(); return successResponse(command.id, { checked: true }); } async function handleUncheck(command: UncheckCommand, browser: BrowserManager): Promise { - const frame = browser.getFrame(); - await frame.uncheck(command.selector); + const locator = browser.getLocator(command.selector); + await locator.uncheck(); return successResponse(command.id, { unchecked: true }); } async function handleUpload(command: UploadCommand, browser: BrowserManager): Promise { - const frame = browser.getFrame(); + const locator = browser.getLocator(command.selector); const files = Array.isArray(command.files) ? command.files : [command.files]; - await frame.setInputFiles(command.selector, files); + await locator.setInputFiles(files); return successResponse(command.id, { uploaded: files }); } @@ -650,14 +659,14 @@ async function handleDoubleClick( command: DoubleClickCommand, browser: BrowserManager ): Promise { - const frame = browser.getFrame(); - await frame.dblclick(command.selector); + const locator = browser.getLocator(command.selector); + await locator.dblclick(); return successResponse(command.id, { clicked: true }); } async function handleFocus(command: FocusCommand, browser: BrowserManager): Promise { - const frame = browser.getFrame(); - await frame.focus(command.selector); + const locator = browser.getLocator(command.selector); + await locator.focus(); return successResponse(command.id, { focused: true }); } @@ -1017,14 +1026,14 @@ async function handleGetAttribute( command: GetAttributeCommand, browser: BrowserManager ): Promise { - const page = browser.getPage(); - const value = await page.getAttribute(command.selector, command.attribute); + const locator = browser.getLocator(command.selector); + const value = await locator.getAttribute(command.attribute); return successResponse(command.id, { attribute: command.attribute, value }); } async function handleGetText(command: GetTextCommand, browser: BrowserManager): Promise { - const page = browser.getPage(); - const text = await page.textContent(command.selector); + const locator = browser.getLocator(command.selector); + const text = await locator.textContent(); return successResponse(command.id, { text }); } @@ -1032,8 +1041,8 @@ async function handleIsVisible( command: IsVisibleCommand, browser: BrowserManager ): Promise { - const page = browser.getPage(); - const visible = await page.isVisible(command.selector); + const locator = browser.getLocator(command.selector); + const visible = await locator.isVisible(); return successResponse(command.id, { visible }); } @@ -1041,8 +1050,8 @@ async function handleIsEnabled( command: IsEnabledCommand, browser: BrowserManager ): Promise { - const page = browser.getPage(); - const enabled = await page.isEnabled(command.selector); + const locator = browser.getLocator(command.selector); + const enabled = await locator.isEnabled(); return successResponse(command.id, { enabled }); } @@ -1050,8 +1059,8 @@ async function handleIsChecked( command: IsCheckedCommand, browser: BrowserManager ): Promise { - const page = browser.getPage(); - const checked = await page.isChecked(command.selector); + const locator = browser.getLocator(command.selector); + const checked = await locator.isChecked(); return successResponse(command.id, { checked }); } diff --git a/src/browser.ts b/src/browser.ts index 1429426..a7ff4d9 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -10,8 +10,10 @@ import { type Dialog, type Request, type Route, + type Locator, } from 'playwright-core'; import type { LaunchCommand } from './types.js'; +import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js'; interface TrackedRequest { url: string; @@ -47,6 +49,8 @@ export class BrowserManager { private consoleMessages: ConsoleMessage[] = []; private pageErrors: PageError[] = []; private isRecordingHar: boolean = false; + private refMap: RefMap = {}; + private lastSnapshot: string = ''; /** * Check if browser is launched @@ -55,6 +59,65 @@ export class BrowserManager { return this.browser !== null; } + /** + * Get enhanced snapshot with refs and cache the ref map + */ + async getSnapshot(): Promise { + const page = this.getPage(); + const snapshot = await getEnhancedSnapshot(page); + this.refMap = snapshot.refs; + this.lastSnapshot = snapshot.tree; + return snapshot; + } + + /** + * Get the cached ref map from last snapshot + */ + getRefMap(): RefMap { + return this.refMap; + } + + /** + * Get a locator from a ref (e.g., "e1", "@e1", "ref=e1") + * Returns null if ref doesn't exist or is invalid + */ + getLocatorFromRef(refArg: string): Locator | null { + const ref = parseRef(refArg); + if (!ref) return null; + + const refData = this.refMap[ref]; + if (!refData) return null; + + const page = this.getPage(); + + // Parse the selector and create locator + if (refData.name) { + return page.getByRole(refData.role as any, { name: refData.name }); + } else { + return page.getByRole(refData.role as any); + } + } + + /** + * Check if a selector looks like a ref + */ + isRef(selector: string): boolean { + return parseRef(selector) !== null; + } + + /** + * Get locator - supports both refs and regular selectors + */ + getLocator(selectorOrRef: string): Locator { + // Check if it's a ref first + const locator = this.getLocatorFromRef(selectorOrRef); + if (locator) return locator; + + // Otherwise treat as regular selector + const page = this.getPage(); + return page.locator(selectorOrRef); + } + /** * Get the current active page, throws if not launched */ @@ -612,5 +675,7 @@ export class BrowserManager { } this.activePageIndex = 0; + this.refMap = {}; + this.lastSnapshot = ''; } } diff --git a/src/index.ts b/src/index.ts index fd81658..c1b06ce 100644 --- a/src/index.ts +++ b/src/index.ts @@ -230,7 +230,7 @@ ${c('yellow', 'Usage:')} agent-browser [options] ${c('yellow', 'Core Commands:')} ${c('cyan', 'open')} Navigate to URL - ${c('cyan', 'click')} Click element + ${c('cyan', 'click')} Click element (or @ref) ${c('cyan', 'type')} Type into element ${c('cyan', 'fill')} Clear and fill ${c('cyan', 'press')} Press key (Enter, Tab, Control+a) @@ -239,10 +239,16 @@ ${c('yellow', 'Core Commands:')} ${c('cyan', 'scroll')} [px] Scroll (up/down/left/right) ${c('cyan', 'wait')} Wait for element or time ${c('cyan', 'screenshot')} [path] Take screenshot - ${c('cyan', 'snapshot')} Accessibility tree (for AI) + ${c('cyan', 'snapshot')} Accessibility tree with refs (for AI) ${c('cyan', 'eval')} Run JavaScript ${c('cyan', 'close')} Close browser +${c('yellow', 'Selectors:')} CSS, XPath, text=, or ${c('green', '@ref')} from snapshot + ${c('dim', 'CSS:')} "#id", ".class", "button" + ${c('dim', 'XPath:')} "xpath=//button" + ${c('dim', 'Text:')} "text=Submit" + ${c('dim', 'Ref:')} ${c('green', '@e1')}, ${c('green', '@e2')} (from snapshot output) + ${c('yellow', 'Get Info:')} agent-browser get [selector] text, html, value, attr, title, url, count, box @@ -286,13 +292,12 @@ ${c('yellow', 'Options:')} ${c('yellow', 'Examples:')} agent-browser open example.com - agent-browser click "#submit" - agent-browser fill "#email" "test@example.com" - agent-browser get text "h1" - agent-browser is visible ".modal" + agent-browser snapshot # Get tree with refs + agent-browser click @e2 # Click by ref from snapshot + agent-browser fill @e3 "test@example.com" # Fill by ref + agent-browser click "#submit" # CSS selector still works + agent-browser get text @e1 # Get text by ref agent-browser find role button click --name Submit - agent-browser wait 2000 - agent-browser wait --load networkidle `); } diff --git a/src/snapshot.ts b/src/snapshot.ts new file mode 100644 index 0000000..3f0a4f1 --- /dev/null +++ b/src/snapshot.ts @@ -0,0 +1,207 @@ +/** + * Enhanced snapshot with element refs for deterministic element selection. + * + * This module generates accessibility snapshots with embedded refs that can be + * used to click/fill/interact with elements without re-querying the DOM. + * + * Example output: + * - heading "Example Domain" [ref=e1] [level=1] + * - paragraph: Some text content + * - button "Submit" [ref=e2] + * - textbox "Email" [ref=e3] + * + * Usage: + * agent-browser snapshot # Get snapshot with refs + * agent-browser click @e2 # Click element by ref + * agent-browser fill @e3 "test" # Fill element by ref + */ + +import type { Page, Locator } from 'playwright-core'; + +export interface RefMap { + [ref: string]: { + selector: string; + role: string; + name?: string; + }; +} + +export interface EnhancedSnapshot { + tree: string; + refs: RefMap; +} + +// Counter for generating refs +let refCounter = 0; + +/** + * Reset ref counter (call at start of each snapshot) + */ +export function resetRefs(): void { + refCounter = 0; +} + +/** + * Generate next ref ID + */ +function nextRef(): string { + return `e${++refCounter}`; +} + +/** + * Roles that are interactive and should get refs + */ +const INTERACTIVE_ROLES = new Set([ + 'button', + 'link', + 'textbox', + 'checkbox', + 'radio', + 'combobox', + 'listbox', + 'menuitem', + 'menuitemcheckbox', + 'menuitemradio', + 'option', + 'searchbox', + 'slider', + 'spinbutton', + 'switch', + 'tab', + 'treeitem', +]); + +/** + * Roles that provide structure/context (get refs for text extraction) + */ +const CONTENT_ROLES = new Set([ + 'heading', + 'cell', + 'gridcell', + 'columnheader', + 'rowheader', + 'listitem', + 'article', + 'region', + 'main', + 'navigation', +]); + +/** + * Build a selector string for storing in ref map + */ +function buildSelector(role: string, name?: string): string { + if (name) { + const escapedName = name.replace(/"/g, '\\"'); + return `getByRole('${role}', { name: "${escapedName}" })`; + } + return `getByRole('${role}')`; +} + +/** + * Get enhanced snapshot with refs + * + * Uses ariaSnapshot() which returns ARIA tree, then parses and adds refs + */ +export async function getEnhancedSnapshot(page: Page): Promise { + resetRefs(); + const refs: RefMap = {}; + + // Get ARIA snapshot from Playwright + const ariaTree = await page.locator(':root').ariaSnapshot(); + + if (!ariaTree) { + return { + tree: '(empty page)', + refs: {}, + }; + } + + // Parse the ARIA tree and add refs to interactive elements + const enhancedTree = addRefsToAriaTree(ariaTree, refs); + + return { tree: enhancedTree, refs }; +} + +/** + * Parse ARIA snapshot and add refs to interactive elements + * + * Input format from ariaSnapshot(): + * - document: + * - heading "Example Domain" [level=1] + * - paragraph: This is text + * - link "More info": + * - /url: https://... + */ +function addRefsToAriaTree(ariaTree: string, refs: RefMap): string { + const lines = ariaTree.split('\n'); + const enhancedLines: string[] = []; + + for (const line of lines) { + // Match lines like: + // - button "Submit" + // - heading "Title" [level=1] + // - link "Click me": + // - textbox "Email" + const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/); + + if (match) { + const [, prefix, role, name, suffix] = match; + const roleLower = role.toLowerCase(); + + // Skip metadata lines (like /url:) + if (role.startsWith('/')) { + enhancedLines.push(line); + continue; + } + + // Add ref for interactive or named content elements + const isInteractive = INTERACTIVE_ROLES.has(roleLower); + const isNamedContent = CONTENT_ROLES.has(roleLower) && name; + + if (isInteractive || isNamedContent) { + const ref = nextRef(); + + // Store ref data for later locator creation + refs[ref] = { + selector: buildSelector(roleLower, name), + role: roleLower, + name, + }; + + // Insert ref tag before any trailing content (like [level=1] or :) + const refTag = `[ref=${ref}]`; + + // Build the enhanced line + let enhanced = `${prefix}${role}`; + if (name) enhanced += ` "${name}"`; + enhanced += ` ${refTag}`; + if (suffix) enhanced += suffix; + + enhancedLines.push(enhanced); + } else { + enhancedLines.push(line); + } + } else { + enhancedLines.push(line); + } + } + + return enhancedLines.join('\n'); +} + +/** + * Parse a ref from command argument (e.g., "@e1" -> "e1") + */ +export function parseRef(arg: string): string | null { + if (arg.startsWith('@')) { + return arg.slice(1); + } + if (arg.startsWith('ref=')) { + return arg.slice(4); + } + if (/^e\d+$/.test(arg)) { + return arg; + } + return null; +}