refs

README.md

@@ -14,13 +14,22 @@ pnpm build
 
 ```bash
 agent-browser open example.com
-agent-browser click "#submit"
-agent-browser fill "#email" "test@example.com"
-agent-browser get text "h1"
+agent-browser snapshot                    # Get accessibility tree with refs
+agent-browser click @e2                   # Click by ref from snapshot
+agent-browser fill @e3 "test@example.com" # Fill by ref
+agent-browser get text @e1                # Get text by ref
 agent-browser screenshot page.png
 agent-browser close
 ```
 
+### Traditional Selectors (also supported)
+
+```bash
+agent-browser click "#submit"
+agent-browser fill "#email" "test@example.com"
+agent-browser find role button click --name "Submit"
+```
+
 ## Commands
 
 ### Core Commands
@@ -48,7 +57,7 @@ agent-browser upload <sel> <files>    # Upload files
 agent-browser download [path]         # Wait for download
 agent-browser screenshot [path]       # Take screenshot (--full for full page)
 agent-browser pdf <path>              # Save as PDF
-agent-browser snapshot                # Accessibility tree (best for AI)
+agent-browser snapshot                # Accessibility tree with refs (best for AI)
 agent-browser eval <js>               # Run JavaScript
 agent-browser close                   # Close browser
 ```
@@ -244,19 +253,49 @@ agent-browser session list
 
 ## Selectors
 
+### Refs (Recommended for AI)
+
+Refs provide deterministic element selection from snapshots:
+
+```bash
+# 1. Get snapshot with refs
+agent-browser snapshot
+# Output:
+# - heading "Example Domain" [ref=e1] [level=1]
+# - button "Submit" [ref=e2]
+# - textbox "Email" [ref=e3]
+# - link "Learn more" [ref=e4]
+
+# 2. Use refs to interact
+agent-browser click @e2                   # Click the button
+agent-browser fill @e3 "test@example.com" # Fill the textbox
+agent-browser get text @e1                # Get heading text
+agent-browser hover @e4                   # Hover the link
+```
+
+**Why use refs?**
+- **Deterministic**: Ref points to exact element from snapshot
+- **Fast**: No DOM re-query needed
+- **AI-friendly**: Snapshot + ref workflow is optimal for LLMs
+
+### CSS Selectors
+
 ```bash
-# CSS
 agent-browser click "#id"
 agent-browser click ".class"
 agent-browser click "div > button"
+```
 
-# Text
+### Text & XPath
+
+```bash
 agent-browser click "text=Submit"
-
-# XPath
 agent-browser click "xpath=//button"
+```
 
-# Semantic (recommended)
+### Semantic Locators
+
+```bash
 agent-browser find role button click --name "Submit"
 agent-browser find label "Email" fill "test@test.com"
 ```
@@ -267,8 +306,26 @@ Use `--json` for machine-readable output:
 
 ```bash
 agent-browser snapshot --json
-agent-browser get text "h1" --json
-agent-browser is visible ".modal" --json
+# Returns: {"success":true,"data":{"snapshot":"...","refs":{"e1":{"role":"heading","name":"Title"},...}}}
+
+agent-browser get text @e1 --json
+agent-browser is visible @e2 --json
+```
+
+### Optimal AI Workflow
+
+```bash
+# 1. Navigate and get snapshot
+agent-browser open example.com
+agent-browser snapshot --json    # AI parses tree and refs
+
+# 2. AI identifies target refs from snapshot
+# 3. Execute actions using refs
+agent-browser click @e2
+agent-browser fill @e3 "input text"
+
+# 4. Get new snapshot if page changed
+agent-browser snapshot --json
 ```
 
 ## License

package.json

@@ -29,12 +29,12 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
-    "playwright-core": "^1.40.0",
+    "playwright-core": "^1.57.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {
     "@types/node": "^20.10.0",
-    "playwright": "^1.40.0",
+    "playwright": "^1.57.0",
     "prettier": "^3.7.4",
     "tsx": "^4.6.0",
     "typescript": "^5.3.0",

src/actions.ts

@@ -108,6 +108,7 @@ import { successResponse, errorResponse } from './protocol.js';
 // Snapshot response type
 interface SnapshotData {
   snapshot: string;
+  refs?: Record<string, { role: string; name?: string }>;
 }
 
 /**
@@ -380,8 +381,10 @@ async function handleNavigate(
 }
 
 async function handleClick(command: ClickCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
-  await page.click(command.selector, {
+  // Support both refs (@e1) and regular selectors
+  const locator = browser.getLocator(command.selector);
+  
+  await locator.click({
     button: command.button,
     clickCount: command.clickCount,
     delay: command.delay,
@@ -391,13 +394,13 @@ async function handleClick(command: ClickCommand, browser: BrowserManager): Prom
 }
 
 async function handleType(command: TypeCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
+  const locator = browser.getLocator(command.selector);
 
   if (command.clear) {
-    await page.fill(command.selector, '');
+    await locator.fill('');
   }
 
-  await page.type(command.selector, command.text, {
+  await locator.pressSequentially(command.text, {
     delay: command.delay,
   });
 
@@ -449,12 +452,18 @@ async function handleSnapshot(
   command: Command & { action: 'snapshot' },
   browser: BrowserManager
 ): Promise<Response<SnapshotData>> {
-  const page = browser.getPage();
-  // Use ariaSnapshot which returns a string representation of the accessibility tree
-  const snapshot = await page.locator(':root').ariaSnapshot();
+  // Use enhanced snapshot with refs
+  const { tree, refs } = await browser.getSnapshot();
+
+  // Simplify refs for output (just role and name)
+  const simpleRefs: Record<string, { role: string; name?: string }> = {};
+  for (const [ref, data] of Object.entries(refs)) {
+    simpleRefs[ref] = { role: data.role, name: data.name };
+  }
 
   return successResponse(command.id, {
-    snapshot: snapshot ?? 'Empty page',
+    snapshot: tree || 'Empty page',
+    refs: Object.keys(simpleRefs).length > 0 ? simpleRefs : undefined,
   });
 }
 
@@ -533,17 +542,17 @@ async function handleScroll(command: ScrollCommand, browser: BrowserManager): Pr
 }
 
 async function handleSelect(command: SelectCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
+  const locator = browser.getLocator(command.selector);
   const values = Array.isArray(command.values) ? command.values : [command.values];
 
-  await page.selectOption(command.selector, values);
+  await locator.selectOption(values);
 
   return successResponse(command.id, { selected: values });
 }
 
 async function handleHover(command: HoverCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
-  await page.hover(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.hover();
 
   return successResponse(command.id, { hovered: true });
 }
@@ -622,27 +631,27 @@ async function handleWindowNew(
 // New handlers for enhanced Playwright parity
 
 async function handleFill(command: FillCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.fill(command.selector, command.value);
+  const locator = browser.getLocator(command.selector);
+  await locator.fill(command.value);
   return successResponse(command.id, { filled: true });
 }
 
 async function handleCheck(command: CheckCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.check(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.check();
   return successResponse(command.id, { checked: true });
 }
 
 async function handleUncheck(command: UncheckCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.uncheck(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.uncheck();
   return successResponse(command.id, { unchecked: true });
 }
 
 async function handleUpload(command: UploadCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
+  const locator = browser.getLocator(command.selector);
   const files = Array.isArray(command.files) ? command.files : [command.files];
-  await frame.setInputFiles(command.selector, files);
+  await locator.setInputFiles(files);
   return successResponse(command.id, { uploaded: files });
 }
 
@@ -650,14 +659,14 @@ async function handleDoubleClick(
   command: DoubleClickCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.dblclick(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.dblclick();
   return successResponse(command.id, { clicked: true });
 }
 
 async function handleFocus(command: FocusCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.focus(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.focus();
   return successResponse(command.id, { focused: true });
 }
 
@@ -1017,14 +1026,14 @@ async function handleGetAttribute(
   command: GetAttributeCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const value = await page.getAttribute(command.selector, command.attribute);
+  const locator = browser.getLocator(command.selector);
+  const value = await locator.getAttribute(command.attribute);
   return successResponse(command.id, { attribute: command.attribute, value });
 }
 
 async function handleGetText(command: GetTextCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
-  const text = await page.textContent(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const text = await locator.textContent();
   return successResponse(command.id, { text });
 }
 
@@ -1032,8 +1041,8 @@ async function handleIsVisible(
   command: IsVisibleCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const visible = await page.isVisible(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const visible = await locator.isVisible();
   return successResponse(command.id, { visible });
 }
 
@@ -1041,8 +1050,8 @@ async function handleIsEnabled(
   command: IsEnabledCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const enabled = await page.isEnabled(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const enabled = await locator.isEnabled();
   return successResponse(command.id, { enabled });
 }
 
@@ -1050,8 +1059,8 @@ async function handleIsChecked(
   command: IsCheckedCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const checked = await page.isChecked(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const checked = await locator.isChecked();
   return successResponse(command.id, { checked });
 }
 

src/browser.ts

@@ -10,8 +10,10 @@ import {
   type Dialog,
   type Request,
   type Route,
+  type Locator,
 } from 'playwright-core';
 import type { LaunchCommand } from './types.js';
+import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js';
 
 interface TrackedRequest {
   url: string;
@@ -47,6 +49,8 @@ export class BrowserManager {
   private consoleMessages: ConsoleMessage[] = [];
   private pageErrors: PageError[] = [];
   private isRecordingHar: boolean = false;
+  private refMap: RefMap = {};
+  private lastSnapshot: string = '';
 
   /**
    * Check if browser is launched
@@ -55,6 +59,65 @@ export class BrowserManager {
     return this.browser !== null;
   }
 
+  /**
+   * Get enhanced snapshot with refs and cache the ref map
+   */
+  async getSnapshot(): Promise<EnhancedSnapshot> {
+    const page = this.getPage();
+    const snapshot = await getEnhancedSnapshot(page);
+    this.refMap = snapshot.refs;
+    this.lastSnapshot = snapshot.tree;
+    return snapshot;
+  }
+
+  /**
+   * Get the cached ref map from last snapshot
+   */
+  getRefMap(): RefMap {
+    return this.refMap;
+  }
+
+  /**
+   * Get a locator from a ref (e.g., "e1", "@e1", "ref=e1")
+   * Returns null if ref doesn't exist or is invalid
+   */
+  getLocatorFromRef(refArg: string): Locator | null {
+    const ref = parseRef(refArg);
+    if (!ref) return null;
+
+    const refData = this.refMap[ref];
+    if (!refData) return null;
+
+    const page = this.getPage();
+    
+    // Parse the selector and create locator
+    if (refData.name) {
+      return page.getByRole(refData.role as any, { name: refData.name });
+    } else {
+      return page.getByRole(refData.role as any);
+    }
+  }
+
+  /**
+   * Check if a selector looks like a ref
+   */
+  isRef(selector: string): boolean {
+    return parseRef(selector) !== null;
+  }
+
+  /**
+   * Get locator - supports both refs and regular selectors
+   */
+  getLocator(selectorOrRef: string): Locator {
+    // Check if it's a ref first
+    const locator = this.getLocatorFromRef(selectorOrRef);
+    if (locator) return locator;
+
+    // Otherwise treat as regular selector
+    const page = this.getPage();
+    return page.locator(selectorOrRef);
+  }
+
   /**
    * Get the current active page, throws if not launched
    */
@@ -612,5 +675,7 @@ export class BrowserManager {
     }
 
     this.activePageIndex = 0;
+    this.refMap = {};
+    this.lastSnapshot = '';
   }
 }

src/index.ts

@@ -230,7 +230,7 @@ ${c('yellow', 'Usage:')} agent-browser <command> [options]
 
 ${c('yellow', 'Core Commands:')}
   ${c('cyan', 'open')} <url>                 Navigate to URL
-  ${c('cyan', 'click')} <sel>                Click element
+  ${c('cyan', 'click')} <sel>                Click element (or @ref)
   ${c('cyan', 'type')} <sel> <text>          Type into element
   ${c('cyan', 'fill')} <sel> <text>          Clear and fill
   ${c('cyan', 'press')} <key>                Press key (Enter, Tab, Control+a)
@@ -239,10 +239,16 @@ ${c('yellow', 'Core Commands:')}
   ${c('cyan', 'scroll')} <dir> [px]          Scroll (up/down/left/right)
   ${c('cyan', 'wait')} <sel|ms>              Wait for element or time
   ${c('cyan', 'screenshot')} [path]          Take screenshot
-  ${c('cyan', 'snapshot')}                   Accessibility tree (for AI)
+  ${c('cyan', 'snapshot')}                   Accessibility tree with refs (for AI)
   ${c('cyan', 'eval')} <js>                  Run JavaScript
   ${c('cyan', 'close')}                      Close browser
 
+${c('yellow', 'Selectors:')} CSS, XPath, text=, or ${c('green', '@ref')} from snapshot
+  ${c('dim', 'CSS:')}     "#id", ".class", "button"
+  ${c('dim', 'XPath:')}   "xpath=//button"
+  ${c('dim', 'Text:')}    "text=Submit"
+  ${c('dim', 'Ref:')}     ${c('green', '@e1')}, ${c('green', '@e2')} (from snapshot output)
+
 ${c('yellow', 'Get Info:')}  agent-browser get <what> [selector]
   text, html, value, attr, title, url, count, box
 
@@ -286,13 +292,12 @@ ${c('yellow', 'Options:')}
 
 ${c('yellow', 'Examples:')}
   agent-browser open example.com
-  agent-browser click "#submit"
-  agent-browser fill "#email" "test@example.com"
-  agent-browser get text "h1"
-  agent-browser is visible ".modal"
+  agent-browser snapshot                    # Get tree with refs
+  agent-browser click @e2                   # Click by ref from snapshot
+  agent-browser fill @e3 "test@example.com" # Fill by ref
+  agent-browser click "#submit"             # CSS selector still works
+  agent-browser get text @e1                # Get text by ref
   agent-browser find role button click --name Submit
-  agent-browser wait 2000
-  agent-browser wait --load networkidle
 `);
 }
 

src/snapshot.ts

@@ -0,0 +1,207 @@
+/**
+ * Enhanced snapshot with element refs for deterministic element selection.
+ *
+ * This module generates accessibility snapshots with embedded refs that can be
+ * used to click/fill/interact with elements without re-querying the DOM.
+ *
+ * Example output:
+ *   - heading "Example Domain" [ref=e1] [level=1]
+ *   - paragraph: Some text content
+ *   - button "Submit" [ref=e2]
+ *   - textbox "Email" [ref=e3]
+ *
+ * Usage:
+ *   agent-browser snapshot          # Get snapshot with refs
+ *   agent-browser click @e2         # Click element by ref
+ *   agent-browser fill @e3 "test"   # Fill element by ref
+ */
+
+import type { Page, Locator } from 'playwright-core';
+
+export interface RefMap {
+  [ref: string]: {
+    selector: string;
+    role: string;
+    name?: string;
+  };
+}
+
+export interface EnhancedSnapshot {
+  tree: string;
+  refs: RefMap;
+}
+
+// Counter for generating refs
+let refCounter = 0;
+
+/**
+ * Reset ref counter (call at start of each snapshot)
+ */
+export function resetRefs(): void {
+  refCounter = 0;
+}
+
+/**
+ * Generate next ref ID
+ */
+function nextRef(): string {
+  return `e${++refCounter}`;
+}
+
+/**
+ * Roles that are interactive and should get refs
+ */
+const INTERACTIVE_ROLES = new Set([
+  'button',
+  'link',
+  'textbox',
+  'checkbox',
+  'radio',
+  'combobox',
+  'listbox',
+  'menuitem',
+  'menuitemcheckbox',
+  'menuitemradio',
+  'option',
+  'searchbox',
+  'slider',
+  'spinbutton',
+  'switch',
+  'tab',
+  'treeitem',
+]);
+
+/**
+ * Roles that provide structure/context (get refs for text extraction)
+ */
+const CONTENT_ROLES = new Set([
+  'heading',
+  'cell',
+  'gridcell',
+  'columnheader',
+  'rowheader',
+  'listitem',
+  'article',
+  'region',
+  'main',
+  'navigation',
+]);
+
+/**
+ * Build a selector string for storing in ref map
+ */
+function buildSelector(role: string, name?: string): string {
+  if (name) {
+    const escapedName = name.replace(/"/g, '\\"');
+    return `getByRole('${role}', { name: "${escapedName}" })`;
+  }
+  return `getByRole('${role}')`;
+}
+
+/**
+ * Get enhanced snapshot with refs
+ *
+ * Uses ariaSnapshot() which returns ARIA tree, then parses and adds refs
+ */
+export async function getEnhancedSnapshot(page: Page): Promise<EnhancedSnapshot> {
+  resetRefs();
+  const refs: RefMap = {};
+
+  // Get ARIA snapshot from Playwright
+  const ariaTree = await page.locator(':root').ariaSnapshot();
+
+  if (!ariaTree) {
+    return {
+      tree: '(empty page)',
+      refs: {},
+    };
+  }
+
+  // Parse the ARIA tree and add refs to interactive elements
+  const enhancedTree = addRefsToAriaTree(ariaTree, refs);
+
+  return { tree: enhancedTree, refs };
+}
+
+/**
+ * Parse ARIA snapshot and add refs to interactive elements
+ *
+ * Input format from ariaSnapshot():
+ *   - document:
+ *     - heading "Example Domain" [level=1]
+ *     - paragraph: This is text
+ *     - link "More info":
+ *       - /url: https://...
+ */
+function addRefsToAriaTree(ariaTree: string, refs: RefMap): string {
+  const lines = ariaTree.split('\n');
+  const enhancedLines: string[] = [];
+
+  for (const line of lines) {
+    // Match lines like:
+    //   - button "Submit"
+    //   - heading "Title" [level=1]
+    //   - link "Click me":
+    //   - textbox "Email"
+    const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
+
+    if (match) {
+      const [, prefix, role, name, suffix] = match;
+      const roleLower = role.toLowerCase();
+
+      // Skip metadata lines (like /url:)
+      if (role.startsWith('/')) {
+        enhancedLines.push(line);
+        continue;
+      }
+
+      // Add ref for interactive or named content elements
+      const isInteractive = INTERACTIVE_ROLES.has(roleLower);
+      const isNamedContent = CONTENT_ROLES.has(roleLower) && name;
+
+      if (isInteractive || isNamedContent) {
+        const ref = nextRef();
+
+        // Store ref data for later locator creation
+        refs[ref] = {
+          selector: buildSelector(roleLower, name),
+          role: roleLower,
+          name,
+        };
+
+        // Insert ref tag before any trailing content (like [level=1] or :)
+        const refTag = `[ref=${ref}]`;
+
+        // Build the enhanced line
+        let enhanced = `${prefix}${role}`;
+        if (name) enhanced += ` "${name}"`;
+        enhanced += ` ${refTag}`;
+        if (suffix) enhanced += suffix;
+
+        enhancedLines.push(enhanced);
+      } else {
+        enhancedLines.push(line);
+      }
+    } else {
+      enhancedLines.push(line);
+    }
+  }
+
+  return enhancedLines.join('\n');
+}
+
+/**
+ * Parse a ref from command argument (e.g., "@e1" -> "e1")
+ */
+export function parseRef(arg: string): string | null {
+  if (arg.startsWith('@')) {
+    return arg.slice(1);
+  }
+  if (arg.startsWith('ref=')) {
+    return arg.slice(4);
+  }
+  if (/^e\d+$/.test(arg)) {
+    return arg;
+  }
+  return null;
+}