feat: add cursor-interactive element detection in snapshots (#374)

* fix: only warn about ignored flags when explicitly passed via CLI

The warning about launch-time options being ignored (when daemon is
already running) was incorrectly shown when options were set via
environment variables like AGENT_BROWSER_EXECUTABLE_PATH, even when
no CLI flag was passed.

Now the warning only appears when flags are explicitly passed on the
command line, not when values come solely from environment variables.

Fixes #372

* feat: add cursor-interactive element detection in snapshots

Add -C/--cursor flag to snapshot command that detects clickable elements
that don't have proper ARIA roles but are interactive based on:
- cursor: pointer CSS style
- onclick attribute/handler
- tabindex attribute

This helps with modern web apps that use custom divs/spans as buttons.

Fixes #366

* fix: add cursor option to getSnapshot type signature

README.md

@@ -302,6 +302,7 @@ The `snapshot` command supports filtering to reduce output size:
 ```bash
 agent-browser snapshot                    # Full accessibility tree
 agent-browser snapshot -i                 # Interactive elements only (buttons, inputs, links)
+agent-browser snapshot -i -C              # Include cursor-interactive elements (divs with onclick, etc.)
 agent-browser snapshot -c                 # Compact (remove empty structural elements)
 agent-browser snapshot -d 3               # Limit depth to 3 levels
 agent-browser snapshot -s "#main"         # Scope to CSS selector
@@ -311,10 +312,13 @@ agent-browser snapshot -i -c -d 5         # Combine options
 | Option | Description |
 |--------|-------------|
 | `-i, --interactive` | Only show interactive elements (buttons, links, inputs) |
+| `-C, --cursor` | Include cursor-interactive elements (cursor:pointer, onclick, tabindex) |
 | `-c, --compact` | Remove empty structural elements |
 | `-d, --depth <n>` | Limit tree depth |
 | `-s, --selector <sel>` | Scope to CSS selector |
 
+The `-C` flag is useful for modern web apps that use custom clickable elements (divs, spans) instead of standard buttons/links.
+
 ## Options
 
 | Option | Description |

cli/src/commands.rs

@@ -403,6 +403,9 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
                     "-c" | "--compact" => {
                         obj.insert("compact".to_string(), json!(true));
                     }
+                    "-C" | "--cursor" => {
+                        obj.insert("cursor".to_string(), json!(true));
+                    }
                     "-d" | "--depth" => {
                         if let Some(d) = rest.get(i + 1) {
                             if let Ok(n) = d.parse::<i32>() {
@@ -1948,6 +1951,21 @@ mod tests {
         assert_eq!(cmd["interactive"], true);
     }
 
+    #[test]
+    fn test_snapshot_cursor() {
+        let cmd = parse_command(&args("snapshot -C"), &default_flags()).unwrap();
+        assert_eq!(cmd["action"], "snapshot");
+        assert_eq!(cmd["cursor"], true);
+    }
+
+    #[test]
+    fn test_snapshot_interactive_cursor() {
+        let cmd = parse_command(&args("snapshot -i -C"), &default_flags()).unwrap();
+        assert_eq!(cmd["action"], "snapshot");
+        assert_eq!(cmd["interactive"], true);
+        assert_eq!(cmd["cursor"], true);
+    }
+
     #[test]
     fn test_snapshot_compact() {
         let cmd = parse_command(&args("snapshot --compact"), &default_flags()).unwrap();

cli/src/output.rs

@@ -905,6 +905,7 @@ Designed for AI agents to understand page structure.
 
 Options:
   -i, --interactive    Only include interactive elements
+  -C, --cursor         Include cursor-interactive elements (cursor:pointer, onclick, tabindex)
   -c, --compact        Remove empty structural elements
   -d, --depth <n>      Limit tree depth
   -s, --selector <sel> Scope snapshot to CSS selector
@@ -916,6 +917,7 @@ Global Options:
 Examples:
   agent-browser snapshot
   agent-browser snapshot -i
+  agent-browser snapshot -i -C         # Interactive + cursor-interactive elements
   agent-browser snapshot --compact --depth 5
   agent-browser snapshot -s "#main-content"
 "##

docs/src/app/snapshots/page.tsx

@@ -13,6 +13,7 @@ export default function Snapshots() {
         <p>Filter output to reduce size:</p>
         <CodeBlock code={`agent-browser snapshot                    # Full accessibility tree
 agent-browser snapshot -i                 # Interactive elements only (recommended)
+agent-browser snapshot -i -C              # Include cursor-interactive elements
 agent-browser snapshot -c                 # Compact (remove empty elements)
 agent-browser snapshot -d 3               # Limit depth to 3 levels
 agent-browser snapshot -s "#main"         # Scope to CSS selector
@@ -30,6 +31,10 @@ agent-browser snapshot -i -c -d 5         # Combine options`} />
               <td><code>-i, --interactive</code></td>
               <td>Only interactive elements (buttons, links, inputs)</td>
             </tr>
+            <tr>
+              <td><code>-C, --cursor</code></td>
+              <td>Include cursor-interactive elements (cursor:pointer, onclick, tabindex)</td>
+            </tr>
             <tr>
               <td><code>-c, --compact</code></td>
               <td>Remove empty structural elements</td>
@@ -45,6 +50,24 @@ agent-browser snapshot -i -c -d 5         # Combine options`} />
           </tbody>
         </table>
 
+        <h2>Cursor-interactive elements</h2>
+        <p>
+          Many modern web apps use custom clickable elements (divs, spans) instead of standard buttons or links.
+          The <code>-C</code> flag detects these by looking for:
+        </p>
+        <ul>
+          <li><code>cursor: pointer</code> CSS style</li>
+          <li><code>onclick</code> attribute or handler</li>
+          <li><code>tabindex</code> attribute (keyboard focusable)</li>
+        </ul>
+        <CodeBlock code={`agent-browser snapshot -i -C
+# Output includes:
+# @e1 [button] "Submit"
+# @e2 [link] "Learn more"
+# Cursor-interactive elements:
+# @e3 [clickable] "Menu Item" [cursor:pointer, onclick]
+# @e4 [clickable] "Card" [cursor:pointer]`} />
+
         <h2>Output format</h2>
         <p>The default text output is compact and AI-friendly:</p>
         <CodeBlock code={`agent-browser snapshot -i

skills/agent-browser/SKILL.md

@@ -36,6 +36,7 @@ agent-browser close                   # Close browser
 
 # Snapshot
 agent-browser snapshot -i             # Interactive elements with refs (recommended)
+agent-browser snapshot -i -C          # Include cursor-interactive elements (divs with onclick, cursor:pointer)
 agent-browser snapshot -s "#selector" # Scope to CSS selector
 
 # Interaction (use @refs from snapshot)

src/actions.ts

@@ -589,6 +589,7 @@ async function handleSnapshot(
   command: Command & {
     action: 'snapshot';
     interactive?: boolean;
+    cursor?: boolean;
     maxDepth?: number;
     compact?: boolean;
     selector?: string;
@@ -598,6 +599,7 @@ async function handleSnapshot(
   // Use enhanced snapshot with refs and optional filtering
   const { tree, refs } = await browser.getSnapshot({
     interactive: command.interactive,
+    cursor: command.cursor,
     maxDepth: command.maxDepth,
     compact: command.compact,
     selector: command.selector,

src/browser.test.ts

@@ -304,6 +304,86 @@ describe('BrowserManager', () => {
       // Compact should be equal or shorter
       expect(compactSnapshot.length).toBeLessThanOrEqual(fullSnapshot.length);
     });
+
+    it('should not capture cursor-interactive elements without cursor flag', async () => {
+      const page = browser.getPage();
+      await page.setContent(`
+        <html>
+          <body>
+            <button id="standard-btn">Standard Button</button>
+            <div id="clickable-div" style="cursor: pointer;" onclick="void(0)">Clickable Div</div>
+          </body>
+        </html>
+      `);
+
+      const { tree, refs } = await browser.getSnapshot({ interactive: true });
+
+      // Standard button should be captured via ARIA
+      expect(tree).toContain('button "Standard Button"');
+
+      // Cursor-interactive elements should NOT be captured without cursor flag
+      expect(tree).not.toContain('Cursor-interactive elements');
+      expect(tree).not.toContain('clickable "Clickable Div"');
+
+      // Should only have refs for ARIA interactive elements
+      const refValues = Object.values(refs);
+      expect(refValues.some((r) => r.role === 'button')).toBe(true);
+      expect(refValues.some((r) => r.role === 'clickable')).toBe(false);
+    });
+
+    it('should capture cursor-interactive elements with cursor flag', async () => {
+      const page = browser.getPage();
+      await page.setContent(`
+        <html>
+          <body>
+            <button id="standard-btn">Standard Button</button>
+            <div id="clickable-div" style="cursor: pointer;" onclick="void(0)">Clickable Div</div>
+            <span onclick="void(0)">Onclick Span</span>
+          </body>
+        </html>
+      `);
+
+      const { tree, refs } = await browser.getSnapshot({ interactive: true, cursor: true });
+
+      // Standard button should be captured via ARIA
+      expect(tree).toContain('button "Standard Button"');
+
+      // Cursor-interactive elements should be captured with cursor flag
+      expect(tree).toContain('Cursor-interactive elements');
+      expect(tree).toContain('clickable "Clickable Div"');
+      expect(tree).toContain('clickable "Onclick Span"');
+
+      // Should have refs for all interactive elements
+      const refValues = Object.values(refs);
+      expect(refValues.some((r) => r.role === 'button')).toBe(true);
+      expect(refValues.some((r) => r.role === 'clickable')).toBe(true);
+    });
+
+    it('should click cursor-interactive elements via refs', async () => {
+      const page = browser.getPage();
+      await page.setContent(`
+        <html>
+          <body>
+            <div id="clickable" style="cursor: pointer;" onclick="document.getElementById('result').textContent = 'clicked'">Click Me</div>
+            <div id="result">not clicked</div>
+          </body>
+        </html>
+      `);
+
+      const { refs } = await browser.getSnapshot({ cursor: true });
+
+      // Find the ref for the clickable element
+      const clickableRef = Object.keys(refs).find((k) => refs[k].name === 'Click Me');
+      expect(clickableRef).toBeDefined();
+
+      // Click using the ref
+      const locator = browser.getLocator(`@${clickableRef}`);
+      await locator.click();
+
+      // Verify click worked
+      const result = await page.locator('#result').textContent();
+      expect(result).toBe('clicked');
+    });
   });
 
   describe('locator resolution', () => {

src/browser.ts

@@ -115,6 +115,7 @@ export class BrowserManager {
    */
   async getSnapshot(options?: {
     interactive?: boolean;
+    cursor?: boolean;
     maxDepth?: number;
     compact?: boolean;
     selector?: string;
@@ -146,6 +147,13 @@ export class BrowserManager {
 
     const page = this.getPage();
 
+    // Check if this is a cursor-interactive element (uses CSS selector, not ARIA role)
+    // These have pseudo-roles 'clickable' or 'focusable' and a CSS selector
+    if (refData.role === 'clickable' || refData.role === 'focusable') {
+      // The selector is a CSS selector, use it directly
+      return page.locator(refData.selector);
+    }
+
     // Build locator with exact: true to avoid substring matches
     let locator: Locator;
     if (refData.name) {

src/snapshot.ts

@@ -37,6 +37,8 @@ export interface EnhancedSnapshot {
 export interface SnapshotOptions {
   /** Only include interactive elements (buttons, links, inputs, etc.) */
   interactive?: boolean;
+  /** Include cursor-interactive elements (cursor:pointer, onclick, tabindex) */
+  cursor?: boolean;
   /** Maximum depth of tree to include (0 = root only) */
   maxDepth?: number;
   /** Remove structural elements without meaningful content */
@@ -136,6 +138,115 @@ function buildSelector(role: string, name?: string): string {
   return `getByRole('${role}')`;
 }
 
+/**
+ * Query the page for clickable elements that might not have proper ARIA roles.
+ * This finds elements with cursor: pointer or onclick handlers.
+ */
+async function findCursorInteractiveElements(
+  page: Page,
+  selector?: string
+): Promise<
+  Array<{
+    selector: string;
+    text: string;
+    tagName: string;
+    hasOnClick: boolean;
+    hasCursorPointer: boolean;
+    hasTabIndex: boolean;
+  }>
+> {
+  const rootSelector = selector || 'body';
+
+  // Use a string function body to avoid TypeScript transpilation issues
+  const scriptBody = `(rootSel) => {
+    const results = [];
+
+    // Elements that already have interactive ARIA roles - skip these
+    const interactiveRoles = new Set([
+      'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'listbox',
+      'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'searchbox',
+      'slider', 'spinbutton', 'switch', 'tab', 'treeitem'
+    ]);
+
+    // Tags that are already interactive by default
+    const interactiveTags = new Set([
+      'a', 'button', 'input', 'select', 'textarea', 'details', 'summary'
+    ]);
+
+    const root = document.querySelector(rootSel) || document.body;
+    const allElements = root.querySelectorAll('*');
+
+    // Build a unique selector for an element
+    const buildSelector = (el) => {
+      const testId = el.getAttribute('data-testid');
+      if (testId) return '[data-testid="' + testId + '"]';
+      if (el.id) return '#' + CSS.escape(el.id);
+
+      const path = [];
+      let current = el;
+      while (current && current !== document.body) {
+        let sel = current.tagName.toLowerCase();
+        const classes = Array.from(current.classList).filter(c => c.trim());
+        if (classes.length > 0) sel += '.' + CSS.escape(classes[0]);
+
+        const parent = current.parentElement;
+        if (parent) {
+          const siblings = Array.from(parent.children);
+          const matching = siblings.filter(s => {
+            if (s.tagName !== current.tagName) return false;
+            if (classes.length > 0 && !s.classList.contains(classes[0])) return false;
+            return true;
+          });
+          if (matching.length > 1) {
+            const idx = matching.indexOf(current) + 1;
+            sel += ':nth-of-type(' + idx + ')';
+          }
+        }
+        path.unshift(sel);
+        current = current.parentElement;
+        if (path.length >= 3) break;
+      }
+      return path.join(' > ');
+    };
+
+    for (const el of allElements) {
+      const tagName = el.tagName.toLowerCase();
+      if (interactiveTags.has(tagName)) continue;
+
+      const role = el.getAttribute('role');
+      if (role && interactiveRoles.has(role.toLowerCase())) continue;
+
+      const computedStyle = getComputedStyle(el);
+      const hasCursorPointer = computedStyle.cursor === 'pointer';
+      const hasOnClick = el.hasAttribute('onclick') || el.onclick !== null;
+      const tabIndex = el.getAttribute('tabindex');
+      const hasTabIndex = tabIndex !== null && tabIndex !== '-1';
+
+      if (!hasCursorPointer && !hasOnClick && !hasTabIndex) continue;
+
+      const text = (el.textContent || '').trim().slice(0, 100);
+      if (!text) continue;
+
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 || rect.height === 0) continue;
+
+      results.push({
+        selector: buildSelector(el),
+        text,
+        tagName,
+        hasOnClick,
+        hasCursorPointer,
+        hasTabIndex
+      });
+    }
+    return results;
+  }`;
+
+  // eslint-disable-next-line @typescript-eslint/no-implied-eval
+  const fn = new Function('return ' + scriptBody)();
+  return page.evaluate(fn, rootSelector);
+}
+
 /**
  * Get enhanced snapshot with refs and optional filtering
  */
@@ -160,6 +271,48 @@ export async function getEnhancedSnapshot(
   // Parse and enhance the ARIA tree
   const enhancedTree = processAriaTree(ariaTree, refs, options);
 
+  // When cursor flag is set, also find cursor-interactive elements
+  // that may not have proper ARIA roles
+  if (options.cursor) {
+    const cursorElements = await findCursorInteractiveElements(page, options.selector);
+
+    // Filter out elements whose text is already captured in the snapshot
+    const existingTexts = new Set(Object.values(refs).map((r) => r.name?.toLowerCase()));
+
+    const additionalLines: string[] = [];
+    for (const el of cursorElements) {
+      // Skip if text already captured (likely already in ARIA tree)
+      if (existingTexts.has(el.text.toLowerCase())) continue;
+
+      const ref = nextRef();
+      const role = el.hasCursorPointer ? 'clickable' : el.hasOnClick ? 'clickable' : 'focusable';
+
+      refs[ref] = {
+        selector: el.selector,
+        role: role,
+        name: el.text,
+      };
+
+      // Build description of why it's interactive
+      const hints: string[] = [];
+      if (el.hasCursorPointer) hints.push('cursor:pointer');
+      if (el.hasOnClick) hints.push('onclick');
+      if (el.hasTabIndex) hints.push('tabindex');
+
+      additionalLines.push(`- ${role} "${el.text}" [ref=${ref}] [${hints.join(', ')}]`);
+    }
+
+    if (additionalLines.length > 0) {
+      const separator =
+        enhancedTree === '(no interactive elements)' ? '' : '\n# Cursor-interactive elements:\n';
+      const base = enhancedTree === '(no interactive elements)' ? '' : enhancedTree;
+      return {
+        tree: base + separator + additionalLines.join('\n'),
+        refs,
+      };
+    }
+  }
+
   return { tree: enhancedTree, refs };
 }