feat: add cursor-interactive element detection in snapshots (#374)

* fix: only warn about ignored flags when explicitly passed via CLI The warning about launch-time options being ignored (when daemon is already running) was incorrectly shown when options were set via environment variables like AGENT_BROWSER_EXECUTABLE_PATH, even when no CLI flag was passed. Now the warning only appears when flags are explicitly passed on the command line, not when values come solely from environment variables. Fixes #372 * feat: add cursor-interactive element detection in snapshots Add -C/--cursor flag to snapshot command that detects clickable elements that don't have proper ARIA roles but are interactive based on: - cursor: pointer CSS style - onclick attribute/handler - tabindex attribute This helps with modern web apps that use custom divs/spans as buttons. Fixes #366 * fix: add cursor option to getSnapshot type signature
2026-02-04 23:44:58 -06:00
parent d34ce8c2d0
commit 74be667c80
9 changed files with 291 additions and 0 deletions
@@ -589,6 +589,7 @@ async function handleSnapshot(
  command: Command & {
    action: 'snapshot';
    interactive?: boolean;
+    cursor?: boolean;
    maxDepth?: number;
    compact?: boolean;
    selector?: string;
@@ -598,6 +599,7 @@ async function handleSnapshot(
  // Use enhanced snapshot with refs and optional filtering
  const { tree, refs } = await browser.getSnapshot({
    interactive: command.interactive,
+    cursor: command.cursor,
    maxDepth: command.maxDepth,
    compact: command.compact,
    selector: command.selector,
@@ -304,6 +304,86 @@ describe('BrowserManager', () => {
      // Compact should be equal or shorter
      expect(compactSnapshot.length).toBeLessThanOrEqual(fullSnapshot.length);
    });
+
+    it('should not capture cursor-interactive elements without cursor flag', async () => {
+      const page = browser.getPage();
+      await page.setContent(`
+        <html>
+          <body>
+            <button id="standard-btn">Standard Button</button>
+            <div id="clickable-div" style="cursor: pointer;" onclick="void(0)">Clickable Div</div>
+          </body>
+        </html>
+      `);
+
+      const { tree, refs } = await browser.getSnapshot({ interactive: true });
+
+      // Standard button should be captured via ARIA
+      expect(tree).toContain('button "Standard Button"');
+
+      // Cursor-interactive elements should NOT be captured without cursor flag
+      expect(tree).not.toContain('Cursor-interactive elements');
+      expect(tree).not.toContain('clickable "Clickable Div"');
+
+      // Should only have refs for ARIA interactive elements
+      const refValues = Object.values(refs);
+      expect(refValues.some((r) => r.role === 'button')).toBe(true);
+      expect(refValues.some((r) => r.role === 'clickable')).toBe(false);
+    });
+
+    it('should capture cursor-interactive elements with cursor flag', async () => {
+      const page = browser.getPage();
+      await page.setContent(`
+        <html>
+          <body>
+            <button id="standard-btn">Standard Button</button>
+            <div id="clickable-div" style="cursor: pointer;" onclick="void(0)">Clickable Div</div>
+            <span onclick="void(0)">Onclick Span</span>
+          </body>
+        </html>
+      `);
+
+      const { tree, refs } = await browser.getSnapshot({ interactive: true, cursor: true });
+
+      // Standard button should be captured via ARIA
+      expect(tree).toContain('button "Standard Button"');
+
+      // Cursor-interactive elements should be captured with cursor flag
+      expect(tree).toContain('Cursor-interactive elements');
+      expect(tree).toContain('clickable "Clickable Div"');
+      expect(tree).toContain('clickable "Onclick Span"');
+
+      // Should have refs for all interactive elements
+      const refValues = Object.values(refs);
+      expect(refValues.some((r) => r.role === 'button')).toBe(true);
+      expect(refValues.some((r) => r.role === 'clickable')).toBe(true);
+    });
+
+    it('should click cursor-interactive elements via refs', async () => {
+      const page = browser.getPage();
+      await page.setContent(`
+        <html>
+          <body>
+            <div id="clickable" style="cursor: pointer;" onclick="document.getElementById('result').textContent = 'clicked'">Click Me</div>
+            <div id="result">not clicked</div>
+          </body>
+        </html>
+      `);
+
+      const { refs } = await browser.getSnapshot({ cursor: true });
+
+      // Find the ref for the clickable element
+      const clickableRef = Object.keys(refs).find((k) => refs[k].name === 'Click Me');
+      expect(clickableRef).toBeDefined();
+
+      // Click using the ref
+      const locator = browser.getLocator(`@${clickableRef}`);
+      await locator.click();
+
+      // Verify click worked
+      const result = await page.locator('#result').textContent();
+      expect(result).toBe('clicked');
+    });
  });

  describe('locator resolution', () => {
@@ -115,6 +115,7 @@ export class BrowserManager {
   */
  async getSnapshot(options?: {
    interactive?: boolean;
+    cursor?: boolean;
    maxDepth?: number;
    compact?: boolean;
    selector?: string;
@@ -146,6 +147,13 @@ export class BrowserManager {

    const page = this.getPage();

+    // Check if this is a cursor-interactive element (uses CSS selector, not ARIA role)
+    // These have pseudo-roles 'clickable' or 'focusable' and a CSS selector
+    if (refData.role === 'clickable' || refData.role === 'focusable') {
+      // The selector is a CSS selector, use it directly
+      return page.locator(refData.selector);
+    }
+
    // Build locator with exact: true to avoid substring matches
    let locator: Locator;
    if (refData.name) {
@@ -37,6 +37,8 @@ export interface EnhancedSnapshot {
 export interface SnapshotOptions {
  /** Only include interactive elements (buttons, links, inputs, etc.) */
  interactive?: boolean;
+  /** Include cursor-interactive elements (cursor:pointer, onclick, tabindex) */
+  cursor?: boolean;
  /** Maximum depth of tree to include (0 = root only) */
  maxDepth?: number;
  /** Remove structural elements without meaningful content */
@@ -136,6 +138,115 @@ function buildSelector(role: string, name?: string): string {
  return `getByRole('${role}')`;
 }

+/**
+ * Query the page for clickable elements that might not have proper ARIA roles.
+ * This finds elements with cursor: pointer or onclick handlers.
+ */
+async function findCursorInteractiveElements(
+  page: Page,
+  selector?: string
+): Promise<
+  Array<{
+    selector: string;
+    text: string;
+    tagName: string;
+    hasOnClick: boolean;
+    hasCursorPointer: boolean;
+    hasTabIndex: boolean;
+  }>
+> {
+  const rootSelector = selector || 'body';
+
+  // Use a string function body to avoid TypeScript transpilation issues
+  const scriptBody = `(rootSel) => {
+    const results = [];
+
+    // Elements that already have interactive ARIA roles - skip these
+    const interactiveRoles = new Set([
+      'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'listbox',
+      'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'searchbox',
+      'slider', 'spinbutton', 'switch', 'tab', 'treeitem'
+    ]);
+
+    // Tags that are already interactive by default
+    const interactiveTags = new Set([
+      'a', 'button', 'input', 'select', 'textarea', 'details', 'summary'
+    ]);
+
+    const root = document.querySelector(rootSel) || document.body;
+    const allElements = root.querySelectorAll('*');
+
+    // Build a unique selector for an element
+    const buildSelector = (el) => {
+      const testId = el.getAttribute('data-testid');
+      if (testId) return '[data-testid="' + testId + '"]';
+      if (el.id) return '#' + CSS.escape(el.id);
+
+      const path = [];
+      let current = el;
+      while (current && current !== document.body) {
+        let sel = current.tagName.toLowerCase();
+        const classes = Array.from(current.classList).filter(c => c.trim());
+        if (classes.length > 0) sel += '.' + CSS.escape(classes[0]);
+
+        const parent = current.parentElement;
+        if (parent) {
+          const siblings = Array.from(parent.children);
+          const matching = siblings.filter(s => {
+            if (s.tagName !== current.tagName) return false;
+            if (classes.length > 0 && !s.classList.contains(classes[0])) return false;
+            return true;
+          });
+          if (matching.length > 1) {
+            const idx = matching.indexOf(current) + 1;
+            sel += ':nth-of-type(' + idx + ')';
+          }
+        }
+        path.unshift(sel);
+        current = current.parentElement;
+        if (path.length >= 3) break;
+      }
+      return path.join(' > ');
+    };
+
+    for (const el of allElements) {
+      const tagName = el.tagName.toLowerCase();
+      if (interactiveTags.has(tagName)) continue;
+
+      const role = el.getAttribute('role');
+      if (role && interactiveRoles.has(role.toLowerCase())) continue;
+
+      const computedStyle = getComputedStyle(el);
+      const hasCursorPointer = computedStyle.cursor === 'pointer';
+      const hasOnClick = el.hasAttribute('onclick') || el.onclick !== null;
+      const tabIndex = el.getAttribute('tabindex');
+      const hasTabIndex = tabIndex !== null && tabIndex !== '-1';
+
+      if (!hasCursorPointer && !hasOnClick && !hasTabIndex) continue;
+
+      const text = (el.textContent || '').trim().slice(0, 100);
+      if (!text) continue;
+
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 || rect.height === 0) continue;
+
+      results.push({
+        selector: buildSelector(el),
+        text,
+        tagName,
+        hasOnClick,
+        hasCursorPointer,
+        hasTabIndex
+      });
+    }
+    return results;
+  }`;
+
+  // eslint-disable-next-line @typescript-eslint/no-implied-eval
+  const fn = new Function('return ' + scriptBody)();
+  return page.evaluate(fn, rootSelector);
+}
+
 /**
 * Get enhanced snapshot with refs and optional filtering
 */
@@ -160,6 +271,48 @@ export async function getEnhancedSnapshot(
  // Parse and enhance the ARIA tree
  const enhancedTree = processAriaTree(ariaTree, refs, options);

+  // When cursor flag is set, also find cursor-interactive elements
+  // that may not have proper ARIA roles
+  if (options.cursor) {
+    const cursorElements = await findCursorInteractiveElements(page, options.selector);
+
+    // Filter out elements whose text is already captured in the snapshot
+    const existingTexts = new Set(Object.values(refs).map((r) => r.name?.toLowerCase()));
+
+    const additionalLines: string[] = [];
+    for (const el of cursorElements) {
+      // Skip if text already captured (likely already in ARIA tree)
+      if (existingTexts.has(el.text.toLowerCase())) continue;
+
+      const ref = nextRef();
+      const role = el.hasCursorPointer ? 'clickable' : el.hasOnClick ? 'clickable' : 'focusable';
+
+      refs[ref] = {
+        selector: el.selector,
+        role: role,
+        name: el.text,
+      };
+
+      // Build description of why it's interactive
+      const hints: string[] = [];
+      if (el.hasCursorPointer) hints.push('cursor:pointer');
+      if (el.hasOnClick) hints.push('onclick');
+      if (el.hasTabIndex) hints.push('tabindex');
+
+      additionalLines.push(`- ${role} "${el.text}" [ref=${ref}] [${hints.join(', ')}]`);
+    }
+
+    if (additionalLines.length > 0) {
+      const separator =
+        enhancedTree === '(no interactive elements)' ? '' : '\n# Cursor-interactive elements:\n';
+      const base = enhancedTree === '(no interactive elements)' ? '' : enhancedTree;
+      return {
+        tree: base + separator + additionalLines.join('\n'),
+        refs,
+      };
+    }
+  }
+
  return { tree: enhancedTree, refs };
 }