diff --git a/README.md b/README.md index 317d6b7..1b779fb 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,7 @@ The `snapshot` command supports filtering to reduce output size: ```bash agent-browser snapshot # Full accessibility tree agent-browser snapshot -i # Interactive elements only (buttons, inputs, links) +agent-browser snapshot -i -C # Include cursor-interactive elements (divs with onclick, etc.) agent-browser snapshot -c # Compact (remove empty structural elements) agent-browser snapshot -d 3 # Limit depth to 3 levels agent-browser snapshot -s "#main" # Scope to CSS selector @@ -311,10 +312,13 @@ agent-browser snapshot -i -c -d 5 # Combine options | Option | Description | |--------|-------------| | `-i, --interactive` | Only show interactive elements (buttons, links, inputs) | +| `-C, --cursor` | Include cursor-interactive elements (cursor:pointer, onclick, tabindex) | | `-c, --compact` | Remove empty structural elements | | `-d, --depth ` | Limit tree depth | | `-s, --selector ` | Scope to CSS selector | +The `-C` flag is useful for modern web apps that use custom clickable elements (divs, spans) instead of standard buttons/links. + ## Options | Option | Description | diff --git a/cli/src/commands.rs b/cli/src/commands.rs index 0d135f9..bbc8b35 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -403,6 +403,9 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result { obj.insert("compact".to_string(), json!(true)); } + "-C" | "--cursor" => { + obj.insert("cursor".to_string(), json!(true)); + } "-d" | "--depth" => { if let Some(d) = rest.get(i + 1) { if let Ok(n) = d.parse::() { @@ -1948,6 +1951,21 @@ mod tests { assert_eq!(cmd["interactive"], true); } + #[test] + fn test_snapshot_cursor() { + let cmd = parse_command(&args("snapshot -C"), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "snapshot"); + assert_eq!(cmd["cursor"], true); + } + + #[test] + fn test_snapshot_interactive_cursor() { + let cmd = parse_command(&args("snapshot -i -C"), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "snapshot"); + assert_eq!(cmd["interactive"], true); + assert_eq!(cmd["cursor"], true); + } + #[test] fn test_snapshot_compact() { let cmd = parse_command(&args("snapshot --compact"), &default_flags()).unwrap(); diff --git a/cli/src/output.rs b/cli/src/output.rs index b23fc88..3b3c499 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -905,6 +905,7 @@ Designed for AI agents to understand page structure. Options: -i, --interactive Only include interactive elements + -C, --cursor Include cursor-interactive elements (cursor:pointer, onclick, tabindex) -c, --compact Remove empty structural elements -d, --depth Limit tree depth -s, --selector Scope snapshot to CSS selector @@ -916,6 +917,7 @@ Global Options: Examples: agent-browser snapshot agent-browser snapshot -i + agent-browser snapshot -i -C # Interactive + cursor-interactive elements agent-browser snapshot --compact --depth 5 agent-browser snapshot -s "#main-content" "## diff --git a/docs/src/app/snapshots/page.tsx b/docs/src/app/snapshots/page.tsx index 777ca92..2120c68 100644 --- a/docs/src/app/snapshots/page.tsx +++ b/docs/src/app/snapshots/page.tsx @@ -13,6 +13,7 @@ export default function Snapshots() {

Filter output to reduce size:

-i, --interactive Only interactive elements (buttons, links, inputs) + + -C, --cursor + Include cursor-interactive elements (cursor:pointer, onclick, tabindex) + -c, --compact Remove empty structural elements @@ -45,6 +50,24 @@ agent-browser snapshot -i -c -d 5 # Combine options`} /> +

Cursor-interactive elements

+

+ Many modern web apps use custom clickable elements (divs, spans) instead of standard buttons or links. + The -C flag detects these by looking for: +

+
    +
  • cursor: pointer CSS style
    • +
  • onclick attribute or handler
    • +
  • tabindex attribute (keyboard focusable)
    • +
+ +

Output format

The default text output is compact and AI-friendly:

{ // Compact should be equal or shorter expect(compactSnapshot.length).toBeLessThanOrEqual(fullSnapshot.length); }); + + it('should not capture cursor-interactive elements without cursor flag', async () => { + const page = browser.getPage(); + await page.setContent(` + + + +
Clickable Div
+ + + `); + + const { tree, refs } = await browser.getSnapshot({ interactive: true }); + + // Standard button should be captured via ARIA + expect(tree).toContain('button "Standard Button"'); + + // Cursor-interactive elements should NOT be captured without cursor flag + expect(tree).not.toContain('Cursor-interactive elements'); + expect(tree).not.toContain('clickable "Clickable Div"'); + + // Should only have refs for ARIA interactive elements + const refValues = Object.values(refs); + expect(refValues.some((r) => r.role === 'button')).toBe(true); + expect(refValues.some((r) => r.role === 'clickable')).toBe(false); + }); + + it('should capture cursor-interactive elements with cursor flag', async () => { + const page = browser.getPage(); + await page.setContent(` + + + +
Clickable Div
+ Onclick Span + + + `); + + const { tree, refs } = await browser.getSnapshot({ interactive: true, cursor: true }); + + // Standard button should be captured via ARIA + expect(tree).toContain('button "Standard Button"'); + + // Cursor-interactive elements should be captured with cursor flag + expect(tree).toContain('Cursor-interactive elements'); + expect(tree).toContain('clickable "Clickable Div"'); + expect(tree).toContain('clickable "Onclick Span"'); + + // Should have refs for all interactive elements + const refValues = Object.values(refs); + expect(refValues.some((r) => r.role === 'button')).toBe(true); + expect(refValues.some((r) => r.role === 'clickable')).toBe(true); + }); + + it('should click cursor-interactive elements via refs', async () => { + const page = browser.getPage(); + await page.setContent(` + + +
Click Me
+
not clicked
+ + + `); + + const { refs } = await browser.getSnapshot({ cursor: true }); + + // Find the ref for the clickable element + const clickableRef = Object.keys(refs).find((k) => refs[k].name === 'Click Me'); + expect(clickableRef).toBeDefined(); + + // Click using the ref + const locator = browser.getLocator(`@${clickableRef}`); + await locator.click(); + + // Verify click worked + const result = await page.locator('#result').textContent(); + expect(result).toBe('clicked'); + }); }); describe('locator resolution', () => { diff --git a/src/browser.ts b/src/browser.ts index 47ac880..134cbaa 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -115,6 +115,7 @@ export class BrowserManager { */ async getSnapshot(options?: { interactive?: boolean; + cursor?: boolean; maxDepth?: number; compact?: boolean; selector?: string; @@ -146,6 +147,13 @@ export class BrowserManager { const page = this.getPage(); + // Check if this is a cursor-interactive element (uses CSS selector, not ARIA role) + // These have pseudo-roles 'clickable' or 'focusable' and a CSS selector + if (refData.role === 'clickable' || refData.role === 'focusable') { + // The selector is a CSS selector, use it directly + return page.locator(refData.selector); + } + // Build locator with exact: true to avoid substring matches let locator: Locator; if (refData.name) { diff --git a/src/snapshot.ts b/src/snapshot.ts index de81aa9..11682aa 100644 --- a/src/snapshot.ts +++ b/src/snapshot.ts @@ -37,6 +37,8 @@ export interface EnhancedSnapshot { export interface SnapshotOptions { /** Only include interactive elements (buttons, links, inputs, etc.) */ interactive?: boolean; + /** Include cursor-interactive elements (cursor:pointer, onclick, tabindex) */ + cursor?: boolean; /** Maximum depth of tree to include (0 = root only) */ maxDepth?: number; /** Remove structural elements without meaningful content */ @@ -136,6 +138,115 @@ function buildSelector(role: string, name?: string): string { return `getByRole('${role}')`; } +/** + * Query the page for clickable elements that might not have proper ARIA roles. + * This finds elements with cursor: pointer or onclick handlers. + */ +async function findCursorInteractiveElements( + page: Page, + selector?: string +): Promise< + Array<{ + selector: string; + text: string; + tagName: string; + hasOnClick: boolean; + hasCursorPointer: boolean; + hasTabIndex: boolean; + }> +> { + const rootSelector = selector || 'body'; + + // Use a string function body to avoid TypeScript transpilation issues + const scriptBody = `(rootSel) => { + const results = []; + + // Elements that already have interactive ARIA roles - skip these + const interactiveRoles = new Set([ + 'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'listbox', + 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'searchbox', + 'slider', 'spinbutton', 'switch', 'tab', 'treeitem' + ]); + + // Tags that are already interactive by default + const interactiveTags = new Set([ + 'a', 'button', 'input', 'select', 'textarea', 'details', 'summary' + ]); + + const root = document.querySelector(rootSel) || document.body; + const allElements = root.querySelectorAll('*'); + + // Build a unique selector for an element + const buildSelector = (el) => { + const testId = el.getAttribute('data-testid'); + if (testId) return '[data-testid="' + testId + '"]'; + if (el.id) return '#' + CSS.escape(el.id); + + const path = []; + let current = el; + while (current && current !== document.body) { + let sel = current.tagName.toLowerCase(); + const classes = Array.from(current.classList).filter(c => c.trim()); + if (classes.length > 0) sel += '.' + CSS.escape(classes[0]); + + const parent = current.parentElement; + if (parent) { + const siblings = Array.from(parent.children); + const matching = siblings.filter(s => { + if (s.tagName !== current.tagName) return false; + if (classes.length > 0 && !s.classList.contains(classes[0])) return false; + return true; + }); + if (matching.length > 1) { + const idx = matching.indexOf(current) + 1; + sel += ':nth-of-type(' + idx + ')'; + } + } + path.unshift(sel); + current = current.parentElement; + if (path.length >= 3) break; + } + return path.join(' > '); + }; + + for (const el of allElements) { + const tagName = el.tagName.toLowerCase(); + if (interactiveTags.has(tagName)) continue; + + const role = el.getAttribute('role'); + if (role && interactiveRoles.has(role.toLowerCase())) continue; + + const computedStyle = getComputedStyle(el); + const hasCursorPointer = computedStyle.cursor === 'pointer'; + const hasOnClick = el.hasAttribute('onclick') || el.onclick !== null; + const tabIndex = el.getAttribute('tabindex'); + const hasTabIndex = tabIndex !== null && tabIndex !== '-1'; + + if (!hasCursorPointer && !hasOnClick && !hasTabIndex) continue; + + const text = (el.textContent || '').trim().slice(0, 100); + if (!text) continue; + + const rect = el.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) continue; + + results.push({ + selector: buildSelector(el), + text, + tagName, + hasOnClick, + hasCursorPointer, + hasTabIndex + }); + } + return results; + }`; + + // eslint-disable-next-line @typescript-eslint/no-implied-eval + const fn = new Function('return ' + scriptBody)(); + return page.evaluate(fn, rootSelector); +} + /** * Get enhanced snapshot with refs and optional filtering */ @@ -160,6 +271,48 @@ export async function getEnhancedSnapshot( // Parse and enhance the ARIA tree const enhancedTree = processAriaTree(ariaTree, refs, options); + // When cursor flag is set, also find cursor-interactive elements + // that may not have proper ARIA roles + if (options.cursor) { + const cursorElements = await findCursorInteractiveElements(page, options.selector); + + // Filter out elements whose text is already captured in the snapshot + const existingTexts = new Set(Object.values(refs).map((r) => r.name?.toLowerCase())); + + const additionalLines: string[] = []; + for (const el of cursorElements) { + // Skip if text already captured (likely already in ARIA tree) + if (existingTexts.has(el.text.toLowerCase())) continue; + + const ref = nextRef(); + const role = el.hasCursorPointer ? 'clickable' : el.hasOnClick ? 'clickable' : 'focusable'; + + refs[ref] = { + selector: el.selector, + role: role, + name: el.text, + }; + + // Build description of why it's interactive + const hints: string[] = []; + if (el.hasCursorPointer) hints.push('cursor:pointer'); + if (el.hasOnClick) hints.push('onclick'); + if (el.hasTabIndex) hints.push('tabindex'); + + additionalLines.push(`- ${role} "${el.text}" [ref=${ref}] [${hints.join(', ')}]`); + } + + if (additionalLines.length > 0) { + const separator = + enhancedTree === '(no interactive elements)' ? '' : '\n# Cursor-interactive elements:\n'; + const base = enhancedTree === '(no interactive elements)' ? '' : enhancedTree; + return { + tree: base + separator + additionalLines.join('\n'), + refs, + }; + } + } + return { tree: enhancedTree, refs }; }