From e2e259f1e28b962e2b385f87f8d022a8b5f18ceb Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 18 Feb 2026 22:20:01 -0600 Subject: [PATCH] annotated screenshots (#503) * screenshot annotation * fixes * fix CI checks * fixes * fixes * fixes * fixes * fixes --- README.md | 23 +++++ cli/src/commands.rs | 11 ++- cli/src/flags.rs | 34 +++++-- cli/src/output.rs | 46 ++++++++- docs/src/app/commands/page.mdx | 2 + docs/src/app/snapshots/page.mdx | 16 ++++ skills/agent-browser/SKILL.md | 20 ++++ src/actions.ts | 164 +++++++++++++++++++++++++++++++- src/browser.test.ts | 127 +++++++++++++++++++++++++ src/protocol.test.ts | 5 + src/protocol.ts | 1 + src/types.ts | 10 ++ 12 files changed, 445 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4b97804..f5b7613 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ agent-browser scrollintoview # Scroll element into view (alias: scrolli agent-browser drag # Drag and drop agent-browser upload # Upload files agent-browser screenshot [path] # Take screenshot (--full for full page, saves to a temporary directory if no path) +agent-browser screenshot --annotate # Annotated screenshot with numbered element labels agent-browser pdf # Save as PDF agent-browser snapshot # Accessibility tree with refs (best for AI) agent-browser eval # Run JavaScript (-b for base64, --stdin for piped input) @@ -401,6 +402,27 @@ agent-browser snapshot -i -c -d 5 # Combine options The `-C` flag is useful for modern web apps that use custom clickable elements (divs, spans) instead of standard buttons/links. +## Annotated Screenshots + +The `--annotate` flag overlays numbered labels on interactive elements in the screenshot. Each label `[N]` corresponds to ref `@eN`, so the same refs work for both visual and text-based workflows. + +```bash +agent-browser screenshot --annotate +# -> Screenshot saved to /tmp/screenshot-2026-02-17T12-00-00-abc123.png +# [1] @e1 button "Submit" +# [2] @e2 link "Home" +# [3] @e3 textbox "Email" +``` + +After an annotated screenshot, refs are cached so you can immediately interact with elements: + +```bash +agent-browser screenshot --annotate ./page.png +agent-browser click @e2 # Click the "Home" link labeled [2] +``` + +This is useful for multimodal AI models that can reason about visual layout, unlabeled icon buttons, canvas elements, or visual state that the text accessibility tree cannot capture. + ## Options | Option | Description | @@ -422,6 +444,7 @@ The `-C` flag is useful for modern web apps that use custom clickable elements ( | `--device ` | iOS device name, e.g. "iPhone 15 Pro" (or `AGENT_BROWSER_IOS_DEVICE` env) | | `--json` | JSON output (for agents) | | `--full, -f` | Full page screenshot | +| `--annotate` | Annotated screenshot with numbered element labels (or `AGENT_BROWSER_ANNOTATE` env) | | `--headed` | Show browser window (not headless) | | `--cdp ` | Connect via Chrome DevTools Protocol (port or WebSocket URL) | | `--auto-connect` | Auto-discover and connect to running Chrome (or `AGENT_BROWSER_AUTO_CONNECT` env) | diff --git a/cli/src/commands.rs b/cli/src/commands.rs index faa44f9..4ac56b4 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -2,6 +2,7 @@ use base64::{engine::general_purpose::STANDARD, Engine}; use serde_json::{json, Value}; use std::io::{self, BufRead}; +use crate::color; use crate::flags::Flags; use crate::validation::{is_valid_session_name, session_name_error}; @@ -82,6 +83,13 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result = args[1..].iter().map(|s| s.as_str()).collect(); let id = gen_id(); + if flags.annotate && cmd != "screenshot" { + eprintln!( + "{} --annotate only applies to the screenshot command", + color::warning_indicator() + ); + } + match cmd { // === Navigation === "open" | "goto" | "navigate" => { @@ -392,7 +400,7 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result (None, None), }; Ok( - json!({ "id": id, "action": "screenshot", "path": path, "selector": selector, "fullPage": flags.full }), + json!({ "id": id, "action": "screenshot", "path": path, "selector": selector, "fullPage": flags.full, "annotate": flags.annotate }), ) } "pdf" => { @@ -1583,6 +1591,7 @@ mod tests { cli_proxy: false, cli_proxy_bypass: false, cli_allow_file_access: false, + annotate: false, } } diff --git a/cli/src/flags.rs b/cli/src/flags.rs index 8214374..14ced63 100644 --- a/cli/src/flags.rs +++ b/cli/src/flags.rs @@ -32,6 +32,7 @@ pub struct Config { pub cdp: Option, pub auto_connect: Option, pub headers: Option, + pub annotate: Option, } impl Config { @@ -64,6 +65,7 @@ impl Config { cdp: other.cdp.or(self.cdp), auto_connect: other.auto_connect.or(self.auto_connect), headers: other.headers.or(self.headers), + annotate: other.annotate.or(self.annotate), } } } @@ -84,6 +86,15 @@ fn read_config_file(path: &Path) -> Option { } } +/// Check if a boolean environment variable is set to a truthy value. +/// Returns false when unset, empty, or set to "0", "false", or "no" (case-insensitive). +fn env_var_is_truthy(name: &str) -> bool { + match env::var(name) { + Ok(val) => !matches!(val.to_lowercase().as_str(), "0" | "false" | "no" | ""), + Err(_) => false, + } +} + /// Parse an optional boolean value after a flag. Returns (value, consumed_next_arg). /// Recognizes "true" as true, "false" as false. Bare flag defaults to true. fn parse_bool_arg(args: &[String], i: usize) -> (bool, bool) { @@ -187,6 +198,7 @@ pub struct Flags { pub device: Option, pub auto_connect: bool, pub session_name: Option, + pub annotate: bool, // Track which launch-time options were explicitly passed via CLI // (as opposed to being set only via environment variables) @@ -224,13 +236,13 @@ pub fn parse_flags(args: &[String]) -> Flags { }; let mut flags = Flags { - json: env::var("AGENT_BROWSER_JSON").is_ok() + json: env_var_is_truthy("AGENT_BROWSER_JSON") || config.json.unwrap_or(false), - full: env::var("AGENT_BROWSER_FULL").is_ok() + full: env_var_is_truthy("AGENT_BROWSER_FULL") || config.full.unwrap_or(false), - headed: env::var("AGENT_BROWSER_HEADED").is_ok() + headed: env_var_is_truthy("AGENT_BROWSER_HEADED") || config.headed.unwrap_or(false), - debug: env::var("AGENT_BROWSER_DEBUG").is_ok() + debug: env_var_is_truthy("AGENT_BROWSER_DEBUG") || config.debug.unwrap_or(false), session: env::var("AGENT_BROWSER_SESSION").ok() .or(config.session) @@ -254,16 +266,18 @@ pub fn parse_flags(args: &[String]) -> Flags { .or(config.user_agent), provider: env::var("AGENT_BROWSER_PROVIDER").ok() .or(config.provider), - ignore_https_errors: env::var("AGENT_BROWSER_IGNORE_HTTPS_ERRORS").is_ok() + ignore_https_errors: env_var_is_truthy("AGENT_BROWSER_IGNORE_HTTPS_ERRORS") || config.ignore_https_errors.unwrap_or(false), - allow_file_access: env::var("AGENT_BROWSER_ALLOW_FILE_ACCESS").is_ok() + allow_file_access: env_var_is_truthy("AGENT_BROWSER_ALLOW_FILE_ACCESS") || config.allow_file_access.unwrap_or(false), device: env::var("AGENT_BROWSER_IOS_DEVICE").ok() .or(config.device), - auto_connect: env::var("AGENT_BROWSER_AUTO_CONNECT").is_ok() + auto_connect: env_var_is_truthy("AGENT_BROWSER_AUTO_CONNECT") || config.auto_connect.unwrap_or(false), session_name: env::var("AGENT_BROWSER_SESSION_NAME").ok() .or(config.session_name), + annotate: env_var_is_truthy("AGENT_BROWSER_ANNOTATE") + || config.annotate.unwrap_or(false), cli_executable_path: false, cli_extensions: false, cli_profile: false, @@ -406,6 +420,11 @@ pub fn parse_flags(args: &[String]) -> Flags { i += 1; } } + "--annotate" => { + let (val, consumed) = parse_bool_arg(args, i); + flags.annotate = val; + if consumed { i += 1; } + } "--config" => { // Already handled by load_config(); skip the value i += 1; @@ -430,6 +449,7 @@ pub fn clean_args(args: &[String]) -> Vec { "--ignore-https-errors", "--allow-file-access", "--auto-connect", + "--annotate", ]; // Global flags that always take a value (need to skip the next arg too) const GLOBAL_FLAGS_WITH_VALUE: &[&str] = &[ diff --git a/cli/src/output.rs b/cli/src/output.rs index 572e213..d271559 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -363,11 +363,37 @@ pub fn print_response(resp: &Response, json_mode: bool, action: Option<&str>) { // Path-based operations (screenshot/pdf/trace/har/download/state/video) if let Some(path) = data.get("path").and_then(|v| v.as_str()) { match action.unwrap_or("") { - "screenshot" => println!( - "{} Screenshot saved to {}", - color::success_indicator(), - color::green(path) - ), + "screenshot" => { + println!( + "{} Screenshot saved to {}", + color::success_indicator(), + color::green(path) + ); + if let Some(annotations) = data.get("annotations").and_then(|v| v.as_array()) { + for ann in annotations { + let num = ann.get("number").and_then(|n| n.as_u64()).unwrap_or(0); + let ref_id = ann.get("ref").and_then(|r| r.as_str()).unwrap_or(""); + let role = ann.get("role").and_then(|r| r.as_str()).unwrap_or(""); + let name = ann.get("name").and_then(|n| n.as_str()).unwrap_or(""); + if name.is_empty() { + println!( + " {} @{} {}", + color::dim(&format!("[{}]", num)), + ref_id, + role, + ); + } else { + println!( + " {} @{} {} {:?}", + color::dim(&format!("[{}]", num)), + ref_id, + role, + name, + ); + } + } + } + } "pdf" => println!( "{} PDF saved to {}", color::success_indicator(), @@ -965,6 +991,10 @@ saves to a temporary directory with a generated filename. Options: --full, -f Capture full page (not just viewport) + --annotate Overlay numbered labels on interactive elements. + Each label [N] corresponds to ref @eN from snapshot. + Prints a legend mapping labels to element roles/names. + With --json, annotations are included in the response. Global Options: --json Output as JSON @@ -974,6 +1004,9 @@ Examples: agent-browser screenshot agent-browser screenshot ./screenshot.png agent-browser screenshot --full ./full-page.png + agent-browser screenshot --annotate # Labeled screenshot + legend + agent-browser screenshot --annotate ./page.png # Save annotated screenshot + agent-browser screenshot --annotate --json # JSON output with annotations "## } "pdf" => { @@ -1932,6 +1965,7 @@ Options: --device iOS device name (e.g., "iPhone 15 Pro") --json JSON output --full, -f Full page screenshot + --annotate Annotated screenshot with numbered labels and legend --headed Show browser window (not headless) --cdp Connect via CDP (Chrome DevTools Protocol) --auto-connect Auto-discover and connect to running Chrome @@ -1970,6 +2004,7 @@ Environment: AGENT_BROWSER_HEADED Show browser window (not headless) AGENT_BROWSER_JSON JSON output AGENT_BROWSER_FULL Full page screenshot + AGENT_BROWSER_ANNOTATE Annotated screenshot with numbered labels and legend AGENT_BROWSER_DEBUG Debug output AGENT_BROWSER_IGNORE_HTTPS_ERRORS Ignore HTTPS certificate errors AGENT_BROWSER_PROVIDER Browser provider (ios, browserbase, kernel, browseruse) @@ -1994,6 +2029,7 @@ Examples: agent-browser find role button click --name Submit agent-browser get text @e1 agent-browser screenshot --full + agent-browser screenshot --annotate # Labeled screenshot for vision models agent-browser wait --load networkidle # Wait for slow pages to load agent-browser --cdp 9222 snapshot # Connect via CDP port agent-browser --auto-connect snapshot # Auto-discover running Chrome diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx index 3c51be6..82ed5f4 100644 --- a/docs/src/app/commands/page.mdx +++ b/docs/src/app/commands/page.mdx @@ -23,6 +23,7 @@ agent-browser scrollintoview # Scroll element into view agent-browser drag # Drag and drop agent-browser upload # Upload files agent-browser screenshot [path] # Screenshot (--full for full page) +agent-browser screenshot --annotate # Annotated screenshot with numbered element labels agent-browser pdf # Save page as PDF agent-browser snapshot # Accessibility tree with refs agent-browser eval # Run JavaScript @@ -236,6 +237,7 @@ agent-browser reload # Reload page --device # iOS device name (e.g., "iPhone 15 Pro") --json # JSON output (for scripts) --full, -f # Full page screenshot +--annotate # Annotated screenshot with numbered element labels --headed # Show browser window (not headless) --cdp # Connect via Chrome DevTools Protocol (port or WebSocket URL) --auto-connect # Auto-discover and connect to running Chrome diff --git a/docs/src/app/snapshots/page.mdx b/docs/src/app/snapshots/page.mdx index 21308f0..c26a5d2 100644 --- a/docs/src/app/snapshots/page.mdx +++ b/docs/src/app/snapshots/page.mdx @@ -78,12 +78,28 @@ agent-browser snapshot -i # Get fresh refs agent-browser click @e1 # Use new refs ``` +## Annotated screenshots + +For visual context alongside text snapshots, use `screenshot --annotate` to overlay numbered labels on interactive elements. Each label `[N]` maps to ref `@eN`: + +```bash +agent-browser screenshot --annotate ./page.png +# -> Screenshot saved to ./page.png +# [1] @e1 button "Submit" +# [2] @e2 link "Home" +# [3] @e3 textbox "Email" +agent-browser click @e2 +``` + +Annotated screenshots also cache refs, so you can interact with elements immediately. This is useful when the text snapshot is insufficient -- unlabeled icons, canvas content, or visual layout verification. + ## Best practices 1. Use `-i` to reduce output to actionable elements 2. Re-snapshot after page changes to get updated refs 3. Scope with `-s` for specific page sections 4. Use `-d` to limit depth on complex pages +5. Use `screenshot --annotate` when visual context is needed alongside refs ## JSON output diff --git a/skills/agent-browser/SKILL.md b/skills/agent-browser/SKILL.md index 9ceeef8..ea452e2 100644 --- a/skills/agent-browser/SKILL.md +++ b/skills/agent-browser/SKILL.md @@ -80,6 +80,7 @@ agent-browser wait 2000 # Wait milliseconds # Capture agent-browser screenshot # Screenshot to temp dir agent-browser screenshot --full # Full page screenshot +agent-browser screenshot --annotate # Annotated screenshot with numbered element labels agent-browser pdf output.pdf # Save as PDF ``` @@ -278,6 +279,25 @@ agent-browser snapshot -i # MUST re-snapshot agent-browser click @e1 # Use new refs ``` +## Annotated Screenshots (Vision Mode) + +Use `--annotate` to take a screenshot with numbered labels overlaid on interactive elements. Each label `[N]` maps to ref `@eN`. This also caches refs, so you can interact with elements immediately without a separate snapshot. + +```bash +agent-browser screenshot --annotate +# Output includes the image path and a legend: +# [1] @e1 button "Submit" +# [2] @e2 link "Home" +# [3] @e3 textbox "Email" +agent-browser click @e2 # Click using ref from annotated screenshot +``` + +Use annotated screenshots when: +- The page has unlabeled icon buttons or visual-only elements +- You need to verify visual layout or styling +- Canvas or chart elements are present (invisible to text snapshots) +- You need spatial reasoning about element positions + ## Semantic Locators (Alternative to Refs) When refs are unavailable or unreliable, use semantic locators: diff --git a/src/actions.ts b/src/actions.ts index 5490fa0..4889b9b 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -123,6 +123,7 @@ import type { RecordingStartCommand, RecordingStopCommand, RecordingRestartCommand, + Annotation, NavigateData, ScreenshotData, EvaluateData, @@ -599,6 +600,16 @@ async function handlePress(command: PressCommand, browser: BrowserManager): Prom return successResponse(command.id, { pressed: true }); } +const ANNOTATION_OVERLAY_ID = '__agent_browser_annotations__'; + +async function removeAnnotationOverlay(page: Page): Promise { + await page + .evaluate( + `(() => { const el = document.getElementById(${JSON.stringify(ANNOTATION_OVERLAY_ID)}); if (el) el.remove(); })()` + ) + .catch(() => {}); +} + async function handleScreenshot( command: ScreenshotCommand, browser: BrowserManager @@ -619,6 +630,8 @@ async function handleScreenshot( target = browser.getLocator(command.selector); } + let overlayInjected = false; + try { let savePath = command.path; if (!savePath) { @@ -631,9 +644,158 @@ async function handleScreenshot( savePath = path.join(screenshotDir, filename); } + let annotations: Annotation[] | undefined; + + if (command.annotate) { + const { refs } = await browser.getSnapshot({ interactive: true }); + + const entries = Object.entries(refs); + const results = await Promise.all( + entries.map(async ([ref, data]): Promise => { + try { + const locator = browser.getLocatorFromRef(ref); + if (!locator) return null; + const box = await locator.boundingBox(); + if (!box || box.width === 0 || box.height === 0) return null; + const num = parseInt(ref.replace('e', ''), 10); + return { + ref, + number: num, + role: data.role, + name: data.name || undefined, + box: { + x: Math.round(box.x), + y: Math.round(box.y), + width: Math.round(box.width), + height: Math.round(box.height), + }, + }; + } catch { + return null; + } + }) + ); + + // When a selector is provided the screenshot is cropped to that element, + // so filter to annotations that overlap the target and shift coordinates. + let targetBox: { x: number; y: number; width: number; height: number } | null = null; + if (command.selector) { + const raw = await browser.getLocator(command.selector).boundingBox(); + if (raw) { + targetBox = { + x: Math.round(raw.x), + y: Math.round(raw.y), + width: Math.round(raw.width), + height: Math.round(raw.height), + }; + } + } + + const filtered = results.filter((a): a is Annotation => a !== null); + + // Filter by selector overlap if needed, but keep viewport-relative coords + // for overlay positioning. Coordinate shifting happens later for metadata only. + let overlayItems: Annotation[]; + if (targetBox) { + const tb = targetBox; + overlayItems = filtered + .filter((a) => { + const ax2 = a.box.x + a.box.width; + const ay2 = a.box.y + a.box.height; + const bx2 = tb.x + tb.width; + const by2 = tb.y + tb.height; + return a.box.x < bx2 && ax2 > tb.x && a.box.y < by2 && ay2 > tb.y; + }) + .sort((a, b) => a.number - b.number); + } else { + overlayItems = filtered.sort((a, b) => a.number - b.number); + } + + if (overlayItems.length > 0) { + const overlayData = overlayItems.map((a) => ({ + number: a.number, + x: a.box.x, + y: a.box.y, + width: a.box.width, + height: a.box.height, + })); + + // Uses position:absolute with document-relative coords so labels render + // correctly for both viewport and fullPage screenshots, and when the + // screenshot is scoped to a selector element. + await page.evaluate(`(() => { + var items = ${JSON.stringify(overlayData)}; + var id = ${JSON.stringify(ANNOTATION_OVERLAY_ID)}; + var sx = window.scrollX || 0; + var sy = window.scrollY || 0; + var c = document.createElement('div'); + c.id = id; + c.style.cssText = 'position:absolute;top:0;left:0;width:0;height:0;pointer-events:none;z-index:2147483647;'; + for (var i = 0; i < items.length; i++) { + var it = items[i]; + var dx = it.x + sx; + var dy = it.y + sy; + var b = document.createElement('div'); + b.style.cssText = 'position:absolute;left:' + dx + 'px;top:' + dy + 'px;width:' + it.width + 'px;height:' + it.height + 'px;border:2px solid rgba(255,0,0,0.8);box-sizing:border-box;pointer-events:none;'; + var l = document.createElement('div'); + l.textContent = String(it.number); + var labelTop = dy < 14 ? '2px' : '-14px'; + l.style.cssText = 'position:absolute;top:' + labelTop + ';left:-2px;background:rgba(255,0,0,0.9);color:#fff;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;white-space:nowrap;'; + b.appendChild(l); + c.appendChild(b); + } + document.documentElement.appendChild(c); + })()`); + overlayInjected = true; + } + + // Build returned annotation metadata with image-relative coordinates. + // Selector: shift to target-element-relative. + // fullPage: convert to document-relative (matching fullPage image origin). + // Default: viewport-relative (unchanged). + if (targetBox) { + const tb = targetBox; + annotations = overlayItems.map((a) => ({ + ...a, + box: { + x: a.box.x - tb.x, + y: a.box.y - tb.y, + width: a.box.width, + height: a.box.height, + }, + })); + } else if (command.fullPage) { + const scroll = (await page.evaluate( + `({x: window.scrollX || 0, y: window.scrollY || 0})` + )) as { x: number; y: number }; + annotations = overlayItems.map((a) => ({ + ...a, + box: { + x: a.box.x + scroll.x, + y: a.box.y + scroll.y, + width: a.box.width, + height: a.box.height, + }, + })); + } else { + annotations = overlayItems; + } + } + await target.screenshot({ ...options, path: savePath }); - return successResponse(command.id, { path: savePath }); + + if (overlayInjected) { + await removeAnnotationOverlay(page); + } + + return successResponse(command.id, { + path: savePath, + ...(annotations && annotations.length > 0 ? { annotations } : {}), + }); } catch (error) { + if (overlayInjected) { + await removeAnnotationOverlay(page); + } if (command.selector) { throw toAIFriendlyError(error, command.selector); } diff --git a/src/browser.test.ts b/src/browser.test.ts index f83d5e5..44e5352 100644 --- a/src/browser.test.ts +++ b/src/browser.test.ts @@ -284,6 +284,133 @@ describe('BrowserManager', () => { }); }); + describe('annotated screenshots', () => { + afterAll(async () => { + await browser.getPage().goto('https://example.com'); + }); + + it('should return annotations with correct shape', async () => { + const page = browser.getPage(); + await page.setContent(` + + + Home + + + `); + + const result = await executeCommand( + { id: 'ann-1', action: 'screenshot', annotate: true }, + browser + ); + + expect(result.success).toBe(true); + const data = result.data as { path?: string; annotations?: unknown[] }; + expect(data.path).toBeDefined(); + expect(data.annotations).toBeDefined(); + expect(data.annotations!.length).toBeGreaterThan(0); + + for (const ann of data.annotations! as Array<{ + ref: string; + number: number; + role: string; + name?: string; + box: { x: number; y: number; width: number; height: number }; + }>) { + expect(ann.ref).toMatch(/^e\d+$/); + expect(typeof ann.number).toBe('number'); + expect(typeof ann.role).toBe('string'); + expect(typeof ann.box.x).toBe('number'); + expect(typeof ann.box.y).toBe('number'); + expect(typeof ann.box.width).toBe('number'); + expect(typeof ann.box.height).toBe('number'); + } + }); + + it('should clean up overlay from DOM after screenshot', async () => { + const page = browser.getPage(); + await page.setContent(` + + + + `); + + await executeCommand({ id: 'ann-2', action: 'screenshot', annotate: true }, browser); + + const overlay = await page.$('#__agent_browser_annotations__'); + expect(overlay).toBeNull(); + }); + + it('should scope annotations to selector element', async () => { + const page = browser.getPage(); + await page.setContent(` + + +
+ +
+ + `); + + const result = await executeCommand( + { id: 'ann-3', action: 'screenshot', annotate: true, selector: '#container' }, + browser + ); + + expect(result.success).toBe(true); + const data = result.data as { annotations?: Array<{ name?: string }> }; + expect(data.annotations).toBeDefined(); + + const names = data.annotations!.map((a) => a.name).filter(Boolean); + expect(names).toContain('Inside'); + expect(names).not.toContain('Outside'); + }); + + it('should succeed with no annotations on static page', async () => { + const page = browser.getPage(); + await page.setContent(` + +

Just some text, no interactive elements.

+ + `); + + const result = await executeCommand( + { id: 'ann-4', action: 'screenshot', annotate: true }, + browser + ); + + expect(result.success).toBe(true); + const data = result.data as { path?: string; annotations?: unknown[] }; + expect(data.path).toBeDefined(); + expect(data.annotations).toBeUndefined(); + }); + + it('should return document-relative coords for fullPage screenshots', async () => { + const page = browser.getPage(); + await page.setContent(` + +
+ + + `); + + const result = await executeCommand( + { id: 'ann-5', action: 'screenshot', annotate: true, fullPage: true }, + browser + ); + + expect(result.success).toBe(true); + const data = result.data as { + annotations?: Array<{ name?: string; box: { y: number } }>; + }; + expect(data.annotations).toBeDefined(); + + const bottom = data.annotations!.find((a) => a.name === 'Bottom'); + expect(bottom).toBeDefined(); + expect(bottom!.box.y).toBeGreaterThanOrEqual(2000); + }); + }); + describe('evaluate', () => { it('should evaluate JavaScript', async () => { const page = browser.getPage(); diff --git a/src/protocol.test.ts b/src/protocol.test.ts index 92ef74b..5c3aac0 100644 --- a/src/protocol.test.ts +++ b/src/protocol.test.ts @@ -129,6 +129,11 @@ describe('parseCommand', () => { ); expect(result.success).toBe(true); }); + + it('should parse screenshot with annotate', () => { + const result = parseCommand(cmd({ id: '1', action: 'screenshot', annotate: true })); + expect(result.success).toBe(true); + }); }); describe('cookies', () => { diff --git a/src/protocol.ts b/src/protocol.ts index 9b72c6a..e43bd2d 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -753,6 +753,7 @@ const screenshotSchema = baseCommandSchema.extend({ selector: z.string().min(1).nullish(), format: z.enum(['png', 'jpeg']).optional(), quality: z.number().min(0).max(100).optional(), + annotate: z.boolean().optional(), }); const snapshotSchema = baseCommandSchema.extend({ diff --git a/src/types.ts b/src/types.ts index 8f20bc4..0723dcc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -807,6 +807,7 @@ export interface ScreenshotCommand extends BaseCommand { selector?: string; format?: 'png' | 'jpeg'; quality?: number; + annotate?: boolean; } export interface SnapshotCommand extends BaseCommand { @@ -1036,9 +1037,18 @@ export interface NavigateData { title: string; } +export interface Annotation { + ref: string; + number: number; + role: string; + name?: string; + box: { x: number; y: number; width: number; height: number }; +} + export interface ScreenshotData { path?: string; base64?: string; + annotations?: Annotation[]; } export interface SnapshotData {