diff --git a/README.md b/README.md index e8ae669..5b05c21 100644 --- a/README.md +++ b/README.md @@ -445,6 +445,8 @@ The `-C` flag is useful for modern web apps that use custom clickable elements ( The `--annotate` flag overlays numbered labels on interactive elements in the screenshot. Each label `[N]` corresponds to ref `@eN`, so the same refs work for both visual and text-based workflows. +In native mode, annotated screenshots are supported on the CDP-backed browser path (`--native` with Chromium/Lightpanda). The Safari/WebDriver backend does not yet support `--annotate`. + ```bash agent-browser screenshot --annotate # -> Screenshot saved to /tmp/screenshot-2026-02-17T12-00-00-abc123.png diff --git a/cli/src/main.rs b/cli/src/main.rs index c4f8735..c1453da 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -30,6 +30,31 @@ use output::{ use std::path::PathBuf; use std::process::Command as ProcessCommand; +fn serialize_json_value(value: &serde_json::Value) -> String { + serde_json::to_string(value).unwrap_or_else(|_| { + r#"{"success":false,"error":"Failed to serialize JSON response"}"#.to_string() + }) +} + +fn print_json_value(value: serde_json::Value) { + println!("{}", serialize_json_value(&value)); +} + +fn print_json_error(message: impl AsRef) { + print_json_value(json!({ + "success": false, + "error": message.as_ref(), + })); +} + +fn print_json_error_with_type(message: impl AsRef, error_type: &str) { + print_json_value(json!({ + "success": false, + "error": message.as_ref(), + "type": error_type, + })); +} + /// Run a local auth command (auth_save/list/show/delete) via node auth-cli.js. /// These commands don't need a browser, so we handle them directly to avoid /// sending passwords through the daemon's Unix socket channel. @@ -63,7 +88,7 @@ fn run_auth_cli(cmd: &serde_json::Value, json_mode: bool) -> ! { Some(p) => p.clone(), None => { if json_mode { - println!(r#"{{"success":false,"error":"auth-cli.js not found"}}"#); + print_json_error("auth-cli.js not found"); } else { eprintln!( "{} auth-cli.js not found. Set AGENT_BROWSER_HOME or run from project directory.", @@ -92,7 +117,7 @@ fn run_auth_cli(cmd: &serde_json::Value, json_mode: bool) -> ! { if stdout.is_empty() { if json_mode { - println!(r#"{{"success":false,"error":"No response from auth-cli"}}"#); + print_json_error("No response from auth-cli"); } else { eprintln!("{} No response from auth-cli", color::error_indicator()); } @@ -125,10 +150,7 @@ fn run_auth_cli(cmd: &serde_json::Value, json_mode: bool) -> ! { } Err(e) => { if json_mode { - println!( - r#"{{"success":false,"error":"Failed to run auth-cli: {}"}}"#, - e - ); + print_json_error(format!("Failed to run auth-cli: {}", e)); } else { eprintln!("{} Failed to run auth-cli: {}", color::error_indicator(), e); } @@ -235,7 +257,12 @@ fn run_session(args: &[String], session: &str, json_mode: bool) { None | Some(_) => { // Just show current session if json_mode { - println!(r#"{{"success":true,"data":{{"session":"{}"}}}}"#, session); + print_json_value(json!({ + "success": true, + "data": { + "session": session, + }, + })); } else { println!("{}", session); } @@ -326,11 +353,7 @@ fn main() { ParseError::InvalidValue { .. } => "invalid_value", ParseError::InvalidSessionName { .. } => "invalid_session_name", }; - println!( - r#"{{"success":false,"error":"{}","type":"{}"}}"#, - e.format().replace('\n', " "), - error_type - ); + print_json_error_with_type(e.format(), error_type); } else { eprintln!("{}", color::red(&e.format())); } @@ -385,10 +408,7 @@ fn main() { if !validation::is_valid_session_name(name) { let msg = validation::session_name_error(name); if flags.json { - println!( - r#"{{"success":false,"error":"{}","type":"invalid_session_name"}}"#, - msg.replace('"', "\\\"") - ); + print_json_error_with_type(msg, "invalid_session_name"); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -423,7 +443,7 @@ fn main() { Ok(result) => result, Err(e) => { if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, e); + print_json_error(e); } else { eprintln!("{} {}", color::error_indicator(), e); } @@ -494,7 +514,7 @@ fn main() { if flags.cdp.is_some() && flags.provider.is_some() { let msg = "Cannot use --cdp and -p/--provider together"; if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -504,7 +524,7 @@ fn main() { if flags.auto_connect && flags.cdp.is_some() { let msg = "Cannot use --auto-connect and --cdp together"; if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -514,7 +534,7 @@ fn main() { if flags.auto_connect && flags.provider.is_some() { let msg = "Cannot use --auto-connect and -p/--provider together"; if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -524,7 +544,7 @@ fn main() { if flags.provider.is_some() && !flags.extensions.is_empty() { let msg = "Cannot use --extension with -p/--provider (extensions require local browser)"; if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -534,7 +554,7 @@ fn main() { if flags.cdp.is_some() && !flags.extensions.is_empty() { let msg = "Cannot use --extension with --cdp (extensions require local browser)"; if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -572,7 +592,7 @@ fn main() { if let Some(msg) = err { if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -600,7 +620,7 @@ fn main() { Ok(0) => { let msg = "Invalid CDP port: port must be greater than 0".to_string(); if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(&msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -612,7 +632,7 @@ fn main() { p ); if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(&msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -625,7 +645,7 @@ fn main() { cdp_value ); if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(&msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -662,7 +682,7 @@ fn main() { if let Some(msg) = err { if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -693,7 +713,7 @@ fn main() { if let Some(msg) = err { if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, msg); + print_json_error(msg); } else { eprintln!("{} {}", color::error_indicator(), msg); } @@ -797,7 +817,7 @@ fn main() { .error .unwrap_or_else(|| "Browser launch failed".to_string()); if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, error_msg); + print_json_error(error_msg); } else { eprintln!("{} {}", color::error_indicator(), error_msg); } @@ -805,7 +825,7 @@ fn main() { } Err(e) => { if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, e); + print_json_error(e); } else { eprintln!( "{} Could not configure browser: {}", @@ -892,7 +912,7 @@ fn main() { } Err(e) => { if flags.json { - println!(r#"{{"success":false,"error":"{}"}}"#, e); + print_json_error(e); } else { eprintln!("{} {}", color::error_indicator(), e); } @@ -958,4 +978,19 @@ mod tests { assert_eq!(result["username"], "user"); assert_eq!(result["password"], "p@ss:w0rd"); } + + #[test] + fn test_serialize_json_value_escapes_control_characters() { + let payload = serialize_json_value(&json!({ + "success": false, + "error": "Daemon process exited during startup:\nline \"quoted\"\u{001b}[2mansi\u{001b}[22m", + })); + + let parsed: serde_json::Value = serde_json::from_str(&payload).unwrap(); + assert_eq!(parsed["success"], false); + assert_eq!( + parsed["error"], + "Daemon process exited during startup:\nline \"quoted\"\u{001b}[2mansi\u{001b}[22m" + ); + } } diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 573974f..f597a78 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -1295,8 +1295,20 @@ async fn handle_snapshot(cmd: &Value, state: &mut DaemonState) -> Result Result { + let annotate = cmd + .get("annotate") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if let Some(ref wb) = state.webdriver_backend { if state.browser.is_none() { + if annotate { + return Err( + "Annotated screenshots are not yet implemented on the WebDriver backend" + .to_string(), + ); + } + let base64_data = wb.screenshot().await?; let path = cmd.get("path").and_then(|v| v.as_str()); if let Some(p) = path { @@ -1349,12 +1361,33 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result Result { @@ -4103,13 +4136,14 @@ async fn handle_diff_screenshot(cmd: &Value, state: &DaemonState) -> Result Result { let session_id = self.active_session_id()?.to_string(); + let mut lifecycle_rx = self.client.subscribe(); let nav_result: PageNavigateResult = self .client @@ -429,7 +430,8 @@ impl BrowserManager { return Err(format!("Navigation failed: {}", error_text)); } - self.wait_for_lifecycle(wait_until, &session_id).await?; + self.wait_for_lifecycle(wait_until, &session_id, &mut lifecycle_rx) + .await?; let page_url = self.get_url().await.unwrap_or_else(|_| url.to_string()); let title = self.get_title().await.unwrap_or_default(); @@ -446,14 +448,14 @@ impl BrowserManager { &self, wait_until: WaitUntil, session_id: &str, + rx: &mut broadcast::Receiver, ) -> Result<(), String> { let event_name = match wait_until { WaitUntil::Load => "Page.loadEventFired", WaitUntil::DomContentLoaded => "Page.domContentEventFired", - WaitUntil::NetworkIdle => return self.wait_for_network_idle(session_id).await, + WaitUntil::NetworkIdle => return self.wait_for_network_idle(session_id, rx).await, }; - let mut rx = self.client.subscribe(); let timeout = tokio::time::Duration::from_millis(self.default_timeout_ms); tokio::time::timeout(timeout, async { @@ -468,8 +470,11 @@ impl BrowserManager { .map_err(|_| format!("Timeout waiting for {}", event_name))? } - async fn wait_for_network_idle(&self, session_id: &str) -> Result<(), String> { - let mut rx = self.client.subscribe(); + async fn wait_for_network_idle( + &self, + session_id: &str, + rx: &mut broadcast::Receiver, + ) -> Result<(), String> { let pending = Arc::new(Mutex::new(HashSet::::new())); let timeout = tokio::time::Duration::from_millis(self.default_timeout_ms); @@ -588,7 +593,9 @@ impl BrowserManager { wait_until: WaitUntil, session_id: &str, ) -> Result<(), String> { - self.wait_for_lifecycle(wait_until, session_id).await + let mut rx = self.client.subscribe(); + self.wait_for_lifecycle(wait_until, session_id, &mut rx) + .await } pub async fn close(&mut self) -> Result<(), String> { diff --git a/cli/src/native/e2e_tests.rs b/cli/src/native/e2e_tests.rs index d448b40..be6998c 100644 --- a/cli/src/native/e2e_tests.rs +++ b/cli/src/native/e2e_tests.rs @@ -197,6 +197,74 @@ async fn e2e_screenshot() { assert!(std::path::Path::new(&tmp_path).exists()); let _ = std::fs::remove_file(&tmp_path); + let resp = execute_command( + &json!({ + "id": "5", + "action": "setcontent", + "html": r##" + + + Home +
+ + "##, + }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "6", "action": "screenshot", "annotate": true }), + &mut state, + ) + .await; + assert_success(&resp); + let annotations = get_data(&resp)["annotations"] + .as_array() + .expect("Annotated screenshot should return annotations"); + assert!( + !annotations.is_empty(), + "Annotated screenshot should have at least one annotation" + ); + + let submit_ref = annotations + .iter() + .find(|ann| ann.get("name").and_then(|v| v.as_str()) == Some("Submit")) + .and_then(|ann| ann.get("ref").and_then(|v| v.as_str())) + .expect("Expected a Submit annotation"); + + let resp = execute_command( + &json!({ + "id": "7", + "action": "evaluate", + "script": "document.getElementById('__agent_browser_annotations__') === null" + }), + &mut state, + ) + .await; + assert_success(&resp); + assert_eq!(get_data(&resp)["result"], true); + + let resp = execute_command( + &json!({ "id": "8", "action": "click", "selector": format!("@{}", submit_ref) }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ + "id": "9", + "action": "evaluate", + "script": "document.getElementById('result').textContent" + }), + &mut state, + ) + .await; + assert_success(&resp); + assert_eq!(get_data(&resp)["result"], "clicked"); + let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await; assert_success(&resp); } diff --git a/cli/src/native/element.rs b/cli/src/native/element.rs index 4fc7249..b21ef00 100644 --- a/cli/src/native/element.rs +++ b/cli/src/native/element.rs @@ -47,10 +47,47 @@ impl RefMap { ); } + pub fn add_selector( + &mut self, + ref_id: String, + selector: String, + role: &str, + name: &str, + nth: Option, + ) { + self.map.insert( + ref_id, + RefEntry { + backend_node_id: None, + role: role.to_string(), + name: name.to_string(), + nth, + selector: Some(selector), + }, + ); + } + pub fn get(&self, ref_id: &str) -> Option<&RefEntry> { self.map.get(ref_id) } + pub fn entries_sorted(&self) -> Vec<(String, RefEntry)> { + let mut entries = self + .map + .iter() + .map(|(ref_id, entry)| (ref_id.clone(), entry.clone())) + .collect::>(); + + entries.sort_by_key(|(ref_id, _)| { + ref_id + .strip_prefix('e') + .and_then(|n| n.parse::().ok()) + .unwrap_or(usize::MAX) + }); + + entries + } + pub fn clear(&mut self) { self.map.clear(); self.next_ref = 1; diff --git a/cli/src/native/screenshot.rs b/cli/src/native/screenshot.rs index d325dbe..3ff3fd3 100644 --- a/cli/src/native/screenshot.rs +++ b/cli/src/native/screenshot.rs @@ -1,3 +1,4 @@ +use serde::Serialize; use serde_json::Value; use std::path::PathBuf; @@ -5,12 +6,57 @@ use super::cdp::client::CdpClient; use super::cdp::types::*; use super::element::RefMap; +const ANNOTATION_OVERLAY_ID: &str = "__agent_browser_annotations__"; + +#[derive(Debug, Clone)] +struct Rect { + x: f64, + y: f64, + width: f64, + height: f64, +} + +#[derive(Debug, Clone)] +struct RawAnnotation { + ref_id: String, + number: u64, + role: String, + name: Option, + rect: Rect, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AnnotationBox { + pub x: i64, + pub y: i64, + pub width: i64, + pub height: i64, +} + +#[derive(Debug, Clone)] +pub struct ScreenshotAnnotation { + pub ref_id: String, + pub number: u64, + pub role: String, + pub name: Option, + pub box_: AnnotationBox, +} + +#[derive(Debug, Clone)] +pub struct ScreenshotResult { + pub path: String, + pub base64: String, + pub annotations: Vec, +} + +#[derive(Debug, Clone)] pub struct ScreenshotOptions { pub selector: Option, pub path: Option, pub full_page: bool, pub format: String, pub quality: Option, + pub annotate: bool, } impl Default for ScreenshotOptions { @@ -21,16 +67,99 @@ impl Default for ScreenshotOptions { full_page: false, format: "png".to_string(), quality: None, + annotate: false, } } } +impl Serialize for ScreenshotAnnotation { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + + let mut state = serializer.serialize_struct("ScreenshotAnnotation", 5)?; + state.serialize_field("ref", &self.ref_id)?; + state.serialize_field("number", &self.number)?; + state.serialize_field("role", &self.role)?; + if let Some(name) = &self.name { + state.serialize_field("name", name)?; + } + state.serialize_field("box", &self.box_)?; + state.end() + } +} + +/// Captures a screenshot via CDP and optionally overlays numbered annotations +/// that mirror the Node.js screenshot `annotate` mode. pub async fn take_screenshot( client: &CdpClient, session_id: &str, ref_map: &RefMap, options: &ScreenshotOptions, -) -> Result<(String, String), String> { +) -> Result { + let target_rect = if options.annotate { + match options.selector.as_deref() { + Some(selector) => get_rect_for_selector(client, session_id, ref_map, selector).await?, + None => None, + } + } else { + None + }; + + let raw_annotations = if options.annotate { + collect_annotations(client, session_id, ref_map).await? + } else { + Vec::new() + }; + + let overlay_items = filter_annotations(raw_annotations, target_rect.as_ref()); + let overlay_injected = if options.annotate && !overlay_items.is_empty() { + inject_annotation_overlay(client, session_id, &overlay_items).await?; + true + } else { + false + }; + + let base64 = capture_screenshot_base64(client, session_id, ref_map, options).await; + + if overlay_injected { + let _ = remove_annotation_overlay(client, session_id).await; + } + + let base64 = base64?; + let annotations = if options.annotate { + let scroll = if options.full_page { + Some(get_scroll_offsets(client, session_id).await?) + } else { + None + }; + project_annotations(&overlay_items, target_rect.as_ref(), scroll) + } else { + Vec::new() + }; + + let ext = if options.format == "jpeg" { + "jpg" + } else { + "png" + }; + let path = save_screenshot(&base64, options.path.as_deref(), ext)?; + + Ok(ScreenshotResult { + path, + base64, + annotations, + }) +} + +async fn capture_screenshot_base64( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, + options: &ScreenshotOptions, +) -> Result { let mut params = CaptureScreenshotParams { format: Some(options.format.clone()), quality: if options.format == "jpeg" { @@ -64,40 +193,12 @@ pub async fn take_screenshot( }); } } else if let Some(ref selector) = options.selector { - // Element screenshot via bounding box - let object_id = - super::element::resolve_element_object_id(client, session_id, ref_map, selector) - .await?; - - let result: EvaluateResult = client - .send_command_typed( - "Runtime.callFunctionOn", - &CallFunctionOnParams { - function_declaration: r#"function() { - const rect = this.getBoundingClientRect(); - return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; - }"# - .to_string(), - object_id: Some(object_id), - arguments: None, - return_by_value: Some(true), - await_promise: Some(false), - }, - Some(session_id), - ) - .await?; - - if let Some(rect) = result.result.value { - let x = rect.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let y = rect.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); - let w = rect.get("width").and_then(|v| v.as_f64()).unwrap_or(100.0); - let h = rect.get("height").and_then(|v| v.as_f64()).unwrap_or(100.0); - + if let Some(rect) = get_rect_for_selector(client, session_id, ref_map, selector).await? { params.clip = Some(Viewport { - x, - y, - width: w, - height: h, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, scale: 1.0, }); } @@ -107,14 +208,280 @@ pub async fn take_screenshot( .send_command_typed("Page.captureScreenshot", ¶ms, Some(session_id)) .await?; - let ext = if options.format == "jpeg" { - "jpg" - } else { - "png" - }; + Ok(result.data) +} - let save_path = match &options.path { - Some(p) => p.clone(), +async fn collect_annotations( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, +) -> Result, String> { + let mut annotations = Vec::new(); + + for (ref_id, entry) in ref_map.entries_sorted() { + let object_id = + match super::element::resolve_element_object_id(client, session_id, ref_map, &ref_id) + .await + { + Ok(id) => id, + Err(_) => continue, + }; + + let Some(rect) = get_rect_for_object(client, session_id, &object_id).await? else { + continue; + }; + + if rect.width <= 0.0 || rect.height <= 0.0 { + continue; + } + + let number = ref_id + .strip_prefix('e') + .and_then(|n| n.parse::().ok()) + .unwrap_or(0); + + annotations.push(RawAnnotation { + ref_id, + number, + role: entry.role, + name: (!entry.name.is_empty()).then_some(entry.name), + rect, + }); + } + + Ok(annotations) +} + +async fn get_rect_for_selector( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, + selector: &str, +) -> Result, String> { + let object_id = + super::element::resolve_element_object_id(client, session_id, ref_map, selector).await?; + get_rect_for_object(client, session_id, &object_id).await +} + +async fn get_rect_for_object( + client: &CdpClient, + session_id: &str, + object_id: &str, +) -> Result, String> { + let result: EvaluateResult = client + .send_command_typed( + "Runtime.callFunctionOn", + &CallFunctionOnParams { + function_declaration: r#"function() { + const rect = this.getBoundingClientRect(); + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; + }"# + .to_string(), + object_id: Some(object_id.to_string()), + arguments: None, + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(session_id), + ) + .await?; + + Ok(result.result.value.as_ref().and_then(parse_rect)) +} + +fn parse_rect(value: &Value) -> Option { + Some(Rect { + x: value.get("x")?.as_f64()?, + y: value.get("y")?.as_f64()?, + width: value.get("width")?.as_f64()?, + height: value.get("height")?.as_f64()?, + }) +} + +fn filter_annotations( + annotations: Vec, + target_rect: Option<&Rect>, +) -> Vec { + let mut items = annotations + .into_iter() + .filter(|annotation| match target_rect { + Some(target) => overlaps(&annotation.rect, target), + None => true, + }) + .collect::>(); + + items.sort_by_key(|annotation| annotation.number); + items +} + +fn overlaps(left: &Rect, right: &Rect) -> bool { + let left_x2 = left.x + left.width; + let left_y2 = left.y + left.height; + let right_x2 = right.x + right.width; + let right_y2 = right.y + right.height; + + left.x < right_x2 && left_x2 > right.x && left.y < right_y2 && left_y2 > right.y +} + +async fn inject_annotation_overlay( + client: &CdpClient, + session_id: &str, + annotations: &[RawAnnotation], +) -> Result<(), String> { + let overlay_data = annotations + .iter() + .map(|annotation| { + serde_json::json!({ + "number": annotation.number, + "x": round(annotation.rect.x), + "y": round(annotation.rect.y), + "width": round(annotation.rect.width), + "height": round(annotation.rect.height), + }) + }) + .collect::>(); + + let expression = format!( + r#"(() => {{ + var items = {items}; + var id = {overlay_id}; + var existing = document.getElementById(id); + if (existing) existing.remove(); + var sx = window.scrollX || 0; + var sy = window.scrollY || 0; + var c = document.createElement('div'); + c.id = id; + c.style.cssText = 'position:absolute;top:0;left:0;width:0;height:0;pointer-events:none;z-index:2147483647;'; + for (var i = 0; i < items.length; i++) {{ + var it = items[i]; + var dx = it.x + sx; + var dy = it.y + sy; + var b = document.createElement('div'); + b.style.cssText = 'position:absolute;left:' + dx + 'px;top:' + dy + 'px;width:' + it.width + 'px;height:' + it.height + 'px;border:2px solid rgba(255,0,0,0.8);box-sizing:border-box;pointer-events:none;'; + var l = document.createElement('div'); + l.textContent = String(it.number); + var labelTop = dy < 14 ? '2px' : '-14px'; + l.style.cssText = 'position:absolute;top:' + labelTop + ';left:-2px;background:rgba(255,0,0,0.9);color:#fff;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;white-space:nowrap;'; + b.appendChild(l); + c.appendChild(b); + }} + document.documentElement.appendChild(c); + return true; + }})()"#, + items = serde_json::to_string(&overlay_data).unwrap_or_else(|_| "[]".to_string()), + overlay_id = + serde_json::to_string(ANNOTATION_OVERLAY_ID).unwrap_or_else(|_| "\"\"".to_string()), + ); + + let _: EvaluateResult = client + .send_command_typed( + "Runtime.evaluate", + &EvaluateParams { + expression, + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(session_id), + ) + .await?; + + Ok(()) +} + +async fn remove_annotation_overlay(client: &CdpClient, session_id: &str) -> Result<(), String> { + let expression = format!( + r#"(() => {{ + var el = document.getElementById({overlay_id}); + if (el) el.remove(); + return true; + }})()"#, + overlay_id = + serde_json::to_string(ANNOTATION_OVERLAY_ID).unwrap_or_else(|_| "\"\"".to_string()), + ); + + let _: EvaluateResult = client + .send_command_typed( + "Runtime.evaluate", + &EvaluateParams { + expression, + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(session_id), + ) + .await?; + + Ok(()) +} + +async fn get_scroll_offsets(client: &CdpClient, session_id: &str) -> Result<(f64, f64), String> { + let result: EvaluateResult = client + .send_command_typed( + "Runtime.evaluate", + &EvaluateParams { + expression: "({x: window.scrollX || 0, y: window.scrollY || 0})".to_string(), + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(session_id), + ) + .await?; + + let value = result.result.value.unwrap_or(Value::Null); + let x = value.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); + let y = value.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); + Ok((x, y)) +} + +fn project_annotations( + annotations: &[RawAnnotation], + target_rect: Option<&Rect>, + scroll: Option<(f64, f64)>, +) -> Vec { + annotations + .iter() + .map(|annotation| { + let rect = if let Some(target) = target_rect { + Rect { + x: annotation.rect.x - target.x, + y: annotation.rect.y - target.y, + width: annotation.rect.width, + height: annotation.rect.height, + } + } else if let Some((scroll_x, scroll_y)) = scroll { + Rect { + x: annotation.rect.x + scroll_x, + y: annotation.rect.y + scroll_y, + width: annotation.rect.width, + height: annotation.rect.height, + } + } else { + annotation.rect.clone() + }; + + ScreenshotAnnotation { + ref_id: annotation.ref_id.clone(), + number: annotation.number, + role: annotation.role.clone(), + name: annotation.name.clone(), + box_: AnnotationBox { + x: round(rect.x), + y: round(rect.y), + width: round(rect.width), + height: round(rect.height), + }, + } + }) + .collect() +} + +fn save_screenshot( + base64_data: &str, + explicit_path: Option<&str>, + ext: &str, +) -> Result { + let save_path = match explicit_path { + Some(path) => path.to_string(), None => { let dir = get_screenshot_dir(); let _ = std::fs::create_dir_all(&dir); @@ -127,13 +494,17 @@ pub async fn take_screenshot( } }; - let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &result.data) + let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, base64_data) .map_err(|e| format!("Failed to decode screenshot: {}", e))?; std::fs::write(&save_path, &bytes) .map_err(|e| format!("Failed to save screenshot to {}: {}", save_path, e))?; - Ok((save_path, result.data)) + Ok(save_path) +} + +fn round(value: f64) -> i64 { + value.round() as i64 } fn get_screenshot_dir() -> PathBuf { @@ -145,3 +516,96 @@ fn get_screenshot_dir() -> PathBuf { .join("screenshots") } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filters_annotations_to_target_overlap() { + let annotations = vec![ + RawAnnotation { + ref_id: "e1".to_string(), + number: 1, + role: "button".to_string(), + name: Some("Inside".to_string()), + rect: Rect { + x: 10.0, + y: 10.0, + width: 50.0, + height: 20.0, + }, + }, + RawAnnotation { + ref_id: "e2".to_string(), + number: 2, + role: "button".to_string(), + name: Some("Outside".to_string()), + rect: Rect { + x: 200.0, + y: 200.0, + width: 40.0, + height: 20.0, + }, + }, + ]; + + let target = Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }; + + let filtered = filter_annotations(annotations, Some(&target)); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].ref_id, "e1"); + } + + #[test] + fn projects_selector_annotations_relative_to_target() { + let annotations = vec![RawAnnotation { + ref_id: "e1".to_string(), + number: 1, + role: "button".to_string(), + name: Some("Inside".to_string()), + rect: Rect { + x: 25.0, + y: 35.0, + width: 40.0, + height: 20.0, + }, + }]; + + let target = Rect { + x: 10.0, + y: 15.0, + width: 100.0, + height: 100.0, + }; + + let projected = project_annotations(&annotations, Some(&target), None); + assert_eq!(projected[0].box_.x, 15); + assert_eq!(projected[0].box_.y, 20); + } + + #[test] + fn projects_full_page_annotations_to_document_space() { + let annotations = vec![RawAnnotation { + ref_id: "e1".to_string(), + number: 1, + role: "button".to_string(), + name: Some("Bottom".to_string()), + rect: Rect { + x: 5.0, + y: 12.0, + width: 40.0, + height: 20.0, + }, + }]; + + let projected = project_annotations(&annotations, None, Some((10.0, 1000.0))); + assert_eq!(projected[0].box_.x, 15); + assert_eq!(projected[0].box_.y, 1012); + } +} diff --git a/cli/src/output.rs b/cli/src/output.rs index 131568c..b251aa5 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -1330,6 +1330,8 @@ Options: Each label [N] corresponds to ref @eN from snapshot. Prints a legend mapping labels to element roles/names. With --json, annotations are included in the response. + In native mode, this is currently supported on the + CDP-backed browser path (Chromium/Lightpanda). Global Options: --json Output as JSON diff --git a/docs/src/app/native-mode/page.mdx b/docs/src/app/native-mode/page.mdx index d74ff7a..a286d42 100644 --- a/docs/src/app/native-mode/page.mdx +++ b/docs/src/app/native-mode/page.mdx @@ -70,6 +70,7 @@ All core commands are supported in native mode: ## Known Limitations - **Firefox and WebKit** are not yet supported (Chromium and Safari only) +- **Annotated screenshots** (`screenshot --annotate`) currently work on the CDP-backed browser path. The Safari/WebDriver backend does not yet support them. - **Playwright trace format** is not available (native tracing uses Chrome's built-in tracing) - **HAR export** is not available - **Network route interception** uses CDP Fetch domain instead of Playwright's route API diff --git a/docs/src/app/snapshots/page.mdx b/docs/src/app/snapshots/page.mdx index a5a9f05..592b367 100644 --- a/docs/src/app/snapshots/page.mdx +++ b/docs/src/app/snapshots/page.mdx @@ -89,6 +89,8 @@ agent-browser click @e1 # Use new refs For visual context alongside text snapshots, use `screenshot --annotate` to overlay numbered labels on interactive elements. Each label `[N]` maps to ref `@eN`: +In native mode, annotated screenshots currently work on the CDP-backed browser path (Chromium/Lightpanda). The Safari/WebDriver backend does not yet support `--annotate`. + ```bash agent-browser screenshot --annotate ./page.png # -> Screenshot saved to ./page.png diff --git a/skills/agent-browser/SKILL.md b/skills/agent-browser/SKILL.md index 52a7a41..da77998 100644 --- a/skills/agent-browser/SKILL.md +++ b/skills/agent-browser/SKILL.md @@ -433,6 +433,8 @@ agent-browser click @e1 # Use new refs Use `--annotate` to take a screenshot with numbered labels overlaid on interactive elements. Each label `[N]` maps to ref `@eN`. This also caches refs, so you can interact with elements immediately without a separate snapshot. +In native mode, this currently works on the CDP-backed browser path (Chromium/Lightpanda). The Safari/WebDriver backend does not yet support `--annotate`. + ```bash agent-browser screenshot --annotate # Output includes the image path and a legend: