oimwiodev/chrome-use
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

annotated screenshots (#503)

Browse Source 
* screenshot annotation

* fixes

* fix CI checks

* fixes

* fixes

* fixes

* fixes

* fixes
This commit is contained in:
Chris Tate
2026-02-18 22:20:01 -06:00
committed by GitHub
parent 06a32f4191
commit e2e259f1e2
12 changed files with 445 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/commands.rs
+10 -1
View File
@@ -2,6 +2,7 @@ use base64::{engine::general_purpose::STANDARD, Engine};
use serde_json::{json, Value};
use std::io::{self, BufRead};
use crate::color;
use crate::flags::Flags;
use crate::validation::{is_valid_session_name, session_name_error};
@@ -82,6 +83,13 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
let rest: Vec<&str> = args[1..].iter().map(|s| s.as_str()).collect();
let id = gen_id();
if flags.annotate && cmd != "screenshot" {
eprintln!(
"{} --annotate only applies to the screenshot command",
color::warning_indicator()
);
}
match cmd {
// === Navigation ===
"open" | "goto" | "navigate" => {
@@ -392,7 +400,7 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
_ => (None, None),
};
Ok(
json!({ "id": id, "action": "screenshot", "path": path, "selector": selector, "fullPage": flags.full }),
json!({ "id": id, "action": "screenshot", "path": path, "selector": selector, "fullPage": flags.full, "annotate": flags.annotate }),
)
}
"pdf" => {
@@ -1583,6 +1591,7 @@ mod tests {
cli_proxy: false,
cli_proxy_bypass: false,
cli_allow_file_access: false,
annotate: false,
}
}
cli/src/flags.rs
+27 -7
View File
@@ -32,6 +32,7 @@ pub struct Config {
pub cdp: Option<String>,
pub auto_connect: Option<bool>,
pub headers: Option<String>,
pub annotate: Option<bool>,
}
impl Config {
@@ -64,6 +65,7 @@ impl Config {
cdp: other.cdp.or(self.cdp),
auto_connect: other.auto_connect.or(self.auto_connect),
headers: other.headers.or(self.headers),
annotate: other.annotate.or(self.annotate),
}
}
}
@@ -84,6 +86,15 @@ fn read_config_file(path: &Path) -> Option<Config> {
}
}
/// Check if a boolean environment variable is set to a truthy value.
/// Returns false when unset, empty, or set to "0", "false", or "no" (case-insensitive).
fn env_var_is_truthy(name: &str) -> bool {
match env::var(name) {
Ok(val) => !matches!(val.to_lowercase().as_str(), "0" | "false" | "no" | ""),
Err(_) => false,
}
}
/// Parse an optional boolean value after a flag. Returns (value, consumed_next_arg).
/// Recognizes "true" as true, "false" as false. Bare flag defaults to true.
fn parse_bool_arg(args: &[String], i: usize) -> (bool, bool) {
@@ -187,6 +198,7 @@ pub struct Flags {
pub device: Option<String>,
pub auto_connect: bool,
pub session_name: Option<String>,
pub annotate: bool,
// Track which launch-time options were explicitly passed via CLI
// (as opposed to being set only via environment variables)
@@ -224,13 +236,13 @@ pub fn parse_flags(args: &[String]) -> Flags {
};
let mut flags = Flags {
json: env::var("AGENT_BROWSER_JSON").is_ok()
json: env_var_is_truthy("AGENT_BROWSER_JSON")
|| config.json.unwrap_or(false),
full: env::var("AGENT_BROWSER_FULL").is_ok()
full: env_var_is_truthy("AGENT_BROWSER_FULL")
|| config.full.unwrap_or(false),
headed: env::var("AGENT_BROWSER_HEADED").is_ok()
headed: env_var_is_truthy("AGENT_BROWSER_HEADED")
|| config.headed.unwrap_or(false),
debug: env::var("AGENT_BROWSER_DEBUG").is_ok()
debug: env_var_is_truthy("AGENT_BROWSER_DEBUG")
|| config.debug.unwrap_or(false),
session: env::var("AGENT_BROWSER_SESSION").ok()
.or(config.session)
@@ -254,16 +266,18 @@ pub fn parse_flags(args: &[String]) -> Flags {
.or(config.user_agent),
provider: env::var("AGENT_BROWSER_PROVIDER").ok()
.or(config.provider),
ignore_https_errors: env::var("AGENT_BROWSER_IGNORE_HTTPS_ERRORS").is_ok()
ignore_https_errors: env_var_is_truthy("AGENT_BROWSER_IGNORE_HTTPS_ERRORS")
|| config.ignore_https_errors.unwrap_or(false),
allow_file_access: env::var("AGENT_BROWSER_ALLOW_FILE_ACCESS").is_ok()
allow_file_access: env_var_is_truthy("AGENT_BROWSER_ALLOW_FILE_ACCESS")
|| config.allow_file_access.unwrap_or(false),
device: env::var("AGENT_BROWSER_IOS_DEVICE").ok()
.or(config.device),
auto_connect: env::var("AGENT_BROWSER_AUTO_CONNECT").is_ok()
auto_connect: env_var_is_truthy("AGENT_BROWSER_AUTO_CONNECT")
|| config.auto_connect.unwrap_or(false),
session_name: env::var("AGENT_BROWSER_SESSION_NAME").ok()
.or(config.session_name),
annotate: env_var_is_truthy("AGENT_BROWSER_ANNOTATE")
|| config.annotate.unwrap_or(false),
cli_executable_path: false,
cli_extensions: false,
cli_profile: false,
@@ -406,6 +420,11 @@ pub fn parse_flags(args: &[String]) -> Flags {
i += 1;
}
}
"--annotate" => {
let (val, consumed) = parse_bool_arg(args, i);
flags.annotate = val;
if consumed { i += 1; }
}
"--config" => {
// Already handled by load_config(); skip the value
i += 1;
@@ -430,6 +449,7 @@ pub fn clean_args(args: &[String]) -> Vec<String> {
"--ignore-https-errors",
"--allow-file-access",
"--auto-connect",
"--annotate",
];
// Global flags that always take a value (need to skip the next arg too)
const GLOBAL_FLAGS_WITH_VALUE: &[&str] = &[
cli/src/output.rs
+41 -5
View File
@@ -363,11 +363,37 @@ pub fn print_response(resp: &Response, json_mode: bool, action: Option<&str>) {
// Path-based operations (screenshot/pdf/trace/har/download/state/video)
if let Some(path) = data.get("path").and_then(|v| v.as_str()) {
match action.unwrap_or("") {
"screenshot" => println!(
"{} Screenshot saved to {}",
color::success_indicator(),
color::green(path)
),
"screenshot" => {
println!(
"{} Screenshot saved to {}",
color::success_indicator(),
color::green(path)
);
if let Some(annotations) = data.get("annotations").and_then(|v| v.as_array()) {
for ann in annotations {
let num = ann.get("number").and_then(|n| n.as_u64()).unwrap_or(0);
let ref_id = ann.get("ref").and_then(|r| r.as_str()).unwrap_or("");
let role = ann.get("role").and_then(|r| r.as_str()).unwrap_or("");
let name = ann.get("name").and_then(|n| n.as_str()).unwrap_or("");
if name.is_empty() {
println!(
" {} @{} {}",
color::dim(&format!("[{}]", num)),
ref_id,
role,
);
} else {
println!(
" {} @{} {} {:?}",
color::dim(&format!("[{}]", num)),
ref_id,
role,
name,
);
}
}
}
}
"pdf" => println!(
"{} PDF saved to {}",
color::success_indicator(),
@@ -965,6 +991,10 @@ saves to a temporary directory with a generated filename.
Options:
--full, -f Capture full page (not just viewport)
--annotate Overlay numbered labels on interactive elements.
Each label [N] corresponds to ref @eN from snapshot.
Prints a legend mapping labels to element roles/names.
With --json, annotations are included in the response.
Global Options:
--json Output as JSON
@@ -974,6 +1004,9 @@ Examples:
agent-browser screenshot
agent-browser screenshot ./screenshot.png
agent-browser screenshot --full ./full-page.png
agent-browser screenshot --annotate # Labeled screenshot + legend
agent-browser screenshot --annotate ./page.png # Save annotated screenshot
agent-browser screenshot --annotate --json # JSON output with annotations
"##
}
"pdf" => {
@@ -1932,6 +1965,7 @@ Options:
--device <name> iOS device name (e.g., "iPhone 15 Pro")
--json JSON output
--full, -f Full page screenshot
--annotate Annotated screenshot with numbered labels and legend
--headed Show browser window (not headless)
--cdp <port> Connect via CDP (Chrome DevTools Protocol)
--auto-connect Auto-discover and connect to running Chrome
@@ -1970,6 +2004,7 @@ Environment:
AGENT_BROWSER_HEADED Show browser window (not headless)
AGENT_BROWSER_JSON JSON output
AGENT_BROWSER_FULL Full page screenshot
AGENT_BROWSER_ANNOTATE Annotated screenshot with numbered labels and legend
AGENT_BROWSER_DEBUG Debug output
AGENT_BROWSER_IGNORE_HTTPS_ERRORS Ignore HTTPS certificate errors
AGENT_BROWSER_PROVIDER Browser provider (ios, browserbase, kernel, browseruse)
@@ -1994,6 +2029,7 @@ Examples:
agent-browser find role button click --name Submit
agent-browser get text @e1
agent-browser screenshot --full
agent-browser screenshot --annotate # Labeled screenshot for vision models
agent-browser wait --load networkidle # Wait for slow pages to load
agent-browser --cdp 9222 snapshot # Connect via CDP port
agent-browser --auto-connect snapshot # Auto-discover running Chrome