Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d8f484eded | ||
|
|
1eb40eabd5 | ||
|
|
601404ba72 | ||
|
|
fd10766762 | ||
|
|
ba9b167ede | ||
|
|
c077593e99 | ||
|
|
02e23ebe11 | ||
|
|
d5cd9cd621 | ||
|
|
284a60a54c | ||
|
|
10d196b6eb | ||
|
|
5be01e292d |
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chrome-use"
|
||||
version = "1.5.21"
|
||||
version = "1.5.27"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "chrome-use"
|
||||
version = "1.5.21"
|
||||
version = "1.5.27"
|
||||
edition = "2021"
|
||||
description = "Fast browser automation CLI for AI agents"
|
||||
license = "Apache-2.0"
|
||||
|
||||
+219
-33
@@ -80,6 +80,11 @@ const KNOWN_COMMANDS: &[&str] = &[
|
||||
"upload",
|
||||
"site",
|
||||
"box",
|
||||
"adopt",
|
||||
"canvas",
|
||||
"viewport",
|
||||
"resize",
|
||||
"keep",
|
||||
];
|
||||
|
||||
/// Levenshtein distance, capped — small inputs only (command names).
|
||||
@@ -1172,6 +1177,54 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
// Canvas/WebGL apps (Figma, games, maps, charts, drawing tools) render to a
|
||||
// <canvas> with no DOM/refs to read. `canvas` extracts what's actually
|
||||
// rendered: `list` enumerates canvases; `capture` saves a canvas to PNG via
|
||||
// toDataURL (full backing-store resolution) with a CDP-screenshot fallback
|
||||
// for WebGL contexts (no preserveDrawingBuffer) or cross-origin-tainted ones.
|
||||
"canvas" => {
|
||||
match rest.first().copied() {
|
||||
Some("list") => Ok(json!({ "id": id, "action": "canvas_list" })),
|
||||
Some("capture") => {
|
||||
let force_screenshot = rest.contains(&"--screenshot");
|
||||
let pos: Vec<&str> = rest[1..]
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|a| !a.starts_with("--"))
|
||||
.collect();
|
||||
// `capture [selector] [path]`: a selector starts with . # @ or is
|
||||
// a tag; a path contains / or ends in an image extension.
|
||||
let is_path = |s: &str| {
|
||||
s.contains('/')
|
||||
|| s.ends_with(".png")
|
||||
|| s.ends_with(".jpg")
|
||||
|| s.ends_with(".jpeg")
|
||||
|| s.ends_with(".webp")
|
||||
};
|
||||
let (selector, path) = match (pos.first(), pos.get(1)) {
|
||||
(Some(a), Some(b)) => (Some(*a), Some(*b)),
|
||||
(Some(a), None) if is_path(a) => (None, Some(*a)),
|
||||
(Some(a), None) => (Some(*a), None),
|
||||
_ => (None, None),
|
||||
};
|
||||
let mut cmd = json!({ "id": id, "action": "canvas_capture" });
|
||||
if let Some(s) = selector {
|
||||
cmd["selector"] = json!(s);
|
||||
}
|
||||
if let Some(p) = path {
|
||||
cmd["path"] = json!(p);
|
||||
}
|
||||
if force_screenshot {
|
||||
cmd["forceScreenshot"] = json!(true);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
_ => Err(ParseError::InvalidValue {
|
||||
message: "canvas needs a subcommand".to_string(),
|
||||
usage: "canvas list | canvas capture [selector] [path] [--screenshot]",
|
||||
}),
|
||||
}
|
||||
}
|
||||
"pdf" => {
|
||||
let path = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "pdf".to_string(),
|
||||
@@ -1317,6 +1370,16 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
||||
Ok(json!({ "id": id, "action": "site", "domain": domain, "script": script }))
|
||||
}
|
||||
|
||||
// `adopt <url|targetId>`: the adoption happens at daemon connect (driven by
|
||||
// the AGENT_BROWSER_ADOPT env main.rs set + a forced-fresh daemon), so by
|
||||
// the time this command runs the tab is already attached. Resolve to a
|
||||
// `url` read so the response confirms which tab got adopted.
|
||||
"adopt" => Ok(json!({ "id": id, "action": "url" })),
|
||||
|
||||
// `keep`: leave the active tab for the user — exempt it from the daemon's
|
||||
// auto-close/idle cleanup and remove it from the session's tab group.
|
||||
"keep" => Ok(json!({ "id": id, "action": "keep" })),
|
||||
|
||||
// === Stealth self-check ===
|
||||
"stealth" => {
|
||||
// `stealth [status]` — local stealth self-check: mode, live probes
|
||||
@@ -1640,6 +1703,13 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
||||
// === Mouse ===
|
||||
"mouse" => parse_mouse(&rest, &id),
|
||||
|
||||
// === Viewport / window size ===
|
||||
// Top-level shortcut for `set viewport` — agents (and the author, in
|
||||
// issue #47) reach for `viewport`/`resize` first. Uses a CDP virtual
|
||||
// viewport, so it also works on the extension relay without yanking the
|
||||
// user's real window around.
|
||||
"viewport" | "resize" => parse_viewport(&rest, &id),
|
||||
|
||||
// === Set (browser settings) ===
|
||||
"set" => parse_set(&rest, &id),
|
||||
|
||||
@@ -3088,6 +3158,97 @@ fn parse_mouse(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a viewport / resize spec, shared by the top-level `viewport` / `resize`
|
||||
/// commands and `set viewport`. Sets a CDP device-metrics override
|
||||
/// (`Emulation.setDeviceMetricsOverride`) — a *virtual* viewport for the tab, so
|
||||
/// it works headless AND on the extension relay without physically resizing the
|
||||
/// user's real Chrome window (issue #47). Forms:
|
||||
/// viewport <width> <height> [scale] [--dpr N] [--mobile]
|
||||
/// viewport <width>x<height> (e.g. 1280x800)
|
||||
/// viewport reset | clear (drop the override, restore real size)
|
||||
fn parse_viewport(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
const USAGE: &str = "viewport <width> <height> [scale] [--dpr N] [--mobile] | viewport reset";
|
||||
|
||||
if matches!(
|
||||
rest.first().copied(),
|
||||
Some("reset") | Some("clear") | Some("off")
|
||||
) {
|
||||
return Ok(json!({ "id": id, "action": "viewport", "reset": true }));
|
||||
}
|
||||
|
||||
// Positional (non-flag) tokens. A `WxH` token counts as one positional.
|
||||
let positionals: Vec<&str> = rest
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|a| !a.starts_with("--"))
|
||||
.collect();
|
||||
|
||||
let (w, h, scale_tok): (i32, i32, Option<&str>) = match positionals.first() {
|
||||
Some(first) if first.contains('x') || first.contains('X') => {
|
||||
let mut parts = first.split(['x', 'X']);
|
||||
let w = parts.next().and_then(|s| s.parse::<i32>().ok());
|
||||
let h = parts.next().and_then(|s| s.parse::<i32>().ok());
|
||||
match (w, h) {
|
||||
(Some(w), Some(h)) => (w, h, positionals.get(1).copied()),
|
||||
_ => {
|
||||
return Err(ParseError::InvalidValue {
|
||||
message: format!("Invalid viewport size: {}", first),
|
||||
usage: USAGE,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(w_str) => {
|
||||
let h_str = positionals.get(1).ok_or(ParseError::MissingArguments {
|
||||
context: "viewport".to_string(),
|
||||
usage: USAGE,
|
||||
})?;
|
||||
let w = w_str.parse::<i32>().map_err(|_| ParseError::InvalidValue {
|
||||
message: format!("Invalid width: {}", w_str),
|
||||
usage: USAGE,
|
||||
})?;
|
||||
let h = h_str.parse::<i32>().map_err(|_| ParseError::InvalidValue {
|
||||
message: format!("Invalid height: {}", h_str),
|
||||
usage: USAGE,
|
||||
})?;
|
||||
(w, h, positionals.get(2).copied())
|
||||
}
|
||||
None => {
|
||||
return Err(ParseError::MissingArguments {
|
||||
context: "viewport".to_string(),
|
||||
usage: USAGE,
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
let mut cmd = json!({ "id": id, "action": "viewport", "width": w, "height": h });
|
||||
|
||||
// Device-scale-factor: positional, overridden by --dpr / --scale.
|
||||
let mut scale: Option<f64> = match scale_tok {
|
||||
Some(s) => Some(s.parse::<f64>().map_err(|_| ParseError::InvalidValue {
|
||||
message: format!("Invalid scale: {}", s),
|
||||
usage: USAGE,
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(i) = rest.iter().position(|a| *a == "--dpr" || *a == "--scale") {
|
||||
let v = rest.get(i + 1).and_then(|s| s.parse::<f64>().ok()).ok_or(
|
||||
ParseError::InvalidValue {
|
||||
message: "--dpr/--scale needs a number".to_string(),
|
||||
usage: USAGE,
|
||||
},
|
||||
)?;
|
||||
scale = Some(v);
|
||||
}
|
||||
if let Some(s) = scale {
|
||||
cmd["deviceScaleFactor"] = json!(s);
|
||||
}
|
||||
if rest.contains(&"--mobile") {
|
||||
cmd["mobile"] = json!(true);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
|
||||
fn parse_set(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
const VALID: &[&str] = &[
|
||||
"viewport",
|
||||
@@ -3102,39 +3263,8 @@ fn parse_set(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
];
|
||||
|
||||
match rest.first().copied() {
|
||||
Some("viewport") => {
|
||||
let w_str = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "set viewport".to_string(),
|
||||
usage: "set viewport <width> <height> [scale]",
|
||||
})?;
|
||||
let h_str = rest.get(2).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "set viewport".to_string(),
|
||||
usage: "set viewport <width> <height> [scale]",
|
||||
})?;
|
||||
let w = w_str
|
||||
.parse::<i32>()
|
||||
.map_err(|_| ParseError::MissingArguments {
|
||||
context: "set viewport".to_string(),
|
||||
usage: "set viewport <width> <height> [scale]",
|
||||
})?;
|
||||
let h = h_str
|
||||
.parse::<i32>()
|
||||
.map_err(|_| ParseError::MissingArguments {
|
||||
context: "set viewport".to_string(),
|
||||
usage: "set viewport <width> <height> [scale]",
|
||||
})?;
|
||||
let mut cmd = json!({ "id": id, "action": "viewport", "width": w, "height": h });
|
||||
if let Some(scale_str) = rest.get(3) {
|
||||
let scale = scale_str
|
||||
.parse::<f64>()
|
||||
.map_err(|_| ParseError::MissingArguments {
|
||||
context: "set viewport".to_string(),
|
||||
usage: "set viewport <width> <height> [scale]",
|
||||
})?;
|
||||
cmd["deviceScaleFactor"] = json!(scale);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
// `set viewport ...` is an alias for the top-level `viewport` command.
|
||||
Some("viewport") => parse_viewport(&rest[1..], id),
|
||||
Some("device") => {
|
||||
let dev = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "set device".to_string(),
|
||||
@@ -5295,6 +5425,62 @@ mod tests {
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_viewport_toplevel() {
|
||||
let cmd = parse_command(&args("viewport 1280 800"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "viewport");
|
||||
assert_eq!(cmd["width"], 1280);
|
||||
assert_eq!(cmd["height"], 800);
|
||||
assert!(cmd.get("deviceScaleFactor").is_none());
|
||||
assert!(cmd.get("mobile").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resize_alias() {
|
||||
let cmd = parse_command(&args("resize 700 800"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "viewport");
|
||||
assert_eq!(cmd["width"], 700);
|
||||
assert_eq!(cmd["height"], 800);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_viewport_wxh_form() {
|
||||
let cmd = parse_command(&args("viewport 375x812"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "viewport");
|
||||
assert_eq!(cmd["width"], 375);
|
||||
assert_eq!(cmd["height"], 812);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_viewport_dpr_and_mobile_flags() {
|
||||
let cmd =
|
||||
parse_command(&args("viewport 375 812 --dpr 3 --mobile"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "viewport");
|
||||
assert_eq!(cmd["width"], 375);
|
||||
assert_eq!(cmd["height"], 812);
|
||||
assert_eq!(cmd["deviceScaleFactor"], 3.0);
|
||||
assert_eq!(cmd["mobile"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_viewport_reset() {
|
||||
for spec in ["viewport reset", "viewport clear", "resize reset"] {
|
||||
let cmd = parse_command(&args(spec), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "viewport", "{spec}");
|
||||
assert_eq!(cmd["reset"], true, "{spec}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_viewport_missing_height() {
|
||||
assert!(parse_command(&args("viewport 1280"), &default_flags()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_viewport_invalid_width() {
|
||||
assert!(parse_command(&args("viewport abc 800"), &default_flags()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_first_no_value() {
|
||||
let cmd = parse_command(&args("find first a click"), &default_flags()).unwrap();
|
||||
|
||||
@@ -595,7 +595,7 @@ fn query_current_url(session: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
/// Kill a running daemon by reading its PID file and sending a kill signal.
|
||||
fn kill_stale_daemon(session: &str) {
|
||||
pub fn kill_stale_daemon(session: &str) {
|
||||
// Remove the socket first so no new connections reach the old daemon
|
||||
#[cfg(unix)]
|
||||
{
|
||||
|
||||
@@ -305,6 +305,49 @@ fn run_session(args: &[String], session: &str, json_mode: bool) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Stop a specific session daemon (issue #48). Graceful: kill_stale_daemon
|
||||
// sends SIGTERM first, so the daemon's shutdown handler runs `close()` and
|
||||
// tidies the tabs IT created (its tab group) before exiting.
|
||||
Some("stop") => {
|
||||
let target = args.get(2).map(|s| s.as_str()).unwrap_or(session);
|
||||
connection::kill_stale_daemon(target);
|
||||
if json_mode {
|
||||
print_json_value(json!({ "success": true, "data": { "stopped": target } }));
|
||||
} else {
|
||||
println!(
|
||||
"{} stopped session daemon: {}",
|
||||
color::success_indicator(),
|
||||
target
|
||||
);
|
||||
}
|
||||
}
|
||||
// Reclaim ALL session daemons now (issue #48) — for clearing the pile of
|
||||
// idle daemons left after a round of automation/debugging without waiting
|
||||
// for the idle timeout. Each is stopped gracefully (closes its own tabs);
|
||||
// they respawn clean on next use. The `__nm-host` relay is not a tracked
|
||||
// session daemon, so the extension/live-Chrome connection survives.
|
||||
Some("prune") => {
|
||||
let sessions: Vec<String> = walk_daemons()
|
||||
.sessions
|
||||
.into_iter()
|
||||
.map(|s| s.name)
|
||||
.collect();
|
||||
for s in &sessions {
|
||||
connection::kill_stale_daemon(s);
|
||||
}
|
||||
if json_mode {
|
||||
print_json_value(json!({ "success": true, "data": { "pruned": sessions } }));
|
||||
} else if sessions.is_empty() {
|
||||
println!("No session daemons to prune");
|
||||
} else {
|
||||
println!(
|
||||
"{} pruned {} session daemon(s): {}",
|
||||
color::success_indicator(),
|
||||
sessions.len(),
|
||||
sessions.join(", ")
|
||||
);
|
||||
}
|
||||
}
|
||||
None | Some(_) => {
|
||||
// Just show current session
|
||||
if json_mode {
|
||||
@@ -981,6 +1024,43 @@ fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
// `adopt <url|targetId>`: read a PRE-EXISTING tab (the user's own, or another
|
||||
// session's) WITHOUT opening a new one. Forces a fresh daemon and points it at
|
||||
// the relay (like `extension connect`); the AGENT_BROWSER_ADOPT env makes the
|
||||
// daemon's first connect ADOPT the matching tab instead of creating an
|
||||
// about:blank. Rewrites into `connect <relay-url>` BEFORE parse_command so the
|
||||
// daemon attaches to the user's real Chrome. Must run before parse_command.
|
||||
if clean.first().map(|s| s.as_str()) == Some("adopt") {
|
||||
match clean.get(1) {
|
||||
Some(spec) if !spec.trim().is_empty() => {
|
||||
std::env::set_var("AGENT_BROWSER_ADOPT", spec.trim());
|
||||
connection::kill_stale_daemon(&flags.session);
|
||||
match connect::relay_url() {
|
||||
Some(url) => {
|
||||
flags.cdp = Some(url.clone());
|
||||
flags.auto_connect = false;
|
||||
clean = vec!["connect".to_string(), url];
|
||||
}
|
||||
None => {
|
||||
eprintln!(
|
||||
"{} extension relay not connected — open Chrome with the ab-connect \
|
||||
extension first (this command reads an EXISTING tab, it won't launch one).",
|
||||
color::error_indicator()
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
eprintln!(
|
||||
"{} usage: chrome-use adopt <url-substring|targetId> (reads an existing tab, no new tab)",
|
||||
color::error_indicator()
|
||||
);
|
||||
exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle session separately (doesn't need daemon)
|
||||
if clean.first().map(|s| s.as_str()) == Some("session") {
|
||||
run_session(&clean, &flags.session, flags.json);
|
||||
|
||||
@@ -1317,9 +1317,12 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
||||
"evaluate" => handle_evaluate(cmd, state).await,
|
||||
"site" => handle_site(cmd, state).await,
|
||||
"close" => handle_close(state).await,
|
||||
"keep" => handle_keep(state).await,
|
||||
"stealth_status" => handle_stealth_status(state).await,
|
||||
"snapshot" => handle_snapshot(cmd, state).await,
|
||||
"screenshot" => handle_screenshot(cmd, state).await,
|
||||
"canvas_list" => handle_canvas_list(state).await,
|
||||
"canvas_capture" => handle_canvas_capture(cmd, state).await,
|
||||
"click" => handle_click(cmd, state).await,
|
||||
"dblclick" => handle_dblclick(cmd, state).await,
|
||||
"fill" => handle_fill(cmd, state).await,
|
||||
@@ -2853,6 +2856,34 @@ async fn handle_stealth_status(state: &DaemonState) -> Result<Value, String> {
|
||||
}))
|
||||
}
|
||||
|
||||
/// `keep` — leave the ACTIVE tab for the user: stop owning it (so the daemon's
|
||||
/// `close()`/idle-shutdown won't close it) and best-effort remove it from this
|
||||
/// session's tab group so it looks like a normal user tab. The "leave for the
|
||||
/// user" half of the auto-close-on-idle cleanup: scratch tabs get closed, tabs
|
||||
/// the agent explicitly `keep`s stay. (Adopted user tabs are never owned, so
|
||||
/// they're already safe.)
|
||||
async fn handle_keep(state: &mut DaemonState) -> Result<Value, String> {
|
||||
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
|
||||
let target_id = mgr.active_target_id()?.to_string();
|
||||
let session_id = mgr.active_session_id()?.to_string();
|
||||
let was_owned = mgr.unown_target(&target_id);
|
||||
// Best-effort: ask the extension to ungroup the tab (relay only; no-ops on a
|
||||
// launched browser or an older extension that doesn't know ABExt.ungroupTab).
|
||||
let _ = mgr
|
||||
.client
|
||||
.send_command_typed::<_, Value>(
|
||||
"ABExt.ungroupTab",
|
||||
&json!({ "sessionId": session_id, "targetId": target_id }),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
Ok(json!({
|
||||
"kept": target_id,
|
||||
"wasOwned": was_owned,
|
||||
"note": "tab left for the user — exempt from auto-close, removed from the session tab group",
|
||||
}))
|
||||
}
|
||||
|
||||
async fn handle_close(state: &mut DaemonState) -> Result<Value, String> {
|
||||
if let Some(ref mgr) = state.browser {
|
||||
if let Some(ref session_name) = state.session_name {
|
||||
@@ -3213,6 +3244,154 @@ fn downscale_screenshot(
|
||||
Some((resized.width(), resized.height()))
|
||||
}
|
||||
|
||||
/// `canvas list` — enumerate <canvas> elements (size, visibility, whether
|
||||
/// toDataURL is usable) so an agent can pick one to capture on a canvas/WebGL app
|
||||
/// where snapshot/DOM reads see nothing.
|
||||
async fn handle_canvas_list(state: &DaemonState) -> Result<Value, String> {
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
let js = r#"(() => [...document.querySelectorAll('canvas')].map((c, i) => {
|
||||
const r = c.getBoundingClientRect();
|
||||
let toDataUrl = true, tainted = false;
|
||||
try { c.toDataURL('image/png'); } catch (e) { toDataUrl = false; tainted = true; }
|
||||
return {
|
||||
index: i,
|
||||
backingWidth: c.width, backingHeight: c.height,
|
||||
cssWidth: Math.round(r.width), cssHeight: Math.round(r.height),
|
||||
visible: r.width > 0 && r.height > 0 && r.bottom > 0 && r.top < innerHeight,
|
||||
id: c.id || null, className: c.className || null,
|
||||
toDataUrl, tainted,
|
||||
};
|
||||
}))()"#;
|
||||
let canvases = mgr.evaluate(js, None).await?;
|
||||
Ok(json!({ "canvases": canvases }))
|
||||
}
|
||||
|
||||
/// `canvas capture [selector] [path]` — save a canvas's rendered pixels to PNG.
|
||||
/// Prefers `toDataURL` (full backing-store resolution); falls back to a CDP
|
||||
/// screenshot of the canvas element when toDataURL is blank (WebGL without
|
||||
/// preserveDrawingBuffer, e.g. Figma) or throws (cross-origin tainted). This is
|
||||
/// how chrome-use "sees" canvas/WebGL apps that expose no DOM or refs.
|
||||
async fn handle_canvas_capture(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||
let selector = cmd
|
||||
.get("selector")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from);
|
||||
let force_screenshot = cmd
|
||||
.get("forceScreenshot")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let out_path = cmd.get("path").and_then(|v| v.as_str()).map(String::from);
|
||||
let id = cmd.get("id").and_then(|v| v.as_str()).unwrap_or("0");
|
||||
|
||||
// Probe: locate the canvas, get its bbox, and try toDataURL (unless forced).
|
||||
let probe = {
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
let sel_lit = match &selector {
|
||||
Some(s) => serde_json::to_string(s).unwrap_or_else(|_| "null".to_string()),
|
||||
None => "null".to_string(),
|
||||
};
|
||||
let js = format!(
|
||||
r#"(() => {{
|
||||
const sel = {sel_lit};
|
||||
const c = sel ? document.querySelector(sel)
|
||||
: [...document.querySelectorAll('canvas')].sort((a,b)=>(b.width*b.height)-(a.width*a.height))[0];
|
||||
if (!c) return {{ found: false, count: document.querySelectorAll('canvas').length }};
|
||||
const r = c.getBoundingClientRect();
|
||||
let dataUrl = null, err = null;
|
||||
if ({try_data}) {{ try {{ dataUrl = c.toDataURL('image/png'); }} catch (e) {{ err = String(e && e.message || e); }} }}
|
||||
return {{ found: true, w: c.width, h: c.height, x: r.x, y: r.y, cw: r.width, ch: r.height, dataUrl, err }};
|
||||
}})()"#,
|
||||
try_data = !force_screenshot
|
||||
);
|
||||
mgr.evaluate(&js, None).await?
|
||||
};
|
||||
|
||||
if !probe
|
||||
.get("found")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
let count = probe.get("count").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
return Err(format!(
|
||||
"canvas: no canvas matched{} ({count} canvas element(s) on the page — try `canvas list`)",
|
||||
selector
|
||||
.as_deref()
|
||||
.map(|s| format!(" `{s}`"))
|
||||
.unwrap_or_default()
|
||||
));
|
||||
}
|
||||
let backing_w = probe.get("w").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
let backing_h = probe.get("h").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
|
||||
// Decode toDataURL if present; a WebGL canvas without preserveDrawingBuffer
|
||||
// returns a blank PNG (tiny when compressed), so reject suspiciously small
|
||||
// results and fall back to the screenshot path.
|
||||
let decoded: Option<Vec<u8>> = probe
|
||||
.get("dataUrl")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(|u| u.split_once(",").map(|(_, b)| b.to_string()))
|
||||
.and_then(|b64| {
|
||||
base64::Engine::decode(&base64::engine::general_purpose::STANDARD, b64.as_bytes()).ok()
|
||||
});
|
||||
let use_data = !force_screenshot && decoded.as_ref().map(|d| d.len() > 1024).unwrap_or(false);
|
||||
|
||||
if use_data {
|
||||
let bytes = decoded.unwrap();
|
||||
let path = out_path.unwrap_or_else(|| {
|
||||
std::env::temp_dir()
|
||||
.join(format!("canvas-{id}.png"))
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
});
|
||||
std::fs::write(&path, &bytes).map_err(|e| format!("canvas: write {path}: {e}"))?;
|
||||
Ok(json!({
|
||||
"path": absolutize_saved_path(&path),
|
||||
"method": "toDataURL",
|
||||
"width": backing_w,
|
||||
"height": backing_h,
|
||||
}))
|
||||
} else {
|
||||
// Fallback: screenshot the canvas element (or its bbox clip).
|
||||
let clip = if selector.is_none() {
|
||||
let x = probe.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let y = probe.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let cw = probe.get("cw").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let ch = probe.get("ch").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
Some((x, y, cw, ch))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let options = ScreenshotOptions {
|
||||
selector: selector.clone(),
|
||||
path: out_path,
|
||||
clip,
|
||||
..ScreenshotOptions::default()
|
||||
};
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
let session_id = mgr.active_session_id()?.to_string();
|
||||
let result = screenshot::take_screenshot(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&state.ref_map,
|
||||
&options,
|
||||
&state.iframe_sessions,
|
||||
)
|
||||
.await?;
|
||||
let why = probe
|
||||
.get("err")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|e| format!("toDataURL failed ({e})"))
|
||||
.unwrap_or_else(|| {
|
||||
"toDataURL blank/unavailable (WebGL no preserveDrawingBuffer)".into()
|
||||
});
|
||||
Ok(json!({
|
||||
"path": absolutize_saved_path(&result.path),
|
||||
"method": "screenshot",
|
||||
"note": format!("captured rendered pixels via screenshot — {why}"),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||
// First-class coordinate click (issue #8.4): click a raw viewport point with
|
||||
// no element resolution. Parsed from `click <x> <y>` / `click --coords x,y`.
|
||||
@@ -5102,6 +5281,19 @@ async fn handle_tab_close(cmd: &Value, state: &mut DaemonState) -> Result<Value,
|
||||
|
||||
async fn handle_viewport(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
|
||||
// `viewport reset` clears the device-metrics override and restores the real
|
||||
// layout viewport (the launched window's size, or — over the relay — the
|
||||
// user's actual Chrome window).
|
||||
if cmd.get("reset").and_then(|v| v.as_bool()).unwrap_or(false) {
|
||||
mgr.clear_viewport().await?;
|
||||
state.viewport = None;
|
||||
if let Some(ref server) = state.stream_server {
|
||||
server.set_viewport(1280, 720).await;
|
||||
}
|
||||
return Ok(json!({ "reset": true }));
|
||||
}
|
||||
|
||||
let width = cmd.get("width").and_then(|v| v.as_i64()).unwrap_or(1280) as i32;
|
||||
let height = cmd.get("height").and_then(|v| v.as_i64()).unwrap_or(720) as i32;
|
||||
let scale = cmd
|
||||
|
||||
+247
-20
@@ -509,6 +509,13 @@ pub struct BrowserManager {
|
||||
/// for `RELAY_PRUNE_MISSES` consecutive snapshots; any snapshot that includes
|
||||
/// it resets the counter. Keyed by stable target_id.
|
||||
relay_target_misses: HashMap<String, u32>,
|
||||
/// Whether the relay accepted this session's group announcement and is
|
||||
/// therefore scoping `Target.getTargets` to our own tab group (issue #40).
|
||||
/// When true the daemon can safely adopt new targets again (follow-popup,
|
||||
/// cross-session adopt) — the relay has already filtered out foreign tabs.
|
||||
/// When false (launch-on-real-CDP, or an older relay that didn't answer the
|
||||
/// announce) the daemon keeps strict daemon-side isolation.
|
||||
relay_scoped: bool,
|
||||
next_tab_id: u32,
|
||||
/// Whether to enable the CDP `Runtime` domain (console / error / exception capture).
|
||||
/// OFF by default for stealth: a live `Runtime.enable` is a detectable CDP signal
|
||||
@@ -646,6 +653,7 @@ impl BrowserManager {
|
||||
created_targets: HashSet::new(),
|
||||
active_target_id: None,
|
||||
relay_target_misses: HashMap::new(),
|
||||
relay_scoped: false,
|
||||
next_tab_id: 1,
|
||||
capture_console: console_capture_enabled(),
|
||||
};
|
||||
@@ -749,6 +757,7 @@ impl BrowserManager {
|
||||
created_targets: HashSet::new(),
|
||||
active_target_id: None,
|
||||
relay_target_misses: HashMap::new(),
|
||||
relay_scoped: false,
|
||||
next_tab_id: 1,
|
||||
capture_console: console_capture_enabled(),
|
||||
};
|
||||
@@ -815,6 +824,106 @@ impl BrowserManager {
|
||||
Ok(by_id.into_values().collect())
|
||||
}
|
||||
|
||||
/// Every tab the relay knows, UNSCOPED (ignores group scoping) — for explicit
|
||||
/// cross-group adoption (`chrome-use adopt`). Falls back to the scoped
|
||||
/// `collect_page_targets` on a relay/browser that doesn't support the
|
||||
/// unscoped query. Retries a few times over the relay (discovery is eventual).
|
||||
async fn collect_all_targets(&self) -> Result<Vec<TargetInfo>, String> {
|
||||
let rounds = if crate::connect::relay_url().is_some() {
|
||||
3
|
||||
} else {
|
||||
1
|
||||
};
|
||||
let mut by_id: HashMap<String, TargetInfo> = HashMap::new();
|
||||
let mut any_ok = false;
|
||||
for i in 0..rounds {
|
||||
if i > 0 {
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
}
|
||||
if let Ok(result) = self
|
||||
.client
|
||||
.send_command_typed::<_, GetTargetsResult>(
|
||||
"ABRelay.getAllTargets",
|
||||
&json!({}),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
any_ok = true;
|
||||
for t in result.target_infos.into_iter().filter(should_track_target) {
|
||||
by_id.entry(t.target_id.clone()).or_insert(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
if any_ok {
|
||||
Ok(by_id.into_values().collect())
|
||||
} else {
|
||||
// Older relay without ABRelay.getAllTargets → best-effort scoped list.
|
||||
self.collect_page_targets().await
|
||||
}
|
||||
}
|
||||
|
||||
/// Adopt a specific pre-existing tab matched by `spec` (an exact CDP
|
||||
/// `targetId`, or a case-insensitive substring of the tab URL) WITHOUT opening
|
||||
/// a new tab — for `chrome-use adopt`. Attaches it (the relay tags it into our
|
||||
/// group), tracks + pins it. Errors if nothing matches (never creates a tab).
|
||||
async fn adopt_existing_target(&mut self, spec: &str) -> Result<(), String> {
|
||||
let all = self.collect_all_targets().await?;
|
||||
let spec_l = spec.to_lowercase();
|
||||
let target = all
|
||||
.iter()
|
||||
.find(|t| t.target_id == spec)
|
||||
.or_else(|| all.iter().find(|t| t.url.to_lowercase().contains(&spec_l)))
|
||||
.ok_or_else(|| {
|
||||
let mut open: Vec<String> = all
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let u = if t.url.len() > 80 {
|
||||
&t.url[..80]
|
||||
} else {
|
||||
&t.url
|
||||
};
|
||||
u.to_string()
|
||||
})
|
||||
.collect();
|
||||
open.sort();
|
||||
open.dedup();
|
||||
format!(
|
||||
"adopt: no open tab matching `{spec}` (by targetId or URL substring).\n\
|
||||
{} tab(s) the extension can see:\n {}",
|
||||
open.len(),
|
||||
open.join("\n ")
|
||||
)
|
||||
})?
|
||||
.clone();
|
||||
|
||||
let attach: AttachToTargetResult = self
|
||||
.client
|
||||
.send_command_typed(
|
||||
"Target.attachToTarget",
|
||||
&AttachToTargetParams {
|
||||
target_id: target.target_id.clone(),
|
||||
flatten: true,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let tab_id = self.assign_tab_id();
|
||||
self.pages.push(PageInfo {
|
||||
tab_id,
|
||||
label: None,
|
||||
target_id: target.target_id.clone(),
|
||||
session_id: attach.session_id.clone(),
|
||||
url: target.url.clone(),
|
||||
title: sanitize_title(&target.title),
|
||||
target_type: target.target_type.clone(),
|
||||
});
|
||||
self.active_page_index = self.pages.len() - 1;
|
||||
self.pin_active_target();
|
||||
self.enable_domains(&attach.session_id).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn discover_and_attach_targets(&mut self) -> Result<(), String> {
|
||||
self.client
|
||||
.send_command_typed::<_, Value>(
|
||||
@@ -824,6 +933,21 @@ impl BrowserManager {
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Announce our group FIRST so the relay scopes the getTargets below to our
|
||||
// own tab group (issue #40). On a launched browser this is a no-op.
|
||||
let scoped = self.announce_group().await;
|
||||
|
||||
// `chrome-use adopt <spec>`: adopt a specific PRE-EXISTING tab instead of
|
||||
// creating one — true zero-new-tab reading of the user's own tab. The
|
||||
// directive rides in via env so it takes effect at first connect (before
|
||||
// any about:blank would be made). If nothing matches, error out rather
|
||||
// than fall back to creating a tab.
|
||||
if let Ok(spec) = std::env::var("AGENT_BROWSER_ADOPT") {
|
||||
if !spec.trim().is_empty() {
|
||||
return self.adopt_existing_target(spec.trim()).await;
|
||||
}
|
||||
}
|
||||
|
||||
let page_targets: Vec<TargetInfo> = self.collect_page_targets().await?;
|
||||
|
||||
if page_targets.is_empty() {
|
||||
@@ -870,19 +994,19 @@ impl BrowserManager {
|
||||
self.active_page_index = 0;
|
||||
self.pin_active_target();
|
||||
self.enable_domains(&attach_result.session_id).await?;
|
||||
} else if self.agent_group().is_some() {
|
||||
// STRICT MULTI-AGENT ISOLATION (relay / the user's real Chrome).
|
||||
// `page_targets` here are the USER's and OTHER agents' tabs. A tab
|
||||
// group belongs to exactly ONE agent, so this session must NOT adopt
|
||||
// any of them — it tracks ONLY tabs it creates (its own colored group)
|
||||
// plus popups it opens. Adopting foreign tabs is precisely what let
|
||||
// another concurrent agent's tab churn drop the tab we were driving and
|
||||
// drift eval/click onto the wrong page (multi-agent failure). Open our
|
||||
// own dedicated background tab in the session's group and pin it; the
|
||||
// user's / other agents' tabs stay invisible to us.
|
||||
} else if self.agent_group().is_some() && !scoped {
|
||||
// STRICT MULTI-AGENT ISOLATION fallback (relay, but the group announce
|
||||
// didn't take — e.g. an older relay). Without relay-side scoping,
|
||||
// `page_targets` could be the USER's and OTHER agents' tabs, so this
|
||||
// session must NOT adopt any of them — adopting foreign tabs is what let
|
||||
// another agent's tab churn drop the tab we were driving (multi-agent
|
||||
// failure). Open our own dedicated background tab and pin it instead.
|
||||
self.tab_new(None, None).await?;
|
||||
} else {
|
||||
// A browser WE launched: every tab is ours, so adopt them all.
|
||||
// Either a browser WE launched (every tab is ours) or the relay has
|
||||
// scoped getTargets to our own tab group (#40) — so `page_targets` are
|
||||
// all ours: adopt them (this restores follow-popup + cross-session
|
||||
// adopt under isolation, since foreign tabs were already filtered out).
|
||||
for target in &page_targets {
|
||||
let attach_result: AttachToTargetResult = self
|
||||
.client
|
||||
@@ -1422,6 +1546,13 @@ impl BrowserManager {
|
||||
.ok_or_else(|| "No active page".to_string())
|
||||
}
|
||||
|
||||
/// Stop owning a tab — drop it from `created_targets` so it survives `close()`
|
||||
/// and idle-shutdown (the agent is leaving it for the user). Returns true if it
|
||||
/// was owned. Used by `keep`.
|
||||
pub fn unown_target(&mut self, target_id: &str) -> bool {
|
||||
self.created_targets.remove(target_id)
|
||||
}
|
||||
|
||||
/// Returns true if this manager was connected via CDP (as opposed to local launch).
|
||||
pub fn is_cdp_connection(&self) -> bool {
|
||||
self.browser_process.is_none()
|
||||
@@ -1580,7 +1711,11 @@ impl BrowserManager {
|
||||
// in the synthesized targetInfo), so don't adopt anything: the agent drives
|
||||
// only tabs it explicitly created, and pop-ups (e.g. an OAuth/login window)
|
||||
// are the user's. A launched browser (every tab ours) still follows pop-ups.
|
||||
if self.agent_group().is_some() {
|
||||
// Strict isolation only when on the relay WITHOUT group scoping: there a
|
||||
// pop-up can't be told apart from a foreign tab, so adopt nothing. When the
|
||||
// relay IS scoping (#40), getTargets returns only our group, so a tab that
|
||||
// appeared after our own action is genuinely ours (a pop-up) — adopt it.
|
||||
if self.agent_group().is_some() && !self.relay_scoped {
|
||||
return None;
|
||||
}
|
||||
let result: GetTargetsResult = self
|
||||
@@ -1658,16 +1793,17 @@ impl BrowserManager {
|
||||
.collect();
|
||||
let live_ids: HashSet<String> = live.iter().map(|t| t.target_id.clone()).collect();
|
||||
let on_relay = self.agent_group().is_some();
|
||||
// When the relay scopes getTargets to our group (#40), `live` is already
|
||||
// only our own tabs, so adopting unknown ones is safe (a freshly-opened
|
||||
// pop-up). Without scoping, keep strict isolation: never adopt a tab we
|
||||
// didn't create — it belongs to the user or another agent.
|
||||
let strict_isolation = on_relay && !self.relay_scoped;
|
||||
|
||||
for target in &live {
|
||||
if self.update_page_target_info(target) {
|
||||
continue;
|
||||
}
|
||||
// STRICT MULTI-AGENT ISOLATION: on the relay (the user's real Chrome,
|
||||
// shared with other agents), NEVER adopt a tab this session didn't
|
||||
// create — it belongs to the user or another agent's group. Only a
|
||||
// browser we launched (every tab ours) adopts unknown targets.
|
||||
if on_relay {
|
||||
if strict_isolation {
|
||||
continue;
|
||||
}
|
||||
let attach_result: AttachToTargetResult = match self
|
||||
@@ -1821,9 +1957,17 @@ impl BrowserManager {
|
||||
/// CDP browser the endpoint is strict, so we must NOT send the custom param —
|
||||
/// hence `None` there. We detect the relay by matching our `ws_url` against
|
||||
/// the live relay URL the native-messaging host published.
|
||||
/// Whether this manager is driving the user's real Chrome through the
|
||||
/// `ab-connect` extension relay (vs. a browser we launched or a direct CDP
|
||||
/// endpoint). Detected by matching our `ws_url` against the live relay URL
|
||||
/// the native-messaging host published. Used to avoid relay-unsafe CDP that
|
||||
/// would disturb the user's window (e.g. Browser.setContentsSize, issue #47).
|
||||
fn via_relay(&self) -> bool {
|
||||
crate::connect::relay_url().as_deref() == Some(self.ws_url.as_str())
|
||||
}
|
||||
|
||||
fn agent_group(&self) -> Option<String> {
|
||||
let via_relay = crate::connect::relay_url().as_deref() == Some(self.ws_url.as_str());
|
||||
if !via_relay {
|
||||
if !self.via_relay() {
|
||||
return None;
|
||||
}
|
||||
let name = DAEMON_SESSION
|
||||
@@ -1837,6 +1981,25 @@ impl BrowserManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Tell the relay which tab group this session owns so it can scope
|
||||
/// `Target.getTargets` to us (issue #40). Only meaningful on the relay; a
|
||||
/// no-op (returns false) on a launched/real-CDP connection. Sets and returns
|
||||
/// `relay_scoped`: when true, the daemon can trust getTargets to contain only
|
||||
/// our group and re-enable adopting new tabs (pop-ups, cross-session adopt).
|
||||
async fn announce_group(&mut self) -> bool {
|
||||
let Some(group) = self.agent_group() else {
|
||||
self.relay_scoped = false;
|
||||
return false;
|
||||
};
|
||||
let ok = self
|
||||
.client
|
||||
.send_command_typed::<_, Value>("ABRelay.setGroup", &json!({ "group": group }), None)
|
||||
.await
|
||||
.is_ok();
|
||||
self.relay_scoped = ok;
|
||||
ok
|
||||
}
|
||||
|
||||
pub async fn tab_new(
|
||||
&mut self,
|
||||
url: Option<&str>,
|
||||
@@ -1907,6 +2070,46 @@ impl BrowserManager {
|
||||
self.active_page_index = index;
|
||||
self.pin_active_target();
|
||||
|
||||
// Close the daemon's leftover initial `about:blank` scratch tab once this
|
||||
// real tab exists, so the session's tab group isn't left showing a stray
|
||||
// blank page beside the work tab (every group otherwise carried one). Only
|
||||
// on the RELAY — there the about:blank is a tab WE created as scratch; on a
|
||||
// launched browser the initial about:blank is the browser's own first tab,
|
||||
// which we must not close. Only when opening a real url, OWNED, still-blank.
|
||||
if target_url != "about:blank" && self.agent_group().is_some() {
|
||||
if let Some(new_tid) = self.pages.get(index).map(|p| p.target_id.clone()) {
|
||||
let blanks: Vec<String> = self
|
||||
.pages
|
||||
.iter()
|
||||
.filter(|p| {
|
||||
p.target_id != new_tid
|
||||
&& self.created_targets.contains(&p.target_id)
|
||||
&& (p.url == "about:blank" || p.url.is_empty())
|
||||
})
|
||||
.map(|p| p.target_id.clone())
|
||||
.collect();
|
||||
for tid in blanks {
|
||||
let _ = self
|
||||
.client
|
||||
.send_command_typed::<_, Value>(
|
||||
"Target.closeTarget",
|
||||
&CloseTargetParams {
|
||||
target_id: tid.clone(),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
self.created_targets.remove(&tid);
|
||||
self.remove_page_by_target_id(&tid);
|
||||
}
|
||||
// Removing earlier pages shifts indices — re-pin the new tab.
|
||||
if let Some(i) = self.pages.iter().position(|p| p.target_id == new_tid) {
|
||||
self.active_page_index = i;
|
||||
self.pin_active_target();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(json!({
|
||||
"tabId": format_tab_id(tab_id),
|
||||
"label": label,
|
||||
@@ -2013,7 +2216,13 @@ impl BrowserManager {
|
||||
.await?;
|
||||
|
||||
// Screencast captures the actual content area, not the emulated CSS
|
||||
// viewport, so resize the content area to match.
|
||||
// viewport, so resize the content area to match — but ONLY for a browser
|
||||
// we launched. Over the ab-connect relay the "window" is the user's real
|
||||
// Chrome window, and Browser.setContentsSize would physically resize it
|
||||
// (issue #47) — the exact thing the CDP device-metrics override exists to
|
||||
// avoid. The Emulation override above already gives the tab the requested
|
||||
// CSS viewport without touching the OS window, so skip the resize there.
|
||||
if !self.via_relay() {
|
||||
if let Ok(target_id) = self.active_target_id() {
|
||||
if let Ok(window_info) = self
|
||||
.client
|
||||
@@ -2043,10 +2252,27 @@ impl BrowserManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clear the CDP device-metrics override (`viewport reset`), restoring the
|
||||
/// tab's real layout viewport. Never touches the OS window, so it is safe on
|
||||
/// the relay (we never physically resized the user's window — see
|
||||
/// `set_viewport`).
|
||||
pub async fn clear_viewport(&self) -> Result<(), String> {
|
||||
let session_id = self.active_session_id()?;
|
||||
self.client
|
||||
.send_command(
|
||||
"Emulation.clearDeviceMetricsOverride",
|
||||
Some(json!({})),
|
||||
Some(session_id),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn set_user_agent(&self, user_agent: &str) -> Result<(), String> {
|
||||
let session_id = self.active_session_id()?;
|
||||
self.client
|
||||
@@ -2630,6 +2856,7 @@ async fn initialize_lightpanda_manager(
|
||||
created_targets: HashSet::new(),
|
||||
active_target_id: None,
|
||||
relay_target_misses: HashMap::new(),
|
||||
relay_scoped: false,
|
||||
next_tab_id: 1,
|
||||
capture_console: console_capture_enabled(),
|
||||
};
|
||||
|
||||
@@ -130,12 +130,21 @@ pub async fn run_daemon(session: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-shutdown the daemon after this many ms of inactivity (no commands received).
|
||||
// Disabled when unset or 0.
|
||||
let idle_timeout_ms = env::var("AGENT_BROWSER_IDLE_TIMEOUT_MS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<u64>().ok())
|
||||
.filter(|&ms| ms > 0);
|
||||
// Auto-shutdown the daemon after this many ms of inactivity (no commands
|
||||
// received). On shutdown the daemon closes the tabs IT created (its per-session
|
||||
// tab group), so an agent that finishes a task and just stops — without ever
|
||||
// calling `close` — no longer leaves a pile of scratch tabs and a lingering
|
||||
// tab group in the user's Chrome. The timer resets on every command, so active
|
||||
// sessions are never interrupted; only genuinely-idle ones clean up.
|
||||
//
|
||||
// Defaults to 10 minutes. Set AGENT_BROWSER_IDLE_TIMEOUT_MS to override, or 0
|
||||
// to disable (keep the daemon alive forever — the old behaviour). Adopted
|
||||
// tabs (the user's own, via `adopt`) are never closed: only `created_targets`.
|
||||
const DEFAULT_IDLE_TIMEOUT_MS: u64 = 600_000;
|
||||
let idle_timeout_ms = match env::var("AGENT_BROWSER_IDLE_TIMEOUT_MS") {
|
||||
Ok(s) => s.trim().parse::<u64>().ok().filter(|&ms| ms > 0),
|
||||
Err(_) => Some(DEFAULT_IDLE_TIMEOUT_MS),
|
||||
};
|
||||
|
||||
let result = run_socket_server(
|
||||
&socket_path,
|
||||
|
||||
+273
-4
@@ -52,6 +52,22 @@ pub struct RelayState {
|
||||
pending: HashMap<i64, (ClientId, Value)>,
|
||||
/// monotonic source of relay-global command ids
|
||||
next_global_id: i64,
|
||||
/// Group-scoped isolation (issue #40). A tab group belongs to exactly one
|
||||
/// agent/session; the relay scopes `Target.getTargets` per client to its own
|
||||
/// group so the daemon can safely adopt new tabs (follow-popup, cross-session
|
||||
/// adopt) without ever seeing the user's or another agent's tabs.
|
||||
///
|
||||
/// clientId -> group name. A client that never announced a group (older
|
||||
/// daemon) is absent here and gets the full, UNSCOPED target list — so this
|
||||
/// is fully backward-compatible.
|
||||
client_groups: HashMap<ClientId, String>,
|
||||
/// targetId -> group name. Created tabs are tagged from `Target.createTarget`'s
|
||||
/// `agentGroup`; an explicitly adopted tab is tagged to the adopter; a pop-up
|
||||
/// inherits its opener's group (needs the extension to report `openerTargetId`).
|
||||
target_group: HashMap<String, String>,
|
||||
/// relay-global id of an in-flight `Target.createTarget` -> the `agentGroup`
|
||||
/// it carried, so the reply's `targetId` can be tagged with that group.
|
||||
pending_create: HashMap<i64, String>,
|
||||
}
|
||||
|
||||
/// What to do with a raw CDP command received from a `CdpClient`.
|
||||
@@ -95,6 +111,7 @@ impl RelayState {
|
||||
/// `pending` entries don't leak.
|
||||
pub fn drop_client(&mut self, client_id: ClientId) {
|
||||
self.pending.retain(|_, (cid, _)| *cid != client_id);
|
||||
self.client_groups.remove(&client_id);
|
||||
}
|
||||
|
||||
/// Route a raw CDP command `{id, method, params?, sessionId?}` from a
|
||||
@@ -123,12 +140,28 @@ impl RelayState {
|
||||
"jsVersion": ""
|
||||
}
|
||||
})),
|
||||
// Non-CDP control message: a daemon announces which tab group
|
||||
// (session) it owns, so getTargets can be scoped to it (issue #40).
|
||||
"ABRelay.setGroup" => {
|
||||
if let Some(g) = params.get("group").and_then(|g| g.as_str()) {
|
||||
if !g.is_empty() {
|
||||
self.client_groups.insert(client_id, g.to_string());
|
||||
}
|
||||
}
|
||||
ClientRoute::Local(json!({ "id": id, "result": {} }))
|
||||
}
|
||||
// Discovery is best-effort and event-driven in real CDP; abs only
|
||||
// reads the getTargets result, so an empty ack is enough here.
|
||||
"Target.setDiscoverTargets" | "Target.setAutoAttach" => {
|
||||
ClientRoute::Local(json!({ "id": id, "result": {} }))
|
||||
}
|
||||
"Target.getTargets" => {
|
||||
// Unscoped discovery for EXPLICIT cross-group adoption (`chrome-use
|
||||
// adopt`): returns every target the extension has attached, ignoring
|
||||
// group scoping, so an agent can find a specific pre-existing tab (the
|
||||
// user's, another session's) by URL/targetId and adopt it. Isolation
|
||||
// is preserved because the daemon only acts on the one tab it then
|
||||
// attaches (which the relay re-tags into the adopter's group).
|
||||
"ABRelay.getAllTargets" => {
|
||||
let infos: Vec<Value> = self
|
||||
.targets
|
||||
.values()
|
||||
@@ -136,15 +169,44 @@ impl RelayState {
|
||||
.collect();
|
||||
ClientRoute::Local(json!({ "id": id, "result": { "targetInfos": infos } }))
|
||||
}
|
||||
"Target.getTargets" => {
|
||||
// Scope to the client's own group when it announced one; an
|
||||
// un-announced (legacy) client gets the full list (back-compat).
|
||||
let scoped = self.client_groups.get(&client_id).cloned();
|
||||
let infos: Vec<Value> = self
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|(tid, _)| match &scoped {
|
||||
Some(g) => self
|
||||
.target_group
|
||||
.get(*tid)
|
||||
.map(|tg| tg == g)
|
||||
.unwrap_or(false),
|
||||
None => true,
|
||||
})
|
||||
.map(|(_, t)| t.target_info.clone())
|
||||
.collect();
|
||||
ClientRoute::Local(json!({ "id": id, "result": { "targetInfos": infos } }))
|
||||
}
|
||||
"Target.attachToTarget" => {
|
||||
let target_id = params
|
||||
.get("targetId")
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("");
|
||||
match self.targets.get(target_id) {
|
||||
Some(entry) => ClientRoute::Local(
|
||||
json!({ "id": id, "result": { "sessionId": entry.session_id } }),
|
||||
),
|
||||
Some(entry) => {
|
||||
let session_id = entry.session_id.clone();
|
||||
// Explicitly adopting a target makes it this client's
|
||||
// (cross-session adopt, #21) — tag it into the adopter's
|
||||
// group so it stays in that client's scoped getTargets and
|
||||
// isn't churn-pruned.
|
||||
if let Some(g) = self.client_groups.get(&client_id).cloned() {
|
||||
self.target_group.insert(target_id.to_string(), g);
|
||||
}
|
||||
ClientRoute::Local(
|
||||
json!({ "id": id, "result": { "sessionId": session_id } }),
|
||||
)
|
||||
}
|
||||
None => ClientRoute::Local(json!({
|
||||
"id": id,
|
||||
"error": { "code": -32602, "message": format!("No such target {target_id}") }
|
||||
@@ -157,6 +219,18 @@ impl RelayState {
|
||||
self.next_global_id += 1;
|
||||
let gid = self.next_global_id;
|
||||
self.pending.insert(gid, (client_id, id));
|
||||
// Remember the group a createTarget carries so the reply's
|
||||
// targetId can be tagged to the creating session (issue #40).
|
||||
if method == "Target.createTarget" {
|
||||
if let Some(g) = params.get("agentGroup").and_then(|g| g.as_str()) {
|
||||
if !g.is_empty() {
|
||||
self.pending_create.insert(gid, g.to_string());
|
||||
self.client_groups
|
||||
.entry(client_id)
|
||||
.or_insert_with(|| g.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ClientRoute::Forward(json!({
|
||||
"id": gid,
|
||||
"method": "forwardCDPCommand",
|
||||
@@ -201,6 +275,17 @@ impl RelayState {
|
||||
&& msg.get("method").is_none()
|
||||
{
|
||||
let gid = msg.get("id").and_then(|i| i.as_i64());
|
||||
// A createTarget reply: tag the new tab's targetId with the group the
|
||||
// command carried, so it lands in the creating session's scope (#40).
|
||||
if let Some(g) = gid.and_then(|g| self.pending_create.remove(&g)) {
|
||||
if let Some(tid) = msg
|
||||
.get("result")
|
||||
.and_then(|r| r.get("targetId"))
|
||||
.and_then(|t| t.as_str())
|
||||
{
|
||||
self.target_group.insert(tid.to_string(), g);
|
||||
}
|
||||
}
|
||||
let (to, orig_id) = match gid.and_then(|g| self.pending.remove(&g)) {
|
||||
Some((client_id, orig)) => (Some(client_id), orig),
|
||||
// No mapping (stale/unknown id) — fall back to broadcasting with
|
||||
@@ -241,6 +326,27 @@ impl RelayState {
|
||||
.and_then(|s| s.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
// Attribute the tab to a group for scoping (issue #40),
|
||||
// unless we already know it (createTarget tag). An
|
||||
// explicit `abGroup` from the extension wins; otherwise a
|
||||
// pop-up inherits its opener's group via `openerTargetId`.
|
||||
if !self.target_group.contains_key(tid) {
|
||||
if let Some(g) = info
|
||||
.get("abGroup")
|
||||
.and_then(|g| g.as_str())
|
||||
.filter(|g| !g.is_empty())
|
||||
{
|
||||
self.target_group.insert(tid.to_string(), g.to_string());
|
||||
} else if let Some(opener) = info
|
||||
.get("openerTargetId")
|
||||
.and_then(|o| o.as_str())
|
||||
.filter(|o| !o.is_empty())
|
||||
{
|
||||
if let Some(g) = self.target_group.get(opener).cloned() {
|
||||
self.target_group.insert(tid.to_string(), g);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.targets.insert(
|
||||
tid.to_string(),
|
||||
TargetEntry {
|
||||
@@ -255,6 +361,15 @@ impl RelayState {
|
||||
"Target.detachedFromTarget" => {
|
||||
let gone = inner_params.get("sessionId").and_then(|s| s.as_str());
|
||||
if let Some(gone) = gone {
|
||||
let gone_tids: Vec<String> = self
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|(_, e)| e.session_id == gone)
|
||||
.map(|(tid, _)| tid.clone())
|
||||
.collect();
|
||||
for tid in gone_tids {
|
||||
self.target_group.remove(&tid);
|
||||
}
|
||||
self.targets.retain(|_, e| e.session_id != gone);
|
||||
}
|
||||
return vec![];
|
||||
@@ -557,4 +672,158 @@ mod tests {
|
||||
_ => panic!("expected ToExt"),
|
||||
}
|
||||
}
|
||||
|
||||
// === Group-scoped isolation (issue #40) ===
|
||||
|
||||
/// Drive the real create path: announce group, createTarget(agentGroup), feed
|
||||
/// the ext reply (tags target→group) + the attachedToTarget event (creates the
|
||||
/// entry). Returns nothing; mutates `s`.
|
||||
fn create_in_group(s: &mut RelayState, client: ClientId, group: &str, tid: &str, sid: &str) {
|
||||
s.route_client_command(
|
||||
client,
|
||||
&json!({ "id": 1, "method": "ABRelay.setGroup", "params": { "group": group } }),
|
||||
);
|
||||
let route = s.route_client_command(
|
||||
client,
|
||||
&json!({ "id": 2, "method": "Target.createTarget",
|
||||
"params": { "url": "about:blank", "agentGroup": group } }),
|
||||
);
|
||||
let gid = match route {
|
||||
ClientRoute::Forward(env) => env["id"].as_i64().unwrap(),
|
||||
_ => panic!("createTarget must forward"),
|
||||
};
|
||||
s.handle_ext_message(&json!({ "id": gid, "result": { "targetId": tid } }), "");
|
||||
s.handle_ext_message(
|
||||
&json!({ "method": "forwardCDPEvent", "params": {
|
||||
"method": "Target.attachedToTarget",
|
||||
"params": { "sessionId": sid, "targetInfo": {
|
||||
"targetId": tid, "type": "page", "url": "about:blank", "attached": true } } } }),
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
fn get_target_ids(s: &mut RelayState, client: ClientId) -> Vec<String> {
|
||||
match s.route_client_command(client, &json!({ "id": 9, "method": "Target.getTargets" })) {
|
||||
ClientRoute::Local(v) => v["result"]["targetInfos"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|t| t["targetId"].as_str().unwrap().to_string())
|
||||
.collect(),
|
||||
_ => panic!("getTargets must be local"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_targets_is_scoped_to_each_clients_group() {
|
||||
let mut s = RelayState::new();
|
||||
create_in_group(&mut s, 1, "agent-a", "ta", "sa");
|
||||
create_in_group(&mut s, 2, "agent-b", "tb", "sb");
|
||||
// Each client sees ONLY its own group's tab — never the other agent's.
|
||||
assert_eq!(get_target_ids(&mut s, 1), vec!["ta"]);
|
||||
assert_eq!(get_target_ids(&mut s, 2), vec!["tb"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_client_without_group_sees_all_targets() {
|
||||
let mut s = RelayState::new();
|
||||
create_in_group(&mut s, 1, "agent-a", "ta", "sa");
|
||||
create_in_group(&mut s, 2, "agent-b", "tb", "sb");
|
||||
// Client 3 never announced a group → full, unscoped list (back-compat).
|
||||
let mut all = get_target_ids(&mut s, 3);
|
||||
all.sort();
|
||||
assert_eq!(all, vec!["ta", "tb"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn popup_inherits_opener_group_and_is_visible_to_that_client_only() {
|
||||
let mut s = RelayState::new();
|
||||
create_in_group(&mut s, 1, "agent-a", "ta", "sa");
|
||||
create_in_group(&mut s, 2, "agent-b", "tb", "sb");
|
||||
// A pop-up that agent-a's tab opened: extension reports openerTargetId=ta.
|
||||
s.handle_ext_message(
|
||||
&json!({ "method": "forwardCDPEvent", "params": {
|
||||
"method": "Target.attachedToTarget",
|
||||
"params": { "sessionId": "sp", "targetInfo": {
|
||||
"targetId": "tp", "type": "page", "url": "https://oauth.example/",
|
||||
"attached": true, "openerTargetId": "ta" } } } }),
|
||||
"",
|
||||
);
|
||||
// Only agent-a sees the pop-up; agent-b never does.
|
||||
let mut a = get_target_ids(&mut s, 1);
|
||||
a.sort();
|
||||
assert_eq!(a, vec!["ta", "tp"]);
|
||||
assert_eq!(get_target_ids(&mut s, 2), vec!["tb"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_attach_tags_target_into_adopter_group() {
|
||||
let mut s = RelayState::new();
|
||||
// A pre-existing, ungrouped tab the extension reported (e.g. user's tab).
|
||||
s.handle_ext_message(
|
||||
&json!({ "method": "forwardCDPEvent", "params": {
|
||||
"method": "Target.attachedToTarget",
|
||||
"params": { "sessionId": "su", "targetInfo": {
|
||||
"targetId": "tu", "type": "page", "url": "https://user.example/", "attached": true } } } }),
|
||||
"",
|
||||
);
|
||||
// Client 1 (group agent-a) explicitly adopts it by targetId (#21).
|
||||
s.route_client_command(
|
||||
1,
|
||||
&json!({ "id": 1, "method": "ABRelay.setGroup", "params": { "group": "agent-a" } }),
|
||||
);
|
||||
s.route_client_command(
|
||||
1,
|
||||
&json!({ "id": 2, "method": "Target.attachToTarget", "params": { "targetId": "tu" } }),
|
||||
);
|
||||
// Now it's in agent-a's scope and survives the scoped getTargets.
|
||||
assert_eq!(get_target_ids(&mut s, 1), vec!["tu"]);
|
||||
// A different agent still doesn't see it.
|
||||
s.route_client_command(
|
||||
2,
|
||||
&json!({ "id": 1, "method": "ABRelay.setGroup", "params": { "group": "agent-b" } }),
|
||||
);
|
||||
assert!(get_target_ids(&mut s, 2).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_all_targets_is_unscoped() {
|
||||
let mut s = RelayState::new();
|
||||
create_in_group(&mut s, 1, "agent-a", "ta", "sa");
|
||||
create_in_group(&mut s, 2, "agent-b", "tb", "sb");
|
||||
// Client 1's scoped getTargets sees only its own group...
|
||||
assert_eq!(get_target_ids(&mut s, 1), vec!["ta"]);
|
||||
// ...but ABRelay.getAllTargets returns EVERY target regardless of group
|
||||
// (for explicit cross-group adoption).
|
||||
let all = match s
|
||||
.route_client_command(1, &json!({ "id": 1, "method": "ABRelay.getAllTargets" }))
|
||||
{
|
||||
ClientRoute::Local(v) => {
|
||||
let mut ids: Vec<String> = v["result"]["targetInfos"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|t| t["targetId"].as_str().unwrap().to_string())
|
||||
.collect();
|
||||
ids.sort();
|
||||
ids
|
||||
}
|
||||
_ => panic!("getAllTargets must be local"),
|
||||
};
|
||||
assert_eq!(all, vec!["ta", "tb"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detach_clears_target_group() {
|
||||
let mut s = RelayState::new();
|
||||
create_in_group(&mut s, 1, "agent-a", "ta", "sa");
|
||||
assert_eq!(get_target_ids(&mut s, 1), vec!["ta"]);
|
||||
s.handle_ext_message(
|
||||
&json!({ "method": "forwardCDPEvent", "params": {
|
||||
"method": "Target.detachedFromTarget", "params": { "sessionId": "sa" } } }),
|
||||
"",
|
||||
);
|
||||
assert!(get_target_ids(&mut s, 1).is_empty());
|
||||
assert!(!s.target_group.contains_key("ta"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3319,9 +3319,16 @@ Core Commands:
|
||||
screenshot [path] Take screenshot (auto-downscaled to ≤2000px long edge;
|
||||
--max-width/--max-height/--scale to override)
|
||||
pdf <path> Save as PDF
|
||||
canvas list List <canvas> elements (size, type) on the page
|
||||
canvas capture [sel] [path] Save a canvas's rendered pixels to PNG — for
|
||||
WebGL/canvas apps (Figma, games, maps, charts) that
|
||||
expose no DOM. toDataURL, with a screenshot fallback.
|
||||
snapshot Accessibility tree with refs (for AI)
|
||||
eval <js> Run JavaScript
|
||||
connect <port|url> Connect to browser via CDP
|
||||
keep Leave the active tab for the user — exempt it from
|
||||
auto-close/idle cleanup + remove it from the session
|
||||
tab group (so scratch tabs get cleaned, this one stays)
|
||||
close [--all] Close browser (--all closes every session)
|
||||
|
||||
Navigation:
|
||||
@@ -3375,6 +3382,10 @@ Tabs:
|
||||
stable targetId, no reload — preserves in-page state
|
||||
open <url> --reuse-tab Reuse an existing tab on that URL instead of spawning
|
||||
a duplicate (matches origin+path; preserves state)
|
||||
adopt <url|targetId> Read a PRE-EXISTING tab (the user's own, or another
|
||||
session's) WITHOUT opening a new one — matches by URL
|
||||
substring or stable targetId, then drives it. e.g.
|
||||
`adopt "github.com/owner/repo"`
|
||||
|
||||
Diff:
|
||||
diff snapshot Compare current vs last snapshot
|
||||
@@ -3444,11 +3455,21 @@ Confirmation:
|
||||
Sessions:
|
||||
session Show current session name
|
||||
session list List active sessions
|
||||
session stop [name] Stop one session daemon (default: current) — graceful,
|
||||
closes the tabs it created
|
||||
session prune Stop ALL session daemons now (closes their tabs; they
|
||||
respawn clean on next use). For clearing idle daemons.
|
||||
sessions List running session daemons (alias of daemon status)
|
||||
daemon status List running session daemons (+ relay state)
|
||||
daemon restart Kill all session daemons; keeps the extension relay
|
||||
up. Clears stale/cross-leaked state after an upgrade.
|
||||
|
||||
Lifecycle: each --session <name> spawns a background daemon that drives that
|
||||
session's tabs. A daemon auto-shuts-down after 10 min idle (no commands) —
|
||||
AGENT_BROWSER_IDLE_TIMEOUT_MS overrides, 0 disables — and on shutdown closes
|
||||
the scratch tabs IT created (its tab group). Use `keep` to leave a tab for the
|
||||
user (exempt from auto-close), `session stop/prune` to reclaim now.
|
||||
|
||||
Chat (AI):
|
||||
chat <message> Send a natural language instruction (single-shot)
|
||||
chat Start interactive chat (REPL mode when stdin is a TTY)
|
||||
|
||||
+88
-8
@@ -31,6 +31,10 @@ fn dirs_home() -> Option<PathBuf> {
|
||||
pub struct Adapter {
|
||||
pub meta: Value,
|
||||
pub func_src: String,
|
||||
/// The adapter's declared `args` keys in DECLARATION order. Parsed from the
|
||||
/// raw @meta text because `serde_json` sorts object keys alphabetically, which
|
||||
/// would otherwise scramble positional-arg mapping for multi-arg adapters.
|
||||
pub arg_order: Vec<String>,
|
||||
}
|
||||
|
||||
impl Adapter {
|
||||
@@ -111,7 +115,66 @@ pub fn parse_adapter(raw: &str, spec: &str) -> Result<Adapter, String> {
|
||||
if func_src.is_empty() {
|
||||
return Err(format!("site: {spec} has no function body after @meta"));
|
||||
}
|
||||
Ok(Adapter { meta, func_src })
|
||||
let arg_order = arg_order_from_meta(&raw[start..end]);
|
||||
Ok(Adapter {
|
||||
meta,
|
||||
func_src,
|
||||
arg_order,
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract the `args` object's keys in DECLARATION order from the raw @meta JSON
|
||||
/// text (serde sorts them, losing order). Brace/string-aware: finds the `"args"`
|
||||
/// value object and collects only its top-level keys.
|
||||
fn arg_order_from_meta(meta_json: &str) -> Vec<String> {
|
||||
let bytes = meta_json.as_bytes();
|
||||
// Locate the `"args"` key, then the `{` that opens its value object.
|
||||
let Some(args_pos) = meta_json.find("\"args\"") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(brace_off) = meta_json[args_pos..].find('{') else {
|
||||
return Vec::new();
|
||||
};
|
||||
let open = args_pos + brace_off;
|
||||
let mut keys = Vec::new();
|
||||
let mut depth = 0i32;
|
||||
let mut in_str = false;
|
||||
let mut esc = false;
|
||||
let mut cur = String::new();
|
||||
let mut last_str: Option<String> = None;
|
||||
for &b in bytes.iter().skip(open) {
|
||||
if in_str {
|
||||
if esc {
|
||||
esc = false;
|
||||
} else if b == b'\\' {
|
||||
esc = true;
|
||||
} else if b == b'"' {
|
||||
in_str = false;
|
||||
last_str = Some(std::mem::take(&mut cur));
|
||||
} else {
|
||||
cur.push(b as char);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match b {
|
||||
b'"' => in_str = true,
|
||||
b'{' => depth += 1,
|
||||
b'}' => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
break; // end of the args object
|
||||
}
|
||||
}
|
||||
// A `:` at depth 1 means the preceding string was a key of `args`.
|
||||
b':' if depth == 1 => {
|
||||
if let Some(k) = last_str.take() {
|
||||
keys.push(k);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
keys
|
||||
}
|
||||
|
||||
/// Build the JS to eval: `(<adapter function>)(<args JSON>)`. The adapter's
|
||||
@@ -325,14 +388,11 @@ pub fn adapters_for_domain(host: &str) -> Vec<String> {
|
||||
/// validates required args itself.
|
||||
pub fn map_args(adapter: &Adapter, positional: &[String], named: &[(String, String)]) -> Value {
|
||||
let mut obj = serde_json::Map::new();
|
||||
let keys: Vec<String> = adapter
|
||||
.meta
|
||||
.get("args")
|
||||
.and_then(|a| a.as_object())
|
||||
.map(|m| m.keys().cloned().collect())
|
||||
.unwrap_or_default();
|
||||
// Positional args fill the adapter's declared args in DECLARATION order
|
||||
// (`arg_order`), not serde's alphabetized key order — otherwise a 2-arg
|
||||
// adapter like `{projectId, path}` would map positionals to `{path, projectId}`.
|
||||
for (i, val) in positional.iter().enumerate() {
|
||||
if let Some(k) = keys.get(i) {
|
||||
if let Some(k) = adapter.arg_order.get(i) {
|
||||
obj.insert(k.clone(), Value::String(val.clone()));
|
||||
}
|
||||
}
|
||||
@@ -382,4 +442,24 @@ async function(args) { return { repo: args.repo }; }"#;
|
||||
assert!(load_adapter("noslash").is_err());
|
||||
assert!(load_adapter("../etc/passwd").is_err());
|
||||
}
|
||||
|
||||
// Regression: positional args must follow DECLARATION order, not serde's
|
||||
// alphabetical key order. With `{projectId, path}` (not alphabetical),
|
||||
// `<uuid> <file>` must map projectId←uuid, path←file — not swapped.
|
||||
#[test]
|
||||
fn positional_args_follow_declaration_order_not_alphabetical() {
|
||||
let raw = r#"/* @meta
|
||||
{
|
||||
"name": "claude-design/get-file",
|
||||
"domain": "claude.ai",
|
||||
"args": { "projectId": {"required": true}, "path": {"required": true} }
|
||||
}
|
||||
*/
|
||||
async function(args){ return args; }"#;
|
||||
let a = parse_adapter(raw, "claude-design/get-file").unwrap();
|
||||
assert_eq!(a.arg_order, vec!["projectId", "path"]);
|
||||
let args = map_args(&a, &["the-uuid".into(), "misonote.dc.html".into()], &[]);
|
||||
assert_eq!(args["projectId"], "the-uuid");
|
||||
assert_eq!(args["path"], "misonote.dc.html");
|
||||
}
|
||||
}
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -81,6 +81,35 @@ async function groupTabInto(tabId, name) {
|
||||
groupIdByName.set(name, gid)
|
||||
}
|
||||
|
||||
// Group-scoped relay isolation hints (issue #40). The relay scopes
|
||||
// Target.getTargets per agent by tab group; report two things in the synthesized
|
||||
// targetInfo so it can attribute each tab:
|
||||
// - abGroup: the tab's Chrome tab-group TITLE (= the owning session name), so
|
||||
// the relay can re-attribute existing tabs after a restart (createTarget
|
||||
// tagging won't re-run for already-open tabs).
|
||||
// - openerTargetId: the targetId of the tab that opened this one, so a pop-up
|
||||
// (window.open / target=_blank / OAuth result) inherits its opener's group
|
||||
// and the agent that opened it can follow it — without foreign tabs leaking.
|
||||
// Best-effort: any failure yields empty strings, which the relay ignores.
|
||||
async function tabScopeHints(tabId) {
|
||||
let openerTargetId = ''
|
||||
let abGroup = ''
|
||||
try {
|
||||
const t = await chrome.tabs.get(tabId)
|
||||
if (t) {
|
||||
if (typeof t.openerTabId === 'number') {
|
||||
const op = tabs.get(t.openerTabId)
|
||||
if (op) openerTargetId = op.targetId
|
||||
}
|
||||
if (t.groupId != null && t.groupId >= 0 && chrome.tabGroups) {
|
||||
const g = await chrome.tabGroups.get(t.groupId).catch(() => null)
|
||||
if (g && g.title) abGroup = g.title
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
return { openerTargetId, abGroup }
|
||||
}
|
||||
|
||||
function postToHost(msg) {
|
||||
try {
|
||||
if (port) port.postMessage(msg)
|
||||
@@ -126,7 +155,7 @@ function connectHost() {
|
||||
} catch {}
|
||||
// Tell the daemon about everything we already have attached, then attach
|
||||
// anything new.
|
||||
reannounceAttachedTabs()
|
||||
void reannounceAttachedTabs()
|
||||
void attachAllTabs()
|
||||
}
|
||||
|
||||
@@ -140,7 +169,7 @@ async function onHostMessage(msg) {
|
||||
// Daemon (re)connected — (re)attach and announce every tab so it discovers
|
||||
// the user's existing tabs rather than racing an empty target list.
|
||||
if (msg.method === 'attachAll') {
|
||||
reannounceAttachedTabs()
|
||||
void reannounceAttachedTabs()
|
||||
await attachAllTabs()
|
||||
return
|
||||
}
|
||||
@@ -259,6 +288,19 @@ async function handleForwardCdpCommand(msg) {
|
||||
const params = msg?.params?.params || undefined
|
||||
const sessionId = typeof msg?.params?.sessionId === 'string' ? msg.params.sessionId : undefined
|
||||
|
||||
// Non-CDP extension commands (ABExt.*) the daemon sends. `ungroupTab` removes a
|
||||
// tab from its per-session tab group so a `keep`-marked tab is left for the user
|
||||
// as a normal, ungrouped tab (the group can then be cleaned up). Best-effort.
|
||||
if (method === 'ABExt.ungroupTab') {
|
||||
const tabId = tabIdFromSession(sessionId) ?? tabForSession(sessionId)
|
||||
if (tabId != null && chrome.tabs.ungroup) {
|
||||
try {
|
||||
await chrome.tabs.ungroup(tabId)
|
||||
} catch {}
|
||||
}
|
||||
return { ungrouped: tabId ?? null }
|
||||
}
|
||||
|
||||
// Browser-level Target methods that map onto chrome.tabs.
|
||||
if (method === 'Target.createTarget') {
|
||||
const url = typeof params?.url === 'string' && params.url ? params.url : 'about:blank'
|
||||
@@ -387,12 +429,16 @@ async function attachTab(tabId) {
|
||||
sessionToTab.set(sessionId, tabId)
|
||||
rememberSessionTarget(sessionId, targetId)
|
||||
setBadge(tabId, port ? 'on' : 'connecting')
|
||||
const { openerTargetId, abGroup } = await tabScopeHints(tabId)
|
||||
postToHost({
|
||||
method: 'forwardCDPEvent',
|
||||
params: {
|
||||
sessionId,
|
||||
method: 'Target.attachedToTarget',
|
||||
params: { sessionId, targetInfo: { ...targetInfo, attached: true } },
|
||||
params: {
|
||||
sessionId,
|
||||
targetInfo: { ...targetInfo, attached: true, openerTargetId, abGroup },
|
||||
},
|
||||
},
|
||||
})
|
||||
return entry
|
||||
@@ -434,14 +480,32 @@ async function attachAllTabs() {
|
||||
}
|
||||
}
|
||||
|
||||
function reannounceAttachedTabs() {
|
||||
for (const [, entry] of tabs.entries()) {
|
||||
async function reannounceAttachedTabs() {
|
||||
for (const [tabId, entry] of tabs.entries()) {
|
||||
// Re-send the group hint too (issue #40) so the relay can rebuild its
|
||||
// targetId→group map after its own restart (createTarget tagging won't
|
||||
// re-run for tabs that are already open). Include the live url/title so the
|
||||
// relay's target list stays matchable by URL after a reconnect (otherwise a
|
||||
// reannounced tab shows a blank url and `adopt <url>` can't find it).
|
||||
const { openerTargetId, abGroup } = await tabScopeHints(tabId)
|
||||
let url = ''
|
||||
let title = ''
|
||||
try {
|
||||
const t = await chrome.tabs.get(tabId)
|
||||
if (t) {
|
||||
url = t.url || t.pendingUrl || ''
|
||||
title = t.title || ''
|
||||
}
|
||||
} catch {}
|
||||
postToHost({
|
||||
method: 'forwardCDPEvent',
|
||||
params: {
|
||||
sessionId: entry.sessionId,
|
||||
method: 'Target.attachedToTarget',
|
||||
params: { sessionId: entry.sessionId, targetInfo: { targetId: entry.targetId, type: 'page', attached: true } },
|
||||
params: {
|
||||
sessionId: entry.sessionId,
|
||||
targetInfo: { targetId: entry.targetId, type: 'page', url, title, attached: true, openerTargetId, abGroup },
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "chrome-use",
|
||||
"version": "0.4.9",
|
||||
"description": "Let chrome-use drive your logged-in Chrome \u2014 install once, no token, no per-use confirmation.",
|
||||
"version": "0.4.12",
|
||||
"description": "Let chrome-use drive your logged-in Chrome — install once, no token, no per-use confirmation.",
|
||||
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6vQIyscGIPYPZdSpPwPL0+0gxUROyRgCpmvCSDoc8XUm4qm97VbKnD9Ijc1lV22lNWZtE78gaRjt6BeSfuMgnBymnhLKjN1gU6AI5QUU0mrJyeHdWKvrKQR5FmsM2A7Xr1ykE2SiiS8zNUS3Y/6O5l+Nva7wrVy6E4a2dkBVQkOsu+DV+nEZvhIyuDY5D5SPXqNwUTWTaglwj5mjvHz36xSwCWlPmrtJ+ED0AUyrb2z4GIOmvk4kqtBVrh/UD058klLo4CkYOnIybB5aV6WYuwarfPY4bF/dLggPem+ewLNTUNBuwrxj/A4nUv0LJTuRO8rR7f8WR9qnRCY0Ic5saQIDAQAB",
|
||||
"icons": {
|
||||
"16": "icons/icon16.png",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "chrome-use",
|
||||
"version": "1.5.21",
|
||||
"version": "1.5.27",
|
||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||
"type": "module",
|
||||
"packageManager": "pnpm@11.1.3",
|
||||
|
||||
@@ -154,7 +154,17 @@ real Chrome concurrently without ever dropping or stealing each other's tabs —
|
||||
another agent's tab churn can't make your bound tab vanish or drift your commands
|
||||
onto the wrong page. Consequence: `tab list` shows only *your* session's tabs; to
|
||||
drive a specific page, navigate to it in your own tab instead of expecting a
|
||||
pre-existing or popped-up tab to appear in the list. **Anti-detection ranking: this real logged-in Chrome (extension
|
||||
pre-existing or popped-up tab to appear in the list.
|
||||
|
||||
> **Need to read a tab the user already has open?** Use `chrome-use adopt
|
||||
> <url-substring|targetId>` — it finds that pre-existing tab (the user's own, or
|
||||
> another session's) across groups and drives it **without opening a new tab**.
|
||||
> e.g. `adopt "claude.ai/design"` then `snapshot`/`eval`/`get text` on it. On no
|
||||
> match it errors and lists the tabs it can see. This is the explicit, opt-in way
|
||||
> through the isolation above (it tags the adopted tab into your group). Great for
|
||||
> "read/extract from the page I'm looking at" without disturbing it.
|
||||
|
||||
**Anti-detection ranking: this real logged-in Chrome (extension
|
||||
connect) > a headed launched browser > headless (forbidden).** A genuine human
|
||||
browser has no headless/automation tells at all, so prefer it for anything
|
||||
anti-bot-sensitive.
|
||||
@@ -444,6 +454,13 @@ tree**, so `snapshot` comes back near-empty and refs are a dead end. `snapshot`
|
||||
detects this and prints a one-line hint. Drive them the screenshot way:
|
||||
|
||||
```bash
|
||||
chrome-use canvas list # enumerate <canvas> elements (size, type)
|
||||
chrome-use canvas capture out.png # save the canvas's RENDERED pixels to PNG —
|
||||
# toDataURL (full backing-store res, e.g.
|
||||
# Figma 2522x1904), screenshot fallback for
|
||||
# WebGL w/o preserveDrawingBuffer / tainted.
|
||||
# Gets the RENDER, not hidden source data
|
||||
# (those live in the app's binary store/API).
|
||||
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
||||
# eval/get text return nothing useful)
|
||||
chrome-use click 640 360 # interact by viewport coordinate
|
||||
@@ -774,6 +791,29 @@ chrome-use snapshot -i
|
||||
chrome-use frame main # back to main frame
|
||||
```
|
||||
|
||||
### Viewport / window size (responsive & overflow debugging)
|
||||
|
||||
To reproduce width-dependent bugs (responsive breakpoints, horizontal-overflow
|
||||
hunts, mobile layouts) set the viewport. This is a **CDP virtual viewport**
|
||||
(`Emulation.setDeviceMetricsOverride`) — it changes the layout viewport *for the
|
||||
tab* without physically resizing the OS window, so it works headless **and** over
|
||||
the extension relay without yanking the user's real Chrome window around.
|
||||
|
||||
```bash
|
||||
chrome-use viewport 1280 800 # set width x height (alias: resize)
|
||||
chrome-use viewport 375x812 # WxH shorthand
|
||||
chrome-use viewport 375 812 --dpr 3 --mobile # retina + mobile emulation
|
||||
chrome-use viewport reset # clear the override, restore real size
|
||||
```
|
||||
|
||||
```bash
|
||||
# Find what's overflowing at a narrow width:
|
||||
chrome-use viewport 375 812
|
||||
chrome-use eval 'document.documentElement.scrollWidth + " vs " + innerWidth'
|
||||
```
|
||||
|
||||
`set viewport <w> <h> [scale]` is an equivalent alias.
|
||||
|
||||
### Dialogs
|
||||
|
||||
`alert` and `beforeunload` are auto-accepted so agents never block. For
|
||||
|
||||
Reference in New Issue
Block a user