feat(canvas): extract WebGL/canvas-app content + fix site arg-order + adopt skill doc
canvas — chrome-use can now read canvas/WebGL apps (Figma, games, maps, charts,
drawing tools) that expose no DOM/refs:
- canvas list: enumerate <canvas> (backing+CSS size, visibility, toDataUrl/tainted)
- canvas capture [selector] [path]: save rendered pixels to PNG — toDataURL
(full backing-store resolution), with a CDP screenshot fallback for WebGL
without preserveDrawingBuffer or cross-origin-tainted canvases. --screenshot
forces the screenshot path. Gets the RENDER, not hidden source data.
Verified live: captured Figma's canvas at full 2522x1904 via toDataURL.
site — fix map_args losing the adapter's declared arg order: serde sorts @meta
keys alphabetically, so a 2-arg adapter like {projectId, path} mapped positionals
to {path, projectId} (swapped). Now parses declaration order from the raw @meta
text (Adapter.arg_order) + regression test. Affects any multi-arg adapter.
skill — core skill now documents `adopt <url|targetId>` (read a pre-existing tab,
the explicit way through strict isolation) and `canvas list`/`canvas capture` in
the canvas/WebGL section.
863 tests pass.
This commit is contained in:
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.5.23"
|
version = "1.5.24"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.5.23"
|
version = "1.5.24"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "Fast browser automation CLI for AI agents"
|
description = "Fast browser automation CLI for AI agents"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
+200
-33
@@ -81,6 +81,9 @@ const KNOWN_COMMANDS: &[&str] = &[
|
|||||||
"site",
|
"site",
|
||||||
"box",
|
"box",
|
||||||
"adopt",
|
"adopt",
|
||||||
|
"canvas",
|
||||||
|
"viewport",
|
||||||
|
"resize",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Levenshtein distance, capped — small inputs only (command names).
|
/// Levenshtein distance, capped — small inputs only (command names).
|
||||||
@@ -1173,6 +1176,54 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
}
|
}
|
||||||
Ok(cmd)
|
Ok(cmd)
|
||||||
}
|
}
|
||||||
|
// Canvas/WebGL apps (Figma, games, maps, charts, drawing tools) render to a
|
||||||
|
// <canvas> with no DOM/refs to read. `canvas` extracts what's actually
|
||||||
|
// rendered: `list` enumerates canvases; `capture` saves a canvas to PNG via
|
||||||
|
// toDataURL (full backing-store resolution) with a CDP-screenshot fallback
|
||||||
|
// for WebGL contexts (no preserveDrawingBuffer) or cross-origin-tainted ones.
|
||||||
|
"canvas" => {
|
||||||
|
match rest.first().copied() {
|
||||||
|
Some("list") => Ok(json!({ "id": id, "action": "canvas_list" })),
|
||||||
|
Some("capture") => {
|
||||||
|
let force_screenshot = rest.contains(&"--screenshot");
|
||||||
|
let pos: Vec<&str> = rest[1..]
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|a| !a.starts_with("--"))
|
||||||
|
.collect();
|
||||||
|
// `capture [selector] [path]`: a selector starts with . # @ or is
|
||||||
|
// a tag; a path contains / or ends in an image extension.
|
||||||
|
let is_path = |s: &str| {
|
||||||
|
s.contains('/')
|
||||||
|
|| s.ends_with(".png")
|
||||||
|
|| s.ends_with(".jpg")
|
||||||
|
|| s.ends_with(".jpeg")
|
||||||
|
|| s.ends_with(".webp")
|
||||||
|
};
|
||||||
|
let (selector, path) = match (pos.first(), pos.get(1)) {
|
||||||
|
(Some(a), Some(b)) => (Some(*a), Some(*b)),
|
||||||
|
(Some(a), None) if is_path(a) => (None, Some(*a)),
|
||||||
|
(Some(a), None) => (Some(*a), None),
|
||||||
|
_ => (None, None),
|
||||||
|
};
|
||||||
|
let mut cmd = json!({ "id": id, "action": "canvas_capture" });
|
||||||
|
if let Some(s) = selector {
|
||||||
|
cmd["selector"] = json!(s);
|
||||||
|
}
|
||||||
|
if let Some(p) = path {
|
||||||
|
cmd["path"] = json!(p);
|
||||||
|
}
|
||||||
|
if force_screenshot {
|
||||||
|
cmd["forceScreenshot"] = json!(true);
|
||||||
|
}
|
||||||
|
Ok(cmd)
|
||||||
|
}
|
||||||
|
_ => Err(ParseError::InvalidValue {
|
||||||
|
message: "canvas needs a subcommand".to_string(),
|
||||||
|
usage: "canvas list | canvas capture [selector] [path] [--screenshot]",
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
"pdf" => {
|
"pdf" => {
|
||||||
let path = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let path = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
context: "pdf".to_string(),
|
context: "pdf".to_string(),
|
||||||
@@ -1647,6 +1698,13 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
// === Mouse ===
|
// === Mouse ===
|
||||||
"mouse" => parse_mouse(&rest, &id),
|
"mouse" => parse_mouse(&rest, &id),
|
||||||
|
|
||||||
|
// === Viewport / window size ===
|
||||||
|
// Top-level shortcut for `set viewport` — agents (and the author, in
|
||||||
|
// issue #47) reach for `viewport`/`resize` first. Uses a CDP virtual
|
||||||
|
// viewport, so it also works on the extension relay without yanking the
|
||||||
|
// user's real window around.
|
||||||
|
"viewport" | "resize" => parse_viewport(&rest, &id),
|
||||||
|
|
||||||
// === Set (browser settings) ===
|
// === Set (browser settings) ===
|
||||||
"set" => parse_set(&rest, &id),
|
"set" => parse_set(&rest, &id),
|
||||||
|
|
||||||
@@ -3095,6 +3153,91 @@ fn parse_mouse(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Parse a viewport / resize spec, shared by the top-level `viewport` / `resize`
|
||||||
|
/// commands and `set viewport`. Sets a CDP device-metrics override
|
||||||
|
/// (`Emulation.setDeviceMetricsOverride`) — a *virtual* viewport for the tab, so
|
||||||
|
/// it works headless AND on the extension relay without physically resizing the
|
||||||
|
/// user's real Chrome window (issue #47). Forms:
|
||||||
|
/// viewport <width> <height> [scale] [--dpr N] [--mobile]
|
||||||
|
/// viewport <width>x<height> (e.g. 1280x800)
|
||||||
|
/// viewport reset | clear (drop the override, restore real size)
|
||||||
|
fn parse_viewport(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||||
|
const USAGE: &str =
|
||||||
|
"viewport <width> <height> [scale] [--dpr N] [--mobile] | viewport reset";
|
||||||
|
|
||||||
|
if matches!(rest.first().copied(), Some("reset") | Some("clear") | Some("off")) {
|
||||||
|
return Ok(json!({ "id": id, "action": "viewport", "reset": true }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Positional (non-flag) tokens. A `WxH` token counts as one positional.
|
||||||
|
let positionals: Vec<&str> = rest.iter().copied().filter(|a| !a.starts_with("--")).collect();
|
||||||
|
|
||||||
|
let (w, h, scale_tok): (i32, i32, Option<&str>) = match positionals.first() {
|
||||||
|
Some(first) if first.contains('x') || first.contains('X') => {
|
||||||
|
let mut parts = first.split(|c| c == 'x' || c == 'X');
|
||||||
|
let w = parts.next().and_then(|s| s.parse::<i32>().ok());
|
||||||
|
let h = parts.next().and_then(|s| s.parse::<i32>().ok());
|
||||||
|
match (w, h) {
|
||||||
|
(Some(w), Some(h)) => (w, h, positionals.get(1).copied()),
|
||||||
|
_ => {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!("Invalid viewport size: {}", first),
|
||||||
|
usage: USAGE,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(w_str) => {
|
||||||
|
let h_str = positionals
|
||||||
|
.get(1)
|
||||||
|
.ok_or(ParseError::MissingArguments { context: "viewport".to_string(), usage: USAGE })?;
|
||||||
|
let w = w_str.parse::<i32>().map_err(|_| ParseError::InvalidValue {
|
||||||
|
message: format!("Invalid width: {}", w_str),
|
||||||
|
usage: USAGE,
|
||||||
|
})?;
|
||||||
|
let h = h_str.parse::<i32>().map_err(|_| ParseError::InvalidValue {
|
||||||
|
message: format!("Invalid height: {}", h_str),
|
||||||
|
usage: USAGE,
|
||||||
|
})?;
|
||||||
|
(w, h, positionals.get(2).copied())
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
return Err(ParseError::MissingArguments {
|
||||||
|
context: "viewport".to_string(),
|
||||||
|
usage: USAGE,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut cmd = json!({ "id": id, "action": "viewport", "width": w, "height": h });
|
||||||
|
|
||||||
|
// Device-scale-factor: positional, overridden by --dpr / --scale.
|
||||||
|
let mut scale: Option<f64> = match scale_tok {
|
||||||
|
Some(s) => Some(s.parse::<f64>().map_err(|_| ParseError::InvalidValue {
|
||||||
|
message: format!("Invalid scale: {}", s),
|
||||||
|
usage: USAGE,
|
||||||
|
})?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
if let Some(i) = rest.iter().position(|a| *a == "--dpr" || *a == "--scale") {
|
||||||
|
let v = rest
|
||||||
|
.get(i + 1)
|
||||||
|
.and_then(|s| s.parse::<f64>().ok())
|
||||||
|
.ok_or(ParseError::InvalidValue {
|
||||||
|
message: "--dpr/--scale needs a number".to_string(),
|
||||||
|
usage: USAGE,
|
||||||
|
})?;
|
||||||
|
scale = Some(v);
|
||||||
|
}
|
||||||
|
if let Some(s) = scale {
|
||||||
|
cmd["deviceScaleFactor"] = json!(s);
|
||||||
|
}
|
||||||
|
if rest.iter().any(|a| *a == "--mobile") {
|
||||||
|
cmd["mobile"] = json!(true);
|
||||||
|
}
|
||||||
|
Ok(cmd)
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_set(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
fn parse_set(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||||
const VALID: &[&str] = &[
|
const VALID: &[&str] = &[
|
||||||
"viewport",
|
"viewport",
|
||||||
@@ -3109,39 +3252,8 @@ fn parse_set(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
|||||||
];
|
];
|
||||||
|
|
||||||
match rest.first().copied() {
|
match rest.first().copied() {
|
||||||
Some("viewport") => {
|
// `set viewport ...` is an alias for the top-level `viewport` command.
|
||||||
let w_str = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
Some("viewport") => parse_viewport(&rest[1..], id),
|
||||||
context: "set viewport".to_string(),
|
|
||||||
usage: "set viewport <width> <height> [scale]",
|
|
||||||
})?;
|
|
||||||
let h_str = rest.get(2).ok_or_else(|| ParseError::MissingArguments {
|
|
||||||
context: "set viewport".to_string(),
|
|
||||||
usage: "set viewport <width> <height> [scale]",
|
|
||||||
})?;
|
|
||||||
let w = w_str
|
|
||||||
.parse::<i32>()
|
|
||||||
.map_err(|_| ParseError::MissingArguments {
|
|
||||||
context: "set viewport".to_string(),
|
|
||||||
usage: "set viewport <width> <height> [scale]",
|
|
||||||
})?;
|
|
||||||
let h = h_str
|
|
||||||
.parse::<i32>()
|
|
||||||
.map_err(|_| ParseError::MissingArguments {
|
|
||||||
context: "set viewport".to_string(),
|
|
||||||
usage: "set viewport <width> <height> [scale]",
|
|
||||||
})?;
|
|
||||||
let mut cmd = json!({ "id": id, "action": "viewport", "width": w, "height": h });
|
|
||||||
if let Some(scale_str) = rest.get(3) {
|
|
||||||
let scale = scale_str
|
|
||||||
.parse::<f64>()
|
|
||||||
.map_err(|_| ParseError::MissingArguments {
|
|
||||||
context: "set viewport".to_string(),
|
|
||||||
usage: "set viewport <width> <height> [scale]",
|
|
||||||
})?;
|
|
||||||
cmd["deviceScaleFactor"] = json!(scale);
|
|
||||||
}
|
|
||||||
Ok(cmd)
|
|
||||||
}
|
|
||||||
Some("device") => {
|
Some("device") => {
|
||||||
let dev = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
let dev = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||||
context: "set device".to_string(),
|
context: "set device".to_string(),
|
||||||
@@ -5302,6 +5414,61 @@ mod tests {
|
|||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_viewport_toplevel() {
|
||||||
|
let cmd = parse_command(&args("viewport 1280 800"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "viewport");
|
||||||
|
assert_eq!(cmd["width"], 1280);
|
||||||
|
assert_eq!(cmd["height"], 800);
|
||||||
|
assert!(cmd.get("deviceScaleFactor").is_none());
|
||||||
|
assert!(cmd.get("mobile").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resize_alias() {
|
||||||
|
let cmd = parse_command(&args("resize 700 800"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "viewport");
|
||||||
|
assert_eq!(cmd["width"], 700);
|
||||||
|
assert_eq!(cmd["height"], 800);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_viewport_wxh_form() {
|
||||||
|
let cmd = parse_command(&args("viewport 375x812"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "viewport");
|
||||||
|
assert_eq!(cmd["width"], 375);
|
||||||
|
assert_eq!(cmd["height"], 812);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_viewport_dpr_and_mobile_flags() {
|
||||||
|
let cmd = parse_command(&args("viewport 375 812 --dpr 3 --mobile"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "viewport");
|
||||||
|
assert_eq!(cmd["width"], 375);
|
||||||
|
assert_eq!(cmd["height"], 812);
|
||||||
|
assert_eq!(cmd["deviceScaleFactor"], 3.0);
|
||||||
|
assert_eq!(cmd["mobile"], true);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_viewport_reset() {
|
||||||
|
for spec in ["viewport reset", "viewport clear", "resize reset"] {
|
||||||
|
let cmd = parse_command(&args(spec), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "viewport", "{spec}");
|
||||||
|
assert_eq!(cmd["reset"], true, "{spec}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_viewport_missing_height() {
|
||||||
|
assert!(parse_command(&args("viewport 1280"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_viewport_invalid_width() {
|
||||||
|
assert!(parse_command(&args("viewport abc 800"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_find_first_no_value() {
|
fn test_find_first_no_value() {
|
||||||
let cmd = parse_command(&args("find first a click"), &default_flags()).unwrap();
|
let cmd = parse_command(&args("find first a click"), &default_flags()).unwrap();
|
||||||
|
|||||||
@@ -1320,6 +1320,8 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
|||||||
"stealth_status" => handle_stealth_status(state).await,
|
"stealth_status" => handle_stealth_status(state).await,
|
||||||
"snapshot" => handle_snapshot(cmd, state).await,
|
"snapshot" => handle_snapshot(cmd, state).await,
|
||||||
"screenshot" => handle_screenshot(cmd, state).await,
|
"screenshot" => handle_screenshot(cmd, state).await,
|
||||||
|
"canvas_list" => handle_canvas_list(state).await,
|
||||||
|
"canvas_capture" => handle_canvas_capture(cmd, state).await,
|
||||||
"click" => handle_click(cmd, state).await,
|
"click" => handle_click(cmd, state).await,
|
||||||
"dblclick" => handle_dblclick(cmd, state).await,
|
"dblclick" => handle_dblclick(cmd, state).await,
|
||||||
"fill" => handle_fill(cmd, state).await,
|
"fill" => handle_fill(cmd, state).await,
|
||||||
@@ -3213,6 +3215,154 @@ fn downscale_screenshot(
|
|||||||
Some((resized.width(), resized.height()))
|
Some((resized.width(), resized.height()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `canvas list` — enumerate <canvas> elements (size, visibility, whether
|
||||||
|
/// toDataURL is usable) so an agent can pick one to capture on a canvas/WebGL app
|
||||||
|
/// where snapshot/DOM reads see nothing.
|
||||||
|
async fn handle_canvas_list(state: &DaemonState) -> Result<Value, String> {
|
||||||
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
|
let js = r#"(() => [...document.querySelectorAll('canvas')].map((c, i) => {
|
||||||
|
const r = c.getBoundingClientRect();
|
||||||
|
let toDataUrl = true, tainted = false;
|
||||||
|
try { c.toDataURL('image/png'); } catch (e) { toDataUrl = false; tainted = true; }
|
||||||
|
return {
|
||||||
|
index: i,
|
||||||
|
backingWidth: c.width, backingHeight: c.height,
|
||||||
|
cssWidth: Math.round(r.width), cssHeight: Math.round(r.height),
|
||||||
|
visible: r.width > 0 && r.height > 0 && r.bottom > 0 && r.top < innerHeight,
|
||||||
|
id: c.id || null, className: c.className || null,
|
||||||
|
toDataUrl, tainted,
|
||||||
|
};
|
||||||
|
}))()"#;
|
||||||
|
let canvases = mgr.evaluate(js, None).await?;
|
||||||
|
Ok(json!({ "canvases": canvases }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `canvas capture [selector] [path]` — save a canvas's rendered pixels to PNG.
|
||||||
|
/// Prefers `toDataURL` (full backing-store resolution); falls back to a CDP
|
||||||
|
/// screenshot of the canvas element when toDataURL is blank (WebGL without
|
||||||
|
/// preserveDrawingBuffer, e.g. Figma) or throws (cross-origin tainted). This is
|
||||||
|
/// how chrome-use "sees" canvas/WebGL apps that expose no DOM or refs.
|
||||||
|
async fn handle_canvas_capture(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
|
let selector = cmd
|
||||||
|
.get("selector")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(String::from);
|
||||||
|
let force_screenshot = cmd
|
||||||
|
.get("forceScreenshot")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let out_path = cmd.get("path").and_then(|v| v.as_str()).map(String::from);
|
||||||
|
let id = cmd.get("id").and_then(|v| v.as_str()).unwrap_or("0");
|
||||||
|
|
||||||
|
// Probe: locate the canvas, get its bbox, and try toDataURL (unless forced).
|
||||||
|
let probe = {
|
||||||
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
|
let sel_lit = match &selector {
|
||||||
|
Some(s) => serde_json::to_string(s).unwrap_or_else(|_| "null".to_string()),
|
||||||
|
None => "null".to_string(),
|
||||||
|
};
|
||||||
|
let js = format!(
|
||||||
|
r#"(() => {{
|
||||||
|
const sel = {sel_lit};
|
||||||
|
const c = sel ? document.querySelector(sel)
|
||||||
|
: [...document.querySelectorAll('canvas')].sort((a,b)=>(b.width*b.height)-(a.width*a.height))[0];
|
||||||
|
if (!c) return {{ found: false, count: document.querySelectorAll('canvas').length }};
|
||||||
|
const r = c.getBoundingClientRect();
|
||||||
|
let dataUrl = null, err = null;
|
||||||
|
if ({try_data}) {{ try {{ dataUrl = c.toDataURL('image/png'); }} catch (e) {{ err = String(e && e.message || e); }} }}
|
||||||
|
return {{ found: true, w: c.width, h: c.height, x: r.x, y: r.y, cw: r.width, ch: r.height, dataUrl, err }};
|
||||||
|
}})()"#,
|
||||||
|
try_data = !force_screenshot
|
||||||
|
);
|
||||||
|
mgr.evaluate(&js, None).await?
|
||||||
|
};
|
||||||
|
|
||||||
|
if !probe
|
||||||
|
.get("found")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
let count = probe.get("count").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
return Err(format!(
|
||||||
|
"canvas: no canvas matched{} ({count} canvas element(s) on the page — try `canvas list`)",
|
||||||
|
selector
|
||||||
|
.as_deref()
|
||||||
|
.map(|s| format!(" `{s}`"))
|
||||||
|
.unwrap_or_default()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let backing_w = probe.get("w").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
let backing_h = probe.get("h").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
|
||||||
|
// Decode toDataURL if present; a WebGL canvas without preserveDrawingBuffer
|
||||||
|
// returns a blank PNG (tiny when compressed), so reject suspiciously small
|
||||||
|
// results and fall back to the screenshot path.
|
||||||
|
let decoded: Option<Vec<u8>> = probe
|
||||||
|
.get("dataUrl")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.and_then(|u| u.split_once(",").map(|(_, b)| b.to_string()))
|
||||||
|
.and_then(|b64| {
|
||||||
|
base64::Engine::decode(&base64::engine::general_purpose::STANDARD, b64.as_bytes()).ok()
|
||||||
|
});
|
||||||
|
let use_data = !force_screenshot && decoded.as_ref().map(|d| d.len() > 1024).unwrap_or(false);
|
||||||
|
|
||||||
|
if use_data {
|
||||||
|
let bytes = decoded.unwrap();
|
||||||
|
let path = out_path.unwrap_or_else(|| {
|
||||||
|
std::env::temp_dir()
|
||||||
|
.join(format!("canvas-{id}.png"))
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned()
|
||||||
|
});
|
||||||
|
std::fs::write(&path, &bytes).map_err(|e| format!("canvas: write {path}: {e}"))?;
|
||||||
|
Ok(json!({
|
||||||
|
"path": absolutize_saved_path(&path),
|
||||||
|
"method": "toDataURL",
|
||||||
|
"width": backing_w,
|
||||||
|
"height": backing_h,
|
||||||
|
}))
|
||||||
|
} else {
|
||||||
|
// Fallback: screenshot the canvas element (or its bbox clip).
|
||||||
|
let clip = if selector.is_none() {
|
||||||
|
let x = probe.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
let y = probe.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
let cw = probe.get("cw").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
let ch = probe.get("ch").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
Some((x, y, cw, ch))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let options = ScreenshotOptions {
|
||||||
|
selector: selector.clone(),
|
||||||
|
path: out_path,
|
||||||
|
clip,
|
||||||
|
..ScreenshotOptions::default()
|
||||||
|
};
|
||||||
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
let result = screenshot::take_screenshot(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.ref_map,
|
||||||
|
&options,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let why = probe
|
||||||
|
.get("err")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|e| format!("toDataURL failed ({e})"))
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
"toDataURL blank/unavailable (WebGL no preserveDrawingBuffer)".into()
|
||||||
|
});
|
||||||
|
Ok(json!({
|
||||||
|
"path": absolutize_saved_path(&result.path),
|
||||||
|
"method": "screenshot",
|
||||||
|
"note": format!("captured rendered pixels via screenshot — {why}"),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
// First-class coordinate click (issue #8.4): click a raw viewport point with
|
// First-class coordinate click (issue #8.4): click a raw viewport point with
|
||||||
// no element resolution. Parsed from `click <x> <y>` / `click --coords x,y`.
|
// no element resolution. Parsed from `click <x> <y>` / `click --coords x,y`.
|
||||||
@@ -5102,6 +5252,19 @@ async fn handle_tab_close(cmd: &Value, state: &mut DaemonState) -> Result<Value,
|
|||||||
|
|
||||||
async fn handle_viewport(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_viewport(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
|
|
||||||
|
// `viewport reset` clears the device-metrics override and restores the real
|
||||||
|
// layout viewport (the launched window's size, or — over the relay — the
|
||||||
|
// user's actual Chrome window).
|
||||||
|
if cmd.get("reset").and_then(|v| v.as_bool()).unwrap_or(false) {
|
||||||
|
mgr.clear_viewport().await?;
|
||||||
|
state.viewport = None;
|
||||||
|
if let Some(ref server) = state.stream_server {
|
||||||
|
server.set_viewport(1280, 720).await;
|
||||||
|
}
|
||||||
|
return Ok(json!({ "reset": true }));
|
||||||
|
}
|
||||||
|
|
||||||
let width = cmd.get("width").and_then(|v| v.as_i64()).unwrap_or(1280) as i32;
|
let width = cmd.get("width").and_then(|v| v.as_i64()).unwrap_or(1280) as i32;
|
||||||
let height = cmd.get("height").and_then(|v| v.as_i64()).unwrap_or(720) as i32;
|
let height = cmd.get("height").and_then(|v| v.as_i64()).unwrap_or(720) as i32;
|
||||||
let scale = cmd
|
let scale = cmd
|
||||||
|
|||||||
@@ -3319,6 +3319,10 @@ Core Commands:
|
|||||||
screenshot [path] Take screenshot (auto-downscaled to ≤2000px long edge;
|
screenshot [path] Take screenshot (auto-downscaled to ≤2000px long edge;
|
||||||
--max-width/--max-height/--scale to override)
|
--max-width/--max-height/--scale to override)
|
||||||
pdf <path> Save as PDF
|
pdf <path> Save as PDF
|
||||||
|
canvas list List <canvas> elements (size, type) on the page
|
||||||
|
canvas capture [sel] [path] Save a canvas's rendered pixels to PNG — for
|
||||||
|
WebGL/canvas apps (Figma, games, maps, charts) that
|
||||||
|
expose no DOM. toDataURL, with a screenshot fallback.
|
||||||
snapshot Accessibility tree with refs (for AI)
|
snapshot Accessibility tree with refs (for AI)
|
||||||
eval <js> Run JavaScript
|
eval <js> Run JavaScript
|
||||||
connect <port|url> Connect to browser via CDP
|
connect <port|url> Connect to browser via CDP
|
||||||
|
|||||||
+88
-8
@@ -31,6 +31,10 @@ fn dirs_home() -> Option<PathBuf> {
|
|||||||
pub struct Adapter {
|
pub struct Adapter {
|
||||||
pub meta: Value,
|
pub meta: Value,
|
||||||
pub func_src: String,
|
pub func_src: String,
|
||||||
|
/// The adapter's declared `args` keys in DECLARATION order. Parsed from the
|
||||||
|
/// raw @meta text because `serde_json` sorts object keys alphabetically, which
|
||||||
|
/// would otherwise scramble positional-arg mapping for multi-arg adapters.
|
||||||
|
pub arg_order: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Adapter {
|
impl Adapter {
|
||||||
@@ -111,7 +115,66 @@ pub fn parse_adapter(raw: &str, spec: &str) -> Result<Adapter, String> {
|
|||||||
if func_src.is_empty() {
|
if func_src.is_empty() {
|
||||||
return Err(format!("site: {spec} has no function body after @meta"));
|
return Err(format!("site: {spec} has no function body after @meta"));
|
||||||
}
|
}
|
||||||
Ok(Adapter { meta, func_src })
|
let arg_order = arg_order_from_meta(&raw[start..end]);
|
||||||
|
Ok(Adapter {
|
||||||
|
meta,
|
||||||
|
func_src,
|
||||||
|
arg_order,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the `args` object's keys in DECLARATION order from the raw @meta JSON
|
||||||
|
/// text (serde sorts them, losing order). Brace/string-aware: finds the `"args"`
|
||||||
|
/// value object and collects only its top-level keys.
|
||||||
|
fn arg_order_from_meta(meta_json: &str) -> Vec<String> {
|
||||||
|
let bytes = meta_json.as_bytes();
|
||||||
|
// Locate the `"args"` key, then the `{` that opens its value object.
|
||||||
|
let Some(args_pos) = meta_json.find("\"args\"") else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let Some(brace_off) = meta_json[args_pos..].find('{') else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let open = args_pos + brace_off;
|
||||||
|
let mut keys = Vec::new();
|
||||||
|
let mut depth = 0i32;
|
||||||
|
let mut in_str = false;
|
||||||
|
let mut esc = false;
|
||||||
|
let mut cur = String::new();
|
||||||
|
let mut last_str: Option<String> = None;
|
||||||
|
for &b in bytes.iter().skip(open) {
|
||||||
|
if in_str {
|
||||||
|
if esc {
|
||||||
|
esc = false;
|
||||||
|
} else if b == b'\\' {
|
||||||
|
esc = true;
|
||||||
|
} else if b == b'"' {
|
||||||
|
in_str = false;
|
||||||
|
last_str = Some(std::mem::take(&mut cur));
|
||||||
|
} else {
|
||||||
|
cur.push(b as char);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match b {
|
||||||
|
b'"' => in_str = true,
|
||||||
|
b'{' => depth += 1,
|
||||||
|
b'}' => {
|
||||||
|
depth -= 1;
|
||||||
|
if depth == 0 {
|
||||||
|
break; // end of the args object
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A `:` at depth 1 means the preceding string was a key of `args`.
|
||||||
|
b':' if depth == 1 => {
|
||||||
|
if let Some(k) = last_str.take() {
|
||||||
|
keys.push(k);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keys
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the JS to eval: `(<adapter function>)(<args JSON>)`. The adapter's
|
/// Build the JS to eval: `(<adapter function>)(<args JSON>)`. The adapter's
|
||||||
@@ -325,14 +388,11 @@ pub fn adapters_for_domain(host: &str) -> Vec<String> {
|
|||||||
/// validates required args itself.
|
/// validates required args itself.
|
||||||
pub fn map_args(adapter: &Adapter, positional: &[String], named: &[(String, String)]) -> Value {
|
pub fn map_args(adapter: &Adapter, positional: &[String], named: &[(String, String)]) -> Value {
|
||||||
let mut obj = serde_json::Map::new();
|
let mut obj = serde_json::Map::new();
|
||||||
let keys: Vec<String> = adapter
|
// Positional args fill the adapter's declared args in DECLARATION order
|
||||||
.meta
|
// (`arg_order`), not serde's alphabetized key order — otherwise a 2-arg
|
||||||
.get("args")
|
// adapter like `{projectId, path}` would map positionals to `{path, projectId}`.
|
||||||
.and_then(|a| a.as_object())
|
|
||||||
.map(|m| m.keys().cloned().collect())
|
|
||||||
.unwrap_or_default();
|
|
||||||
for (i, val) in positional.iter().enumerate() {
|
for (i, val) in positional.iter().enumerate() {
|
||||||
if let Some(k) = keys.get(i) {
|
if let Some(k) = adapter.arg_order.get(i) {
|
||||||
obj.insert(k.clone(), Value::String(val.clone()));
|
obj.insert(k.clone(), Value::String(val.clone()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -382,4 +442,24 @@ async function(args) { return { repo: args.repo }; }"#;
|
|||||||
assert!(load_adapter("noslash").is_err());
|
assert!(load_adapter("noslash").is_err());
|
||||||
assert!(load_adapter("../etc/passwd").is_err());
|
assert!(load_adapter("../etc/passwd").is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Regression: positional args must follow DECLARATION order, not serde's
|
||||||
|
// alphabetical key order. With `{projectId, path}` (not alphabetical),
|
||||||
|
// `<uuid> <file>` must map projectId←uuid, path←file — not swapped.
|
||||||
|
#[test]
|
||||||
|
fn positional_args_follow_declaration_order_not_alphabetical() {
|
||||||
|
let raw = r#"/* @meta
|
||||||
|
{
|
||||||
|
"name": "claude-design/get-file",
|
||||||
|
"domain": "claude.ai",
|
||||||
|
"args": { "projectId": {"required": true}, "path": {"required": true} }
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
async function(args){ return args; }"#;
|
||||||
|
let a = parse_adapter(raw, "claude-design/get-file").unwrap();
|
||||||
|
assert_eq!(a.arg_order, vec!["projectId", "path"]);
|
||||||
|
let args = map_args(&a, &["the-uuid".into(), "misonote.dc.html".into()], &[]);
|
||||||
|
assert_eq!(args["projectId"], "the-uuid");
|
||||||
|
assert_eq!(args["path"], "misonote.dc.html");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "1.5.23",
|
"version": "1.5.24",
|
||||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"packageManager": "pnpm@11.1.3",
|
"packageManager": "pnpm@11.1.3",
|
||||||
|
|||||||
@@ -154,7 +154,17 @@ real Chrome concurrently without ever dropping or stealing each other's tabs —
|
|||||||
another agent's tab churn can't make your bound tab vanish or drift your commands
|
another agent's tab churn can't make your bound tab vanish or drift your commands
|
||||||
onto the wrong page. Consequence: `tab list` shows only *your* session's tabs; to
|
onto the wrong page. Consequence: `tab list` shows only *your* session's tabs; to
|
||||||
drive a specific page, navigate to it in your own tab instead of expecting a
|
drive a specific page, navigate to it in your own tab instead of expecting a
|
||||||
pre-existing or popped-up tab to appear in the list. **Anti-detection ranking: this real logged-in Chrome (extension
|
pre-existing or popped-up tab to appear in the list.
|
||||||
|
|
||||||
|
> **Need to read a tab the user already has open?** Use `chrome-use adopt
|
||||||
|
> <url-substring|targetId>` — it finds that pre-existing tab (the user's own, or
|
||||||
|
> another session's) across groups and drives it **without opening a new tab**.
|
||||||
|
> e.g. `adopt "claude.ai/design"` then `snapshot`/`eval`/`get text` on it. On no
|
||||||
|
> match it errors and lists the tabs it can see. This is the explicit, opt-in way
|
||||||
|
> through the isolation above (it tags the adopted tab into your group). Great for
|
||||||
|
> "read/extract from the page I'm looking at" without disturbing it.
|
||||||
|
|
||||||
|
**Anti-detection ranking: this real logged-in Chrome (extension
|
||||||
connect) > a headed launched browser > headless (forbidden).** A genuine human
|
connect) > a headed launched browser > headless (forbidden).** A genuine human
|
||||||
browser has no headless/automation tells at all, so prefer it for anything
|
browser has no headless/automation tells at all, so prefer it for anything
|
||||||
anti-bot-sensitive.
|
anti-bot-sensitive.
|
||||||
@@ -444,6 +454,13 @@ tree**, so `snapshot` comes back near-empty and refs are a dead end. `snapshot`
|
|||||||
detects this and prints a one-line hint. Drive them the screenshot way:
|
detects this and prints a one-line hint. Drive them the screenshot way:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
chrome-use canvas list # enumerate <canvas> elements (size, type)
|
||||||
|
chrome-use canvas capture out.png # save the canvas's RENDERED pixels to PNG —
|
||||||
|
# toDataURL (full backing-store res, e.g.
|
||||||
|
# Figma 2522x1904), screenshot fallback for
|
||||||
|
# WebGL w/o preserveDrawingBuffer / tainted.
|
||||||
|
# Gets the RENDER, not hidden source data
|
||||||
|
# (those live in the app's binary store/API).
|
||||||
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
||||||
# eval/get text return nothing useful)
|
# eval/get text return nothing useful)
|
||||||
chrome-use click 640 360 # interact by viewport coordinate
|
chrome-use click 640 360 # interact by viewport coordinate
|
||||||
|
|||||||
Reference in New Issue
Block a user