Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
997373fd57 | ||
|
|
5a858af93f | ||
|
|
58dc02bfdc | ||
|
|
c47601bd7b | ||
|
|
0296bc7a88 | ||
|
|
32e203b908 | ||
|
|
fc51cd63ba | ||
|
|
f714c7920b | ||
|
|
1ac8ef7732 | ||
|
|
9f24e66033 | ||
|
|
70ab38d35f | ||
|
|
6830df50ea | ||
|
|
cd47ec43d0 | ||
|
|
2cd361817d | ||
|
|
42f47c49aa | ||
|
|
e29800df72 | ||
|
|
e7e849ea39 | ||
|
|
ebb02c65c8 |
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.5.7"
|
version = "1.5.15"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.5.7"
|
version = "1.5.15"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "Fast browser automation CLI for AI agents"
|
description = "Fast browser automation CLI for AI agents"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
+216
-20
@@ -34,11 +34,50 @@ pub enum ParseError {
|
|||||||
/// suggestions on an unknown command (issue #29). Not exhaustive — just the
|
/// suggestions on an unknown command (issue #29). Not exhaustive — just the
|
||||||
/// common verbs plus a few known wrong-guesses mapped to the real command.
|
/// common verbs plus a few known wrong-guesses mapped to the real command.
|
||||||
const KNOWN_COMMANDS: &[&str] = &[
|
const KNOWN_COMMANDS: &[&str] = &[
|
||||||
"open", "navigate", "click", "fill", "type", "press", "snapshot", "screenshot", "eval", "get",
|
"open",
|
||||||
"text", "html", "frames", "find", "wait", "scroll", "hover", "select", "check", "uncheck",
|
"navigate",
|
||||||
"tab", "tabs", "close", "back", "forward", "reload", "sessions", "status", "daemon", "doctor",
|
"click",
|
||||||
"upgrade", "connect", "cookies", "mouse", "keyboard", "stream", "frame", "profiles", "title",
|
"fill",
|
||||||
"url", "is", "drag", "dialog", "upload",
|
"type",
|
||||||
|
"press",
|
||||||
|
"snapshot",
|
||||||
|
"screenshot",
|
||||||
|
"eval",
|
||||||
|
"get",
|
||||||
|
"text",
|
||||||
|
"html",
|
||||||
|
"frames",
|
||||||
|
"find",
|
||||||
|
"wait",
|
||||||
|
"scroll",
|
||||||
|
"hover",
|
||||||
|
"select",
|
||||||
|
"check",
|
||||||
|
"uncheck",
|
||||||
|
"tab",
|
||||||
|
"tabs",
|
||||||
|
"close",
|
||||||
|
"back",
|
||||||
|
"forward",
|
||||||
|
"reload",
|
||||||
|
"sessions",
|
||||||
|
"status",
|
||||||
|
"daemon",
|
||||||
|
"doctor",
|
||||||
|
"upgrade",
|
||||||
|
"connect",
|
||||||
|
"cookies",
|
||||||
|
"mouse",
|
||||||
|
"keyboard",
|
||||||
|
"stream",
|
||||||
|
"frame",
|
||||||
|
"profiles",
|
||||||
|
"title",
|
||||||
|
"url",
|
||||||
|
"is",
|
||||||
|
"drag",
|
||||||
|
"dialog",
|
||||||
|
"upload",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Levenshtein distance, capped — small inputs only (command names).
|
/// Levenshtein distance, capped — small inputs only (command names).
|
||||||
@@ -525,20 +564,31 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": rest[1..].join(" ") }))
|
Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": rest[1..].join(" ") }))
|
||||||
}
|
}
|
||||||
"type" => {
|
"type" => {
|
||||||
|
// `--key-events` (alias `--keys`): send real per-character keystrokes
|
||||||
|
// instead of Input.insertText, so autocomplete/combobox widgets that
|
||||||
|
// only react to key events fire (e.g. Google address postal lookup).
|
||||||
|
let key_events = rest.iter().any(|a| *a == "--key-events" || *a == "--keys");
|
||||||
|
let rest: Vec<&str> = rest
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|a| *a != "--key-events" && *a != "--keys")
|
||||||
|
.collect();
|
||||||
// `type --focused <text>` types into whatever element currently has
|
// `type --focused <text>` types into whatever element currently has
|
||||||
// focus (no selector) — for custom widgets that move focus to a hidden
|
// focus (no selector) — for custom widgets that move focus to a hidden
|
||||||
// input after you open them.
|
// input after you open them.
|
||||||
if rest.first() == Some(&"--focused") {
|
if rest.first() == Some(&"--focused") {
|
||||||
return Ok(json!({
|
return Ok(json!({
|
||||||
"id": id, "action": "type", "focused": true,
|
"id": id, "action": "type", "focused": true,
|
||||||
"text": rest[1..].join(" "),
|
"text": rest[1..].join(" "), "keyEvents": key_events,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
context: "type".to_string(),
|
context: "type".to_string(),
|
||||||
usage: "type <selector> <text> (or: type --focused <text>)",
|
usage: "type <selector> <text> (or: type --focused <text>) [--key-events]",
|
||||||
})?;
|
})?;
|
||||||
Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" ") }))
|
Ok(
|
||||||
|
json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" "), "keyEvents": key_events }),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
"pick" => {
|
"pick" => {
|
||||||
// pick <selector|@ref> --option "<text>" — atomic combobox select:
|
// pick <selector|@ref> --option "<text>" — atomic combobox select:
|
||||||
@@ -729,10 +779,57 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
} else {
|
} else {
|
||||||
return Err(ParseError::MissingArguments {
|
return Err(ParseError::MissingArguments {
|
||||||
context: "scroll --selector".to_string(),
|
context: "scroll --selector".to_string(),
|
||||||
usage: "scroll [direction] [amount] [--selector <sel>]",
|
usage: "scroll [direction] [amount] [--selector <sel>] [--at <x,y>] [--frame <n>]",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
"--at" => {
|
||||||
|
// `--at x,y`: dispatch the wheel at this viewport pixel, so it
|
||||||
|
// scrolls whatever element/iframe is under the pointer — including
|
||||||
|
// cross-origin iframes that `window.scrollBy` can't reach (#36).
|
||||||
|
let val = rest.get(i + 1).ok_or(ParseError::MissingArguments {
|
||||||
|
context: "scroll --at".to_string(),
|
||||||
|
usage: "scroll [direction] [amount] --at <x,y>",
|
||||||
|
})?;
|
||||||
|
let mut parts = val.split(',');
|
||||||
|
match (
|
||||||
|
parts.next().and_then(|s| s.trim().parse::<f64>().ok()),
|
||||||
|
parts.next().and_then(|s| s.trim().parse::<f64>().ok()),
|
||||||
|
) {
|
||||||
|
(Some(x), Some(y)) => {
|
||||||
|
obj.insert("at".to_string(), json!([x, y]));
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!("scroll --at: invalid coordinate `{}`", val),
|
||||||
|
usage:
|
||||||
|
"scroll [direction] [amount] --at <x,y> (e.g. --at 640,400)",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
"--frame" => {
|
||||||
|
// `--frame n`: scroll the n-th frame from `chrome-use frames` by
|
||||||
|
// dispatching the wheel at that frame's center — reaches content in
|
||||||
|
// a cross-origin iframe without needing a selector into it (#36).
|
||||||
|
let val = rest.get(i + 1).ok_or(ParseError::MissingArguments {
|
||||||
|
context: "scroll --frame".to_string(),
|
||||||
|
usage: "scroll [direction] [amount] --frame <n>",
|
||||||
|
})?;
|
||||||
|
match val.trim().parse::<usize>() {
|
||||||
|
Ok(n) => {
|
||||||
|
obj.insert("frame".to_string(), json!(n));
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!("scroll --frame: invalid index `{}`", val),
|
||||||
|
usage: "scroll [direction] [amount] --frame <n> (index from `chrome-use frames`)",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
arg if arg.starts_with('-') => {}
|
arg if arg.starts_with('-') => {}
|
||||||
_ => {
|
_ => {
|
||||||
match positional_index {
|
match positional_index {
|
||||||
@@ -907,17 +1004,41 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
// selector: @ref or CSS selector
|
// selector: @ref or CSS selector
|
||||||
// path: file path (contains / or . or ends with known extension)
|
// path: file path (contains / or . or ends with known extension)
|
||||||
let mut full_page = false;
|
let mut full_page = false;
|
||||||
let positional: Vec<&str> = rest
|
let mut clip: Option<Value> = None;
|
||||||
.iter()
|
let mut positional: Vec<&str> = Vec::new();
|
||||||
.filter(|arg| match **arg {
|
let mut i = 0;
|
||||||
"--full" | "-f" => {
|
while i < rest.len() {
|
||||||
full_page = true;
|
match rest[i] {
|
||||||
false
|
"--full" | "-f" => full_page = true,
|
||||||
|
// `--clip x,y,w,h` captures a pixel region (issue #34).
|
||||||
|
"--clip" => {
|
||||||
|
let raw = rest
|
||||||
|
.get(i + 1)
|
||||||
|
.ok_or_else(|| ParseError::MissingArguments {
|
||||||
|
context: "screenshot --clip".to_string(),
|
||||||
|
usage: "screenshot --clip <x,y,w,h> [path]",
|
||||||
|
})?;
|
||||||
|
let nums: Vec<f64> = raw
|
||||||
|
.split(',')
|
||||||
|
.filter_map(|n| n.trim().parse::<f64>().ok())
|
||||||
|
.collect();
|
||||||
|
if nums.len() != 4 {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!(
|
||||||
|
"--clip expects 'x,y,w,h' (4 numbers), got '{raw}'"
|
||||||
|
),
|
||||||
|
usage: "screenshot --clip <x,y,w,h> [path]",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
clip = Some(json!({
|
||||||
|
"x": nums[0], "y": nums[1], "width": nums[2], "height": nums[3]
|
||||||
|
}));
|
||||||
|
i += 1;
|
||||||
}
|
}
|
||||||
_ => true,
|
other => positional.push(other),
|
||||||
})
|
}
|
||||||
.copied()
|
i += 1;
|
||||||
.collect();
|
}
|
||||||
let (selector, path) = match (positional.first(), positional.get(1)) {
|
let (selector, path) = match (positional.first(), positional.get(1)) {
|
||||||
(Some(first), Some(second)) => {
|
(Some(first), Some(second)) => {
|
||||||
// Two args: first is selector, second is path
|
// Two args: first is selector, second is path
|
||||||
@@ -948,6 +1069,9 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
"path": path, "selector": selector,
|
"path": path, "selector": selector,
|
||||||
"fullPage": full_page, "annotate": flags.annotate
|
"fullPage": full_page, "annotate": flags.annotate
|
||||||
});
|
});
|
||||||
|
if let Some(c) = clip {
|
||||||
|
cmd["clip"] = c;
|
||||||
|
}
|
||||||
if let Some(ref fmt) = flags.screenshot_format {
|
if let Some(ref fmt) = flags.screenshot_format {
|
||||||
cmd["format"] = json!(fmt);
|
cmd["format"] = json!(fmt);
|
||||||
}
|
}
|
||||||
@@ -4043,6 +4167,28 @@ mod tests {
|
|||||||
assert_eq!(cmd["action"], "type");
|
assert_eq!(cmd["action"], "type");
|
||||||
assert_eq!(cmd["selector"], "#input");
|
assert_eq!(cmd["selector"], "#input");
|
||||||
assert_eq!(cmd["text"], "some text");
|
assert_eq!(cmd["text"], "some text");
|
||||||
|
assert_eq!(cmd["keyEvents"], false);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_type_key_events() {
|
||||||
|
// --key-events sends real keystrokes (for autocomplete/combobox) and must
|
||||||
|
// not be swallowed into the typed text.
|
||||||
|
let cmd = parse_command(
|
||||||
|
&args("type #postal 201-0001 --key-events"),
|
||||||
|
&default_flags(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cmd["action"], "type");
|
||||||
|
assert_eq!(cmd["selector"], "#postal");
|
||||||
|
assert_eq!(cmd["text"], "201-0001");
|
||||||
|
assert_eq!(cmd["keyEvents"], true);
|
||||||
|
|
||||||
|
let focused =
|
||||||
|
parse_command(&args("type --focused 201-0001 --keys"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(focused["focused"], true);
|
||||||
|
assert_eq!(focused["text"], "201-0001");
|
||||||
|
assert_eq!(focused["keyEvents"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -4327,6 +4473,24 @@ mod tests {
|
|||||||
assert_eq!(cmd["fullPage"], true);
|
assert_eq!(cmd["fullPage"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_screenshot_clip() {
|
||||||
|
// `--clip x,y,w,h` captures a pixel region (issue #34); the path still parses.
|
||||||
|
let cmd = parse_command(
|
||||||
|
&args("screenshot --clip 10,20,200,40 out.png"),
|
||||||
|
&default_flags(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cmd["action"], "screenshot");
|
||||||
|
assert_eq!(cmd["clip"]["x"], 10.0);
|
||||||
|
assert_eq!(cmd["clip"]["y"], 20.0);
|
||||||
|
assert_eq!(cmd["clip"]["width"], 200.0);
|
||||||
|
assert_eq!(cmd["clip"]["height"], 40.0);
|
||||||
|
assert_eq!(cmd["path"], "out.png");
|
||||||
|
// Bad clip is a clear error, not silent.
|
||||||
|
assert!(parse_command(&args("screenshot --clip 1,2,3"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_screenshot_with_ref() {
|
fn test_screenshot_with_ref() {
|
||||||
let cmd = parse_command(&args("screenshot @e1"), &default_flags()).unwrap();
|
let cmd = parse_command(&args("screenshot @e1"), &default_flags()).unwrap();
|
||||||
@@ -4894,7 +5058,10 @@ mod tests {
|
|||||||
assert_eq!(nearest_command("sesions").as_deref(), Some("sessions"));
|
assert_eq!(nearest_command("sesions").as_deref(), Some("sessions"));
|
||||||
assert_eq!(nearest_command("session").as_deref(), Some("sessions"));
|
assert_eq!(nearest_command("session").as_deref(), Some("sessions"));
|
||||||
assert_eq!(nearest_command("clik").as_deref(), Some("click"));
|
assert_eq!(nearest_command("clik").as_deref(), Some("click"));
|
||||||
assert_eq!(nearest_command("screenshits").as_deref(), Some("screenshot"));
|
assert_eq!(
|
||||||
|
nearest_command("screenshits").as_deref(),
|
||||||
|
Some("screenshot")
|
||||||
|
);
|
||||||
// Nonsense with no close match stays silent.
|
// Nonsense with no close match stays silent.
|
||||||
assert_eq!(nearest_command("xyzzy"), None);
|
assert_eq!(nearest_command("xyzzy"), None);
|
||||||
// The unknown-command error embeds the suggestion.
|
// The unknown-command error embeds the suggestion.
|
||||||
@@ -5830,6 +5997,35 @@ mod tests {
|
|||||||
assert_eq!(cmd["selector"], ".sidebar");
|
assert_eq!(cmd["selector"], ".sidebar");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_at_coordinate() {
|
||||||
|
// `--at x,y` carries a [x, y] array for a wheel dispatched at that pixel
|
||||||
|
// (issue #36: cross-origin iframe scroll).
|
||||||
|
let cmd = parse_command(&args("scroll down 700 --at 640,400"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "scroll");
|
||||||
|
assert_eq!(cmd["direction"], "down");
|
||||||
|
assert_eq!(cmd["amount"], 700);
|
||||||
|
assert_eq!(cmd["at"], json!([640.0, 400.0]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_at_rejects_garbage() {
|
||||||
|
assert!(parse_command(&args("scroll --at nope"), &default_flags()).is_err());
|
||||||
|
assert!(parse_command(&args("scroll --at 1"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_frame_index() {
|
||||||
|
let cmd = parse_command(&args("scroll down 700 --frame 2"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "scroll");
|
||||||
|
assert_eq!(cmd["frame"], 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_frame_rejects_non_integer() {
|
||||||
|
assert!(parse_command(&args("scroll --frame two"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_scroll_selector_before_positional() {
|
fn test_scroll_selector_before_positional() {
|
||||||
let cmd =
|
let cmd =
|
||||||
|
|||||||
@@ -1369,6 +1369,20 @@ fn main() {
|
|||||||
&& flags.provider.is_none()
|
&& flags.provider.is_none()
|
||||||
&& (flags.force_launch || !flags.auto_connect)
|
&& (flags.force_launch || !flags.auto_connect)
|
||||||
{
|
{
|
||||||
|
// Launching a debug-port Chrome pops Chrome's "Allow remote debugging?"
|
||||||
|
// consent modal (Chrome 136+). When the ab-connect relay is already up,
|
||||||
|
// this is almost always unintended — the relay drives the user's real
|
||||||
|
// Chrome with NO modal. Warn so the modal is self-explained and the
|
||||||
|
// caller (often a stray --launch / --no-auto-connect) is fixable (#32).
|
||||||
|
if !flags.json && connect::relay_url().is_some() {
|
||||||
|
eprintln!(
|
||||||
|
"{} launching a new Chrome with a debug port — this pops Chrome's \
|
||||||
|
\"Allow remote debugging?\" modal.\n The ab-connect relay is up; \
|
||||||
|
drop --launch/--new (and don't pass --no-auto-connect) to drive your \
|
||||||
|
real Chrome with no modal.",
|
||||||
|
color::warning_indicator()
|
||||||
|
);
|
||||||
|
}
|
||||||
let mut launch_cmd = json!({
|
let mut launch_cmd = json!({
|
||||||
"id": gen_id(),
|
"id": gen_id(),
|
||||||
"action": "launch",
|
"action": "launch",
|
||||||
|
|||||||
+234
-12
@@ -3000,6 +3000,14 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
|||||||
.get("screenshotDir")
|
.get("screenshotDir")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.map(String::from),
|
.map(String::from),
|
||||||
|
clip: cmd.get("clip").and_then(|c| {
|
||||||
|
Some((
|
||||||
|
c.get("x")?.as_f64()?,
|
||||||
|
c.get("y")?.as_f64()?,
|
||||||
|
c.get("width")?.as_f64()?,
|
||||||
|
c.get("height")?.as_f64()?,
|
||||||
|
))
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
if annotate {
|
if annotate {
|
||||||
@@ -3217,6 +3225,14 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
|
||||||
|
// `--key-events`: dispatch real per-character keyDown/keyUp instead of
|
||||||
|
// Input.insertText, so autocomplete/combobox widgets that only react to key
|
||||||
|
// events fire (e.g. Google's address postal-code lookup) (issue #4/#36).
|
||||||
|
let key_events = cmd
|
||||||
|
.get("keyEvents")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
// `type --focused <text>`: type into the currently-focused element without a
|
// `type --focused <text>`: type into the currently-focused element without a
|
||||||
// selector (custom widgets that move focus to a hidden input on open).
|
// selector (custom widgets that move focus to a hidden input on open).
|
||||||
if cmd
|
if cmd
|
||||||
@@ -3228,7 +3244,14 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
.get("text")
|
.get("text")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'text' parameter")?;
|
.ok_or("Missing 'text' parameter")?;
|
||||||
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None).await?;
|
interaction::type_text_into_active_context(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
text,
|
||||||
|
None,
|
||||||
|
key_events,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
return Ok(json!({ "typed": text, "focused": true }));
|
return Ok(json!({ "typed": text, "focused": true }));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3252,6 +3275,7 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
clear,
|
clear,
|
||||||
delay,
|
delay,
|
||||||
&state.iframe_sessions,
|
&state.iframe_sessions,
|
||||||
|
key_events,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(json!({ "typed": text }))
|
Ok(json!({ "typed": text }))
|
||||||
@@ -3459,17 +3483,182 @@ async fn handle_scroll(cmd: &Value, state: &mut DaemonState) -> Result<Value, St
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An explicit `--selector` keeps the precise element-scroll path (scrollBy on
|
||||||
|
// the resolved node, same-origin only).
|
||||||
|
if let Some(sel) = selector {
|
||||||
|
interaction::scroll(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.ref_map,
|
||||||
|
Some(sel),
|
||||||
|
dx,
|
||||||
|
dy,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(json!({ "scrolled": true, "via": "selector" }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// `--at x,y` / `--frame n`: dispatch a real (isTrusted) wheel at a viewport
|
||||||
|
// coordinate. This hits the compositor and scrolls whatever scroll container
|
||||||
|
// is under the pointer — including cross-origin iframes that `window.scrollBy`
|
||||||
|
// on the top document silently no-ops on (issue #36).
|
||||||
|
if cmd.get("at").is_some() || cmd.get("frame").is_some() {
|
||||||
|
let (x, y, via) = if let Some(at) = cmd.get("at").and_then(|v| v.as_array()) {
|
||||||
|
let x = at.first().and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
let y = at.get(1).and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
(x, y, "at")
|
||||||
|
} else {
|
||||||
|
let n = cmd.get("frame").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
let (x, y) = frame_center(mgr, &session_id, &state.iframe_sessions, n as usize).await?;
|
||||||
|
(x, y, "frame")
|
||||||
|
};
|
||||||
|
dispatch_wheel(&mgr.client, &session_id, x, y, dx, dy).await?;
|
||||||
|
return Ok(json!({ "scrolled": true, "via": via, "at": [x, y] }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default (no selector/at/frame): scroll the page with `window.scrollBy`. This
|
||||||
|
// is the reliable path for ordinary page scrolling; a coordinate wheel at the
|
||||||
|
// viewport centre is NOT a dependable substitute (it no-ops on some pages,
|
||||||
|
// e.g. headless), so the wheel stays opt-in via `--at`/`--frame` for the
|
||||||
|
// cross-origin-iframe case (issue #36).
|
||||||
interaction::scroll(
|
interaction::scroll(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
&session_id,
|
&session_id,
|
||||||
&state.ref_map,
|
&state.ref_map,
|
||||||
selector,
|
None,
|
||||||
dx,
|
dx,
|
||||||
dy,
|
dy,
|
||||||
&state.iframe_sessions,
|
&state.iframe_sessions,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(json!({ "scrolled": true }))
|
Ok(json!({ "scrolled": true, "via": "page" }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Viewport center in CSS pixels, used as the default wheel landing point for
|
||||||
|
/// `scroll` (issue #36). Falls back to a sane 640×400 center if the page can't
|
||||||
|
/// be evaluated (e.g. a restricted document).
|
||||||
|
async fn viewport_center(mgr: &BrowserManager, session_id: &str) -> Result<(f64, f64), String> {
|
||||||
|
let dims = mgr
|
||||||
|
.client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
&super::cdp::types::EvaluateParams {
|
||||||
|
expression: "[window.innerWidth, window.innerHeight]".to_string(),
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
let arr = dims
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("result"))
|
||||||
|
.and_then(|v| v.get("value"))
|
||||||
|
.and_then(|v| v.as_array());
|
||||||
|
let w = arr
|
||||||
|
.and_then(|a| a.first())
|
||||||
|
.and_then(|v| v.as_f64())
|
||||||
|
.filter(|w| *w > 0.0)
|
||||||
|
.unwrap_or(1280.0);
|
||||||
|
let h = arr
|
||||||
|
.and_then(|a| a.get(1))
|
||||||
|
.and_then(|v| v.as_f64())
|
||||||
|
.filter(|h| *h > 0.0)
|
||||||
|
.unwrap_or(800.0);
|
||||||
|
Ok((w / 2.0, h / 2.0))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Center of the `n`-th frame (as listed by `chrome-use frames`) in top-viewport
|
||||||
|
/// CSS pixels, so `scroll --frame n` lands its wheel inside a cross-origin iframe
|
||||||
|
/// without needing a selector into it (issue #36). Resolves the frame's owning
|
||||||
|
/// `<iframe>` element box via `DOM.getFrameOwner` + `DOM.getBoxModel` — exact for
|
||||||
|
/// a frame nested directly under the top document; for a deeper nesting the box is
|
||||||
|
/// relative to the intermediate frame, so prefer `--at x,y` from a screenshot.
|
||||||
|
async fn frame_center(
|
||||||
|
mgr: &BrowserManager,
|
||||||
|
session_id: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
n: usize,
|
||||||
|
) -> Result<(f64, f64), String> {
|
||||||
|
let frames =
|
||||||
|
super::element::collect_all_frames_text(&mgr.client, session_id, iframe_sessions).await?;
|
||||||
|
let frame = frames.get(n).ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"frame index {} out of range (run `chrome-use frames`: {} frame(s))",
|
||||||
|
n,
|
||||||
|
frames.len()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
if n == 0 {
|
||||||
|
// Frame 0 is the top document — there's no owner element; scroll its center.
|
||||||
|
return viewport_center(mgr, session_id).await;
|
||||||
|
}
|
||||||
|
let owner = mgr
|
||||||
|
.client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
&json!({ "frameId": frame.frame_id }),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("can't locate frame {}'s owner element: {}", n, e))?;
|
||||||
|
let backend_node_id = owner
|
||||||
|
.get("backendNodeId")
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
|
.ok_or_else(|| format!("frame {} has no owner <iframe> element", n))?;
|
||||||
|
let box_model = mgr
|
||||||
|
.client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"DOM.getBoxModel",
|
||||||
|
&json!({ "backendNodeId": backend_node_id }),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("can't measure frame {}'s box: {}", n, e))?;
|
||||||
|
let content = box_model
|
||||||
|
.get("model")
|
||||||
|
.and_then(|m| m.get("content"))
|
||||||
|
.and_then(|c| c.as_array())
|
||||||
|
.ok_or_else(|| format!("frame {} box model has no content quad", n))?;
|
||||||
|
let coord = |i: usize| content.get(i).and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
// content quad is [x1,y1, x2,y2, x3,y3, x4,y4]; opposite corners are 0 and 2.
|
||||||
|
let cx = (coord(0) + coord(4)) / 2.0;
|
||||||
|
let cy = (coord(1) + coord(5)) / 2.0;
|
||||||
|
Ok((cx, cy))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dispatch a trusted mouse wheel at `(x, y)`, humanized like `handle_wheel`.
|
||||||
|
async fn dispatch_wheel(
|
||||||
|
client: &super::cdp::client::CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
x: f64,
|
||||||
|
y: f64,
|
||||||
|
delta_x: f64,
|
||||||
|
delta_y: f64,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let level = humanize::active_level();
|
||||||
|
let seed = humanize::next_seed();
|
||||||
|
for (dx, dy, delay) in humanize::scroll_segments(delta_x, delta_y, level, seed) {
|
||||||
|
client
|
||||||
|
.send_command(
|
||||||
|
"Input.dispatchMouseEvent",
|
||||||
|
Some(json!({
|
||||||
|
"type": "mouseWheel",
|
||||||
|
"x": x,
|
||||||
|
"y": y,
|
||||||
|
"deltaX": dx,
|
||||||
|
"deltaY": dy,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if !delay.is_zero() {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_select(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_select(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
@@ -3666,12 +3855,9 @@ async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, S
|
|||||||
async fn handle_frames(_cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_frames(_cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
let frames = super::element::collect_all_frames_text(
|
let frames =
|
||||||
&mgr.client,
|
super::element::collect_all_frames_text(&mgr.client, &session_id, &state.iframe_sessions)
|
||||||
&session_id,
|
.await?;
|
||||||
&state.iframe_sessions,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
let list: Vec<Value> = frames
|
let list: Vec<Value> = frames
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
@@ -4167,7 +4353,10 @@ async fn handle_cf_status(_cmd: &Value, state: &mut DaemonState) -> Result<Value
|
|||||||
let url = mgr.get_url().await.unwrap_or_default();
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
|
||||||
// 1. Is the page a Cloudflare challenge right now?
|
// 1. Is the page a Cloudflare challenge right now?
|
||||||
let probe_raw = mgr.evaluate(CF_CHALLENGE_JS, None).await.unwrap_or(Value::Null);
|
let probe_raw = mgr
|
||||||
|
.evaluate(CF_CHALLENGE_JS, None)
|
||||||
|
.await
|
||||||
|
.unwrap_or(Value::Null);
|
||||||
let probe = parse_json_string(probe_raw, "cf challenge probe").unwrap_or(Value::Null);
|
let probe = parse_json_string(probe_raw, "cf challenge probe").unwrap_or(Value::Null);
|
||||||
let challenged = probe
|
let challenged = probe
|
||||||
.get("challenged")
|
.get("challenged")
|
||||||
@@ -4562,8 +4751,18 @@ async fn handle_keyboard(cmd: &Value, state: &DaemonState) -> Result<Value, Stri
|
|||||||
.get("text")
|
.get("text")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'text' parameter")?;
|
.ok_or("Missing 'text' parameter")?;
|
||||||
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None)
|
let key_events = cmd
|
||||||
.await?;
|
.get("keyEvents")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
interaction::type_text_into_active_context(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
text,
|
||||||
|
None,
|
||||||
|
key_events,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
return Ok(json!({ "typed": text }));
|
return Ok(json!({ "typed": text }));
|
||||||
}
|
}
|
||||||
Some("insertText") => {
|
Some("insertText") => {
|
||||||
@@ -7092,6 +7291,28 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'target' parameter")?;
|
.ok_or("Missing 'target' parameter")?;
|
||||||
|
|
||||||
|
// Over the relay (or into an iframe) a coordinate drag drifts to the
|
||||||
|
// foreground tab and can't reach an OOPIF — DOM-dispatch an HTML5 drag in the
|
||||||
|
// element's own session instead (issues #31/#36). `coord` mode forces the
|
||||||
|
// coordinate path for pointer-driven drags (canvas/sliders) on a launched
|
||||||
|
// browser.
|
||||||
|
if std::env::var("AGENT_BROWSER_CLICK_MODE").as_deref() != Ok("coord")
|
||||||
|
&& (crate::connect::relay_url().is_some()
|
||||||
|
|| state.ref_map.ref_is_in_iframe(source)
|
||||||
|
|| state.ref_map.ref_is_in_iframe(target))
|
||||||
|
{
|
||||||
|
super::interaction::dom_drag(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.ref_map,
|
||||||
|
source,
|
||||||
|
target,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(json!({ "dragged": { "source": source, "target": target }, "via": "dom" }));
|
||||||
|
}
|
||||||
|
|
||||||
let (sx, sy, _, _, source_session_id) = super::element::resolve_element_center(
|
let (sx, sy, _, _, source_session_id) = super::element::resolve_element_center(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
&session_id,
|
&session_id,
|
||||||
@@ -7446,6 +7667,7 @@ async fn handle_diff_screenshot(cmd: &Value, state: &DaemonState) -> Result<Valu
|
|||||||
quality: None,
|
quality: None,
|
||||||
annotate: false,
|
annotate: false,
|
||||||
output_dir: None,
|
output_dir: None,
|
||||||
|
clip: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
let result = screenshot::take_screenshot(
|
let result = screenshot::take_screenshot(
|
||||||
|
|||||||
+412
-53
@@ -121,7 +121,7 @@ fn normalize_url_for_match(url: &str) -> String {
|
|||||||
fn update_page_target_info_in_pages(pages: &mut [PageInfo], target: &TargetInfo) -> bool {
|
fn update_page_target_info_in_pages(pages: &mut [PageInfo], target: &TargetInfo) -> bool {
|
||||||
if let Some(page) = pages.iter_mut().find(|p| p.target_id == target.target_id) {
|
if let Some(page) = pages.iter_mut().find(|p| p.target_id == target.target_id) {
|
||||||
page.url = target.url.clone();
|
page.url = target.url.clone();
|
||||||
page.title = target.title.clone();
|
page.title = sanitize_title(&target.title);
|
||||||
page.target_type = target.target_type.clone();
|
page.target_type = target.target_type.clone();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -166,6 +166,54 @@ fn resolve_active_index(
|
|||||||
active_page_index
|
active_page_index
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Strip zero-width / invisible / bidi-format Unicode from a page title before
|
||||||
|
/// we store it. Some sites prepend runs of ZWJ / word-joiner / invisible-times /
|
||||||
|
/// BOM to `document.title` (badging, watermarking, anti-scrape); left in, they
|
||||||
|
/// pollute `tab list`, break text matching, and wreck column alignment (#33).
|
||||||
|
fn sanitize_title(s: &str) -> String {
|
||||||
|
s.chars()
|
||||||
|
.filter(|&c| {
|
||||||
|
!matches!(c as u32,
|
||||||
|
0x00AD // soft hyphen
|
||||||
|
| 0x200B..=0x200F // ZWSP, ZWNJ, ZWJ, LRM, RLM
|
||||||
|
| 0x2028 | 0x2029 // line / paragraph separators
|
||||||
|
| 0x202A..=0x202E // bidi embedding/override
|
||||||
|
| 0x2060..=0x2064 // word joiner, invisible operators
|
||||||
|
| 0x2066..=0x2069 // bidi isolates
|
||||||
|
| 0x180E // Mongolian vowel separator
|
||||||
|
| 0xFEFF // BOM / ZW no-break space
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<String>()
|
||||||
|
.trim()
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Best-effort MIME type from a filename extension, for the relay file-upload
|
||||||
|
/// fallback (the page-constructed `File` needs a sensible `type`). Covers the
|
||||||
|
/// common upload kinds; anything unknown falls back to a generic binary type.
|
||||||
|
fn mime_for_path(name: &str) -> &'static str {
|
||||||
|
let ext = name.rsplit('.').next().unwrap_or("").to_lowercase();
|
||||||
|
match ext.as_str() {
|
||||||
|
"png" => "image/png",
|
||||||
|
"jpg" | "jpeg" => "image/jpeg",
|
||||||
|
"gif" => "image/gif",
|
||||||
|
"webp" => "image/webp",
|
||||||
|
"svg" => "image/svg+xml",
|
||||||
|
"bmp" => "image/bmp",
|
||||||
|
"pdf" => "application/pdf",
|
||||||
|
"txt" => "text/plain",
|
||||||
|
"csv" => "text/csv",
|
||||||
|
"json" => "application/json",
|
||||||
|
"mp4" => "video/mp4",
|
||||||
|
"webm" => "video/webm",
|
||||||
|
"mov" => "video/quicktime",
|
||||||
|
"mp3" => "audio/mpeg",
|
||||||
|
"zip" => "application/zip",
|
||||||
|
_ => "application/octet-stream",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Target ids to prune after a `Target.getTargets` resync: tracked pages whose
|
/// Target ids to prune after a `Target.getTargets` resync: tracked pages whose
|
||||||
/// target is no longer in the live set — EXCEPT the explicitly-pinned active
|
/// target is no longer in the live set — EXCEPT the explicitly-pinned active
|
||||||
/// target, which is protected. The relay against a busy real Chrome occasionally
|
/// target, which is protected. The relay against a busy real Chrome occasionally
|
||||||
@@ -206,6 +254,21 @@ fn active_index_is_owned(
|
|||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a CDP error means the bound relay target is gone — the tab was
|
||||||
|
/// closed, navigated across processes (renderer swap), or lost after an
|
||||||
|
/// extension/service-worker restart, and the relay could not re-attach. The
|
||||||
|
/// ab-connect relay surfaces these as `stale sessionId … its tab is gone`,
|
||||||
|
/// `unknown sessionId …`, or `no attached tab …`. `navigate` keys its
|
||||||
|
/// auto-reattach recovery off this (issue #35) so a dead session rebinds to a
|
||||||
|
/// fresh tab instead of erroring on every command until the user runs `tab new`.
|
||||||
|
fn is_stale_target_error(error: &str) -> bool {
|
||||||
|
let lower = error.to_lowercase();
|
||||||
|
lower.contains("its tab is gone")
|
||||||
|
|| lower.contains("stale sessionid")
|
||||||
|
|| lower.contains("unknown sessionid")
|
||||||
|
|| lower.contains("no attached tab")
|
||||||
|
}
|
||||||
|
|
||||||
/// Converts common error messages into AI-friendly, actionable descriptions.
|
/// Converts common error messages into AI-friendly, actionable descriptions.
|
||||||
pub fn to_ai_friendly_error(error: &str) -> String {
|
pub fn to_ai_friendly_error(error: &str) -> String {
|
||||||
let lower = error.to_lowercase();
|
let lower = error.to_lowercase();
|
||||||
@@ -516,7 +579,10 @@ impl BrowserManager {
|
|||||||
crate::connect::log_connect_mode(
|
crate::connect::log_connect_mode(
|
||||||
&ws_url,
|
&ws_url,
|
||||||
true,
|
true,
|
||||||
DAEMON_SESSION.get().map(String::as_str).unwrap_or("default"),
|
DAEMON_SESSION
|
||||||
|
.get()
|
||||||
|
.map(String::as_str)
|
||||||
|
.unwrap_or("default"),
|
||||||
);
|
);
|
||||||
let manager = if engine == "lightpanda" {
|
let manager = if engine == "lightpanda" {
|
||||||
initialize_lightpanda_manager(ws_url, process).await?
|
initialize_lightpanda_manager(ws_url, process).await?
|
||||||
@@ -618,7 +684,10 @@ impl BrowserManager {
|
|||||||
crate::connect::log_connect_mode(
|
crate::connect::log_connect_mode(
|
||||||
&ws_url,
|
&ws_url,
|
||||||
false,
|
false,
|
||||||
DAEMON_SESSION.get().map(String::as_str).unwrap_or("default"),
|
DAEMON_SESSION
|
||||||
|
.get()
|
||||||
|
.map(String::as_str)
|
||||||
|
.unwrap_or("default"),
|
||||||
);
|
);
|
||||||
let client = Arc::new(CdpClient::connect_with_headers(&ws_url, headers).await?);
|
let client = Arc::new(CdpClient::connect_with_headers(&ws_url, headers).await?);
|
||||||
let mut manager = Self {
|
let mut manager = Self {
|
||||||
@@ -662,6 +731,43 @@ impl BrowserManager {
|
|||||||
Self::connect_cdp(&ws_url).await
|
Self::connect_cdp(&ws_url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Page targets to adopt, merging several `Target.getTargets` snapshots over
|
||||||
|
/// the extension relay. A single relay snapshot is flaky on a busy real Chrome
|
||||||
|
/// — it can omit live tabs (a different window's set, or a partial list; issue
|
||||||
|
/// #31) — so a tab the daemon should adopt would silently vanish (e.g. after a
|
||||||
|
/// daemon restart the page being driven disappeared from the tab list). Taking
|
||||||
|
/// the union of a few snapshots makes adoption resilient to a transient miss.
|
||||||
|
/// Off the relay (a browser we launched) one snapshot is authoritative.
|
||||||
|
async fn collect_page_targets(&self) -> Result<Vec<TargetInfo>, String> {
|
||||||
|
let rounds = if crate::connect::relay_url().is_some() {
|
||||||
|
3
|
||||||
|
} else {
|
||||||
|
1
|
||||||
|
};
|
||||||
|
let mut by_id: HashMap<String, TargetInfo> = HashMap::new();
|
||||||
|
let mut any_ok = false;
|
||||||
|
for i in 0..rounds {
|
||||||
|
if i > 0 {
|
||||||
|
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||||
|
}
|
||||||
|
match self
|
||||||
|
.client
|
||||||
|
.send_command_typed::<_, GetTargetsResult>("Target.getTargets", &json!({}), None)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(result) => {
|
||||||
|
any_ok = true;
|
||||||
|
for t in result.target_infos.into_iter().filter(should_track_target) {
|
||||||
|
by_id.entry(t.target_id.clone()).or_insert(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) if i == rounds - 1 && !any_ok => return Err(e),
|
||||||
|
Err(_) => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(by_id.into_values().collect())
|
||||||
|
}
|
||||||
|
|
||||||
async fn discover_and_attach_targets(&mut self) -> Result<(), String> {
|
async fn discover_and_attach_targets(&mut self) -> Result<(), String> {
|
||||||
self.client
|
self.client
|
||||||
.send_command_typed::<_, Value>(
|
.send_command_typed::<_, Value>(
|
||||||
@@ -671,16 +777,7 @@ impl BrowserManager {
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let result: GetTargetsResult = self
|
let page_targets: Vec<TargetInfo> = self.collect_page_targets().await?;
|
||||||
.client
|
|
||||||
.send_command_typed("Target.getTargets", &json!({}), None)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let page_targets: Vec<TargetInfo> = result
|
|
||||||
.target_infos
|
|
||||||
.into_iter()
|
|
||||||
.filter(should_track_target)
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
if page_targets.is_empty() {
|
if page_targets.is_empty() {
|
||||||
// Create a new tab
|
// Create a new tab
|
||||||
@@ -748,15 +845,28 @@ impl BrowserManager {
|
|||||||
target_id: target.target_id.clone(),
|
target_id: target.target_id.clone(),
|
||||||
session_id: attach_result.session_id.clone(),
|
session_id: attach_result.session_id.clone(),
|
||||||
url: target.url.clone(),
|
url: target.url.clone(),
|
||||||
title: target.title.clone(),
|
title: sanitize_title(&target.title),
|
||||||
target_type: target.target_type.clone(),
|
target_type: target.target_type.clone(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
self.active_page_index = 0;
|
if self.agent_group().is_some() {
|
||||||
self.pin_active_target();
|
// Relay: the adopted tabs above are the USER's, in their real
|
||||||
let session_id = self.pages[0].session_id.clone();
|
// Chrome. NEVER make one of them the agent's working tab — that is
|
||||||
self.enable_domains(&session_id).await?;
|
// how commands drifted onto whatever page the user was viewing
|
||||||
|
// between steps (eval/click/get landed on the user's foreground
|
||||||
|
// tab; #35). Open our own dedicated background tab in the session's
|
||||||
|
// group and pin THAT as active. The user's tabs stay adopted (so
|
||||||
|
// `tab list` / explicit `tab switch` can reach them) but are never
|
||||||
|
// auto-selected — the agent only ever drives a tab it owns.
|
||||||
|
self.tab_new(None, None).await?;
|
||||||
|
} else {
|
||||||
|
// A browser we launched: every tab is ours, so the first is fine.
|
||||||
|
self.active_page_index = 0;
|
||||||
|
self.pin_active_target();
|
||||||
|
let session_id = self.pages[0].session_id.clone();
|
||||||
|
self.enable_domains(&session_id).await?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -867,6 +977,24 @@ impl BrowserManager {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Drop the page bound to `session_id` from the tracked list — used when the
|
||||||
|
/// relay reports its tab is gone (issue #35) so the stale entry can't keep
|
||||||
|
/// resolving as active. Forgets ownership, unpins it if it was pinned, and
|
||||||
|
/// keeps `active_page_index` in range.
|
||||||
|
fn drop_page_by_session(&mut self, session_id: &str) {
|
||||||
|
let Some(pos) = self.pages.iter().position(|p| p.session_id == session_id) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let target_id = self.pages[pos].target_id.clone();
|
||||||
|
self.pages.remove(pos);
|
||||||
|
self.created_targets.remove(&target_id);
|
||||||
|
if self.active_target_id.as_deref() == Some(target_id.as_str()) {
|
||||||
|
self.active_target_id = None;
|
||||||
|
}
|
||||||
|
self.active_page_index =
|
||||||
|
active_page_index_after_removal(self.active_page_index, pos, self.pages.len());
|
||||||
|
}
|
||||||
|
|
||||||
/// Pin the current active page by target_id so later commands stick to it.
|
/// Pin the current active page by target_id so later commands stick to it.
|
||||||
/// Call after any explicit open / tab new / tab switch.
|
/// Call after any explicit open / tab new / tab switch.
|
||||||
fn pin_active_target(&mut self) {
|
fn pin_active_target(&mut self) {
|
||||||
@@ -896,10 +1024,10 @@ impl BrowserManager {
|
|||||||
if self.agent_group().is_some() && !self.active_is_session_owned() {
|
if self.agent_group().is_some() && !self.active_is_session_owned() {
|
||||||
self.tab_new(None, None).await?;
|
self.tab_new(None, None).await?;
|
||||||
}
|
}
|
||||||
let session_id = self.active_session_id()?.to_string();
|
let mut session_id = self.active_session_id()?.to_string();
|
||||||
let mut lifecycle_rx = self.client.subscribe();
|
let mut lifecycle_rx = self.client.subscribe();
|
||||||
|
|
||||||
let nav_result: PageNavigateResult = self
|
let nav_result: PageNavigateResult = match self
|
||||||
.client
|
.client
|
||||||
.send_command_typed(
|
.send_command_typed(
|
||||||
"Page.navigate",
|
"Page.navigate",
|
||||||
@@ -909,7 +1037,38 @@ impl BrowserManager {
|
|||||||
},
|
},
|
||||||
Some(&session_id),
|
Some(&session_id),
|
||||||
)
|
)
|
||||||
.await?;
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) => r,
|
||||||
|
// Auto-reattach when the bound tab is gone (issue #35). On the shared
|
||||||
|
// real browser the human can close/swap the agent's tab, and a
|
||||||
|
// cross-process nav can destroy the target without a re-attachable
|
||||||
|
// tabId — both leave the cached `cb-tab-<id>` session stale, so every
|
||||||
|
// command (including `open`) failed on it and only `tab new`
|
||||||
|
// recovered. The relay error literally says "re-open your target URL
|
||||||
|
// to re-attach"; fulfil that here: drop the dead page, open a fresh
|
||||||
|
// owned tab in this session's group, and navigate THAT. Gated on the
|
||||||
|
// relay (`agent_group`) and on the explicit navigation intent — read
|
||||||
|
// commands deliberately still fail loudly rather than silently
|
||||||
|
// recover onto a blank tab and return wrong data (issue #8.1).
|
||||||
|
Err(e) if self.agent_group().is_some() && is_stale_target_error(&e) => {
|
||||||
|
self.drop_page_by_session(&session_id);
|
||||||
|
self.tab_new(None, None).await?;
|
||||||
|
session_id = self.active_session_id()?.to_string();
|
||||||
|
lifecycle_rx = self.client.subscribe();
|
||||||
|
self.client
|
||||||
|
.send_command_typed(
|
||||||
|
"Page.navigate",
|
||||||
|
&PageNavigateParams {
|
||||||
|
url: url.to_string(),
|
||||||
|
referrer: None,
|
||||||
|
},
|
||||||
|
Some(&session_id),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
}
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
|
||||||
if let Some(ref error_text) = nav_result.error_text {
|
if let Some(ref error_text) = nav_result.error_text {
|
||||||
return Err(format!("Navigation failed: {}", error_text));
|
return Err(format!("Navigation failed: {}", error_text));
|
||||||
@@ -974,7 +1133,7 @@ impl BrowserManager {
|
|||||||
self.active_page_index = self.resolved_active_index();
|
self.active_page_index = self.resolved_active_index();
|
||||||
if let Some(page) = self.pages.get_mut(self.active_page_index) {
|
if let Some(page) = self.pages.get_mut(self.active_page_index) {
|
||||||
page.url = page_url.clone();
|
page.url = page_url.clone();
|
||||||
page.title = title.clone();
|
page.title = sanitize_title(&title);
|
||||||
}
|
}
|
||||||
self.pin_active_target();
|
self.pin_active_target();
|
||||||
|
|
||||||
@@ -1036,7 +1195,7 @@ impl BrowserManager {
|
|||||||
|
|
||||||
pub async fn get_title(&self) -> Result<String, String> {
|
pub async fn get_title(&self) -> Result<String, String> {
|
||||||
let result = self.evaluate_simple("document.title").await?;
|
let result = self.evaluate_simple("document.title").await?;
|
||||||
Ok(result.as_str().unwrap_or("").to_string())
|
Ok(sanitize_title(result.as_str().unwrap_or("")))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_content(&self) -> Result<String, String> {
|
pub async fn get_content(&self) -> Result<String, String> {
|
||||||
@@ -1049,15 +1208,31 @@ impl BrowserManager {
|
|||||||
pub async fn evaluate(&self, script: &str, _args: Option<Value>) -> Result<Value, String> {
|
pub async fn evaluate(&self, script: &str, _args: Option<Value>) -> Result<Value, String> {
|
||||||
let session_id = self.active_session_id()?.to_string();
|
let session_id = self.active_session_id()?.to_string();
|
||||||
|
|
||||||
|
// `replMode: true` lets successive `eval`s re-declare top-level
|
||||||
|
// `let`/`const` instead of throwing "Identifier 'x' has already been
|
||||||
|
// declared" (issue #38 — independent `eval` steps in a test suite collided
|
||||||
|
// in the page's shared lexical scope). BUT replMode and `awaitPromise` are
|
||||||
|
// mutually exclusive in Chrome: under replMode a returned promise is NOT
|
||||||
|
// awaited (it serialises to `{}`), which breaks `fetch(...).then(...)` and
|
||||||
|
// every other async eval. So enable replMode ONLY for synchronous scripts
|
||||||
|
// that declare a top-level `let`/`const`; promise-returning scripts keep
|
||||||
|
// `awaitPromise` (no replMode) — exactly the pre-#38 behaviour.
|
||||||
|
let mentions_async = script.contains("await")
|
||||||
|
|| script.contains(".then(")
|
||||||
|
|| script.contains("fetch(")
|
||||||
|
|| script.contains("Promise");
|
||||||
|
let declares = script.contains("let ") || script.contains("const ");
|
||||||
|
let repl_mode = declares && !mentions_async;
|
||||||
let result: EvaluateResult = self
|
let result: EvaluateResult = self
|
||||||
.client
|
.client
|
||||||
.send_command_typed(
|
.send_command_typed(
|
||||||
"Runtime.evaluate",
|
"Runtime.evaluate",
|
||||||
&EvaluateParams {
|
&json!({
|
||||||
expression: script.to_string(),
|
"expression": script,
|
||||||
return_by_value: Some(true),
|
"returnByValue": true,
|
||||||
await_promise: Some(true),
|
"awaitPromise": !repl_mode,
|
||||||
},
|
"replMode": repl_mode,
|
||||||
|
}),
|
||||||
Some(&session_id),
|
Some(&session_id),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
@@ -1379,7 +1554,7 @@ impl BrowserManager {
|
|||||||
target_id: target.target_id.clone(),
|
target_id: target.target_id.clone(),
|
||||||
session_id: attach.session_id.clone(),
|
session_id: attach.session_id.clone(),
|
||||||
url: target.url.clone(),
|
url: target.url.clone(),
|
||||||
title: target.title.clone(),
|
title: sanitize_title(&target.title),
|
||||||
target_type: target.target_type.clone(),
|
target_type: target.target_type.clone(),
|
||||||
};
|
};
|
||||||
self.add_background_page(page.clone());
|
self.add_background_page(page.clone());
|
||||||
@@ -1440,7 +1615,7 @@ impl BrowserManager {
|
|||||||
target_id: target.target_id.clone(),
|
target_id: target.target_id.clone(),
|
||||||
session_id: attach_result.session_id.clone(),
|
session_id: attach_result.session_id.clone(),
|
||||||
url: target.url.clone(),
|
url: target.url.clone(),
|
||||||
title: target.title.clone(),
|
title: sanitize_title(&target.title),
|
||||||
target_type: target.target_type.clone(),
|
target_type: target.target_type.clone(),
|
||||||
});
|
});
|
||||||
let _ = self.enable_domains(&attach_result.session_id).await;
|
let _ = self.enable_domains(&attach_result.session_id).await;
|
||||||
@@ -1483,7 +1658,7 @@ impl BrowserManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(t) = ti.get("title").and_then(|v| v.as_str()) {
|
if let Some(t) = ti.get("title").and_then(|v| v.as_str()) {
|
||||||
page.title = t.to_string();
|
page.title = sanitize_title(t);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1671,7 +1846,7 @@ impl BrowserManager {
|
|||||||
|
|
||||||
if let Some(page) = self.pages.get_mut(index) {
|
if let Some(page) = self.pages.get_mut(index) {
|
||||||
page.url = url.clone();
|
page.url = url.clone();
|
||||||
page.title = title.clone();
|
page.title = sanitize_title(&title);
|
||||||
}
|
}
|
||||||
|
|
||||||
let page = &self.pages[index];
|
let page = &self.pages[index];
|
||||||
@@ -1926,7 +2101,8 @@ impl BrowserManager {
|
|||||||
.and_then(|v| v.as_i64())
|
.and_then(|v| v.as_i64())
|
||||||
.ok_or("Could not get backendNodeId for file input")?;
|
.ok_or("Could not get backendNodeId for file input")?;
|
||||||
|
|
||||||
self.client
|
let set_files = self
|
||||||
|
.client
|
||||||
.send_command(
|
.send_command(
|
||||||
"DOM.setFileInputFiles",
|
"DOM.setFileInputFiles",
|
||||||
Some(json!({
|
Some(json!({
|
||||||
@@ -1935,26 +2111,153 @@ impl BrowserManager {
|
|||||||
})),
|
})),
|
||||||
Some(&effective_session_id),
|
Some(&effective_session_id),
|
||||||
)
|
)
|
||||||
.await
|
.await;
|
||||||
.map_err(|e| {
|
|
||||||
// Chrome's chrome.debugger API (the extension-relay transport)
|
|
||||||
// forbids DOM.setFileInputFiles for security, surfacing as an
|
|
||||||
// opaque `-32000 "Not allowed"`. Translate it into an actionable
|
|
||||||
// message rather than leaking the raw CDP error (issue #13).
|
|
||||||
if e.contains("Not allowed") || e.contains("-32000") {
|
|
||||||
"file upload isn't supported over the extension relay — \
|
|
||||||
Chrome's chrome.debugger API forbids DOM.setFileInputFiles. \
|
|
||||||
Use a direct-CDP session instead: \
|
|
||||||
`chrome-use --session up --launch open <url>` (carry your \
|
|
||||||
login over with `cookies export` | `cookies set --curl`), \
|
|
||||||
then run `upload` in that session. \
|
|
||||||
See https://github.com/leeguooooo/chrome-use/issues/13"
|
|
||||||
.to_string()
|
|
||||||
} else {
|
|
||||||
e
|
|
||||||
}
|
|
||||||
})?;
|
|
||||||
|
|
||||||
|
if let Err(e) = set_files {
|
||||||
|
// Chrome's chrome.debugger API (the extension-relay transport) forbids
|
||||||
|
// DOM.setFileInputFiles for security, surfacing as an opaque
|
||||||
|
// `-32000 "Not allowed"`. Fall back to constructing the File entirely
|
||||||
|
// IN THE PAGE and assigning it to the input — the standard
|
||||||
|
// Playwright/Cypress trick, which needs no privileged CDP and so works
|
||||||
|
// over the relay (issue #13).
|
||||||
|
if e.contains("Not allowed") || e.contains("-32000") {
|
||||||
|
return self
|
||||||
|
.upload_files_via_page(object_id, files, &effective_session_id)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Relay-safe file upload: read each file locally, hand its bytes to the page
|
||||||
|
/// as base64, and rebuild a `File` there — then either assign it to a file
|
||||||
|
/// `<input>` (Chrome allows `input.files = dataTransfer.files`) or, for a
|
||||||
|
/// dropzone/composer, dispatch synthetic `paste`/`drop` events carrying the
|
||||||
|
/// `DataTransfer`. No `DOM.setFileInputFiles`, so chrome.debugger permits it.
|
||||||
|
async fn upload_files_via_page(
|
||||||
|
&self,
|
||||||
|
object_id: String,
|
||||||
|
files: &[String],
|
||||||
|
session_id: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
use base64::Engine;
|
||||||
|
// The relay tunnels every CDP message through Chrome native messaging,
|
||||||
|
// which caps a single message at ~1 MiB. A whole image's base64 blows
|
||||||
|
// past that ("CDP response channel closed"), so we STREAM the bytes into
|
||||||
|
// a page-side buffer in sub-limit chunks, then assemble the File from it.
|
||||||
|
const CHUNK: usize = 96 * 1024; // base64 chars per message; safe under 1 MiB
|
||||||
|
|
||||||
|
// Reset the staging buffer.
|
||||||
|
self.client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(json!({ "expression": "window.__cuUpload = [];", "returnByValue": true })),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("relay upload (reset) failed: {}", e))?;
|
||||||
|
|
||||||
|
for path in files {
|
||||||
|
let bytes = std::fs::read(path).map_err(|e| format!("cannot read {}: {}", path, e))?;
|
||||||
|
let name = std::path::Path::new(path)
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.unwrap_or("upload.bin")
|
||||||
|
.to_string();
|
||||||
|
let mime = mime_for_path(&name);
|
||||||
|
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||||
|
|
||||||
|
// Push the file's metadata with an empty buffer.
|
||||||
|
let init = format!(
|
||||||
|
"window.__cuUpload.push({{ name: {}, type: {}, b64: '' }});",
|
||||||
|
serde_json::to_string(&name).unwrap_or_default(),
|
||||||
|
serde_json::to_string(mime).unwrap_or_default(),
|
||||||
|
);
|
||||||
|
self.client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(json!({ "expression": init, "returnByValue": true })),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("relay upload (init) failed: {}", e))?;
|
||||||
|
|
||||||
|
// Stream the base64 in chunks. base64's alphabet (A–Za–z0–9+/=) needs
|
||||||
|
// no escaping inside a single-quoted JS string, so concatenation is safe.
|
||||||
|
let idx = "window.__cuUpload[window.__cuUpload.length-1].b64";
|
||||||
|
let mut start = 0;
|
||||||
|
while start < b64.len() {
|
||||||
|
let end = (start + CHUNK).min(b64.len());
|
||||||
|
let chunk = &b64[start..end];
|
||||||
|
let expr = format!("{idx} += '{chunk}';");
|
||||||
|
self.client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(json!({ "expression": expr, "returnByValue": true })),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("relay upload (chunk) failed: {}", e))?;
|
||||||
|
start = end;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assemble the Files from the buffer and attach to the element, then clean up.
|
||||||
|
let func = r#"function() {
|
||||||
|
const filesData = window.__cuUpload || [];
|
||||||
|
const dt = new DataTransfer();
|
||||||
|
for (const f of filesData) {
|
||||||
|
const bin = atob(f.b64);
|
||||||
|
const arr = new Uint8Array(bin.length);
|
||||||
|
for (let i = 0; i < bin.length; i++) arr[i] = bin.charCodeAt(i);
|
||||||
|
dt.items.add(new File([arr], f.name, { type: f.type }));
|
||||||
|
}
|
||||||
|
try { delete window.__cuUpload; } catch (e) { window.__cuUpload = undefined; }
|
||||||
|
const el = this;
|
||||||
|
if (el.tagName === 'INPUT' && el.type === 'file') {
|
||||||
|
el.files = dt.files;
|
||||||
|
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||||
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||||
|
return 'input:' + dt.files.length;
|
||||||
|
}
|
||||||
|
// Dropzone / rich composer: replay paste then drop with the files.
|
||||||
|
try { el.dispatchEvent(new ClipboardEvent('paste', { bubbles: true, clipboardData: dt })); } catch (e) {}
|
||||||
|
try {
|
||||||
|
const ev = new DragEvent('drop', { bubbles: true, cancelable: true });
|
||||||
|
Object.defineProperty(ev, 'dataTransfer', { value: dt });
|
||||||
|
el.dispatchEvent(ev);
|
||||||
|
} catch (e) {}
|
||||||
|
return 'event:' + dt.files.length;
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
let result: EvaluateResult = self
|
||||||
|
.client
|
||||||
|
.send_command_typed(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: func.to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("relay file-injection failed: {}", e))?;
|
||||||
|
|
||||||
|
if let Some(ref details) = result.exception_details {
|
||||||
|
return Err(format!(
|
||||||
|
"relay file-injection threw: {}",
|
||||||
|
details
|
||||||
|
.exception
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|ex| ex.description.as_deref())
|
||||||
|
.unwrap_or(&details.text)
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2517,6 +2820,33 @@ mod tests {
|
|||||||
assert_eq!(active_page_index_after_removal(0, 0, 0), 0);
|
assert_eq!(active_page_index_after_removal(0, 0, 0), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stale_target_error_matches_relay_signatures() {
|
||||||
|
// The exact relay error `open` must recover from (issue #35), as wrapped
|
||||||
|
// by send_command's `CDP error (Page.navigate): …` prefix.
|
||||||
|
assert!(is_stale_target_error(
|
||||||
|
"CDP error (Page.navigate): stale sessionId cb-tab-1655244623 for Page.navigate: \
|
||||||
|
its tab is gone (closed, navigated across processes, or lost after an extension \
|
||||||
|
restart). Re-attach by re-opening your target URL before retrying."
|
||||||
|
));
|
||||||
|
assert!(is_stale_target_error(
|
||||||
|
"unknown sessionId cb-tab-7 for Page.navigate"
|
||||||
|
));
|
||||||
|
assert!(is_stale_target_error("no attached tab for Page.navigate"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stale_target_error_ignores_unrelated_failures() {
|
||||||
|
// A genuine navigation failure (bad URL, DNS, blocked) must NOT trigger
|
||||||
|
// the open-a-fresh-tab recovery — that would mask the real error.
|
||||||
|
assert!(!is_stale_target_error(
|
||||||
|
"Navigation failed: net::ERR_NAME_NOT_RESOLVED"
|
||||||
|
));
|
||||||
|
assert!(!is_stale_target_error(
|
||||||
|
"CDP command timed out: Page.navigate"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
fn page(target_id: &str) -> PageInfo {
|
fn page(target_id: &str) -> PageInfo {
|
||||||
PageInfo {
|
PageInfo {
|
||||||
tab_id: 1,
|
tab_id: 1,
|
||||||
@@ -2614,6 +2944,32 @@ mod tests {
|
|||||||
assert!(!active_index_is_owned(&[], None, 0, &created));
|
assert!(!active_index_is_owned(&[], None, 0, &created));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sanitize_title() {
|
||||||
|
// The exact pollution from #33: ZWJ / word-joiner / invisible-times / BOM
|
||||||
|
// prepended to "GitHub".
|
||||||
|
let dirty = "\u{200d}\u{2061}\u{200d}\u{2063}\u{200b}\u{2062}\u{feff}GitHub";
|
||||||
|
assert_eq!(sanitize_title(dirty), "GitHub");
|
||||||
|
// Clean titles (incl. CJK + normal punctuation) pass through untouched.
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_title("購入手続きへ - メルカリ"),
|
||||||
|
"購入手続きへ - メルカリ"
|
||||||
|
);
|
||||||
|
assert_eq!(sanitize_title(" Hello World "), "Hello World");
|
||||||
|
// Emoji and real content survive; only the invisibles are dropped.
|
||||||
|
assert_eq!(sanitize_title("✓ Done\u{200b}"), "✓ Done");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_mime_for_path() {
|
||||||
|
assert_eq!(mime_for_path("a.png"), "image/png");
|
||||||
|
assert_eq!(mime_for_path("PHOTO.JPG"), "image/jpeg");
|
||||||
|
assert_eq!(mime_for_path("clip.webp"), "image/webp");
|
||||||
|
assert_eq!(mime_for_path("doc.pdf"), "application/pdf");
|
||||||
|
assert_eq!(mime_for_path("noext"), "application/octet-stream");
|
||||||
|
assert_eq!(mime_for_path("weird.xyz"), "application/octet-stream");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn prune_protects_pinned_target_on_transient_snapshot() {
|
fn prune_protects_pinned_target_on_transient_snapshot() {
|
||||||
// The relay returned a getTargets snapshot missing the pinned tab "A"
|
// The relay returned a getTargets snapshot missing the pinned tab "A"
|
||||||
@@ -2630,7 +2986,10 @@ mod tests {
|
|||||||
// A pinned target that IS in the live set is simply not prunable anyway.
|
// A pinned target that IS in the live set is simply not prunable anyway.
|
||||||
let mut live2 = HashSet::new();
|
let mut live2 = HashSet::new();
|
||||||
live2.insert("A".to_string());
|
live2.insert("A".to_string());
|
||||||
assert_eq!(prunable_target_ids(&pages, &live2, Some("A")), vec!["B".to_string()]);
|
assert_eq!(
|
||||||
|
prunable_target_ids(&pages, &live2, Some("A")),
|
||||||
|
vec!["B".to_string()]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ fn native_test_fixture_html(name: &str) -> &'static str {
|
|||||||
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
|
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
|
||||||
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
|
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
|
||||||
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
|
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
|
||||||
|
"iframe_button_probe" => include_str!("test_fixtures/iframe_button_probe.html"),
|
||||||
_ => panic!("Unknown native test fixture: {}", name),
|
_ => panic!("Unknown native test fixture: {}", name),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -573,6 +574,76 @@ async fn e2e_snapshot_and_click_ref() {
|
|||||||
assert_success(&resp);
|
assert_success(&resp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clicking a button INSIDE an iframe by `@ref` must deliver a TRUSTED activation
|
||||||
|
/// (`event.isTrusted === true`), not a synthetic DOM `.click()`. Security-sensitive
|
||||||
|
/// embedded forms (Google Payments' `保存`) reject `isTrusted:false` clicks, so an
|
||||||
|
/// enabled submit button silently no-op'd (issue #39). The fix routes iframe-ref
|
||||||
|
/// clicks to a real `Input.dispatchMouseEvent` on the element's own frame session.
|
||||||
|
/// The fixture's iframe button writes `clicked:<isTrusted>` into its own text on
|
||||||
|
/// click, which the cross-frame snapshot reads back.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore]
|
||||||
|
async fn e2e_iframe_button_click_is_trusted() {
|
||||||
|
let mut state = DaemonState::new();
|
||||||
|
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "1", "action": "launch", "headless": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "2", "action": "navigate", "url": native_test_fixture_url("iframe_button_probe") }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
// Snapshot (interactive) — the button lives in the iframe and must appear with
|
||||||
|
// a ref; that ref carries the frame_id so the click resolves into the frame.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "3", "action": "snapshot", "interactive": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
let snapshot = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||||
|
let ref_id = snapshot
|
||||||
|
.lines()
|
||||||
|
.find(|l| l.contains("button \"save\""))
|
||||||
|
.and_then(|l| l.split("ref=").nth(1))
|
||||||
|
.map(|r| r.trim_end_matches(']').trim())
|
||||||
|
.unwrap_or_else(|| panic!("iframe button not found in snapshot:\n{snapshot}"));
|
||||||
|
|
||||||
|
// Click it by ref.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "4", "action": "click", "selector": ref_id }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(300)).await;
|
||||||
|
|
||||||
|
// The button rewrote its own text with the click's isTrusted flag; read it
|
||||||
|
// back across frames.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "5", "action": "snapshot", "interactive": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
let after = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||||
|
assert!(
|
||||||
|
after.contains("clicked:true"),
|
||||||
|
"iframe button click must be trusted (isTrusted:true); snapshot:\n{after}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
|
||||||
|
assert_success(&resp);
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Screenshot
|
// Screenshot
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -100,6 +100,15 @@ impl RefMap {
|
|||||||
self.map.get(ref_id)
|
self.map.get(ref_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether `selector_or_ref` is a `@ref` whose snapshot entry lives inside an
|
||||||
|
/// iframe (has a `frame_id`). Pointer interactions use this to choose
|
||||||
|
/// DOM-dispatch over coordinates for OOPIF elements (issue #36).
|
||||||
|
pub fn ref_is_in_iframe(&self, selector_or_ref: &str) -> bool {
|
||||||
|
parse_ref(selector_or_ref)
|
||||||
|
.and_then(|r| self.map.get(&r).map(|e| e.frame_id.is_some()))
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn entries_sorted(&self) -> Vec<(String, RefEntry)> {
|
pub fn entries_sorted(&self) -> Vec<(String, RefEntry)> {
|
||||||
let mut entries = self
|
let mut entries = self
|
||||||
.map
|
.map
|
||||||
@@ -1019,7 +1028,9 @@ async fn eval_text_in_frame(client: &CdpClient, session_id: &str, frame_id: &str
|
|||||||
.await
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|v| v.get("executionContextId").and_then(|c| c.as_i64()));
|
.and_then(|v| v.get("executionContextId").and_then(|c| c.as_i64()));
|
||||||
let Some(ctx_id) = ctx else { return String::new() };
|
let Some(ctx_id) = ctx else {
|
||||||
|
return String::new();
|
||||||
|
};
|
||||||
let res = client
|
let res = client
|
||||||
.send_command(
|
.send_command(
|
||||||
"Runtime.evaluate",
|
"Runtime.evaluate",
|
||||||
@@ -1090,7 +1101,10 @@ pub async fn collect_all_frames_text(
|
|||||||
let (kind, text) = if is_top {
|
let (kind, text) = if is_top {
|
||||||
("top", eval_text_default(client, top_session).await)
|
("top", eval_text_default(client, top_session).await)
|
||||||
} else {
|
} else {
|
||||||
("inline", eval_text_in_frame(client, top_session, &fid).await)
|
(
|
||||||
|
"inline",
|
||||||
|
eval_text_in_frame(client, top_session, &fid).await,
|
||||||
|
)
|
||||||
};
|
};
|
||||||
out.push(FrameText {
|
out.push(FrameText {
|
||||||
frame_id: fid,
|
frame_id: fid,
|
||||||
|
|||||||
@@ -7,6 +7,17 @@ use super::cdp::types::*;
|
|||||||
use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap};
|
use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap};
|
||||||
use super::humanize;
|
use super::humanize;
|
||||||
|
|
||||||
|
/// Whether a pointer interaction should be DOM-dispatched (invoke the event on
|
||||||
|
/// the element in its own session) rather than dispatched at a viewport
|
||||||
|
/// coordinate via `Input.dispatchMouseEvent`. True when the target is inside an
|
||||||
|
/// iframe (an OOPIF element's box can't be mapped to a top-viewport point) or we
|
||||||
|
/// drive over the extension relay (a coordinate Input event isn't confined to the
|
||||||
|
/// target tab on a busy real Chrome — it drifts onto the foreground tab; issues
|
||||||
|
/// #31/#36). DOM-dispatch always hits the right element in the right tab.
|
||||||
|
fn prefer_dom_dispatch(ref_map: &RefMap, selector_or_ref: &str) -> bool {
|
||||||
|
ref_map.ref_is_in_iframe(selector_or_ref) || crate::connect::relay_url().is_some()
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn click(
|
pub async fn click(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -45,6 +56,45 @@ pub async fn click(
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An element INSIDE an iframe needs a TRUSTED activation: a DOM `.click()` is
|
||||||
|
// `isTrusted:false`, which security-sensitive embedded forms reject — Google
|
||||||
|
// Payments' enabled `保存` button silently no-ops on a synthetic click (issue
|
||||||
|
// #39). A coordinate `Input.dispatchMouseEvent` can't help either: `getBoxModel`
|
||||||
|
// for a sub-frame node returns frame-local coordinates that don't compose the
|
||||||
|
// iframe's offset, so the click lands in the wrong place. The frame-agnostic
|
||||||
|
// trusted path is keyboard activation — focus the element in its own frame, then
|
||||||
|
// dispatch a real Enter on the page session; Chrome routes the key to the
|
||||||
|
// focused element regardless of frame (same as `type --focused`), and Enter on a
|
||||||
|
// focused button/link fires a trusted `click`. `coord` mode opts out.
|
||||||
|
let in_iframe = ref_map.ref_is_in_iframe(selector_or_ref);
|
||||||
|
if mode != "coord" && button == "left" && click_count == 1 && in_iframe {
|
||||||
|
return dom_activate(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
// On the relay (the user's real Chrome) a TOP-document coordinate click used to
|
||||||
|
// drift onto the foreground tab; that root cause is fixed (#5: the agent drives
|
||||||
|
// its own pinned tab), but DOM-dispatch stays the conservative default here.
|
||||||
|
if mode != "coord"
|
||||||
|
&& button == "left"
|
||||||
|
&& click_count == 1
|
||||||
|
&& crate::connect::relay_url().is_some()
|
||||||
|
{
|
||||||
|
return dom_click(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
let resolved = resolve_element_center(
|
let resolved = resolve_element_center(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
@@ -235,6 +285,112 @@ async fn dom_click(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Trusted activation of an element inside an iframe (issue #39). Focuses the
|
||||||
|
/// element in its own frame session, then dispatches a real Enter/Space on the
|
||||||
|
/// page session — Chrome routes the key to the focused element across frames, and
|
||||||
|
/// Enter/Space on a focused button/link/checkbox fires a `click` with
|
||||||
|
/// `isTrusted: true`, which security-sensitive embedded forms (Google Payments
|
||||||
|
/// `保存`) require. Non-activatable roles (a `div[onclick]`) can't be keyboard-
|
||||||
|
/// activated, so they fall back to a DOM `.click()`.
|
||||||
|
async fn dom_activate(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
selector_or_ref: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let role = parse_ref(selector_or_ref)
|
||||||
|
.and_then(|r| ref_map.get(&r).map(|e| e.role.clone()))
|
||||||
|
.unwrap_or_default();
|
||||||
|
// Space toggles checkbox-like controls; Enter activates buttons/links/menus.
|
||||||
|
let key = match role.as_str() {
|
||||||
|
"checkbox" | "radio" | "switch" | "option" | "menuitemcheckbox" | "menuitemradio" => {
|
||||||
|
Some("space")
|
||||||
|
}
|
||||||
|
"button" | "link" | "menuitem" | "tab" | "treeitem" => Some("enter"),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let Some(key) = key else {
|
||||||
|
// Not keyboard-activatable — best effort via DOM .click() (untrusted).
|
||||||
|
return dom_click(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
};
|
||||||
|
|
||||||
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// Focus the element in its OWN frame session so the keystroke lands on it.
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: "function() { this.focus(); }".to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&effective_session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// Trusted key on the page session — routed to the focused (in-frame) element.
|
||||||
|
press_key(client, session_id, key).await?;
|
||||||
|
wait_for_paint_settled(client, &effective_session_id).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DOM-dispatch a double-click on the element in its own session (no coordinates)
|
||||||
|
/// — the relay/iframe-safe counterpart to a coordinate dblclick. Fires the full
|
||||||
|
/// click,click,dblclick sequence so handlers bound to any of them respond.
|
||||||
|
async fn dom_dblclick(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
selector_or_ref: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: r#"function() {
|
||||||
|
const opts = { bubbles: true, cancelable: true, view: window };
|
||||||
|
this.dispatchEvent(new MouseEvent('click', opts));
|
||||||
|
this.dispatchEvent(new MouseEvent('click', { ...opts, detail: 2 }));
|
||||||
|
this.dispatchEvent(new MouseEvent('dblclick', opts));
|
||||||
|
}"#
|
||||||
|
.to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&effective_session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
wait_for_paint_settled(client, &effective_session_id).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn dblclick(
|
pub async fn dblclick(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -242,6 +398,20 @@ pub async fn dblclick(
|
|||||||
selector_or_ref: &str,
|
selector_or_ref: &str,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
|
// Same relay/iframe drift hazard as a single click — DOM-dispatch the
|
||||||
|
// double-click there instead of a coordinate one (issues #31/#36).
|
||||||
|
if std::env::var("AGENT_BROWSER_CLICK_MODE").as_deref() != Ok("coord")
|
||||||
|
&& prefer_dom_dispatch(ref_map, selector_or_ref)
|
||||||
|
{
|
||||||
|
return dom_dblclick(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
click(
|
click(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
@@ -254,6 +424,50 @@ pub async fn dblclick(
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// DOM-dispatch a hover (pointer/mouse enter+move) on the element in its own
|
||||||
|
/// session — reaches OOPIF elements and never drifts to the foreground tab over
|
||||||
|
/// the relay, unlike a coordinate `mouseMoved` (issues #31/#36).
|
||||||
|
async fn dom_hover(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
selector_or_ref: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: r#"function() {
|
||||||
|
const r = this.getBoundingClientRect();
|
||||||
|
const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
|
||||||
|
const base = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy };
|
||||||
|
this.dispatchEvent(new PointerEvent('pointerover', base));
|
||||||
|
this.dispatchEvent(new PointerEvent('pointerenter', { ...base, bubbles: false }));
|
||||||
|
this.dispatchEvent(new MouseEvent('mouseover', base));
|
||||||
|
this.dispatchEvent(new MouseEvent('mouseenter', { ...base, bubbles: false }));
|
||||||
|
this.dispatchEvent(new MouseEvent('mousemove', base));
|
||||||
|
}"#
|
||||||
|
.to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&effective_session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn hover(
|
pub async fn hover(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -261,6 +475,18 @@ pub async fn hover(
|
|||||||
selector_or_ref: &str,
|
selector_or_ref: &str,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
|
// Coordinate `mouseMoved` drifts to the foreground tab over the relay and
|
||||||
|
// can't reach an OOPIF — DOM-dispatch the hover there (issues #31/#36).
|
||||||
|
if prefer_dom_dispatch(ref_map, selector_or_ref) {
|
||||||
|
return dom_hover(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
let (x, y, _w, _h, effective_session_id) = resolve_element_center(
|
let (x, y, _w, _h, effective_session_id) = resolve_element_center(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
@@ -289,6 +515,63 @@ pub async fn hover(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// DOM-dispatch an HTML5 drag-and-drop from `source` to `target` in their shared
|
||||||
|
/// session — the relay/iframe-safe counterpart to the coordinate drag, which
|
||||||
|
/// drifts to the foreground tab over the relay and can't reach an OOPIF (issues
|
||||||
|
/// #31/#36). Covers HTML5 DnD (sortable lists, file/card boards); pointer-driven
|
||||||
|
/// drag (canvas, sliders) still needs the coordinate path. Errors if source and
|
||||||
|
/// target live in different frames — a synthetic cross-frame DnD isn't reliable.
|
||||||
|
pub async fn dom_drag(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
source: &str,
|
||||||
|
target: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (src_obj, src_session) =
|
||||||
|
resolve_element_object_id(client, session_id, ref_map, source, iframe_sessions).await?;
|
||||||
|
let (tgt_obj, tgt_session) =
|
||||||
|
resolve_element_object_id(client, session_id, ref_map, target, iframe_sessions).await?;
|
||||||
|
if src_session != tgt_session {
|
||||||
|
return Err(
|
||||||
|
"drag source and target are in different frames; cross-frame drag-and-drop over the \
|
||||||
|
relay isn't supported — drag within a single frame, or use a launched browser with \
|
||||||
|
AGENT_BROWSER_CLICK_MODE=coord"
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: r#"function(target) {
|
||||||
|
const dt = new DataTransfer();
|
||||||
|
const ev = (type, el) => el.dispatchEvent(
|
||||||
|
new DragEvent(type, { bubbles: true, cancelable: true, dataTransfer: dt }));
|
||||||
|
ev('dragstart', this);
|
||||||
|
ev('drag', this);
|
||||||
|
ev('dragenter', target);
|
||||||
|
ev('dragover', target);
|
||||||
|
ev('drop', target);
|
||||||
|
ev('dragend', this);
|
||||||
|
}"#
|
||||||
|
.to_string(),
|
||||||
|
object_id: Some(src_obj),
|
||||||
|
arguments: Some(vec![CallArgument {
|
||||||
|
value: None,
|
||||||
|
object_id: Some(tgt_obj),
|
||||||
|
}]),
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&src_session),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
wait_for_paint_settled(client, &src_session).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn fill(
|
pub async fn fill(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -372,6 +655,7 @@ pub async fn type_text(
|
|||||||
clear: bool,
|
clear: bool,
|
||||||
delay_ms: Option<u64>,
|
delay_ms: Option<u64>,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
key_events: bool,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let (object_id, effective_session_id) = resolve_element_object_id(
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
client,
|
client,
|
||||||
@@ -418,7 +702,7 @@ pub async fn type_text(
|
|||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
type_text_into_active_context(client, session_id, text, delay_ms).await
|
type_text_into_active_context(client, session_id, text, delay_ms, key_events).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn type_text_into_active_context(
|
pub async fn type_text_into_active_context(
|
||||||
@@ -426,6 +710,7 @@ pub async fn type_text_into_active_context(
|
|||||||
session_id: &str,
|
session_id: &str,
|
||||||
text: &str,
|
text: &str,
|
||||||
delay_ms: Option<u64>,
|
delay_ms: Option<u64>,
|
||||||
|
key_events: bool,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
// Per-character timing: an explicit `delay_ms` wins (caller asked for a
|
// Per-character timing: an explicit `delay_ms` wins (caller asked for a
|
||||||
// fixed cadence); otherwise fall back to humanize — variable, human-like
|
// fixed cadence); otherwise fall back to humanize — variable, human-like
|
||||||
@@ -475,6 +760,46 @@ pub async fn type_text_into_active_context(
|
|||||||
Some(session_id),
|
Some(session_id),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
} else if key_events {
|
||||||
|
// Real keystrokes (keyDown+keyUp carrying `text`) for autocomplete /
|
||||||
|
// combobox widgets that only react to key events and ignore the
|
||||||
|
// `input` that `Input.insertText` fires — e.g. Google's address
|
||||||
|
// postal-code → city/prefecture lookup (issue #36 / #4). The keyDown's
|
||||||
|
// `text` still inserts the character, so the field also fills.
|
||||||
|
let (key, code, key_code) = char_to_key_info(ch);
|
||||||
|
let s = ch.to_string();
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
&DispatchKeyEventParams {
|
||||||
|
event_type: "keyDown".to_string(),
|
||||||
|
key: Some(key.clone()),
|
||||||
|
code: Some(code.clone()),
|
||||||
|
text: Some(s.clone()),
|
||||||
|
unmodified_text: Some(s),
|
||||||
|
windows_virtual_key_code: Some(key_code),
|
||||||
|
native_virtual_key_code: Some(key_code),
|
||||||
|
modifiers: None,
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
&DispatchKeyEventParams {
|
||||||
|
event_type: "keyUp".to_string(),
|
||||||
|
key: Some(key),
|
||||||
|
code: Some(code),
|
||||||
|
text: None,
|
||||||
|
unmodified_text: None,
|
||||||
|
windows_virtual_key_code: Some(key_code),
|
||||||
|
native_virtual_key_code: Some(key_code),
|
||||||
|
modifiers: None,
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
// VS Code/Electron webviews reject repeated dispatchKeyEvent calls
|
// VS Code/Electron webviews reject repeated dispatchKeyEvent calls
|
||||||
// carrying printable `text`. Insert printable characters directly
|
// carrying printable `text`. Insert printable characters directly
|
||||||
|
|||||||
@@ -60,6 +60,9 @@ pub struct ScreenshotOptions {
|
|||||||
pub quality: Option<i32>,
|
pub quality: Option<i32>,
|
||||||
pub annotate: bool,
|
pub annotate: bool,
|
||||||
pub output_dir: Option<String>,
|
pub output_dir: Option<String>,
|
||||||
|
/// Explicit pixel region (x, y, width, height) — `--clip` (issue #34). Takes
|
||||||
|
/// precedence over selector/full_page.
|
||||||
|
pub clip: Option<(f64, f64, f64, f64)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ScreenshotOptions {
|
impl Default for ScreenshotOptions {
|
||||||
@@ -72,6 +75,7 @@ impl Default for ScreenshotOptions {
|
|||||||
quality: None,
|
quality: None,
|
||||||
annotate: false,
|
annotate: false,
|
||||||
output_dir: None,
|
output_dir: None,
|
||||||
|
clip: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -187,7 +191,16 @@ async fn capture_screenshot_base64(
|
|||||||
capture_beyond_viewport: if options.full_page { Some(true) } else { None },
|
capture_beyond_viewport: if options.full_page { Some(true) } else { None },
|
||||||
};
|
};
|
||||||
|
|
||||||
if options.full_page {
|
if let Some((x, y, width, height)) = options.clip {
|
||||||
|
// Explicit pixel region wins over selector/full_page (issue #34).
|
||||||
|
params.clip = Some(Viewport {
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
scale: 1.0,
|
||||||
|
});
|
||||||
|
} else if options.full_page {
|
||||||
let metrics: Value = client
|
let metrics: Value = client
|
||||||
.send_command_no_params("Page.getLayoutMetrics", Some(session_id))
|
.send_command_no_params("Page.getLayoutMetrics", Some(session_id))
|
||||||
.await?;
|
.await?;
|
||||||
|
|||||||
@@ -330,6 +330,13 @@ impl RoleNameTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Max iframe nesting depth `take_snapshot` expands. Embedded payment/checkout
|
||||||
|
/// widgets nest a few frames deep (e.g. AdSense → payments.google.com → an inner
|
||||||
|
/// form frame); expanding past the first level is what gives those inner refs a
|
||||||
|
/// `frame_id` so clicks resolve into the right frame (issue #36). Capped to keep
|
||||||
|
/// a pathological frame tree from blowing up the snapshot.
|
||||||
|
const MAX_IFRAME_DEPTH: usize = 3;
|
||||||
|
|
||||||
pub async fn take_snapshot(
|
pub async fn take_snapshot(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -337,6 +344,28 @@ pub async fn take_snapshot(
|
|||||||
ref_map: &mut RefMap,
|
ref_map: &mut RefMap,
|
||||||
frame_id: Option<&str>,
|
frame_id: Option<&str>,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
take_snapshot_at_depth(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
options,
|
||||||
|
ref_map,
|
||||||
|
frame_id,
|
||||||
|
iframe_sessions,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn take_snapshot_at_depth(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
options: &SnapshotOptions,
|
||||||
|
ref_map: &mut RefMap,
|
||||||
|
frame_id: Option<&str>,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
depth: usize,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
client
|
client
|
||||||
.send_command_no_params("DOM.enable", Some(session_id))
|
.send_command_no_params("DOM.enable", Some(session_id))
|
||||||
@@ -606,10 +635,11 @@ pub async fn take_snapshot(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Recurse into child iframes: for each Iframe node with a backend_node_id,
|
// Recurse into child iframes: for each Iframe node with a backend_node_id,
|
||||||
// resolve the child frame ID and take a snapshot of its content.
|
// resolve the child frame ID and snapshot its content. Recurse to
|
||||||
// We only recurse from the main frame (frame_id == None) to avoid
|
// MAX_IFRAME_DEPTH (not just the main frame) so refs inside nested
|
||||||
// unbounded depth; nested iframes within iframes are not expanded.
|
// payment/checkout widgets get a `frame_id` and clicks resolve into the right
|
||||||
if frame_id.is_none() {
|
// frame (issue #36); the cap bounds a pathological frame tree.
|
||||||
|
if depth < MAX_IFRAME_DEPTH {
|
||||||
let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot)
|
let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot)
|
||||||
for node in tree_nodes.iter() {
|
for node in tree_nodes.iter() {
|
||||||
if node.role != "Iframe" || !node.has_ref {
|
if node.role != "Iframe" || !node.has_ref {
|
||||||
@@ -622,13 +652,14 @@ pub async fn take_snapshot(
|
|||||||
if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await {
|
if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await {
|
||||||
// Snapshot the child frame; errors are silently ignored
|
// Snapshot the child frame; errors are silently ignored
|
||||||
// (e.g. cross-origin iframes)
|
// (e.g. cross-origin iframes)
|
||||||
if let Ok(child_text) = Box::pin(take_snapshot(
|
if let Ok(child_text) = Box::pin(take_snapshot_at_depth(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
options,
|
options,
|
||||||
ref_map,
|
ref_map,
|
||||||
Some(&child_fid),
|
Some(&child_fid),
|
||||||
iframe_sessions,
|
iframe_sessions,
|
||||||
|
depth + 1,
|
||||||
))
|
))
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>iframe button probe</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>iframe button probe</h1>
|
||||||
|
<iframe
|
||||||
|
id="frame"
|
||||||
|
width="320"
|
||||||
|
height="140"
|
||||||
|
srcdoc="
|
||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<body style='margin:24px'>
|
||||||
|
<button id='b' style='padding:24px;font-size:22px'>save</button>
|
||||||
|
<script>
|
||||||
|
document.getElementById('b').addEventListener('click', function (e) {
|
||||||
|
this.textContent = 'clicked:' + e.isTrusted;
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"
|
||||||
|
></iframe>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
+62
-9
@@ -228,13 +228,28 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
// because its response carries `url`/`title`, which later generic
|
// because its response carries `url`/`title`, which later generic
|
||||||
// renderers would otherwise swallow.
|
// renderers would otherwise swallow.
|
||||||
if action == Some("cf_status") {
|
if action == Some("cf_status") {
|
||||||
let challenged = data.get("challenged").and_then(|v| v.as_bool()).unwrap_or(false);
|
let challenged = data
|
||||||
let rec = data.get("recommendation").and_then(|v| v.as_str()).unwrap_or("?");
|
.get("challenged")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let rec = data
|
||||||
|
.get("recommendation")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("?");
|
||||||
let cl = data.get("clearance");
|
let cl = data.get("clearance");
|
||||||
let present = cl.and_then(|c| c.get("present")).and_then(|v| v.as_bool()).unwrap_or(false);
|
let present = cl
|
||||||
let expired = cl.and_then(|c| c.get("expired")).and_then(|v| v.as_bool()).unwrap_or(false);
|
.and_then(|c| c.get("present"))
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let expired = cl
|
||||||
|
.and_then(|c| c.get("expired"))
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
let expires_in = cl.and_then(|c| c.get("expiresIn")).and_then(|v| v.as_i64());
|
let expires_in = cl.and_then(|c| c.get("expiresIn")).and_then(|v| v.as_i64());
|
||||||
let device = data.get("deviceVerified").and_then(|v| v.as_bool()).unwrap_or(false);
|
let device = data
|
||||||
|
.get("deviceVerified")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
let (icon, headline) = match rec {
|
let (icon, headline) = match rec {
|
||||||
"proceed" => (color::success_indicator().to_string(), "cleared — no challenge, proceed"),
|
"proceed" => (color::success_indicator().to_string(), "cleared — no challenge, proceed"),
|
||||||
@@ -243,7 +258,10 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
_ => (color::cyan("•").to_string(), "unknown"),
|
_ => (color::cyan("•").to_string(), "unknown"),
|
||||||
};
|
};
|
||||||
println!("{} {}", icon, headline);
|
println!("{} {}", icon, headline);
|
||||||
println!(" challenged: {}", if challenged { "yes" } else { "no" });
|
println!(
|
||||||
|
" challenged: {}",
|
||||||
|
if challenged { "yes" } else { "no" }
|
||||||
|
);
|
||||||
let cl_desc = if !present {
|
let cl_desc = if !present {
|
||||||
"absent".to_string()
|
"absent".to_string()
|
||||||
} else if expired {
|
} else if expired {
|
||||||
@@ -254,7 +272,14 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
"present (session)".to_string()
|
"present (session)".to_string()
|
||||||
};
|
};
|
||||||
println!(" cf_clearance: {}", cl_desc);
|
println!(" cf_clearance: {}", cl_desc);
|
||||||
println!(" device trusted: {}", if device { "yes (CF_VERIFIED_DEVICE)" } else { "no" });
|
println!(
|
||||||
|
" device trusted: {}",
|
||||||
|
if device {
|
||||||
|
"yes (CF_VERIFIED_DEVICE)"
|
||||||
|
} else {
|
||||||
|
"no"
|
||||||
|
}
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,7 +407,11 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
let count = list.len();
|
let count = list.len();
|
||||||
println!(
|
println!(
|
||||||
"{}",
|
"{}",
|
||||||
color::bold(&format!("{} frame{}", count, if count == 1 { "" } else { "s" }))
|
color::bold(&format!(
|
||||||
|
"{} frame{}",
|
||||||
|
count,
|
||||||
|
if count == 1 { "" } else { "s" }
|
||||||
|
))
|
||||||
);
|
);
|
||||||
for f in list {
|
for f in list {
|
||||||
let idx = f.get("index").and_then(|v| v.as_i64()).unwrap_or(0);
|
let idx = f.get("index").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||||
@@ -1464,6 +1493,12 @@ Usage: chrome-use type <selector> <text>
|
|||||||
Types text into the specified element character by character.
|
Types text into the specified element character by character.
|
||||||
Unlike fill, this does not clear existing content first.
|
Unlike fill, this does not clear existing content first.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--key-events Send real per-character keyDown/keyUp instead of
|
||||||
|
(alias --keys) Input.insertText. Use for autocomplete / combobox fields
|
||||||
|
that only react to key events — e.g. a postal-code box
|
||||||
|
that auto-fills city/prefecture, or Google Places.
|
||||||
|
|
||||||
Global Options:
|
Global Options:
|
||||||
--json Output as JSON
|
--json Output as JSON
|
||||||
--session <name> Use specific session
|
--session <name> Use specific session
|
||||||
@@ -1471,6 +1506,7 @@ Global Options:
|
|||||||
Examples:
|
Examples:
|
||||||
chrome-use type "#search" "hello"
|
chrome-use type "#search" "hello"
|
||||||
chrome-use type @e2 "additional text"
|
chrome-use type @e2 "additional text"
|
||||||
|
chrome-use type @e5 "201-0001" --key-events # trigger the address autocomplete
|
||||||
|
|
||||||
See Also:
|
See Also:
|
||||||
For typing into contenteditable editors (Lexical, ProseMirror, etc.)
|
For typing into contenteditable editors (Lexical, ProseMirror, etc.)
|
||||||
@@ -1735,12 +1771,23 @@ Usage: chrome-use scroll [direction] [amount] [options]
|
|||||||
|
|
||||||
Scrolls the page or a specific element in the specified direction.
|
Scrolls the page or a specific element in the specified direction.
|
||||||
|
|
||||||
|
Without --selector, scroll dispatches a real (isTrusted) mouse wheel at a
|
||||||
|
viewport coordinate, so it scrolls whatever container is under the pointer —
|
||||||
|
including cross-origin iframes (Google Payments, Stripe, embedded checkout/KYC)
|
||||||
|
that plain page scroll can't reach.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
direction up, down, left, right (default: down)
|
direction up, down, left, right (default: down)
|
||||||
amount Pixels to scroll (default: 300)
|
amount Pixels to scroll (default: 300)
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
-s, --selector <sel> CSS selector for a scrollable container
|
-s, --selector <sel> CSS selector for a scrollable container (same-origin)
|
||||||
|
--at <x,y> Dispatch the wheel at this viewport pixel (read it from a
|
||||||
|
screenshot) — precise way into a cross-origin iframe
|
||||||
|
--frame <n> Scroll the n-th frame from `chrome-use frames` (wheel at
|
||||||
|
that frame's center)
|
||||||
|
|
||||||
|
Without --selector/--at/--frame the wheel lands at the viewport center.
|
||||||
|
|
||||||
Global Options:
|
Global Options:
|
||||||
--json Output as JSON
|
--json Output as JSON
|
||||||
@@ -1752,6 +1799,8 @@ Examples:
|
|||||||
chrome-use scroll up 200
|
chrome-use scroll up 200
|
||||||
chrome-use scroll left 100
|
chrome-use scroll left 100
|
||||||
chrome-use scroll down 500 --selector "div.scroll-container"
|
chrome-use scroll down 500 --selector "div.scroll-container"
|
||||||
|
chrome-use scroll down 700 --at 640,400 # wheel at a pixel over an iframe
|
||||||
|
chrome-use scroll down 700 --frame 2 # scroll frame 2 from `frames`
|
||||||
"##
|
"##
|
||||||
}
|
}
|
||||||
"scrollintoview" | "scrollinto" => {
|
"scrollintoview" | "scrollinto" => {
|
||||||
@@ -1832,6 +1881,8 @@ Pass --hide-scrollbars false when launching to keep native scrollbars visible.
|
|||||||
|
|
||||||
Options:
|
Options:
|
||||||
--full, -f Capture full page (not just viewport)
|
--full, -f Capture full page (not just viewport)
|
||||||
|
[selector] Capture just an element (CSS or @ref), e.g. `screenshot ".header" h.png`
|
||||||
|
--clip <x,y,w,h> Capture a pixel region, e.g. `screenshot --clip 0,0,200,40 corner.png`
|
||||||
--annotate Overlay numbered labels on interactive elements.
|
--annotate Overlay numbered labels on interactive elements.
|
||||||
Each label [N] corresponds to ref @eN from snapshot.
|
Each label [N] corresponds to ref @eN from snapshot.
|
||||||
Prints a legend mapping labels to element roles/names.
|
Prints a legend mapping labels to element roles/names.
|
||||||
@@ -1852,6 +1903,8 @@ Examples:
|
|||||||
chrome-use screenshot
|
chrome-use screenshot
|
||||||
chrome-use screenshot ./screenshot.png
|
chrome-use screenshot ./screenshot.png
|
||||||
chrome-use screenshot --full ./full-page.png
|
chrome-use screenshot --full ./full-page.png
|
||||||
|
chrome-use screenshot ".header .indicator" corner.png # just one element
|
||||||
|
chrome-use screenshot --clip 1600,0,200,40 corner.png # a pixel region
|
||||||
chrome-use screenshot --annotate # Labeled screenshot + legend
|
chrome-use screenshot --annotate # Labeled screenshot + legend
|
||||||
chrome-use screenshot --annotate ./page.png # Save annotated screenshot
|
chrome-use screenshot --annotate ./page.png # Save annotated screenshot
|
||||||
chrome-use screenshot --annotate --json # JSON output with annotations
|
chrome-use screenshot --annotate --json # JSON output with annotations
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "1.5.7",
|
"version": "1.5.15",
|
||||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"packageManager": "pnpm@11.1.3",
|
"packageManager": "pnpm@11.1.3",
|
||||||
|
|||||||
@@ -36,6 +36,29 @@ Refs (`@e1`, `@e2`, ...) are assigned fresh on every snapshot. They become
|
|||||||
submits, dynamic re-renders, dialog opens. Always re-snapshot before your
|
submits, dynamic re-renders, dialog opens. Always re-snapshot before your
|
||||||
next ref interaction.
|
next ref interaction.
|
||||||
|
|
||||||
|
> **Hard rule: snapshot-first, never screenshot-to-locate.** For form fields and
|
||||||
|
> buttons, ALWAYS `snapshot -i` and act on refs/selectors. Do **not** reach for
|
||||||
|
> `screenshot` + coordinate clicks to find or hit an element — `snapshot -i` now
|
||||||
|
> pierces **cross-origin iframes** (embedded Google Payments / Stripe / checkout /
|
||||||
|
> KYC forms) and lists their elements by `@ref`, including input values. Use
|
||||||
|
> coordinates only for canvas/WebGL, or when `snapshot` genuinely returns nothing
|
||||||
|
> for your target. Screenshots are for *visual verification you report*, never the
|
||||||
|
> agent's own input — and a full-page `screenshot` of a real retina browser is
|
||||||
|
> often too large for an image reader anyway. (If you ever feel you *need* a
|
||||||
|
> screenshot to read state or locate something, that's a bug — please file it.)
|
||||||
|
|
||||||
|
> **Snapshot-first, always. Never default to `screenshot` + coordinate clicking
|
||||||
|
> for form fields or buttons.** Run `snapshot -i` and act on `@refs`. Use
|
||||||
|
> coordinates only for canvas/WebGL, or when `snapshot` genuinely returns nothing
|
||||||
|
> for your target. This holds **even inside cross-origin embedded iframes** —
|
||||||
|
> since v1.5.12 `snapshot -i` pierces out-of-process iframes (Google Payments,
|
||||||
|
> Stripe, embedded checkout/KYC) and lists their elements with refs, so
|
||||||
|
> `click @e` / `type @e` / `fill @e` work directly. A screenshot is for a genuine
|
||||||
|
> *visual* check you report to the user — not your own input. (Full-page
|
||||||
|
> screenshots of a real retina Chrome are often too large for the image reader
|
||||||
|
> anyway.) Driving off pixels on the relay also risks a coordinate event drifting
|
||||||
|
> onto the user's foreground tab — refs never do. See issue #37.
|
||||||
|
|
||||||
## Before you automate: pick the cheapest tool
|
## Before you automate: pick the cheapest tool
|
||||||
|
|
||||||
Driving a browser is the heavy option. chrome-use earns its keep when you
|
Driving a browser is the heavy option. chrome-use earns its keep when you
|
||||||
@@ -264,6 +287,10 @@ chrome-use hover @e1 # hover
|
|||||||
chrome-use focus @e1 # focus (useful before keyboard input)
|
chrome-use focus @e1 # focus (useful before keyboard input)
|
||||||
chrome-use fill @e2 "hello" # clear then type
|
chrome-use fill @e2 "hello" # clear then type
|
||||||
chrome-use type @e2 " world" # type without clearing
|
chrome-use type @e2 " world" # type without clearing
|
||||||
|
chrome-use type @e5 "201-0001" --key-events # real keystrokes (not insertText) —
|
||||||
|
# use for autocomplete/combobox fields that
|
||||||
|
# only react to key events (e.g. a postal box
|
||||||
|
# that auto-fills city/prefecture, Google Places)
|
||||||
chrome-use press Enter # press a key at current focus (down+up)
|
chrome-use press Enter # press a key at current focus (down+up)
|
||||||
chrome-use press Control+a # key combination
|
chrome-use press Control+a # key combination
|
||||||
chrome-use keydown d # HOLD a key down (no auto-release)
|
chrome-use keydown d # HOLD a key down (no auto-release)
|
||||||
@@ -281,16 +308,32 @@ chrome-use pick @e4 --option "Europe" # ANY combobox (react-select / ARIA /
|
|||||||
# (no silent no-op). Use this for custom
|
# (no silent no-op). Use this for custom
|
||||||
# dropdowns where `select` returns ✓ but
|
# dropdowns where `select` returns ✓ but
|
||||||
# changes nothing.
|
# changes nothing.
|
||||||
chrome-use upload @e5 file1.pdf # upload file(s) — NOTE: needs a --launch/direct-CDP
|
chrome-use upload @e5 file1.pdf # upload file(s) — works over the extension relay too:
|
||||||
# session. Over the extension relay it CANNOT work
|
# chrome.debugger forbids setFileInputFiles, so the
|
||||||
# (Chrome's chrome.debugger forbids it); chrome-use
|
# file's bytes are streamed into the page and rebuilt as
|
||||||
# errors with a hint. Carry your login into a launched
|
# a File there (chunked under native-messaging's 1 MiB cap).
|
||||||
# session via `cookies export` | `cookies set --curl`.
|
# Works on file <input>s and drop/paste composers (e.g. X).
|
||||||
chrome-use scroll down 500 # scroll page (up/down/left/right)
|
chrome-use scroll down 500 # scroll page (up/down/left/right)
|
||||||
|
chrome-use scroll down 700 --at 640,400 # wheel at a pixel — scrolls a cross-origin
|
||||||
|
# iframe (Payments/Stripe/checkout/KYC) that
|
||||||
|
# plain page scroll can't reach
|
||||||
|
chrome-use scroll down 700 --frame 2 # scroll frame 2 from `chrome-use frames`
|
||||||
chrome-use scrollintoview @e1 # scroll element into view
|
chrome-use scrollintoview @e1 # scroll element into view
|
||||||
chrome-use drag @e1 @e2 # drag and drop
|
chrome-use drag @e1 @e2 # drag and drop
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Cross-origin iframes (embedded payment / checkout / KYC widgets — Google
|
||||||
|
Payments, Stripe, etc.) — drive them by ref, never by screenshot.** `snapshot -i`
|
||||||
|
pierces these out-of-process iframes and lists their elements by `@ref`
|
||||||
|
(including input values); `get text --all-frames` reads their text. Then just act
|
||||||
|
on the refs: `click @e`, `type @e`, `hover @e`, `dblclick @e`, `drag @a @b` all
|
||||||
|
work into the iframe. Over the extension relay these are dispatched through the
|
||||||
|
DOM (in the element's own frame), so they hit the right element in the right tab
|
||||||
|
— a coordinate click/scroll there can drift onto whatever tab is in the
|
||||||
|
foreground, so prefer refs. For below-the-fold content in such a frame, scroll it
|
||||||
|
with `scroll down N --at x,y` (a pixel over the frame) or `--frame n`. For a
|
||||||
|
postal/autocomplete box inside the frame, `type @e "…" --key-events`.
|
||||||
|
|
||||||
### When refs don't work or you don't want to snapshot
|
### When refs don't work or you don't want to snapshot
|
||||||
|
|
||||||
Use semantic locators:
|
Use semantic locators:
|
||||||
|
|||||||
Reference in New Issue
Block a user