Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fd2cdcde77 | ||
|
|
8e001e3d88 | ||
|
|
eb2bc343a0 | ||
|
|
d86c9c4be2 | ||
|
|
32c25a6627 | ||
|
|
a8ce3dd3f8 | ||
|
|
997373fd57 | ||
|
|
5a858af93f | ||
|
|
58dc02bfdc | ||
|
|
c47601bd7b | ||
|
|
0296bc7a88 | ||
|
|
32e203b908 | ||
|
|
fc51cd63ba | ||
|
|
f714c7920b | ||
|
|
1ac8ef7732 | ||
|
|
9f24e66033 | ||
|
|
70ab38d35f | ||
|
|
6830df50ea | ||
|
|
cd47ec43d0 | ||
|
|
2cd361817d | ||
|
|
42f47c49aa | ||
|
|
e29800df72 | ||
|
|
e7e849ea39 | ||
|
|
ebb02c65c8 | ||
|
|
4317db636f | ||
|
|
c99838a034 | ||
|
|
dc2aa4cade | ||
|
|
7085f3bf36 | ||
|
|
29815ff5f3 | ||
|
|
2a338d4c29 | ||
|
|
6fcf52db60 | ||
|
|
af8823b27b | ||
|
|
c85e3faa82 | ||
|
|
b25958946c | ||
|
|
d4ff49caa8 | ||
|
|
4e949fffbf | ||
|
|
af46490812 | ||
|
|
57c52d6517 | ||
|
|
2707ceb1c4 | ||
|
|
f7a657ac46 | ||
|
|
4e295ce139 | ||
|
|
3d82f11ff2 | ||
|
|
33269adc1a | ||
|
|
9ab8753b48 | ||
|
|
770708b8e6 | ||
|
|
7c594820da | ||
|
|
81d18bbd2e | ||
|
|
d99a223d23 | ||
|
|
4e7e80a596 | ||
|
|
9bf79a4242 | ||
|
|
5b4ffdb2bb | ||
|
|
62e7229b47 | ||
|
|
23ab4ce68f | ||
|
|
e272546b5c | ||
|
|
c7de19b099 | ||
|
|
6b9de10c73 | ||
|
|
63e0dd5921 | ||
|
|
c0ee65d0d8 | ||
|
|
7601919a04 | ||
|
|
345c0d62a2 |
@@ -39,7 +39,6 @@ jobs:
|
|||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
with:
|
with:
|
||||||
ref: ${{ github.event.inputs.tag || github.ref }}
|
ref: ${{ github.event.inputs.tag || github.ref }}
|
||||||
fetch-depth: 0 # full history so the changelog step can diff tags
|
|
||||||
|
|
||||||
- name: Setup Rust toolchain
|
- name: Setup Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@stable
|
||||||
@@ -117,6 +116,16 @@ jobs:
|
|||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
|
# The release job is separate from the build matrix and has no repo by
|
||||||
|
# default — check it out (full history + tags) so the changelog step has a
|
||||||
|
# git repo to diff. Without this, `git` failed with "not a git repository"
|
||||||
|
# and the changelog came out empty.
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
ref: ${{ github.event.inputs.tag || github.ref }}
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Download all artifacts
|
- name: Download all artifacts
|
||||||
uses: actions/download-artifact@v8
|
uses: actions/download-artifact@v8
|
||||||
with:
|
with:
|
||||||
@@ -138,16 +147,31 @@ jobs:
|
|||||||
git fetch --tags --force --quiet origin 2>/dev/null || true
|
git fetch --tags --force --quiet origin 2>/dev/null || true
|
||||||
TAG="${{ github.event.inputs.tag || github.ref_name }}"
|
TAG="${{ github.event.inputs.tag || github.ref_name }}"
|
||||||
PREV="$(git describe --tags --abbrev=0 "${TAG}^" 2>/dev/null || true)"
|
PREV="$(git describe --tags --abbrev=0 "${TAG}^" 2>/dev/null || true)"
|
||||||
|
RANGE="${TAG}"
|
||||||
|
[ -n "$PREV" ] && RANGE="${PREV}..${TAG}"
|
||||||
|
# Group commit subjects by conventional-commit type so the notes are
|
||||||
|
# scannable ("what's new / what's fixed") instead of a flat dev log.
|
||||||
|
LOG="$(git log "$RANGE" --no-merges --pretty='%s' | grep -v '^chore(release)' || true)"
|
||||||
|
# NOTE: the job runs under `bash -e`. grep returning 1 (no match) and
|
||||||
|
# the `[ -n "$body" ]` test returning 1 (empty section) must NOT abort
|
||||||
|
# the script — otherwise a release whose commit range lacks a whole
|
||||||
|
# category (e.g. only `feat`, no `fix`) dies before writing the closing
|
||||||
|
# heredoc delimiter and the whole release step fails. `|| true` +
|
||||||
|
# `return 0` keep section() always-succeeding.
|
||||||
|
section() { # $1=header $2=grep-pattern
|
||||||
|
local body; body="$(printf '%s\n' "$LOG" | grep -E "$2" | sed 's/^/- /' || true)"
|
||||||
|
[ -n "$body" ] && printf '\n### %s\n%s\n' "$1" "$body"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
{
|
{
|
||||||
echo "notes<<__NOTES_EOF__"
|
echo "notes<<__NOTES_EOF__"
|
||||||
echo "## What changed"
|
echo "## What changed"
|
||||||
echo ""
|
section "✨ Features" '^feat'
|
||||||
|
section "🐛 Fixes" '^fix'
|
||||||
|
section "🔧 Other" '^(perf|refactor|docs|build|ci|test|style|revert)'
|
||||||
if [ -n "$PREV" ]; then
|
if [ -n "$PREV" ]; then
|
||||||
git log "${PREV}..${TAG}" --no-merges --pretty='- %s' | grep -v '^- chore(release)' || true
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "**Full changelog**: https://github.com/${{ github.repository }}/compare/${PREV}...${TAG}"
|
echo "**Full changelog**: https://github.com/${{ github.repository }}/compare/${PREV}...${TAG}"
|
||||||
else
|
|
||||||
git log "${TAG}" --no-merges --pretty='- %s' | grep -v '^- chore(release)' || true
|
|
||||||
fi
|
fi
|
||||||
echo "__NOTES_EOF__"
|
echo "__NOTES_EOF__"
|
||||||
} >> "$GITHUB_OUTPUT"
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
|||||||
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.2.3"
|
version = "1.5.18"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.2.3"
|
version = "1.5.18"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "Fast browser automation CLI for AI agents"
|
description = "Fast browser automation CLI for AI agents"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
@@ -3,6 +3,23 @@ use std::env;
|
|||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
/// Embed the version of the `ab-connect` extension this CLI ships alongside, so
|
||||||
|
/// `doctor` can tell a connected extension "you're older than what this CLI
|
||||||
|
/// expects, update it." Read from the extension manifest at build time so it
|
||||||
|
/// stays in sync with whatever extension version is in the same checkout/release
|
||||||
|
/// (the ext is on its own 0.4.x line, separate from the CLI version). Falls back
|
||||||
|
/// to "unknown" if the manifest can't be read.
|
||||||
|
fn embed_extension_version() {
|
||||||
|
let manifest = Path::new("../extensions/ab-connect/manifest.json");
|
||||||
|
println!("cargo:rerun-if-changed=../extensions/ab-connect/manifest.json");
|
||||||
|
let version = fs::read_to_string(manifest)
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
|
||||||
|
.and_then(|v| v.get("version").and_then(|x| x.as_str()).map(String::from))
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
println!("cargo:rustc-env=AB_CONNECT_VERSION={}", version);
|
||||||
|
}
|
||||||
|
|
||||||
/// Ensure `packages/dashboard/out/` exists so `rust-embed` doesn't fail during
|
/// Ensure `packages/dashboard/out/` exists so `rust-embed` doesn't fail during
|
||||||
/// Rust-only dev builds where the dashboard hasn't been built. The placeholder
|
/// Rust-only dev builds where the dashboard hasn't been built. The placeholder
|
||||||
/// `index.html` is only written when the directory is completely absent.
|
/// `index.html` is only written when the directory is completely absent.
|
||||||
@@ -20,6 +37,7 @@ fn ensure_dashboard_dir() {
|
|||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
ensure_dashboard_dir();
|
ensure_dashboard_dir();
|
||||||
|
embed_extension_version();
|
||||||
|
|
||||||
let protocol_dir = Path::new("cdp-protocol");
|
let protocol_dir = Path::new("cdp-protocol");
|
||||||
let out_dir = env::var("OUT_DIR").unwrap();
|
let out_dir = env::var("OUT_DIR").unwrap();
|
||||||
|
|||||||
+541
-44
@@ -30,12 +30,104 @@ pub enum ParseError {
|
|||||||
InvalidSessionName { name: String },
|
InvalidSessionName { name: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Top-level commands an agent is likely to mistype, used for "did you mean"
|
||||||
|
/// suggestions on an unknown command (issue #29). Not exhaustive — just the
|
||||||
|
/// common verbs plus a few known wrong-guesses mapped to the real command.
|
||||||
|
const KNOWN_COMMANDS: &[&str] = &[
|
||||||
|
"open",
|
||||||
|
"navigate",
|
||||||
|
"click",
|
||||||
|
"fill",
|
||||||
|
"type",
|
||||||
|
"press",
|
||||||
|
"snapshot",
|
||||||
|
"screenshot",
|
||||||
|
"eval",
|
||||||
|
"get",
|
||||||
|
"text",
|
||||||
|
"html",
|
||||||
|
"frames",
|
||||||
|
"find",
|
||||||
|
"wait",
|
||||||
|
"scroll",
|
||||||
|
"hover",
|
||||||
|
"select",
|
||||||
|
"check",
|
||||||
|
"uncheck",
|
||||||
|
"tab",
|
||||||
|
"tabs",
|
||||||
|
"close",
|
||||||
|
"back",
|
||||||
|
"forward",
|
||||||
|
"reload",
|
||||||
|
"sessions",
|
||||||
|
"status",
|
||||||
|
"daemon",
|
||||||
|
"doctor",
|
||||||
|
"upgrade",
|
||||||
|
"connect",
|
||||||
|
"cookies",
|
||||||
|
"mouse",
|
||||||
|
"keyboard",
|
||||||
|
"stream",
|
||||||
|
"frame",
|
||||||
|
"profiles",
|
||||||
|
"title",
|
||||||
|
"url",
|
||||||
|
"is",
|
||||||
|
"drag",
|
||||||
|
"dialog",
|
||||||
|
"upload",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Levenshtein distance, capped — small inputs only (command names).
|
||||||
|
fn edit_distance(a: &str, b: &str) -> usize {
|
||||||
|
let a: Vec<char> = a.chars().collect();
|
||||||
|
let b: Vec<char> = b.chars().collect();
|
||||||
|
let mut prev: Vec<usize> = (0..=b.len()).collect();
|
||||||
|
let mut curr = vec![0usize; b.len() + 1];
|
||||||
|
for (i, &ca) in a.iter().enumerate() {
|
||||||
|
curr[0] = i + 1;
|
||||||
|
for (j, &cb) in b.iter().enumerate() {
|
||||||
|
let cost = if ca == cb { 0 } else { 1 };
|
||||||
|
curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(prev[j] + cost);
|
||||||
|
}
|
||||||
|
std::mem::swap(&mut prev, &mut curr);
|
||||||
|
}
|
||||||
|
prev[b.len()]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Closest known command within a small edit distance, or a prefix/substring
|
||||||
|
/// match — `None` if nothing is close enough to suggest confidently.
|
||||||
|
fn nearest_command(input: &str) -> Option<String> {
|
||||||
|
let lower = input.to_lowercase();
|
||||||
|
// Exact prefix/substring hits first (e.g. "session" -> "sessions").
|
||||||
|
if let Some(c) = KNOWN_COMMANDS
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.starts_with(&lower) || lower.starts_with(**c))
|
||||||
|
{
|
||||||
|
return Some(c.to_string());
|
||||||
|
}
|
||||||
|
// Tolerance scales with length: short words get distance 1, longer get 2.
|
||||||
|
let max_dist = if lower.len() <= 4 { 1 } else { 2 };
|
||||||
|
KNOWN_COMMANDS
|
||||||
|
.iter()
|
||||||
|
.map(|c| (*c, edit_distance(&lower, c)))
|
||||||
|
.filter(|(_, d)| *d <= max_dist)
|
||||||
|
.min_by_key(|(_, d)| *d)
|
||||||
|
.map(|(c, _)| c.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
impl ParseError {
|
impl ParseError {
|
||||||
pub fn format(&self) -> String {
|
pub fn format(&self) -> String {
|
||||||
match self {
|
match self {
|
||||||
ParseError::UnknownCommand { command } => {
|
ParseError::UnknownCommand { command } => match nearest_command(command) {
|
||||||
format!("Unknown command: {}", command)
|
Some(suggestion) => format!(
|
||||||
}
|
"Unknown command: {}\nDid you mean: chrome-use {}?",
|
||||||
|
command, suggestion
|
||||||
|
),
|
||||||
|
None => format!("Unknown command: {}", command),
|
||||||
|
},
|
||||||
ParseError::UnknownSubcommand {
|
ParseError::UnknownSubcommand {
|
||||||
subcommand,
|
subcommand,
|
||||||
valid_options,
|
valid_options,
|
||||||
@@ -370,6 +462,12 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
if flags.provider.is_some() {
|
if flags.provider.is_some() {
|
||||||
nav_cmd["waitUntil"] = json!("none");
|
nav_cmd["waitUntil"] = json!("none");
|
||||||
}
|
}
|
||||||
|
// `--reuse-tab`: adopt an existing tab already on this URL instead of
|
||||||
|
// navigating/spawning a new one (issue #21 — avoids duplicate tabs on
|
||||||
|
// rebind, preserves in-page state).
|
||||||
|
if rest.iter().any(|a| *a == "--reuse-tab" || *a == "--reuse") {
|
||||||
|
nav_cmd["reuseTab"] = json!(true);
|
||||||
|
}
|
||||||
// Explicit readiness override (issue #10): SPAs whose `load` event
|
// Explicit readiness override (issue #10): SPAs whose `load` event
|
||||||
// never fires (a long-lived XHR/websocket holds it open) hang out the
|
// never fires (a long-lived XHR/websocket holds it open) hang out the
|
||||||
// load-event wait. `--wait-until domcontentloaded` returns as soon as
|
// load-event wait. `--wait-until domcontentloaded` returns as soon as
|
||||||
@@ -418,6 +516,9 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
// === Core Actions ===
|
// === Core Actions ===
|
||||||
"click" => {
|
"click" => {
|
||||||
let new_tab = rest.contains(&"--new-tab");
|
let new_tab = rest.contains(&"--new-tab");
|
||||||
|
// `--follow`: if the click opens a new tab, switch the active tab to
|
||||||
|
// it (default reports the opened tab but stays put) (issue #24-A).
|
||||||
|
let follow = rest.contains(&"--follow");
|
||||||
// Coordinate click as a first-class form (issue #8.4): when the only
|
// Coordinate click as a first-class form (issue #8.4): when the only
|
||||||
// handle is a pixel position, no element/selector is needed.
|
// handle is a pixel position, no element/selector is needed.
|
||||||
// click <x> <y> e.g. click 449 320
|
// click <x> <y> e.g. click 449 320
|
||||||
@@ -426,23 +527,27 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
let coord_args: Vec<&str> = rest
|
let coord_args: Vec<&str> = rest
|
||||||
.iter()
|
.iter()
|
||||||
.copied()
|
.copied()
|
||||||
.filter(|a| *a != "--new-tab" && *a != "--coords")
|
.filter(|a| !a.starts_with("--"))
|
||||||
.collect();
|
.collect();
|
||||||
if let Some((x, y)) = parse_coords(&coord_args) {
|
if let Some((x, y)) = parse_coords(&coord_args) {
|
||||||
return Ok(json!({ "id": id, "action": "click", "x": x, "y": y }));
|
return Ok(json!({ "id": id, "action": "click", "x": x, "y": y }));
|
||||||
}
|
}
|
||||||
let sel = rest
|
let sel = rest
|
||||||
.iter()
|
.iter()
|
||||||
.find(|arg| **arg != "--new-tab")
|
.find(|arg| !arg.starts_with("--"))
|
||||||
.ok_or_else(|| ParseError::MissingArguments {
|
.ok_or_else(|| ParseError::MissingArguments {
|
||||||
context: "click".to_string(),
|
context: "click".to_string(),
|
||||||
usage: "click <selector> | click <x> <y> | click --coords <x>,<y> [--new-tab]",
|
usage:
|
||||||
|
"click <selector> | click <x> <y> | click --coords <x>,<y> [--new-tab] [--follow]",
|
||||||
})?;
|
})?;
|
||||||
|
let mut cmd = json!({ "id": id, "action": "click", "selector": sel });
|
||||||
if new_tab {
|
if new_tab {
|
||||||
Ok(json!({ "id": id, "action": "click", "selector": sel, "newTab": true }))
|
cmd["newTab"] = json!(true);
|
||||||
} else {
|
|
||||||
Ok(json!({ "id": id, "action": "click", "selector": sel }))
|
|
||||||
}
|
}
|
||||||
|
if follow {
|
||||||
|
cmd["follow"] = json!(true);
|
||||||
|
}
|
||||||
|
Ok(cmd)
|
||||||
}
|
}
|
||||||
"dblclick" => {
|
"dblclick" => {
|
||||||
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
@@ -459,20 +564,31 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": rest[1..].join(" ") }))
|
Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": rest[1..].join(" ") }))
|
||||||
}
|
}
|
||||||
"type" => {
|
"type" => {
|
||||||
|
// `--key-events` (alias `--keys`): send real per-character keystrokes
|
||||||
|
// instead of Input.insertText, so autocomplete/combobox widgets that
|
||||||
|
// only react to key events fire (e.g. Google address postal lookup).
|
||||||
|
let key_events = rest.iter().any(|a| *a == "--key-events" || *a == "--keys");
|
||||||
|
let rest: Vec<&str> = rest
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|a| *a != "--key-events" && *a != "--keys")
|
||||||
|
.collect();
|
||||||
// `type --focused <text>` types into whatever element currently has
|
// `type --focused <text>` types into whatever element currently has
|
||||||
// focus (no selector) — for custom widgets that move focus to a hidden
|
// focus (no selector) — for custom widgets that move focus to a hidden
|
||||||
// input after you open them.
|
// input after you open them.
|
||||||
if rest.first() == Some(&"--focused") {
|
if rest.first() == Some(&"--focused") {
|
||||||
return Ok(json!({
|
return Ok(json!({
|
||||||
"id": id, "action": "type", "focused": true,
|
"id": id, "action": "type", "focused": true,
|
||||||
"text": rest[1..].join(" "),
|
"text": rest[1..].join(" "), "keyEvents": key_events,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
context: "type".to_string(),
|
context: "type".to_string(),
|
||||||
usage: "type <selector> <text> (or: type --focused <text>)",
|
usage: "type <selector> <text> (or: type --focused <text>) [--key-events]",
|
||||||
})?;
|
})?;
|
||||||
Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" ") }))
|
Ok(
|
||||||
|
json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" "), "keyEvents": key_events }),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
"pick" => {
|
"pick" => {
|
||||||
// pick <selector|@ref> --option "<text>" — atomic combobox select:
|
// pick <selector|@ref> --option "<text>" — atomic combobox select:
|
||||||
@@ -577,11 +693,27 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
|
|
||||||
// === Keyboard ===
|
// === Keyboard ===
|
||||||
"press" | "key" => {
|
"press" | "key" => {
|
||||||
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let key = rest.iter().find(|a| !a.starts_with("--")).ok_or_else(|| {
|
||||||
context: "press".to_string(),
|
ParseError::MissingArguments {
|
||||||
usage: "press <key>",
|
context: "press".to_string(),
|
||||||
|
usage: "press <key> [--hold <ms>]",
|
||||||
|
}
|
||||||
})?;
|
})?;
|
||||||
Ok(json!({ "id": id, "action": "press", "key": key }))
|
let mut c = json!({ "id": id, "action": "press", "key": key });
|
||||||
|
// `--hold <ms>`: hold the key down for <ms> then release, timed inside
|
||||||
|
// the daemon (one round-trip, no shell-sleep jitter) — for games and
|
||||||
|
// hold-to-charge where keydown+sleep+keyup over 3 round-trips is too
|
||||||
|
// imprecise.
|
||||||
|
if let Some(i) = rest.iter().position(|a| *a == "--hold") {
|
||||||
|
let ms = rest.get(i + 1).and_then(|s| s.parse::<u64>().ok()).ok_or(
|
||||||
|
ParseError::MissingArguments {
|
||||||
|
context: "press --hold".to_string(),
|
||||||
|
usage: "press <key> --hold <ms>",
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
c["hold"] = json!(ms);
|
||||||
|
}
|
||||||
|
Ok(c)
|
||||||
}
|
}
|
||||||
"keydown" => {
|
"keydown" => {
|
||||||
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
@@ -647,10 +779,57 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
} else {
|
} else {
|
||||||
return Err(ParseError::MissingArguments {
|
return Err(ParseError::MissingArguments {
|
||||||
context: "scroll --selector".to_string(),
|
context: "scroll --selector".to_string(),
|
||||||
usage: "scroll [direction] [amount] [--selector <sel>]",
|
usage: "scroll [direction] [amount] [--selector <sel>] [--at <x,y>] [--frame <n>]",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
"--at" => {
|
||||||
|
// `--at x,y`: dispatch the wheel at this viewport pixel, so it
|
||||||
|
// scrolls whatever element/iframe is under the pointer — including
|
||||||
|
// cross-origin iframes that `window.scrollBy` can't reach (#36).
|
||||||
|
let val = rest.get(i + 1).ok_or(ParseError::MissingArguments {
|
||||||
|
context: "scroll --at".to_string(),
|
||||||
|
usage: "scroll [direction] [amount] --at <x,y>",
|
||||||
|
})?;
|
||||||
|
let mut parts = val.split(',');
|
||||||
|
match (
|
||||||
|
parts.next().and_then(|s| s.trim().parse::<f64>().ok()),
|
||||||
|
parts.next().and_then(|s| s.trim().parse::<f64>().ok()),
|
||||||
|
) {
|
||||||
|
(Some(x), Some(y)) => {
|
||||||
|
obj.insert("at".to_string(), json!([x, y]));
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!("scroll --at: invalid coordinate `{}`", val),
|
||||||
|
usage:
|
||||||
|
"scroll [direction] [amount] --at <x,y> (e.g. --at 640,400)",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
"--frame" => {
|
||||||
|
// `--frame n`: scroll the n-th frame from `chrome-use frames` by
|
||||||
|
// dispatching the wheel at that frame's center — reaches content in
|
||||||
|
// a cross-origin iframe without needing a selector into it (#36).
|
||||||
|
let val = rest.get(i + 1).ok_or(ParseError::MissingArguments {
|
||||||
|
context: "scroll --frame".to_string(),
|
||||||
|
usage: "scroll [direction] [amount] --frame <n>",
|
||||||
|
})?;
|
||||||
|
match val.trim().parse::<usize>() {
|
||||||
|
Ok(n) => {
|
||||||
|
obj.insert("frame".to_string(), json!(n));
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!("scroll --frame: invalid index `{}`", val),
|
||||||
|
usage: "scroll [direction] [amount] --frame <n> (index from `chrome-use frames`)",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
arg if arg.starts_with('-') => {}
|
arg if arg.starts_with('-') => {}
|
||||||
_ => {
|
_ => {
|
||||||
match positional_index {
|
match positional_index {
|
||||||
@@ -825,17 +1004,41 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
// selector: @ref or CSS selector
|
// selector: @ref or CSS selector
|
||||||
// path: file path (contains / or . or ends with known extension)
|
// path: file path (contains / or . or ends with known extension)
|
||||||
let mut full_page = false;
|
let mut full_page = false;
|
||||||
let positional: Vec<&str> = rest
|
let mut clip: Option<Value> = None;
|
||||||
.iter()
|
let mut positional: Vec<&str> = Vec::new();
|
||||||
.filter(|arg| match **arg {
|
let mut i = 0;
|
||||||
"--full" | "-f" => {
|
while i < rest.len() {
|
||||||
full_page = true;
|
match rest[i] {
|
||||||
false
|
"--full" | "-f" => full_page = true,
|
||||||
|
// `--clip x,y,w,h` captures a pixel region (issue #34).
|
||||||
|
"--clip" => {
|
||||||
|
let raw = rest
|
||||||
|
.get(i + 1)
|
||||||
|
.ok_or_else(|| ParseError::MissingArguments {
|
||||||
|
context: "screenshot --clip".to_string(),
|
||||||
|
usage: "screenshot --clip <x,y,w,h> [path]",
|
||||||
|
})?;
|
||||||
|
let nums: Vec<f64> = raw
|
||||||
|
.split(',')
|
||||||
|
.filter_map(|n| n.trim().parse::<f64>().ok())
|
||||||
|
.collect();
|
||||||
|
if nums.len() != 4 {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!(
|
||||||
|
"--clip expects 'x,y,w,h' (4 numbers), got '{raw}'"
|
||||||
|
),
|
||||||
|
usage: "screenshot --clip <x,y,w,h> [path]",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
clip = Some(json!({
|
||||||
|
"x": nums[0], "y": nums[1], "width": nums[2], "height": nums[3]
|
||||||
|
}));
|
||||||
|
i += 1;
|
||||||
}
|
}
|
||||||
_ => true,
|
other => positional.push(other),
|
||||||
})
|
}
|
||||||
.copied()
|
i += 1;
|
||||||
.collect();
|
}
|
||||||
let (selector, path) = match (positional.first(), positional.get(1)) {
|
let (selector, path) = match (positional.first(), positional.get(1)) {
|
||||||
(Some(first), Some(second)) => {
|
(Some(first), Some(second)) => {
|
||||||
// Two args: first is selector, second is path
|
// Two args: first is selector, second is path
|
||||||
@@ -866,6 +1069,9 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
"path": path, "selector": selector,
|
"path": path, "selector": selector,
|
||||||
"fullPage": full_page, "annotate": flags.annotate
|
"fullPage": full_page, "annotate": flags.annotate
|
||||||
});
|
});
|
||||||
|
if let Some(c) = clip {
|
||||||
|
cmd["clip"] = c;
|
||||||
|
}
|
||||||
if let Some(ref fmt) = flags.screenshot_format {
|
if let Some(ref fmt) = flags.screenshot_format {
|
||||||
cmd["format"] = json!(fmt);
|
cmd["format"] = json!(fmt);
|
||||||
}
|
}
|
||||||
@@ -996,8 +1202,31 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
Ok(json!({ "id": id, "action": "stealth_status" }))
|
Ok(json!({ "id": id, "action": "stealth_status" }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// `cf-status` — Cloudflare challenge/clearance preflight: is the page
|
||||||
|
// currently a CF challenge, and is there a still-valid cf_clearance (the
|
||||||
|
// HttpOnly persistence cookie)? Lets an agent SKIP re-solving when already
|
||||||
|
// cleared, and know when it must solve. Persistence optimization.
|
||||||
|
"cf-status" | "cf" | "cloudflare-status" | "clearance" => {
|
||||||
|
Ok(json!({ "id": id, "action": "cf_status" }))
|
||||||
|
}
|
||||||
|
|
||||||
// === Close ===
|
// === Close ===
|
||||||
"close" | "quit" | "exit" => Ok(json!({ "id": id, "action": "close" })),
|
"close" | "quit" | "exit" => {
|
||||||
|
// `close <tab>` closes only that tab (and the output says "Tab
|
||||||
|
// closed"); bare `close` closes the browser/session. `close --all` is
|
||||||
|
// intercepted earlier in the dispatcher. Previously `close t12` still
|
||||||
|
// ran a browser close and alarmingly printed "Browser closed" (#26).
|
||||||
|
if let Some(tab_ref) = rest.iter().find(|a| !a.starts_with("--")) {
|
||||||
|
Ok(json!({ "id": id, "action": "tab_close", "tabId": tab_ref }))
|
||||||
|
} else {
|
||||||
|
Ok(json!({ "id": id, "action": "close" }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The active tab's stable handle — `targetId` survives cross-process
|
||||||
|
// navigation and is reusable across sessions, so an agent can hold it
|
||||||
|
// instead of re-deriving "which tab is live" from `tabs` each step (#26).
|
||||||
|
"current" => Ok(json!({ "id": id, "action": "current" })),
|
||||||
|
|
||||||
// === Inspect ===
|
// === Inspect ===
|
||||||
"inspect" => Ok(json!({ "id": id, "action": "inspect" })),
|
"inspect" => Ok(json!({ "id": id, "action": "inspect" })),
|
||||||
@@ -1252,6 +1481,11 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
// === Get ===
|
// === Get ===
|
||||||
"get" => parse_get(&rest, &id),
|
"get" => parse_get(&rest, &id),
|
||||||
|
|
||||||
|
// List every frame the session can reach (top + same-process child
|
||||||
|
// frames + out-of-process iframes), with a text-length per frame so you
|
||||||
|
// can see where a listing's description actually lives (issue #27).
|
||||||
|
"frames" => Ok(json!({ "id": id, "action": "frames" })),
|
||||||
|
|
||||||
// Top-level shortcuts for `get <x>` status reads — users naturally type
|
// Top-level shortcuts for `get <x>` status reads — users naturally type
|
||||||
// `chrome-use url` / `cdp-url` / `title` without the `get` prefix
|
// `chrome-use url` / `cdp-url` / `title` without the `get` prefix
|
||||||
// (and expect `cdp-url`/`cdp_url` to work interchangeably).
|
// (and expect `cdp-url`/`cdp_url` to work interchangeably).
|
||||||
@@ -1553,11 +1787,16 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
}
|
}
|
||||||
Ok(cmd)
|
Ok(cmd)
|
||||||
}
|
}
|
||||||
Some(tab_ref) => Ok(json!({
|
Some(tab_ref) => {
|
||||||
"id": id,
|
// `tab <ref> --activate` (alias `--front`) switches to the tab
|
||||||
"action": "tab_switch",
|
// AND raises it to the foreground — for handing a specific tab
|
||||||
"tabId": tab_ref,
|
// to the human (SMS code, captcha) (issue #24-C).
|
||||||
})),
|
let mut cmd = json!({ "id": id, "action": "tab_switch", "tabId": tab_ref });
|
||||||
|
if rest.iter().any(|a| *a == "--activate" || *a == "--front") {
|
||||||
|
cmd["activate"] = json!(true);
|
||||||
|
}
|
||||||
|
Ok(cmd)
|
||||||
|
}
|
||||||
None => {
|
None => {
|
||||||
let mut cmd = json!({ "id": id, "action": "tab_list" });
|
let mut cmd = json!({ "id": id, "action": "tab_list" });
|
||||||
if full {
|
if full {
|
||||||
@@ -2339,11 +2578,42 @@ fn parse_get(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
|||||||
|
|
||||||
match rest.first().copied() {
|
match rest.first().copied() {
|
||||||
Some("text") => {
|
Some("text") => {
|
||||||
let sel = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
// `get text --all-frames` aggregates visible text across every
|
||||||
context: "get text".to_string(),
|
// frame, including out-of-process iframes invisible to the top
|
||||||
usage: "get text <selector>",
|
// document (issue #27). The selector is ignored in this mode.
|
||||||
})?;
|
let all_frames = rest[1..]
|
||||||
Ok(json!({ "id": id, "action": "gettext", "selector": sel }))
|
.iter()
|
||||||
|
.any(|a| matches!(*a, "--all-frames" | "--frames" | "-a"));
|
||||||
|
if all_frames {
|
||||||
|
return Ok(json!({ "id": id, "action": "gettext", "allFrames": true }));
|
||||||
|
}
|
||||||
|
// `get text --pierce` reads through CLOSED shadow DOM / child docs
|
||||||
|
// via the CDP DOM tree — content eval/innerText can't reach, e.g. an
|
||||||
|
// extension's injected panel in a closed shadow root (issue #30).
|
||||||
|
let pierce = rest[1..]
|
||||||
|
.iter()
|
||||||
|
.any(|a| matches!(*a, "--pierce" | "--shadow" | "--deep"));
|
||||||
|
if pierce {
|
||||||
|
return Ok(json!({ "id": id, "action": "gettext", "pierce": true }));
|
||||||
|
}
|
||||||
|
// `get text --main` returns the main-content region (readability),
|
||||||
|
// skipping header/nav/footer/sidebar boilerplate (issue #27).
|
||||||
|
let main = rest[1..]
|
||||||
|
.iter()
|
||||||
|
.any(|a| matches!(*a, "--main" | "--readable" | "-m"));
|
||||||
|
if main {
|
||||||
|
return Ok(json!({ "id": id, "action": "gettext", "main": true }));
|
||||||
|
}
|
||||||
|
// `get text` with no selector reads the WHOLE PAGE and now defaults
|
||||||
|
// to cross-frame aggregation, so an agent gets a page's iframed
|
||||||
|
// content (listing descriptions etc.) without having to know about
|
||||||
|
// `--all-frames` (#27). On a single-frame page this is identical to
|
||||||
|
// the old body read; multi-frame pages get the child frames too —
|
||||||
|
// a strict superset. An explicit selector stays element-scoped.
|
||||||
|
match rest.iter().skip(1).find(|a| !a.starts_with("--")).copied() {
|
||||||
|
Some(sel) => Ok(json!({ "id": id, "action": "gettext", "selector": sel })),
|
||||||
|
None => Ok(json!({ "id": id, "action": "gettext", "allFrames": true })),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Some("html") => {
|
Some("html") => {
|
||||||
let sel = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
let sel = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||||
@@ -3575,6 +3845,40 @@ mod tests {
|
|||||||
assert_eq!(cmd["url"], "https://example.com");
|
assert_eq!(cmd["url"], "https://example.com");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_press_plain_and_hold() {
|
||||||
|
let cmd = parse_command(&args("press d"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "press");
|
||||||
|
assert_eq!(cmd["key"], "d");
|
||||||
|
assert!(cmd.get("hold").is_none());
|
||||||
|
|
||||||
|
let held = parse_command(&args("press d --hold 800"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(held["key"], "d");
|
||||||
|
assert_eq!(held["hold"], 800);
|
||||||
|
|
||||||
|
// Missing/invalid duration is an error, not a silent no-hold.
|
||||||
|
assert!(parse_command(&args("press d --hold"), &default_flags()).is_err());
|
||||||
|
assert!(parse_command(&args("press d --hold abc"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_navigate_reuse_tab_flag() {
|
||||||
|
let cmd = parse_command(
|
||||||
|
&args("open https://example.com --reuse-tab"),
|
||||||
|
&default_flags(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cmd["action"], "navigate");
|
||||||
|
assert_eq!(cmd["reuseTab"], true);
|
||||||
|
// Alias.
|
||||||
|
let cmd2 =
|
||||||
|
parse_command(&args("open https://example.com --reuse"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd2["reuseTab"], true);
|
||||||
|
// Absent by default.
|
||||||
|
let cmd3 = parse_command(&args("open https://example.com"), &default_flags()).unwrap();
|
||||||
|
assert!(cmd3.get("reuseTab").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_navigate_with_headers() {
|
fn test_navigate_with_headers() {
|
||||||
let mut flags = default_flags();
|
let mut flags = default_flags();
|
||||||
@@ -3723,6 +4027,21 @@ mod tests {
|
|||||||
assert!(cmd.get("x").is_none());
|
assert!(cmd.get("x").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_click_follow_flag() {
|
||||||
|
// `--follow` sets the flag; the selector is still found even with the flag
|
||||||
|
// before it (issue #24-A).
|
||||||
|
let cmd = parse_command(&args("click @e5 --follow"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["selector"], "@e5");
|
||||||
|
assert_eq!(cmd["follow"], true);
|
||||||
|
let cmd2 = parse_command(&args("click --follow @e5"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd2["selector"], "@e5");
|
||||||
|
assert_eq!(cmd2["follow"], true);
|
||||||
|
// Absent by default.
|
||||||
|
let plain = parse_command(&args("click @e5"), &default_flags()).unwrap();
|
||||||
|
assert!(plain.get("follow").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_tabs_alias_lists() {
|
fn test_tabs_alias_lists() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -3848,6 +4167,28 @@ mod tests {
|
|||||||
assert_eq!(cmd["action"], "type");
|
assert_eq!(cmd["action"], "type");
|
||||||
assert_eq!(cmd["selector"], "#input");
|
assert_eq!(cmd["selector"], "#input");
|
||||||
assert_eq!(cmd["text"], "some text");
|
assert_eq!(cmd["text"], "some text");
|
||||||
|
assert_eq!(cmd["keyEvents"], false);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_type_key_events() {
|
||||||
|
// --key-events sends real keystrokes (for autocomplete/combobox) and must
|
||||||
|
// not be swallowed into the typed text.
|
||||||
|
let cmd = parse_command(
|
||||||
|
&args("type #postal 201-0001 --key-events"),
|
||||||
|
&default_flags(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cmd["action"], "type");
|
||||||
|
assert_eq!(cmd["selector"], "#postal");
|
||||||
|
assert_eq!(cmd["text"], "201-0001");
|
||||||
|
assert_eq!(cmd["keyEvents"], true);
|
||||||
|
|
||||||
|
let focused =
|
||||||
|
parse_command(&args("type --focused 201-0001 --keys"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(focused["focused"], true);
|
||||||
|
assert_eq!(focused["text"], "201-0001");
|
||||||
|
assert_eq!(focused["keyEvents"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -3931,6 +4272,28 @@ mod tests {
|
|||||||
assert_eq!(cmd["tabId"], "docs");
|
assert_eq!(cmd["tabId"], "docs");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_close_tab_vs_browser() {
|
||||||
|
// `close <tab>` closes that tab (says "Tab closed"); bare `close` closes
|
||||||
|
// the browser (#26).
|
||||||
|
let tab = parse_command(&args("close t12"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(tab["action"], "tab_close");
|
||||||
|
assert_eq!(tab["tabId"], "t12");
|
||||||
|
let browser = parse_command(&args("close"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(browser["action"], "close");
|
||||||
|
// `quit`/`exit` aliases still browser-close.
|
||||||
|
assert_eq!(
|
||||||
|
parse_command(&args("quit"), &default_flags()).unwrap()["action"],
|
||||||
|
"close"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_current_command() {
|
||||||
|
let cmd = parse_command(&args("current"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "current");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_tab_sends_string_tab_id() {
|
fn test_tab_sends_string_tab_id() {
|
||||||
let cmd = parse_command(&args("tab t2"), &default_flags()).unwrap();
|
let cmd = parse_command(&args("tab t2"), &default_flags()).unwrap();
|
||||||
@@ -4110,6 +4473,24 @@ mod tests {
|
|||||||
assert_eq!(cmd["fullPage"], true);
|
assert_eq!(cmd["fullPage"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_screenshot_clip() {
|
||||||
|
// `--clip x,y,w,h` captures a pixel region (issue #34); the path still parses.
|
||||||
|
let cmd = parse_command(
|
||||||
|
&args("screenshot --clip 10,20,200,40 out.png"),
|
||||||
|
&default_flags(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cmd["action"], "screenshot");
|
||||||
|
assert_eq!(cmd["clip"]["x"], 10.0);
|
||||||
|
assert_eq!(cmd["clip"]["y"], 20.0);
|
||||||
|
assert_eq!(cmd["clip"]["width"], 200.0);
|
||||||
|
assert_eq!(cmd["clip"]["height"], 40.0);
|
||||||
|
assert_eq!(cmd["path"], "out.png");
|
||||||
|
// Bad clip is a clear error, not silent.
|
||||||
|
assert!(parse_command(&args("screenshot --clip 1,2,3"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_screenshot_with_ref() {
|
fn test_screenshot_with_ref() {
|
||||||
let cmd = parse_command(&args("screenshot @e1"), &default_flags()).unwrap();
|
let cmd = parse_command(&args("screenshot @e1"), &default_flags()).unwrap();
|
||||||
@@ -4630,12 +5011,99 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_text_missing_selector() {
|
fn test_get_text_defaults_to_all_frames() {
|
||||||
let result = parse_command(&args("get text"), &default_flags());
|
// `get text` with no selector now reads the whole page across ALL frames
|
||||||
assert!(result.is_err());
|
// by default (#27), so iframed content isn't silently missed. (Was: a
|
||||||
let err = result.unwrap_err();
|
// top-frame `body` read, #24-D.)
|
||||||
assert!(matches!(err, ParseError::MissingArguments { .. }));
|
let cmd = parse_command(&args("get text"), &default_flags()).unwrap();
|
||||||
assert!(err.format().contains("get text"));
|
assert_eq!(cmd["action"], "gettext");
|
||||||
|
assert_eq!(cmd["allFrames"], true);
|
||||||
|
assert!(cmd.get("selector").is_none());
|
||||||
|
// An explicit selector still wins and stays element-scoped.
|
||||||
|
let cmd2 = parse_command(&args("get text h1"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd2["selector"], "h1");
|
||||||
|
assert!(cmd2.get("allFrames").is_none());
|
||||||
|
// `text` top-level shortcut behaves the same.
|
||||||
|
let cmd3 = parse_command(&args("text"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd3["allFrames"], true);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_text_all_frames() {
|
||||||
|
// `--all-frames` switches to whole-page, cross-frame aggregation and
|
||||||
|
// drops the selector (issue #27).
|
||||||
|
for variant in ["get text --all-frames", "get text --frames", "text -a"] {
|
||||||
|
let cmd = parse_command(&args(variant), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "gettext", "{variant}");
|
||||||
|
assert_eq!(cmd["allFrames"], true, "{variant}");
|
||||||
|
assert!(cmd.get("selector").is_none(), "{variant}");
|
||||||
|
}
|
||||||
|
// A flag mixed with a selector still triggers all-frames.
|
||||||
|
let cmd = parse_command(&args("get text body --all-frames"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["allFrames"], true);
|
||||||
|
// Without the flag, a leading flag-like token is skipped for the selector.
|
||||||
|
let cmd = parse_command(&args("get text main"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["selector"], "main");
|
||||||
|
assert!(cmd.get("allFrames").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_frames_command() {
|
||||||
|
let cmd = parse_command(&args("frames"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "frames");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_nearest_command_suggestions() {
|
||||||
|
assert_eq!(nearest_command("sesions").as_deref(), Some("sessions"));
|
||||||
|
assert_eq!(nearest_command("session").as_deref(), Some("sessions"));
|
||||||
|
assert_eq!(nearest_command("clik").as_deref(), Some("click"));
|
||||||
|
assert_eq!(
|
||||||
|
nearest_command("screenshits").as_deref(),
|
||||||
|
Some("screenshot")
|
||||||
|
);
|
||||||
|
// Nonsense with no close match stays silent.
|
||||||
|
assert_eq!(nearest_command("xyzzy"), None);
|
||||||
|
// The unknown-command error embeds the suggestion.
|
||||||
|
let err = ParseError::UnknownCommand {
|
||||||
|
command: "sesions".to_string(),
|
||||||
|
};
|
||||||
|
assert!(err.format().contains("Did you mean: chrome-use sessions?"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_text_main() {
|
||||||
|
for variant in ["get text --main", "get text --readable", "text -m"] {
|
||||||
|
let cmd = parse_command(&args(variant), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "gettext", "{variant}");
|
||||||
|
assert_eq!(cmd["main"], true, "{variant}");
|
||||||
|
assert!(cmd.get("selector").is_none(), "{variant}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_text_pierce() {
|
||||||
|
for variant in ["get text --pierce", "get text --shadow", "text --deep"] {
|
||||||
|
let cmd = parse_command(&args(variant), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "gettext", "{variant}");
|
||||||
|
assert_eq!(cmd["pierce"], true, "{variant}");
|
||||||
|
assert!(cmd.get("selector").is_none(), "{variant}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tab_activate_flag() {
|
||||||
|
let plain = parse_command(&args("tab t3"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(plain["action"], "tab_switch");
|
||||||
|
assert!(plain.get("activate").is_none());
|
||||||
|
|
||||||
|
let act = parse_command(&args("tab t3 --activate"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(act["action"], "tab_switch");
|
||||||
|
assert_eq!(act["tabId"], "t3");
|
||||||
|
assert_eq!(act["activate"], true);
|
||||||
|
// `--front` alias.
|
||||||
|
let front = parse_command(&args("tab t3 --front"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(front["activate"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// === Protocol alignment tests ===
|
// === Protocol alignment tests ===
|
||||||
@@ -5529,6 +5997,35 @@ mod tests {
|
|||||||
assert_eq!(cmd["selector"], ".sidebar");
|
assert_eq!(cmd["selector"], ".sidebar");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_at_coordinate() {
|
||||||
|
// `--at x,y` carries a [x, y] array for a wheel dispatched at that pixel
|
||||||
|
// (issue #36: cross-origin iframe scroll).
|
||||||
|
let cmd = parse_command(&args("scroll down 700 --at 640,400"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "scroll");
|
||||||
|
assert_eq!(cmd["direction"], "down");
|
||||||
|
assert_eq!(cmd["amount"], 700);
|
||||||
|
assert_eq!(cmd["at"], json!([640.0, 400.0]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_at_rejects_garbage() {
|
||||||
|
assert!(parse_command(&args("scroll --at nope"), &default_flags()).is_err());
|
||||||
|
assert!(parse_command(&args("scroll --at 1"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_frame_index() {
|
||||||
|
let cmd = parse_command(&args("scroll down 700 --frame 2"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "scroll");
|
||||||
|
assert_eq!(cmd["frame"], 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scroll_frame_rejects_non_integer() {
|
||||||
|
assert!(parse_command(&args("scroll --frame two"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_scroll_selector_before_positional() {
|
fn test_scroll_selector_before_positional() {
|
||||||
let cmd =
|
let cmd =
|
||||||
|
|||||||
@@ -449,6 +449,69 @@ pub fn relay_url() -> Option<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Append a one-line record of how a CDP connection was established, to
|
||||||
|
/// `~/.chrome-use/connect-mode.log`. This is the smoking-gun detector for the
|
||||||
|
/// "Allow remote debugging?" consent modal: that modal ONLY appears on a raw
|
||||||
|
/// remote-debugging attach / a browser we launched with a debug port — NEVER on
|
||||||
|
/// the extension relay. When the modal reappears, this log says which session
|
||||||
|
/// took which path and when, so we can tell a code regression (`raw-port` /
|
||||||
|
/// `launched` while the relay was up) from Chrome's own extension-debugger
|
||||||
|
/// consent UX. Low volume (one line per connection); best-effort, never fails a
|
||||||
|
/// connection.
|
||||||
|
pub fn log_connect_mode(ws_url: &str, launched: bool, session: &str) {
|
||||||
|
let relay = relay_url();
|
||||||
|
let relay_up = relay.is_some();
|
||||||
|
let mode = if launched {
|
||||||
|
"launched(debug-port)"
|
||||||
|
} else if relay.as_deref() == Some(ws_url) {
|
||||||
|
"relay"
|
||||||
|
} else if ws_url.contains("127.0.0.1") || ws_url.contains("localhost") {
|
||||||
|
"raw-port-attach"
|
||||||
|
} else {
|
||||||
|
"remote-ws"
|
||||||
|
};
|
||||||
|
// A raw-port attach or a self-launch while the relay was available is the
|
||||||
|
// exact thing that pops the consent modal — flag it loudly in the line.
|
||||||
|
let suspect = (mode == "raw-port-attach" || launched) && relay_up;
|
||||||
|
let line = format!(
|
||||||
|
"session={session} mode={mode} relay_up={relay_up}{} ws={ws_url}\n",
|
||||||
|
if suspect { " CONSENT-MODAL-RISK" } else { "" }
|
||||||
|
);
|
||||||
|
if let Some(home) = dirs::home_dir() {
|
||||||
|
let path = home.join(".chrome-use").join("connect-mode.log");
|
||||||
|
use std::io::Write;
|
||||||
|
if let Ok(mut f) = std::fs::OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.append(true)
|
||||||
|
.open(&path)
|
||||||
|
{
|
||||||
|
let _ = f.write_all(line.as_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sidecar recording the connected extension's version, written by the host when
|
||||||
|
/// it receives the extension's `hello` (sibling of `relay-cdp-url`). Lets
|
||||||
|
/// `doctor` surface which extension build is live without a CDP round-trip.
|
||||||
|
fn relay_ext_version_path() -> PathBuf {
|
||||||
|
relay_url_path().with_file_name("relay-ext-version")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Version of the connected `ab-connect` extension, if the host learned it from
|
||||||
|
/// the extension's `hello`. `None` when no extension has connected since the
|
||||||
|
/// host started, or the extension predates version reporting.
|
||||||
|
pub fn relay_ext_version() -> Option<String> {
|
||||||
|
let s = std::fs::read_to_string(relay_ext_version_path())
|
||||||
|
.ok()?
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
if s.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Hidden `__nm-host` mode: launched by Chrome for the ab-connect extension.
|
/// Hidden `__nm-host` mode: launched by Chrome for the ab-connect extension.
|
||||||
///
|
///
|
||||||
/// Bridges the extension (native-messaging stdio, envelope protocol) to a local
|
/// Bridges the extension (native-messaging stdio, envelope protocol) to a local
|
||||||
@@ -570,6 +633,15 @@ async fn nm_host_main() {
|
|||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
Err(_) => continue,
|
Err(_) => continue,
|
||||||
};
|
};
|
||||||
|
// Extension version handshake: record it next to the relay URL so
|
||||||
|
// `doctor` can report which extension build is live (and whether it's
|
||||||
|
// behind). Best-effort; the message carries no CDP payload.
|
||||||
|
if v.get("method").and_then(|m| m.as_str()) == Some("hello") {
|
||||||
|
if let Some(ver) = v.get("version").and_then(|x| x.as_str()) {
|
||||||
|
let _ = std::fs::write(relay_ext_version_path(), ver);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let outs = {
|
let outs = {
|
||||||
let mut s = state.lock().await;
|
let mut s = state.lock().await;
|
||||||
s.handle_ext_message(&v, "")
|
s.handle_ext_message(&v, "")
|
||||||
@@ -602,6 +674,7 @@ async fn nm_host_main() {
|
|||||||
}
|
}
|
||||||
nm_log("[nm-host] stdin EOF — Chrome closed the port");
|
nm_log("[nm-host] stdin EOF — Chrome closed the port");
|
||||||
let _ = std::fs::remove_file(relay_url_path());
|
let _ = std::fs::remove_file(relay_url_path());
|
||||||
|
let _ = std::fs::remove_file(relay_ext_version_path());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
|||||||
@@ -642,6 +642,22 @@ fn kill_stale_daemon(session: &str) {
|
|||||||
cleanup_stale_files(session);
|
cleanup_stale_files(session);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Kill every per-session daemon worker (SIGTERM→SIGKILL + sidecar cleanup),
|
||||||
|
/// leaving the Chrome-launched `__nm-host` native-messaging bridge alone — it's
|
||||||
|
/// not a tracked session daemon, so the extension relay stays up. Returns the
|
||||||
|
/// session names that were stopped. Powers `chrome-use daemon restart`, which
|
||||||
|
/// clears corrupted/cross-leaked daemon state (e.g. after a version-mismatch
|
||||||
|
/// restart) without the user resorting to `pgrep`/`kill` (issue #20).
|
||||||
|
pub fn restart_all_daemons() -> Vec<String> {
|
||||||
|
let inventory = walk_daemons();
|
||||||
|
let mut stopped = Vec::new();
|
||||||
|
for session in &inventory.sessions {
|
||||||
|
kill_stale_daemon(&session.name);
|
||||||
|
stopped.push(session.name.clone());
|
||||||
|
}
|
||||||
|
stopped
|
||||||
|
}
|
||||||
|
|
||||||
pub fn ensure_daemon(session: &str, opts: &DaemonOptions) -> Result<DaemonResult, String> {
|
pub fn ensure_daemon(session: &str, opts: &DaemonOptions) -> Result<DaemonResult, String> {
|
||||||
// Socket connectivity is the sole liveness check — no PID check — so
|
// Socket connectivity is the sole liveness check — no PID check — so
|
||||||
// callers in a different PID namespace (e.g. unshare) can still reuse
|
// callers in a different PID namespace (e.g. unshare) can still reuse
|
||||||
@@ -1182,6 +1198,55 @@ mod tests {
|
|||||||
let _ = fs::remove_dir(&dir);
|
let _ = fs::remove_dir(&dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_restart_all_daemons_empty_dir() {
|
||||||
|
let dir = std::env::temp_dir().join("ab-test-restart-empty");
|
||||||
|
let _ = fs::create_dir_all(&dir);
|
||||||
|
let _guard = EnvGuard::new(&["AGENT_BROWSER_SOCKET_DIR", "XDG_RUNTIME_DIR"]);
|
||||||
|
_guard.set("AGENT_BROWSER_SOCKET_DIR", dir.to_str().unwrap());
|
||||||
|
|
||||||
|
// No daemons registered → nothing to stop, and it must not blow up.
|
||||||
|
assert!(restart_all_daemons().is_empty());
|
||||||
|
|
||||||
|
let _ = fs::remove_dir(&dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn test_restart_all_daemons_kills_live_session() {
|
||||||
|
let dir = std::env::temp_dir().join("ab-test-restart-live");
|
||||||
|
let _ = fs::create_dir_all(&dir);
|
||||||
|
let _guard = EnvGuard::new(&["AGENT_BROWSER_SOCKET_DIR", "XDG_RUNTIME_DIR"]);
|
||||||
|
_guard.set("AGENT_BROWSER_SOCKET_DIR", dir.to_str().unwrap());
|
||||||
|
|
||||||
|
// Spawn a real, killable child and register it as a session daemon.
|
||||||
|
let mut child = Command::new("sleep")
|
||||||
|
.arg("30")
|
||||||
|
.spawn()
|
||||||
|
.expect("spawn sleep");
|
||||||
|
let pid = child.id();
|
||||||
|
let _ = fs::write(dir.join("rktest.pid"), pid.to_string());
|
||||||
|
let _ = fs::write(get_socket_path("rktest"), b"");
|
||||||
|
|
||||||
|
let stopped = restart_all_daemons();
|
||||||
|
assert!(
|
||||||
|
stopped.contains(&"rktest".to_string()),
|
||||||
|
"stopped: {:?}",
|
||||||
|
stopped
|
||||||
|
);
|
||||||
|
|
||||||
|
// Reap the killed child first — until the parent waits, it lingers as a
|
||||||
|
// zombie that still answers `kill(pid, 0)`, so is_pid_alive would lie.
|
||||||
|
let _ = child.wait();
|
||||||
|
assert!(!is_pid_alive(pid));
|
||||||
|
|
||||||
|
// Sidecars are cleaned up.
|
||||||
|
assert!(!dir.join("rktest.pid").exists());
|
||||||
|
assert!(!get_socket_path("rktest").exists());
|
||||||
|
|
||||||
|
let _ = fs::remove_dir(&dir);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_cleanup_stale_files_removes_version() {
|
fn test_cleanup_stale_files_removes_version() {
|
||||||
let dir = std::env::temp_dir().join("ab-test-cleanup-version");
|
let dir = std::env::temp_dir().join("ab-test-cleanup-version");
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ mod launch;
|
|||||||
mod network;
|
mod network;
|
||||||
mod providers;
|
mod providers;
|
||||||
mod security;
|
mod security;
|
||||||
|
mod versions;
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
@@ -97,6 +98,7 @@ pub fn run_doctor(opts: DoctorOptions) -> i32 {
|
|||||||
let mut fixed: Vec<String> = Vec::new();
|
let mut fixed: Vec<String> = Vec::new();
|
||||||
|
|
||||||
environment::check(&mut checks);
|
environment::check(&mut checks);
|
||||||
|
versions::check(&mut checks);
|
||||||
chrome::check(&mut checks);
|
chrome::check(&mut checks);
|
||||||
daemon::check(&mut checks);
|
daemon::check(&mut checks);
|
||||||
config::check(&mut checks);
|
config::check(&mut checks);
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
//! Version-coherence checks across all four moving parts: the CLI binary, the
|
||||||
|
//! per-session daemons (covered by `daemon.rs`), the connected `ab-connect`
|
||||||
|
//! extension, and the bundled skill. The extension was previously a black box —
|
||||||
|
//! nothing reported which build was live — so a user could sit on an old
|
||||||
|
//! extension with no signal. The extension now reports its version over the
|
||||||
|
//! relay (`hello`), the host records it, and this surfaces it in one place.
|
||||||
|
|
||||||
|
use super::{Check, Status};
|
||||||
|
use crate::{connect, upgrade};
|
||||||
|
|
||||||
|
pub(super) fn check(checks: &mut Vec<Check>) {
|
||||||
|
let category = "Versions";
|
||||||
|
let cli_version = env!("CARGO_PKG_VERSION");
|
||||||
|
|
||||||
|
// CLI — compare against the latest seen by the background update check.
|
||||||
|
match upgrade::cached_latest_version() {
|
||||||
|
Some(latest) if upgrade::version_is_newer(&latest, cli_version) => {
|
||||||
|
checks.push(
|
||||||
|
Check::new(
|
||||||
|
"versions.cli",
|
||||||
|
category,
|
||||||
|
Status::Warn,
|
||||||
|
format!("CLI {cli_version} (newer available: {latest})"),
|
||||||
|
)
|
||||||
|
.with_fix("chrome-use upgrade".to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.cli",
|
||||||
|
category,
|
||||||
|
Status::Pass,
|
||||||
|
format!("CLI {cli_version}"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extension — the build this CLI shipped alongside (embedded at compile time
|
||||||
|
// from the extension manifest) is what we expect to be running.
|
||||||
|
let expected_ext = env!("AB_CONNECT_VERSION");
|
||||||
|
match connect::relay_ext_version() {
|
||||||
|
Some(ext) if upgrade::version_is_newer(expected_ext, &ext) => {
|
||||||
|
checks.push(
|
||||||
|
Check::new(
|
||||||
|
"versions.extension",
|
||||||
|
category,
|
||||||
|
Status::Warn,
|
||||||
|
format!("extension {ext} is behind the bundled {expected_ext}"),
|
||||||
|
)
|
||||||
|
.with_fix(
|
||||||
|
"update ab-connect in Chrome: chrome://extensions \u{2192} reload \
|
||||||
|
(or wait for the Web Store auto-update)"
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Some(ext) => {
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.extension",
|
||||||
|
category,
|
||||||
|
Status::Pass,
|
||||||
|
format!("extension {ext}"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.extension",
|
||||||
|
category,
|
||||||
|
Status::Info,
|
||||||
|
format!(
|
||||||
|
"extension not connected (or it predates version reporting — \
|
||||||
|
expected {expected_ext})"
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skill — ships inside the same release artifact as the binary, so it's
|
||||||
|
// version-locked here. Copies made elsewhere via `skills add` aren't.
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.skill",
|
||||||
|
category,
|
||||||
|
Status::Info,
|
||||||
|
format!(
|
||||||
|
"skills bundled with this CLI ({cli_version}); copies made via `skills add` \
|
||||||
|
elsewhere may be stale — re-run to refresh"
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
+118
-2
@@ -29,8 +29,8 @@ use windows_sys::Win32::System::Threading::OpenProcess;
|
|||||||
|
|
||||||
use commands::{gen_id, parse_command, ParseError};
|
use commands::{gen_id, parse_command, ParseError};
|
||||||
use connection::{
|
use connection::{
|
||||||
cleanup_stale_files, ensure_daemon, get_socket_dir, is_pid_alive, send_command, walk_daemons,
|
cleanup_stale_files, ensure_daemon, get_socket_dir, is_pid_alive, restart_all_daemons,
|
||||||
DaemonOptions,
|
send_command, walk_daemons, DaemonOptions,
|
||||||
};
|
};
|
||||||
use flags::{clean_args, parse_flags, Flags};
|
use flags::{clean_args, parse_flags, Flags};
|
||||||
use install::run_install;
|
use install::run_install;
|
||||||
@@ -320,6 +320,94 @@ fn run_session(args: &[String], session: &str, json_mode: bool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `chrome-use daemon <restart|status>` — manage the per-session daemon workers
|
||||||
|
/// without resorting to `pgrep`/`kill`. `restart` clears corrupted or
|
||||||
|
/// cross-leaked daemon state (e.g. after a mid-session `chrome-use upgrade`
|
||||||
|
/// where stale tab handles bleed across sessions, issue #20) by killing every
|
||||||
|
/// session worker. The Chrome-launched `__nm-host` native-messaging bridge is
|
||||||
|
/// NOT a tracked session daemon, so the extension relay survives a restart —
|
||||||
|
/// the next command spins up a fresh, clean daemon against the same live Chrome.
|
||||||
|
fn run_daemon(args: &[String], json_mode: bool) {
|
||||||
|
match args.get(1).map(|s| s.as_str()) {
|
||||||
|
Some("restart") => {
|
||||||
|
let stopped = restart_all_daemons();
|
||||||
|
let relay_up = connect::relay_url().is_some();
|
||||||
|
if json_mode {
|
||||||
|
print_json_value(json!({
|
||||||
|
"success": true,
|
||||||
|
"data": { "stopped": stopped, "count": stopped.len(), "relay": relay_up },
|
||||||
|
}));
|
||||||
|
} else if stopped.is_empty() {
|
||||||
|
println!("No session daemons running — nothing to restart.");
|
||||||
|
if relay_up {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
color::dim("Extension relay still up; next command starts a fresh daemon.")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for s in &stopped {
|
||||||
|
println!("{} Stopped daemon: {}", color::green("✓"), s);
|
||||||
|
}
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
color::dim(if relay_up {
|
||||||
|
"Extension relay (__nm-host) left running; next command starts a fresh daemon."
|
||||||
|
} else {
|
||||||
|
"Next command starts a fresh daemon."
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some("status") | Some("list") => {
|
||||||
|
let inventory = walk_daemons();
|
||||||
|
let relay_up = connect::relay_url().is_some();
|
||||||
|
if json_mode {
|
||||||
|
let sessions: Vec<_> = inventory
|
||||||
|
.sessions
|
||||||
|
.iter()
|
||||||
|
.map(|s| json!({ "name": s.name, "pid": s.pid, "version": s.version }))
|
||||||
|
.collect();
|
||||||
|
print_json_value(json!({
|
||||||
|
"success": true,
|
||||||
|
"data": { "sessions": sessions, "relay": relay_up },
|
||||||
|
}));
|
||||||
|
} else if inventory.sessions.is_empty() {
|
||||||
|
println!("No session daemons running.");
|
||||||
|
if relay_up {
|
||||||
|
println!("{}", color::dim("Extension relay (__nm-host): up"));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
println!("Session daemons:");
|
||||||
|
for s in &inventory.sessions {
|
||||||
|
let ver = s
|
||||||
|
.version
|
||||||
|
.as_deref()
|
||||||
|
.map(|v| format!(" {}", color::dim(&format!("(v{})", v))))
|
||||||
|
.unwrap_or_default();
|
||||||
|
println!(" {} pid {}{}", s.name, s.pid, ver);
|
||||||
|
}
|
||||||
|
if relay_up {
|
||||||
|
println!("{}", color::dim("Extension relay (__nm-host): up"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
eprintln!(
|
||||||
|
"{} usage: chrome-use daemon <restart|status>",
|
||||||
|
color::error_indicator()
|
||||||
|
);
|
||||||
|
if let Some(unknown) = other {
|
||||||
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
color::dim(&format!(" unknown subcommand: {}", unknown))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn get_dashboard_pid_path() -> std::path::PathBuf {
|
fn get_dashboard_pid_path() -> std::path::PathBuf {
|
||||||
get_socket_dir().join("dashboard.pid")
|
get_socket_dir().join("dashboard.pid")
|
||||||
}
|
}
|
||||||
@@ -799,6 +887,20 @@ fn main() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handle daemon management (doesn't talk to a daemon — it manages them).
|
||||||
|
if clean.first().map(|s| s.as_str()) == Some("daemon") {
|
||||||
|
run_daemon(&clean, flags.json);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// `sessions` is a natural top-level guess for "list my sessions" (the skill
|
||||||
|
// advertises sessions as a feature) — route it to the daemon inventory the
|
||||||
|
// same way `daemon status` does (issue #29).
|
||||||
|
if clean.first().map(|s| s.as_str()) == Some("sessions") {
|
||||||
|
run_daemon(&["sessions".to_string(), "status".to_string()], flags.json);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Handle close --all: close all active sessions
|
// Handle close --all: close all active sessions
|
||||||
if matches!(
|
if matches!(
|
||||||
clean.first().map(|s| s.as_str()),
|
clean.first().map(|s| s.as_str()),
|
||||||
@@ -1267,6 +1369,20 @@ fn main() {
|
|||||||
&& flags.provider.is_none()
|
&& flags.provider.is_none()
|
||||||
&& (flags.force_launch || !flags.auto_connect)
|
&& (flags.force_launch || !flags.auto_connect)
|
||||||
{
|
{
|
||||||
|
// Launching a debug-port Chrome pops Chrome's "Allow remote debugging?"
|
||||||
|
// consent modal (Chrome 136+). When the ab-connect relay is already up,
|
||||||
|
// this is almost always unintended — the relay drives the user's real
|
||||||
|
// Chrome with NO modal. Warn so the modal is self-explained and the
|
||||||
|
// caller (often a stray --launch / --no-auto-connect) is fixable (#32).
|
||||||
|
if !flags.json && connect::relay_url().is_some() {
|
||||||
|
eprintln!(
|
||||||
|
"{} launching a new Chrome with a debug port — this pops Chrome's \
|
||||||
|
\"Allow remote debugging?\" modal.\n The ab-connect relay is up; \
|
||||||
|
drop --launch/--new (and don't pass --no-auto-connect) to drive your \
|
||||||
|
real Chrome with no modal.",
|
||||||
|
color::warning_indicator()
|
||||||
|
);
|
||||||
|
}
|
||||||
let mut launch_cmd = json!({
|
let mut launch_cmd = json!({
|
||||||
"id": gen_id(),
|
"id": gen_id(),
|
||||||
"action": "launch",
|
"action": "launch",
|
||||||
|
|||||||
+568
-27
@@ -1332,6 +1332,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
|||||||
"uncheck" => handle_uncheck(cmd, state).await,
|
"uncheck" => handle_uncheck(cmd, state).await,
|
||||||
"wait" => handle_wait(cmd, state).await,
|
"wait" => handle_wait(cmd, state).await,
|
||||||
"gettext" => handle_gettext(cmd, state).await,
|
"gettext" => handle_gettext(cmd, state).await,
|
||||||
|
"frames" => handle_frames(cmd, state).await,
|
||||||
"getattribute" => handle_getattribute(cmd, state).await,
|
"getattribute" => handle_getattribute(cmd, state).await,
|
||||||
"isvisible" => handle_isvisible(cmd, state).await,
|
"isvisible" => handle_isvisible(cmd, state).await,
|
||||||
"isenabled" => handle_isenabled(cmd, state).await,
|
"isenabled" => handle_isenabled(cmd, state).await,
|
||||||
@@ -1340,6 +1341,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
|||||||
"forward" => handle_forward(state).await,
|
"forward" => handle_forward(state).await,
|
||||||
"reload" => handle_reload(state).await,
|
"reload" => handle_reload(state).await,
|
||||||
"cookies_get" => handle_cookies_get(cmd, state).await,
|
"cookies_get" => handle_cookies_get(cmd, state).await,
|
||||||
|
"cf_status" => handle_cf_status(cmd, state).await,
|
||||||
"cookies_set" => handle_cookies_set(cmd, state).await,
|
"cookies_set" => handle_cookies_set(cmd, state).await,
|
||||||
"cookies_clear" => handle_cookies_clear(state).await,
|
"cookies_clear" => handle_cookies_clear(state).await,
|
||||||
"storage_get" => handle_storage_get(cmd, state).await,
|
"storage_get" => handle_storage_get(cmd, state).await,
|
||||||
@@ -1395,6 +1397,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
|||||||
"count" => handle_count(cmd, state).await,
|
"count" => handle_count(cmd, state).await,
|
||||||
"styles" => handle_styles(cmd, state).await,
|
"styles" => handle_styles(cmd, state).await,
|
||||||
"bringtofront" => handle_bringtofront(state).await,
|
"bringtofront" => handle_bringtofront(state).await,
|
||||||
|
"current" => handle_current(state).await,
|
||||||
"timezone" => handle_timezone(cmd, state).await,
|
"timezone" => handle_timezone(cmd, state).await,
|
||||||
"locale" => handle_locale(cmd, state).await,
|
"locale" => handle_locale(cmd, state).await,
|
||||||
"geolocation" => handle_geolocation(cmd, state).await,
|
"geolocation" => handle_geolocation(cmd, state).await,
|
||||||
@@ -2531,6 +2534,20 @@ async fn handle_navigate(cmd: &Value, state: &mut DaemonState) -> Result<Value,
|
|||||||
state.ref_map.clear();
|
state.ref_map.clear();
|
||||||
state.iframe_sessions.clear();
|
state.iframe_sessions.clear();
|
||||||
state.active_frame_id = None;
|
state.active_frame_id = None;
|
||||||
|
|
||||||
|
// `--reuse-tab`: if a tab already shows this URL (same origin+path), switch
|
||||||
|
// to it instead of navigating — preserves any in-page state and stops
|
||||||
|
// re-`open` from piling up duplicate tabs on rebind (issue #21).
|
||||||
|
if cmd
|
||||||
|
.get("reuseTab")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
if let Ok(Some(switched)) = mgr.reuse_tab_for_url(url).await {
|
||||||
|
return Ok(switched);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let result = mgr.navigate(url, wait_until).await?;
|
let result = mgr.navigate(url, wait_until).await?;
|
||||||
// Adaptive humanize: sample the freshly loaded page for known behavioural
|
// Adaptive humanize: sample the freshly loaded page for known behavioural
|
||||||
// anti-bot vendors and escalate this session to Human if any are present.
|
// anti-bot vendors and escalate this session to Human if any are present.
|
||||||
@@ -2874,7 +2891,32 @@ async fn handle_snapshot(cmd: &Value, state: &mut DaemonState) -> Result<Value,
|
|||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
Ok(json!({ "snapshot": tree, "origin": url, "refs": refs }))
|
let ref_count = refs.len();
|
||||||
|
let mut out = json!({ "snapshot": tree, "origin": url, "refs": refs });
|
||||||
|
|
||||||
|
// Canvas/WebGL apps (games, map/3D viewers, drawing tools) paint to a
|
||||||
|
// <canvas> and expose almost no accessibility tree, so `snapshot` comes back
|
||||||
|
// near-empty and agents get stuck looking for refs that will never exist
|
||||||
|
// (dogfood: the Dead Cell game). When the tree is sparse but a canvas
|
||||||
|
// dominates the viewport, tell them to switch to the screenshot-driven path.
|
||||||
|
if ref_count < 3 {
|
||||||
|
let canvas_js =
|
||||||
|
"(() => { const c = document.querySelector('canvas'); if (!c) return false; \
|
||||||
|
const r = c.getBoundingClientRect(); \
|
||||||
|
return r.width * r.height > innerWidth * innerHeight * 0.5; })()";
|
||||||
|
if let Ok(v) = mgr.evaluate(canvas_js, None).await {
|
||||||
|
if v.as_bool() == Some(true) {
|
||||||
|
out["note"] = json!(
|
||||||
|
"This page renders to a <canvas> (game / WebGL / editor) and exposes almost no \
|
||||||
|
accessibility tree — refs won't help. Use `screenshot` to see it, coordinate \
|
||||||
|
`click <x> <y>` to interact, and `keydown`/`keyup`/`press` for keyboard \
|
||||||
|
(hold-to-move: `keydown d` … `keyup d`)."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve a (possibly relative) saved-file path to an absolute one so the CLI
|
/// Resolve a (possibly relative) saved-file path to an absolute one so the CLI
|
||||||
@@ -2958,6 +3000,14 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
|||||||
.get("screenshotDir")
|
.get("screenshotDir")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.map(String::from),
|
.map(String::from),
|
||||||
|
clip: cmd.get("clip").and_then(|c| {
|
||||||
|
Some((
|
||||||
|
c.get("x")?.as_f64()?,
|
||||||
|
c.get("y")?.as_f64()?,
|
||||||
|
c.get("width")?.as_f64()?,
|
||||||
|
c.get("height")?.as_f64()?,
|
||||||
|
))
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
if annotate {
|
if annotate {
|
||||||
@@ -3077,6 +3127,15 @@ async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
|
|||||||
|
|
||||||
let button = cmd.get("button").and_then(|v| v.as_str()).unwrap_or("left");
|
let button = cmd.get("button").and_then(|v| v.as_str()).unwrap_or("left");
|
||||||
let click_count = cmd.get("clickCount").and_then(|v| v.as_i64()).unwrap_or(1) as i32;
|
let click_count = cmd.get("clickCount").and_then(|v| v.as_i64()).unwrap_or(1) as i32;
|
||||||
|
let follow = cmd.get("follow").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||||
|
|
||||||
|
// Snapshot tracked targets so we can tell if this click opened a NEW tab
|
||||||
|
// (target=_blank link / window.open). On the relay the new tab is discovered
|
||||||
|
// passively and doesn't steal focus (#7/#8.1), so without surfacing it the
|
||||||
|
// post-click snapshot shows the OLD page and looks like the click failed
|
||||||
|
// (issue #24-A).
|
||||||
|
let before: std::collections::HashSet<String> =
|
||||||
|
mgr.pages_list().into_iter().map(|p| p.target_id).collect();
|
||||||
|
|
||||||
interaction::click(
|
interaction::click(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
@@ -3089,7 +3148,26 @@ async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(json!({ "clicked": selector }))
|
// Give a just-opened tab a moment to register, then look for it.
|
||||||
|
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(150)).await;
|
||||||
|
let opened = mgr.adopt_newly_opened(&before).await;
|
||||||
|
|
||||||
|
let mut out = json!({ "clicked": selector });
|
||||||
|
if let Some(page) = opened {
|
||||||
|
let tab_id = super::browser::format_tab_id(page.tab_id);
|
||||||
|
out["openedTab"] = json!({ "tabId": tab_id, "url": page.url, "title": page.title });
|
||||||
|
// `--follow`: switch the active tab to the newly-opened one (default is
|
||||||
|
// to report it but stay put, so multi-tab flows aren't hijacked).
|
||||||
|
if follow {
|
||||||
|
state.ref_map.clear();
|
||||||
|
state.iframe_sessions.clear();
|
||||||
|
state.active_frame_id = None;
|
||||||
|
let _ = mgr.tab_switch_by_id(page.tab_id).await;
|
||||||
|
out["followed"] = json!(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_dblclick(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_dblclick(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
@@ -3147,6 +3225,14 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
|
||||||
|
// `--key-events`: dispatch real per-character keyDown/keyUp instead of
|
||||||
|
// Input.insertText, so autocomplete/combobox widgets that only react to key
|
||||||
|
// events fire (e.g. Google's address postal-code lookup) (issue #4/#36).
|
||||||
|
let key_events = cmd
|
||||||
|
.get("keyEvents")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
// `type --focused <text>`: type into the currently-focused element without a
|
// `type --focused <text>`: type into the currently-focused element without a
|
||||||
// selector (custom widgets that move focus to a hidden input on open).
|
// selector (custom widgets that move focus to a hidden input on open).
|
||||||
if cmd
|
if cmd
|
||||||
@@ -3158,7 +3244,14 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
.get("text")
|
.get("text")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'text' parameter")?;
|
.ok_or("Missing 'text' parameter")?;
|
||||||
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None).await?;
|
interaction::type_text_into_active_context(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
text,
|
||||||
|
None,
|
||||||
|
key_events,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
return Ok(json!({ "typed": text, "focused": true }));
|
return Ok(json!({ "typed": text, "focused": true }));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3182,6 +3275,7 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
clear,
|
clear,
|
||||||
delay,
|
delay,
|
||||||
&state.iframe_sessions,
|
&state.iframe_sessions,
|
||||||
|
key_events,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(json!({ "typed": text }))
|
Ok(json!({ "typed": text }))
|
||||||
@@ -3295,6 +3389,16 @@ async fn handle_press(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
|
|||||||
// Parse modifier+key chords like "Control+a", "Shift+Enter", "Control+Shift+a"
|
// Parse modifier+key chords like "Control+a", "Shift+Enter", "Control+Shift+a"
|
||||||
let (actual_key, modifiers) = parse_key_chord(key);
|
let (actual_key, modifiers) = parse_key_chord(key);
|
||||||
|
|
||||||
|
// `--hold <ms>`: keyDown, wait, keyUp — all inside the daemon so the hold
|
||||||
|
// duration is precise (no shell-sleep / round-trip jitter). For games
|
||||||
|
// (hold-to-move/charge) and any press-and-hold interaction.
|
||||||
|
if let Some(ms) = cmd.get("hold").and_then(|v| v.as_u64()) {
|
||||||
|
interaction::dispatch_single_key(&mgr.client, &session_id, &actual_key, "keyDown").await?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(ms)).await;
|
||||||
|
interaction::dispatch_single_key(&mgr.client, &session_id, &actual_key, "keyUp").await?;
|
||||||
|
return Ok(json!({ "pressed": key, "heldMs": ms }));
|
||||||
|
}
|
||||||
|
|
||||||
interaction::press_key_with_modifiers(&mgr.client, &session_id, &actual_key, modifiers).await?;
|
interaction::press_key_with_modifiers(&mgr.client, &session_id, &actual_key, modifiers).await?;
|
||||||
Ok(json!({ "pressed": key }))
|
Ok(json!({ "pressed": key }))
|
||||||
}
|
}
|
||||||
@@ -3379,17 +3483,182 @@ async fn handle_scroll(cmd: &Value, state: &mut DaemonState) -> Result<Value, St
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An explicit `--selector` keeps the precise element-scroll path (scrollBy on
|
||||||
|
// the resolved node, same-origin only).
|
||||||
|
if let Some(sel) = selector {
|
||||||
|
interaction::scroll(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.ref_map,
|
||||||
|
Some(sel),
|
||||||
|
dx,
|
||||||
|
dy,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(json!({ "scrolled": true, "via": "selector" }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// `--at x,y` / `--frame n`: dispatch a real (isTrusted) wheel at a viewport
|
||||||
|
// coordinate. This hits the compositor and scrolls whatever scroll container
|
||||||
|
// is under the pointer — including cross-origin iframes that `window.scrollBy`
|
||||||
|
// on the top document silently no-ops on (issue #36).
|
||||||
|
if cmd.get("at").is_some() || cmd.get("frame").is_some() {
|
||||||
|
let (x, y, via) = if let Some(at) = cmd.get("at").and_then(|v| v.as_array()) {
|
||||||
|
let x = at.first().and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
let y = at.get(1).and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
(x, y, "at")
|
||||||
|
} else {
|
||||||
|
let n = cmd.get("frame").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
let (x, y) = frame_center(mgr, &session_id, &state.iframe_sessions, n as usize).await?;
|
||||||
|
(x, y, "frame")
|
||||||
|
};
|
||||||
|
dispatch_wheel(&mgr.client, &session_id, x, y, dx, dy).await?;
|
||||||
|
return Ok(json!({ "scrolled": true, "via": via, "at": [x, y] }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default (no selector/at/frame): scroll the page with `window.scrollBy`. This
|
||||||
|
// is the reliable path for ordinary page scrolling; a coordinate wheel at the
|
||||||
|
// viewport centre is NOT a dependable substitute (it no-ops on some pages,
|
||||||
|
// e.g. headless), so the wheel stays opt-in via `--at`/`--frame` for the
|
||||||
|
// cross-origin-iframe case (issue #36).
|
||||||
interaction::scroll(
|
interaction::scroll(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
&session_id,
|
&session_id,
|
||||||
&state.ref_map,
|
&state.ref_map,
|
||||||
selector,
|
None,
|
||||||
dx,
|
dx,
|
||||||
dy,
|
dy,
|
||||||
&state.iframe_sessions,
|
&state.iframe_sessions,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(json!({ "scrolled": true }))
|
Ok(json!({ "scrolled": true, "via": "page" }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Viewport center in CSS pixels, used as the default wheel landing point for
|
||||||
|
/// `scroll` (issue #36). Falls back to a sane 640×400 center if the page can't
|
||||||
|
/// be evaluated (e.g. a restricted document).
|
||||||
|
async fn viewport_center(mgr: &BrowserManager, session_id: &str) -> Result<(f64, f64), String> {
|
||||||
|
let dims = mgr
|
||||||
|
.client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
&super::cdp::types::EvaluateParams {
|
||||||
|
expression: "[window.innerWidth, window.innerHeight]".to_string(),
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
let arr = dims
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("result"))
|
||||||
|
.and_then(|v| v.get("value"))
|
||||||
|
.and_then(|v| v.as_array());
|
||||||
|
let w = arr
|
||||||
|
.and_then(|a| a.first())
|
||||||
|
.and_then(|v| v.as_f64())
|
||||||
|
.filter(|w| *w > 0.0)
|
||||||
|
.unwrap_or(1280.0);
|
||||||
|
let h = arr
|
||||||
|
.and_then(|a| a.get(1))
|
||||||
|
.and_then(|v| v.as_f64())
|
||||||
|
.filter(|h| *h > 0.0)
|
||||||
|
.unwrap_or(800.0);
|
||||||
|
Ok((w / 2.0, h / 2.0))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Center of the `n`-th frame (as listed by `chrome-use frames`) in top-viewport
|
||||||
|
/// CSS pixels, so `scroll --frame n` lands its wheel inside a cross-origin iframe
|
||||||
|
/// without needing a selector into it (issue #36). Resolves the frame's owning
|
||||||
|
/// `<iframe>` element box via `DOM.getFrameOwner` + `DOM.getBoxModel` — exact for
|
||||||
|
/// a frame nested directly under the top document; for a deeper nesting the box is
|
||||||
|
/// relative to the intermediate frame, so prefer `--at x,y` from a screenshot.
|
||||||
|
async fn frame_center(
|
||||||
|
mgr: &BrowserManager,
|
||||||
|
session_id: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
n: usize,
|
||||||
|
) -> Result<(f64, f64), String> {
|
||||||
|
let frames =
|
||||||
|
super::element::collect_all_frames_text(&mgr.client, session_id, iframe_sessions).await?;
|
||||||
|
let frame = frames.get(n).ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"frame index {} out of range (run `chrome-use frames`: {} frame(s))",
|
||||||
|
n,
|
||||||
|
frames.len()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
if n == 0 {
|
||||||
|
// Frame 0 is the top document — there's no owner element; scroll its center.
|
||||||
|
return viewport_center(mgr, session_id).await;
|
||||||
|
}
|
||||||
|
let owner = mgr
|
||||||
|
.client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
&json!({ "frameId": frame.frame_id }),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("can't locate frame {}'s owner element: {}", n, e))?;
|
||||||
|
let backend_node_id = owner
|
||||||
|
.get("backendNodeId")
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
|
.ok_or_else(|| format!("frame {} has no owner <iframe> element", n))?;
|
||||||
|
let box_model = mgr
|
||||||
|
.client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"DOM.getBoxModel",
|
||||||
|
&json!({ "backendNodeId": backend_node_id }),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("can't measure frame {}'s box: {}", n, e))?;
|
||||||
|
let content = box_model
|
||||||
|
.get("model")
|
||||||
|
.and_then(|m| m.get("content"))
|
||||||
|
.and_then(|c| c.as_array())
|
||||||
|
.ok_or_else(|| format!("frame {} box model has no content quad", n))?;
|
||||||
|
let coord = |i: usize| content.get(i).and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
// content quad is [x1,y1, x2,y2, x3,y3, x4,y4]; opposite corners are 0 and 2.
|
||||||
|
let cx = (coord(0) + coord(4)) / 2.0;
|
||||||
|
let cy = (coord(1) + coord(5)) / 2.0;
|
||||||
|
Ok((cx, cy))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dispatch a trusted mouse wheel at `(x, y)`, humanized like `handle_wheel`.
|
||||||
|
async fn dispatch_wheel(
|
||||||
|
client: &super::cdp::client::CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
x: f64,
|
||||||
|
y: f64,
|
||||||
|
delta_x: f64,
|
||||||
|
delta_y: f64,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let level = humanize::active_level();
|
||||||
|
let seed = humanize::next_seed();
|
||||||
|
for (dx, dy, delay) in humanize::scroll_segments(delta_x, delta_y, level, seed) {
|
||||||
|
client
|
||||||
|
.send_command(
|
||||||
|
"Input.dispatchMouseEvent",
|
||||||
|
Some(json!({
|
||||||
|
"type": "mouseWheel",
|
||||||
|
"x": x,
|
||||||
|
"y": y,
|
||||||
|
"deltaX": dx,
|
||||||
|
"deltaY": dy,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if !delay.is_zero() {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_select(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_select(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
@@ -3516,6 +3785,56 @@ async fn handle_wait(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
|
||||||
|
// `get text --all-frames` aggregates visible text across every frame the
|
||||||
|
// session can reach — including out-of-process iframes that never show up
|
||||||
|
// in the top document (#27: Yahoo/Rakuten/Mercari listing descriptions).
|
||||||
|
if cmd.get("allFrames").and_then(|v| v.as_bool()) == Some(true) {
|
||||||
|
let frames = super::element::collect_all_frames_text(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let mut combined = String::new();
|
||||||
|
let mut frame_count = 0usize;
|
||||||
|
for f in &frames {
|
||||||
|
let t = f.text.trim();
|
||||||
|
if t.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
frame_count += 1;
|
||||||
|
if f.kind != "top" {
|
||||||
|
combined.push_str(&format!("\n\n----- frame [{}] {} -----\n", f.kind, f.url));
|
||||||
|
}
|
||||||
|
combined.push_str(t);
|
||||||
|
}
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
return Ok(json!({
|
||||||
|
"text": combined,
|
||||||
|
"origin": url,
|
||||||
|
"frames": frame_count,
|
||||||
|
"allFrames": true,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// `get text --pierce` reads text through CLOSED shadow roots and child
|
||||||
|
// documents via the CDP DOM tree — content `innerText`/`eval` can't see,
|
||||||
|
// e.g. an extension's injected panel in a closed shadow DOM (#30).
|
||||||
|
if cmd.get("pierce").and_then(|v| v.as_bool()) == Some(true) {
|
||||||
|
let text = super::element::get_pierced_text(&mgr.client, &session_id).await?;
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
return Ok(json!({ "text": text, "origin": url, "pierce": true }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// `get text --main` returns the page's main-content region (readability-lite),
|
||||||
|
// skipping global header/nav/footer/sidebar boilerplate (#27).
|
||||||
|
if cmd.get("main").and_then(|v| v.as_bool()) == Some(true) {
|
||||||
|
let text = super::element::get_main_content_text(&mgr.client, &session_id).await?;
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
return Ok(json!({ "text": text, "origin": url, "main": true }));
|
||||||
|
}
|
||||||
|
|
||||||
let selector = cmd
|
let selector = cmd
|
||||||
.get("selector")
|
.get("selector")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
@@ -3533,6 +3852,29 @@ async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, S
|
|||||||
Ok(json!({ "text": text, "origin": url }))
|
Ok(json!({ "text": text, "origin": url }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn handle_frames(_cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
let frames =
|
||||||
|
super::element::collect_all_frames_text(&mgr.client, &session_id, &state.iframe_sessions)
|
||||||
|
.await?;
|
||||||
|
let list: Vec<Value> = frames
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, f)| {
|
||||||
|
json!({
|
||||||
|
"index": i,
|
||||||
|
"kind": f.kind,
|
||||||
|
"url": f.url,
|
||||||
|
"frameId": f.frame_id,
|
||||||
|
"textLen": f.text.trim().chars().count(),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
Ok(json!({ "frames": list, "count": list.len(), "origin": url }))
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
@@ -3967,6 +4309,111 @@ async fn handle_cookies_clear(state: &DaemonState) -> Result<Value, String> {
|
|||||||
Ok(json!({ "cleared": true }))
|
Ok(json!({ "cleared": true }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Detect whether the active page is *currently* a Cloudflare challenge
|
||||||
|
// (the full-page "Just a moment…" / "正在进行安全验证" interstitial), so an agent
|
||||||
|
// knows whether it must solve or can proceed. Runs in the top frame main world.
|
||||||
|
const CF_CHALLENGE_JS: &str = r#"(function(){
|
||||||
|
var t = document.title || '';
|
||||||
|
var challenged =
|
||||||
|
/just a moment|attention required|checking (your|if)|verify you are human|正在进行安全验证|安全验证|请稍候|请完成|人机验证/i.test(t) ||
|
||||||
|
!!document.querySelector('#challenge-form, #challenge-running, #cf-challenge-running, [id^="cf-chl"], script[src*="/cdn-cgi/challenge-platform/"]');
|
||||||
|
var turnstile = !!document.querySelector('.cf-turnstile, [data-sitekey]');
|
||||||
|
return JSON.stringify({ title: t, challenged: challenged, turnstile: turnstile, readyState: document.readyState });
|
||||||
|
})()"#;
|
||||||
|
|
||||||
|
/// Recommendation for a Cloudflare-gated page, from the current challenge state
|
||||||
|
/// and whether a still-valid `cf_clearance` exists. Pure so it's unit-testable.
|
||||||
|
/// - not challenged → "proceed" (the page is cleared/loaded)
|
||||||
|
/// - challenged, valid cookie → "reissue" (clearance present but page still
|
||||||
|
/// blocks → it's stale or the IP/UA no longer matches what it was issued for)
|
||||||
|
/// - challenged, no cookie → "solve"
|
||||||
|
fn cf_recommendation(challenged: bool, clearance_valid: bool) -> &'static str {
|
||||||
|
if !challenged {
|
||||||
|
"proceed"
|
||||||
|
} else if clearance_valid {
|
||||||
|
"reissue"
|
||||||
|
} else {
|
||||||
|
"solve"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `cf_clearance` validity for a cookie's expiry (epoch seconds; <=0 = session
|
||||||
|
/// cookie, treated as non-expiring). Returns (present, expired). Pure.
|
||||||
|
fn clearance_state(expires: Option<f64>, now: f64) -> (bool, bool) {
|
||||||
|
match expires {
|
||||||
|
None => (false, false),
|
||||||
|
Some(e) if e <= 0.0 => (true, false), // session cookie: no expiry
|
||||||
|
Some(e) => (true, e < now),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_cf_status(_cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
|
||||||
|
// 1. Is the page a Cloudflare challenge right now?
|
||||||
|
let probe_raw = mgr
|
||||||
|
.evaluate(CF_CHALLENGE_JS, None)
|
||||||
|
.await
|
||||||
|
.unwrap_or(Value::Null);
|
||||||
|
let probe = parse_json_string(probe_raw, "cf challenge probe").unwrap_or(Value::Null);
|
||||||
|
let challenged = probe
|
||||||
|
.get("challenged")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let turnstile = probe
|
||||||
|
.get("turnstile")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let title = probe
|
||||||
|
.get("title")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
// 2. Persistence artifacts: cf_clearance (HttpOnly → must read via CDP, not
|
||||||
|
// document.cookie) + CF_VERIFIED_DEVICE. Scope to the current URL.
|
||||||
|
let urls = if url.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(vec![url.clone()])
|
||||||
|
};
|
||||||
|
let cookies = super::cookies::get_cookies(&mgr.client, &session_id, urls)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
let clearance = cookies.iter().find(|c| c.name == "cf_clearance");
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs_f64())
|
||||||
|
.unwrap_or(0.0);
|
||||||
|
let (present, expired) = clearance_state(clearance.map(|c| c.expires), now);
|
||||||
|
let expires_in = clearance
|
||||||
|
.filter(|_| present && !expired)
|
||||||
|
.map(|c| (c.expires - now).max(0.0) as i64);
|
||||||
|
let device_verified = cookies
|
||||||
|
.iter()
|
||||||
|
.any(|c| c.name.starts_with("CF_VERIFIED_DEVICE"));
|
||||||
|
|
||||||
|
let clearance_valid = present && !expired;
|
||||||
|
let recommendation = cf_recommendation(challenged, clearance_valid);
|
||||||
|
|
||||||
|
Ok(json!({
|
||||||
|
"url": url,
|
||||||
|
"title": title,
|
||||||
|
"challenged": challenged,
|
||||||
|
"turnstile": turnstile,
|
||||||
|
"clearance": {
|
||||||
|
"present": present,
|
||||||
|
"expired": expired,
|
||||||
|
"expiresIn": expires_in,
|
||||||
|
"httpOnly": clearance.map(|c| c.http_only).unwrap_or(false),
|
||||||
|
},
|
||||||
|
"deviceVerified": device_verified,
|
||||||
|
"recommendation": recommendation,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_storage_get(cmd: &Value, state: &DaemonState) -> Result<Value, String> {
|
async fn handle_storage_get(cmd: &Value, state: &DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
@@ -4304,8 +4751,18 @@ async fn handle_keyboard(cmd: &Value, state: &DaemonState) -> Result<Value, Stri
|
|||||||
.get("text")
|
.get("text")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'text' parameter")?;
|
.ok_or("Missing 'text' parameter")?;
|
||||||
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None)
|
let key_events = cmd
|
||||||
.await?;
|
.get("keyEvents")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
interaction::type_text_into_active_context(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
text,
|
||||||
|
None,
|
||||||
|
key_events,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
return Ok(json!({ "typed": text }));
|
return Ok(json!({ "typed": text }));
|
||||||
}
|
}
|
||||||
Some("insertText") => {
|
Some("insertText") => {
|
||||||
@@ -4355,8 +4812,12 @@ async fn handle_keyboard(cmd: &Value, state: &DaemonState) -> Result<Value, Stri
|
|||||||
// Phase 5 handlers
|
// Phase 5 handlers
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async fn handle_tab_list(cmd: &Value, state: &DaemonState) -> Result<Value, String> {
|
async fn handle_tab_list(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
|
||||||
|
// Re-sync with the live browser so the list reflects tabs opened by other
|
||||||
|
// sessions or re-attached after a cross-process nav, and drops gone ones
|
||||||
|
// (issue #21). Best-effort: a stale list still beats erroring the command.
|
||||||
|
mgr.resync_targets().await.ok();
|
||||||
let tabs = mgr.tab_list();
|
let tabs = mgr.tab_list();
|
||||||
// Echo `full` so the formatter prints untruncated URLs (issue #19).
|
// Echo `full` so the formatter prints untruncated URLs (issue #19).
|
||||||
if cmd.get("full").and_then(|v| v.as_bool()).unwrap_or(false) {
|
if cmd.get("full").and_then(|v| v.as_bool()).unwrap_or(false) {
|
||||||
@@ -4394,13 +4855,50 @@ async fn handle_tab_switch(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
|||||||
let tab_ref_str = cmd
|
let tab_ref_str = cmd
|
||||||
.get("tabId")
|
.get("tabId")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'tabId' parameter (expected `t<N>` or a label)")?;
|
.ok_or("Missing 'tabId' parameter (expected `t<N>`, a label, or a targetId)")?;
|
||||||
let tab_ref = super::browser::TabRef::parse(tab_ref_str)?;
|
// Re-sync first so a tab opened by another session, or one that re-attached
|
||||||
let tab_id = mgr.resolve_tab_ref(&tab_ref)?;
|
// after a cross-process nav, is adoptable from here (issue #21).
|
||||||
|
mgr.resync_targets().await.ok();
|
||||||
|
// A CDP `targetId` (shown in `tab list`) is stable across sessions, so accept
|
||||||
|
// it directly for adopting a specific pre-existing tab — falling back to the
|
||||||
|
// per-session `t<N>` / label form.
|
||||||
|
let tab_id = match mgr.tab_id_for_target(tab_ref_str) {
|
||||||
|
Some(id) => id,
|
||||||
|
None => {
|
||||||
|
let tab_ref = super::browser::TabRef::parse(tab_ref_str)?;
|
||||||
|
mgr.resolve_tab_ref(&tab_ref)?
|
||||||
|
}
|
||||||
|
};
|
||||||
state.ref_map.clear();
|
state.ref_map.clear();
|
||||||
state.iframe_sessions.clear();
|
state.iframe_sessions.clear();
|
||||||
state.active_frame_id = None;
|
state.active_frame_id = None;
|
||||||
let result = mgr.tab_switch_by_id(tab_id).await?;
|
let mut result = mgr.tab_switch_by_id(tab_id).await?;
|
||||||
|
|
||||||
|
// Liveness probe: confirm the new session actually answers before we report
|
||||||
|
// success, so `tab <id>` doesn't print a misleading ✓ for a session that's
|
||||||
|
// stale and will fail on the very next command (issue #29.3). On the churned
|
||||||
|
// -tabId case the ext-0.4.9 targetId recovery (#24) self-heals within ~6s, so
|
||||||
|
// we surface a warning rather than a hard error to avoid a false failure
|
||||||
|
// during that window.
|
||||||
|
if mgr.evaluate("1", None).await.is_err() {
|
||||||
|
if let Some(obj) = result.as_object_mut() {
|
||||||
|
obj.insert(
|
||||||
|
"warning".to_string(),
|
||||||
|
json!("switched tab is not responding yet (session re-attaching); retry the next command"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `--activate`: raise this tab to the foreground (the switch made it active;
|
||||||
|
// bring_to_front acts on the active tab) — for handing a specific tab to the
|
||||||
|
// human (issue #24-C). Best-effort; don't fail the switch if it can't.
|
||||||
|
if cmd
|
||||||
|
.get("activate")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
let _ = mgr.bring_to_front().await;
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(ref server) = state.stream_server {
|
if let Some(ref server) = state.stream_server {
|
||||||
if let Ok(dims) = mgr
|
if let Ok(dims) = mgr
|
||||||
@@ -5230,6 +5728,18 @@ async fn handle_bringtofront(state: &DaemonState) -> Result<Value, String> {
|
|||||||
Ok(json!({ "broughtToFront": true }))
|
Ok(json!({ "broughtToFront": true }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn handle_current(state: &mut DaemonState) -> Result<Value, String> {
|
||||||
|
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
|
||||||
|
// Refresh so `current` reflects the live URL/title even after a cross-process
|
||||||
|
// nav (the relay's cached target_info can lag) (#26).
|
||||||
|
mgr.resync_targets().await.ok();
|
||||||
|
let mut info = mgr.active_page_info().ok_or("No active tab")?;
|
||||||
|
if let Some(obj) = info.as_object_mut() {
|
||||||
|
obj.insert("current".to_string(), json!(true));
|
||||||
|
}
|
||||||
|
Ok(info)
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_timezone(cmd: &Value, state: &DaemonState) -> Result<Value, String> {
|
async fn handle_timezone(cmd: &Value, state: &DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let timezone = cmd
|
let timezone = cmd
|
||||||
@@ -6781,6 +7291,28 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'target' parameter")?;
|
.ok_or("Missing 'target' parameter")?;
|
||||||
|
|
||||||
|
// Over the relay (or into an iframe) a coordinate drag drifts to the
|
||||||
|
// foreground tab and can't reach an OOPIF — DOM-dispatch an HTML5 drag in the
|
||||||
|
// element's own session instead (issues #31/#36). `coord` mode forces the
|
||||||
|
// coordinate path for pointer-driven drags (canvas/sliders) on a launched
|
||||||
|
// browser.
|
||||||
|
if std::env::var("AGENT_BROWSER_CLICK_MODE").as_deref() != Ok("coord")
|
||||||
|
&& (crate::connect::relay_url().is_some()
|
||||||
|
|| state.ref_map.ref_is_in_iframe(source)
|
||||||
|
|| state.ref_map.ref_is_in_iframe(target))
|
||||||
|
{
|
||||||
|
super::interaction::dom_drag(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.ref_map,
|
||||||
|
source,
|
||||||
|
target,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(json!({ "dragged": { "source": source, "target": target }, "via": "dom" }));
|
||||||
|
}
|
||||||
|
|
||||||
let (sx, sy, _, _, source_session_id) = super::element::resolve_element_center(
|
let (sx, sy, _, _, source_session_id) = super::element::resolve_element_center(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
&session_id,
|
&session_id,
|
||||||
@@ -7135,6 +7667,7 @@ async fn handle_diff_screenshot(cmd: &Value, state: &DaemonState) -> Result<Valu
|
|||||||
quality: None,
|
quality: None,
|
||||||
annotate: false,
|
annotate: false,
|
||||||
output_dir: None,
|
output_dir: None,
|
||||||
|
clip: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
let result = screenshot::take_screenshot(
|
let result = screenshot::take_screenshot(
|
||||||
@@ -8731,13 +9264,7 @@ async fn handle_keydown(cmd: &Value, state: &DaemonState) -> Result<Value, Strin
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'key' parameter")?;
|
.ok_or("Missing 'key' parameter")?;
|
||||||
|
|
||||||
mgr.client
|
interaction::dispatch_single_key(&mgr.client, &session_id, key, "keyDown").await?;
|
||||||
.send_command(
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
Some(json!({ "type": "keyDown", "key": key })),
|
|
||||||
Some(&session_id),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
Ok(json!({ "keydown": key }))
|
Ok(json!({ "keydown": key }))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -8749,13 +9276,7 @@ async fn handle_keyup(cmd: &Value, state: &DaemonState) -> Result<Value, String>
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'key' parameter")?;
|
.ok_or("Missing 'key' parameter")?;
|
||||||
|
|
||||||
mgr.client
|
interaction::dispatch_single_key(&mgr.client, &session_id, key, "keyUp").await?;
|
||||||
.send_command(
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
Some(json!({ "type": "keyUp", "key": key })),
|
|
||||||
Some(&session_id),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
Ok(json!({ "keyup": key }))
|
Ok(json!({ "keyup": key }))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -8873,6 +9394,26 @@ mod tests {
|
|||||||
use crate::test_utils::EnvGuard;
|
use crate::test_utils::EnvGuard;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cf_recommendation() {
|
||||||
|
assert_eq!(cf_recommendation(false, false), "proceed");
|
||||||
|
assert_eq!(cf_recommendation(false, true), "proceed");
|
||||||
|
assert_eq!(cf_recommendation(true, false), "solve");
|
||||||
|
assert_eq!(cf_recommendation(true, true), "reissue");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_clearance_state() {
|
||||||
|
// no cookie
|
||||||
|
assert_eq!(clearance_state(None, 1000.0), (false, false));
|
||||||
|
// session cookie (expires <= 0) → present, never expired
|
||||||
|
assert_eq!(clearance_state(Some(-1.0), 1000.0), (true, false));
|
||||||
|
// valid: expiry in the future
|
||||||
|
assert_eq!(clearance_state(Some(2000.0), 1000.0), (true, false));
|
||||||
|
// expired: expiry in the past
|
||||||
|
assert_eq!(clearance_state(Some(500.0), 1000.0), (true, true));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_url_glob_to_regex() {
|
fn test_url_glob_to_regex() {
|
||||||
assert_eq!(url_glob_to_regex("**/dashboard"), "^.*/dashboard$");
|
assert_eq!(url_glob_to_regex("**/dashboard"), "^.*/dashboard$");
|
||||||
|
|||||||
+923
-45
File diff suppressed because it is too large
Load Diff
@@ -35,6 +35,7 @@ fn native_test_fixture_html(name: &str) -> &'static str {
|
|||||||
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
|
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
|
||||||
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
|
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
|
||||||
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
|
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
|
||||||
|
"iframe_button_probe" => include_str!("test_fixtures/iframe_button_probe.html"),
|
||||||
_ => panic!("Unknown native test fixture: {}", name),
|
_ => panic!("Unknown native test fixture: {}", name),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -573,6 +574,76 @@ async fn e2e_snapshot_and_click_ref() {
|
|||||||
assert_success(&resp);
|
assert_success(&resp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clicking a button INSIDE an iframe by `@ref` must deliver a TRUSTED activation
|
||||||
|
/// (`event.isTrusted === true`), not a synthetic DOM `.click()`. Security-sensitive
|
||||||
|
/// embedded forms (Google Payments' `保存`) reject `isTrusted:false` clicks, so an
|
||||||
|
/// enabled submit button silently no-op'd (issue #39). The fix routes iframe-ref
|
||||||
|
/// clicks to a real `Input.dispatchMouseEvent` on the element's own frame session.
|
||||||
|
/// The fixture's iframe button writes `clicked:<isTrusted>` into its own text on
|
||||||
|
/// click, which the cross-frame snapshot reads back.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore]
|
||||||
|
async fn e2e_iframe_button_click_is_trusted() {
|
||||||
|
let mut state = DaemonState::new();
|
||||||
|
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "1", "action": "launch", "headless": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "2", "action": "navigate", "url": native_test_fixture_url("iframe_button_probe") }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
// Snapshot (interactive) — the button lives in the iframe and must appear with
|
||||||
|
// a ref; that ref carries the frame_id so the click resolves into the frame.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "3", "action": "snapshot", "interactive": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
let snapshot = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||||
|
let ref_id = snapshot
|
||||||
|
.lines()
|
||||||
|
.find(|l| l.contains("button \"save\""))
|
||||||
|
.and_then(|l| l.split("ref=").nth(1))
|
||||||
|
.map(|r| r.trim_end_matches(']').trim())
|
||||||
|
.unwrap_or_else(|| panic!("iframe button not found in snapshot:\n{snapshot}"));
|
||||||
|
|
||||||
|
// Click it by ref.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "4", "action": "click", "selector": ref_id }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(300)).await;
|
||||||
|
|
||||||
|
// The button rewrote its own text with the click's isTrusted flag; read it
|
||||||
|
// back across frames.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "5", "action": "snapshot", "interactive": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
let after = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||||
|
assert!(
|
||||||
|
after.contains("clicked:true"),
|
||||||
|
"iframe button click must be trusted (isTrusted:true); snapshot:\n{after}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
|
||||||
|
assert_success(&resp);
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Screenshot
|
// Screenshot
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
+351
-8
@@ -100,6 +100,15 @@ impl RefMap {
|
|||||||
self.map.get(ref_id)
|
self.map.get(ref_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether `selector_or_ref` is a `@ref` whose snapshot entry lives inside an
|
||||||
|
/// iframe (has a `frame_id`). Pointer interactions use this to choose
|
||||||
|
/// DOM-dispatch over coordinates for OOPIF elements (issue #36).
|
||||||
|
pub fn ref_is_in_iframe(&self, selector_or_ref: &str) -> bool {
|
||||||
|
parse_ref(selector_or_ref)
|
||||||
|
.and_then(|r| self.map.get(&r).map(|e| e.frame_id.is_some()))
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn entries_sorted(&self) -> Vec<(String, RefEntry)> {
|
pub fn entries_sorted(&self) -> Vec<(String, RefEntry)> {
|
||||||
let mut entries = self
|
let mut entries = self
|
||||||
.map
|
.map
|
||||||
@@ -796,16 +805,43 @@ pub(super) fn extract_ax_string(value: &Option<AXValue>) -> String {
|
|||||||
/// Build a JS expression that finds a DOM element by CSS selector or XPath.
|
/// Build a JS expression that finds a DOM element by CSS selector or XPath.
|
||||||
fn build_find_element_js(selector: &str) -> String {
|
fn build_find_element_js(selector: &str) -> String {
|
||||||
if let Some(xpath) = selector.strip_prefix("xpath=") {
|
if let Some(xpath) = selector.strip_prefix("xpath=") {
|
||||||
format!(
|
return format!(
|
||||||
"document.evaluate({}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue",
|
"document.evaluate({}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue",
|
||||||
serde_json::to_string(xpath).unwrap_or_default()
|
serde_json::to_string(xpath).unwrap_or_default()
|
||||||
)
|
);
|
||||||
} else {
|
|
||||||
format!(
|
|
||||||
"document.querySelector({})",
|
|
||||||
serde_json::to_string(selector).unwrap_or_default()
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
// Bare string (or explicit `text=`): try CSS first, then fall back to
|
||||||
|
// matching an interactive element by its VISIBLE TEXT. snapshot exposes
|
||||||
|
// buttons/links by their name, so `click "購入手続きへ"` should resolve by
|
||||||
|
// that label — previously it was fed straight to `querySelector` as CSS and
|
||||||
|
// failed as an invalid selector even though the button was right there
|
||||||
|
// (issue #24-B). CSS still wins when it matches, so existing selectors are
|
||||||
|
// unaffected; nested/non-ASCII labels now resolve too.
|
||||||
|
let text_only = selector.strip_prefix("text=");
|
||||||
|
let force_text = text_only.is_some();
|
||||||
|
let sel_json = serde_json::to_string(selector).unwrap_or_default();
|
||||||
|
let want_json = serde_json::to_string(text_only.unwrap_or(selector)).unwrap_or_default();
|
||||||
|
format!(
|
||||||
|
r#"(() => {{
|
||||||
|
const sel = {sel};
|
||||||
|
const css = {force_text} ? null : (() => {{ try {{ return document.querySelector(sel); }} catch (_e) {{ return null; }} }})();
|
||||||
|
if (css) return css;
|
||||||
|
const norm = s => (s == null ? '' : String(s)).replace(/\s+/g, ' ').trim();
|
||||||
|
const w = norm({want}); if (!w) return null;
|
||||||
|
const wl = w.toLowerCase();
|
||||||
|
const interactive = Array.from(document.querySelectorAll(
|
||||||
|
'button,a,[role=button],[role=link],[role=menuitem],[role=tab],[role=option],input[type=submit],input[type=button],input[type=reset],summary,label,[onclick]'));
|
||||||
|
const textOf = e => norm(e.innerText || e.textContent) || norm(e.value) ||
|
||||||
|
norm(e.getAttribute && e.getAttribute('aria-label')) || norm(e.getAttribute && e.getAttribute('title'));
|
||||||
|
let hit = interactive.find(e => textOf(e) === w) || interactive.find(e => textOf(e).toLowerCase().includes(wl));
|
||||||
|
if (hit) return hit;
|
||||||
|
const leaves = Array.from(document.querySelectorAll('*')).filter(e => !e.children.length);
|
||||||
|
return leaves.find(e => norm(e.textContent) === w) || leaves.find(e => norm(e.textContent).toLowerCase().includes(wl)) || null;
|
||||||
|
}})()"#,
|
||||||
|
sel = sel_json,
|
||||||
|
want = want_json,
|
||||||
|
force_text = force_text
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a JS expression that counts matching DOM elements by CSS selector or XPath.
|
/// Build a JS expression that counts matching DOM elements by CSS selector or XPath.
|
||||||
@@ -948,6 +984,268 @@ pub async fn get_element_text(
|
|||||||
.unwrap_or_default())
|
.unwrap_or_default())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Text content collected from a single frame of the page.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct FrameText {
|
||||||
|
pub frame_id: String,
|
||||||
|
pub url: String,
|
||||||
|
/// "top" | "inline" (same-process child frame) | "oopif" (out-of-process).
|
||||||
|
pub kind: &'static str,
|
||||||
|
pub text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// The expression we run in every frame to read its visible text. innerText
|
||||||
|
// honors CSS visibility (skips display:none), textContent is the fallback.
|
||||||
|
const FRAME_INNERTEXT_JS: &str = "(function(){try{var b=document.body||document.documentElement;return b?(b.innerText||b.textContent||''):'';}catch(e){return '';}})()";
|
||||||
|
|
||||||
|
async fn eval_text_default(client: &CdpClient, session_id: &str) -> String {
|
||||||
|
let res = client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(serde_json::json!({
|
||||||
|
"expression": FRAME_INNERTEXT_JS,
|
||||||
|
"returnByValue": true,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
res.ok()
|
||||||
|
.and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned())
|
||||||
|
.and_then(|v| v.as_str().map(|s| s.to_string()))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same-process child frames share the top renderer but live in their own
|
||||||
|
// execution context. Page.createIsolatedWorld hands us a context id bound to
|
||||||
|
// that frame so Runtime.evaluate reads the child document, not the parent.
|
||||||
|
async fn eval_text_in_frame(client: &CdpClient, session_id: &str, frame_id: &str) -> String {
|
||||||
|
let ctx = client
|
||||||
|
.send_command(
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
Some(serde_json::json!({ "frameId": frame_id, "worldName": "chrome_use_text" })),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.get("executionContextId").and_then(|c| c.as_i64()));
|
||||||
|
let Some(ctx_id) = ctx else {
|
||||||
|
return String::new();
|
||||||
|
};
|
||||||
|
let res = client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(serde_json::json!({
|
||||||
|
"expression": FRAME_INNERTEXT_JS,
|
||||||
|
"returnByValue": true,
|
||||||
|
"contextId": ctx_id,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
res.ok()
|
||||||
|
.and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned())
|
||||||
|
.and_then(|v| v.as_str().map(|s| s.to_string()))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flatten_frame_tree(node: &Value, is_top: bool, out: &mut Vec<(String, String, bool)>) {
|
||||||
|
if let Some(frame) = node.get("frame") {
|
||||||
|
if let Some(id) = frame.get("id").and_then(|v| v.as_str()) {
|
||||||
|
let url = frame
|
||||||
|
.get("url")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
out.push((id.to_string(), url, is_top));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(children) = node.get("childFrames").and_then(|v| v.as_array()) {
|
||||||
|
for child in children {
|
||||||
|
flatten_frame_tree(child, false, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collect visible text from every frame reachable in the active session,
|
||||||
|
/// including out-of-process iframes (which never appear in the top frame's
|
||||||
|
/// `Page.getFrameTree` and so are invisible to `document.body.innerText`).
|
||||||
|
///
|
||||||
|
/// Same-process child frames are read through `Page.createIsolatedWorld`;
|
||||||
|
/// OOPIFs are read through their own auto-attached debugger session
|
||||||
|
/// (`iframe_sessions`, keyed by frameId == targetId). This is the engine
|
||||||
|
/// behind `get text --all-frames` and `chrome-use frames` — the fix for
|
||||||
|
/// listing/marketplace pages whose description lives in a child frame (#27).
|
||||||
|
pub async fn collect_all_frames_text(
|
||||||
|
client: &CdpClient,
|
||||||
|
top_session: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<Vec<FrameText>, String> {
|
||||||
|
let mut out: Vec<FrameText> = Vec::new();
|
||||||
|
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
// 1. Top session: the top frame plus its same-process descendants. OOPIF
|
||||||
|
// frames that happen to surface here are skipped — they're read via
|
||||||
|
// their dedicated session in step 2 (cross-process isolated worlds fail).
|
||||||
|
let tree = client
|
||||||
|
.send_command_no_params("Page.getFrameTree", Some(top_session))
|
||||||
|
.await?;
|
||||||
|
let mut frames: Vec<(String, String, bool)> = Vec::new();
|
||||||
|
flatten_frame_tree(&tree["frameTree"], true, &mut frames);
|
||||||
|
for (fid, url, is_top) in frames {
|
||||||
|
if iframe_sessions.contains_key(&fid) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !seen.insert(fid.clone()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let (kind, text) = if is_top {
|
||||||
|
("top", eval_text_default(client, top_session).await)
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
"inline",
|
||||||
|
eval_text_in_frame(client, top_session, &fid).await,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
out.push(FrameText {
|
||||||
|
frame_id: fid,
|
||||||
|
url,
|
||||||
|
kind,
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Each out-of-process iframe, read through its own session.
|
||||||
|
for (fid, sid) in iframe_sessions {
|
||||||
|
if !seen.insert(fid.clone()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let url = client
|
||||||
|
.send_command_no_params("Page.getFrameTree", Some(sid))
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.and_then(|t| {
|
||||||
|
t.get("frameTree")
|
||||||
|
.and_then(|ft| ft.get("frame"))
|
||||||
|
.and_then(|f| f.get("url"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let text = eval_text_default(client, sid).await;
|
||||||
|
out.push(FrameText {
|
||||||
|
frame_id: fid.clone(),
|
||||||
|
url,
|
||||||
|
kind: "oopif",
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Readability-lite: prefer the page's semantic main-content region over the
|
||||||
|
// whole body so global header/nav/footer chrome (and, on many listing pages,
|
||||||
|
// the "related items" sidebar) doesn't drown out the actual content. Runs on
|
||||||
|
// the live, rendered tree (innerText needs layout — a detached clone returns
|
||||||
|
// empty), so we pick the densest <main>/<article> region rather than cloning
|
||||||
|
// and stripping. Falls back to <body> when no substantial main region exists.
|
||||||
|
const MAIN_CONTENT_JS: &str = r#"(function(){
|
||||||
|
function txt(el){try{return (el.innerText||'').trim();}catch(e){return '';}}
|
||||||
|
var sels=['main','[role=main]','article','#main','#contents','#l-content'];
|
||||||
|
var best=null,bestLen=0;
|
||||||
|
for(var i=0;i<sels.length;i++){
|
||||||
|
var els=document.querySelectorAll(sels[i]);
|
||||||
|
for(var j=0;j<els.length;j++){var l=txt(els[j]).length;if(l>bestLen){bestLen=l;best=els[j];}}
|
||||||
|
}
|
||||||
|
if(best&&bestLen>200)return txt(best);
|
||||||
|
return txt(document.body);
|
||||||
|
})()"#;
|
||||||
|
|
||||||
|
/// Extract the page's main-content text (readability-lite), preferring a
|
||||||
|
/// semantic `<main>`/`<article>` region over the full body. Used by
|
||||||
|
/// `get text --main` to avoid header/nav/sidebar boilerplate (#27).
|
||||||
|
pub async fn get_main_content_text(client: &CdpClient, session_id: &str) -> Result<String, String> {
|
||||||
|
let res = client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(serde_json::json!({
|
||||||
|
"expression": MAIN_CONTENT_JS,
|
||||||
|
"returnByValue": true,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(res
|
||||||
|
.get("result")
|
||||||
|
.and_then(|r| r.get("value"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Text nodes whose parent is one of these carry no visible content.
|
||||||
|
fn is_noise_tag(name: &str) -> bool {
|
||||||
|
matches!(name, "SCRIPT" | "STYLE" | "NOSCRIPT" | "TEMPLATE" | "HEAD")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk a CDP DOM.Node tree, collecting text-node values. Unlike `innerText`
|
||||||
|
// (JS, blocked by CLOSED shadow roots), the CDP DOM tree from
|
||||||
|
// `DOM.getDocument(pierce:true)` includes closed shadow roots and child
|
||||||
|
// documents — so this reaches text JS can't. `parent_noise` carries whether an
|
||||||
|
// ancestor was <script>/<style>/etc so their text is skipped.
|
||||||
|
fn collect_dom_text(node: &Value, parent_noise: bool, out: &mut String) {
|
||||||
|
let node_type = node.get("nodeType").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||||
|
let node_name = node.get("nodeName").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
if node_type == 3 {
|
||||||
|
if !parent_noise {
|
||||||
|
if let Some(t) = node.get("nodeValue").and_then(|v| v.as_str()) {
|
||||||
|
let t = t.trim();
|
||||||
|
if !t.is_empty() {
|
||||||
|
if !out.is_empty() {
|
||||||
|
out.push(' ');
|
||||||
|
}
|
||||||
|
out.push_str(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let noise = parent_noise || is_noise_tag(node_name);
|
||||||
|
if let Some(children) = node.get("children").and_then(|v| v.as_array()) {
|
||||||
|
for child in children {
|
||||||
|
collect_dom_text(child, noise, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(shadow) = node.get("shadowRoots").and_then(|v| v.as_array()) {
|
||||||
|
for sr in shadow {
|
||||||
|
collect_dom_text(sr, noise, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(doc) = node.get("contentDocument") {
|
||||||
|
collect_dom_text(doc, noise, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract text from the page via the CDP DOM tree with `pierce:true`, which
|
||||||
|
/// reaches into CLOSED shadow roots and child documents that `innerText`/`eval`
|
||||||
|
/// cannot. Lets an agent read content rendered into a closed shadow DOM (e.g. an
|
||||||
|
/// extension's injected debug panel) without any extra Chrome permission — it
|
||||||
|
/// rides the per-tab debugger session that's already attached (#30).
|
||||||
|
pub async fn get_pierced_text(client: &CdpClient, session_id: &str) -> Result<String, String> {
|
||||||
|
let doc = client
|
||||||
|
.send_command(
|
||||||
|
"DOM.getDocument",
|
||||||
|
Some(serde_json::json!({ "depth": -1, "pierce": true })),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let mut out = String::new();
|
||||||
|
if let Some(root) = doc.get("root") {
|
||||||
|
collect_dom_text(root, false, &mut out);
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get_element_attribute(
|
pub async fn get_element_attribute(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -1421,6 +1719,31 @@ mod tests {
|
|||||||
assert_eq!(parse_ref("@e123"), Some("e123".to_string()));
|
assert_eq!(parse_ref("@e123"), Some("e123".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_collect_dom_text_pierces_closed_shadow_and_skips_noise() {
|
||||||
|
// A CDP DOM.Node tree: a host element whose CLOSED shadow root holds the
|
||||||
|
// text, plus a <script> whose text must be skipped.
|
||||||
|
let tree = serde_json::json!({
|
||||||
|
"nodeType": 1, "nodeName": "BODY",
|
||||||
|
"children": [
|
||||||
|
{ "nodeType": 1, "nodeName": "SCRIPT",
|
||||||
|
"children": [ { "nodeType": 3, "nodeName": "#text", "nodeValue": "var secret=1;" } ] },
|
||||||
|
{ "nodeType": 1, "nodeName": "DIV",
|
||||||
|
"shadowRoots": [
|
||||||
|
{ "nodeType": 11, "nodeName": "#document-fragment",
|
||||||
|
"children": [
|
||||||
|
{ "nodeType": 1, "nodeName": "SPAN",
|
||||||
|
"children": [ { "nodeType": 3, "nodeName": "#text", "nodeValue": "DECRYPTED 42" } ] }
|
||||||
|
] }
|
||||||
|
] }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
let mut out = String::new();
|
||||||
|
collect_dom_text(&tree, false, &mut out);
|
||||||
|
assert_eq!(out, "DECRYPTED 42");
|
||||||
|
assert!(!out.contains("secret"), "script text must be skipped");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_ref_equals_prefix() {
|
fn test_parse_ref_equals_prefix() {
|
||||||
assert_eq!(parse_ref("ref=e1"), Some("e1".to_string()));
|
assert_eq!(parse_ref("ref=e1"), Some("e1".to_string()));
|
||||||
@@ -1452,10 +1775,30 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_build_selector_js_css() {
|
fn test_build_selector_js_css() {
|
||||||
let js = build_selector_js("#submit-btn");
|
let js = build_selector_js("#submit-btn");
|
||||||
assert!(js.contains("document.querySelector(\"#submit-btn\")"));
|
// CSS is now tried via a `sel` variable, with a visible-text fallback
|
||||||
|
// appended (issue #24-B). It must still use querySelector (not xpath).
|
||||||
|
assert!(js.contains("const sel = \"#submit-btn\""));
|
||||||
|
assert!(js.contains("document.querySelector(sel)"));
|
||||||
assert!(!js.contains("document.evaluate"));
|
assert!(!js.contains("document.evaluate"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_find_element_js_text_fallback() {
|
||||||
|
// A bare label gets a text-matching fallback so `click "購入手続きへ"`
|
||||||
|
// resolves by visible text, not just CSS (issue #24-B).
|
||||||
|
let js = build_find_element_js("購入手続きへ");
|
||||||
|
assert!(js.contains("購入手続きへ"));
|
||||||
|
assert!(js.contains("interactive")); // the text-match branch
|
||||||
|
assert!(js.contains("textOf"));
|
||||||
|
// `text=` forces the text path (skips CSS).
|
||||||
|
let forced = build_find_element_js("text=Buy now");
|
||||||
|
assert!(forced.contains("true ? null")); // force_text => css skipped
|
||||||
|
// xpath is unchanged.
|
||||||
|
let xp = build_find_element_js("xpath=//button");
|
||||||
|
assert!(xp.contains("document.evaluate"));
|
||||||
|
assert!(!xp.contains("interactive"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_build_selector_js_xpath() {
|
fn test_build_selector_js_xpath() {
|
||||||
let js = build_selector_js("xpath=//button[@id='ok']");
|
let js = build_selector_js("xpath=//button[@id='ok']");
|
||||||
|
|||||||
+407
-33
@@ -7,6 +7,17 @@ use super::cdp::types::*;
|
|||||||
use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap};
|
use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap};
|
||||||
use super::humanize;
|
use super::humanize;
|
||||||
|
|
||||||
|
/// Whether a pointer interaction should be DOM-dispatched (invoke the event on
|
||||||
|
/// the element in its own session) rather than dispatched at a viewport
|
||||||
|
/// coordinate via `Input.dispatchMouseEvent`. True when the target is inside an
|
||||||
|
/// iframe (an OOPIF element's box can't be mapped to a top-viewport point) or we
|
||||||
|
/// drive over the extension relay (a coordinate Input event isn't confined to the
|
||||||
|
/// target tab on a busy real Chrome — it drifts onto the foreground tab; issues
|
||||||
|
/// #31/#36). DOM-dispatch always hits the right element in the right tab.
|
||||||
|
fn prefer_dom_dispatch(ref_map: &RefMap, selector_or_ref: &str) -> bool {
|
||||||
|
ref_map.ref_is_in_iframe(selector_or_ref) || crate::connect::relay_url().is_some()
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn click(
|
pub async fn click(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -45,6 +56,45 @@ pub async fn click(
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An element INSIDE an iframe needs a TRUSTED activation: a DOM `.click()` is
|
||||||
|
// `isTrusted:false`, which security-sensitive embedded forms reject — Google
|
||||||
|
// Payments' enabled `保存` button silently no-ops on a synthetic click (issue
|
||||||
|
// #39). A coordinate `Input.dispatchMouseEvent` can't help either: `getBoxModel`
|
||||||
|
// for a sub-frame node returns frame-local coordinates that don't compose the
|
||||||
|
// iframe's offset, so the click lands in the wrong place. The frame-agnostic
|
||||||
|
// trusted path is keyboard activation — focus the element in its own frame, then
|
||||||
|
// dispatch a real Enter on the page session; Chrome routes the key to the
|
||||||
|
// focused element regardless of frame (same as `type --focused`), and Enter on a
|
||||||
|
// focused button/link fires a trusted `click`. `coord` mode opts out.
|
||||||
|
let in_iframe = ref_map.ref_is_in_iframe(selector_or_ref);
|
||||||
|
if mode != "coord" && button == "left" && click_count == 1 && in_iframe {
|
||||||
|
return dom_activate(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
// On the relay (the user's real Chrome) a TOP-document coordinate click used to
|
||||||
|
// drift onto the foreground tab; that root cause is fixed (#5: the agent drives
|
||||||
|
// its own pinned tab), but DOM-dispatch stays the conservative default here.
|
||||||
|
if mode != "coord"
|
||||||
|
&& button == "left"
|
||||||
|
&& click_count == 1
|
||||||
|
&& crate::connect::relay_url().is_some()
|
||||||
|
{
|
||||||
|
return dom_click(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
let resolved = resolve_element_center(
|
let resolved = resolve_element_center(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
@@ -235,6 +285,112 @@ async fn dom_click(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Trusted activation of an element inside an iframe (issue #39). Focuses the
|
||||||
|
/// element in its own frame session, then dispatches a real Enter/Space on the
|
||||||
|
/// page session — Chrome routes the key to the focused element across frames, and
|
||||||
|
/// Enter/Space on a focused button/link/checkbox fires a `click` with
|
||||||
|
/// `isTrusted: true`, which security-sensitive embedded forms (Google Payments
|
||||||
|
/// `保存`) require. Non-activatable roles (a `div[onclick]`) can't be keyboard-
|
||||||
|
/// activated, so they fall back to a DOM `.click()`.
|
||||||
|
async fn dom_activate(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
selector_or_ref: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let role = parse_ref(selector_or_ref)
|
||||||
|
.and_then(|r| ref_map.get(&r).map(|e| e.role.clone()))
|
||||||
|
.unwrap_or_default();
|
||||||
|
// Space toggles checkbox-like controls; Enter activates buttons/links/menus.
|
||||||
|
let key = match role.as_str() {
|
||||||
|
"checkbox" | "radio" | "switch" | "option" | "menuitemcheckbox" | "menuitemradio" => {
|
||||||
|
Some("space")
|
||||||
|
}
|
||||||
|
"button" | "link" | "menuitem" | "tab" | "treeitem" => Some("enter"),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let Some(key) = key else {
|
||||||
|
// Not keyboard-activatable — best effort via DOM .click() (untrusted).
|
||||||
|
return dom_click(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
};
|
||||||
|
|
||||||
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// Focus the element in its OWN frame session so the keystroke lands on it.
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: "function() { this.focus(); }".to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&effective_session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// Trusted key on the page session — routed to the focused (in-frame) element.
|
||||||
|
press_key(client, session_id, key).await?;
|
||||||
|
wait_for_paint_settled(client, &effective_session_id).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DOM-dispatch a double-click on the element in its own session (no coordinates)
|
||||||
|
/// — the relay/iframe-safe counterpart to a coordinate dblclick. Fires the full
|
||||||
|
/// click,click,dblclick sequence so handlers bound to any of them respond.
|
||||||
|
async fn dom_dblclick(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
selector_or_ref: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: r#"function() {
|
||||||
|
const opts = { bubbles: true, cancelable: true, view: window };
|
||||||
|
this.dispatchEvent(new MouseEvent('click', opts));
|
||||||
|
this.dispatchEvent(new MouseEvent('click', { ...opts, detail: 2 }));
|
||||||
|
this.dispatchEvent(new MouseEvent('dblclick', opts));
|
||||||
|
}"#
|
||||||
|
.to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&effective_session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
wait_for_paint_settled(client, &effective_session_id).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn dblclick(
|
pub async fn dblclick(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -242,6 +398,20 @@ pub async fn dblclick(
|
|||||||
selector_or_ref: &str,
|
selector_or_ref: &str,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
|
// Same relay/iframe drift hazard as a single click — DOM-dispatch the
|
||||||
|
// double-click there instead of a coordinate one (issues #31/#36).
|
||||||
|
if std::env::var("AGENT_BROWSER_CLICK_MODE").as_deref() != Ok("coord")
|
||||||
|
&& prefer_dom_dispatch(ref_map, selector_or_ref)
|
||||||
|
{
|
||||||
|
return dom_dblclick(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
click(
|
click(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
@@ -254,6 +424,50 @@ pub async fn dblclick(
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// DOM-dispatch a hover (pointer/mouse enter+move) on the element in its own
|
||||||
|
/// session — reaches OOPIF elements and never drifts to the foreground tab over
|
||||||
|
/// the relay, unlike a coordinate `mouseMoved` (issues #31/#36).
|
||||||
|
async fn dom_hover(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
selector_or_ref: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: r#"function() {
|
||||||
|
const r = this.getBoundingClientRect();
|
||||||
|
const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
|
||||||
|
const base = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy };
|
||||||
|
this.dispatchEvent(new PointerEvent('pointerover', base));
|
||||||
|
this.dispatchEvent(new PointerEvent('pointerenter', { ...base, bubbles: false }));
|
||||||
|
this.dispatchEvent(new MouseEvent('mouseover', base));
|
||||||
|
this.dispatchEvent(new MouseEvent('mouseenter', { ...base, bubbles: false }));
|
||||||
|
this.dispatchEvent(new MouseEvent('mousemove', base));
|
||||||
|
}"#
|
||||||
|
.to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&effective_session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn hover(
|
pub async fn hover(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -261,6 +475,18 @@ pub async fn hover(
|
|||||||
selector_or_ref: &str,
|
selector_or_ref: &str,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
|
// Coordinate `mouseMoved` drifts to the foreground tab over the relay and
|
||||||
|
// can't reach an OOPIF — DOM-dispatch the hover there (issues #31/#36).
|
||||||
|
if prefer_dom_dispatch(ref_map, selector_or_ref) {
|
||||||
|
return dom_hover(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
let (x, y, _w, _h, effective_session_id) = resolve_element_center(
|
let (x, y, _w, _h, effective_session_id) = resolve_element_center(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
@@ -289,6 +515,63 @@ pub async fn hover(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// DOM-dispatch an HTML5 drag-and-drop from `source` to `target` in their shared
|
||||||
|
/// session — the relay/iframe-safe counterpart to the coordinate drag, which
|
||||||
|
/// drifts to the foreground tab over the relay and can't reach an OOPIF (issues
|
||||||
|
/// #31/#36). Covers HTML5 DnD (sortable lists, file/card boards); pointer-driven
|
||||||
|
/// drag (canvas, sliders) still needs the coordinate path. Errors if source and
|
||||||
|
/// target live in different frames — a synthetic cross-frame DnD isn't reliable.
|
||||||
|
pub async fn dom_drag(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
source: &str,
|
||||||
|
target: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (src_obj, src_session) =
|
||||||
|
resolve_element_object_id(client, session_id, ref_map, source, iframe_sessions).await?;
|
||||||
|
let (tgt_obj, tgt_session) =
|
||||||
|
resolve_element_object_id(client, session_id, ref_map, target, iframe_sessions).await?;
|
||||||
|
if src_session != tgt_session {
|
||||||
|
return Err(
|
||||||
|
"drag source and target are in different frames; cross-frame drag-and-drop over the \
|
||||||
|
relay isn't supported — drag within a single frame, or use a launched browser with \
|
||||||
|
AGENT_BROWSER_CLICK_MODE=coord"
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: r#"function(target) {
|
||||||
|
const dt = new DataTransfer();
|
||||||
|
const ev = (type, el) => el.dispatchEvent(
|
||||||
|
new DragEvent(type, { bubbles: true, cancelable: true, dataTransfer: dt }));
|
||||||
|
ev('dragstart', this);
|
||||||
|
ev('drag', this);
|
||||||
|
ev('dragenter', target);
|
||||||
|
ev('dragover', target);
|
||||||
|
ev('drop', target);
|
||||||
|
ev('dragend', this);
|
||||||
|
}"#
|
||||||
|
.to_string(),
|
||||||
|
object_id: Some(src_obj),
|
||||||
|
arguments: Some(vec![CallArgument {
|
||||||
|
value: None,
|
||||||
|
object_id: Some(tgt_obj),
|
||||||
|
}]),
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&src_session),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
wait_for_paint_settled(client, &src_session).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn fill(
|
pub async fn fill(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -306,32 +589,50 @@ pub async fn fill(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Focus the element
|
// Emulate a real edit so framework-controlled inputs (React/Vue) and
|
||||||
client
|
// site-side listeners actually see the change (issue #25): the old path set
|
||||||
.send_command_typed::<_, Value>(
|
// `this.value` directly and used Input.insertText, which left React's
|
||||||
"Runtime.callFunctionOn",
|
// internal value-tracker out of sync and never fired change/blur — so
|
||||||
&CallFunctionOnParams {
|
// dependent logic (e.g. Mercari's postal-code → 都道府県 autocomplete) never
|
||||||
function_declaration: "function() { this.focus(); }".to_string(),
|
// ran even though the value was visible. Set the value through the element's
|
||||||
object_id: Some(object_id.clone()),
|
// PROTOTYPE setter (which React's _valueTracker hooks), then dispatch
|
||||||
arguments: None,
|
// input → change → blur/focusout. `type <sel> <text>` remains for sites that
|
||||||
return_by_value: Some(true),
|
// need per-keystroke events.
|
||||||
await_promise: Some(false),
|
let fill_js = format!(
|
||||||
},
|
r#"function() {{
|
||||||
Some(&effective_session_id),
|
const el = this;
|
||||||
)
|
const v = {val};
|
||||||
.await?;
|
try {{ el.focus(); }} catch (e) {{}}
|
||||||
|
const tag = el.tagName;
|
||||||
|
const fire = (type, ctor) => el.dispatchEvent(new (ctor || Event)(type, {{ bubbles: true }}));
|
||||||
|
if (tag === 'SELECT') {{
|
||||||
|
el.value = v; fire('input'); fire('change'); return true;
|
||||||
|
}}
|
||||||
|
if (el.isContentEditable) {{
|
||||||
|
el.textContent = v; fire('input', window.InputEvent || Event); fire('change');
|
||||||
|
try {{ el.blur(); }} catch (e) {{}} fire('focusout'); return true;
|
||||||
|
}}
|
||||||
|
const proto = tag === 'TEXTAREA' ? window.HTMLTextAreaElement.prototype
|
||||||
|
: window.HTMLInputElement.prototype;
|
||||||
|
const desc = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||||
|
const set = desc && desc.set ? (x) => desc.set.call(el, x) : (x) => {{ el.value = x; }};
|
||||||
|
set(''); // reset the framework tracker
|
||||||
|
fire('input', window.InputEvent || Event);
|
||||||
|
set(v); // native setter → React/Vue registers
|
||||||
|
fire('input', window.InputEvent || Event);
|
||||||
|
fire('change');
|
||||||
|
try {{ el.blur(); }} catch (e) {{}}
|
||||||
|
fire('focusout'); // blur-triggered lookups/validation
|
||||||
|
return true;
|
||||||
|
}}"#,
|
||||||
|
val = serde_json::to_string(value).unwrap_or_default()
|
||||||
|
);
|
||||||
|
|
||||||
// Select all + delete to clear
|
|
||||||
client
|
client
|
||||||
.send_command_typed::<_, Value>(
|
.send_command_typed::<_, Value>(
|
||||||
"Runtime.callFunctionOn",
|
"Runtime.callFunctionOn",
|
||||||
&CallFunctionOnParams {
|
&CallFunctionOnParams {
|
||||||
function_declaration: r#"function() {
|
function_declaration: fill_js,
|
||||||
this.select && this.select();
|
|
||||||
this.value = '';
|
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
||||||
}"#
|
|
||||||
.to_string(),
|
|
||||||
object_id: Some(object_id),
|
object_id: Some(object_id),
|
||||||
arguments: None,
|
arguments: None,
|
||||||
return_by_value: Some(true),
|
return_by_value: Some(true),
|
||||||
@@ -341,17 +642,6 @@ pub async fn fill(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Insert text (keyboard input dispatched at page level, use parent session_id)
|
|
||||||
client
|
|
||||||
.send_command_typed::<_, Value>(
|
|
||||||
"Input.insertText",
|
|
||||||
&InsertTextParams {
|
|
||||||
text: value.to_string(),
|
|
||||||
},
|
|
||||||
Some(session_id),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,6 +655,7 @@ pub async fn type_text(
|
|||||||
clear: bool,
|
clear: bool,
|
||||||
delay_ms: Option<u64>,
|
delay_ms: Option<u64>,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
key_events: bool,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let (object_id, effective_session_id) = resolve_element_object_id(
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
client,
|
client,
|
||||||
@@ -411,7 +702,7 @@ pub async fn type_text(
|
|||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
type_text_into_active_context(client, session_id, text, delay_ms).await
|
type_text_into_active_context(client, session_id, text, delay_ms, key_events).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn type_text_into_active_context(
|
pub async fn type_text_into_active_context(
|
||||||
@@ -419,6 +710,7 @@ pub async fn type_text_into_active_context(
|
|||||||
session_id: &str,
|
session_id: &str,
|
||||||
text: &str,
|
text: &str,
|
||||||
delay_ms: Option<u64>,
|
delay_ms: Option<u64>,
|
||||||
|
key_events: bool,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
// Per-character timing: an explicit `delay_ms` wins (caller asked for a
|
// Per-character timing: an explicit `delay_ms` wins (caller asked for a
|
||||||
// fixed cadence); otherwise fall back to humanize — variable, human-like
|
// fixed cadence); otherwise fall back to humanize — variable, human-like
|
||||||
@@ -468,6 +760,46 @@ pub async fn type_text_into_active_context(
|
|||||||
Some(session_id),
|
Some(session_id),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
} else if key_events {
|
||||||
|
// Real keystrokes (keyDown+keyUp carrying `text`) for autocomplete /
|
||||||
|
// combobox widgets that only react to key events and ignore the
|
||||||
|
// `input` that `Input.insertText` fires — e.g. Google's address
|
||||||
|
// postal-code → city/prefecture lookup (issue #36 / #4). The keyDown's
|
||||||
|
// `text` still inserts the character, so the field also fills.
|
||||||
|
let (key, code, key_code) = char_to_key_info(ch);
|
||||||
|
let s = ch.to_string();
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
&DispatchKeyEventParams {
|
||||||
|
event_type: "keyDown".to_string(),
|
||||||
|
key: Some(key.clone()),
|
||||||
|
code: Some(code.clone()),
|
||||||
|
text: Some(s.clone()),
|
||||||
|
unmodified_text: Some(s),
|
||||||
|
windows_virtual_key_code: Some(key_code),
|
||||||
|
native_virtual_key_code: Some(key_code),
|
||||||
|
modifiers: None,
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
&DispatchKeyEventParams {
|
||||||
|
event_type: "keyUp".to_string(),
|
||||||
|
key: Some(key),
|
||||||
|
code: Some(code),
|
||||||
|
text: None,
|
||||||
|
unmodified_text: None,
|
||||||
|
windows_virtual_key_code: Some(key_code),
|
||||||
|
native_virtual_key_code: Some(key_code),
|
||||||
|
modifiers: None,
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
// VS Code/Electron webviews reject repeated dispatchKeyEvent calls
|
// VS Code/Electron webviews reject repeated dispatchKeyEvent calls
|
||||||
// carrying printable `text`. Insert printable characters directly
|
// carrying printable `text`. Insert printable characters directly
|
||||||
@@ -557,6 +889,48 @@ pub async fn press_key_with_modifiers(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Dispatch a SINGLE key event (`keyDown` or `keyUp`) carrying the full key
|
||||||
|
/// descriptor — `key`, `code`, `windowsVirtualKeyCode`/`nativeVirtualKeyCode`,
|
||||||
|
/// and (on key-down) printable `text`. Powers the `keydown`/`keyup` commands.
|
||||||
|
///
|
||||||
|
/// The previous implementation sent only `{key}`, so games and shortcut handlers
|
||||||
|
/// that read `event.code` (e.g. `"KeyD"`, `"ArrowRight"`) or `event.keyCode` saw
|
||||||
|
/// nothing — a held key set no movement flag and did nothing (dogfood: holding a
|
||||||
|
/// direction in a canvas platformer barely nudged the player). Sending the same
|
||||||
|
/// descriptor `press` uses makes hold-to-move work regardless of which field the
|
||||||
|
/// page keys off.
|
||||||
|
pub async fn dispatch_single_key(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
key: &str,
|
||||||
|
event_type: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (key_name, code, key_code) = named_key_info(key);
|
||||||
|
// Printable text is only meaningful on key-down; key-up never inserts.
|
||||||
|
let text = if event_type == "keyDown" {
|
||||||
|
key_text(&key_name)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
&DispatchKeyEventParams {
|
||||||
|
event_type: event_type.to_string(),
|
||||||
|
key: Some(key_name),
|
||||||
|
code: Some(code),
|
||||||
|
text: text.clone(),
|
||||||
|
unmodified_text: text,
|
||||||
|
windows_virtual_key_code: Some(key_code),
|
||||||
|
native_virtual_key_code: Some(key_code),
|
||||||
|
modifiers: None,
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn scroll(
|
pub async fn scroll(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
|
|||||||
@@ -60,6 +60,9 @@ pub struct ScreenshotOptions {
|
|||||||
pub quality: Option<i32>,
|
pub quality: Option<i32>,
|
||||||
pub annotate: bool,
|
pub annotate: bool,
|
||||||
pub output_dir: Option<String>,
|
pub output_dir: Option<String>,
|
||||||
|
/// Explicit pixel region (x, y, width, height) — `--clip` (issue #34). Takes
|
||||||
|
/// precedence over selector/full_page.
|
||||||
|
pub clip: Option<(f64, f64, f64, f64)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ScreenshotOptions {
|
impl Default for ScreenshotOptions {
|
||||||
@@ -72,6 +75,7 @@ impl Default for ScreenshotOptions {
|
|||||||
quality: None,
|
quality: None,
|
||||||
annotate: false,
|
annotate: false,
|
||||||
output_dir: None,
|
output_dir: None,
|
||||||
|
clip: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -187,7 +191,16 @@ async fn capture_screenshot_base64(
|
|||||||
capture_beyond_viewport: if options.full_page { Some(true) } else { None },
|
capture_beyond_viewport: if options.full_page { Some(true) } else { None },
|
||||||
};
|
};
|
||||||
|
|
||||||
if options.full_page {
|
if let Some((x, y, width, height)) = options.clip {
|
||||||
|
// Explicit pixel region wins over selector/full_page (issue #34).
|
||||||
|
params.clip = Some(Viewport {
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
scale: 1.0,
|
||||||
|
});
|
||||||
|
} else if options.full_page {
|
||||||
let metrics: Value = client
|
let metrics: Value = client
|
||||||
.send_command_no_params("Page.getLayoutMetrics", Some(session_id))
|
.send_command_no_params("Page.getLayoutMetrics", Some(session_id))
|
||||||
.await?;
|
.await?;
|
||||||
|
|||||||
@@ -330,6 +330,13 @@ impl RoleNameTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Max iframe nesting depth `take_snapshot` expands. Embedded payment/checkout
|
||||||
|
/// widgets nest a few frames deep (e.g. AdSense → payments.google.com → an inner
|
||||||
|
/// form frame); expanding past the first level is what gives those inner refs a
|
||||||
|
/// `frame_id` so clicks resolve into the right frame (issue #36). Capped to keep
|
||||||
|
/// a pathological frame tree from blowing up the snapshot.
|
||||||
|
const MAX_IFRAME_DEPTH: usize = 3;
|
||||||
|
|
||||||
pub async fn take_snapshot(
|
pub async fn take_snapshot(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -337,6 +344,28 @@ pub async fn take_snapshot(
|
|||||||
ref_map: &mut RefMap,
|
ref_map: &mut RefMap,
|
||||||
frame_id: Option<&str>,
|
frame_id: Option<&str>,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
take_snapshot_at_depth(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
options,
|
||||||
|
ref_map,
|
||||||
|
frame_id,
|
||||||
|
iframe_sessions,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn take_snapshot_at_depth(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
options: &SnapshotOptions,
|
||||||
|
ref_map: &mut RefMap,
|
||||||
|
frame_id: Option<&str>,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
depth: usize,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
client
|
client
|
||||||
.send_command_no_params("DOM.enable", Some(session_id))
|
.send_command_no_params("DOM.enable", Some(session_id))
|
||||||
@@ -606,10 +635,11 @@ pub async fn take_snapshot(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Recurse into child iframes: for each Iframe node with a backend_node_id,
|
// Recurse into child iframes: for each Iframe node with a backend_node_id,
|
||||||
// resolve the child frame ID and take a snapshot of its content.
|
// resolve the child frame ID and snapshot its content. Recurse to
|
||||||
// We only recurse from the main frame (frame_id == None) to avoid
|
// MAX_IFRAME_DEPTH (not just the main frame) so refs inside nested
|
||||||
// unbounded depth; nested iframes within iframes are not expanded.
|
// payment/checkout widgets get a `frame_id` and clicks resolve into the right
|
||||||
if frame_id.is_none() {
|
// frame (issue #36); the cap bounds a pathological frame tree.
|
||||||
|
if depth < MAX_IFRAME_DEPTH {
|
||||||
let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot)
|
let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot)
|
||||||
for node in tree_nodes.iter() {
|
for node in tree_nodes.iter() {
|
||||||
if node.role != "Iframe" || !node.has_ref {
|
if node.role != "Iframe" || !node.has_ref {
|
||||||
@@ -622,13 +652,14 @@ pub async fn take_snapshot(
|
|||||||
if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await {
|
if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await {
|
||||||
// Snapshot the child frame; errors are silently ignored
|
// Snapshot the child frame; errors are silently ignored
|
||||||
// (e.g. cross-origin iframes)
|
// (e.g. cross-origin iframes)
|
||||||
if let Ok(child_text) = Box::pin(take_snapshot(
|
if let Ok(child_text) = Box::pin(take_snapshot_at_depth(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
options,
|
options,
|
||||||
ref_map,
|
ref_map,
|
||||||
Some(&child_fid),
|
Some(&child_fid),
|
||||||
iframe_sessions,
|
iframe_sessions,
|
||||||
|
depth + 1,
|
||||||
))
|
))
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>iframe button probe</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>iframe button probe</h1>
|
||||||
|
<iframe
|
||||||
|
id="frame"
|
||||||
|
width="320"
|
||||||
|
height="140"
|
||||||
|
srcdoc="
|
||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<body style='margin:24px'>
|
||||||
|
<button id='b' style='padding:24px;font-size:22px'>save</button>
|
||||||
|
<script>
|
||||||
|
document.getElementById('b').addEventListener('click', function (e) {
|
||||||
|
this.textContent = 'clicked:' + e.isTrusted;
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"
|
||||||
|
></iframe>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
+227
-17
@@ -186,6 +186,103 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(data) = &resp.data {
|
if let Some(data) = &resp.data {
|
||||||
|
// A click that opened a new tab: surface it so the agent doesn't read the
|
||||||
|
// unchanged old page as a failed click (issue #24-A).
|
||||||
|
if let Some(opened) = data.get("openedTab") {
|
||||||
|
let tid = opened.get("tabId").and_then(|v| v.as_str()).unwrap_or("?");
|
||||||
|
let url = opened.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let followed = data
|
||||||
|
.get("followed")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let verb = if followed {
|
||||||
|
"switched to new tab"
|
||||||
|
} else {
|
||||||
|
"opened new tab"
|
||||||
|
};
|
||||||
|
eprintln!(
|
||||||
|
"{} {} [{}] {}",
|
||||||
|
color::cyan("→"),
|
||||||
|
verb,
|
||||||
|
tid,
|
||||||
|
color::dim(url)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// `current`: the active tab's stable handle (#26).
|
||||||
|
if data
|
||||||
|
.get("current")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
let tid = data.get("tabId").and_then(|v| v.as_str()).unwrap_or("?");
|
||||||
|
let title = data.get("title").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let url = data.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let target = data.get("targetId").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
println!("{} [{}] {} - {}", color::cyan("→"), tid, title, url);
|
||||||
|
println!(" {}", color::dim(&format!("target: {}", target)));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cloudflare challenge/clearance preflight (`cf-status`). Checked early
|
||||||
|
// because its response carries `url`/`title`, which later generic
|
||||||
|
// renderers would otherwise swallow.
|
||||||
|
if action == Some("cf_status") {
|
||||||
|
let challenged = data
|
||||||
|
.get("challenged")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let rec = data
|
||||||
|
.get("recommendation")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("?");
|
||||||
|
let cl = data.get("clearance");
|
||||||
|
let present = cl
|
||||||
|
.and_then(|c| c.get("present"))
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let expired = cl
|
||||||
|
.and_then(|c| c.get("expired"))
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let expires_in = cl.and_then(|c| c.get("expiresIn")).and_then(|v| v.as_i64());
|
||||||
|
let device = data
|
||||||
|
.get("deviceVerified")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
let (icon, headline) = match rec {
|
||||||
|
"proceed" => (color::success_indicator().to_string(), "cleared — no challenge, proceed"),
|
||||||
|
"solve" => (color::warning_indicator().to_string(), "Cloudflare challenge active, no valid clearance — solve it"),
|
||||||
|
"reissue" => (color::warning_indicator().to_string(), "challenge active but a clearance cookie exists — stale (IP/UA changed?), re-solve"),
|
||||||
|
_ => (color::cyan("•").to_string(), "unknown"),
|
||||||
|
};
|
||||||
|
println!("{} {}", icon, headline);
|
||||||
|
println!(
|
||||||
|
" challenged: {}",
|
||||||
|
if challenged { "yes" } else { "no" }
|
||||||
|
);
|
||||||
|
let cl_desc = if !present {
|
||||||
|
"absent".to_string()
|
||||||
|
} else if expired {
|
||||||
|
"present but EXPIRED".to_string()
|
||||||
|
} else if let Some(s) = expires_in {
|
||||||
|
format!("valid, expires in {}m {}s", s / 60, s % 60)
|
||||||
|
} else {
|
||||||
|
"present (session)".to_string()
|
||||||
|
};
|
||||||
|
println!(" cf_clearance: {}", cl_desc);
|
||||||
|
println!(
|
||||||
|
" device trusted: {}",
|
||||||
|
if device {
|
||||||
|
"yes (CF_VERIFIED_DEVICE)"
|
||||||
|
} else {
|
||||||
|
"no"
|
||||||
|
}
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Dialog status response
|
// Dialog status response
|
||||||
if action == Some("dialog") {
|
if action == Some("dialog") {
|
||||||
if let Some(has_dialog) = data.get("hasDialog").and_then(|v| v.as_bool()) {
|
if let Some(has_dialog) = data.get("hasDialog").and_then(|v| v.as_bool()) {
|
||||||
@@ -297,6 +394,43 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
// Snapshot
|
// Snapshot
|
||||||
if let Some(snapshot) = data.get("snapshot").and_then(|v| v.as_str()) {
|
if let Some(snapshot) = data.get("snapshot").and_then(|v| v.as_str()) {
|
||||||
print_with_boundaries(snapshot, origin, opts);
|
print_with_boundaries(snapshot, origin, opts);
|
||||||
|
// Canvas-app hint: the tree was near-empty but the page paints to a
|
||||||
|
// <canvas>, so refs are a dead end — point at the screenshot path.
|
||||||
|
if let Some(note) = data.get("note").and_then(|v| v.as_str()) {
|
||||||
|
eprintln!("{}", color::dim(note));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Frame list (`chrome-use frames`)
|
||||||
|
if action == Some("frames") {
|
||||||
|
if let Some(list) = data.get("frames").and_then(|v| v.as_array()) {
|
||||||
|
let count = list.len();
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
color::bold(&format!(
|
||||||
|
"{} frame{}",
|
||||||
|
count,
|
||||||
|
if count == 1 { "" } else { "s" }
|
||||||
|
))
|
||||||
|
);
|
||||||
|
for f in list {
|
||||||
|
let idx = f.get("index").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||||
|
let kind = f.get("kind").and_then(|v| v.as_str()).unwrap_or("?");
|
||||||
|
let url = f.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let len = f.get("textLen").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||||
|
println!(
|
||||||
|
" [{}] {:<6} {} chars {}",
|
||||||
|
idx,
|
||||||
|
kind,
|
||||||
|
len,
|
||||||
|
color::dim(if url.is_empty() { "(about:blank)" } else { url })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
color::dim("read everything with: chrome-use get text --all-frames")
|
||||||
|
);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Title
|
// Title
|
||||||
@@ -512,25 +646,35 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
} else {
|
} else {
|
||||||
println!("{} [{}] {} - {}", marker, tab_id, title, url);
|
println!("{} [{}] {} - {}", marker, tab_id, title, url);
|
||||||
}
|
}
|
||||||
|
// `--full` also surfaces the stable cross-session CDP targetId so
|
||||||
|
// a stranded tab can be adopted from another session via
|
||||||
|
// `tab <targetId>` (issue #21).
|
||||||
|
if full {
|
||||||
|
if let Some(target_id) = tab.get("targetId").and_then(|v| v.as_str()) {
|
||||||
|
println!(" {}", color::dim(&format!("target: {}", target_id)));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Tab switch
|
// Tab switch
|
||||||
if action == Some("tab_switch") {
|
if action == Some("tab_switch") {
|
||||||
if let Some(tab_id) = data.get("tabId").and_then(|v| v.as_str()) {
|
if let Some(tab_id) = data.get("tabId").and_then(|v| v.as_str()) {
|
||||||
if let Some(url) = data.get("url").and_then(|v| v.as_str()) {
|
let warning = data.get("warning").and_then(|v| v.as_str());
|
||||||
println!(
|
// A non-responding session isn't a real success — show a warning
|
||||||
"{} Switched to tab [{}] ({})",
|
// indicator instead of the green ✓ (issue #29.3).
|
||||||
color::success_indicator(),
|
let indicator = if warning.is_some() {
|
||||||
tab_id,
|
color::warning_indicator()
|
||||||
url
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
println!(
|
color::success_indicator()
|
||||||
"{} Switched to tab [{}]",
|
};
|
||||||
color::success_indicator(),
|
if let Some(url) = data.get("url").and_then(|v| v.as_str()) {
|
||||||
tab_id
|
println!("{} Switched to tab [{}] ({})", indicator, tab_id, url);
|
||||||
);
|
} else {
|
||||||
|
println!("{} Switched to tab [{}]", indicator, tab_id);
|
||||||
|
}
|
||||||
|
if let Some(w) = warning {
|
||||||
|
eprintln!("{}", color::dim(w));
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1349,6 +1493,12 @@ Usage: chrome-use type <selector> <text>
|
|||||||
Types text into the specified element character by character.
|
Types text into the specified element character by character.
|
||||||
Unlike fill, this does not clear existing content first.
|
Unlike fill, this does not clear existing content first.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--key-events Send real per-character keyDown/keyUp instead of
|
||||||
|
(alias --keys) Input.insertText. Use for autocomplete / combobox fields
|
||||||
|
that only react to key events — e.g. a postal-code box
|
||||||
|
that auto-fills city/prefecture, or Google Places.
|
||||||
|
|
||||||
Global Options:
|
Global Options:
|
||||||
--json Output as JSON
|
--json Output as JSON
|
||||||
--session <name> Use specific session
|
--session <name> Use specific session
|
||||||
@@ -1356,6 +1506,7 @@ Global Options:
|
|||||||
Examples:
|
Examples:
|
||||||
chrome-use type "#search" "hello"
|
chrome-use type "#search" "hello"
|
||||||
chrome-use type @e2 "additional text"
|
chrome-use type @e2 "additional text"
|
||||||
|
chrome-use type @e5 "201-0001" --key-events # trigger the address autocomplete
|
||||||
|
|
||||||
See Also:
|
See Also:
|
||||||
For typing into contenteditable editors (Lexical, ProseMirror, etc.)
|
For typing into contenteditable editors (Lexical, ProseMirror, etc.)
|
||||||
@@ -1620,12 +1771,23 @@ Usage: chrome-use scroll [direction] [amount] [options]
|
|||||||
|
|
||||||
Scrolls the page or a specific element in the specified direction.
|
Scrolls the page or a specific element in the specified direction.
|
||||||
|
|
||||||
|
Without --selector, scroll dispatches a real (isTrusted) mouse wheel at a
|
||||||
|
viewport coordinate, so it scrolls whatever container is under the pointer —
|
||||||
|
including cross-origin iframes (Google Payments, Stripe, embedded checkout/KYC)
|
||||||
|
that plain page scroll can't reach.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
direction up, down, left, right (default: down)
|
direction up, down, left, right (default: down)
|
||||||
amount Pixels to scroll (default: 300)
|
amount Pixels to scroll (default: 300)
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
-s, --selector <sel> CSS selector for a scrollable container
|
-s, --selector <sel> CSS selector for a scrollable container (same-origin)
|
||||||
|
--at <x,y> Dispatch the wheel at this viewport pixel (read it from a
|
||||||
|
screenshot) — precise way into a cross-origin iframe
|
||||||
|
--frame <n> Scroll the n-th frame from `chrome-use frames` (wheel at
|
||||||
|
that frame's center)
|
||||||
|
|
||||||
|
Without --selector/--at/--frame the wheel lands at the viewport center.
|
||||||
|
|
||||||
Global Options:
|
Global Options:
|
||||||
--json Output as JSON
|
--json Output as JSON
|
||||||
@@ -1637,6 +1799,8 @@ Examples:
|
|||||||
chrome-use scroll up 200
|
chrome-use scroll up 200
|
||||||
chrome-use scroll left 100
|
chrome-use scroll left 100
|
||||||
chrome-use scroll down 500 --selector "div.scroll-container"
|
chrome-use scroll down 500 --selector "div.scroll-container"
|
||||||
|
chrome-use scroll down 700 --at 640,400 # wheel at a pixel over an iframe
|
||||||
|
chrome-use scroll down 700 --frame 2 # scroll frame 2 from `frames`
|
||||||
"##
|
"##
|
||||||
}
|
}
|
||||||
"scrollintoview" | "scrollinto" => {
|
"scrollintoview" | "scrollinto" => {
|
||||||
@@ -1717,6 +1881,8 @@ Pass --hide-scrollbars false when launching to keep native scrollbars visible.
|
|||||||
|
|
||||||
Options:
|
Options:
|
||||||
--full, -f Capture full page (not just viewport)
|
--full, -f Capture full page (not just viewport)
|
||||||
|
[selector] Capture just an element (CSS or @ref), e.g. `screenshot ".header" h.png`
|
||||||
|
--clip <x,y,w,h> Capture a pixel region, e.g. `screenshot --clip 0,0,200,40 corner.png`
|
||||||
--annotate Overlay numbered labels on interactive elements.
|
--annotate Overlay numbered labels on interactive elements.
|
||||||
Each label [N] corresponds to ref @eN from snapshot.
|
Each label [N] corresponds to ref @eN from snapshot.
|
||||||
Prints a legend mapping labels to element roles/names.
|
Prints a legend mapping labels to element roles/names.
|
||||||
@@ -1737,6 +1903,8 @@ Examples:
|
|||||||
chrome-use screenshot
|
chrome-use screenshot
|
||||||
chrome-use screenshot ./screenshot.png
|
chrome-use screenshot ./screenshot.png
|
||||||
chrome-use screenshot --full ./full-page.png
|
chrome-use screenshot --full ./full-page.png
|
||||||
|
chrome-use screenshot ".header .indicator" corner.png # just one element
|
||||||
|
chrome-use screenshot --clip 1600,0,200,40 corner.png # a pixel region
|
||||||
chrome-use screenshot --annotate # Labeled screenshot + legend
|
chrome-use screenshot --annotate # Labeled screenshot + legend
|
||||||
chrome-use screenshot --annotate ./page.png # Save annotated screenshot
|
chrome-use screenshot --annotate ./page.png # Save annotated screenshot
|
||||||
chrome-use screenshot --annotate --json # JSON output with annotations
|
chrome-use screenshot --annotate --json # JSON output with annotations
|
||||||
@@ -1878,7 +2046,9 @@ Usage: chrome-use get <subcommand> [args]
|
|||||||
Retrieves various types of information from elements or the page.
|
Retrieves various types of information from elements or the page.
|
||||||
|
|
||||||
Subcommands:
|
Subcommands:
|
||||||
text <selector> Get text content of element
|
text [selector] Element text; no selector = WHOLE PAGE, all frames
|
||||||
|
text --main Main-content text only (skip nav/header/sidebar)
|
||||||
|
text --pierce Read through CLOSED shadow DOM (injected panels)
|
||||||
html <selector> Get inner HTML of element
|
html <selector> Get inner HTML of element
|
||||||
value <selector> Get value of input element
|
value <selector> Get value of input element
|
||||||
attr <selector> <name> Get attribute value
|
attr <selector> <name> Get attribute value
|
||||||
@@ -1894,7 +2064,10 @@ Global Options:
|
|||||||
--session <name> Use specific session
|
--session <name> Use specific session
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
chrome-use get text @e1
|
chrome-use get text # whole page across ALL frames (default)
|
||||||
|
chrome-use get text @e1 # one element
|
||||||
|
chrome-use get text --main # main content, no nav/sidebar boilerplate
|
||||||
|
chrome-use frames # list frames + where the text lives
|
||||||
chrome-use get html "#content"
|
chrome-use get html "#content"
|
||||||
chrome-use get value "#email-input"
|
chrome-use get value "#email-input"
|
||||||
chrome-use get attr "#link" href
|
chrome-use get attr "#link" href
|
||||||
@@ -2765,6 +2938,20 @@ Notes:
|
|||||||
- Streaming is always enabled. Set AGENT_BROWSER_STREAM_PORT to bind to a
|
- Streaming is always enabled. Set AGENT_BROWSER_STREAM_PORT to bind to a
|
||||||
specific port instead of the default OS-assigned port.
|
specific port instead of the default OS-assigned port.
|
||||||
|
|
||||||
|
The WS is BIDIRECTIONAL — the high-throughput way to drive a live/real-time page
|
||||||
|
(games, canvas apps) instead of one screenshot + one CLI call per action:
|
||||||
|
- Server -> client (JSON text frames):
|
||||||
|
{"type":"frame","data":"<base64 jpeg>"} live screencast (~60fps)
|
||||||
|
plus status / tabs messages.
|
||||||
|
- Client -> server (send JSON text):
|
||||||
|
{"type":"input_keyboard","eventType":"keyDown|keyUp","key":" ","code":"Space",
|
||||||
|
"windowsVirtualKeyCode":32}
|
||||||
|
{"type":"input_mouse","eventType":"mousePressed|mouseReleased|mouseMoved",
|
||||||
|
"x":640,"y":360,"button":"left","clickCount":1}
|
||||||
|
{"type":"input_touch","eventType":"touchStart|touchEnd","touchPoints":[...]}
|
||||||
|
Connect once and run a tight local loop: read frames, send timed input — no
|
||||||
|
per-action process spawn, no round-trip. Works over the extension relay too.
|
||||||
|
|
||||||
Global Options:
|
Global Options:
|
||||||
--json Output as JSON
|
--json Output as JSON
|
||||||
--session <name> Use specific session
|
--session <name> Use specific session
|
||||||
@@ -3057,7 +3244,12 @@ Core Commands:
|
|||||||
dblclick <sel> Double-click element
|
dblclick <sel> Double-click element
|
||||||
type <sel> <text> Type into element
|
type <sel> <text> Type into element
|
||||||
fill <sel> <text> Clear and fill
|
fill <sel> <text> Clear and fill
|
||||||
press <key> Press key (Enter, Tab, Control+a)
|
press <key> [--hold <ms>] Press key (Enter, Tab, Control+a). --hold keeps it
|
||||||
|
down <ms> then releases — precise (in-daemon), for
|
||||||
|
games/charge: `press d --hold 800`
|
||||||
|
keydown <key> Hold a key down (no auto-release) — for games/shortcuts
|
||||||
|
keyup <key> Release a held key. Pair with keydown to hold-to-move:
|
||||||
|
`keydown d` … `keyup d`
|
||||||
keyboard type <text> Type text with real keystrokes (no selector)
|
keyboard type <text> Type text with real keystrokes (no selector)
|
||||||
keyboard inserttext <text> Insert text without key events
|
keyboard inserttext <text> Insert text without key events
|
||||||
hover <sel> Hover element
|
hover <sel> Hover element
|
||||||
@@ -3085,10 +3277,15 @@ Navigation:
|
|||||||
|
|
||||||
Get Info: chrome-use get <what> [selector]
|
Get Info: chrome-use get <what> [selector]
|
||||||
text, html, value, attr <name>, title, url, count, box, styles, cdp-url
|
text, html, value, attr <name>, title, url, count, box, styles, cdp-url
|
||||||
|
text (no selector = whole page, all frames), text --main, frames (list)
|
||||||
|
|
||||||
Check State: chrome-use is <what> <selector>
|
Check State: chrome-use is <what> <selector>
|
||||||
visible, enabled, checked
|
visible, enabled, checked
|
||||||
|
|
||||||
|
Anti-bot: chrome-use stealth | cf-status
|
||||||
|
stealth stealth self-check (webdriver/UA/plugins + overrides)
|
||||||
|
cf-status Cloudflare challenge + cf_clearance preflight (skip re-solving)
|
||||||
|
|
||||||
Find Elements: chrome-use find <locator> <value> <action> [text]
|
Find Elements: chrome-use find <locator> <value> <action> [text]
|
||||||
role, text, label, placeholder, alt, title, testid, first, last, nth
|
role, text, label, placeholder, alt, title, testid, first, last, nth
|
||||||
|
|
||||||
@@ -3116,7 +3313,12 @@ Storage:
|
|||||||
storage <local|session> Manage web storage
|
storage <local|session> Manage web storage
|
||||||
|
|
||||||
Tabs:
|
Tabs:
|
||||||
tab [new|list|close|<n>] Manage tabs
|
tab [new|list|close|<ref>] Manage tabs (<ref> = t<N>, a label, or a CDP targetId)
|
||||||
|
tab list --full Full URLs + stable cross-session targetId per tab
|
||||||
|
tab <targetId> Adopt a specific tab (incl. another session's) by its
|
||||||
|
stable targetId, no reload — preserves in-page state
|
||||||
|
open <url> --reuse-tab Reuse an existing tab on that URL instead of spawning
|
||||||
|
a duplicate (matches origin+path; preserves state)
|
||||||
|
|
||||||
Diff:
|
Diff:
|
||||||
diff snapshot Compare current vs last snapshot
|
diff snapshot Compare current vs last snapshot
|
||||||
@@ -3178,6 +3380,10 @@ Confirmation:
|
|||||||
Sessions:
|
Sessions:
|
||||||
session Show current session name
|
session Show current session name
|
||||||
session list List active sessions
|
session list List active sessions
|
||||||
|
sessions List running session daemons (alias of daemon status)
|
||||||
|
daemon status List running session daemons (+ relay state)
|
||||||
|
daemon restart Kill all session daemons; keeps the extension relay
|
||||||
|
up. Clears stale/cross-leaked state after an upgrade.
|
||||||
|
|
||||||
Chat (AI):
|
Chat (AI):
|
||||||
chat <message> Send a natural language instruction (single-shot)
|
chat <message> Send a natural language instruction (single-shot)
|
||||||
@@ -3371,6 +3577,9 @@ iOS Simulator (requires Xcode and Appium):
|
|||||||
chrome-use -p ios device list # List simulators
|
chrome-use -p ios device list # List simulators
|
||||||
chrome-use -p ios swipe up # Swipe gesture
|
chrome-use -p ios swipe up # Swipe gesture
|
||||||
chrome-use -p ios tap @e1 # Touch element
|
chrome-use -p ios tap @e1 # Touch element
|
||||||
|
|
||||||
|
Hit a bug or rough edge? A 30-second issue genuinely sharpens this tool:
|
||||||
|
https://github.com/leeguooooo/chrome-use/issues
|
||||||
"#
|
"#
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -3453,6 +3662,7 @@ fn print_screenshot_diff(data: &serde_json::Map<String, serde_json::Value>) {
|
|||||||
|
|
||||||
pub fn print_version() {
|
pub fn print_version() {
|
||||||
println!("chrome-use {}", env!("CARGO_PKG_VERSION"));
|
println!("chrome-use {}", env!("CARGO_PKG_VERSION"));
|
||||||
|
println!("report bugs / rough edges: https://github.com/leeguooooo/chrome-use/issues");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
+22
-1
@@ -52,6 +52,27 @@ fn is_newer(latest: &str, current: &str) -> bool {
|
|||||||
matches!((parse_version(latest), parse_version(current)), (Some(l), Some(c)) if l > c)
|
matches!((parse_version(latest), parse_version(current)), (Some(l), Some(c)) if l > c)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Public semver-ish comparison (`latest` strictly newer than `current`), so
|
||||||
|
/// `doctor` can flag a stale extension/CLI without re-implementing parsing.
|
||||||
|
pub fn version_is_newer(latest: &str, current: &str) -> bool {
|
||||||
|
is_newer(latest, current)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The latest CLI version recorded by the background update check, if any.
|
||||||
|
/// `doctor` uses it to show "a newer chrome-use is available" without a network
|
||||||
|
/// call (the `__update-check` worker refreshes the cache out of band).
|
||||||
|
pub fn cached_latest_version() -> Option<String> {
|
||||||
|
std::fs::read_to_string(update_cache_path())
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
|
||||||
|
.and_then(|j| {
|
||||||
|
j.get("latest")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
})
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
/// Hidden `__update-check` subcommand: fetch the latest release tag and cache it.
|
/// Hidden `__update-check` subcommand: fetch the latest release tag and cache it.
|
||||||
/// Spawned detached by [`maybe_notify_update`] so the network call never blocks a
|
/// Spawned detached by [`maybe_notify_update`] so the network call never blocks a
|
||||||
/// real command. Uses `curl` (no extra deps, matches `upgrade`).
|
/// real command. Uses `curl` (no extra deps, matches `upgrade`).
|
||||||
@@ -100,7 +121,7 @@ pub fn maybe_notify_update() {
|
|||||||
if first.starts_with("__")
|
if first.starts_with("__")
|
||||||
|| matches!(
|
|| matches!(
|
||||||
first.as_str(),
|
first.as_str(),
|
||||||
"upgrade" | "install" | "doctor" | "dashboard"
|
"upgrade" | "install" | "doctor" | "dashboard" | "daemon"
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
@@ -30,6 +30,16 @@ const tabs = new Map()
|
|||||||
const sessionToTab = new Map()
|
const sessionToTab = new Map()
|
||||||
/** child (OOPIF/worker) sessionId -> tabId */
|
/** child (OOPIF/worker) sessionId -> tabId */
|
||||||
const childSessionToTab = new Map()
|
const childSessionToTab = new Map()
|
||||||
|
/** sessionId -> CDP targetId, kept ACROSS detach so a dead `cb-tab-<oldTabId>`
|
||||||
|
* session can be recovered by its stable targetId when the cross-process nav
|
||||||
|
* gave the tab a new Chrome tabId (issue #24). Capped to bound memory. */
|
||||||
|
const sessionTargets = new Map()
|
||||||
|
function rememberSessionTarget(sessionId, targetId) {
|
||||||
|
if (!sessionId || !targetId) return
|
||||||
|
sessionTargets.delete(sessionId)
|
||||||
|
sessionTargets.set(sessionId, targetId)
|
||||||
|
if (sessionTargets.size > 256) sessionTargets.delete(sessionTargets.keys().next().value)
|
||||||
|
}
|
||||||
/** tab-group name -> chrome tabGroups id (best-effort cache) */
|
/** tab-group name -> chrome tabGroups id (best-effort cache) */
|
||||||
const groupIdByName = new Map()
|
const groupIdByName = new Map()
|
||||||
|
|
||||||
@@ -108,6 +118,12 @@ function connectHost() {
|
|||||||
// reconnect. Keep chrome.debugger attached so reconnect is cheap.
|
// reconnect. Keep chrome.debugger attached so reconnect is cheap.
|
||||||
for (const tabId of tabs.keys()) setBadge(tabId, 'connecting')
|
for (const tabId of tabs.keys()) setBadge(tabId, 'connecting')
|
||||||
})
|
})
|
||||||
|
// Report our version so the host can tell the CLI/`doctor` which extension
|
||||||
|
// build is live (otherwise the extension version is a black box — the user
|
||||||
|
// can't tell they're on an old one). Best-effort; ignored by older hosts.
|
||||||
|
try {
|
||||||
|
postToHost({ method: 'hello', version: chrome.runtime.getManifest().version })
|
||||||
|
} catch {}
|
||||||
// Tell the daemon about everything we already have attached, then attach
|
// Tell the daemon about everything we already have attached, then attach
|
||||||
// anything new.
|
// anything new.
|
||||||
reannounceAttachedTabs()
|
reannounceAttachedTabs()
|
||||||
@@ -149,6 +165,90 @@ function tabForTarget(targetId) {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The STABLE Chrome tabId encoded in a `cb-tab-<tabId>` session id (#17), or
|
||||||
|
// null for any other session shape (child/iframe sessions). The tabId is the
|
||||||
|
// real source of truth: it survives the renderer-process swaps (cross-origin
|
||||||
|
// OAuth/SSO navs) that tear down the page's CDP target — which is why binding to
|
||||||
|
// it (like claude-in-chrome) rides through the hop that killed the old
|
||||||
|
// target/sessionId binding (issue #23).
|
||||||
|
function tabIdFromSession(sessionId) {
|
||||||
|
const m = /^cb-tab-(\d+)$/.exec(sessionId || '')
|
||||||
|
return m ? Number(m[1]) : null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the debugger is attached to a `cb-tab-<tabId>` session's tab, re-attaching
|
||||||
|
// across the transient window of a process swap (with a couple of short retries).
|
||||||
|
// Returns the tabId on success, or null when the tab is genuinely gone
|
||||||
|
// (closed / restricted). (issues #20.1, #23)
|
||||||
|
async function recoverSessionTab(sessionId) {
|
||||||
|
const tabId = tabIdFromSession(sessionId)
|
||||||
|
// 1) Fast path: the encoded Chrome tabId still exists — re-attach it (covers
|
||||||
|
// the common renderer-process swap where the tabId is preserved, #23).
|
||||||
|
if (tabId != null) {
|
||||||
|
for (let i = 0; i < 3; i++) {
|
||||||
|
const tab = await chrome.tabs.get(tabId).catch(() => null)
|
||||||
|
if (!eligible(tab)) break // tabId is gone — fall through to targetId recovery
|
||||||
|
try {
|
||||||
|
await attachTab(tabId)
|
||||||
|
if (tabs.has(tabId)) return tabId
|
||||||
|
} catch {
|
||||||
|
// mid-swap: tab exists but isn't attachable yet — back off and retry.
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 120 + i * 150))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 2) The Chrome tabId is gone, but the CDP targetId is STABLE across the nav.
|
||||||
|
// Some cross-process hops (Mercari's signin token exchange) give the tab a
|
||||||
|
// NEW tabId while keeping the same target, so `cb-tab-<oldTabId>` can't be
|
||||||
|
// recovered by tabId. Find the tab now hosting our remembered targetId via
|
||||||
|
// chrome.debugger.getTargets(), attach it, and ALIAS the dead session to it
|
||||||
|
// so the daemon's session id keeps resolving. Longer window: this hop can
|
||||||
|
// take several seconds to settle (issue #24).
|
||||||
|
const targetId = sessionTargets.get(sessionId)
|
||||||
|
if (targetId) {
|
||||||
|
for (let i = 0; i < 6; i++) {
|
||||||
|
const targets = await chrome.debugger.getTargets().catch(() => null)
|
||||||
|
const t = targets && targets.find((x) => x.id === targetId && x.tabId != null)
|
||||||
|
if (t && t.tabId != null) {
|
||||||
|
const tab = await chrome.tabs.get(t.tabId).catch(() => null)
|
||||||
|
if (eligible(tab)) {
|
||||||
|
try {
|
||||||
|
await attachTab(t.tabId)
|
||||||
|
if (tabs.has(t.tabId)) {
|
||||||
|
sessionToTab.set(sessionId, t.tabId) // alias dead session -> live tab
|
||||||
|
return t.tabId
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// not attachable yet — keep waiting for the swap to settle.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 300 + i * 300))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send a CDP command to a tab, riding a debugger detach that can happen between
|
||||||
|
// our attach check and the command itself (a renderer-process swap mid-flight).
|
||||||
|
// On a detached-style failure, drop the stale handle, re-attach the stable tab,
|
||||||
|
// and retry once — so a cross-process nav never surfaces as a hard error (#23).
|
||||||
|
async function sendCdpToTab(tabId, method, params) {
|
||||||
|
const dbg = { tabId }
|
||||||
|
try {
|
||||||
|
return await chrome.debugger.sendCommand(dbg, method, params)
|
||||||
|
} catch (e) {
|
||||||
|
const msg = String((e && e.message) || e)
|
||||||
|
if (!/detached|not attached|target.*(closed|gone)|no target|cannot access|frame.*detached/i.test(msg)) {
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
detachTab(tabId, false)
|
||||||
|
const ok = await recoverSessionTab(`cb-tab-${tabId}`)
|
||||||
|
if (!ok) throw e
|
||||||
|
return await chrome.debugger.sendCommand(dbg, method, params)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function anyConnectedTab() {
|
function anyConnectedTab() {
|
||||||
const it = tabs.keys().next()
|
const it = tabs.keys().next()
|
||||||
return it.done ? null : it.value
|
return it.done ? null : it.value
|
||||||
@@ -207,13 +307,26 @@ async function handleForwardCdpCommand(msg) {
|
|||||||
// Fail loudly instead so the agent sees an actionable error, not bad data.
|
// Fail loudly instead so the agent sees an actionable error, not bad data.
|
||||||
let tabId
|
let tabId
|
||||||
if (sessionId) {
|
if (sessionId) {
|
||||||
tabId = tabForSession(sessionId)
|
// The stable Chrome tabId encoded in `cb-tab-<tabId>` is the source of truth
|
||||||
if (!tabId) {
|
// (it survives renderer-process swaps; the CDP target/sessionId does not).
|
||||||
throw new Error(
|
// Resolve via it primarily — don't depend on a session→tab map entry that the
|
||||||
`stale sessionId ${sessionId} for ${method}: its tab is gone (closed, ` +
|
// detach handler may have cleared — and ensure the debugger is attached,
|
||||||
`navigated across processes, or lost after an extension restart). ` +
|
// re-attaching across a cross-process nav before failing (issues #20.1, #23).
|
||||||
`Re-attach by re-opening your target URL before retrying.`,
|
// `tabForSession` still covers child/iframe sessions that aren't `cb-tab-*`.
|
||||||
)
|
tabId = tabIdFromSession(sessionId) ?? tabForSession(sessionId)
|
||||||
|
if (tabId == null) {
|
||||||
|
throw new Error(`unknown sessionId ${sessionId} for ${method}`)
|
||||||
|
}
|
||||||
|
if (!tabs.has(tabId)) {
|
||||||
|
const recovered = await recoverSessionTab(sessionId)
|
||||||
|
if (!recovered) {
|
||||||
|
throw new Error(
|
||||||
|
`stale sessionId ${sessionId} for ${method}: its tab is gone (closed, ` +
|
||||||
|
`navigated across processes, or lost after an extension restart). ` +
|
||||||
|
`Re-attach by re-opening your target URL before retrying.`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
tabId = recovered
|
||||||
}
|
}
|
||||||
} else if (typeof params?.targetId === 'string') {
|
} else if (typeof params?.targetId === 'string') {
|
||||||
tabId = tabForTarget(params.targetId)
|
tabId = tabForTarget(params.targetId)
|
||||||
@@ -223,18 +336,17 @@ async function handleForwardCdpCommand(msg) {
|
|||||||
// applies to any attached tab.
|
// applies to any attached tab.
|
||||||
tabId = anyConnectedTab()
|
tabId = anyConnectedTab()
|
||||||
}
|
}
|
||||||
if (!tabId) throw new Error(`no attached tab for ${method}`)
|
if (tabId == null) throw new Error(`no attached tab for ${method}`)
|
||||||
const dbg = { tabId }
|
|
||||||
|
|
||||||
// Re-enabling Runtime can leave a stale state; bounce it (matches upstream).
|
// Re-enabling Runtime can leave a stale state; bounce it (matches upstream).
|
||||||
if (method === 'Runtime.enable') {
|
if (method === 'Runtime.enable') {
|
||||||
try {
|
try {
|
||||||
await chrome.debugger.sendCommand(dbg, 'Runtime.disable')
|
await sendCdpToTab(tabId, 'Runtime.disable', undefined)
|
||||||
await new Promise((r) => setTimeout(r, 30))
|
await new Promise((r) => setTimeout(r, 30))
|
||||||
} catch {}
|
} catch {}
|
||||||
return await chrome.debugger.sendCommand(dbg, 'Runtime.enable', params)
|
return await sendCdpToTab(tabId, 'Runtime.enable', params)
|
||||||
}
|
}
|
||||||
return await chrome.debugger.sendCommand(dbg, method, params)
|
return await sendCdpToTab(tabId, method, params)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- attach / detach ------------------------------------------------------
|
// ---- attach / detach ------------------------------------------------------
|
||||||
@@ -273,6 +385,7 @@ async function attachTab(tabId) {
|
|||||||
const entry = { sessionId, targetId }
|
const entry = { sessionId, targetId }
|
||||||
tabs.set(tabId, entry)
|
tabs.set(tabId, entry)
|
||||||
sessionToTab.set(sessionId, tabId)
|
sessionToTab.set(sessionId, tabId)
|
||||||
|
rememberSessionTarget(sessionId, targetId)
|
||||||
setBadge(tabId, port ? 'on' : 'connecting')
|
setBadge(tabId, port ? 'on' : 'connecting')
|
||||||
postToHost({
|
postToHost({
|
||||||
method: 'forwardCDPEvent',
|
method: 'forwardCDPEvent',
|
||||||
@@ -355,9 +468,33 @@ chrome.debugger.onEvent.addListener((source, method, params) =>
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
chrome.debugger.onDetach.addListener((source) =>
|
chrome.debugger.onDetach.addListener((source, reason) =>
|
||||||
void whenReady(() => {
|
void whenReady(async () => {
|
||||||
if (source.tabId) detachTab(source.tabId, true)
|
const tabId = source.tabId
|
||||||
|
if (!tabId) return
|
||||||
|
detachTab(tabId, true)
|
||||||
|
// A cross-process navigation (e.g. an SSO redirect like
|
||||||
|
// login.account.rakuten.com that swaps the render process / spawns OOPIFs)
|
||||||
|
// detaches the debugger, but the TAB survives. Without re-attaching, the
|
||||||
|
// session goes permanently stale and even open/navigate fails — exactly the
|
||||||
|
// #19 follow-up. So proactively re-attach (the stable `cb-tab-<tabId>`
|
||||||
|
// session id then restores the daemon's binding). Don't fight a detach the
|
||||||
|
// user or DevTools initiated.
|
||||||
|
if (reason === 'canceled_by_user' || reason === 'replaced_with_devtools') return
|
||||||
|
if (!port) return
|
||||||
|
// The swapped-in process needs a moment to settle; retry with backoff.
|
||||||
|
for (let i = 0; i < 6; i++) {
|
||||||
|
await new Promise((r) => setTimeout(r, 250 + i * 200))
|
||||||
|
if (tabs.has(tabId)) return // already re-attached (e.g. via onUpdated)
|
||||||
|
const tab = await chrome.tabs.get(tabId).catch(() => null)
|
||||||
|
if (!tab || !eligible(tab)) return // tab gone or now a restricted page
|
||||||
|
try {
|
||||||
|
await attachTab(tabId)
|
||||||
|
return
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(`ab-connect: reattach attempt ${i + 1} for tab ${tabId} failed:`, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest_version": 3,
|
"manifest_version": 3,
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "0.4.4",
|
"version": "0.4.9",
|
||||||
"description": "Let chrome-use drive your logged-in Chrome \u2014 install once, no token, no per-use confirmation.",
|
"description": "Let chrome-use drive your logged-in Chrome \u2014 install once, no token, no per-use confirmation.",
|
||||||
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6vQIyscGIPYPZdSpPwPL0+0gxUROyRgCpmvCSDoc8XUm4qm97VbKnD9Ijc1lV22lNWZtE78gaRjt6BeSfuMgnBymnhLKjN1gU6AI5QUU0mrJyeHdWKvrKQR5FmsM2A7Xr1ykE2SiiS8zNUS3Y/6O5l+Nva7wrVy6E4a2dkBVQkOsu+DV+nEZvhIyuDY5D5SPXqNwUTWTaglwj5mjvHz36xSwCWlPmrtJ+ED0AUyrb2z4GIOmvk4kqtBVrh/UD058klLo4CkYOnIybB5aV6WYuwarfPY4bF/dLggPem+ewLNTUNBuwrxj/A4nUv0LJTuRO8rR7f8WR9qnRCY0Ic5saQIDAQAB",
|
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6vQIyscGIPYPZdSpPwPL0+0gxUROyRgCpmvCSDoc8XUm4qm97VbKnD9Ijc1lV22lNWZtE78gaRjt6BeSfuMgnBymnhLKjN1gU6AI5QUU0mrJyeHdWKvrKQR5FmsM2A7Xr1ykE2SiiS8zNUS3Y/6O5l+Nva7wrVy6E4a2dkBVQkOsu+DV+nEZvhIyuDY5D5SPXqNwUTWTaglwj5mjvHz36xSwCWlPmrtJ+ED0AUyrb2z4GIOmvk4kqtBVrh/UD058klLo4CkYOnIybB5aV6WYuwarfPY4bF/dLggPem+ewLNTUNBuwrxj/A4nUv0LJTuRO8rR7f8WR9qnRCY0Ic5saQIDAQAB",
|
||||||
"icons": {
|
"icons": {
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "1.2.3",
|
"version": "1.5.18",
|
||||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"packageManager": "pnpm@11.1.3",
|
"packageManager": "pnpm@11.1.3",
|
||||||
|
|||||||
+187
-8
@@ -36,6 +36,29 @@ Refs (`@e1`, `@e2`, ...) are assigned fresh on every snapshot. They become
|
|||||||
submits, dynamic re-renders, dialog opens. Always re-snapshot before your
|
submits, dynamic re-renders, dialog opens. Always re-snapshot before your
|
||||||
next ref interaction.
|
next ref interaction.
|
||||||
|
|
||||||
|
> **Hard rule: snapshot-first, never screenshot-to-locate.** For form fields and
|
||||||
|
> buttons, ALWAYS `snapshot -i` and act on refs/selectors. Do **not** reach for
|
||||||
|
> `screenshot` + coordinate clicks to find or hit an element — `snapshot -i` now
|
||||||
|
> pierces **cross-origin iframes** (embedded Google Payments / Stripe / checkout /
|
||||||
|
> KYC forms) and lists their elements by `@ref`, including input values. Use
|
||||||
|
> coordinates only for canvas/WebGL, or when `snapshot` genuinely returns nothing
|
||||||
|
> for your target. Screenshots are for *visual verification you report*, never the
|
||||||
|
> agent's own input — and a full-page `screenshot` of a real retina browser is
|
||||||
|
> often too large for an image reader anyway. (If you ever feel you *need* a
|
||||||
|
> screenshot to read state or locate something, that's a bug — please file it.)
|
||||||
|
|
||||||
|
> **Snapshot-first, always. Never default to `screenshot` + coordinate clicking
|
||||||
|
> for form fields or buttons.** Run `snapshot -i` and act on `@refs`. Use
|
||||||
|
> coordinates only for canvas/WebGL, or when `snapshot` genuinely returns nothing
|
||||||
|
> for your target. This holds **even inside cross-origin embedded iframes** —
|
||||||
|
> since v1.5.12 `snapshot -i` pierces out-of-process iframes (Google Payments,
|
||||||
|
> Stripe, embedded checkout/KYC) and lists their elements with refs, so
|
||||||
|
> `click @e` / `type @e` / `fill @e` work directly. A screenshot is for a genuine
|
||||||
|
> *visual* check you report to the user — not your own input. (Full-page
|
||||||
|
> screenshots of a real retina Chrome are often too large for the image reader
|
||||||
|
> anyway.) Driving off pixels on the relay also risks a coordinate event drifting
|
||||||
|
> onto the user's foreground tab — refs never do. See issue #37.
|
||||||
|
|
||||||
## Before you automate: pick the cheapest tool
|
## Before you automate: pick the cheapest tool
|
||||||
|
|
||||||
Driving a browser is the heavy option. chrome-use earns its keep when you
|
Driving a browser is the heavy option. chrome-use earns its keep when you
|
||||||
@@ -108,7 +131,17 @@ Each `--session` that connects gets its **own colored Chrome tab group** (named
|
|||||||
after the session) and drives only its own tabs — multiple agents share the one
|
after the session) and drives only its own tabs — multiple agents share the one
|
||||||
real browser without cross-talk, and the user's own tabs are never grouped. CDP
|
real browser without cross-talk, and the user's own tabs are never grouped. CDP
|
||||||
drives the page without moving the user's mouse/keyboard, so it doesn't fight
|
drives the page without moving the user's mouse/keyboard, so it doesn't fight
|
||||||
them for control. **Anti-detection ranking: this real logged-in Chrome (extension
|
them for control.
|
||||||
|
|
||||||
|
**Strict multi-agent isolation.** A session over the relay tracks and drives
|
||||||
|
**only the tabs it created** (its own group). It does **not** adopt the user's
|
||||||
|
existing tabs, other agents' tabs, or pop-ups (e.g. an OAuth/login window — that's
|
||||||
|
the user's), so several agents (and other tools opening tabs) can work in the same
|
||||||
|
real Chrome concurrently without ever dropping or stealing each other's tabs —
|
||||||
|
another agent's tab churn can't make your bound tab vanish or drift your commands
|
||||||
|
onto the wrong page. Consequence: `tab list` shows only *your* session's tabs; to
|
||||||
|
drive a specific page, navigate to it in your own tab instead of expecting a
|
||||||
|
pre-existing or popped-up tab to appear in the list. **Anti-detection ranking: this real logged-in Chrome (extension
|
||||||
connect) > a headed launched browser > headless (forbidden).** A genuine human
|
connect) > a headed launched browser > headless (forbidden).** A genuine human
|
||||||
browser has no headless/automation tells at all, so prefer it for anything
|
browser has no headless/automation tells at all, so prefer it for anything
|
||||||
anti-bot-sensitive.
|
anti-bot-sensitive.
|
||||||
@@ -127,6 +160,19 @@ cadence, and scroll/drag ease. Default `off`; a per-navigation detector
|
|||||||
auto-escalates pages guarded by Akamai/PerimeterX/DataDome to `human`. Leave it
|
auto-escalates pages guarded by Akamai/PerimeterX/DataDome to `human`. Leave it
|
||||||
on auto; force `human` only when you already know the target scores behaviour.
|
on auto; force `human` only when you already know the target scores behaviour.
|
||||||
|
|
||||||
|
**Cloudflare clearance — solve once, reuse.** Passing a Cloudflare challenge
|
||||||
|
mints a `cf_clearance` cookie (HttpOnly — invisible to `eval`/`document.cookie`;
|
||||||
|
read it via `chrome-use cookies`). It's bound to your **IP + User-Agent**: reuse
|
||||||
|
the same exit IP and UA and you skip the challenge until it expires. Driving the
|
||||||
|
user's real Chrome (relay) persists it natively; for isolated sessions,
|
||||||
|
`--session-name <name>` save/restores it. Before spending effort solving, run
|
||||||
|
`chrome-use cf-status` (aliases `cf`, `clearance`): it reports whether the page
|
||||||
|
is *currently* a Cloudflare challenge and whether a still-valid `cf_clearance`
|
||||||
|
exists, with a recommendation — `proceed` (already cleared, don't re-solve),
|
||||||
|
`solve` (challenge up, no clearance), or `reissue` (clearance present but page
|
||||||
|
still blocks → IP/UA drifted, re-solve). Use it as a preflight to avoid
|
||||||
|
re-solving what you already cleared.
|
||||||
|
|
||||||
## Two ways to drive a page — and when to drop to `eval`
|
## Two ways to drive a page — and when to drop to `eval`
|
||||||
|
|
||||||
You have a **real Chrome with the user's DOM**. Two layers, mix them freely:
|
You have a **real Chrome with the user's DOM**. Two layers, mix them freely:
|
||||||
@@ -207,7 +253,11 @@ assigned fresh on every snapshot.
|
|||||||
For unstructured reading (no refs needed):
|
For unstructured reading (no refs needed):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
chrome-use get text @e1 # visible text of an element
|
chrome-use get text # WHOLE PAGE — all frames by default (see below)
|
||||||
|
chrome-use get text @e1 # visible text of one element (or a CSS selector)
|
||||||
|
chrome-use get text --main # main content only — skip nav/header/sidebar
|
||||||
|
chrome-use get text --pierce # read through CLOSED shadow DOM (injected panels)
|
||||||
|
chrome-use frames # list every frame + where the text lives
|
||||||
chrome-use get html @e1 # innerHTML
|
chrome-use get html @e1 # innerHTML
|
||||||
chrome-use get attr @e1 href # any attribute
|
chrome-use get attr @e1 href # any attribute
|
||||||
chrome-use get value @e1 # input value
|
chrome-use get value @e1 # input value
|
||||||
@@ -216,6 +266,27 @@ chrome-use get url # current URL
|
|||||||
chrome-use get count ".item" # count matching elements
|
chrome-use get count ".item" # count matching elements
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Whole-page text is cross-frame by default.** `chrome-use get text` with no
|
||||||
|
selector aggregates visible text across **every** frame — top document plus
|
||||||
|
same-process child frames plus cross-origin iframes — so you never silently miss
|
||||||
|
content that lives in an iframe (Yahoo Auctions / Rakuten / Mercari shop
|
||||||
|
descriptions, embedded checkout/spec frames). Each child frame is delimited with
|
||||||
|
a `----- frame [kind] url -----` marker. You do **not** need to remember a flag —
|
||||||
|
the default already reads all frames. (`--all-frames` is still accepted as an
|
||||||
|
explicit alias.)
|
||||||
|
|
||||||
|
So: when text looks missing or wrong, you don't have to guess — just
|
||||||
|
`chrome-use get text` reads everything. To **see** the structure (which frame
|
||||||
|
holds what), run `chrome-use frames`. To **cut boilerplate** (global nav/header/
|
||||||
|
footer, "related items" sidebars), use `chrome-use get text --main`. If content
|
||||||
|
is lazy-loaded, `scroll` it into view first, then read.
|
||||||
|
|
||||||
|
**Closed shadow DOM.** Some injected UI (browser-extension debug panels, web
|
||||||
|
components) renders into a *closed* shadow root that `eval`/`innerText` cannot
|
||||||
|
read. `chrome-use get text --pierce` reads through closed shadow roots and child
|
||||||
|
documents via the CDP DOM tree — use it when content is clearly on screen (you
|
||||||
|
see it in a screenshot) but `get text`/`eval` come back empty.
|
||||||
|
|
||||||
## Interacting
|
## Interacting
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -226,8 +297,15 @@ chrome-use hover @e1 # hover
|
|||||||
chrome-use focus @e1 # focus (useful before keyboard input)
|
chrome-use focus @e1 # focus (useful before keyboard input)
|
||||||
chrome-use fill @e2 "hello" # clear then type
|
chrome-use fill @e2 "hello" # clear then type
|
||||||
chrome-use type @e2 " world" # type without clearing
|
chrome-use type @e2 " world" # type without clearing
|
||||||
chrome-use press Enter # press a key at current focus
|
chrome-use type @e5 "201-0001" --key-events # real keystrokes (not insertText) —
|
||||||
|
# use for autocomplete/combobox fields that
|
||||||
|
# only react to key events (e.g. a postal box
|
||||||
|
# that auto-fills city/prefecture, Google Places)
|
||||||
|
chrome-use press Enter # press a key at current focus (down+up)
|
||||||
chrome-use press Control+a # key combination
|
chrome-use press Control+a # key combination
|
||||||
|
chrome-use keydown d # HOLD a key down (no auto-release)
|
||||||
|
chrome-use keyup d # release it — pair them to hold-to-move
|
||||||
|
# in a game: `keydown d; sleep; keyup d`
|
||||||
chrome-use check @e3 # check checkbox
|
chrome-use check @e3 # check checkbox
|
||||||
chrome-use uncheck @e3 # uncheck
|
chrome-use uncheck @e3 # uncheck
|
||||||
chrome-use select @e4 "option-value" # native <select> only
|
chrome-use select @e4 "option-value" # native <select> only
|
||||||
@@ -240,16 +318,32 @@ chrome-use pick @e4 --option "Europe" # ANY combobox (react-select / ARIA /
|
|||||||
# (no silent no-op). Use this for custom
|
# (no silent no-op). Use this for custom
|
||||||
# dropdowns where `select` returns ✓ but
|
# dropdowns where `select` returns ✓ but
|
||||||
# changes nothing.
|
# changes nothing.
|
||||||
chrome-use upload @e5 file1.pdf # upload file(s) — NOTE: needs a --launch/direct-CDP
|
chrome-use upload @e5 file1.pdf # upload file(s) — works over the extension relay too:
|
||||||
# session. Over the extension relay it CANNOT work
|
# chrome.debugger forbids setFileInputFiles, so the
|
||||||
# (Chrome's chrome.debugger forbids it); chrome-use
|
# file's bytes are streamed into the page and rebuilt as
|
||||||
# errors with a hint. Carry your login into a launched
|
# a File there (chunked under native-messaging's 1 MiB cap).
|
||||||
# session via `cookies export` | `cookies set --curl`.
|
# Works on file <input>s and drop/paste composers (e.g. X).
|
||||||
chrome-use scroll down 500 # scroll page (up/down/left/right)
|
chrome-use scroll down 500 # scroll page (up/down/left/right)
|
||||||
|
chrome-use scroll down 700 --at 640,400 # wheel at a pixel — scrolls a cross-origin
|
||||||
|
# iframe (Payments/Stripe/checkout/KYC) that
|
||||||
|
# plain page scroll can't reach
|
||||||
|
chrome-use scroll down 700 --frame 2 # scroll frame 2 from `chrome-use frames`
|
||||||
chrome-use scrollintoview @e1 # scroll element into view
|
chrome-use scrollintoview @e1 # scroll element into view
|
||||||
chrome-use drag @e1 @e2 # drag and drop
|
chrome-use drag @e1 @e2 # drag and drop
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Cross-origin iframes (embedded payment / checkout / KYC widgets — Google
|
||||||
|
Payments, Stripe, etc.) — drive them by ref, never by screenshot.** `snapshot -i`
|
||||||
|
pierces these out-of-process iframes and lists their elements by `@ref`
|
||||||
|
(including input values); `get text --all-frames` reads their text. Then just act
|
||||||
|
on the refs: `click @e`, `type @e`, `hover @e`, `dblclick @e`, `drag @a @b` all
|
||||||
|
work into the iframe. Over the extension relay these are dispatched through the
|
||||||
|
DOM (in the element's own frame), so they hit the right element in the right tab
|
||||||
|
— a coordinate click/scroll there can drift onto whatever tab is in the
|
||||||
|
foreground, so prefer refs. For below-the-fold content in such a frame, scroll it
|
||||||
|
with `scroll down N --at x,y` (a pixel over the frame) or `--frame n`. For a
|
||||||
|
postal/autocomplete box inside the frame, `type @e "…" --key-events`.
|
||||||
|
|
||||||
### When refs don't work or you don't want to snapshot
|
### When refs don't work or you don't want to snapshot
|
||||||
|
|
||||||
Use semantic locators:
|
Use semantic locators:
|
||||||
@@ -296,6 +390,54 @@ chrome-use click --coords 449,320 # same, explicit flag
|
|||||||
|
|
||||||
A bare-number argument is always a coordinate, never a selector.
|
A bare-number argument is always a coordinate, never a selector.
|
||||||
|
|
||||||
|
### Canvas / WebGL apps (games, map & 3D viewers, drawing tools)
|
||||||
|
|
||||||
|
These paint everything to a `<canvas>` and expose **almost no accessibility
|
||||||
|
tree**, so `snapshot` comes back near-empty and refs are a dead end. `snapshot`
|
||||||
|
detects this and prints a one-line hint. Drive them the screenshot way:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
||||||
|
# eval/get text return nothing useful)
|
||||||
|
chrome-use click 640 360 # interact by viewport coordinate
|
||||||
|
chrome-use press d --hold 800 # hold-to-move, precise (timed in-daemon —
|
||||||
|
# NOT keydown+shell-sleep+keyup, which
|
||||||
|
# adds ~250ms jitter per round-trip)
|
||||||
|
chrome-use press Space # discrete actions (jump/attack/confirm)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Don't drive frame-by-frame with one CLI call per action** — that's the slowest,
|
||||||
|
lowest-fidelity way (each call is a process spawn + round-trip). Script a *timed
|
||||||
|
sequence in a single round-trip* with `batch` (it sends each step to the running
|
||||||
|
daemon; `press --hold` and `wait` block in-daemon, so timing is precise):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chrome-use batch "press d --hold 900" "press j" "press j" "wait 200" "press d --hold 500"
|
||||||
|
```
|
||||||
|
|
||||||
|
Also try reading real state instead of pixels: `eval` runs in the page's main
|
||||||
|
world, so for a framework/engine game you can often reach its globals (e.g. a
|
||||||
|
Phaser/PIXI/Three instance, a store, `window.__GAME__`) and read positions/score
|
||||||
|
directly — far better than guessing from a screenshot.
|
||||||
|
|
||||||
|
**For genuinely real-time driving, drop the CLI entirely and use the WebSocket.**
|
||||||
|
`chrome-use stream enable` opens a bidirectional WS (`stream status` prints the
|
||||||
|
`ws://127.0.0.1:<port>`). Connect once and you get a live ~60fps screencast AND
|
||||||
|
can send input on the same socket — no per-action process spawn, no round-trip,
|
||||||
|
works over the extension relay:
|
||||||
|
|
||||||
|
```js
|
||||||
|
// node (global WebSocket): live frames + locally-timed input
|
||||||
|
const ws = new WebSocket("ws://127.0.0.1:PORT")
|
||||||
|
ws.onmessage = e => { const m = JSON.parse(e.data); if (m.type==="frame") {/* base64 jpeg */} }
|
||||||
|
const k = (eventType,key,code,vk) => ws.send(JSON.stringify({type:"input_keyboard",eventType,key,code,windowsVirtualKeyCode:vk}))
|
||||||
|
k("keyDown"," ","Space",32); setTimeout(()=>k("keyUp"," ","Space",32), 80) // a jump
|
||||||
|
// also: {type:"input_mouse",eventType:"mousePressed",x,y,button:"left",clickCount:1}
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the difference between watching a slideshow and playing the game. Reserve
|
||||||
|
screenshots for one-off checks; use the WS for any sustained real-time control.
|
||||||
|
|
||||||
## Waiting (read this)
|
## Waiting (read this)
|
||||||
|
|
||||||
Agents fail more often from bad waits than from bad selectors. Pick the
|
Agents fail more often from bad waits than from bad selectors. Pick the
|
||||||
@@ -501,6 +643,43 @@ the same browser's existing targets, so a second session's first `open` can
|
|||||||
navigate a sibling's tab. For concurrent agents on one real Chrome, use the
|
navigate a sibling's tab. For concurrent agents on one real Chrome, use the
|
||||||
extension (each with a distinct `--session`), not raw `--cdp`.
|
extension (each with a distinct `--session`), not raw `--cdp`.
|
||||||
|
|
||||||
|
Each session owns its own tab group and assigns its own `t<N>` indices (the same
|
||||||
|
physical tab is `t8` in one session, `t1` in another), so `t<N>` is **not** a
|
||||||
|
stable cross-session handle. To reach a *specific* tab from another session — e.g.
|
||||||
|
a tab that was filled in a session whose handle later died — use the **stable CDP
|
||||||
|
`targetId`**:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chrome-use tab list --full --session B # re-syncs live tabs; prints `target: <id>` per row
|
||||||
|
chrome-use tab <targetId> --session B # adopt that exact tab, NO reload (state preserved)
|
||||||
|
```
|
||||||
|
|
||||||
|
`tab list` re-discovers the live tab set on every call, so a fresh session sees
|
||||||
|
tabs other sessions opened (and re-attached ones), not just its own. Adopting by
|
||||||
|
`targetId` lands session B on the stranded tab without reloading it, so a
|
||||||
|
half-filled form survives. Still, the simplest recovery for a session whose own
|
||||||
|
tab died is to recover *that* session (reload / re-`open` / `daemon restart`).
|
||||||
|
|
||||||
|
To avoid piling up duplicate tabs when you re-`open` the same entry URL on
|
||||||
|
rebind, pass **`--reuse-tab`**: if a tab already shows that URL (matched by
|
||||||
|
origin+path), it switches to it instead of spawning a new one.
|
||||||
|
|
||||||
|
### Reset stuck daemon state
|
||||||
|
|
||||||
|
Each session runs a background daemon worker that holds the page handles. If a
|
||||||
|
session starts misbehaving — commands hit the wrong tab, refs/handles look stale,
|
||||||
|
or you upgraded `chrome-use` mid-session and old workers linger — restart the
|
||||||
|
daemons instead of hunting PIDs with `pgrep`/`kill`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chrome-use daemon status # list running session daemons (+ relay state)
|
||||||
|
chrome-use daemon restart # kill every session daemon worker
|
||||||
|
```
|
||||||
|
|
||||||
|
`daemon restart` leaves the extension's native-messaging bridge (`__nm-host`)
|
||||||
|
alone, so the relay to your live Chrome stays up — the next command just spins up
|
||||||
|
a fresh, clean daemon against the same browser. It does **not** close any tabs.
|
||||||
|
|
||||||
### Mock network requests
|
### Mock network requests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
Reference in New Issue
Block a user