Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57c52d6517 | ||
|
|
2707ceb1c4 | ||
|
|
f7a657ac46 | ||
|
|
4e295ce139 | ||
|
|
3d82f11ff2 | ||
|
|
33269adc1a | ||
|
|
9ab8753b48 | ||
|
|
770708b8e6 | ||
|
|
7c594820da | ||
|
|
81d18bbd2e |
@@ -147,16 +147,24 @@ jobs:
|
|||||||
git fetch --tags --force --quiet origin 2>/dev/null || true
|
git fetch --tags --force --quiet origin 2>/dev/null || true
|
||||||
TAG="${{ github.event.inputs.tag || github.ref_name }}"
|
TAG="${{ github.event.inputs.tag || github.ref_name }}"
|
||||||
PREV="$(git describe --tags --abbrev=0 "${TAG}^" 2>/dev/null || true)"
|
PREV="$(git describe --tags --abbrev=0 "${TAG}^" 2>/dev/null || true)"
|
||||||
|
RANGE="${TAG}"
|
||||||
|
[ -n "$PREV" ] && RANGE="${PREV}..${TAG}"
|
||||||
|
# Group commit subjects by conventional-commit type so the notes are
|
||||||
|
# scannable ("what's new / what's fixed") instead of a flat dev log.
|
||||||
|
LOG="$(git log "$RANGE" --no-merges --pretty='%s' | grep -v '^chore(release)' || true)"
|
||||||
|
section() { # $1=header $2=grep-pattern
|
||||||
|
local body; body="$(printf '%s\n' "$LOG" | grep -E "$2" | sed 's/^/- /')"
|
||||||
|
[ -n "$body" ] && { printf '\n### %s\n%s\n' "$1" "$body"; }
|
||||||
|
}
|
||||||
{
|
{
|
||||||
echo "notes<<__NOTES_EOF__"
|
echo "notes<<__NOTES_EOF__"
|
||||||
echo "## What changed"
|
echo "## What changed"
|
||||||
echo ""
|
section "✨ Features" '^feat'
|
||||||
|
section "🐛 Fixes" '^fix'
|
||||||
|
section "🔧 Other" '^(perf|refactor|docs|build|ci|test|style|revert)'
|
||||||
if [ -n "$PREV" ]; then
|
if [ -n "$PREV" ]; then
|
||||||
git log "${PREV}..${TAG}" --no-merges --pretty='- %s' | grep -v '^- chore(release)' || true
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "**Full changelog**: https://github.com/${{ github.repository }}/compare/${PREV}...${TAG}"
|
echo "**Full changelog**: https://github.com/${{ github.repository }}/compare/${PREV}...${TAG}"
|
||||||
else
|
|
||||||
git log "${TAG}" --no-merges --pretty='- %s' | grep -v '^- chore(release)' || true
|
|
||||||
fi
|
fi
|
||||||
echo "__NOTES_EOF__"
|
echo "__NOTES_EOF__"
|
||||||
} >> "$GITHUB_OUTPUT"
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
|||||||
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.4.1"
|
version = "1.5.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.4.1"
|
version = "1.5.1"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "Fast browser automation CLI for AI agents"
|
description = "Fast browser automation CLI for AI agents"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
+202
-26
@@ -424,6 +424,9 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
// === Core Actions ===
|
// === Core Actions ===
|
||||||
"click" => {
|
"click" => {
|
||||||
let new_tab = rest.contains(&"--new-tab");
|
let new_tab = rest.contains(&"--new-tab");
|
||||||
|
// `--follow`: if the click opens a new tab, switch the active tab to
|
||||||
|
// it (default reports the opened tab but stays put) (issue #24-A).
|
||||||
|
let follow = rest.contains(&"--follow");
|
||||||
// Coordinate click as a first-class form (issue #8.4): when the only
|
// Coordinate click as a first-class form (issue #8.4): when the only
|
||||||
// handle is a pixel position, no element/selector is needed.
|
// handle is a pixel position, no element/selector is needed.
|
||||||
// click <x> <y> e.g. click 449 320
|
// click <x> <y> e.g. click 449 320
|
||||||
@@ -432,23 +435,27 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
let coord_args: Vec<&str> = rest
|
let coord_args: Vec<&str> = rest
|
||||||
.iter()
|
.iter()
|
||||||
.copied()
|
.copied()
|
||||||
.filter(|a| *a != "--new-tab" && *a != "--coords")
|
.filter(|a| !a.starts_with("--"))
|
||||||
.collect();
|
.collect();
|
||||||
if let Some((x, y)) = parse_coords(&coord_args) {
|
if let Some((x, y)) = parse_coords(&coord_args) {
|
||||||
return Ok(json!({ "id": id, "action": "click", "x": x, "y": y }));
|
return Ok(json!({ "id": id, "action": "click", "x": x, "y": y }));
|
||||||
}
|
}
|
||||||
let sel = rest
|
let sel = rest
|
||||||
.iter()
|
.iter()
|
||||||
.find(|arg| **arg != "--new-tab")
|
.find(|arg| !arg.starts_with("--"))
|
||||||
.ok_or_else(|| ParseError::MissingArguments {
|
.ok_or_else(|| ParseError::MissingArguments {
|
||||||
context: "click".to_string(),
|
context: "click".to_string(),
|
||||||
usage: "click <selector> | click <x> <y> | click --coords <x>,<y> [--new-tab]",
|
usage:
|
||||||
|
"click <selector> | click <x> <y> | click --coords <x>,<y> [--new-tab] [--follow]",
|
||||||
})?;
|
})?;
|
||||||
|
let mut cmd = json!({ "id": id, "action": "click", "selector": sel });
|
||||||
if new_tab {
|
if new_tab {
|
||||||
Ok(json!({ "id": id, "action": "click", "selector": sel, "newTab": true }))
|
cmd["newTab"] = json!(true);
|
||||||
} else {
|
|
||||||
Ok(json!({ "id": id, "action": "click", "selector": sel }))
|
|
||||||
}
|
}
|
||||||
|
if follow {
|
||||||
|
cmd["follow"] = json!(true);
|
||||||
|
}
|
||||||
|
Ok(cmd)
|
||||||
}
|
}
|
||||||
"dblclick" => {
|
"dblclick" => {
|
||||||
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
@@ -583,11 +590,27 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
|
|
||||||
// === Keyboard ===
|
// === Keyboard ===
|
||||||
"press" | "key" => {
|
"press" | "key" => {
|
||||||
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let key = rest.iter().find(|a| !a.starts_with("--")).ok_or_else(|| {
|
||||||
context: "press".to_string(),
|
ParseError::MissingArguments {
|
||||||
usage: "press <key>",
|
context: "press".to_string(),
|
||||||
|
usage: "press <key> [--hold <ms>]",
|
||||||
|
}
|
||||||
})?;
|
})?;
|
||||||
Ok(json!({ "id": id, "action": "press", "key": key }))
|
let mut c = json!({ "id": id, "action": "press", "key": key });
|
||||||
|
// `--hold <ms>`: hold the key down for <ms> then release, timed inside
|
||||||
|
// the daemon (one round-trip, no shell-sleep jitter) — for games and
|
||||||
|
// hold-to-charge where keydown+sleep+keyup over 3 round-trips is too
|
||||||
|
// imprecise.
|
||||||
|
if let Some(i) = rest.iter().position(|a| *a == "--hold") {
|
||||||
|
let ms = rest.get(i + 1).and_then(|s| s.parse::<u64>().ok()).ok_or(
|
||||||
|
ParseError::MissingArguments {
|
||||||
|
context: "press --hold".to_string(),
|
||||||
|
usage: "press <key> --hold <ms>",
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
c["hold"] = json!(ms);
|
||||||
|
}
|
||||||
|
Ok(c)
|
||||||
}
|
}
|
||||||
"keydown" => {
|
"keydown" => {
|
||||||
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
@@ -1003,7 +1026,22 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
}
|
}
|
||||||
|
|
||||||
// === Close ===
|
// === Close ===
|
||||||
"close" | "quit" | "exit" => Ok(json!({ "id": id, "action": "close" })),
|
"close" | "quit" | "exit" => {
|
||||||
|
// `close <tab>` closes only that tab (and the output says "Tab
|
||||||
|
// closed"); bare `close` closes the browser/session. `close --all` is
|
||||||
|
// intercepted earlier in the dispatcher. Previously `close t12` still
|
||||||
|
// ran a browser close and alarmingly printed "Browser closed" (#26).
|
||||||
|
if let Some(tab_ref) = rest.iter().find(|a| !a.starts_with("--")) {
|
||||||
|
Ok(json!({ "id": id, "action": "tab_close", "tabId": tab_ref }))
|
||||||
|
} else {
|
||||||
|
Ok(json!({ "id": id, "action": "close" }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The active tab's stable handle — `targetId` survives cross-process
|
||||||
|
// navigation and is reusable across sessions, so an agent can hold it
|
||||||
|
// instead of re-deriving "which tab is live" from `tabs` each step (#26).
|
||||||
|
"current" => Ok(json!({ "id": id, "action": "current" })),
|
||||||
|
|
||||||
// === Inspect ===
|
// === Inspect ===
|
||||||
"inspect" => Ok(json!({ "id": id, "action": "inspect" })),
|
"inspect" => Ok(json!({ "id": id, "action": "inspect" })),
|
||||||
@@ -1258,6 +1296,11 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
// === Get ===
|
// === Get ===
|
||||||
"get" => parse_get(&rest, &id),
|
"get" => parse_get(&rest, &id),
|
||||||
|
|
||||||
|
// List every frame the session can reach (top + same-process child
|
||||||
|
// frames + out-of-process iframes), with a text-length per frame so you
|
||||||
|
// can see where a listing's description actually lives (issue #27).
|
||||||
|
"frames" => Ok(json!({ "id": id, "action": "frames" })),
|
||||||
|
|
||||||
// Top-level shortcuts for `get <x>` status reads — users naturally type
|
// Top-level shortcuts for `get <x>` status reads — users naturally type
|
||||||
// `chrome-use url` / `cdp-url` / `title` without the `get` prefix
|
// `chrome-use url` / `cdp-url` / `title` without the `get` prefix
|
||||||
// (and expect `cdp-url`/`cdp_url` to work interchangeably).
|
// (and expect `cdp-url`/`cdp_url` to work interchangeably).
|
||||||
@@ -1559,11 +1602,16 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
}
|
}
|
||||||
Ok(cmd)
|
Ok(cmd)
|
||||||
}
|
}
|
||||||
Some(tab_ref) => Ok(json!({
|
Some(tab_ref) => {
|
||||||
"id": id,
|
// `tab <ref> --activate` (alias `--front`) switches to the tab
|
||||||
"action": "tab_switch",
|
// AND raises it to the foreground — for handing a specific tab
|
||||||
"tabId": tab_ref,
|
// to the human (SMS code, captcha) (issue #24-C).
|
||||||
})),
|
let mut cmd = json!({ "id": id, "action": "tab_switch", "tabId": tab_ref });
|
||||||
|
if rest.iter().any(|a| *a == "--activate" || *a == "--front") {
|
||||||
|
cmd["activate"] = json!(true);
|
||||||
|
}
|
||||||
|
Ok(cmd)
|
||||||
|
}
|
||||||
None => {
|
None => {
|
||||||
let mut cmd = json!({ "id": id, "action": "tab_list" });
|
let mut cmd = json!({ "id": id, "action": "tab_list" });
|
||||||
if full {
|
if full {
|
||||||
@@ -2345,10 +2393,32 @@ fn parse_get(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
|||||||
|
|
||||||
match rest.first().copied() {
|
match rest.first().copied() {
|
||||||
Some("text") => {
|
Some("text") => {
|
||||||
let sel = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
// `get text --all-frames` aggregates visible text across every
|
||||||
context: "get text".to_string(),
|
// frame, including out-of-process iframes invisible to the top
|
||||||
usage: "get text <selector>",
|
// document (issue #27). The selector is ignored in this mode.
|
||||||
})?;
|
let all_frames = rest[1..]
|
||||||
|
.iter()
|
||||||
|
.any(|a| matches!(*a, "--all-frames" | "--frames" | "-a"));
|
||||||
|
if all_frames {
|
||||||
|
return Ok(json!({ "id": id, "action": "gettext", "allFrames": true }));
|
||||||
|
}
|
||||||
|
// `get text --main` returns the main-content region (readability),
|
||||||
|
// skipping header/nav/footer/sidebar boilerplate (issue #27).
|
||||||
|
let main = rest[1..]
|
||||||
|
.iter()
|
||||||
|
.any(|a| matches!(*a, "--main" | "--readable" | "-m"));
|
||||||
|
if main {
|
||||||
|
return Ok(json!({ "id": id, "action": "gettext", "main": true }));
|
||||||
|
}
|
||||||
|
// `get text` with no selector returns the whole page's text (body) —
|
||||||
|
// a common convenience; previously it errored without a selector
|
||||||
|
// (issue #24-D).
|
||||||
|
let sel = rest
|
||||||
|
.iter()
|
||||||
|
.skip(1)
|
||||||
|
.find(|a| !a.starts_with("--"))
|
||||||
|
.copied()
|
||||||
|
.unwrap_or("body");
|
||||||
Ok(json!({ "id": id, "action": "gettext", "selector": sel }))
|
Ok(json!({ "id": id, "action": "gettext", "selector": sel }))
|
||||||
}
|
}
|
||||||
Some("html") => {
|
Some("html") => {
|
||||||
@@ -3581,6 +3651,22 @@ mod tests {
|
|||||||
assert_eq!(cmd["url"], "https://example.com");
|
assert_eq!(cmd["url"], "https://example.com");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_press_plain_and_hold() {
|
||||||
|
let cmd = parse_command(&args("press d"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "press");
|
||||||
|
assert_eq!(cmd["key"], "d");
|
||||||
|
assert!(cmd.get("hold").is_none());
|
||||||
|
|
||||||
|
let held = parse_command(&args("press d --hold 800"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(held["key"], "d");
|
||||||
|
assert_eq!(held["hold"], 800);
|
||||||
|
|
||||||
|
// Missing/invalid duration is an error, not a silent no-hold.
|
||||||
|
assert!(parse_command(&args("press d --hold"), &default_flags()).is_err());
|
||||||
|
assert!(parse_command(&args("press d --hold abc"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_navigate_reuse_tab_flag() {
|
fn test_navigate_reuse_tab_flag() {
|
||||||
let cmd = parse_command(
|
let cmd = parse_command(
|
||||||
@@ -3747,6 +3833,21 @@ mod tests {
|
|||||||
assert!(cmd.get("x").is_none());
|
assert!(cmd.get("x").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_click_follow_flag() {
|
||||||
|
// `--follow` sets the flag; the selector is still found even with the flag
|
||||||
|
// before it (issue #24-A).
|
||||||
|
let cmd = parse_command(&args("click @e5 --follow"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["selector"], "@e5");
|
||||||
|
assert_eq!(cmd["follow"], true);
|
||||||
|
let cmd2 = parse_command(&args("click --follow @e5"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd2["selector"], "@e5");
|
||||||
|
assert_eq!(cmd2["follow"], true);
|
||||||
|
// Absent by default.
|
||||||
|
let plain = parse_command(&args("click @e5"), &default_flags()).unwrap();
|
||||||
|
assert!(plain.get("follow").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_tabs_alias_lists() {
|
fn test_tabs_alias_lists() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -3955,6 +4056,28 @@ mod tests {
|
|||||||
assert_eq!(cmd["tabId"], "docs");
|
assert_eq!(cmd["tabId"], "docs");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_close_tab_vs_browser() {
|
||||||
|
// `close <tab>` closes that tab (says "Tab closed"); bare `close` closes
|
||||||
|
// the browser (#26).
|
||||||
|
let tab = parse_command(&args("close t12"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(tab["action"], "tab_close");
|
||||||
|
assert_eq!(tab["tabId"], "t12");
|
||||||
|
let browser = parse_command(&args("close"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(browser["action"], "close");
|
||||||
|
// `quit`/`exit` aliases still browser-close.
|
||||||
|
assert_eq!(
|
||||||
|
parse_command(&args("quit"), &default_flags()).unwrap()["action"],
|
||||||
|
"close"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_current_command() {
|
||||||
|
let cmd = parse_command(&args("current"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "current");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_tab_sends_string_tab_id() {
|
fn test_tab_sends_string_tab_id() {
|
||||||
let cmd = parse_command(&args("tab t2"), &default_flags()).unwrap();
|
let cmd = parse_command(&args("tab t2"), &default_flags()).unwrap();
|
||||||
@@ -4654,12 +4777,65 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_text_missing_selector() {
|
fn test_get_text_defaults_to_body() {
|
||||||
let result = parse_command(&args("get text"), &default_flags());
|
// `get text` with no selector now returns the whole page (body) instead
|
||||||
assert!(result.is_err());
|
// of erroring (issue #24-D).
|
||||||
let err = result.unwrap_err();
|
let cmd = parse_command(&args("get text"), &default_flags()).unwrap();
|
||||||
assert!(matches!(err, ParseError::MissingArguments { .. }));
|
assert_eq!(cmd["action"], "gettext");
|
||||||
assert!(err.format().contains("get text"));
|
assert_eq!(cmd["selector"], "body");
|
||||||
|
// An explicit selector still wins.
|
||||||
|
let cmd2 = parse_command(&args("get text h1"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd2["selector"], "h1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_text_all_frames() {
|
||||||
|
// `--all-frames` switches to whole-page, cross-frame aggregation and
|
||||||
|
// drops the selector (issue #27).
|
||||||
|
for variant in ["get text --all-frames", "get text --frames", "text -a"] {
|
||||||
|
let cmd = parse_command(&args(variant), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "gettext", "{variant}");
|
||||||
|
assert_eq!(cmd["allFrames"], true, "{variant}");
|
||||||
|
assert!(cmd.get("selector").is_none(), "{variant}");
|
||||||
|
}
|
||||||
|
// A flag mixed with a selector still triggers all-frames.
|
||||||
|
let cmd = parse_command(&args("get text body --all-frames"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["allFrames"], true);
|
||||||
|
// Without the flag, a leading flag-like token is skipped for the selector.
|
||||||
|
let cmd = parse_command(&args("get text main"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["selector"], "main");
|
||||||
|
assert!(cmd.get("allFrames").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_frames_command() {
|
||||||
|
let cmd = parse_command(&args("frames"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "frames");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_text_main() {
|
||||||
|
for variant in ["get text --main", "get text --readable", "text -m"] {
|
||||||
|
let cmd = parse_command(&args(variant), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "gettext", "{variant}");
|
||||||
|
assert_eq!(cmd["main"], true, "{variant}");
|
||||||
|
assert!(cmd.get("selector").is_none(), "{variant}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tab_activate_flag() {
|
||||||
|
let plain = parse_command(&args("tab t3"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(plain["action"], "tab_switch");
|
||||||
|
assert!(plain.get("activate").is_none());
|
||||||
|
|
||||||
|
let act = parse_command(&args("tab t3 --activate"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(act["action"], "tab_switch");
|
||||||
|
assert_eq!(act["tabId"], "t3");
|
||||||
|
assert_eq!(act["activate"], true);
|
||||||
|
// `--front` alias.
|
||||||
|
let front = parse_command(&args("tab t3 --front"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(front["activate"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// === Protocol alignment tests ===
|
// === Protocol alignment tests ===
|
||||||
|
|||||||
+131
-1
@@ -1332,6 +1332,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
|||||||
"uncheck" => handle_uncheck(cmd, state).await,
|
"uncheck" => handle_uncheck(cmd, state).await,
|
||||||
"wait" => handle_wait(cmd, state).await,
|
"wait" => handle_wait(cmd, state).await,
|
||||||
"gettext" => handle_gettext(cmd, state).await,
|
"gettext" => handle_gettext(cmd, state).await,
|
||||||
|
"frames" => handle_frames(cmd, state).await,
|
||||||
"getattribute" => handle_getattribute(cmd, state).await,
|
"getattribute" => handle_getattribute(cmd, state).await,
|
||||||
"isvisible" => handle_isvisible(cmd, state).await,
|
"isvisible" => handle_isvisible(cmd, state).await,
|
||||||
"isenabled" => handle_isenabled(cmd, state).await,
|
"isenabled" => handle_isenabled(cmd, state).await,
|
||||||
@@ -1395,6 +1396,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
|||||||
"count" => handle_count(cmd, state).await,
|
"count" => handle_count(cmd, state).await,
|
||||||
"styles" => handle_styles(cmd, state).await,
|
"styles" => handle_styles(cmd, state).await,
|
||||||
"bringtofront" => handle_bringtofront(state).await,
|
"bringtofront" => handle_bringtofront(state).await,
|
||||||
|
"current" => handle_current(state).await,
|
||||||
"timezone" => handle_timezone(cmd, state).await,
|
"timezone" => handle_timezone(cmd, state).await,
|
||||||
"locale" => handle_locale(cmd, state).await,
|
"locale" => handle_locale(cmd, state).await,
|
||||||
"geolocation" => handle_geolocation(cmd, state).await,
|
"geolocation" => handle_geolocation(cmd, state).await,
|
||||||
@@ -3116,6 +3118,15 @@ async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
|
|||||||
|
|
||||||
let button = cmd.get("button").and_then(|v| v.as_str()).unwrap_or("left");
|
let button = cmd.get("button").and_then(|v| v.as_str()).unwrap_or("left");
|
||||||
let click_count = cmd.get("clickCount").and_then(|v| v.as_i64()).unwrap_or(1) as i32;
|
let click_count = cmd.get("clickCount").and_then(|v| v.as_i64()).unwrap_or(1) as i32;
|
||||||
|
let follow = cmd.get("follow").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||||
|
|
||||||
|
// Snapshot tracked targets so we can tell if this click opened a NEW tab
|
||||||
|
// (target=_blank link / window.open). On the relay the new tab is discovered
|
||||||
|
// passively and doesn't steal focus (#7/#8.1), so without surfacing it the
|
||||||
|
// post-click snapshot shows the OLD page and looks like the click failed
|
||||||
|
// (issue #24-A).
|
||||||
|
let before: std::collections::HashSet<String> =
|
||||||
|
mgr.pages_list().into_iter().map(|p| p.target_id).collect();
|
||||||
|
|
||||||
interaction::click(
|
interaction::click(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
@@ -3128,7 +3139,26 @@ async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(json!({ "clicked": selector }))
|
// Give a just-opened tab a moment to register, then look for it.
|
||||||
|
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(150)).await;
|
||||||
|
let opened = mgr.adopt_newly_opened(&before).await;
|
||||||
|
|
||||||
|
let mut out = json!({ "clicked": selector });
|
||||||
|
if let Some(page) = opened {
|
||||||
|
let tab_id = super::browser::format_tab_id(page.tab_id);
|
||||||
|
out["openedTab"] = json!({ "tabId": tab_id, "url": page.url, "title": page.title });
|
||||||
|
// `--follow`: switch the active tab to the newly-opened one (default is
|
||||||
|
// to report it but stay put, so multi-tab flows aren't hijacked).
|
||||||
|
if follow {
|
||||||
|
state.ref_map.clear();
|
||||||
|
state.iframe_sessions.clear();
|
||||||
|
state.active_frame_id = None;
|
||||||
|
let _ = mgr.tab_switch_by_id(page.tab_id).await;
|
||||||
|
out["followed"] = json!(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_dblclick(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_dblclick(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
@@ -3334,6 +3364,16 @@ async fn handle_press(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
|
|||||||
// Parse modifier+key chords like "Control+a", "Shift+Enter", "Control+Shift+a"
|
// Parse modifier+key chords like "Control+a", "Shift+Enter", "Control+Shift+a"
|
||||||
let (actual_key, modifiers) = parse_key_chord(key);
|
let (actual_key, modifiers) = parse_key_chord(key);
|
||||||
|
|
||||||
|
// `--hold <ms>`: keyDown, wait, keyUp — all inside the daemon so the hold
|
||||||
|
// duration is precise (no shell-sleep / round-trip jitter). For games
|
||||||
|
// (hold-to-move/charge) and any press-and-hold interaction.
|
||||||
|
if let Some(ms) = cmd.get("hold").and_then(|v| v.as_u64()) {
|
||||||
|
interaction::dispatch_single_key(&mgr.client, &session_id, &actual_key, "keyDown").await?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(ms)).await;
|
||||||
|
interaction::dispatch_single_key(&mgr.client, &session_id, &actual_key, "keyUp").await?;
|
||||||
|
return Ok(json!({ "pressed": key, "heldMs": ms }));
|
||||||
|
}
|
||||||
|
|
||||||
interaction::press_key_with_modifiers(&mgr.client, &session_id, &actual_key, modifiers).await?;
|
interaction::press_key_with_modifiers(&mgr.client, &session_id, &actual_key, modifiers).await?;
|
||||||
Ok(json!({ "pressed": key }))
|
Ok(json!({ "pressed": key }))
|
||||||
}
|
}
|
||||||
@@ -3555,6 +3595,47 @@ async fn handle_wait(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
|
||||||
|
// `get text --all-frames` aggregates visible text across every frame the
|
||||||
|
// session can reach — including out-of-process iframes that never show up
|
||||||
|
// in the top document (#27: Yahoo/Rakuten/Mercari listing descriptions).
|
||||||
|
if cmd.get("allFrames").and_then(|v| v.as_bool()) == Some(true) {
|
||||||
|
let frames = super::element::collect_all_frames_text(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let mut combined = String::new();
|
||||||
|
let mut frame_count = 0usize;
|
||||||
|
for f in &frames {
|
||||||
|
let t = f.text.trim();
|
||||||
|
if t.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
frame_count += 1;
|
||||||
|
if f.kind != "top" {
|
||||||
|
combined.push_str(&format!("\n\n----- frame [{}] {} -----\n", f.kind, f.url));
|
||||||
|
}
|
||||||
|
combined.push_str(t);
|
||||||
|
}
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
return Ok(json!({
|
||||||
|
"text": combined,
|
||||||
|
"origin": url,
|
||||||
|
"frames": frame_count,
|
||||||
|
"allFrames": true,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// `get text --main` returns the page's main-content region (readability-lite),
|
||||||
|
// skipping global header/nav/footer/sidebar boilerplate (#27).
|
||||||
|
if cmd.get("main").and_then(|v| v.as_bool()) == Some(true) {
|
||||||
|
let text = super::element::get_main_content_text(&mgr.client, &session_id).await?;
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
return Ok(json!({ "text": text, "origin": url, "main": true }));
|
||||||
|
}
|
||||||
|
|
||||||
let selector = cmd
|
let selector = cmd
|
||||||
.get("selector")
|
.get("selector")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
@@ -3572,6 +3653,32 @@ async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, S
|
|||||||
Ok(json!({ "text": text, "origin": url }))
|
Ok(json!({ "text": text, "origin": url }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn handle_frames(_cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
|
let frames = super::element::collect_all_frames_text(
|
||||||
|
&mgr.client,
|
||||||
|
&session_id,
|
||||||
|
&state.iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let list: Vec<Value> = frames
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, f)| {
|
||||||
|
json!({
|
||||||
|
"index": i,
|
||||||
|
"kind": f.kind,
|
||||||
|
"url": f.url,
|
||||||
|
"frameId": f.frame_id,
|
||||||
|
"textLen": f.text.trim().chars().count(),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let url = mgr.get_url().await.unwrap_or_default();
|
||||||
|
Ok(json!({ "frames": list, "count": list.len(), "origin": url }))
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let session_id = mgr.active_session_id()?.to_string();
|
let session_id = mgr.active_session_id()?.to_string();
|
||||||
@@ -4456,6 +4563,17 @@ async fn handle_tab_switch(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
|||||||
state.active_frame_id = None;
|
state.active_frame_id = None;
|
||||||
let result = mgr.tab_switch_by_id(tab_id).await?;
|
let result = mgr.tab_switch_by_id(tab_id).await?;
|
||||||
|
|
||||||
|
// `--activate`: raise this tab to the foreground (the switch made it active;
|
||||||
|
// bring_to_front acts on the active tab) — for handing a specific tab to the
|
||||||
|
// human (issue #24-C). Best-effort; don't fail the switch if it can't.
|
||||||
|
if cmd
|
||||||
|
.get("activate")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
let _ = mgr.bring_to_front().await;
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(ref server) = state.stream_server {
|
if let Some(ref server) = state.stream_server {
|
||||||
if let Ok(dims) = mgr
|
if let Ok(dims) = mgr
|
||||||
.evaluate(
|
.evaluate(
|
||||||
@@ -5284,6 +5402,18 @@ async fn handle_bringtofront(state: &DaemonState) -> Result<Value, String> {
|
|||||||
Ok(json!({ "broughtToFront": true }))
|
Ok(json!({ "broughtToFront": true }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn handle_current(state: &mut DaemonState) -> Result<Value, String> {
|
||||||
|
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
|
||||||
|
// Refresh so `current` reflects the live URL/title even after a cross-process
|
||||||
|
// nav (the relay's cached target_info can lag) (#26).
|
||||||
|
mgr.resync_targets().await.ok();
|
||||||
|
let mut info = mgr.active_page_info().ok_or("No active tab")?;
|
||||||
|
if let Some(obj) = info.as_object_mut() {
|
||||||
|
obj.insert("current".to_string(), json!(true));
|
||||||
|
}
|
||||||
|
Ok(info)
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_timezone(cmd: &Value, state: &DaemonState) -> Result<Value, String> {
|
async fn handle_timezone(cmd: &Value, state: &DaemonState) -> Result<Value, String> {
|
||||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||||
let timezone = cmd
|
let timezone = cmd
|
||||||
|
|||||||
@@ -1263,6 +1263,22 @@ impl BrowserManager {
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The active tab's stable handle + current location, for `chrome-use
|
||||||
|
/// current` (#26). `targetId` survives cross-process navigation, so it's the
|
||||||
|
/// handle an agent should hold across a multi-step flow.
|
||||||
|
pub fn active_page_info(&self) -> Option<Value> {
|
||||||
|
let i = self.resolved_active_index();
|
||||||
|
self.pages.get(i).map(|p| {
|
||||||
|
json!({
|
||||||
|
"tabId": format_tab_id(p.tab_id),
|
||||||
|
"targetId": p.target_id,
|
||||||
|
"label": p.label,
|
||||||
|
"url": p.url,
|
||||||
|
"title": p.title,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Stable `tab_id` for a page identified by its CDP `targetId`, if tracked.
|
/// Stable `tab_id` for a page identified by its CDP `targetId`, if tracked.
|
||||||
/// Lets callers adopt a tab by the cross-session-stable target id.
|
/// Lets callers adopt a tab by the cross-session-stable target id.
|
||||||
pub fn tab_id_for_target(&self, target_id: &str) -> Option<u32> {
|
pub fn tab_id_for_target(&self, target_id: &str) -> Option<u32> {
|
||||||
@@ -1279,6 +1295,67 @@ impl BrowserManager {
|
|||||||
/// the active tab is preserved, and re-pinned if it was pruned. Powers a live
|
/// the active tab is preserved, and re-pinned if it was pruned. Powers a live
|
||||||
/// `tab list` and adopt-by-targetId so a fresh session can reach a stranded,
|
/// `tab list` and adopt-by-targetId so a fresh session can reach a stranded,
|
||||||
/// still-filled tab without reloading it (issue #21).
|
/// still-filled tab without reloading it (issue #21).
|
||||||
|
/// Detect targets that appeared since the `before` set (e.g. a click that
|
||||||
|
/// opened a new tab via a `target=_blank` link or `window.open`), attach +
|
||||||
|
/// track each in the background, and return the first newly-opened page.
|
||||||
|
///
|
||||||
|
/// Lighter than [`resync_targets`] — one `getTargets` and work only on the
|
||||||
|
/// new targets, no whole-tab url/title refresh — so it's cheap enough to run
|
||||||
|
/// after every click. The new tab is added in the background (never steals
|
||||||
|
/// the active tab, per #7/#8.1); the caller surfaces it so the agent knows a
|
||||||
|
/// tab opened instead of seeing the old page (issue #24-A).
|
||||||
|
pub async fn adopt_newly_opened(&mut self, before: &HashSet<String>) -> Option<PageInfo> {
|
||||||
|
let result: GetTargetsResult = self
|
||||||
|
.client
|
||||||
|
.send_command_typed("Target.getTargets", &json!({}), None)
|
||||||
|
.await
|
||||||
|
.ok()?;
|
||||||
|
let live: Vec<TargetInfo> = result
|
||||||
|
.target_infos
|
||||||
|
.into_iter()
|
||||||
|
.filter(should_track_target)
|
||||||
|
.collect();
|
||||||
|
let mut opened: Option<PageInfo> = None;
|
||||||
|
for target in &live {
|
||||||
|
if before.contains(&target.target_id)
|
||||||
|
|| self.pages.iter().any(|p| p.target_id == target.target_id)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let attach: AttachToTargetResult = match self
|
||||||
|
.client
|
||||||
|
.send_command_typed(
|
||||||
|
"Target.attachToTarget",
|
||||||
|
&AttachToTargetParams {
|
||||||
|
target_id: target.target_id.clone(),
|
||||||
|
flatten: true,
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
let tab_id = self.assign_tab_id();
|
||||||
|
let page = PageInfo {
|
||||||
|
tab_id,
|
||||||
|
label: None,
|
||||||
|
target_id: target.target_id.clone(),
|
||||||
|
session_id: attach.session_id.clone(),
|
||||||
|
url: target.url.clone(),
|
||||||
|
title: target.title.clone(),
|
||||||
|
target_type: target.target_type.clone(),
|
||||||
|
};
|
||||||
|
self.add_background_page(page.clone());
|
||||||
|
let _ = self.enable_domains(&attach.session_id).await;
|
||||||
|
if opened.is_none() {
|
||||||
|
opened = Some(page);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opened
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn resync_targets(&mut self) -> Result<(), String> {
|
pub async fn resync_targets(&mut self) -> Result<(), String> {
|
||||||
self.client
|
self.client
|
||||||
.send_command_typed::<_, Value>(
|
.send_command_typed::<_, Value>(
|
||||||
|
|||||||
+249
-8
@@ -796,16 +796,43 @@ pub(super) fn extract_ax_string(value: &Option<AXValue>) -> String {
|
|||||||
/// Build a JS expression that finds a DOM element by CSS selector or XPath.
|
/// Build a JS expression that finds a DOM element by CSS selector or XPath.
|
||||||
fn build_find_element_js(selector: &str) -> String {
|
fn build_find_element_js(selector: &str) -> String {
|
||||||
if let Some(xpath) = selector.strip_prefix("xpath=") {
|
if let Some(xpath) = selector.strip_prefix("xpath=") {
|
||||||
format!(
|
return format!(
|
||||||
"document.evaluate({}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue",
|
"document.evaluate({}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue",
|
||||||
serde_json::to_string(xpath).unwrap_or_default()
|
serde_json::to_string(xpath).unwrap_or_default()
|
||||||
)
|
);
|
||||||
} else {
|
|
||||||
format!(
|
|
||||||
"document.querySelector({})",
|
|
||||||
serde_json::to_string(selector).unwrap_or_default()
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
// Bare string (or explicit `text=`): try CSS first, then fall back to
|
||||||
|
// matching an interactive element by its VISIBLE TEXT. snapshot exposes
|
||||||
|
// buttons/links by their name, so `click "購入手続きへ"` should resolve by
|
||||||
|
// that label — previously it was fed straight to `querySelector` as CSS and
|
||||||
|
// failed as an invalid selector even though the button was right there
|
||||||
|
// (issue #24-B). CSS still wins when it matches, so existing selectors are
|
||||||
|
// unaffected; nested/non-ASCII labels now resolve too.
|
||||||
|
let text_only = selector.strip_prefix("text=");
|
||||||
|
let force_text = text_only.is_some();
|
||||||
|
let sel_json = serde_json::to_string(selector).unwrap_or_default();
|
||||||
|
let want_json = serde_json::to_string(text_only.unwrap_or(selector)).unwrap_or_default();
|
||||||
|
format!(
|
||||||
|
r#"(() => {{
|
||||||
|
const sel = {sel};
|
||||||
|
const css = {force_text} ? null : (() => {{ try {{ return document.querySelector(sel); }} catch (_e) {{ return null; }} }})();
|
||||||
|
if (css) return css;
|
||||||
|
const norm = s => (s == null ? '' : String(s)).replace(/\s+/g, ' ').trim();
|
||||||
|
const w = norm({want}); if (!w) return null;
|
||||||
|
const wl = w.toLowerCase();
|
||||||
|
const interactive = Array.from(document.querySelectorAll(
|
||||||
|
'button,a,[role=button],[role=link],[role=menuitem],[role=tab],[role=option],input[type=submit],input[type=button],input[type=reset],summary,label,[onclick]'));
|
||||||
|
const textOf = e => norm(e.innerText || e.textContent) || norm(e.value) ||
|
||||||
|
norm(e.getAttribute && e.getAttribute('aria-label')) || norm(e.getAttribute && e.getAttribute('title'));
|
||||||
|
let hit = interactive.find(e => textOf(e) === w) || interactive.find(e => textOf(e).toLowerCase().includes(wl));
|
||||||
|
if (hit) return hit;
|
||||||
|
const leaves = Array.from(document.querySelectorAll('*')).filter(e => !e.children.length);
|
||||||
|
return leaves.find(e => norm(e.textContent) === w) || leaves.find(e => norm(e.textContent).toLowerCase().includes(wl)) || null;
|
||||||
|
}})()"#,
|
||||||
|
sel = sel_json,
|
||||||
|
want = want_json,
|
||||||
|
force_text = force_text
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a JS expression that counts matching DOM elements by CSS selector or XPath.
|
/// Build a JS expression that counts matching DOM elements by CSS selector or XPath.
|
||||||
@@ -948,6 +975,200 @@ pub async fn get_element_text(
|
|||||||
.unwrap_or_default())
|
.unwrap_or_default())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Text content collected from a single frame of the page.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct FrameText {
|
||||||
|
pub frame_id: String,
|
||||||
|
pub url: String,
|
||||||
|
/// "top" | "inline" (same-process child frame) | "oopif" (out-of-process).
|
||||||
|
pub kind: &'static str,
|
||||||
|
pub text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// The expression we run in every frame to read its visible text. innerText
|
||||||
|
// honors CSS visibility (skips display:none), textContent is the fallback.
|
||||||
|
const FRAME_INNERTEXT_JS: &str = "(function(){try{var b=document.body||document.documentElement;return b?(b.innerText||b.textContent||''):'';}catch(e){return '';}})()";
|
||||||
|
|
||||||
|
async fn eval_text_default(client: &CdpClient, session_id: &str) -> String {
|
||||||
|
let res = client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(serde_json::json!({
|
||||||
|
"expression": FRAME_INNERTEXT_JS,
|
||||||
|
"returnByValue": true,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
res.ok()
|
||||||
|
.and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned())
|
||||||
|
.and_then(|v| v.as_str().map(|s| s.to_string()))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same-process child frames share the top renderer but live in their own
|
||||||
|
// execution context. Page.createIsolatedWorld hands us a context id bound to
|
||||||
|
// that frame so Runtime.evaluate reads the child document, not the parent.
|
||||||
|
async fn eval_text_in_frame(client: &CdpClient, session_id: &str, frame_id: &str) -> String {
|
||||||
|
let ctx = client
|
||||||
|
.send_command(
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
Some(serde_json::json!({ "frameId": frame_id, "worldName": "chrome_use_text" })),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.get("executionContextId").and_then(|c| c.as_i64()));
|
||||||
|
let Some(ctx_id) = ctx else { return String::new() };
|
||||||
|
let res = client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(serde_json::json!({
|
||||||
|
"expression": FRAME_INNERTEXT_JS,
|
||||||
|
"returnByValue": true,
|
||||||
|
"contextId": ctx_id,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
res.ok()
|
||||||
|
.and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned())
|
||||||
|
.and_then(|v| v.as_str().map(|s| s.to_string()))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flatten_frame_tree(node: &Value, is_top: bool, out: &mut Vec<(String, String, bool)>) {
|
||||||
|
if let Some(frame) = node.get("frame") {
|
||||||
|
if let Some(id) = frame.get("id").and_then(|v| v.as_str()) {
|
||||||
|
let url = frame
|
||||||
|
.get("url")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
out.push((id.to_string(), url, is_top));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(children) = node.get("childFrames").and_then(|v| v.as_array()) {
|
||||||
|
for child in children {
|
||||||
|
flatten_frame_tree(child, false, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collect visible text from every frame reachable in the active session,
|
||||||
|
/// including out-of-process iframes (which never appear in the top frame's
|
||||||
|
/// `Page.getFrameTree` and so are invisible to `document.body.innerText`).
|
||||||
|
///
|
||||||
|
/// Same-process child frames are read through `Page.createIsolatedWorld`;
|
||||||
|
/// OOPIFs are read through their own auto-attached debugger session
|
||||||
|
/// (`iframe_sessions`, keyed by frameId == targetId). This is the engine
|
||||||
|
/// behind `get text --all-frames` and `chrome-use frames` — the fix for
|
||||||
|
/// listing/marketplace pages whose description lives in a child frame (#27).
|
||||||
|
pub async fn collect_all_frames_text(
|
||||||
|
client: &CdpClient,
|
||||||
|
top_session: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<Vec<FrameText>, String> {
|
||||||
|
let mut out: Vec<FrameText> = Vec::new();
|
||||||
|
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
// 1. Top session: the top frame plus its same-process descendants. OOPIF
|
||||||
|
// frames that happen to surface here are skipped — they're read via
|
||||||
|
// their dedicated session in step 2 (cross-process isolated worlds fail).
|
||||||
|
let tree = client
|
||||||
|
.send_command_no_params("Page.getFrameTree", Some(top_session))
|
||||||
|
.await?;
|
||||||
|
let mut frames: Vec<(String, String, bool)> = Vec::new();
|
||||||
|
flatten_frame_tree(&tree["frameTree"], true, &mut frames);
|
||||||
|
for (fid, url, is_top) in frames {
|
||||||
|
if iframe_sessions.contains_key(&fid) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !seen.insert(fid.clone()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let (kind, text) = if is_top {
|
||||||
|
("top", eval_text_default(client, top_session).await)
|
||||||
|
} else {
|
||||||
|
("inline", eval_text_in_frame(client, top_session, &fid).await)
|
||||||
|
};
|
||||||
|
out.push(FrameText {
|
||||||
|
frame_id: fid,
|
||||||
|
url,
|
||||||
|
kind,
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Each out-of-process iframe, read through its own session.
|
||||||
|
for (fid, sid) in iframe_sessions {
|
||||||
|
if !seen.insert(fid.clone()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let url = client
|
||||||
|
.send_command_no_params("Page.getFrameTree", Some(sid))
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.and_then(|t| {
|
||||||
|
t.get("frameTree")
|
||||||
|
.and_then(|ft| ft.get("frame"))
|
||||||
|
.and_then(|f| f.get("url"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let text = eval_text_default(client, sid).await;
|
||||||
|
out.push(FrameText {
|
||||||
|
frame_id: fid.clone(),
|
||||||
|
url,
|
||||||
|
kind: "oopif",
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Readability-lite: prefer the page's semantic main-content region over the
|
||||||
|
// whole body so global header/nav/footer chrome (and, on many listing pages,
|
||||||
|
// the "related items" sidebar) doesn't drown out the actual content. Runs on
|
||||||
|
// the live, rendered tree (innerText needs layout — a detached clone returns
|
||||||
|
// empty), so we pick the densest <main>/<article> region rather than cloning
|
||||||
|
// and stripping. Falls back to <body> when no substantial main region exists.
|
||||||
|
const MAIN_CONTENT_JS: &str = r#"(function(){
|
||||||
|
function txt(el){try{return (el.innerText||'').trim();}catch(e){return '';}}
|
||||||
|
var sels=['main','[role=main]','article','#main','#contents','#l-content'];
|
||||||
|
var best=null,bestLen=0;
|
||||||
|
for(var i=0;i<sels.length;i++){
|
||||||
|
var els=document.querySelectorAll(sels[i]);
|
||||||
|
for(var j=0;j<els.length;j++){var l=txt(els[j]).length;if(l>bestLen){bestLen=l;best=els[j];}}
|
||||||
|
}
|
||||||
|
if(best&&bestLen>200)return txt(best);
|
||||||
|
return txt(document.body);
|
||||||
|
})()"#;
|
||||||
|
|
||||||
|
/// Extract the page's main-content text (readability-lite), preferring a
|
||||||
|
/// semantic `<main>`/`<article>` region over the full body. Used by
|
||||||
|
/// `get text --main` to avoid header/nav/sidebar boilerplate (#27).
|
||||||
|
pub async fn get_main_content_text(client: &CdpClient, session_id: &str) -> Result<String, String> {
|
||||||
|
let res = client
|
||||||
|
.send_command(
|
||||||
|
"Runtime.evaluate",
|
||||||
|
Some(serde_json::json!({
|
||||||
|
"expression": MAIN_CONTENT_JS,
|
||||||
|
"returnByValue": true,
|
||||||
|
})),
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(res
|
||||||
|
.get("result")
|
||||||
|
.and_then(|r| r.get("value"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get_element_attribute(
|
pub async fn get_element_attribute(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
@@ -1452,10 +1673,30 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_build_selector_js_css() {
|
fn test_build_selector_js_css() {
|
||||||
let js = build_selector_js("#submit-btn");
|
let js = build_selector_js("#submit-btn");
|
||||||
assert!(js.contains("document.querySelector(\"#submit-btn\")"));
|
// CSS is now tried via a `sel` variable, with a visible-text fallback
|
||||||
|
// appended (issue #24-B). It must still use querySelector (not xpath).
|
||||||
|
assert!(js.contains("const sel = \"#submit-btn\""));
|
||||||
|
assert!(js.contains("document.querySelector(sel)"));
|
||||||
assert!(!js.contains("document.evaluate"));
|
assert!(!js.contains("document.evaluate"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_find_element_js_text_fallback() {
|
||||||
|
// A bare label gets a text-matching fallback so `click "購入手続きへ"`
|
||||||
|
// resolves by visible text, not just CSS (issue #24-B).
|
||||||
|
let js = build_find_element_js("購入手続きへ");
|
||||||
|
assert!(js.contains("購入手続きへ"));
|
||||||
|
assert!(js.contains("interactive")); // the text-match branch
|
||||||
|
assert!(js.contains("textOf"));
|
||||||
|
// `text=` forces the text path (skips CSS).
|
||||||
|
let forced = build_find_element_js("text=Buy now");
|
||||||
|
assert!(forced.contains("true ? null")); // force_text => css skipped
|
||||||
|
// xpath is unchanged.
|
||||||
|
let xp = build_find_element_js("xpath=//button");
|
||||||
|
assert!(xp.contains("document.evaluate"));
|
||||||
|
assert!(!xp.contains("interactive"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_build_selector_js_xpath() {
|
fn test_build_selector_js_xpath() {
|
||||||
let js = build_selector_js("xpath=//button[@id='ok']");
|
let js = build_selector_js("xpath=//button[@id='ok']");
|
||||||
|
|||||||
@@ -306,32 +306,50 @@ pub async fn fill(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Focus the element
|
// Emulate a real edit so framework-controlled inputs (React/Vue) and
|
||||||
client
|
// site-side listeners actually see the change (issue #25): the old path set
|
||||||
.send_command_typed::<_, Value>(
|
// `this.value` directly and used Input.insertText, which left React's
|
||||||
"Runtime.callFunctionOn",
|
// internal value-tracker out of sync and never fired change/blur — so
|
||||||
&CallFunctionOnParams {
|
// dependent logic (e.g. Mercari's postal-code → 都道府県 autocomplete) never
|
||||||
function_declaration: "function() { this.focus(); }".to_string(),
|
// ran even though the value was visible. Set the value through the element's
|
||||||
object_id: Some(object_id.clone()),
|
// PROTOTYPE setter (which React's _valueTracker hooks), then dispatch
|
||||||
arguments: None,
|
// input → change → blur/focusout. `type <sel> <text>` remains for sites that
|
||||||
return_by_value: Some(true),
|
// need per-keystroke events.
|
||||||
await_promise: Some(false),
|
let fill_js = format!(
|
||||||
},
|
r#"function() {{
|
||||||
Some(&effective_session_id),
|
const el = this;
|
||||||
)
|
const v = {val};
|
||||||
.await?;
|
try {{ el.focus(); }} catch (e) {{}}
|
||||||
|
const tag = el.tagName;
|
||||||
|
const fire = (type, ctor) => el.dispatchEvent(new (ctor || Event)(type, {{ bubbles: true }}));
|
||||||
|
if (tag === 'SELECT') {{
|
||||||
|
el.value = v; fire('input'); fire('change'); return true;
|
||||||
|
}}
|
||||||
|
if (el.isContentEditable) {{
|
||||||
|
el.textContent = v; fire('input', window.InputEvent || Event); fire('change');
|
||||||
|
try {{ el.blur(); }} catch (e) {{}} fire('focusout'); return true;
|
||||||
|
}}
|
||||||
|
const proto = tag === 'TEXTAREA' ? window.HTMLTextAreaElement.prototype
|
||||||
|
: window.HTMLInputElement.prototype;
|
||||||
|
const desc = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||||
|
const set = desc && desc.set ? (x) => desc.set.call(el, x) : (x) => {{ el.value = x; }};
|
||||||
|
set(''); // reset the framework tracker
|
||||||
|
fire('input', window.InputEvent || Event);
|
||||||
|
set(v); // native setter → React/Vue registers
|
||||||
|
fire('input', window.InputEvent || Event);
|
||||||
|
fire('change');
|
||||||
|
try {{ el.blur(); }} catch (e) {{}}
|
||||||
|
fire('focusout'); // blur-triggered lookups/validation
|
||||||
|
return true;
|
||||||
|
}}"#,
|
||||||
|
val = serde_json::to_string(value).unwrap_or_default()
|
||||||
|
);
|
||||||
|
|
||||||
// Select all + delete to clear
|
|
||||||
client
|
client
|
||||||
.send_command_typed::<_, Value>(
|
.send_command_typed::<_, Value>(
|
||||||
"Runtime.callFunctionOn",
|
"Runtime.callFunctionOn",
|
||||||
&CallFunctionOnParams {
|
&CallFunctionOnParams {
|
||||||
function_declaration: r#"function() {
|
function_declaration: fill_js,
|
||||||
this.select && this.select();
|
|
||||||
this.value = '';
|
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
||||||
}"#
|
|
||||||
.to_string(),
|
|
||||||
object_id: Some(object_id),
|
object_id: Some(object_id),
|
||||||
arguments: None,
|
arguments: None,
|
||||||
return_by_value: Some(true),
|
return_by_value: Some(true),
|
||||||
@@ -341,17 +359,6 @@ pub async fn fill(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Insert text (keyboard input dispatched at page level, use parent session_id)
|
|
||||||
client
|
|
||||||
.send_command_typed::<_, Value>(
|
|
||||||
"Input.insertText",
|
|
||||||
&InsertTextParams {
|
|
||||||
text: value.to_string(),
|
|
||||||
},
|
|
||||||
Some(session_id),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+89
-1
@@ -186,6 +186,44 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(data) = &resp.data {
|
if let Some(data) = &resp.data {
|
||||||
|
// A click that opened a new tab: surface it so the agent doesn't read the
|
||||||
|
// unchanged old page as a failed click (issue #24-A).
|
||||||
|
if let Some(opened) = data.get("openedTab") {
|
||||||
|
let tid = opened.get("tabId").and_then(|v| v.as_str()).unwrap_or("?");
|
||||||
|
let url = opened.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let followed = data
|
||||||
|
.get("followed")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
let verb = if followed {
|
||||||
|
"switched to new tab"
|
||||||
|
} else {
|
||||||
|
"opened new tab"
|
||||||
|
};
|
||||||
|
eprintln!(
|
||||||
|
"{} {} [{}] {}",
|
||||||
|
color::cyan("→"),
|
||||||
|
verb,
|
||||||
|
tid,
|
||||||
|
color::dim(url)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// `current`: the active tab's stable handle (#26).
|
||||||
|
if data
|
||||||
|
.get("current")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
let tid = data.get("tabId").and_then(|v| v.as_str()).unwrap_or("?");
|
||||||
|
let title = data.get("title").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let url = data.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let target = data.get("targetId").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
println!("{} [{}] {} - {}", color::cyan("→"), tid, title, url);
|
||||||
|
println!(" {}", color::dim(&format!("target: {}", target)));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Dialog status response
|
// Dialog status response
|
||||||
if action == Some("dialog") {
|
if action == Some("dialog") {
|
||||||
if let Some(has_dialog) = data.get("hasDialog").and_then(|v| v.as_bool()) {
|
if let Some(has_dialog) = data.get("hasDialog").and_then(|v| v.as_bool()) {
|
||||||
@@ -304,6 +342,34 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// Frame list (`chrome-use frames`)
|
||||||
|
if action == Some("frames") {
|
||||||
|
if let Some(list) = data.get("frames").and_then(|v| v.as_array()) {
|
||||||
|
let count = list.len();
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
color::bold(&format!("{} frame{}", count, if count == 1 { "" } else { "s" }))
|
||||||
|
);
|
||||||
|
for f in list {
|
||||||
|
let idx = f.get("index").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||||
|
let kind = f.get("kind").and_then(|v| v.as_str()).unwrap_or("?");
|
||||||
|
let url = f.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let len = f.get("textLen").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||||
|
println!(
|
||||||
|
" [{}] {:<6} {} chars {}",
|
||||||
|
idx,
|
||||||
|
kind,
|
||||||
|
len,
|
||||||
|
color::dim(if url.is_empty() { "(about:blank)" } else { url })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
color::dim("read everything with: chrome-use get text --all-frames")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
// Title
|
// Title
|
||||||
if let Some(title) = data.get("title").and_then(|v| v.as_str()) {
|
if let Some(title) = data.get("title").and_then(|v| v.as_str()) {
|
||||||
println!("{}", title);
|
println!("{}", title);
|
||||||
@@ -1892,6 +1958,8 @@ Retrieves various types of information from elements or the page.
|
|||||||
|
|
||||||
Subcommands:
|
Subcommands:
|
||||||
text <selector> Get text content of element
|
text <selector> Get text content of element
|
||||||
|
text --all-frames Aggregate text across ALL frames (incl. iframes)
|
||||||
|
text --main Main-content text only (skip nav/header/sidebar)
|
||||||
html <selector> Get inner HTML of element
|
html <selector> Get inner HTML of element
|
||||||
value <selector> Get value of input element
|
value <selector> Get value of input element
|
||||||
attr <selector> <name> Get attribute value
|
attr <selector> <name> Get attribute value
|
||||||
@@ -1908,6 +1976,9 @@ Global Options:
|
|||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
chrome-use get text @e1
|
chrome-use get text @e1
|
||||||
|
chrome-use get text --all-frames # read iframed content (listing pages)
|
||||||
|
chrome-use get text --main # main content, no nav/sidebar boilerplate
|
||||||
|
chrome-use frames # list frames + where the text lives
|
||||||
chrome-use get html "#content"
|
chrome-use get html "#content"
|
||||||
chrome-use get value "#email-input"
|
chrome-use get value "#email-input"
|
||||||
chrome-use get attr "#link" href
|
chrome-use get attr "#link" href
|
||||||
@@ -2778,6 +2849,20 @@ Notes:
|
|||||||
- Streaming is always enabled. Set AGENT_BROWSER_STREAM_PORT to bind to a
|
- Streaming is always enabled. Set AGENT_BROWSER_STREAM_PORT to bind to a
|
||||||
specific port instead of the default OS-assigned port.
|
specific port instead of the default OS-assigned port.
|
||||||
|
|
||||||
|
The WS is BIDIRECTIONAL — the high-throughput way to drive a live/real-time page
|
||||||
|
(games, canvas apps) instead of one screenshot + one CLI call per action:
|
||||||
|
- Server -> client (JSON text frames):
|
||||||
|
{"type":"frame","data":"<base64 jpeg>"} live screencast (~60fps)
|
||||||
|
plus status / tabs messages.
|
||||||
|
- Client -> server (send JSON text):
|
||||||
|
{"type":"input_keyboard","eventType":"keyDown|keyUp","key":" ","code":"Space",
|
||||||
|
"windowsVirtualKeyCode":32}
|
||||||
|
{"type":"input_mouse","eventType":"mousePressed|mouseReleased|mouseMoved",
|
||||||
|
"x":640,"y":360,"button":"left","clickCount":1}
|
||||||
|
{"type":"input_touch","eventType":"touchStart|touchEnd","touchPoints":[...]}
|
||||||
|
Connect once and run a tight local loop: read frames, send timed input — no
|
||||||
|
per-action process spawn, no round-trip. Works over the extension relay too.
|
||||||
|
|
||||||
Global Options:
|
Global Options:
|
||||||
--json Output as JSON
|
--json Output as JSON
|
||||||
--session <name> Use specific session
|
--session <name> Use specific session
|
||||||
@@ -3070,7 +3155,9 @@ Core Commands:
|
|||||||
dblclick <sel> Double-click element
|
dblclick <sel> Double-click element
|
||||||
type <sel> <text> Type into element
|
type <sel> <text> Type into element
|
||||||
fill <sel> <text> Clear and fill
|
fill <sel> <text> Clear and fill
|
||||||
press <key> Press key (Enter, Tab, Control+a)
|
press <key> [--hold <ms>] Press key (Enter, Tab, Control+a). --hold keeps it
|
||||||
|
down <ms> then releases — precise (in-daemon), for
|
||||||
|
games/charge: `press d --hold 800`
|
||||||
keydown <key> Hold a key down (no auto-release) — for games/shortcuts
|
keydown <key> Hold a key down (no auto-release) — for games/shortcuts
|
||||||
keyup <key> Release a held key. Pair with keydown to hold-to-move:
|
keyup <key> Release a held key. Pair with keydown to hold-to-move:
|
||||||
`keydown d` … `keyup d`
|
`keydown d` … `keyup d`
|
||||||
@@ -3101,6 +3188,7 @@ Navigation:
|
|||||||
|
|
||||||
Get Info: chrome-use get <what> [selector]
|
Get Info: chrome-use get <what> [selector]
|
||||||
text, html, value, attr <name>, title, url, count, box, styles, cdp-url
|
text, html, value, attr <name>, title, url, count, box, styles, cdp-url
|
||||||
|
text --all-frames (cross-frame), text --main (no boilerplate), frames (list)
|
||||||
|
|
||||||
Check State: chrome-use is <what> <selector>
|
Check State: chrome-use is <what> <selector>
|
||||||
visible, enabled, checked
|
visible, enabled, checked
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
@@ -30,6 +30,16 @@ const tabs = new Map()
|
|||||||
const sessionToTab = new Map()
|
const sessionToTab = new Map()
|
||||||
/** child (OOPIF/worker) sessionId -> tabId */
|
/** child (OOPIF/worker) sessionId -> tabId */
|
||||||
const childSessionToTab = new Map()
|
const childSessionToTab = new Map()
|
||||||
|
/** sessionId -> CDP targetId, kept ACROSS detach so a dead `cb-tab-<oldTabId>`
|
||||||
|
* session can be recovered by its stable targetId when the cross-process nav
|
||||||
|
* gave the tab a new Chrome tabId (issue #24). Capped to bound memory. */
|
||||||
|
const sessionTargets = new Map()
|
||||||
|
function rememberSessionTarget(sessionId, targetId) {
|
||||||
|
if (!sessionId || !targetId) return
|
||||||
|
sessionTargets.delete(sessionId)
|
||||||
|
sessionTargets.set(sessionId, targetId)
|
||||||
|
if (sessionTargets.size > 256) sessionTargets.delete(sessionTargets.keys().next().value)
|
||||||
|
}
|
||||||
/** tab-group name -> chrome tabGroups id (best-effort cache) */
|
/** tab-group name -> chrome tabGroups id (best-effort cache) */
|
||||||
const groupIdByName = new Map()
|
const groupIdByName = new Map()
|
||||||
|
|
||||||
@@ -172,17 +182,49 @@ function tabIdFromSession(sessionId) {
|
|||||||
// (closed / restricted). (issues #20.1, #23)
|
// (closed / restricted). (issues #20.1, #23)
|
||||||
async function recoverSessionTab(sessionId) {
|
async function recoverSessionTab(sessionId) {
|
||||||
const tabId = tabIdFromSession(sessionId)
|
const tabId = tabIdFromSession(sessionId)
|
||||||
if (tabId == null) return null
|
// 1) Fast path: the encoded Chrome tabId still exists — re-attach it (covers
|
||||||
for (let i = 0; i < 3; i++) {
|
// the common renderer-process swap where the tabId is preserved, #23).
|
||||||
const tab = await chrome.tabs.get(tabId).catch(() => null)
|
if (tabId != null) {
|
||||||
if (!eligible(tab)) return null
|
for (let i = 0; i < 3; i++) {
|
||||||
try {
|
const tab = await chrome.tabs.get(tabId).catch(() => null)
|
||||||
await attachTab(tabId)
|
if (!eligible(tab)) break // tabId is gone — fall through to targetId recovery
|
||||||
if (tabs.has(tabId)) return tabId
|
try {
|
||||||
} catch {
|
await attachTab(tabId)
|
||||||
// mid-swap: the tab exists but isn't attachable yet — back off and retry.
|
if (tabs.has(tabId)) return tabId
|
||||||
|
} catch {
|
||||||
|
// mid-swap: tab exists but isn't attachable yet — back off and retry.
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 120 + i * 150))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 2) The Chrome tabId is gone, but the CDP targetId is STABLE across the nav.
|
||||||
|
// Some cross-process hops (Mercari's signin token exchange) give the tab a
|
||||||
|
// NEW tabId while keeping the same target, so `cb-tab-<oldTabId>` can't be
|
||||||
|
// recovered by tabId. Find the tab now hosting our remembered targetId via
|
||||||
|
// chrome.debugger.getTargets(), attach it, and ALIAS the dead session to it
|
||||||
|
// so the daemon's session id keeps resolving. Longer window: this hop can
|
||||||
|
// take several seconds to settle (issue #24).
|
||||||
|
const targetId = sessionTargets.get(sessionId)
|
||||||
|
if (targetId) {
|
||||||
|
for (let i = 0; i < 6; i++) {
|
||||||
|
const targets = await chrome.debugger.getTargets().catch(() => null)
|
||||||
|
const t = targets && targets.find((x) => x.id === targetId && x.tabId != null)
|
||||||
|
if (t && t.tabId != null) {
|
||||||
|
const tab = await chrome.tabs.get(t.tabId).catch(() => null)
|
||||||
|
if (eligible(tab)) {
|
||||||
|
try {
|
||||||
|
await attachTab(t.tabId)
|
||||||
|
if (tabs.has(t.tabId)) {
|
||||||
|
sessionToTab.set(sessionId, t.tabId) // alias dead session -> live tab
|
||||||
|
return t.tabId
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// not attachable yet — keep waiting for the swap to settle.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 300 + i * 300))
|
||||||
}
|
}
|
||||||
await new Promise((r) => setTimeout(r, 120 + i * 150))
|
|
||||||
}
|
}
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
@@ -343,6 +385,7 @@ async function attachTab(tabId) {
|
|||||||
const entry = { sessionId, targetId }
|
const entry = { sessionId, targetId }
|
||||||
tabs.set(tabId, entry)
|
tabs.set(tabId, entry)
|
||||||
sessionToTab.set(sessionId, tabId)
|
sessionToTab.set(sessionId, tabId)
|
||||||
|
rememberSessionTarget(sessionId, targetId)
|
||||||
setBadge(tabId, port ? 'on' : 'connecting')
|
setBadge(tabId, port ? 'on' : 'connecting')
|
||||||
postToHost({
|
postToHost({
|
||||||
method: 'forwardCDPEvent',
|
method: 'forwardCDPEvent',
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest_version": 3,
|
"manifest_version": 3,
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "0.4.8",
|
"version": "0.4.9",
|
||||||
"description": "Let chrome-use drive your logged-in Chrome \u2014 install once, no token, no per-use confirmation.",
|
"description": "Let chrome-use drive your logged-in Chrome \u2014 install once, no token, no per-use confirmation.",
|
||||||
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6vQIyscGIPYPZdSpPwPL0+0gxUROyRgCpmvCSDoc8XUm4qm97VbKnD9Ijc1lV22lNWZtE78gaRjt6BeSfuMgnBymnhLKjN1gU6AI5QUU0mrJyeHdWKvrKQR5FmsM2A7Xr1ykE2SiiS8zNUS3Y/6O5l+Nva7wrVy6E4a2dkBVQkOsu+DV+nEZvhIyuDY5D5SPXqNwUTWTaglwj5mjvHz36xSwCWlPmrtJ+ED0AUyrb2z4GIOmvk4kqtBVrh/UD058klLo4CkYOnIybB5aV6WYuwarfPY4bF/dLggPem+ewLNTUNBuwrxj/A4nUv0LJTuRO8rR7f8WR9qnRCY0Ic5saQIDAQAB",
|
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6vQIyscGIPYPZdSpPwPL0+0gxUROyRgCpmvCSDoc8XUm4qm97VbKnD9Ijc1lV22lNWZtE78gaRjt6BeSfuMgnBymnhLKjN1gU6AI5QUU0mrJyeHdWKvrKQR5FmsM2A7Xr1ykE2SiiS8zNUS3Y/6O5l+Nva7wrVy6E4a2dkBVQkOsu+DV+nEZvhIyuDY5D5SPXqNwUTWTaglwj5mjvHz36xSwCWlPmrtJ+ED0AUyrb2z4GIOmvk4kqtBVrh/UD058klLo4CkYOnIybB5aV6WYuwarfPY4bF/dLggPem+ewLNTUNBuwrxj/A4nUv0LJTuRO8rR7f8WR9qnRCY0Ic5saQIDAQAB",
|
||||||
"icons": {
|
"icons": {
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "1.4.1",
|
"version": "1.5.1",
|
||||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"packageManager": "pnpm@11.1.3",
|
"packageManager": "pnpm@11.1.3",
|
||||||
|
|||||||
@@ -208,6 +208,9 @@ For unstructured reading (no refs needed):
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
chrome-use get text @e1 # visible text of an element
|
chrome-use get text @e1 # visible text of an element
|
||||||
|
chrome-use get text --all-frames # whole page, aggregated across ALL frames
|
||||||
|
chrome-use get text --main # main content only — skip nav/header/sidebar
|
||||||
|
chrome-use frames # list every frame + where the text lives
|
||||||
chrome-use get html @e1 # innerHTML
|
chrome-use get html @e1 # innerHTML
|
||||||
chrome-use get attr @e1 href # any attribute
|
chrome-use get attr @e1 href # any attribute
|
||||||
chrome-use get value @e1 # input value
|
chrome-use get value @e1 # input value
|
||||||
@@ -216,6 +219,14 @@ chrome-use get url # current URL
|
|||||||
chrome-use get count ".item" # count matching elements
|
chrome-use get count ".item" # count matching elements
|
||||||
```
|
```
|
||||||
|
|
||||||
|
On listing/marketplace pages (Yahoo Auctions, Rakuten, Mercari shops) the seller's
|
||||||
|
description often lives in a **child frame** or is buried under a "related items"
|
||||||
|
sidebar, so a plain `get text body` returns only header/nav boilerplate. When the
|
||||||
|
text you expect is missing: run `chrome-use frames` to see where it is, then
|
||||||
|
`get text --all-frames` (reads every reachable frame incl. cross-origin iframes)
|
||||||
|
or `get text --main` (drops the global chrome). If the content is lazy-loaded,
|
||||||
|
`scroll` it into view first.
|
||||||
|
|
||||||
## Interacting
|
## Interacting
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -309,12 +320,43 @@ detects this and prints a one-line hint. Drive them the screenshot way:
|
|||||||
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
||||||
# eval/get text return nothing useful)
|
# eval/get text return nothing useful)
|
||||||
chrome-use click 640 360 # interact by viewport coordinate
|
chrome-use click 640 360 # interact by viewport coordinate
|
||||||
chrome-use keydown d; sleep 0.6; chrome-use keyup d # hold-to-move
|
chrome-use press d --hold 800 # hold-to-move, precise (timed in-daemon —
|
||||||
|
# NOT keydown+shell-sleep+keyup, which
|
||||||
|
# adds ~250ms jitter per round-trip)
|
||||||
chrome-use press Space # discrete actions (jump/attack/confirm)
|
chrome-use press Space # discrete actions (jump/attack/confirm)
|
||||||
```
|
```
|
||||||
|
|
||||||
Each command is a ~250ms round-trip, so this is fine for turn-based / canvas
|
**Don't drive frame-by-frame with one CLI call per action** — that's the slowest,
|
||||||
*apps* but too slow to play a real-time 60fps action game frame-by-frame.
|
lowest-fidelity way (each call is a process spawn + round-trip). Script a *timed
|
||||||
|
sequence in a single round-trip* with `batch` (it sends each step to the running
|
||||||
|
daemon; `press --hold` and `wait` block in-daemon, so timing is precise):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chrome-use batch "press d --hold 900" "press j" "press j" "wait 200" "press d --hold 500"
|
||||||
|
```
|
||||||
|
|
||||||
|
Also try reading real state instead of pixels: `eval` runs in the page's main
|
||||||
|
world, so for a framework/engine game you can often reach its globals (e.g. a
|
||||||
|
Phaser/PIXI/Three instance, a store, `window.__GAME__`) and read positions/score
|
||||||
|
directly — far better than guessing from a screenshot.
|
||||||
|
|
||||||
|
**For genuinely real-time driving, drop the CLI entirely and use the WebSocket.**
|
||||||
|
`chrome-use stream enable` opens a bidirectional WS (`stream status` prints the
|
||||||
|
`ws://127.0.0.1:<port>`). Connect once and you get a live ~60fps screencast AND
|
||||||
|
can send input on the same socket — no per-action process spawn, no round-trip,
|
||||||
|
works over the extension relay:
|
||||||
|
|
||||||
|
```js
|
||||||
|
// node (global WebSocket): live frames + locally-timed input
|
||||||
|
const ws = new WebSocket("ws://127.0.0.1:PORT")
|
||||||
|
ws.onmessage = e => { const m = JSON.parse(e.data); if (m.type==="frame") {/* base64 jpeg */} }
|
||||||
|
const k = (eventType,key,code,vk) => ws.send(JSON.stringify({type:"input_keyboard",eventType,key,code,windowsVirtualKeyCode:vk}))
|
||||||
|
k("keyDown"," ","Space",32); setTimeout(()=>k("keyUp"," ","Space",32), 80) // a jump
|
||||||
|
// also: {type:"input_mouse",eventType:"mousePressed",x,y,button:"left",clickCount:1}
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the difference between watching a slideshow and playing the game. Reserve
|
||||||
|
screenshots for one-off checks; use the WS for any sustained real-time control.
|
||||||
|
|
||||||
## Waiting (read this)
|
## Waiting (read this)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user