oimwiodev/chrome-use
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(relay): reach cross-origin iframes; auto-reattach open (#35, #36)

Browse Source 
#35: `open` auto-reattaches when the bound relay tab is gone — drops the dead
page, opens a fresh tab in the session's group, and navigates it, instead of
only `tab new` recovering.

#36: scroll and click now reach content inside cross-origin OOPIFs:
- scroll dispatches a real wheel at a viewport point (default center, --at x,y,
  or --frame n) so it scrolls the iframe under the pointer, which
  window.scrollBy on the top document silently no-ops on.
- over the extension relay, clicks always use DOM-dispatch instead of
  coordinate Input events — a coordinate event isn't confined to the target tab
  on a busy real Chrome (it drifted onto the foreground tab) and an OOPIF
  element's box can't be mapped to a top-viewport point.
This commit is contained in:
leeguooooo
2026-06-16 18:05:06 +09:00
parent cd47ec43d0
commit 6830df50ea
8 changed files with 372 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/Cargo.lock
Generated
+1 -1
View File
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrome-use"
version = "1.5.10"
version = "1.5.11"
dependencies = [
"aes",
"aes-gcm",
cli/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "chrome-use"
version = "1.5.10"
version = "1.5.11"
edition = "2021"
description = "Fast browser automation CLI for AI agents"
license = "Apache-2.0"
cli/src/commands.rs
+76 -1
View File
@@ -729,10 +729,56 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
} else {
return Err(ParseError::MissingArguments {
context: "scroll --selector".to_string(),
usage: "scroll [direction] [amount] [--selector <sel>]",
usage: "scroll [direction] [amount] [--selector <sel>] [--at <x,y>] [--frame <n>]",
});
}
}
"--at" => {
// `--at x,y`: dispatch the wheel at this viewport pixel, so it
// scrolls whatever element/iframe is under the pointer — including
// cross-origin iframes that `window.scrollBy` can't reach (#36).
let val = rest.get(i + 1).ok_or(ParseError::MissingArguments {
context: "scroll --at".to_string(),
usage: "scroll [direction] [amount] --at <x,y>",
})?;
let mut parts = val.split(',');
match (
parts.next().and_then(|s| s.trim().parse::<f64>().ok()),
parts.next().and_then(|s| s.trim().parse::<f64>().ok()),
) {
(Some(x), Some(y)) => {
obj.insert("at".to_string(), json!([x, y]));
}
_ => {
return Err(ParseError::InvalidValue {
message: format!("scroll --at: invalid coordinate `{}`", val),
usage: "scroll [direction] [amount] --at <x,y> (e.g. --at 640,400)",
})
}
}
i += 1;
}
"--frame" => {
// `--frame n`: scroll the n-th frame from `chrome-use frames` by
// dispatching the wheel at that frame's center — reaches content in
// a cross-origin iframe without needing a selector into it (#36).
let val = rest.get(i + 1).ok_or(ParseError::MissingArguments {
context: "scroll --frame".to_string(),
usage: "scroll [direction] [amount] --frame <n>",
})?;
match val.trim().parse::<usize>() {
Ok(n) => {
obj.insert("frame".to_string(), json!(n));
}
Err(_) => {
return Err(ParseError::InvalidValue {
message: format!("scroll --frame: invalid index `{}`", val),
usage: "scroll [direction] [amount] --frame <n> (index from `chrome-use frames`)",
})
}
}
i += 1;
}
arg if arg.starts_with('-') => {}
_ => {
match positional_index {
@@ -5868,6 +5914,35 @@ mod tests {
assert_eq!(cmd["selector"], ".sidebar");
}
#[test]
fn test_scroll_at_coordinate() {
// `--at x,y` carries a [x, y] array for a wheel dispatched at that pixel
// (issue #36: cross-origin iframe scroll).
let cmd = parse_command(&args("scroll down 700 --at 640,400"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "scroll");
assert_eq!(cmd["direction"], "down");
assert_eq!(cmd["amount"], 700);
assert_eq!(cmd["at"], json!([640.0, 400.0]));
}
#[test]
fn test_scroll_at_rejects_garbage() {
assert!(parse_command(&args("scroll --at nope"), &default_flags()).is_err());
assert!(parse_command(&args("scroll --at 1"), &default_flags()).is_err());
}
#[test]
fn test_scroll_frame_index() {
let cmd = parse_command(&args("scroll down 700 --frame 2"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "scroll");
assert_eq!(cmd["frame"], 2);
}
#[test]
fn test_scroll_frame_rejects_non_integer() {
assert!(parse_command(&args("scroll --frame two"), &default_flags()).is_err());
}
#[test]
fn test_scroll_selector_before_positional() {
let cmd =
cli/src/native/actions.rs
+163 -11
View File
@@ -3467,17 +3467,169 @@ async fn handle_scroll(cmd: &Value, state: &mut DaemonState) -> Result<Value, St
}
}
interaction::scroll(
&mgr.client,
&session_id,
&state.ref_map,
selector,
dx,
dy,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "scrolled": true }))
// An explicit `--selector` keeps the precise element-scroll path (scrollBy on
// the resolved node, same-origin only).
if let Some(sel) = selector {
interaction::scroll(
&mgr.client,
&session_id,
&state.ref_map,
Some(sel),
dx,
dy,
&state.iframe_sessions,
)
.await?;
return Ok(json!({ "scrolled": true, "via": "selector" }));
}
// No selector: dispatch a real (isTrusted) wheel at a viewport coordinate.
// This hits the compositor and scrolls whatever scroll container is under the
// pointer — including cross-origin iframes that `window.scrollBy` on the top
// document silently no-ops on (issue #36). The coordinate is, in priority:
// --at x,y → that exact pixel
// --frame n → the center of frame n from `chrome-use frames`
// default → the viewport center
let (x, y, via) = if let Some(at) = cmd.get("at").and_then(|v| v.as_array()) {
let x = at.first().and_then(|v| v.as_f64()).unwrap_or(0.0);
let y = at.get(1).and_then(|v| v.as_f64()).unwrap_or(0.0);
(x, y, "at")
} else if let Some(n) = cmd.get("frame").and_then(|v| v.as_u64()) {
let (x, y) = frame_center(mgr, &session_id, &state.iframe_sessions, n as usize).await?;
(x, y, "frame")
} else {
let (x, y) = viewport_center(mgr, &session_id).await?;
(x, y, "center")
};
dispatch_wheel(&mgr.client, &session_id, x, y, dx, dy).await?;
Ok(json!({ "scrolled": true, "via": via, "at": [x, y] }))
}
/// Viewport center in CSS pixels, used as the default wheel landing point for
/// `scroll` (issue #36). Falls back to a sane 640×400 center if the page can't
/// be evaluated (e.g. a restricted document).
async fn viewport_center(mgr: &BrowserManager, session_id: &str) -> Result<(f64, f64), String> {
let dims = mgr
.client
.send_command_typed::<_, Value>(
"Runtime.evaluate",
&super::cdp::types::EvaluateParams {
expression: "[window.innerWidth, window.innerHeight]".to_string(),
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
)
.await
.ok();
let arr = dims
.as_ref()
.and_then(|v| v.get("result"))
.and_then(|v| v.get("value"))
.and_then(|v| v.as_array());
let w = arr
.and_then(|a| a.first())
.and_then(|v| v.as_f64())
.filter(|w| *w > 0.0)
.unwrap_or(1280.0);
let h = arr
.and_then(|a| a.get(1))
.and_then(|v| v.as_f64())
.filter(|h| *h > 0.0)
.unwrap_or(800.0);
Ok((w / 2.0, h / 2.0))
}
/// Center of the `n`-th frame (as listed by `chrome-use frames`) in top-viewport
/// CSS pixels, so `scroll --frame n` lands its wheel inside a cross-origin iframe
/// without needing a selector into it (issue #36). Resolves the frame's owning
/// `<iframe>` element box via `DOM.getFrameOwner` + `DOM.getBoxModel` — exact for
/// a frame nested directly under the top document; for a deeper nesting the box is
/// relative to the intermediate frame, so prefer `--at x,y` from a screenshot.
async fn frame_center(
mgr: &BrowserManager,
session_id: &str,
iframe_sessions: &HashMap<String, String>,
n: usize,
) -> Result<(f64, f64), String> {
let frames =
super::element::collect_all_frames_text(&mgr.client, session_id, iframe_sessions).await?;
let frame = frames.get(n).ok_or_else(|| {
format!(
"frame index {} out of range (run `chrome-use frames`: {} frame(s))",
n,
frames.len()
)
})?;
if n == 0 {
// Frame 0 is the top document — there's no owner element; scroll its center.
return viewport_center(mgr, session_id).await;
}
let owner = mgr
.client
.send_command_typed::<_, Value>(
"DOM.getFrameOwner",
&json!({ "frameId": frame.frame_id }),
Some(session_id),
)
.await
.map_err(|e| format!("can't locate frame {}'s owner element: {}", n, e))?;
let backend_node_id = owner
.get("backendNodeId")
.and_then(|v| v.as_i64())
.ok_or_else(|| format!("frame {} has no owner <iframe> element", n))?;
let box_model = mgr
.client
.send_command_typed::<_, Value>(
"DOM.getBoxModel",
&json!({ "backendNodeId": backend_node_id }),
Some(session_id),
)
.await
.map_err(|e| format!("can't measure frame {}'s box: {}", n, e))?;
let content = box_model
.get("model")
.and_then(|m| m.get("content"))
.and_then(|c| c.as_array())
.ok_or_else(|| format!("frame {} box model has no content quad", n))?;
let coord = |i: usize| content.get(i).and_then(|v| v.as_f64()).unwrap_or(0.0);
// content quad is [x1,y1, x2,y2, x3,y3, x4,y4]; opposite corners are 0 and 2.
let cx = (coord(0) + coord(4)) / 2.0;
let cy = (coord(1) + coord(5)) / 2.0;
Ok((cx, cy))
}
/// Dispatch a trusted mouse wheel at `(x, y)`, humanized like `handle_wheel`.
async fn dispatch_wheel(
client: &super::cdp::client::CdpClient,
session_id: &str,
x: f64,
y: f64,
delta_x: f64,
delta_y: f64,
) -> Result<(), String> {
let level = humanize::active_level();
let seed = humanize::next_seed();
for (dx, dy, delay) in humanize::scroll_segments(delta_x, delta_y, level, seed) {
client
.send_command(
"Input.dispatchMouseEvent",
Some(json!({
"type": "mouseWheel",
"x": x,
"y": y,
"deltaX": dx,
"deltaY": dy,
})),
Some(session_id),
)
.await?;
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
}
Ok(())
}
async fn handle_select(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
cli/src/native/browser.rs
+88 -3
View File
@@ -254,6 +254,21 @@ fn active_index_is_owned(
.unwrap_or(false)
}
/// Whether a CDP error means the bound relay target is gone — the tab was
/// closed, navigated across processes (renderer swap), or lost after an
/// extension/service-worker restart, and the relay could not re-attach. The
/// ab-connect relay surfaces these as `stale sessionId … its tab is gone`,
/// `unknown sessionId …`, or `no attached tab …`. `navigate` keys its
/// auto-reattach recovery off this (issue #35) so a dead session rebinds to a
/// fresh tab instead of erroring on every command until the user runs `tab new`.
fn is_stale_target_error(error: &str) -> bool {
let lower = error.to_lowercase();
lower.contains("its tab is gone")
|| lower.contains("stale sessionid")
|| lower.contains("unknown sessionid")
|| lower.contains("no attached tab")
}
/// Converts common error messages into AI-friendly, actionable descriptions.
pub fn to_ai_friendly_error(error: &str) -> String {
let lower = error.to_lowercase();
@@ -915,6 +930,24 @@ impl BrowserManager {
)
}
/// Drop the page bound to `session_id` from the tracked list — used when the
/// relay reports its tab is gone (issue #35) so the stale entry can't keep
/// resolving as active. Forgets ownership, unpins it if it was pinned, and
/// keeps `active_page_index` in range.
fn drop_page_by_session(&mut self, session_id: &str) {
let Some(pos) = self.pages.iter().position(|p| p.session_id == session_id) else {
return;
};
let target_id = self.pages[pos].target_id.clone();
self.pages.remove(pos);
self.created_targets.remove(&target_id);
if self.active_target_id.as_deref() == Some(target_id.as_str()) {
self.active_target_id = None;
}
self.active_page_index =
active_page_index_after_removal(self.active_page_index, pos, self.pages.len());
}
/// Pin the current active page by target_id so later commands stick to it.
/// Call after any explicit open / tab new / tab switch.
fn pin_active_target(&mut self) {
@@ -944,10 +977,10 @@ impl BrowserManager {
if self.agent_group().is_some() && !self.active_is_session_owned() {
self.tab_new(None, None).await?;
}
let session_id = self.active_session_id()?.to_string();
let mut session_id = self.active_session_id()?.to_string();
let mut lifecycle_rx = self.client.subscribe();
let nav_result: PageNavigateResult = self
let nav_result: PageNavigateResult = match self
.client
.send_command_typed(
"Page.navigate",
@@ -957,7 +990,38 @@ impl BrowserManager {
},
Some(&session_id),
)
.await?;
.await
{
Ok(r) => r,
// Auto-reattach when the bound tab is gone (issue #35). On the shared
// real browser the human can close/swap the agent's tab, and a
// cross-process nav can destroy the target without a re-attachable
// tabId — both leave the cached `cb-tab-<id>` session stale, so every
// command (including `open`) failed on it and only `tab new`
// recovered. The relay error literally says "re-open your target URL
// to re-attach"; fulfil that here: drop the dead page, open a fresh
// owned tab in this session's group, and navigate THAT. Gated on the
// relay (`agent_group`) and on the explicit navigation intent — read
// commands deliberately still fail loudly rather than silently
// recover onto a blank tab and return wrong data (issue #8.1).
Err(e) if self.agent_group().is_some() && is_stale_target_error(&e) => {
self.drop_page_by_session(&session_id);
self.tab_new(None, None).await?;
session_id = self.active_session_id()?.to_string();
lifecycle_rx = self.client.subscribe();
self.client
.send_command_typed(
"Page.navigate",
&PageNavigateParams {
url: url.to_string(),
referrer: None,
},
Some(&session_id),
)
.await?
}
Err(e) => return Err(e),
};
if let Some(ref error_text) = nav_result.error_text {
return Err(format!("Navigation failed: {}", error_text));
@@ -2693,6 +2757,27 @@ mod tests {
assert_eq!(active_page_index_after_removal(0, 0, 0), 0);
}
#[test]
fn stale_target_error_matches_relay_signatures() {
// The exact relay error `open` must recover from (issue #35), as wrapped
// by send_command's `CDP error (Page.navigate): …` prefix.
assert!(is_stale_target_error(
"CDP error (Page.navigate): stale sessionId cb-tab-1655244623 for Page.navigate: \
its tab is gone (closed, navigated across processes, or lost after an extension \
restart). Re-attach by re-opening your target URL before retrying."
));
assert!(is_stale_target_error("unknown sessionId cb-tab-7 for Page.navigate"));
assert!(is_stale_target_error("no attached tab for Page.navigate"));
}
#[test]
fn stale_target_error_ignores_unrelated_failures() {
// A genuine navigation failure (bad URL, DNS, blocked) must NOT trigger
// the open-a-fresh-tab recovery — that would mask the real error.
assert!(!is_stale_target_error("Navigation failed: net::ERR_NAME_NOT_RESOLVED"));
assert!(!is_stale_target_error("CDP command timed out: Page.navigate"));
}
fn page(target_id: &str) -> PageInfo {
PageInfo {
tab_id: 1,
cli/src/native/interaction.rs
+25
View File
@@ -45,6 +45,31 @@ pub async fn click(
.await;
}
// Over the extension relay we drive the user's real, in-use Chrome, where a
// coordinate `Input.dispatchMouseEvent` is NOT reliably confined to our target
// tab — it can be delivered to whatever tab is in the foreground, and an OOPIF
// element's box can't be mapped to a top-viewport point at all. This twice
// opened an unrelated tab on the user's busy Chrome (issues #31/#36). So on the
// relay, never use coordinates for a normal left click: DOM-dispatch invokes
// the element's click in its own (frame) session, always hitting the right
// element in the right tab. Double/right clicks still need true pointer
// semantics, and `coord` mode is an explicit opt-out.
if mode != "coord" && button == "left" && click_count == 1 {
let in_iframe = parse_ref(selector_or_ref)
.and_then(|r| ref_map.get(&r).map(|e| e.frame_id.is_some()))
.unwrap_or(false);
if in_iframe || crate::connect::relay_url().is_some() {
return dom_click(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await;
}
}
let resolved = resolve_element_center(
client,
session_id,
cli/src/output.rs
+14 -1
View File
@@ -1735,12 +1735,23 @@ Usage: chrome-use scroll [direction] [amount] [options]
Scrolls the page or a specific element in the specified direction.
Without --selector, scroll dispatches a real (isTrusted) mouse wheel at a
viewport coordinate, so it scrolls whatever container is under the pointer —
including cross-origin iframes (Google Payments, Stripe, embedded checkout/KYC)
that plain page scroll can't reach.
Arguments:
direction up, down, left, right (default: down)
amount Pixels to scroll (default: 300)
Options:
-s, --selector <sel> CSS selector for a scrollable container
-s, --selector <sel> CSS selector for a scrollable container (same-origin)
--at <x,y> Dispatch the wheel at this viewport pixel (read it from a
screenshot) — precise way into a cross-origin iframe
--frame <n> Scroll the n-th frame from `chrome-use frames` (wheel at
that frame's center)
Without --selector/--at/--frame the wheel lands at the viewport center.
Global Options:
--json Output as JSON
@@ -1752,6 +1763,8 @@ Examples:
chrome-use scroll up 200
chrome-use scroll left 100
chrome-use scroll down 500 --selector "div.scroll-container"
chrome-use scroll down 700 --at 640,400 # wheel at a pixel over an iframe
chrome-use scroll down 700 --frame 2 # scroll frame 2 from `frames`
"##
}
"scrollintoview" | "scrollinto" => {