Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
997373fd57 | ||
|
|
5a858af93f |
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chrome-use"
|
||||
version = "1.5.14"
|
||||
version = "1.5.15"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "chrome-use"
|
||||
version = "1.5.14"
|
||||
version = "1.5.15"
|
||||
edition = "2021"
|
||||
description = "Fast browser automation CLI for AI agents"
|
||||
license = "Apache-2.0"
|
||||
|
||||
@@ -35,6 +35,7 @@ fn native_test_fixture_html(name: &str) -> &'static str {
|
||||
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
|
||||
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
|
||||
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
|
||||
"iframe_button_probe" => include_str!("test_fixtures/iframe_button_probe.html"),
|
||||
_ => panic!("Unknown native test fixture: {}", name),
|
||||
}
|
||||
}
|
||||
@@ -573,6 +574,76 @@ async fn e2e_snapshot_and_click_ref() {
|
||||
assert_success(&resp);
|
||||
}
|
||||
|
||||
/// Clicking a button INSIDE an iframe by `@ref` must deliver a TRUSTED activation
|
||||
/// (`event.isTrusted === true`), not a synthetic DOM `.click()`. Security-sensitive
|
||||
/// embedded forms (Google Payments' `保存`) reject `isTrusted:false` clicks, so an
|
||||
/// enabled submit button silently no-op'd (issue #39). The fix routes iframe-ref
|
||||
/// clicks to a real `Input.dispatchMouseEvent` on the element's own frame session.
|
||||
/// The fixture's iframe button writes `clicked:<isTrusted>` into its own text on
|
||||
/// click, which the cross-frame snapshot reads back.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn e2e_iframe_button_click_is_trusted() {
|
||||
let mut state = DaemonState::new();
|
||||
|
||||
let resp = execute_command(
|
||||
&json!({ "id": "1", "action": "launch", "headless": true }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&resp);
|
||||
|
||||
let resp = execute_command(
|
||||
&json!({ "id": "2", "action": "navigate", "url": native_test_fixture_url("iframe_button_probe") }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&resp);
|
||||
|
||||
// Snapshot (interactive) — the button lives in the iframe and must appear with
|
||||
// a ref; that ref carries the frame_id so the click resolves into the frame.
|
||||
let resp = execute_command(
|
||||
&json!({ "id": "3", "action": "snapshot", "interactive": true }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&resp);
|
||||
let snapshot = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||
let ref_id = snapshot
|
||||
.lines()
|
||||
.find(|l| l.contains("button \"save\""))
|
||||
.and_then(|l| l.split("ref=").nth(1))
|
||||
.map(|r| r.trim_end_matches(']').trim())
|
||||
.unwrap_or_else(|| panic!("iframe button not found in snapshot:\n{snapshot}"));
|
||||
|
||||
// Click it by ref.
|
||||
let resp = execute_command(
|
||||
&json!({ "id": "4", "action": "click", "selector": ref_id }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&resp);
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(300)).await;
|
||||
|
||||
// The button rewrote its own text with the click's isTrusted flag; read it
|
||||
// back across frames.
|
||||
let resp = execute_command(
|
||||
&json!({ "id": "5", "action": "snapshot", "interactive": true }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&resp);
|
||||
let after = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||
assert!(
|
||||
after.contains("clicked:true"),
|
||||
"iframe button click must be trusted (isTrusted:true); snapshot:\n{after}"
|
||||
);
|
||||
|
||||
let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
|
||||
assert_success(&resp);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Screenshot
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -56,19 +56,34 @@ pub async fn click(
|
||||
.await;
|
||||
}
|
||||
|
||||
// Over the extension relay we drive the user's real, in-use Chrome, where a
|
||||
// coordinate `Input.dispatchMouseEvent` is NOT reliably confined to our target
|
||||
// tab — it can be delivered to whatever tab is in the foreground, and an OOPIF
|
||||
// element's box can't be mapped to a top-viewport point at all. This twice
|
||||
// opened an unrelated tab on the user's busy Chrome (issues #31/#36). So on the
|
||||
// relay, never use coordinates for a normal left click: DOM-dispatch invokes
|
||||
// the element's click in its own (frame) session, always hitting the right
|
||||
// element in the right tab. Double/right clicks still need true pointer
|
||||
// semantics, and `coord` mode is an explicit opt-out.
|
||||
// An element INSIDE an iframe needs a TRUSTED activation: a DOM `.click()` is
|
||||
// `isTrusted:false`, which security-sensitive embedded forms reject — Google
|
||||
// Payments' enabled `保存` button silently no-ops on a synthetic click (issue
|
||||
// #39). A coordinate `Input.dispatchMouseEvent` can't help either: `getBoxModel`
|
||||
// for a sub-frame node returns frame-local coordinates that don't compose the
|
||||
// iframe's offset, so the click lands in the wrong place. The frame-agnostic
|
||||
// trusted path is keyboard activation — focus the element in its own frame, then
|
||||
// dispatch a real Enter on the page session; Chrome routes the key to the
|
||||
// focused element regardless of frame (same as `type --focused`), and Enter on a
|
||||
// focused button/link fires a trusted `click`. `coord` mode opts out.
|
||||
let in_iframe = ref_map.ref_is_in_iframe(selector_or_ref);
|
||||
if mode != "coord" && button == "left" && click_count == 1 && in_iframe {
|
||||
return dom_activate(
|
||||
client,
|
||||
session_id,
|
||||
ref_map,
|
||||
selector_or_ref,
|
||||
iframe_sessions,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// On the relay (the user's real Chrome) a TOP-document coordinate click used to
|
||||
// drift onto the foreground tab; that root cause is fixed (#5: the agent drives
|
||||
// its own pinned tab), but DOM-dispatch stays the conservative default here.
|
||||
if mode != "coord"
|
||||
&& button == "left"
|
||||
&& click_count == 1
|
||||
&& prefer_dom_dispatch(ref_map, selector_or_ref)
|
||||
&& crate::connect::relay_url().is_some()
|
||||
{
|
||||
return dom_click(
|
||||
client,
|
||||
@@ -270,6 +285,71 @@ async fn dom_click(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Trusted activation of an element inside an iframe (issue #39). Focuses the
|
||||
/// element in its own frame session, then dispatches a real Enter/Space on the
|
||||
/// page session — Chrome routes the key to the focused element across frames, and
|
||||
/// Enter/Space on a focused button/link/checkbox fires a `click` with
|
||||
/// `isTrusted: true`, which security-sensitive embedded forms (Google Payments
|
||||
/// `保存`) require. Non-activatable roles (a `div[onclick]`) can't be keyboard-
|
||||
/// activated, so they fall back to a DOM `.click()`.
|
||||
async fn dom_activate(
|
||||
client: &CdpClient,
|
||||
session_id: &str,
|
||||
ref_map: &RefMap,
|
||||
selector_or_ref: &str,
|
||||
iframe_sessions: &HashMap<String, String>,
|
||||
) -> Result<(), String> {
|
||||
let role = parse_ref(selector_or_ref)
|
||||
.and_then(|r| ref_map.get(&r).map(|e| e.role.clone()))
|
||||
.unwrap_or_default();
|
||||
// Space toggles checkbox-like controls; Enter activates buttons/links/menus.
|
||||
let key = match role.as_str() {
|
||||
"checkbox" | "radio" | "switch" | "option" | "menuitemcheckbox" | "menuitemradio" => {
|
||||
Some("space")
|
||||
}
|
||||
"button" | "link" | "menuitem" | "tab" | "treeitem" => Some("enter"),
|
||||
_ => None,
|
||||
};
|
||||
let Some(key) = key else {
|
||||
// Not keyboard-activatable — best effort via DOM .click() (untrusted).
|
||||
return dom_click(
|
||||
client,
|
||||
session_id,
|
||||
ref_map,
|
||||
selector_or_ref,
|
||||
iframe_sessions,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
|
||||
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||
client,
|
||||
session_id,
|
||||
ref_map,
|
||||
selector_or_ref,
|
||||
iframe_sessions,
|
||||
)
|
||||
.await?;
|
||||
// Focus the element in its OWN frame session so the keystroke lands on it.
|
||||
client
|
||||
.send_command_typed::<_, Value>(
|
||||
"Runtime.callFunctionOn",
|
||||
&CallFunctionOnParams {
|
||||
function_declaration: "function() { this.focus(); }".to_string(),
|
||||
object_id: Some(object_id),
|
||||
arguments: None,
|
||||
return_by_value: Some(true),
|
||||
await_promise: Some(false),
|
||||
},
|
||||
Some(&effective_session_id),
|
||||
)
|
||||
.await?;
|
||||
// Trusted key on the page session — routed to the focused (in-frame) element.
|
||||
press_key(client, session_id, key).await?;
|
||||
wait_for_paint_settled(client, &effective_session_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// DOM-dispatch a double-click on the element in its own session (no coordinates)
|
||||
/// — the relay/iframe-safe counterpart to a coordinate dblclick. Fires the full
|
||||
/// click,click,dblclick sequence so handlers bound to any of them respond.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>iframe button probe</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>iframe button probe</h1>
|
||||
<iframe
|
||||
id="frame"
|
||||
width="320"
|
||||
height="140"
|
||||
srcdoc="
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body style='margin:24px'>
|
||||
<button id='b' style='padding:24px;font-size:22px'>save</button>
|
||||
<script>
|
||||
document.getElementById('b').addEventListener('click', function (e) {
|
||||
this.textContent = 'clicked:' + e.isTrusted;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"
|
||||
></iframe>
|
||||
</body>
|
||||
</html>
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "chrome-use",
|
||||
"version": "1.5.14",
|
||||
"version": "1.5.15",
|
||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||
"type": "module",
|
||||
"packageManager": "pnpm@11.1.3",
|
||||
|
||||
Reference in New Issue
Block a user