Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eb2bc343a0 | ||
|
|
d86c9c4be2 | ||
|
|
32c25a6627 | ||
|
|
a8ce3dd3f8 | ||
|
|
997373fd57 | ||
|
|
5a858af93f |
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.5.14"
|
version = "1.5.17"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.5.14"
|
version = "1.5.17"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "Fast browser automation CLI for AI agents"
|
description = "Fast browser automation CLI for AI agents"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
+146
-26
@@ -235,6 +235,43 @@ fn prunable_target_ids(
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Consecutive missing `getTargets` snapshots before an owned relay tab is
|
||||||
|
/// pruned. >1 so a single churning/partial snapshot (other agents opening/closing
|
||||||
|
/// tabs) or a brief cross-process-nav gap can't drop the tab the agent is driving.
|
||||||
|
const RELAY_PRUNE_MISSES: u32 = 3;
|
||||||
|
|
||||||
|
/// Debounced prune for the relay: target ids to drop, mutating per-target miss
|
||||||
|
/// counters. A tab in `live_ids` resets to 0; an absent (non-pinned) tab
|
||||||
|
/// increments and is pruned only at `RELAY_PRUNE_MISSES`. Counters for
|
||||||
|
/// no-longer-tracked targets are forgotten. Pure, so the multi-agent churn
|
||||||
|
/// tolerance is unit-testable without a live browser.
|
||||||
|
fn debounced_prune_ids(
|
||||||
|
pages: &[PageInfo],
|
||||||
|
live_ids: &HashSet<String>,
|
||||||
|
pinned: Option<&str>,
|
||||||
|
misses: &mut HashMap<String, u32>,
|
||||||
|
) -> Vec<String> {
|
||||||
|
let tracked: HashSet<&str> = pages.iter().map(|p| p.target_id.as_str()).collect();
|
||||||
|
misses.retain(|tid, _| tracked.contains(tid.as_str()));
|
||||||
|
let mut prune = Vec::new();
|
||||||
|
for p in pages {
|
||||||
|
let tid = p.target_id.as_str();
|
||||||
|
if live_ids.contains(tid) {
|
||||||
|
misses.remove(tid);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if pinned == Some(tid) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let c = misses.entry(p.target_id.clone()).or_insert(0);
|
||||||
|
*c += 1;
|
||||||
|
if *c >= RELAY_PRUNE_MISSES {
|
||||||
|
prune.push(p.target_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prune
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether the resolved active page is a tab the session created (its target_id
|
/// Whether the resolved active page is a tab the session created (its target_id
|
||||||
/// is in `created_targets`). Pure core of [`BrowserManager::active_is_session_owned`]
|
/// is in `created_targets`). Pure core of [`BrowserManager::active_is_session_owned`]
|
||||||
/// so the relay no-hijack rule is unit-testable without a live browser.
|
/// so the relay no-hijack rule is unit-testable without a live browser.
|
||||||
@@ -464,6 +501,14 @@ pub struct BrowserManager {
|
|||||||
/// the session's commands onto the wrong page — the wrong-origin-fetch hazard
|
/// the session's commands onto the wrong page — the wrong-origin-fetch hazard
|
||||||
/// in the dogfood reports. Falls back to the index if the pinned tab is gone.
|
/// in the dogfood reports. Falls back to the index if the pinned tab is gone.
|
||||||
active_target_id: Option<String>,
|
active_target_id: Option<String>,
|
||||||
|
/// Per-target count of CONSECUTIVE `resync_targets` snapshots in which an
|
||||||
|
/// owned tab was missing from `Target.getTargets`. Over the relay a single
|
||||||
|
/// snapshot routinely omits live tabs (multi-agent churn, a cross-process nav
|
||||||
|
/// briefly dropping the target), so we must not prune on one miss — that lost
|
||||||
|
/// the tab the agent was driving. A tab is removed only after it's been absent
|
||||||
|
/// for `RELAY_PRUNE_MISSES` consecutive snapshots; any snapshot that includes
|
||||||
|
/// it resets the counter. Keyed by stable target_id.
|
||||||
|
relay_target_misses: HashMap<String, u32>,
|
||||||
next_tab_id: u32,
|
next_tab_id: u32,
|
||||||
/// Whether to enable the CDP `Runtime` domain (console / error / exception capture).
|
/// Whether to enable the CDP `Runtime` domain (console / error / exception capture).
|
||||||
/// OFF by default for stealth: a live `Runtime.enable` is a detectable CDP signal
|
/// OFF by default for stealth: a live `Runtime.enable` is a detectable CDP signal
|
||||||
@@ -600,6 +645,7 @@ impl BrowserManager {
|
|||||||
visited_origins: HashSet::new(),
|
visited_origins: HashSet::new(),
|
||||||
created_targets: HashSet::new(),
|
created_targets: HashSet::new(),
|
||||||
active_target_id: None,
|
active_target_id: None,
|
||||||
|
relay_target_misses: HashMap::new(),
|
||||||
next_tab_id: 1,
|
next_tab_id: 1,
|
||||||
capture_console: console_capture_enabled(),
|
capture_console: console_capture_enabled(),
|
||||||
};
|
};
|
||||||
@@ -702,6 +748,7 @@ impl BrowserManager {
|
|||||||
visited_origins: HashSet::new(),
|
visited_origins: HashSet::new(),
|
||||||
created_targets: HashSet::new(),
|
created_targets: HashSet::new(),
|
||||||
active_target_id: None,
|
active_target_id: None,
|
||||||
|
relay_target_misses: HashMap::new(),
|
||||||
next_tab_id: 1,
|
next_tab_id: 1,
|
||||||
capture_console: console_capture_enabled(),
|
capture_console: console_capture_enabled(),
|
||||||
};
|
};
|
||||||
@@ -823,7 +870,19 @@ impl BrowserManager {
|
|||||||
self.active_page_index = 0;
|
self.active_page_index = 0;
|
||||||
self.pin_active_target();
|
self.pin_active_target();
|
||||||
self.enable_domains(&attach_result.session_id).await?;
|
self.enable_domains(&attach_result.session_id).await?;
|
||||||
|
} else if self.agent_group().is_some() {
|
||||||
|
// STRICT MULTI-AGENT ISOLATION (relay / the user's real Chrome).
|
||||||
|
// `page_targets` here are the USER's and OTHER agents' tabs. A tab
|
||||||
|
// group belongs to exactly ONE agent, so this session must NOT adopt
|
||||||
|
// any of them — it tracks ONLY tabs it creates (its own colored group)
|
||||||
|
// plus popups it opens. Adopting foreign tabs is precisely what let
|
||||||
|
// another concurrent agent's tab churn drop the tab we were driving and
|
||||||
|
// drift eval/click onto the wrong page (multi-agent failure). Open our
|
||||||
|
// own dedicated background tab in the session's group and pin it; the
|
||||||
|
// user's / other agents' tabs stay invisible to us.
|
||||||
|
self.tab_new(None, None).await?;
|
||||||
} else {
|
} else {
|
||||||
|
// A browser WE launched: every tab is ours, so adopt them all.
|
||||||
for target in &page_targets {
|
for target in &page_targets {
|
||||||
let attach_result: AttachToTargetResult = self
|
let attach_result: AttachToTargetResult = self
|
||||||
.client
|
.client
|
||||||
@@ -849,24 +908,10 @@ impl BrowserManager {
|
|||||||
target_type: target.target_type.clone(),
|
target_type: target.target_type.clone(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
self.active_page_index = 0;
|
||||||
if self.agent_group().is_some() {
|
self.pin_active_target();
|
||||||
// Relay: the adopted tabs above are the USER's, in their real
|
let session_id = self.pages[0].session_id.clone();
|
||||||
// Chrome. NEVER make one of them the agent's working tab — that is
|
self.enable_domains(&session_id).await?;
|
||||||
// how commands drifted onto whatever page the user was viewing
|
|
||||||
// between steps (eval/click/get landed on the user's foreground
|
|
||||||
// tab; #35). Open our own dedicated background tab in the session's
|
|
||||||
// group and pin THAT as active. The user's tabs stay adopted (so
|
|
||||||
// `tab list` / explicit `tab switch` can reach them) but are never
|
|
||||||
// auto-selected — the agent only ever drives a tab it owns.
|
|
||||||
self.tab_new(None, None).await?;
|
|
||||||
} else {
|
|
||||||
// A browser we launched: every tab is ours, so the first is fine.
|
|
||||||
self.active_page_index = 0;
|
|
||||||
self.pin_active_target();
|
|
||||||
let session_id = self.pages[0].session_id.clone();
|
|
||||||
self.enable_domains(&session_id).await?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1515,6 +1560,18 @@ impl BrowserManager {
|
|||||||
/// the active tab, per #7/#8.1); the caller surfaces it so the agent knows a
|
/// the active tab, per #7/#8.1); the caller surfaces it so the agent knows a
|
||||||
/// tab opened instead of seeing the old page (issue #24-A).
|
/// tab opened instead of seeing the old page (issue #24-A).
|
||||||
pub async fn adopt_newly_opened(&mut self, before: &HashSet<String>) -> Option<PageInfo> {
|
pub async fn adopt_newly_opened(&mut self, before: &HashSet<String>) -> Option<PageInfo> {
|
||||||
|
// STRICT MULTI-AGENT ISOLATION: on the relay this session's `before` set is
|
||||||
|
// only its OWN tabs, so EVERY foreign tab (the user's, other agents') looks
|
||||||
|
// "new" relative to it and would be adopted here — exactly the leak where a
|
||||||
|
// concurrent agent's tabs (github/Lark/iphone-use) showed up in this
|
||||||
|
// session mid-flow. A tab the agent itself opened (a pop-up) can't be
|
||||||
|
// distinguished from a foreign tab over the relay (no opener/window/group
|
||||||
|
// in the synthesized targetInfo), so don't adopt anything: the agent drives
|
||||||
|
// only tabs it explicitly created, and pop-ups (e.g. an OAuth/login window)
|
||||||
|
// are the user's. A launched browser (every tab ours) still follows pop-ups.
|
||||||
|
if self.agent_group().is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
let result: GetTargetsResult = self
|
let result: GetTargetsResult = self
|
||||||
.client
|
.client
|
||||||
.send_command_typed("Target.getTargets", &json!({}), None)
|
.send_command_typed("Target.getTargets", &json!({}), None)
|
||||||
@@ -1557,6 +1614,11 @@ impl BrowserManager {
|
|||||||
title: sanitize_title(&target.title),
|
title: sanitize_title(&target.title),
|
||||||
target_type: target.target_type.clone(),
|
target_type: target.target_type.clone(),
|
||||||
};
|
};
|
||||||
|
// A tab that appeared right after THIS session's action (a click that
|
||||||
|
// opened a popup/new tab) is ours — record it as owned so it's tracked,
|
||||||
|
// protected from churn-pruning, and cleaned up on close, consistent with
|
||||||
|
// strict multi-agent isolation (we only ever own tabs we created/opened).
|
||||||
|
self.created_targets.insert(target.target_id.clone());
|
||||||
self.add_background_page(page.clone());
|
self.add_background_page(page.clone());
|
||||||
let _ = self.enable_domains(&attach.session_id).await;
|
let _ = self.enable_domains(&attach.session_id).await;
|
||||||
if opened.is_none() {
|
if opened.is_none() {
|
||||||
@@ -1584,13 +1646,19 @@ impl BrowserManager {
|
|||||||
.filter(should_track_target)
|
.filter(should_track_target)
|
||||||
.collect();
|
.collect();
|
||||||
let live_ids: HashSet<String> = live.iter().map(|t| t.target_id.clone()).collect();
|
let live_ids: HashSet<String> = live.iter().map(|t| t.target_id.clone()).collect();
|
||||||
|
let on_relay = self.agent_group().is_some();
|
||||||
|
|
||||||
for target in &live {
|
for target in &live {
|
||||||
if self.update_page_target_info(target) {
|
if self.update_page_target_info(target) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// A target this session hasn't tracked yet — attach and add it in the
|
// STRICT MULTI-AGENT ISOLATION: on the relay (the user's real Chrome,
|
||||||
// background so it's listable/adoptable without stealing the active tab.
|
// shared with other agents), NEVER adopt a tab this session didn't
|
||||||
|
// create — it belongs to the user or another agent's group. Only a
|
||||||
|
// browser we launched (every tab ours) adopts unknown targets.
|
||||||
|
if on_relay {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let attach_result: AttachToTargetResult = match self
|
let attach_result: AttachToTargetResult = match self
|
||||||
.client
|
.client
|
||||||
.send_command_typed(
|
.send_command_typed(
|
||||||
@@ -1621,12 +1689,26 @@ impl BrowserManager {
|
|||||||
let _ = self.enable_domains(&attach_result.session_id).await;
|
let _ = self.enable_domains(&attach_result.session_id).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop tabs that no longer exist so `tab list` doesn't show phantom rows —
|
// Prune tabs that are gone. On a LAUNCHED browser a missing target really
|
||||||
// but never prune the explicitly-pinned active target on a transient
|
// is closed, so prune immediately. On the RELAY a single `getTargets`
|
||||||
// getTargets snapshot (issue #31; see `prunable_target_ids`).
|
// snapshot routinely omits live tabs (multi-agent churn, a brief
|
||||||
let gone = prunable_target_ids(&self.pages, &live_ids, self.active_target_id.as_deref());
|
// cross-process-nav gap) — dropping the tab we're driving on one bad
|
||||||
for tid in gone {
|
// snapshot is the failure we're fixing — so prune only after the tab has
|
||||||
self.remove_page_by_target_id(&tid);
|
// been absent for several CONSECUTIVE snapshots (debounced). The pinned
|
||||||
|
// active target is protected either way (issue #31).
|
||||||
|
let gone = if on_relay {
|
||||||
|
debounced_prune_ids(
|
||||||
|
&self.pages,
|
||||||
|
&live_ids,
|
||||||
|
self.active_target_id.as_deref(),
|
||||||
|
&mut self.relay_target_misses,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
prunable_target_ids(&self.pages, &live_ids, self.active_target_id.as_deref())
|
||||||
|
};
|
||||||
|
for tid in &gone {
|
||||||
|
self.relay_target_misses.remove(tid);
|
||||||
|
self.remove_page_by_target_id(tid);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Refresh url/title from each live tab. The relay only stamps target_info
|
// Refresh url/title from each live tab. The relay only stamps target_info
|
||||||
@@ -2536,6 +2618,7 @@ async fn initialize_lightpanda_manager(
|
|||||||
visited_origins: HashSet::new(),
|
visited_origins: HashSet::new(),
|
||||||
created_targets: HashSet::new(),
|
created_targets: HashSet::new(),
|
||||||
active_target_id: None,
|
active_target_id: None,
|
||||||
|
relay_target_misses: HashMap::new(),
|
||||||
next_tab_id: 1,
|
next_tab_id: 1,
|
||||||
capture_console: console_capture_enabled(),
|
capture_console: console_capture_enabled(),
|
||||||
};
|
};
|
||||||
@@ -2992,6 +3075,43 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn debounced_prune_tolerates_transient_churn() {
|
||||||
|
// Multi-agent churn: a single getTargets snapshot omits our owned tab "B"
|
||||||
|
// (another agent opened/closed tabs). It must NOT be pruned on one miss.
|
||||||
|
let pages = vec![page("A"), page("B")];
|
||||||
|
let mut misses = HashMap::new();
|
||||||
|
let empty: HashSet<String> = HashSet::new();
|
||||||
|
// Misses 1 and 2: B absent but under threshold → not pruned.
|
||||||
|
assert!(debounced_prune_ids(&pages, &empty, Some("A"), &mut misses).is_empty());
|
||||||
|
assert!(debounced_prune_ids(&pages, &empty, Some("A"), &mut misses).is_empty());
|
||||||
|
// Miss 3 (== RELAY_PRUNE_MISSES): genuinely gone → pruned.
|
||||||
|
assert_eq!(
|
||||||
|
debounced_prune_ids(&pages, &empty, Some("A"), &mut misses),
|
||||||
|
vec!["B".to_string()]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn debounced_prune_resets_on_reappearance_and_protects_pin() {
|
||||||
|
let pages = vec![page("A"), page("B")];
|
||||||
|
let mut misses = HashMap::new();
|
||||||
|
let empty: HashSet<String> = HashSet::new();
|
||||||
|
let mut live_b: HashSet<String> = HashSet::new();
|
||||||
|
live_b.insert("B".to_string());
|
||||||
|
// Two misses for B, then it reappears → counter resets, so it survives
|
||||||
|
// indefinitely under intermittent churn.
|
||||||
|
debounced_prune_ids(&pages, &empty, Some("A"), &mut misses);
|
||||||
|
debounced_prune_ids(&pages, &empty, Some("A"), &mut misses);
|
||||||
|
assert!(debounced_prune_ids(&pages, &live_b, Some("A"), &mut misses).is_empty());
|
||||||
|
assert!(debounced_prune_ids(&pages, &empty, Some("A"), &mut misses).is_empty()); // back to miss 1
|
||||||
|
// The pinned active "A" is never pruned no matter how many misses.
|
||||||
|
for _ in 0..5 {
|
||||||
|
let gone = debounced_prune_ids(&pages, &empty, Some("A"), &mut misses);
|
||||||
|
assert!(!gone.contains(&"A".to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolve_active_index_pin_survives_passive_background_tab() {
|
fn resolve_active_index_pin_survives_passive_background_tab() {
|
||||||
// A foreign tab ("Z") gets appended by passive discovery after we pinned
|
// A foreign tab ("Z") gets appended by passive discovery after we pinned
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ fn native_test_fixture_html(name: &str) -> &'static str {
|
|||||||
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
|
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
|
||||||
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
|
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
|
||||||
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
|
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
|
||||||
|
"iframe_button_probe" => include_str!("test_fixtures/iframe_button_probe.html"),
|
||||||
_ => panic!("Unknown native test fixture: {}", name),
|
_ => panic!("Unknown native test fixture: {}", name),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -573,6 +574,76 @@ async fn e2e_snapshot_and_click_ref() {
|
|||||||
assert_success(&resp);
|
assert_success(&resp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clicking a button INSIDE an iframe by `@ref` must deliver a TRUSTED activation
|
||||||
|
/// (`event.isTrusted === true`), not a synthetic DOM `.click()`. Security-sensitive
|
||||||
|
/// embedded forms (Google Payments' `保存`) reject `isTrusted:false` clicks, so an
|
||||||
|
/// enabled submit button silently no-op'd (issue #39). The fix routes iframe-ref
|
||||||
|
/// clicks to a real `Input.dispatchMouseEvent` on the element's own frame session.
|
||||||
|
/// The fixture's iframe button writes `clicked:<isTrusted>` into its own text on
|
||||||
|
/// click, which the cross-frame snapshot reads back.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore]
|
||||||
|
async fn e2e_iframe_button_click_is_trusted() {
|
||||||
|
let mut state = DaemonState::new();
|
||||||
|
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "1", "action": "launch", "headless": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "2", "action": "navigate", "url": native_test_fixture_url("iframe_button_probe") }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
// Snapshot (interactive) — the button lives in the iframe and must appear with
|
||||||
|
// a ref; that ref carries the frame_id so the click resolves into the frame.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "3", "action": "snapshot", "interactive": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
let snapshot = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||||
|
let ref_id = snapshot
|
||||||
|
.lines()
|
||||||
|
.find(|l| l.contains("button \"save\""))
|
||||||
|
.and_then(|l| l.split("ref=").nth(1))
|
||||||
|
.map(|r| r.trim_end_matches(']').trim())
|
||||||
|
.unwrap_or_else(|| panic!("iframe button not found in snapshot:\n{snapshot}"));
|
||||||
|
|
||||||
|
// Click it by ref.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "4", "action": "click", "selector": ref_id }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(300)).await;
|
||||||
|
|
||||||
|
// The button rewrote its own text with the click's isTrusted flag; read it
|
||||||
|
// back across frames.
|
||||||
|
let resp = execute_command(
|
||||||
|
&json!({ "id": "5", "action": "snapshot", "interactive": true }),
|
||||||
|
&mut state,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_success(&resp);
|
||||||
|
let after = get_data(&resp)["snapshot"].as_str().unwrap_or("");
|
||||||
|
assert!(
|
||||||
|
after.contains("clicked:true"),
|
||||||
|
"iframe button click must be trusted (isTrusted:true); snapshot:\n{after}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
|
||||||
|
assert_success(&resp);
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Screenshot
|
// Screenshot
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -56,19 +56,34 @@ pub async fn click(
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Over the extension relay we drive the user's real, in-use Chrome, where a
|
// An element INSIDE an iframe needs a TRUSTED activation: a DOM `.click()` is
|
||||||
// coordinate `Input.dispatchMouseEvent` is NOT reliably confined to our target
|
// `isTrusted:false`, which security-sensitive embedded forms reject — Google
|
||||||
// tab — it can be delivered to whatever tab is in the foreground, and an OOPIF
|
// Payments' enabled `保存` button silently no-ops on a synthetic click (issue
|
||||||
// element's box can't be mapped to a top-viewport point at all. This twice
|
// #39). A coordinate `Input.dispatchMouseEvent` can't help either: `getBoxModel`
|
||||||
// opened an unrelated tab on the user's busy Chrome (issues #31/#36). So on the
|
// for a sub-frame node returns frame-local coordinates that don't compose the
|
||||||
// relay, never use coordinates for a normal left click: DOM-dispatch invokes
|
// iframe's offset, so the click lands in the wrong place. The frame-agnostic
|
||||||
// the element's click in its own (frame) session, always hitting the right
|
// trusted path is keyboard activation — focus the element in its own frame, then
|
||||||
// element in the right tab. Double/right clicks still need true pointer
|
// dispatch a real Enter on the page session; Chrome routes the key to the
|
||||||
// semantics, and `coord` mode is an explicit opt-out.
|
// focused element regardless of frame (same as `type --focused`), and Enter on a
|
||||||
|
// focused button/link fires a trusted `click`. `coord` mode opts out.
|
||||||
|
let in_iframe = ref_map.ref_is_in_iframe(selector_or_ref);
|
||||||
|
if mode != "coord" && button == "left" && click_count == 1 && in_iframe {
|
||||||
|
return dom_activate(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
// On the relay (the user's real Chrome) a TOP-document coordinate click used to
|
||||||
|
// drift onto the foreground tab; that root cause is fixed (#5: the agent drives
|
||||||
|
// its own pinned tab), but DOM-dispatch stays the conservative default here.
|
||||||
if mode != "coord"
|
if mode != "coord"
|
||||||
&& button == "left"
|
&& button == "left"
|
||||||
&& click_count == 1
|
&& click_count == 1
|
||||||
&& prefer_dom_dispatch(ref_map, selector_or_ref)
|
&& crate::connect::relay_url().is_some()
|
||||||
{
|
{
|
||||||
return dom_click(
|
return dom_click(
|
||||||
client,
|
client,
|
||||||
@@ -270,6 +285,71 @@ async fn dom_click(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Trusted activation of an element inside an iframe (issue #39). Focuses the
|
||||||
|
/// element in its own frame session, then dispatches a real Enter/Space on the
|
||||||
|
/// page session — Chrome routes the key to the focused element across frames, and
|
||||||
|
/// Enter/Space on a focused button/link/checkbox fires a `click` with
|
||||||
|
/// `isTrusted: true`, which security-sensitive embedded forms (Google Payments
|
||||||
|
/// `保存`) require. Non-activatable roles (a `div[onclick]`) can't be keyboard-
|
||||||
|
/// activated, so they fall back to a DOM `.click()`.
|
||||||
|
async fn dom_activate(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
ref_map: &RefMap,
|
||||||
|
selector_or_ref: &str,
|
||||||
|
iframe_sessions: &HashMap<String, String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let role = parse_ref(selector_or_ref)
|
||||||
|
.and_then(|r| ref_map.get(&r).map(|e| e.role.clone()))
|
||||||
|
.unwrap_or_default();
|
||||||
|
// Space toggles checkbox-like controls; Enter activates buttons/links/menus.
|
||||||
|
let key = match role.as_str() {
|
||||||
|
"checkbox" | "radio" | "switch" | "option" | "menuitemcheckbox" | "menuitemradio" => {
|
||||||
|
Some("space")
|
||||||
|
}
|
||||||
|
"button" | "link" | "menuitem" | "tab" | "treeitem" => Some("enter"),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let Some(key) = key else {
|
||||||
|
// Not keyboard-activatable — best effort via DOM .click() (untrusted).
|
||||||
|
return dom_click(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
};
|
||||||
|
|
||||||
|
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||||
|
client,
|
||||||
|
session_id,
|
||||||
|
ref_map,
|
||||||
|
selector_or_ref,
|
||||||
|
iframe_sessions,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// Focus the element in its OWN frame session so the keystroke lands on it.
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Runtime.callFunctionOn",
|
||||||
|
&CallFunctionOnParams {
|
||||||
|
function_declaration: "function() { this.focus(); }".to_string(),
|
||||||
|
object_id: Some(object_id),
|
||||||
|
arguments: None,
|
||||||
|
return_by_value: Some(true),
|
||||||
|
await_promise: Some(false),
|
||||||
|
},
|
||||||
|
Some(&effective_session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// Trusted key on the page session — routed to the focused (in-frame) element.
|
||||||
|
press_key(client, session_id, key).await?;
|
||||||
|
wait_for_paint_settled(client, &effective_session_id).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// DOM-dispatch a double-click on the element in its own session (no coordinates)
|
/// DOM-dispatch a double-click on the element in its own session (no coordinates)
|
||||||
/// — the relay/iframe-safe counterpart to a coordinate dblclick. Fires the full
|
/// — the relay/iframe-safe counterpart to a coordinate dblclick. Fires the full
|
||||||
/// click,click,dblclick sequence so handlers bound to any of them respond.
|
/// click,click,dblclick sequence so handlers bound to any of them respond.
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>iframe button probe</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>iframe button probe</h1>
|
||||||
|
<iframe
|
||||||
|
id="frame"
|
||||||
|
width="320"
|
||||||
|
height="140"
|
||||||
|
srcdoc="
|
||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<body style='margin:24px'>
|
||||||
|
<button id='b' style='padding:24px;font-size:22px'>save</button>
|
||||||
|
<script>
|
||||||
|
document.getElementById('b').addEventListener('click', function (e) {
|
||||||
|
this.textContent = 'clicked:' + e.isTrusted;
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"
|
||||||
|
></iframe>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "1.5.14",
|
"version": "1.5.17",
|
||||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"packageManager": "pnpm@11.1.3",
|
"packageManager": "pnpm@11.1.3",
|
||||||
|
|||||||
@@ -131,7 +131,17 @@ Each `--session` that connects gets its **own colored Chrome tab group** (named
|
|||||||
after the session) and drives only its own tabs — multiple agents share the one
|
after the session) and drives only its own tabs — multiple agents share the one
|
||||||
real browser without cross-talk, and the user's own tabs are never grouped. CDP
|
real browser without cross-talk, and the user's own tabs are never grouped. CDP
|
||||||
drives the page without moving the user's mouse/keyboard, so it doesn't fight
|
drives the page without moving the user's mouse/keyboard, so it doesn't fight
|
||||||
them for control. **Anti-detection ranking: this real logged-in Chrome (extension
|
them for control.
|
||||||
|
|
||||||
|
**Strict multi-agent isolation.** A session over the relay tracks and drives
|
||||||
|
**only the tabs it created** (its own group). It does **not** adopt the user's
|
||||||
|
existing tabs, other agents' tabs, or pop-ups (e.g. an OAuth/login window — that's
|
||||||
|
the user's), so several agents (and other tools opening tabs) can work in the same
|
||||||
|
real Chrome concurrently without ever dropping or stealing each other's tabs —
|
||||||
|
another agent's tab churn can't make your bound tab vanish or drift your commands
|
||||||
|
onto the wrong page. Consequence: `tab list` shows only *your* session's tabs; to
|
||||||
|
drive a specific page, navigate to it in your own tab instead of expecting a
|
||||||
|
pre-existing or popped-up tab to appear in the list. **Anti-detection ranking: this real logged-in Chrome (extension
|
||||||
connect) > a headed launched browser > headless (forbidden).** A genuine human
|
connect) > a headed launched browser > headless (forbidden).** A genuine human
|
||||||
browser has no headless/automation tells at all, so prefer it for anything
|
browser has no headless/automation tells at all, so prefer it for anything
|
||||||
anti-bot-sensitive.
|
anti-bot-sensitive.
|
||||||
|
|||||||
Reference in New Issue
Block a user