feat: support cross-origin iframe snapshots and interactions via Target.setAutoAttach (#949)

* feat: support cross-origin iframe snapshots and interactions via Target.setAutoAttach (#925)

Enable Target.setAutoAttach with flatten: true on page sessions so Chrome
auto-creates dedicated CDP sessions for cross-origin iframe targets.

- Add DrainedEvents struct, iframe_sessions map, attach/detach event handling
- Extract resolve_ax_session (shared) and resolve_frame_session helpers
- resolve_element_object_id returns (object_id, effective_session) tuple
- resolve_element_center returns (x, y, effective_session) tuple
- Input dispatch (click/hover/tap) uses effective session for correct coordinates
- Thread iframe_sessions through element.rs, interaction.rs, screenshot.rs
- Clear iframe sessions on navigate, tab switch, tab new, tab close
- Ignore Target.setAutoAttach failure for non-Chrome backends (Lightpanda)
- Unit tests for session resolution logic

* fix

* fix

* fix

---------

Co-authored-by: hyunjinee <leehj0110@kakao.com>
This commit is contained in:
jin.2
2026-03-20 16:37:33 -05:00
committed by GitHub
co-authored by hyunjinee
parent 39e54113e6
commit 59b6e5a034
6 changed files with 919 additions and 168 deletions
+366 -65
View File
@@ -133,6 +133,17 @@ pub struct MouseState {
pub buttons: i32,
}
#[derive(Default)]
struct DrainedEvents {
pending_acks: Vec<i64>,
new_targets: Vec<TargetCreatedEvent>,
destroyed_targets: Vec<String>,
/// Cross-origin iframe (frame_id, session_id) pairs from Target.attachedToTarget.
attached_iframe_sessions: Vec<(String, String)>,
/// Session IDs from Target.detachedFromTarget.
detached_iframe_sessions: Vec<String>,
}
pub struct DaemonState {
pub browser: Option<BrowserManager>,
pub appium: Option<AppiumManager>,
@@ -158,6 +169,9 @@ pub struct DaemonState {
pub tracked_requests: Vec<TrackedRequest>,
pub request_tracking: bool,
pub active_frame_id: Option<String>,
/// Cross-origin iframe frame_id → dedicated CDP session_id.
/// Populated by Target.attachedToTarget events from Target.setAutoAttach.
pub iframe_sessions: HashMap<String, String>,
/// Origin-scoped extra HTTP headers set via `--headers` on navigate.
/// Key is the origin (scheme + host + port), value is the headers map.
/// Wrapped in Arc<RwLock<>> so the background Fetch handler can read it.
@@ -205,6 +219,7 @@ impl DaemonState {
tracked_requests: Vec::new(),
request_tracking: false,
active_frame_id: None,
iframe_sessions: HashMap::new(),
origin_headers: Arc::new(RwLock::new(HashMap::new())),
fetch_handler_task: None,
mouse_state: MouseState::default(),
@@ -355,15 +370,17 @@ impl DaemonState {
recording::stop_recording_task(&mut self.recording_state).await
}
fn drain_cdp_events(&mut self) -> (Vec<i64>, Vec<TargetCreatedEvent>, Vec<String>) {
fn drain_cdp_events(&mut self) -> DrainedEvents {
let rx = match self.event_rx.as_mut() {
Some(rx) => rx,
None => return (Vec::new(), Vec::new(), Vec::new()),
None => return DrainedEvents::default(),
};
let mut pending_acks: Vec<i64> = Vec::new();
let mut new_targets: Vec<TargetCreatedEvent> = Vec::new();
let mut destroyed_targets: Vec<String> = Vec::new();
let mut attached_iframe_sessions: Vec<(String, String)> = Vec::new();
let mut detached_iframe_sessions: Vec<String> = Vec::new();
loop {
match rx.try_recv() {
@@ -397,6 +414,36 @@ impl DaemonState {
}
continue;
}
"Target.attachedToTarget" => {
if let (Some(sid), Some(target_info)) = (
event.params.get("sessionId").and_then(|v| v.as_str()),
event.params.get("targetInfo"),
) {
let target_type = target_info
.get("type")
.and_then(|v| v.as_str())
.unwrap_or("");
if target_type == "iframe" {
// For OOPIF targets, Chrome uses the frameId as
// the targetId, so we can key iframe_sessions by it.
if let Some(target_id) =
target_info.get("targetId").and_then(|v| v.as_str())
{
attached_iframe_sessions
.push((target_id.to_string(), sid.to_string()));
}
}
}
continue;
}
"Target.detachedFromTarget" => {
if let Some(sid) =
event.params.get("sessionId").and_then(|v| v.as_str())
{
detached_iframe_sessions.push(sid.to_string());
}
continue;
}
_ => {}
}
@@ -636,7 +683,13 @@ impl DaemonState {
}
}
(pending_acks, new_targets, destroyed_targets)
DrainedEvents {
pending_acks,
new_targets,
destroyed_targets,
attached_iframe_sessions,
detached_iframe_sessions,
}
}
}
@@ -659,7 +712,13 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
.to_string();
// Drain pending CDP events (console, errors, screencast frames, target lifecycle)
let (pending_acks, new_targets, destroyed_targets) = state.drain_cdp_events();
let DrainedEvents {
pending_acks,
new_targets,
destroyed_targets,
attached_iframe_sessions,
detached_iframe_sessions,
} = state.drain_cdp_events();
if !pending_acks.is_empty() {
if let Some(ref browser) = state.browser {
if let Ok(session_id) = browser.active_session_id() {
@@ -677,6 +736,26 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
}
}
// Track cross-origin iframe sessions
for (frame_id, iframe_sid) in &attached_iframe_sessions {
state
.iframe_sessions
.insert(frame_id.clone(), iframe_sid.clone());
if let Some(ref mgr) = state.browser {
let _ = mgr
.client
.send_command_no_params("DOM.enable", Some(iframe_sid.as_str()))
.await;
let _ = mgr
.client
.send_command_no_params("Accessibility.enable", Some(iframe_sid.as_str()))
.await;
}
}
for sid in &detached_iframe_sessions {
state.iframe_sessions.retain(|_, v| v != sid);
}
for te in &new_targets {
if let Some(ref mut mgr) = state.browser {
let attach_result: Result<AttachToTargetResult, String> = mgr
@@ -1483,6 +1562,8 @@ async fn handle_navigate(cmd: &Value, state: &mut DaemonState) -> Result<Value,
}
state.ref_map.clear();
state.iframe_sessions.clear();
state.active_frame_id = None;
mgr.navigate(url, wait_until).await
}
@@ -1680,6 +1761,7 @@ async fn handle_snapshot(cmd: &Value, state: &mut DaemonState) -> Result<Value,
&options,
&mut state.ref_map,
state.active_frame_id.as_deref(),
&state.iframe_sessions,
)
.await?;
@@ -1786,12 +1868,19 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
},
&mut state.ref_map,
state.active_frame_id.as_deref(),
&state.iframe_sessions,
)
.await?;
}
let result =
screenshot::take_screenshot(&mgr.client, &session_id, &state.ref_map, &options).await?;
let result = screenshot::take_screenshot(
&mgr.client,
&session_id,
&state.ref_map,
&options,
&state.iframe_sessions,
)
.await?;
let mut response = json!({ "path": result.path });
if !result.annotations.is_empty() {
@@ -1822,8 +1911,14 @@ async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
if new_tab {
use super::element::resolve_element_object_id;
let object_id =
resolve_element_object_id(&mgr.client, &session_id, &state.ref_map, selector).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
let call_params = json!({
"objectId": object_id,
"functionDeclaration": "function() { var h = this.getAttribute('href'); if (!h) return null; try { return new URL(h, document.baseURI).toString(); } catch(e) { return null; } }",
@@ -1834,7 +1929,7 @@ async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
.send_command(
"Runtime.callFunctionOn",
Some(call_params),
Some(&session_id),
Some(&effective_session_id),
)
.await?;
let href = call_result
@@ -1866,6 +1961,7 @@ async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
selector,
button,
click_count,
&state.iframe_sessions,
)
.await?;
@@ -1880,7 +1976,14 @@ async fn handle_dblclick(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::dblclick(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::dblclick(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "clicked": selector }))
}
@@ -1904,7 +2007,15 @@ async fn handle_fill(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
let session_id = mgr.active_session_id()?.to_string();
interaction::fill(&mgr.client, &session_id, &state.ref_map, selector, value).await?;
interaction::fill(
&mgr.client,
&session_id,
&state.ref_map,
selector,
value,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "filled": selector }))
}
@@ -1930,6 +2041,7 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
text,
clear,
delay,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "typed": text }))
@@ -1955,7 +2067,14 @@ async fn handle_hover(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::hover(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::hover(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "hovered": selector }))
}
@@ -1980,7 +2099,16 @@ async fn handle_scroll(cmd: &Value, state: &mut DaemonState) -> Result<Value, St
}
}
interaction::scroll(&mgr.client, &session_id, &state.ref_map, selector, dx, dy).await?;
interaction::scroll(
&mgr.client,
&session_id,
&state.ref_map,
selector,
dx,
dy,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "scrolled": true }))
}
@@ -2005,7 +2133,15 @@ async fn handle_select(cmd: &Value, state: &mut DaemonState) -> Result<Value, St
.unwrap_or_default(),
};
interaction::select_option(&mgr.client, &session_id, &state.ref_map, selector, &values).await?;
interaction::select_option(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&values,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "selected": values }))
}
@@ -2017,7 +2153,14 @@ async fn handle_check(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::check(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::check(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "checked": selector }))
}
@@ -2029,7 +2172,14 @@ async fn handle_uncheck(cmd: &Value, state: &mut DaemonState) -> Result<Value, S
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::uncheck(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::uncheck(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "unchecked": selector }))
}
@@ -2082,8 +2232,14 @@ async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, S
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
let text = super::element::get_element_text(&mgr.client, &session_id, &state.ref_map, selector)
.await?;
let text = super::element::get_element_text(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
let url = mgr.get_url().await.unwrap_or_default();
Ok(json!({ "text": text, "origin": url }))
}
@@ -2106,6 +2262,7 @@ async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result<Val
&state.ref_map,
selector,
attribute,
&state.iframe_sessions,
)
.await?;
let url = mgr.get_url().await.unwrap_or_default();
@@ -2120,9 +2277,14 @@ async fn handle_isvisible(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
let visible =
super::element::is_element_visible(&mgr.client, &session_id, &state.ref_map, selector)
.await?;
let visible = super::element::is_element_visible(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
let url = mgr.get_url().await.unwrap_or_default();
Ok(json!({ "visible": visible, "origin": url }))
}
@@ -2135,9 +2297,14 @@ async fn handle_isenabled(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
let enabled =
super::element::is_element_enabled(&mgr.client, &session_id, &state.ref_map, selector)
.await?;
let enabled = super::element::is_element_enabled(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
let url = mgr.get_url().await.unwrap_or_default();
Ok(json!({ "enabled": enabled, "origin": url }))
}
@@ -2150,9 +2317,14 @@ async fn handle_ischecked(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
let checked =
super::element::is_element_checked(&mgr.client, &session_id, &state.ref_map, selector)
.await?;
let checked = super::element::is_element_checked(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
let url = mgr.get_url().await.unwrap_or_default();
Ok(json!({ "checked": checked, "origin": url }))
}
@@ -2582,6 +2754,7 @@ async fn handle_diff_snapshot(cmd: &Value, state: &mut DaemonState) -> Result<Va
&options,
&mut state.ref_map,
state.active_frame_id.as_deref(),
&state.iframe_sessions,
)
.await?;
@@ -2627,16 +2800,28 @@ async fn handle_diff_url(cmd: &Value, state: &mut DaemonState) -> Result<Value,
mgr.navigate(url1, wait_until).await?;
let session_id = mgr.active_session_id()?.to_string();
let options = SnapshotOptions::default();
let snap1 =
snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map, None)
.await?;
let snap1 = snapshot::take_snapshot(
&mgr.client,
&session_id,
&options,
&mut state.ref_map,
None,
&state.iframe_sessions,
)
.await?;
// Navigate to URL2 and snapshot
mgr.navigate(url2, wait_until).await?;
state.ref_map.clear();
let snap2 =
snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map, None)
.await?;
let snap2 = snapshot::take_snapshot(
&mgr.client,
&session_id,
&options,
&mut state.ref_map,
None,
&state.iframe_sessions,
)
.await?;
let result = diff::diff_text(&snap1, &snap2);
Ok(json!({
@@ -2767,6 +2952,8 @@ async fn handle_tab_new(cmd: &Value, state: &mut DaemonState) -> Result<Value, S
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
let url = cmd.get("url").and_then(|v| v.as_str());
state.ref_map.clear();
state.iframe_sessions.clear();
state.active_frame_id = None;
mgr.tab_new(url).await
}
@@ -2777,6 +2964,8 @@ async fn handle_tab_switch(cmd: &Value, state: &mut DaemonState) -> Result<Value
.and_then(|v| v.as_u64())
.ok_or("Missing 'index' parameter")? as usize;
state.ref_map.clear();
state.iframe_sessions.clear();
state.active_frame_id = None;
mgr.tab_switch(index).await
}
@@ -2787,6 +2976,8 @@ async fn handle_tab_close(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_u64())
.map(|i| i as usize);
state.ref_map.clear();
state.iframe_sessions.clear();
state.active_frame_id = None;
mgr.tab_close(index).await
}
@@ -3076,7 +3267,14 @@ async fn handle_focus(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::focus(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::focus(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "focused": selector }))
}
@@ -3088,7 +3286,14 @@ async fn handle_clear(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::clear(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::clear(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "cleared": selector }))
}
@@ -3100,7 +3305,14 @@ async fn handle_selectall(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::select_all(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::select_all(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "selected": selector }))
}
@@ -3112,7 +3324,14 @@ async fn handle_scrollintoview(cmd: &Value, state: &mut DaemonState) -> Result<V
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::scroll_into_view(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::scroll_into_view(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "scrolled": selector }))
}
@@ -3137,6 +3356,7 @@ async fn handle_dispatch(cmd: &Value, state: &mut DaemonState) -> Result<Value,
selector,
event_type,
event_init,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "dispatched": event_type, "selector": selector }))
@@ -3150,7 +3370,14 @@ async fn handle_highlight(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
interaction::highlight(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::highlight(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "highlighted": selector }))
}
@@ -3171,7 +3398,14 @@ async fn handle_tap(cmd: &Value, state: &mut DaemonState) -> Result<Value, Strin
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
let session_id = mgr.active_session_id()?.to_string();
interaction::tap_touch(&mgr.client, &session_id, &state.ref_map, sel).await?;
interaction::tap_touch(
&mgr.client,
&session_id,
&state.ref_map,
sel,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "tapped": sel }))
}
@@ -3188,6 +3422,7 @@ async fn handle_boundingbox(cmd: &Value, state: &mut DaemonState) -> Result<Valu
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(bbox)
@@ -3201,9 +3436,14 @@ async fn handle_innertext(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
let text =
super::element::get_element_inner_text(&mgr.client, &session_id, &state.ref_map, selector)
.await?;
let text = super::element::get_element_inner_text(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "text": text }))
}
@@ -3215,9 +3455,14 @@ async fn handle_innerhtml(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
let html =
super::element::get_element_inner_html(&mgr.client, &session_id, &state.ref_map, selector)
.await?;
let html = super::element::get_element_inner_html(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "html": html }))
}
@@ -3229,9 +3474,14 @@ async fn handle_inputvalue(cmd: &Value, state: &mut DaemonState) -> Result<Value
.and_then(|v| v.as_str())
.ok_or("Missing 'selector' parameter")?;
let value =
super::element::get_element_input_value(&mgr.client, &session_id, &state.ref_map, selector)
.await?;
let value = super::element::get_element_input_value(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "value": value }))
}
@@ -3247,8 +3497,15 @@ async fn handle_setvalue(cmd: &Value, state: &mut DaemonState) -> Result<Value,
.and_then(|v| v.as_str())
.ok_or("Missing 'value' parameter")?;
super::element::set_element_value(&mgr.client, &session_id, &state.ref_map, selector, value)
.await?;
super::element::set_element_value(
&mgr.client,
&session_id,
&state.ref_map,
selector,
value,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "set": selector, "value": value }))
}
@@ -3284,6 +3541,7 @@ async fn handle_styles(cmd: &Value, state: &mut DaemonState) -> Result<Value, St
&state.ref_map,
selector,
properties,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "styles": styles }))
@@ -3872,6 +4130,7 @@ async fn execute_subaction(
selector,
"left",
1,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "clicked": selector }))
@@ -3881,15 +4140,37 @@ async fn execute_subaction(
.get("value")
.and_then(|v| v.as_str())
.ok_or("Missing 'value' for fill subaction")?;
interaction::fill(&mgr.client, &session_id, &state.ref_map, selector, value).await?;
interaction::fill(
&mgr.client,
&session_id,
&state.ref_map,
selector,
value,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "filled": selector }))
}
"check" => {
interaction::check(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::check(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "checked": selector }))
}
"hover" => {
interaction::hover(&mgr.client, &session_id, &state.ref_map, selector).await?;
interaction::hover(
&mgr.client,
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "hovered": selector }))
}
"text" => {
@@ -3898,6 +4179,7 @@ async fn execute_subaction(
&session_id,
&state.ref_map,
selector,
&state.iframe_sessions,
)
.await?;
Ok(json!({ "text": text }))
@@ -4351,26 +4633,36 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
.and_then(|v| v.as_str())
.ok_or("Missing 'target' parameter")?;
let (sx, sy) =
super::element::resolve_element_center(&mgr.client, &session_id, &state.ref_map, source)
.await?;
let (tx, ty) =
super::element::resolve_element_center(&mgr.client, &session_id, &state.ref_map, target)
.await?;
let (sx, sy, source_session_id) = super::element::resolve_element_center(
&mgr.client,
&session_id,
&state.ref_map,
source,
&state.iframe_sessions,
)
.await?;
let (tx, ty, target_session_id) = super::element::resolve_element_center(
&mgr.client,
&session_id,
&state.ref_map,
target,
&state.iframe_sessions,
)
.await?;
// Mouse down at source
mgr.client
.send_command(
"Input.dispatchMouseEvent",
Some(json!({ "type": "mouseMoved", "x": sx, "y": sy })),
Some(&session_id),
Some(&source_session_id),
)
.await?;
mgr.client
.send_command(
"Input.dispatchMouseEvent",
Some(json!({ "type": "mousePressed", "x": sx, "y": sy, "button": "left", "clickCount": 1 })),
Some(&session_id),
Some(&source_session_id),
)
.await?;
@@ -4383,7 +4675,7 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
.send_command(
"Input.dispatchMouseEvent",
Some(json!({ "type": "mouseMoved", "x": cx, "y": cy })),
Some(&session_id),
Some(&target_session_id),
)
.await?;
tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
@@ -4394,7 +4686,7 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
.send_command(
"Input.dispatchMouseEvent",
Some(json!({ "type": "mouseReleased", "x": tx, "y": ty, "button": "left", "clickCount": 1 })),
Some(&session_id),
Some(&target_session_id),
)
.await?;
@@ -4665,8 +4957,14 @@ async fn handle_diff_screenshot(cmd: &Value, state: &DaemonState) -> Result<Valu
output_dir: None,
};
let result =
screenshot::take_screenshot(&mgr.client, &session_id, &state.ref_map, &options).await?;
let result = screenshot::take_screenshot(
&mgr.client,
&session_id,
&state.ref_map,
&options,
&state.iframe_sessions,
)
.await?;
let current_bytes =
base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &result.base64)
@@ -5670,6 +5968,7 @@ async fn handle_auth_login(cmd: &Value, state: &mut DaemonState) -> Result<Value
&state.ref_map,
&user_sel,
&username,
&state.iframe_sessions,
)
.await?;
@@ -5690,6 +5989,7 @@ async fn handle_auth_login(cmd: &Value, state: &mut DaemonState) -> Result<Value
&state.ref_map,
&pass_sel,
&password,
&state.iframe_sessions,
)
.await?;
@@ -5721,6 +6021,7 @@ async fn handle_auth_login(cmd: &Value, state: &mut DaemonState) -> Result<Value
&sub_sel,
"left",
1,
&state.iframe_sessions,
)
.await?;
+15
View File
@@ -420,6 +420,21 @@ impl BrowserManager {
self.client
.send_command_no_params("Network.enable", Some(session_id))
.await?;
// Enable auto-attach for cross-origin iframe support.
// flatten: true gives each iframe its own session_id.
// Ignored on engines that don't support it (e.g. Lightpanda).
let _ = self
.client
.send_command(
"Target.setAutoAttach",
Some(json!({
"autoAttach": true,
"waitForDebuggerOnStart": false,
"flatten": true
})),
Some(session_id),
)
.await;
Ok(())
}
+227 -47
View File
@@ -147,12 +147,16 @@ pub async fn resolve_element_center(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
) -> Result<(f64, f64), String> {
iframe_sessions: &HashMap<String, String>,
) -> Result<(f64, f64, String), String> {
if let Some(ref_id) = parse_ref(selector_or_ref) {
let entry = ref_map
.get(&ref_id)
.ok_or_else(|| format!("Unknown ref: {}", ref_id))?;
let effective_session_id =
resolve_frame_session(entry.frame_id.as_deref(), session_id, iframe_sessions);
// Try cached backend_node_id first (fast path)
if let Some(backend_node_id) = entry.backend_node_id {
let result: Result<DomGetBoxModelResult, String> = client
@@ -163,25 +167,26 @@ pub async fn resolve_element_center(
node_id: None,
object_id: None,
},
Some(session_id),
Some(effective_session_id),
)
.await;
if let Ok(r) = result {
return Ok(box_model_center(&r.model));
let (x, y) = box_model_center(&r.model);
return Ok((x, y, effective_session_id.to_string()));
}
// backend_node_id is stale; re-query the accessibility tree below
}
// Fallback: re-query the accessibility tree to find a fresh node by role/name
let ref_frame_id = entry.frame_id.clone();
let fresh_id = find_node_id_by_role_name(
client,
session_id,
&entry.role,
&entry.name,
entry.nth,
ref_frame_id.as_deref(),
entry.frame_id.as_deref(),
iframe_sessions,
)
.await?;
let result: DomGetBoxModelResult = client
@@ -192,14 +197,16 @@ pub async fn resolve_element_center(
node_id: None,
object_id: None,
},
Some(session_id),
Some(effective_session_id),
)
.await?;
return Ok(box_model_center(&result.model));
let (x, y) = box_model_center(&result.model);
return Ok((x, y, effective_session_id.to_string()));
}
// CSS selector
resolve_by_selector(client, session_id, selector_or_ref).await
let (x, y) = resolve_by_selector(client, session_id, selector_or_ref).await?;
Ok((x, y, session_id.to_string()))
}
pub async fn resolve_element_object_id(
@@ -207,12 +214,16 @@ pub async fn resolve_element_object_id(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
) -> Result<String, String> {
iframe_sessions: &HashMap<String, String>,
) -> Result<(String, String), String> {
if let Some(ref_id) = parse_ref(selector_or_ref) {
let entry = ref_map
.get(&ref_id)
.ok_or_else(|| format!("Unknown ref: {}", ref_id))?;
let effective_session_id =
resolve_frame_session(entry.frame_id.as_deref(), session_id, iframe_sessions);
// Try cached backend_node_id first (fast path)
if let Some(backend_node_id) = entry.backend_node_id {
let result: Result<DomResolveNodeResult, String> = client
@@ -223,27 +234,27 @@ pub async fn resolve_element_object_id(
node_id: None,
object_group: Some("agent-browser".to_string()),
},
Some(session_id),
Some(effective_session_id),
)
.await;
if let Ok(r) = result {
if let Some(oid) = r.object.object_id {
return Ok(oid);
if let Some(object_id) = r.object.object_id {
return Ok((object_id, effective_session_id.to_string()));
}
}
// backend_node_id is stale; re-query the accessibility tree below
}
// Fallback: re-query the accessibility tree to find a fresh node by role/name
let ref_frame_id = entry.frame_id.clone();
let fresh_id = find_node_id_by_role_name(
client,
session_id,
&entry.role,
&entry.name,
entry.nth,
ref_frame_id.as_deref(),
entry.frame_id.as_deref(),
iframe_sessions,
)
.await?;
let result: DomResolveNodeResult = client
@@ -254,13 +265,14 @@ pub async fn resolve_element_object_id(
node_id: None,
object_group: Some("agent-browser".to_string()),
},
Some(session_id),
Some(effective_session_id),
)
.await?;
return result
let object_id = result
.object
.object_id
.ok_or_else(|| format!("No objectId for ref {}", ref_id));
.ok_or_else(|| format!("No objectId for ref {}", ref_id))?;
return Ok((object_id, effective_session_id.to_string()));
}
// Selector fallback (CSS or XPath)
@@ -277,10 +289,44 @@ pub async fn resolve_element_object_id(
)
.await?;
result
let object_id = result
.result
.object_id
.ok_or_else(|| format!("Element not found: {}", selector_or_ref))
.ok_or_else(|| format!("Element not found: {}", selector_or_ref))?;
Ok((object_id, session_id.to_string()))
}
/// Determine which CDP session and parameters to use for an AX tree query.
/// Cross-origin iframes have a dedicated session (no frameId needed);
/// same-origin iframes use the parent session with a frameId parameter.
pub(super) fn resolve_ax_session<'a>(
frame_id: Option<&str>,
session_id: &'a str,
iframe_sessions: &'a HashMap<String, String>,
) -> (serde_json::Value, &'a str) {
if let Some(frame_id) = frame_id {
if let Some(iframe_sid) = iframe_sessions.get(frame_id) {
(serde_json::json!({}), iframe_sid.as_str())
} else {
(serde_json::json!({ "frameId": frame_id }), session_id)
}
} else {
(serde_json::json!({}), session_id)
}
}
/// Resolve the effective CDP session for an element's frame.
/// If the element's frame_id has a dedicated cross-origin iframe session, return it.
/// Otherwise, return the parent session.
fn resolve_frame_session<'a>(
frame_id: Option<&str>,
session_id: &'a str,
iframe_sessions: &'a HashMap<String, String>,
) -> &'a str {
frame_id
.and_then(|fid| iframe_sessions.get(fid))
.map(|s| s.as_str())
.unwrap_or(session_id)
}
/// Re-query the accessibility tree to find a node matching role+name+nth,
@@ -294,14 +340,16 @@ async fn find_node_id_by_role_name(
name: &str,
nth: Option<usize>,
frame_id: Option<&str>,
iframe_sessions: &HashMap<String, String>,
) -> Result<i64, String> {
let ax_params = if let Some(fid) = frame_id {
serde_json::json!({ "frameId": fid })
} else {
serde_json::json!({})
};
let (ax_params, effective_session_id) =
resolve_ax_session(frame_id, session_id, iframe_sessions);
let ax_tree: GetFullAXTreeResult = client
.send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id))
.send_command_typed(
"Accessibility.getFullAXTree",
&ax_params,
Some(effective_session_id),
)
.await?;
let nth_index = nth.unwrap_or(0);
@@ -431,8 +479,16 @@ pub async fn get_element_text(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -445,7 +501,7 @@ pub async fn get_element_text(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -462,8 +518,16 @@ pub async fn get_element_attribute(
ref_map: &RefMap,
selector_or_ref: &str,
attribute: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<Value, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -478,7 +542,7 @@ pub async fn get_element_attribute(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -490,8 +554,16 @@ pub async fn is_element_visible(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<bool, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -511,7 +583,7 @@ pub async fn is_element_visible(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -527,8 +599,16 @@ pub async fn is_element_enabled(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<bool, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -540,7 +620,7 @@ pub async fn is_element_enabled(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -556,8 +636,16 @@ pub async fn is_element_checked(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<bool, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// Mirrors Playwright's getChecked() with follow-label retargeting:
// 1. If element is a native checkbox/radio input, return .checked
@@ -602,7 +690,7 @@ pub async fn is_element_checked(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -618,8 +706,16 @@ pub async fn get_element_inner_text(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -631,7 +727,7 @@ pub async fn get_element_inner_text(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -647,8 +743,16 @@ pub async fn get_element_inner_html(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -660,7 +764,7 @@ pub async fn get_element_inner_html(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -676,8 +780,16 @@ pub async fn get_element_input_value(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -691,7 +803,7 @@ pub async fn get_element_input_value(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -708,8 +820,16 @@ pub async fn set_element_value(
ref_map: &RefMap,
selector_or_ref: &str,
value: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let js = format!(
"function() {{ this.value = {}; this.dispatchEvent(new Event('input', {{bubbles: true}})); this.dispatchEvent(new Event('change', {{bubbles: true}})); }}",
@@ -726,7 +846,7 @@ pub async fn set_element_value(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -738,8 +858,16 @@ pub async fn get_element_bounding_box(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<Value, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -755,7 +883,7 @@ pub async fn get_element_bounding_box(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -793,8 +921,16 @@ pub async fn get_element_styles(
ref_map: &RefMap,
selector_or_ref: &str,
properties: Option<Vec<String>>,
iframe_sessions: &HashMap<String, String>,
) -> Result<Value, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let js = match properties {
Some(props) => {
@@ -832,7 +968,7 @@ pub async fn get_element_styles(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -932,4 +1068,48 @@ mod tests {
assert!((x - 60.0).abs() < 0.01);
assert!((y - 40.0).abs() < 0.01);
}
// -----------------------------------------------------------------------
// resolve_frame_session tests (Issue #925)
// Cross-origin iframe elements must resolve to the dedicated session.
// -----------------------------------------------------------------------
#[test]
fn test_cross_origin_element_uses_dedicated_session() {
let mut iframe_sessions = HashMap::new();
iframe_sessions.insert(
"cross-origin-frame".to_string(),
"iframe-session".to_string(),
);
let session = resolve_frame_session(
Some("cross-origin-frame"),
"parent-session",
&iframe_sessions,
);
assert_eq!(session, "iframe-session");
}
#[test]
fn test_same_origin_element_uses_parent_session() {
let iframe_sessions = HashMap::new();
let session = resolve_frame_session(
Some("same-origin-frame"),
"parent-session",
&iframe_sessions,
);
assert_eq!(session, "parent-session");
}
#[test]
fn test_main_frame_element_uses_parent_session() {
let iframe_sessions = HashMap::new();
let session = resolve_frame_session(None, "parent-session", &iframe_sessions);
assert_eq!(session, "parent-session");
}
}
+222 -43
View File
@@ -1,3 +1,5 @@
use std::collections::HashMap;
use serde_json::Value;
use super::cdp::client::CdpClient;
@@ -11,9 +13,17 @@ pub async fn click(
selector_or_ref: &str,
button: &str,
click_count: i32,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (x, y) = resolve_element_center(client, session_id, ref_map, selector_or_ref).await?;
dispatch_click(client, session_id, x, y, button, click_count).await
let (x, y, effective_session_id) = resolve_element_center(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
dispatch_click(client, &effective_session_id, x, y, button, click_count).await
}
pub async fn dblclick(
@@ -21,8 +31,18 @@ pub async fn dblclick(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
click(client, session_id, ref_map, selector_or_ref, "left", 2).await
click(
client,
session_id,
ref_map,
selector_or_ref,
"left",
2,
iframe_sessions,
)
.await
}
pub async fn hover(
@@ -30,8 +50,16 @@ pub async fn hover(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (x, y) = resolve_element_center(client, session_id, ref_map, selector_or_ref).await?;
let (x, y, effective_session_id) = resolve_element_center(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
"Input.dispatchMouseEvent",
@@ -46,7 +74,7 @@ pub async fn hover(
delta_y: None,
modifiers: None,
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
Ok(())
@@ -58,8 +86,16 @@ pub async fn fill(
ref_map: &RefMap,
selector_or_ref: &str,
value: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// Focus the element
client
@@ -72,7 +108,7 @@ pub async fn fill(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -92,11 +128,11 @@ pub async fn fill(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
// Insert text
// Insert text (keyboard input dispatched at page level, use parent session_id)
client
.send_command_typed::<_, Value>(
"Input.insertText",
@@ -119,8 +155,16 @@ pub async fn type_text(
text: &str,
clear: bool,
delay_ms: Option<u64>,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// Focus
client
@@ -133,7 +177,7 @@ pub async fn type_text(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -153,7 +197,7 @@ pub async fn type_text(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
}
@@ -283,9 +327,11 @@ pub async fn scroll(
selector_or_ref: Option<&str>,
delta_x: f64,
delta_y: f64,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
if let Some(sel) = selector_or_ref {
let object_id = resolve_element_object_id(client, session_id, ref_map, sel).await?;
let (object_id, effective_session_id) =
resolve_element_object_id(client, session_id, ref_map, sel, iframe_sessions).await?;
let js = "function(dx, dy) { this.scrollBy(dx, dy); }".to_string();
client
.send_command_typed::<_, Value>(
@@ -306,7 +352,7 @@ pub async fn scroll(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
} else {
@@ -332,8 +378,16 @@ pub async fn select_option(
ref_map: &RefMap,
selector_or_ref: &str,
values: &[String],
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let js = r#"function(vals) {
const options = Array.from(this.options);
@@ -357,7 +411,7 @@ pub async fn select_option(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -369,18 +423,48 @@ pub async fn check(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let is_checked =
super::element::is_element_checked(client, session_id, ref_map, selector_or_ref).await?;
let is_checked = super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
if !is_checked {
click(client, session_id, ref_map, selector_or_ref, "left", 1).await?;
click(
client,
session_id,
ref_map,
selector_or_ref,
"left",
1,
iframe_sessions,
)
.await?;
// Verify the click changed the state (Playwright parity: _setChecked re-checks).
// If the coordinate-based click missed (e.g. hidden input, overlay), retry
// with a JS .click() on the element and its associated input.
if !super::element::is_element_checked(client, session_id, ref_map, selector_or_ref).await?
if !super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?
{
js_click_checkbox(client, session_id, ref_map, selector_or_ref).await?;
js_click_checkbox(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
}
}
Ok(())
@@ -391,15 +475,46 @@ pub async fn uncheck(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let is_checked =
super::element::is_element_checked(client, session_id, ref_map, selector_or_ref).await?;
let is_checked = super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
if is_checked {
click(client, session_id, ref_map, selector_or_ref, "left", 1).await?;
click(
client,
session_id,
ref_map,
selector_or_ref,
"left",
1,
iframe_sessions,
)
.await?;
// Same verify-and-retry as check().
if super::element::is_element_checked(client, session_id, ref_map, selector_or_ref).await? {
js_click_checkbox(client, session_id, ref_map, selector_or_ref).await?;
if super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?
{
js_click_checkbox(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
}
}
Ok(())
@@ -419,8 +534,16 @@ async fn js_click_checkbox(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let js = r#"function() {
var el = this;
@@ -456,7 +579,7 @@ async fn js_click_checkbox(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -468,8 +591,16 @@ pub async fn focus(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
@@ -481,7 +612,7 @@ pub async fn focus(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -493,8 +624,16 @@ pub async fn clear(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
@@ -512,7 +651,7 @@ pub async fn clear(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -524,8 +663,16 @@ pub async fn select_all(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
@@ -549,7 +696,7 @@ pub async fn select_all(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -561,8 +708,16 @@ pub async fn scroll_into_view(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
@@ -576,7 +731,7 @@ pub async fn scroll_into_view(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -590,8 +745,16 @@ pub async fn dispatch_event(
selector_or_ref: &str,
event_type: &str,
event_init: Option<&Value>,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let init_json = event_init
.map(|v| serde_json::to_string(v).unwrap_or("{}".to_string()))
@@ -613,7 +776,7 @@ pub async fn dispatch_event(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -625,8 +788,16 @@ pub async fn highlight(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
@@ -647,7 +818,7 @@ pub async fn highlight(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -659,8 +830,16 @@ pub async fn tap_touch(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (x, y) = resolve_element_center(client, session_id, ref_map, selector_or_ref).await?;
let (x, y, effective_session_id) = resolve_element_center(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command(
@@ -669,7 +848,7 @@ pub async fn tap_touch(
"type": "touchStart",
"touchPoints": [{ "x": x, "y": y }],
})),
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -680,7 +859,7 @@ pub async fn tap_touch(
"type": "touchEnd",
"touchPoints": [],
})),
Some(session_id),
Some(&effective_session_id),
)
.await?;
+23 -6
View File
@@ -2,6 +2,8 @@ use serde::Serialize;
use serde_json::Value;
use std::path::PathBuf;
use std::collections::HashMap;
use super::cdp::client::CdpClient;
use super::cdp::types::*;
use super::element::RefMap;
@@ -100,10 +102,14 @@ pub async fn take_screenshot(
session_id: &str,
ref_map: &RefMap,
options: &ScreenshotOptions,
iframe_sessions: &HashMap<String, String>,
) -> Result<ScreenshotResult, String> {
let target_rect = if options.annotate {
match options.selector.as_deref() {
Some(selector) => get_rect_for_selector(client, session_id, ref_map, selector).await?,
Some(selector) => {
get_rect_for_selector(client, session_id, ref_map, selector, iframe_sessions)
.await?
}
None => None,
}
} else {
@@ -124,7 +130,8 @@ pub async fn take_screenshot(
false
};
let base64 = capture_screenshot_base64(client, session_id, ref_map, options).await;
let base64 =
capture_screenshot_base64(client, session_id, ref_map, options, iframe_sessions).await;
if overlay_injected {
let _ = remove_annotation_overlay(client, session_id).await;
@@ -166,6 +173,7 @@ async fn capture_screenshot_base64(
session_id: &str,
ref_map: &RefMap,
options: &ScreenshotOptions,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let mut params = CaptureScreenshotParams {
format: Some(options.format.clone()),
@@ -200,7 +208,9 @@ async fn capture_screenshot_base64(
});
}
} else if let Some(ref selector) = options.selector {
if let Some(rect) = get_rect_for_selector(client, session_id, ref_map, selector).await? {
if let Some(rect) =
get_rect_for_selector(client, session_id, ref_map, selector, iframe_sessions).await?
{
params.clip = Some(Viewport {
x: rect.x,
y: rect.y,
@@ -316,10 +326,17 @@ async fn get_rect_for_selector(
session_id: &str,
ref_map: &RefMap,
selector: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<Option<Rect>, String> {
let object_id =
super::element::resolve_element_object_id(client, session_id, ref_map, selector).await?;
get_rect_for_object(client, session_id, &object_id).await
let (object_id, effective_session_id) = super::element::resolve_element_object_id(
client,
session_id,
ref_map,
selector,
iframe_sessions,
)
.await?;
get_rect_for_object(client, &effective_session_id, &object_id).await
}
async fn get_rect_for_object(
+66 -7
View File
@@ -6,7 +6,7 @@ use super::cdp::client::CdpClient;
use super::cdp::types::{
AXNode, AXProperty, AXValue, EvaluateParams, EvaluateResult, GetFullAXTreeResult,
};
use super::element::RefMap;
use super::element::{resolve_ax_session, RefMap};
const INTERACTIVE_ROLES: &[&str] = &[
"button",
@@ -139,6 +139,7 @@ pub async fn take_snapshot(
options: &SnapshotOptions,
ref_map: &mut RefMap,
frame_id: Option<&str>,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
client
.send_command_no_params("DOM.enable", Some(session_id))
@@ -202,13 +203,24 @@ pub async fn take_snapshot(
None
};
let ax_params = if let Some(fid) = frame_id {
serde_json::json!({ "frameId": fid })
} else {
serde_json::json!({})
};
let (ax_params, effective_session_id) =
resolve_ax_session(frame_id, session_id, iframe_sessions);
// Ensure domains are enabled on the iframe session (defensive fallback
// in case the attach-time enable in execute_command was missed).
if effective_session_id != session_id {
let _ = client
.send_command_no_params("DOM.enable", Some(effective_session_id))
.await;
let _ = client
.send_command_no_params("Accessibility.enable", Some(effective_session_id))
.await;
}
let ax_tree: GetFullAXTreeResult = client
.send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id))
.send_command_typed(
"Accessibility.getFullAXTree",
&ax_params,
Some(effective_session_id),
)
.await?;
let (tree_nodes, root_indices) = build_tree(&ax_tree.nodes);
@@ -350,6 +362,7 @@ pub async fn take_snapshot(
options,
ref_map,
Some(&child_fid),
iframe_sessions,
))
.await
{
@@ -1161,4 +1174,50 @@ mod tests {
assert_eq!(set.len(), 1);
assert!(set.contains("ok"));
}
// -----------------------------------------------------------------------
// resolve_ax_session tests (Issue #925 regression guard)
// Cross-origin iframes must use a dedicated session without frameId.
// Same-origin iframes must use the parent session with frameId.
// -----------------------------------------------------------------------
#[test]
fn test_cross_origin_iframe_uses_dedicated_session() {
let parent_session = "parent-session";
let iframe_frame_id = "cross-origin-iframe-frame";
let iframe_session = "cross-origin-iframe-session";
let mut iframe_sessions = HashMap::new();
iframe_sessions.insert(iframe_frame_id.to_string(), iframe_session.to_string());
let (params, session) =
resolve_ax_session(Some(iframe_frame_id), parent_session, &iframe_sessions);
assert_eq!(session, iframe_session);
assert_eq!(params, serde_json::json!({}));
}
#[test]
fn test_same_origin_iframe_uses_parent_session_with_frame_id() {
let parent_session = "parent-session";
let iframe_frame_id = "same-origin-iframe-frame";
let iframe_sessions = HashMap::new();
let (params, session) =
resolve_ax_session(Some(iframe_frame_id), parent_session, &iframe_sessions);
assert_eq!(session, parent_session);
assert_eq!(params, serde_json::json!({ "frameId": iframe_frame_id }));
}
#[test]
fn test_main_frame_uses_parent_session() {
let parent_session = "parent-session";
let iframe_sessions = HashMap::new();
let (params, session) = resolve_ax_session(None, parent_session, &iframe_sessions);
assert_eq!(session, parent_session);
assert_eq!(params, serde_json::json!({}));
}
}