oimwiodev/chrome-use
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: support cross-origin iframe snapshots and interactions via Target.setAutoAttach (#949)

Browse Source 
* feat: support cross-origin iframe snapshots and interactions via Target.setAutoAttach (#925)

Enable Target.setAutoAttach with flatten: true on page sessions so Chrome
auto-creates dedicated CDP sessions for cross-origin iframe targets.

- Add DrainedEvents struct, iframe_sessions map, attach/detach event handling
- Extract resolve_ax_session (shared) and resolve_frame_session helpers
- resolve_element_object_id returns (object_id, effective_session) tuple
- resolve_element_center returns (x, y, effective_session) tuple
- Input dispatch (click/hover/tap) uses effective session for correct coordinates
- Thread iframe_sessions through element.rs, interaction.rs, screenshot.rs
- Clear iframe sessions on navigate, tab switch, tab new, tab close
- Ignore Target.setAutoAttach failure for non-Chrome backends (Lightpanda)
- Unit tests for session resolution logic

* fix

* fix

* fix

---------

Co-authored-by: hyunjinee <leehj0110@kakao.com>
This commit is contained in:
jin.2
2026-03-20 16:37:33 -05:00
committed by GitHub
co-authored by hyunjinee
parent 39e54113e6
commit 59b6e5a034
6 changed files with 919 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/native/element.rs
+227 -47
View File
@@ -147,12 +147,16 @@ pub async fn resolve_element_center(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
) -> Result<(f64, f64), String> {
iframe_sessions: &HashMap<String, String>,
) -> Result<(f64, f64, String), String> {
if let Some(ref_id) = parse_ref(selector_or_ref) {
let entry = ref_map
.get(&ref_id)
.ok_or_else(|| format!("Unknown ref: {}", ref_id))?;
let effective_session_id =
resolve_frame_session(entry.frame_id.as_deref(), session_id, iframe_sessions);
// Try cached backend_node_id first (fast path)
if let Some(backend_node_id) = entry.backend_node_id {
let result: Result<DomGetBoxModelResult, String> = client
@@ -163,25 +167,26 @@ pub async fn resolve_element_center(
node_id: None,
object_id: None,
},
Some(session_id),
Some(effective_session_id),
)
.await;
if let Ok(r) = result {
return Ok(box_model_center(&r.model));
let (x, y) = box_model_center(&r.model);
return Ok((x, y, effective_session_id.to_string()));
}
// backend_node_id is stale; re-query the accessibility tree below
}
// Fallback: re-query the accessibility tree to find a fresh node by role/name
let ref_frame_id = entry.frame_id.clone();
let fresh_id = find_node_id_by_role_name(
client,
session_id,
&entry.role,
&entry.name,
entry.nth,
ref_frame_id.as_deref(),
entry.frame_id.as_deref(),
iframe_sessions,
)
.await?;
let result: DomGetBoxModelResult = client
@@ -192,14 +197,16 @@ pub async fn resolve_element_center(
node_id: None,
object_id: None,
},
Some(session_id),
Some(effective_session_id),
)
.await?;
return Ok(box_model_center(&result.model));
let (x, y) = box_model_center(&result.model);
return Ok((x, y, effective_session_id.to_string()));
}
// CSS selector
resolve_by_selector(client, session_id, selector_or_ref).await
let (x, y) = resolve_by_selector(client, session_id, selector_or_ref).await?;
Ok((x, y, session_id.to_string()))
}
pub async fn resolve_element_object_id(
@@ -207,12 +214,16 @@ pub async fn resolve_element_object_id(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
) -> Result<String, String> {
iframe_sessions: &HashMap<String, String>,
) -> Result<(String, String), String> {
if let Some(ref_id) = parse_ref(selector_or_ref) {
let entry = ref_map
.get(&ref_id)
.ok_or_else(|| format!("Unknown ref: {}", ref_id))?;
let effective_session_id =
resolve_frame_session(entry.frame_id.as_deref(), session_id, iframe_sessions);
// Try cached backend_node_id first (fast path)
if let Some(backend_node_id) = entry.backend_node_id {
let result: Result<DomResolveNodeResult, String> = client
@@ -223,27 +234,27 @@ pub async fn resolve_element_object_id(
node_id: None,
object_group: Some("agent-browser".to_string()),
},
Some(session_id),
Some(effective_session_id),
)
.await;
if let Ok(r) = result {
if let Some(oid) = r.object.object_id {
return Ok(oid);
if let Some(object_id) = r.object.object_id {
return Ok((object_id, effective_session_id.to_string()));
}
}
// backend_node_id is stale; re-query the accessibility tree below
}
// Fallback: re-query the accessibility tree to find a fresh node by role/name
let ref_frame_id = entry.frame_id.clone();
let fresh_id = find_node_id_by_role_name(
client,
session_id,
&entry.role,
&entry.name,
entry.nth,
ref_frame_id.as_deref(),
entry.frame_id.as_deref(),
iframe_sessions,
)
.await?;
let result: DomResolveNodeResult = client
@@ -254,13 +265,14 @@ pub async fn resolve_element_object_id(
node_id: None,
object_group: Some("agent-browser".to_string()),
},
Some(session_id),
Some(effective_session_id),
)
.await?;
return result
let object_id = result
.object
.object_id
.ok_or_else(|| format!("No objectId for ref {}", ref_id));
.ok_or_else(|| format!("No objectId for ref {}", ref_id))?;
return Ok((object_id, effective_session_id.to_string()));
}
// Selector fallback (CSS or XPath)
@@ -277,10 +289,44 @@ pub async fn resolve_element_object_id(
)
.await?;
result
let object_id = result
.result
.object_id
.ok_or_else(|| format!("Element not found: {}", selector_or_ref))
.ok_or_else(|| format!("Element not found: {}", selector_or_ref))?;
Ok((object_id, session_id.to_string()))
}
/// Determine which CDP session and parameters to use for an AX tree query.
/// Cross-origin iframes have a dedicated session (no frameId needed);
/// same-origin iframes use the parent session with a frameId parameter.
pub(super) fn resolve_ax_session<'a>(
frame_id: Option<&str>,
session_id: &'a str,
iframe_sessions: &'a HashMap<String, String>,
) -> (serde_json::Value, &'a str) {
if let Some(frame_id) = frame_id {
if let Some(iframe_sid) = iframe_sessions.get(frame_id) {
(serde_json::json!({}), iframe_sid.as_str())
} else {
(serde_json::json!({ "frameId": frame_id }), session_id)
}
} else {
(serde_json::json!({}), session_id)
}
}
/// Resolve the effective CDP session for an element's frame.
/// If the element's frame_id has a dedicated cross-origin iframe session, return it.
/// Otherwise, return the parent session.
fn resolve_frame_session<'a>(
frame_id: Option<&str>,
session_id: &'a str,
iframe_sessions: &'a HashMap<String, String>,
) -> &'a str {
frame_id
.and_then(|fid| iframe_sessions.get(fid))
.map(|s| s.as_str())
.unwrap_or(session_id)
}
/// Re-query the accessibility tree to find a node matching role+name+nth,
@@ -294,14 +340,16 @@ async fn find_node_id_by_role_name(
name: &str,
nth: Option<usize>,
frame_id: Option<&str>,
iframe_sessions: &HashMap<String, String>,
) -> Result<i64, String> {
let ax_params = if let Some(fid) = frame_id {
serde_json::json!({ "frameId": fid })
} else {
serde_json::json!({})
};
let (ax_params, effective_session_id) =
resolve_ax_session(frame_id, session_id, iframe_sessions);
let ax_tree: GetFullAXTreeResult = client
.send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id))
.send_command_typed(
"Accessibility.getFullAXTree",
&ax_params,
Some(effective_session_id),
)
.await?;
let nth_index = nth.unwrap_or(0);
@@ -431,8 +479,16 @@ pub async fn get_element_text(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -445,7 +501,7 @@ pub async fn get_element_text(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -462,8 +518,16 @@ pub async fn get_element_attribute(
ref_map: &RefMap,
selector_or_ref: &str,
attribute: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<Value, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -478,7 +542,7 @@ pub async fn get_element_attribute(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -490,8 +554,16 @@ pub async fn is_element_visible(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<bool, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -511,7 +583,7 @@ pub async fn is_element_visible(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -527,8 +599,16 @@ pub async fn is_element_enabled(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<bool, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -540,7 +620,7 @@ pub async fn is_element_enabled(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -556,8 +636,16 @@ pub async fn is_element_checked(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<bool, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// Mirrors Playwright's getChecked() with follow-label retargeting:
// 1. If element is a native checkbox/radio input, return .checked
@@ -602,7 +690,7 @@ pub async fn is_element_checked(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -618,8 +706,16 @@ pub async fn get_element_inner_text(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -631,7 +727,7 @@ pub async fn get_element_inner_text(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -647,8 +743,16 @@ pub async fn get_element_inner_html(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -660,7 +764,7 @@ pub async fn get_element_inner_html(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -676,8 +780,16 @@ pub async fn get_element_input_value(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<String, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -691,7 +803,7 @@ pub async fn get_element_input_value(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -708,8 +820,16 @@ pub async fn set_element_value(
ref_map: &RefMap,
selector_or_ref: &str,
value: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let js = format!(
"function() {{ this.value = {}; this.dispatchEvent(new Event('input', {{bubbles: true}})); this.dispatchEvent(new Event('change', {{bubbles: true}})); }}",
@@ -726,7 +846,7 @@ pub async fn set_element_value(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -738,8 +858,16 @@ pub async fn get_element_bounding_box(
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<Value, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let result: EvaluateResult = client
.send_command_typed(
@@ -755,7 +883,7 @@ pub async fn get_element_bounding_box(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -793,8 +921,16 @@ pub async fn get_element_styles(
ref_map: &RefMap,
selector_or_ref: &str,
properties: Option<Vec<String>>,
iframe_sessions: &HashMap<String, String>,
) -> Result<Value, String> {
let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let js = match properties {
Some(props) => {
@@ -832,7 +968,7 @@ pub async fn get_element_styles(
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -932,4 +1068,48 @@ mod tests {
assert!((x - 60.0).abs() < 0.01);
assert!((y - 40.0).abs() < 0.01);
}
// -----------------------------------------------------------------------
// resolve_frame_session tests (Issue #925)
// Cross-origin iframe elements must resolve to the dedicated session.
// -----------------------------------------------------------------------
#[test]
fn test_cross_origin_element_uses_dedicated_session() {
let mut iframe_sessions = HashMap::new();
iframe_sessions.insert(
"cross-origin-frame".to_string(),
"iframe-session".to_string(),
);
let session = resolve_frame_session(
Some("cross-origin-frame"),
"parent-session",
&iframe_sessions,
);
assert_eq!(session, "iframe-session");
}
#[test]
fn test_same_origin_element_uses_parent_session() {
let iframe_sessions = HashMap::new();
let session = resolve_frame_session(
Some("same-origin-frame"),
"parent-session",
&iframe_sessions,
);
assert_eq!(session, "parent-session");
}
#[test]
fn test_main_frame_element_uses_parent_session() {
let iframe_sessions = HashMap::new();
let session = resolve_frame_session(None, "parent-session", &iframe_sessions);
assert_eq!(session, "parent-session");
}
}