Add iframe support for CLI interactions and snapshots (#869)
* Add iframe support for CLI interactions and snapshots This PR adds comprehensive iframe support to the agent browser CLI, allowing users to interact with elements inside iframes seamlessly. ## Problem Users couldn't interact with elements inside iframes via the command line. The existing `frame` command was non-functional as it set `active_frame_id` but no other code read this value. ## Changes Made ### Enhanced Frame Context Tracking - Added `frame_id` field to `RefEntry` to track which frame each element reference belongs to - Updated `RefMap::add` and related methods to accept and store frame context - Modified element resolution functions to use frame context from ref entries ### Improved Frame Command - Fixed the existing `frame` command to actually work by threading `active_frame_id` through snapshot operations - Added support for iframe element references (e.g., `frame @e2`) in addition to CSS selectors - Enhanced frame detection to work with both named frames and iframe elements ### Updated Snapshot Behavior - Modified `take_snapshot` to accept optional frame context parameter - Updated all snapshot call sites to pass appropriate frame context - Maintained backward compatibility while enabling frame-scoped operations ### Element Resolution Updates - Updated `resolve_element_center` and `resolve_element_object_id` to use frame context from ref entries - Modified `find_node_id_by_role_name` to support frame-specific element lookup - Ensured all interaction functions work correctly within iframe contexts ## Implementation Details - Frame context is now properly propagated through the entire element interaction pipeline - The `frame` command can accept both CSS selectors and element references - All existing functionality remains intact while adding iframe capabilities - Added `Iframe` to interactive roles for better element discovery Fixes #863 * docs: add iframe support documentation Document the new iframe capabilities across all documentation surfaces: - Auto-inlining of iframe content in snapshots - Direct interaction with iframe element refs - frame command support for element refs (@e3) - Scoped snapshots via frame switching * fix: pass active frame context to diff snapshots and fix nameless iframe lookup - handle_diff_snapshot now respects active_frame_id instead of always passing None, so diff snapshots work correctly inside iframes - Nameless/id-less iframes now fall back to src URL (or null) instead of the literal string 'frame' which never matched any frame in the tree * fix: resolve iframe frame ID via DOM.describeNode and reduce code duplication - handle_frame: Use DOM.describeNode + contentDocument.frameId to resolve iframe frame IDs directly, fixing failures for nameless iframes that lack name/id/src attributes - element.rs: Deduplicate add() by delegating to add_with_frame() - snapshot.rs: Guard against out-of-bounds insert_str when iframe marker is on the last line without a trailing newline --------- Co-authored-by: ctate <366502+ctate@users.noreply.github.com>
This commit is contained in:
+42
-11
@@ -12,6 +12,7 @@ pub struct RefEntry {
|
||||
pub name: String,
|
||||
pub nth: Option<usize>,
|
||||
pub selector: Option<String>,
|
||||
pub frame_id: Option<String>,
|
||||
}
|
||||
|
||||
pub struct RefMap {
|
||||
@@ -34,6 +35,18 @@ impl RefMap {
|
||||
role: &str,
|
||||
name: &str,
|
||||
nth: Option<usize>,
|
||||
) {
|
||||
self.add_with_frame(ref_id, backend_node_id, role, name, nth, None);
|
||||
}
|
||||
|
||||
pub fn add_with_frame(
|
||||
&mut self,
|
||||
ref_id: String,
|
||||
backend_node_id: Option<i64>,
|
||||
role: &str,
|
||||
name: &str,
|
||||
nth: Option<usize>,
|
||||
frame_id: Option<&str>,
|
||||
) {
|
||||
self.map.insert(
|
||||
ref_id,
|
||||
@@ -43,6 +56,7 @@ impl RefMap {
|
||||
name: name.to_string(),
|
||||
nth,
|
||||
selector: None,
|
||||
frame_id: frame_id.map(|s| s.to_string()),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -63,6 +77,7 @@ impl RefMap {
|
||||
name: name.to_string(),
|
||||
nth,
|
||||
selector: Some(selector),
|
||||
frame_id: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -159,9 +174,16 @@ pub async fn resolve_element_center(
|
||||
}
|
||||
|
||||
// Fallback: re-query the accessibility tree to find a fresh node by role/name
|
||||
let fresh_id =
|
||||
find_node_id_by_role_name(client, session_id, &entry.role, &entry.name, entry.nth)
|
||||
.await?;
|
||||
let ref_frame_id = entry.frame_id.clone();
|
||||
let fresh_id = find_node_id_by_role_name(
|
||||
client,
|
||||
session_id,
|
||||
&entry.role,
|
||||
&entry.name,
|
||||
entry.nth,
|
||||
ref_frame_id.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
let result: DomGetBoxModelResult = client
|
||||
.send_command_typed(
|
||||
"DOM.getBoxModel",
|
||||
@@ -214,9 +236,16 @@ pub async fn resolve_element_object_id(
|
||||
}
|
||||
|
||||
// Fallback: re-query the accessibility tree to find a fresh node by role/name
|
||||
let fresh_id =
|
||||
find_node_id_by_role_name(client, session_id, &entry.role, &entry.name, entry.nth)
|
||||
.await?;
|
||||
let ref_frame_id = entry.frame_id.clone();
|
||||
let fresh_id = find_node_id_by_role_name(
|
||||
client,
|
||||
session_id,
|
||||
&entry.role,
|
||||
&entry.name,
|
||||
entry.nth,
|
||||
ref_frame_id.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
let result: DomResolveNodeResult = client
|
||||
.send_command_typed(
|
||||
"DOM.resolveNode",
|
||||
@@ -267,13 +296,15 @@ async fn find_node_id_by_role_name(
|
||||
role: &str,
|
||||
name: &str,
|
||||
nth: Option<usize>,
|
||||
frame_id: Option<&str>,
|
||||
) -> Result<i64, String> {
|
||||
let ax_params = if let Some(fid) = frame_id {
|
||||
serde_json::json!({ "frameId": fid })
|
||||
} else {
|
||||
serde_json::json!({})
|
||||
};
|
||||
let ax_tree: GetFullAXTreeResult = client
|
||||
.send_command_typed(
|
||||
"Accessibility.getFullAXTree",
|
||||
&serde_json::json!({}),
|
||||
Some(session_id),
|
||||
)
|
||||
.send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id))
|
||||
.await?;
|
||||
|
||||
let nth_index = nth.unwrap_or(0);
|
||||
|
||||
Reference in New Issue
Block a user