Add iframe support for CLI interactions and snapshots (#869)

* Add iframe support for CLI interactions and snapshots

This PR adds comprehensive iframe support to the agent browser CLI, allowing users to interact with elements inside iframes seamlessly.

## Problem
Users couldn't interact with elements inside iframes via the command line. The existing `frame` command was non-functional as it set `active_frame_id` but no other code read this value.

## Changes Made

### Enhanced Frame Context Tracking
- Added `frame_id` field to `RefEntry` to track which frame each element reference belongs to
- Updated `RefMap::add` and related methods to accept and store frame context
- Modified element resolution functions to use frame context from ref entries

### Improved Frame Command
- Fixed the existing `frame` command to actually work by threading `active_frame_id` through snapshot operations
- Added support for iframe element references (e.g., `frame @e2`) in addition to CSS selectors
- Enhanced frame detection to work with both named frames and iframe elements

### Updated Snapshot Behavior
- Modified `take_snapshot` to accept optional frame context parameter
- Updated all snapshot call sites to pass appropriate frame context
- Maintained backward compatibility while enabling frame-scoped operations

### Element Resolution Updates
- Updated `resolve_element_center` and `resolve_element_object_id` to use frame context from ref entries
- Modified `find_node_id_by_role_name` to support frame-specific element lookup
- Ensured all interaction functions work correctly within iframe contexts

## Implementation Details
- Frame context is now properly propagated through the entire element interaction pipeline
- The `frame` command can accept both CSS selectors and element references
- All existing functionality remains intact while adding iframe capabilities
- Added `Iframe` to interactive roles for better element discovery

Fixes #863

* docs: add iframe support documentation

Document the new iframe capabilities across all documentation surfaces:
- Auto-inlining of iframe content in snapshots
- Direct interaction with iframe element refs
- frame command support for element refs (@e3)
- Scoped snapshots via frame switching

* fix: pass active frame context to diff snapshots and fix nameless iframe lookup

- handle_diff_snapshot now respects active_frame_id instead of always
  passing None, so diff snapshots work correctly inside iframes
- Nameless/id-less iframes now fall back to src URL (or null) instead of
  the literal string 'frame' which never matched any frame in the tree

* fix: resolve iframe frame ID via DOM.describeNode and reduce code duplication

- handle_frame: Use DOM.describeNode + contentDocument.frameId to resolve
  iframe frame IDs directly, fixing failures for nameless iframes that
  lack name/id/src attributes
- element.rs: Deduplicate add() by delegating to add_with_frame()
- snapshot.rs: Guard against out-of-bounds insert_str when iframe marker
  is on the last line without a trailing newline

---------

Co-authored-by: ctate <366502+ctate@users.noreply.github.com>

cli/src/native/actions.rs

@@ -1500,8 +1500,14 @@ async fn handle_snapshot(cmd: &Value, state: &mut DaemonState) -> Result<Value,
     };
 
     state.ref_map.clear();
-    let tree =
-        snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map).await?;
+    let tree = snapshot::take_snapshot(
+        &mgr.client,
+        &session_id,
+        &options,
+        &mut state.ref_map,
+        state.active_frame_id.as_deref(),
+    )
+    .await?;
 
     let url = mgr.get_url().await.unwrap_or_default();
 
@@ -1605,6 +1611,7 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
                 ..SnapshotOptions::default()
             },
             &mut state.ref_map,
+            state.active_frame_id.as_deref(),
         )
         .await?;
     }
@@ -2395,8 +2402,14 @@ async fn handle_diff_snapshot(cmd: &Value, state: &mut DaemonState) -> Result<Va
         selector,
         ..SnapshotOptions::default()
     };
-    let current =
-        snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map).await?;
+    let current = snapshot::take_snapshot(
+        &mgr.client,
+        &session_id,
+        &options,
+        &mut state.ref_map,
+        state.active_frame_id.as_deref(),
+    )
+    .await?;
 
     let baseline = cmd.get("baseline").and_then(|v| v.as_str());
 
@@ -2441,13 +2454,15 @@ async fn handle_diff_url(cmd: &Value, state: &mut DaemonState) -> Result<Value,
     let session_id = mgr.active_session_id()?.to_string();
     let options = SnapshotOptions::default();
     let snap1 =
-        snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map).await?;
+        snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map, None)
+            .await?;
 
     // Navigate to URL2 and snapshot
     mgr.navigate(url2, wait_until).await?;
     state.ref_map.clear();
     let snap2 =
-        snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map).await?;
+        snapshot::take_snapshot(&mgr.client, &session_id, &options, &mut state.ref_map, None)
+            .await?;
 
     let result = diff::diff_text(&snap1, &snap2);
     Ok(json!({
@@ -3558,14 +3573,79 @@ async fn handle_frame(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
 
     let frame_tree = &tree_result["frameTree"];
 
-    // If selector, resolve via JS to find the iframe's contentWindow
+    // If selector is a ref (@e1), resolve the iframe element from the ref map
     if let Some(sel) = selector {
+        if let Some(ref_id) = super::element::parse_ref(sel) {
+            let entry = state
+                .ref_map
+                .get(&ref_id)
+                .ok_or_else(|| format!("Unknown ref: {}", ref_id))?;
+            let backend_node_id = entry
+                .backend_node_id
+                .ok_or_else(|| format!("Ref {} has no backend node id", ref_id))?;
+
+            // Use DOM.describeNode to resolve the child frame ID directly.
+            // This works reliably for all iframes, including those without
+            // name, id, or src attributes.
+            let describe: Value = mgr
+                .client
+                .send_command(
+                    "DOM.describeNode",
+                    Some(json!({ "backendNodeId": backend_node_id, "depth": 1 })),
+                    Some(&session_id),
+                )
+                .await?;
+
+            // Verify this is an iframe/frame element
+            let node_name = describe
+                .get("node")
+                .and_then(|n| n.get("nodeName"))
+                .and_then(|v| v.as_str())
+                .unwrap_or("");
+            if node_name != "IFRAME" && node_name != "FRAME" {
+                return Err("Ref does not point to an iframe element".to_string());
+            }
+
+            // Try contentDocument.frameId first (standard for iframes)
+            let frame_id = describe
+                .get("node")
+                .and_then(|n| n.get("contentDocument"))
+                .and_then(|cd| cd.get("frameId"))
+                .and_then(|v| v.as_str())
+                // Fallback: the node itself may carry a frameId
+                .or_else(|| {
+                    describe
+                        .get("node")
+                        .and_then(|n| n.get("frameId"))
+                        .and_then(|v| v.as_str())
+                })
+                .ok_or("Could not resolve frame ID for iframe element")?;
+
+            let label = describe
+                .get("node")
+                .and_then(|n| n.get("attributes"))
+                .and_then(|a| a.as_array())
+                .and_then(|attrs| {
+                    attrs
+                        .iter()
+                        .enumerate()
+                        .find(|(_, v)| v.as_str() == Some("name"))
+                        .and_then(|(i, _)| attrs.get(i + 1))
+                        .and_then(|v| v.as_str())
+                })
+                .unwrap_or(&ref_id);
+
+            state.active_frame_id = Some(frame_id.to_string());
+            return Ok(json!({ "frame": label }));
+        }
+
+        // CSS selector path
         let js = format!(
             r#"(() => {{
                 const el = document.querySelector({});
                 if (!el) return null;
                 if (el.tagName === 'IFRAME' || el.tagName === 'FRAME') {{
-                    return el.name || el.id || 'frame';
+                    return el.name || el.id || el.src || null;
                 }}
                 return null;
             }})()"#,

cli/src/native/element.rs

@@ -12,6 +12,7 @@ pub struct RefEntry {
     pub name: String,
     pub nth: Option<usize>,
     pub selector: Option<String>,
+    pub frame_id: Option<String>,
 }
 
 pub struct RefMap {
@@ -34,6 +35,18 @@ impl RefMap {
         role: &str,
         name: &str,
         nth: Option<usize>,
+    ) {
+        self.add_with_frame(ref_id, backend_node_id, role, name, nth, None);
+    }
+
+    pub fn add_with_frame(
+        &mut self,
+        ref_id: String,
+        backend_node_id: Option<i64>,
+        role: &str,
+        name: &str,
+        nth: Option<usize>,
+        frame_id: Option<&str>,
     ) {
         self.map.insert(
             ref_id,
@@ -43,6 +56,7 @@ impl RefMap {
                 name: name.to_string(),
                 nth,
                 selector: None,
+                frame_id: frame_id.map(|s| s.to_string()),
             },
         );
     }
@@ -63,6 +77,7 @@ impl RefMap {
                 name: name.to_string(),
                 nth,
                 selector: Some(selector),
+                frame_id: None,
             },
         );
     }
@@ -159,9 +174,16 @@ pub async fn resolve_element_center(
         }
 
         // Fallback: re-query the accessibility tree to find a fresh node by role/name
-        let fresh_id =
-            find_node_id_by_role_name(client, session_id, &entry.role, &entry.name, entry.nth)
-                .await?;
+        let ref_frame_id = entry.frame_id.clone();
+        let fresh_id = find_node_id_by_role_name(
+            client,
+            session_id,
+            &entry.role,
+            &entry.name,
+            entry.nth,
+            ref_frame_id.as_deref(),
+        )
+        .await?;
         let result: DomGetBoxModelResult = client
             .send_command_typed(
                 "DOM.getBoxModel",
@@ -214,9 +236,16 @@ pub async fn resolve_element_object_id(
         }
 
         // Fallback: re-query the accessibility tree to find a fresh node by role/name
-        let fresh_id =
-            find_node_id_by_role_name(client, session_id, &entry.role, &entry.name, entry.nth)
-                .await?;
+        let ref_frame_id = entry.frame_id.clone();
+        let fresh_id = find_node_id_by_role_name(
+            client,
+            session_id,
+            &entry.role,
+            &entry.name,
+            entry.nth,
+            ref_frame_id.as_deref(),
+        )
+        .await?;
         let result: DomResolveNodeResult = client
             .send_command_typed(
                 "DOM.resolveNode",
@@ -267,13 +296,15 @@ async fn find_node_id_by_role_name(
     role: &str,
     name: &str,
     nth: Option<usize>,
+    frame_id: Option<&str>,
 ) -> Result<i64, String> {
+    let ax_params = if let Some(fid) = frame_id {
+        serde_json::json!({ "frameId": fid })
+    } else {
+        serde_json::json!({})
+    };
     let ax_tree: GetFullAXTreeResult = client
-        .send_command_typed(
-            "Accessibility.getFullAXTree",
-            &serde_json::json!({}),
-            Some(session_id),
-        )
+        .send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id))
         .await?;
 
     let nth_index = nth.unwrap_or(0);

cli/src/native/interaction.rs

@@ -110,6 +110,7 @@ pub async fn fill(
     Ok(())
 }
 
+#[allow(clippy::too_many_arguments)]
 pub async fn type_text(
     client: &CdpClient,
     session_id: &str,

cli/src/native/snapshot.rs

@@ -26,6 +26,7 @@ const INTERACTIVE_ROLES: &[&str] = &[
     "switch",
     "tab",
     "treeitem",
+    "Iframe",
 ];
 
 const CONTENT_ROLES: &[&str] = &[
@@ -137,6 +138,7 @@ pub async fn take_snapshot(
     session_id: &str,
     options: &SnapshotOptions,
     ref_map: &mut RefMap,
+    frame_id: Option<&str>,
 ) -> Result<String, String> {
     client
         .send_command_no_params("DOM.enable", Some(session_id))
@@ -200,12 +202,13 @@ pub async fn take_snapshot(
             None
         };
 
+    let ax_params = if let Some(fid) = frame_id {
+        serde_json::json!({ "frameId": fid })
+    } else {
+        serde_json::json!({})
+    };
     let ax_tree: GetFullAXTreeResult = client
-        .send_command_typed(
-            "Accessibility.getFullAXTree",
-            &serde_json::json!({}),
-            Some(session_id),
-        )
+        .send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id))
         .await?;
 
     let (tree_nodes, root_indices) = build_tree(&ax_tree.nodes);
@@ -293,12 +296,13 @@ pub async fn take_snapshot(
         let ref_id = format!("e{}", next_ref);
         next_ref += 1;
 
-        ref_map.add(
+        ref_map.add_with_frame(
             ref_id.clone(),
             tree_nodes[*idx].backend_node_id,
             &tree_nodes[*idx].role,
             &tree_nodes[*idx].name,
             actual_nth,
+            frame_id,
         );
 
         tree_nodes[*idx].has_ref = true;
@@ -323,6 +327,74 @@ pub async fn take_snapshot(
         render_tree(&tree_nodes, root_idx, 0, &mut output, options);
     }
 
+    // Recurse into child iframes: for each Iframe node with a backend_node_id,
+    // resolve the child frame ID and take a snapshot of its content.
+    // We only recurse from the main frame (frame_id == None) to avoid
+    // unbounded depth; nested iframes within iframes are not expanded.
+    if frame_id.is_none() {
+        let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot)
+        for node in tree_nodes.iter() {
+            if node.role != "Iframe" || !node.has_ref {
+                continue;
+            }
+            let Some(bid) = node.backend_node_id else {
+                continue;
+            };
+            let ref_id = node.ref_id.as_deref().unwrap_or("");
+            if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await {
+                // Snapshot the child frame; errors are silently ignored
+                // (e.g. cross-origin iframes)
+                if let Ok(child_text) = Box::pin(take_snapshot(
+                    client,
+                    session_id,
+                    options,
+                    ref_map,
+                    Some(&child_fid),
+                ))
+                .await
+                {
+                    if !child_text.is_empty()
+                        && child_text != "(empty page)"
+                        && child_text != "(no interactive elements)"
+                    {
+                        iframe_snapshots.push((ref_id.to_string(), child_text));
+                    }
+                }
+            }
+        }
+
+        // Insert each child snapshot after its Iframe line in the output
+        for (ref_id, child_text) in iframe_snapshots {
+            let marker = format!("[ref={}]", ref_id);
+            if let Some(pos) = output.find(&marker) {
+                // Find the end of the Iframe line
+                let line_end = output[pos..]
+                    .find('\n')
+                    .map(|i| pos + i)
+                    .unwrap_or(output.len());
+                // Determine the indent of the Iframe line
+                let line_start = output[..pos].rfind('\n').map(|i| i + 1).unwrap_or(0);
+                let iframe_line = &output[line_start..line_end];
+                let iframe_indent = iframe_line.len() - iframe_line.trim_start().len();
+                let child_indent = iframe_indent + 2; // one level deeper
+                let prefix = " ".repeat(child_indent);
+
+                let indented_child: String = child_text
+                    .lines()
+                    .map(|line| format!("{}{}\n", prefix, line))
+                    .collect();
+
+                // Ensure there's a newline to insert after
+                if line_end == output.len() {
+                    output.push('\n');
+                    output.push_str(&indented_child);
+                } else {
+                    output.insert_str(line_end + 1, &indented_child);
+                }
+            }
+        }
+    }
+
     if options.compact {
         output = compact_tree(&output, options.interactive);
     }
@@ -339,6 +411,40 @@ pub async fn take_snapshot(
     Ok(trimmed)
 }
 
+/// Resolve the child frame ID for an iframe element given its backendNodeId.
+async fn resolve_iframe_frame_id(
+    client: &CdpClient,
+    session_id: &str,
+    backend_node_id: i64,
+) -> Result<String, String> {
+    // depth: 1 ensures contentDocument is included in the response
+    let describe: Value = client
+        .send_command(
+            "DOM.describeNode",
+            Some(serde_json::json!({ "backendNodeId": backend_node_id, "depth": 1 })),
+            Some(session_id),
+        )
+        .await?;
+
+    // Try contentDocument.frameId first (standard for iframes)
+    if let Some(frame_id) = describe
+        .get("node")
+        .and_then(|n| n.get("contentDocument"))
+        .and_then(|cd| cd.get("frameId"))
+        .and_then(|v| v.as_str())
+    {
+        return Ok(frame_id.to_string());
+    }
+
+    // Fallback: the node itself may have a frameId
+    describe
+        .get("node")
+        .and_then(|n| n.get("frameId"))
+        .and_then(|v| v.as_str())
+        .map(|s| s.to_string())
+        .ok_or_else(|| "Could not resolve iframe frame ID".to_string())
+}
+
 async fn find_cursor_interactive_elements(
     client: &CdpClient,
     session_id: &str,