diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 5210823..9081683 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -1500,8 +1500,14 @@ async fn handle_snapshot(cmd: &Value, state: &mut DaemonState) -> Result Result Result Result Result {{ const el = document.querySelector({}); if (!el) return null; if (el.tagName === 'IFRAME' || el.tagName === 'FRAME') {{ - return el.name || el.id || 'frame'; + return el.name || el.id || el.src || null; }} return null; }})()"#, diff --git a/cli/src/native/element.rs b/cli/src/native/element.rs index b40d714..5f17ff9 100644 --- a/cli/src/native/element.rs +++ b/cli/src/native/element.rs @@ -12,6 +12,7 @@ pub struct RefEntry { pub name: String, pub nth: Option, pub selector: Option, + pub frame_id: Option, } pub struct RefMap { @@ -34,6 +35,18 @@ impl RefMap { role: &str, name: &str, nth: Option, + ) { + self.add_with_frame(ref_id, backend_node_id, role, name, nth, None); + } + + pub fn add_with_frame( + &mut self, + ref_id: String, + backend_node_id: Option, + role: &str, + name: &str, + nth: Option, + frame_id: Option<&str>, ) { self.map.insert( ref_id, @@ -43,6 +56,7 @@ impl RefMap { name: name.to_string(), nth, selector: None, + frame_id: frame_id.map(|s| s.to_string()), }, ); } @@ -63,6 +77,7 @@ impl RefMap { name: name.to_string(), nth, selector: Some(selector), + frame_id: None, }, ); } @@ -159,9 +174,16 @@ pub async fn resolve_element_center( } // Fallback: re-query the accessibility tree to find a fresh node by role/name - let fresh_id = - find_node_id_by_role_name(client, session_id, &entry.role, &entry.name, entry.nth) - .await?; + let ref_frame_id = entry.frame_id.clone(); + let fresh_id = find_node_id_by_role_name( + client, + session_id, + &entry.role, + &entry.name, + entry.nth, + ref_frame_id.as_deref(), + ) + .await?; let result: DomGetBoxModelResult = client .send_command_typed( "DOM.getBoxModel", @@ -214,9 +236,16 @@ pub async fn resolve_element_object_id( } // Fallback: re-query the accessibility tree to find a fresh node by role/name - let fresh_id = - find_node_id_by_role_name(client, session_id, &entry.role, &entry.name, entry.nth) - .await?; + let ref_frame_id = entry.frame_id.clone(); + let fresh_id = find_node_id_by_role_name( + client, + session_id, + &entry.role, + &entry.name, + entry.nth, + ref_frame_id.as_deref(), + ) + .await?; let result: DomResolveNodeResult = client .send_command_typed( "DOM.resolveNode", @@ -267,13 +296,15 @@ async fn find_node_id_by_role_name( role: &str, name: &str, nth: Option, + frame_id: Option<&str>, ) -> Result { + let ax_params = if let Some(fid) = frame_id { + serde_json::json!({ "frameId": fid }) + } else { + serde_json::json!({}) + }; let ax_tree: GetFullAXTreeResult = client - .send_command_typed( - "Accessibility.getFullAXTree", - &serde_json::json!({}), - Some(session_id), - ) + .send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id)) .await?; let nth_index = nth.unwrap_or(0); diff --git a/cli/src/native/interaction.rs b/cli/src/native/interaction.rs index f058023..1d9dc78 100644 --- a/cli/src/native/interaction.rs +++ b/cli/src/native/interaction.rs @@ -110,6 +110,7 @@ pub async fn fill( Ok(()) } +#[allow(clippy::too_many_arguments)] pub async fn type_text( client: &CdpClient, session_id: &str, diff --git a/cli/src/native/snapshot.rs b/cli/src/native/snapshot.rs index 7a6196f..addcf6d 100644 --- a/cli/src/native/snapshot.rs +++ b/cli/src/native/snapshot.rs @@ -26,6 +26,7 @@ const INTERACTIVE_ROLES: &[&str] = &[ "switch", "tab", "treeitem", + "Iframe", ]; const CONTENT_ROLES: &[&str] = &[ @@ -137,6 +138,7 @@ pub async fn take_snapshot( session_id: &str, options: &SnapshotOptions, ref_map: &mut RefMap, + frame_id: Option<&str>, ) -> Result { client .send_command_no_params("DOM.enable", Some(session_id)) @@ -200,12 +202,13 @@ pub async fn take_snapshot( None }; + let ax_params = if let Some(fid) = frame_id { + serde_json::json!({ "frameId": fid }) + } else { + serde_json::json!({}) + }; let ax_tree: GetFullAXTreeResult = client - .send_command_typed( - "Accessibility.getFullAXTree", - &serde_json::json!({}), - Some(session_id), - ) + .send_command_typed("Accessibility.getFullAXTree", &ax_params, Some(session_id)) .await?; let (tree_nodes, root_indices) = build_tree(&ax_tree.nodes); @@ -293,12 +296,13 @@ pub async fn take_snapshot( let ref_id = format!("e{}", next_ref); next_ref += 1; - ref_map.add( + ref_map.add_with_frame( ref_id.clone(), tree_nodes[*idx].backend_node_id, &tree_nodes[*idx].role, &tree_nodes[*idx].name, actual_nth, + frame_id, ); tree_nodes[*idx].has_ref = true; @@ -323,6 +327,74 @@ pub async fn take_snapshot( render_tree(&tree_nodes, root_idx, 0, &mut output, options); } + // Recurse into child iframes: for each Iframe node with a backend_node_id, + // resolve the child frame ID and take a snapshot of its content. + // We only recurse from the main frame (frame_id == None) to avoid + // unbounded depth; nested iframes within iframes are not expanded. + if frame_id.is_none() { + let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot) + for node in tree_nodes.iter() { + if node.role != "Iframe" || !node.has_ref { + continue; + } + let Some(bid) = node.backend_node_id else { + continue; + }; + let ref_id = node.ref_id.as_deref().unwrap_or(""); + if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await { + // Snapshot the child frame; errors are silently ignored + // (e.g. cross-origin iframes) + if let Ok(child_text) = Box::pin(take_snapshot( + client, + session_id, + options, + ref_map, + Some(&child_fid), + )) + .await + { + if !child_text.is_empty() + && child_text != "(empty page)" + && child_text != "(no interactive elements)" + { + iframe_snapshots.push((ref_id.to_string(), child_text)); + } + } + } + } + + // Insert each child snapshot after its Iframe line in the output + for (ref_id, child_text) in iframe_snapshots { + let marker = format!("[ref={}]", ref_id); + if let Some(pos) = output.find(&marker) { + // Find the end of the Iframe line + let line_end = output[pos..] + .find('\n') + .map(|i| pos + i) + .unwrap_or(output.len()); + // Determine the indent of the Iframe line + let line_start = output[..pos].rfind('\n').map(|i| i + 1).unwrap_or(0); + let iframe_line = &output[line_start..line_end]; + let iframe_indent = iframe_line.len() - iframe_line.trim_start().len(); + let child_indent = iframe_indent + 2; // one level deeper + let prefix = " ".repeat(child_indent); + + let indented_child: String = child_text + .lines() + .map(|line| format!("{}{}\n", prefix, line)) + .collect(); + + // Ensure there's a newline to insert after + if line_end == output.len() { + output.push('\n'); + output.push_str(&indented_child); + } else { + output.insert_str(line_end + 1, &indented_child); + } + } + } + } + if options.compact { output = compact_tree(&output, options.interactive); } @@ -339,6 +411,40 @@ pub async fn take_snapshot( Ok(trimmed) } +/// Resolve the child frame ID for an iframe element given its backendNodeId. +async fn resolve_iframe_frame_id( + client: &CdpClient, + session_id: &str, + backend_node_id: i64, +) -> Result { + // depth: 1 ensures contentDocument is included in the response + let describe: Value = client + .send_command( + "DOM.describeNode", + Some(serde_json::json!({ "backendNodeId": backend_node_id, "depth": 1 })), + Some(session_id), + ) + .await?; + + // Try contentDocument.frameId first (standard for iframes) + if let Some(frame_id) = describe + .get("node") + .and_then(|n| n.get("contentDocument")) + .and_then(|cd| cd.get("frameId")) + .and_then(|v| v.as_str()) + { + return Ok(frame_id.to_string()); + } + + // Fallback: the node itself may have a frameId + describe + .get("node") + .and_then(|n| n.get("frameId")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .ok_or_else(|| "Could not resolve iframe frame ID".to_string()) +} + async fn find_cursor_interactive_elements( client: &CdpClient, session_id: &str, diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx index dd67f27..985aa98 100644 --- a/docs/src/app/commands/page.mdx +++ b/docs/src/app/commands/page.mdx @@ -189,10 +189,35 @@ agent-browser tab new [url] # New tab agent-browser tab # Switch to tab agent-browser tab close [n] # Close tab agent-browser window new # Open new browser window -agent-browser frame # Switch to iframe +agent-browser frame # Switch to iframe by CSS selector +agent-browser frame @e3 # Switch to iframe by element ref agent-browser frame main # Back to main frame ``` +### Iframe support + +Iframes are detected automatically during snapshots. `Iframe` nodes are resolved and their content is +inlined beneath the iframe element in the snapshot output. Refs assigned to elements inside iframes carry +frame context, so `click`, `fill`, and other interactions work without manually switching frames. + +```bash +agent-browser snapshot -i +# @e3 [Iframe] "payment-frame" +# @e4 [input] "Card number" +# @e5 [button] "Pay" + +# Interact directly using refs — no frame switch needed +agent-browser fill @e4 "4111111111111111" +agent-browser click @e5 + +# Or switch frame context for scoped snapshots +agent-browser frame @e3 +agent-browser snapshot -i # Only elements inside that iframe +agent-browser frame main # Return to main frame +``` + +The `frame` command accepts element refs (`@e3`), CSS selectors (`"#my-iframe"`), or frame name/URL. + ## Dialogs ```bash diff --git a/docs/src/app/snapshots/page.mdx b/docs/src/app/snapshots/page.mdx index 592b367..00d8d04 100644 --- a/docs/src/app/snapshots/page.mdx +++ b/docs/src/app/snapshots/page.mdx @@ -102,6 +102,31 @@ agent-browser click @e2 Annotated screenshots also cache refs, so you can interact with elements immediately. This is useful when the text snapshot is insufficient -- unlabeled icons, canvas content, or visual layout verification. +## Iframes + +Snapshots automatically detect and inline iframe content. Each `Iframe` node in the main frame is resolved and its child accessibility tree is included directly beneath it. Refs assigned to elements inside iframes carry frame context, so interactions work without switching frames first. + +```bash +agent-browser snapshot -i +# @e1 [heading] "Checkout" +# @e2 [Iframe] "payment-frame" +# @e3 [input] "Card number" +# @e4 [button] "Pay" + +agent-browser fill @e3 "4111111111111111" +agent-browser click @e4 +``` + +Only one level of iframe nesting is expanded. Cross-origin iframes that block accessibility tree access and empty iframes are silently omitted. + +To scope a snapshot to a single iframe, switch into it first: + +```bash +agent-browser frame @e2 +agent-browser snapshot -i # Only elements inside that iframe +agent-browser frame main # Return to main frame +``` + ## Best practices 1. Use `-i` to reduce output to actionable elements diff --git a/skills/agent-browser/SKILL.md b/skills/agent-browser/SKILL.md index 899053f..2b4b304 100644 --- a/skills/agent-browser/SKILL.md +++ b/skills/agent-browser/SKILL.md @@ -251,6 +251,30 @@ agent-browser state clear myapp agent-browser state clean --older-than 7 ``` +### Working with Iframes + +Iframe content is automatically inlined in snapshots. Refs inside iframes carry frame context, so you can interact with them directly. + +```bash +agent-browser open https://example.com/checkout +agent-browser snapshot -i +# @e1 [heading] "Checkout" +# @e2 [Iframe] "payment-frame" +# @e3 [input] "Card number" +# @e4 [input] "Expiry" +# @e5 [button] "Pay" + +# Interact directly — no frame switch needed +agent-browser fill @e3 "4111111111111111" +agent-browser fill @e4 "12/28" +agent-browser click @e5 + +# To scope a snapshot to one iframe: +agent-browser frame @e2 +agent-browser snapshot -i # Only iframe content +agent-browser frame main # Return to main frame +``` + ### Data Extraction ```bash diff --git a/skills/agent-browser/references/commands.md b/skills/agent-browser/references/commands.md index 383a748..46de5f1 100644 --- a/skills/agent-browser/references/commands.md +++ b/skills/agent-browser/references/commands.md @@ -177,10 +177,36 @@ agent-browser window new # New window ## Frames ```bash -agent-browser frame "#iframe" # Switch to iframe +agent-browser frame "#iframe" # Switch to iframe by CSS selector +agent-browser frame @e3 # Switch to iframe by element ref agent-browser frame main # Back to main frame ``` +### Iframe support + +Iframes are detected automatically during snapshots. When the main-frame snapshot runs, `Iframe` nodes are resolved and their content is inlined beneath the iframe element in the output (one level of nesting; iframes within iframes are not expanded). + +```bash +agent-browser snapshot -i +# @e3 [Iframe] "payment-frame" +# @e4 [input] "Card number" +# @e5 [button] "Pay" + +# Interact directly — refs inside iframes already work +agent-browser fill @e4 "4111111111111111" +agent-browser click @e5 + +# Or switch frame context for scoped snapshots +agent-browser frame @e3 # Switch using element ref +agent-browser snapshot -i # Snapshot scoped to that iframe +agent-browser frame main # Return to main frame +``` + +The `frame` command accepts: +- **Element refs** — `frame @e3` resolves the ref to an iframe element +- **CSS selectors** — `frame "#payment-iframe"` finds the iframe by selector +- **Frame name/URL** — matches against the browser's frame tree + ## Dialogs ```bash diff --git a/skills/agent-browser/references/snapshot-refs.md b/skills/agent-browser/references/snapshot-refs.md index c5868d5..3cc0fea 100644 --- a/skills/agent-browser/references/snapshot-refs.md +++ b/skills/agent-browser/references/snapshot-refs.md @@ -162,6 +162,31 @@ agent-browser snapshot @e9 @e10 [radio] selected # Selected radio ``` +## Iframes + +Snapshots automatically detect and inline iframe content. When the main-frame snapshot runs, each `Iframe` node is resolved and its child accessibility tree is included directly beneath it in the output. Refs assigned to elements inside iframes carry frame context, so interactions like `click`, `fill`, and `type` work without manually switching frames. + +```bash +agent-browser snapshot -i +# @e1 [heading] "Checkout" +# @e2 [Iframe] "payment-frame" +# @e3 [input] "Card number" +# @e4 [input] "Expiry" +# @e5 [button] "Pay" +# @e6 [button] "Cancel" + +# Interact with iframe elements directly using their refs +agent-browser fill @e3 "4111111111111111" +agent-browser fill @e4 "12/28" +agent-browser click @e5 +``` + +**Key details:** +- Only one level of iframe nesting is expanded (iframes within iframes are not recursed) +- Cross-origin iframes that block accessibility tree access are silently skipped +- Empty iframes or iframes with no interactive content are omitted from the output +- To scope a snapshot to a single iframe, use `frame @ref` then `snapshot -i` + ## Troubleshooting ### "Ref not found" Error