use std::collections::HashMap; use serde_json::Value; use super::cdp::client::CdpClient; use super::cdp::types::{ AXNode, AXProperty, AXValue, EvaluateParams, EvaluateResult, GetFullAXTreeResult, }; use super::adaptive::ElementFingerprint; use super::element::{resolve_ax_session, RefMap}; const INTERACTIVE_ROLES: &[&str] = &[ "button", "link", "textbox", "checkbox", "radio", "combobox", "listbox", "menuitem", "menuitemcheckbox", "menuitemradio", "option", "searchbox", "slider", "spinbutton", "switch", "tab", "treeitem", "Iframe", ]; const CONTENT_ROLES: &[&str] = &[ "heading", "cell", "gridcell", "columnheader", "rowheader", "listitem", "article", "region", "main", "navigation", ]; const STRUCTURAL_ROLES: &[&str] = &[ "generic", "group", "list", "table", "row", "rowgroup", "grid", "treegrid", "menu", "menubar", "toolbar", "tablist", "tree", "directory", "document", "application", "presentation", "none", "WebArea", "RootWebArea", ]; const INVISIBLE_CHARS: &[char] = &[ '\u{FEFF}', // BOM / Zero Width No-Break Space '\u{200B}', // Zero Width Space '\u{200C}', // Zero Width Non-Joiner '\u{200D}', // Zero Width Joiner '\u{2060}', // Word Joiner '\u{00A0}', // Non-Breaking Space ( ) ]; #[derive(Default)] pub struct SnapshotOptions { pub selector: Option, pub interactive: bool, pub compact: bool, pub depth: Option, pub urls: bool, } struct TreeNode { role: String, name: String, level: Option, checked: Option, expanded: Option, selected: Option, disabled: Option, required: Option, value_text: Option, backend_node_id: Option, children: Vec, parent_idx: Option, has_ref: bool, ref_id: Option, depth: usize, cursor_info: Option, url: Option, } impl TreeNode { // Create an empty node fn empty() -> Self { Self { role: String::new(), name: String::new(), level: None, checked: None, expanded: None, selected: None, disabled: None, required: None, value_text: None, backend_node_id: None, children: Vec::new(), parent_idx: None, has_ref: false, ref_id: None, depth: 0, cursor_info: None, url: None, } } fn clear(&mut self) { self.role = String::new(); self.name = String::new(); self.level = None; self.checked = None; self.expanded = None; self.selected = None; self.disabled = None; self.required = None; self.value_text = None; self.backend_node_id = None; self.children.clear(); self.parent_idx = None; self.has_ref = false; self.url = None; self.ref_id = None; self.depth = 0; self.cursor_info = None; } } /// Build an AX fingerprint for a tree node, used by adaptive @ref relocation. /// Pulls only data already in the AX tree (no extra CDP calls): role as `tag`, /// accessible name as `text`, a few discriminating AX properties as `attrs`, and /// the ancestor/parent/sibling structure from the tree links. fn build_ax_fingerprint(tree_nodes: &[TreeNode], idx: usize) -> ElementFingerprint { let node = &tree_nodes[idx]; let mut attrs = std::collections::BTreeMap::new(); if let Some(v) = &node.value_text { if !v.is_empty() { attrs.insert("value".to_string(), v.clone()); } } if let Some(u) = &node.url { if !u.is_empty() { attrs.insert("url".to_string(), u.clone()); } } if let Some(l) = node.level { attrs.insert("level".to_string(), l.to_string()); } if let Some(c) = &node.checked { attrs.insert("checked".to_string(), c.clone()); } // Ancestor roles, nearest first, capped to keep the signature stable. let mut ancestors = Vec::new(); let mut cur = node.parent_idx; while let Some(pidx) = cur { if ancestors.len() >= 6 { break; } let role = tree_nodes[pidx].role.clone(); if !role.is_empty() { ancestors.push(role); } cur = tree_nodes[pidx].parent_idx; } let (parent_tag, parent_text) = node .parent_idx .map(|pidx| (tree_nodes[pidx].role.clone(), tree_nodes[pidx].name.clone())) .unwrap_or_default(); // Position among same-role siblings under the same parent. let (sibling_index, sibling_count) = match node.parent_idx { Some(pidx) => { let mut count = 0u32; let mut index = 0u32; for &child in &tree_nodes[pidx].children { if tree_nodes[child].role == node.role { if child == idx { index = count; } count += 1; } } (index, count) } None => (0, 0), }; ElementFingerprint { tag: node.role.clone(), text: node.name.clone(), attrs, ancestors, parent_tag, parent_text, sibling_index, sibling_count, } } /// Collect AX fingerprints for every node that has a backend node id, used as the /// candidate set when relocating a stale @ref. Reuses the same extraction as the /// baseline so the two are scored in the same space. fn collect_fingerprints(tree_nodes: &[TreeNode]) -> Vec<(i64, ElementFingerprint)> { tree_nodes .iter() .enumerate() .filter_map(|(idx, n)| { n.backend_node_id .map(|bid| (bid, build_ax_fingerprint(tree_nodes, idx))) }) .collect() } /// Fetch a fresh AX tree for the given frame and return `(backend_node_id, /// fingerprint)` for every node — the candidate set for adaptive @ref /// relocation. One `getFullAXTree` call, no per-element work. pub(super) async fn collect_current_fingerprints( client: &CdpClient, session_id: &str, frame_id: Option<&str>, iframe_sessions: &HashMap, ) -> Result, String> { let (ax_params, effective_session_id) = resolve_ax_session(frame_id, session_id, iframe_sessions); let _ = client .send_command_no_params("DOM.enable", Some(effective_session_id)) .await; let _ = client .send_command_no_params("Accessibility.enable", Some(effective_session_id)) .await; let ax_tree: GetFullAXTreeResult = client .send_command_typed( "Accessibility.getFullAXTree", &ax_params, Some(effective_session_id), ) .await?; let (tree_nodes, _roots) = build_tree(&ax_tree.nodes); Ok(collect_fingerprints(&tree_nodes)) } /// The type of a hidden form input found inside a cursor-interactive element. #[derive(Clone, Copy)] enum HiddenInputKind { Radio, Checkbox, } impl HiddenInputKind { fn parse(s: &str) -> Option { match s { "radio" => Some(Self::Radio), "checkbox" => Some(Self::Checkbox), _ => None, } } fn as_role(&self) -> &str { match self { Self::Radio => "radio", Self::Checkbox => "checkbox", } } } /// Information about a cursor-interactive element (elements with cursor:pointer, onclick, tabindex, etc.) #[derive(Clone)] struct CursorElementInfo { kind: String, // "clickable", "focusable", "editable" hints: Vec, text: String, // textContent from the DOM element (fallback when ARIA name is empty) hidden_input_kind: Option, hidden_input_checked: Option, // "true", "false", or "mixed" (tristate) } struct RoleNameTracker { counts: HashMap, entries: Vec<(usize, String)>, } impl RoleNameTracker { fn new() -> Self { Self { counts: HashMap::new(), entries: Vec::new(), } } fn track(&mut self, role: &str, name: &str, node_idx: usize) -> usize { let key = format!("{}:{}", role, name); let count = self.counts.entry(key.clone()).or_insert(0); let nth = *count; *count += 1; self.entries.push((node_idx, key)); nth } fn get_duplicates(&self) -> HashMap { self.counts .iter() .filter(|(_, &count)| count > 1) .map(|(key, &count)| (key.clone(), count)) .collect() } } pub async fn take_snapshot( client: &CdpClient, session_id: &str, options: &SnapshotOptions, ref_map: &mut RefMap, frame_id: Option<&str>, iframe_sessions: &HashMap, ) -> Result { client .send_command_no_params("DOM.enable", Some(session_id)) .await?; client .send_command_no_params("Accessibility.enable", Some(session_id)) .await?; // If a CSS selector is provided, resolve the set of backendNodeIds that // belong to the DOM subtree rooted at the matched element. We use this // set to pick the right AX subtree root(s) later. let selector_backend_ids: Option> = if let Some(ref selector) = options.selector { let js = format!( "document.querySelector({})", serde_json::to_string(selector).unwrap_or_default() ); let result: EvaluateResult = client .send_command_typed( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(false), await_promise: Some(false), }, Some(session_id), ) .await?; let object_id = result .result .object_id .ok_or_else(|| format!("Selector '{}' did not match any element", selector))?; // Request the full DOM subtree (depth: -1) so we can collect all // backendNodeIds that live under the matched element. let describe: Value = client .send_command( "DOM.describeNode", Some(serde_json::json!({ "objectId": object_id, "depth": -1 })), Some(session_id), ) .await?; let root_node = describe .get("node") .ok_or_else(|| format!("Could not resolve DOM node for selector '{}'", selector))?; let mut ids = std::collections::HashSet::new(); collect_backend_node_ids(root_node, &mut ids); if ids.is_empty() { return Err(format!( "Could not resolve backendNodeId for selector '{}'", selector )); } Some(ids) } else { None }; let (ax_params, effective_session_id) = resolve_ax_session(frame_id, session_id, iframe_sessions); // Ensure domains are enabled on the iframe session (defensive fallback // in case the attach-time enable in execute_command was missed). if effective_session_id != session_id { let _ = client .send_command_no_params("DOM.enable", Some(effective_session_id)) .await; let _ = client .send_command_no_params("Accessibility.enable", Some(effective_session_id)) .await; } let ax_tree: GetFullAXTreeResult = client .send_command_typed( "Accessibility.getFullAXTree", &ax_params, Some(effective_session_id), ) .await?; let (mut tree_nodes, root_indices) = build_tree(&ax_tree.nodes); // When a selector is given, find AX nodes whose backendDOMNodeId falls // within the target DOM subtree and pick the top-level ones as roots. let effective_roots = if let Some(ref id_set) = selector_backend_ids { // Mark which tree_nodes belong to the target DOM subtree. let in_subtree: Vec = tree_nodes .iter() .map(|n| n.backend_node_id.is_some_and(|bid| id_set.contains(&bid))) .collect(); // An AX node is a "top-level" match if it is in the subtree but its // parent (in the AX tree) is not. let mut roots = Vec::new(); for (idx, node) in tree_nodes.iter().enumerate() { if !in_subtree[idx] { continue; } let parent_in_subtree = node.parent_idx.is_some_and(|pidx| in_subtree[pidx]); if !parent_in_subtree { roots.push(idx); } } if roots.is_empty() { return Err(format!( "No accessibility node found for selector '{}'", options.selector.as_deref().unwrap_or("") )); } roots } else { root_indices }; let mut tracker = RoleNameTracker::new(); let mut next_ref: usize = ref_map.next_ref_num(); let mut nodes_with_refs: Vec<(usize, usize)> = Vec::new(); // Pre-collect cursor-interactive elements so we can mark them with refs during tree building let cursor_elements: HashMap = find_cursor_interactive_elements(client, session_id) .await .unwrap_or_default(); promote_hidden_inputs(&mut tree_nodes, &cursor_elements); for (idx, node) in tree_nodes.iter().enumerate() { let role = node.role.as_str(); let mut should_ref = if INTERACTIVE_ROLES.contains(&role) { true } else if CONTENT_ROLES.contains(&role) { !node.name.is_empty() } else { false }; if node .backend_node_id .is_some_and(|bid| cursor_elements.contains_key(&bid)) { // ref elements that are cursor-interactive should_ref = true; } if should_ref { let nth = tracker.track(role, &node.name, idx); nodes_with_refs.push((idx, nth)); } } let duplicates = tracker.get_duplicates(); for (idx, nth) in &nodes_with_refs { let node = &tree_nodes[*idx]; let key = format!("{}:{}", node.role, node.name); let actual_nth = if duplicates.contains_key(&key) { Some(*nth) } else { None }; let ref_id = format!("e{}", next_ref); next_ref += 1; ref_map.add_with_frame( ref_id.clone(), tree_nodes[*idx].backend_node_id, &tree_nodes[*idx].role, &tree_nodes[*idx].name, actual_nth, frame_id, ); ref_map.set_fingerprint(&ref_id, build_ax_fingerprint(&tree_nodes, *idx)); tree_nodes[*idx].has_ref = true; tree_nodes[*idx].ref_id = Some(ref_id); } // Populate cursor_info for ref-bearing nodes for (idx, _) in &nodes_with_refs { if let Some(bid) = tree_nodes[*idx].backend_node_id { if let Some(cursor_info) = cursor_elements.get(&bid) { tree_nodes[*idx].cursor_info = Some((*cursor_info).clone()); } } } ref_map.set_next_ref_num(next_ref); if options.urls { let link_nodes: Vec<(usize, i64)> = tree_nodes .iter() .enumerate() .filter(|(_, n)| n.role == "link" && n.has_ref && n.backend_node_id.is_some()) .filter_map(|(i, n)| n.backend_node_id.map(|bid| (i, bid))) .collect(); if !link_nodes.is_empty() { // CDP has no batch resolve API, so we parallelize individual calls. // Phase 1: resolve all backend node IDs to JS object IDs in parallel. let resolve_futs = link_nodes.iter().map(|&(idx, bid)| async move { let resolved = client .send_command( "DOM.resolveNode", Some(serde_json::json!({ "backendNodeId": bid })), Some(session_id), ) .await; let obj_id = resolved.ok().and_then(|r| { r.get("object") .and_then(|o| o.get("objectId")) .and_then(|v| v.as_str()) .map(|s| s.to_string()) }); (idx, obj_id) }); let resolved: Vec<(usize, Option)> = futures_util::future::join_all(resolve_futs).await; // Phase 2: fetch hrefs for all resolved objects in parallel. let href_futs: Vec<_> = resolved .iter() .filter_map(|(idx, obj_id)| { let oid = obj_id.as_ref()?; Some(async move { let result = client .send_command( "Runtime.callFunctionOn", Some(serde_json::json!({ "objectId": oid, "functionDeclaration": "function() { return this.href || ''; }", "returnByValue": true, })), Some(session_id), ) .await; let href = result.ok().and_then(|r| { r.get("result") .and_then(|r| r.get("value")) .and_then(|v| v.as_str()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) }); (*idx, href) }) }) .collect(); let hrefs: Vec<(usize, Option)> = futures_util::future::join_all(href_futs).await; for (idx, href) in hrefs { if let Some(url) = href { tree_nodes[idx].url = Some(url); } } } } let mut output = String::new(); for &root_idx in &effective_roots { render_tree(&tree_nodes, root_idx, 0, &mut output, options); } // Recurse into child iframes: for each Iframe node with a backend_node_id, // resolve the child frame ID and take a snapshot of its content. // We only recurse from the main frame (frame_id == None) to avoid // unbounded depth; nested iframes within iframes are not expanded. if frame_id.is_none() { let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot) for node in tree_nodes.iter() { if node.role != "Iframe" || !node.has_ref { continue; } let Some(bid) = node.backend_node_id else { continue; }; let ref_id = node.ref_id.as_deref().unwrap_or(""); if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await { // Snapshot the child frame; errors are silently ignored // (e.g. cross-origin iframes) if let Ok(child_text) = Box::pin(take_snapshot( client, session_id, options, ref_map, Some(&child_fid), iframe_sessions, )) .await { if !child_text.is_empty() && child_text != "(empty page)" && child_text != "(no interactive elements)" { iframe_snapshots.push((ref_id.to_string(), child_text)); } } } } // Insert each child snapshot after its Iframe line in the output for (ref_id, child_text) in iframe_snapshots { let marker = format!("[ref={}]", ref_id); if let Some(pos) = output.find(&marker) { // Find the end of the Iframe line let line_end = output[pos..] .find('\n') .map(|i| pos + i) .unwrap_or(output.len()); // Determine the indent of the Iframe line let line_start = output[..pos].rfind('\n').map(|i| i + 1).unwrap_or(0); let iframe_line = &output[line_start..line_end]; let iframe_indent = iframe_line.len() - iframe_line.trim_start().len(); let child_indent = iframe_indent + 2; // one level deeper let prefix = " ".repeat(child_indent); let indented_child: String = child_text .lines() .map(|line| format!("{}{}\n", prefix, line)) .collect(); // Ensure there's a newline to insert after if line_end == output.len() { output.push('\n'); output.push_str(&indented_child); } else { output.insert_str(line_end + 1, &indented_child); } } } } if options.compact { output = compact_tree(&output, options.interactive); } let trimmed = output.trim().to_string(); if trimmed.is_empty() { if options.interactive { return Ok("(no interactive elements)".to_string()); } return Ok("(empty page)".to_string()); } Ok(trimmed) } /// Resolve the child frame ID for an iframe element given its backendNodeId. async fn resolve_iframe_frame_id( client: &CdpClient, session_id: &str, backend_node_id: i64, ) -> Result { // depth: 1 ensures contentDocument is included in the response let describe: Value = client .send_command( "DOM.describeNode", Some(serde_json::json!({ "backendNodeId": backend_node_id, "depth": 1 })), Some(session_id), ) .await?; // Try contentDocument.frameId first (standard for iframes) if let Some(frame_id) = describe .get("node") .and_then(|n| n.get("contentDocument")) .and_then(|cd| cd.get("frameId")) .and_then(|v| v.as_str()) { return Ok(frame_id.to_string()); } // Fallback: the node itself may have a frameId describe .get("node") .and_then(|n| n.get("frameId")) .and_then(|v| v.as_str()) .map(|s| s.to_string()) .ok_or_else(|| "Could not resolve iframe frame ID".to_string()) } async fn find_cursor_interactive_elements( client: &CdpClient, session_id: &str, ) -> Result, String> { // Single JS evaluation that matches the v0.19.0 Node.js findCursorInteractiveElements(): // - Uses querySelectorAll('*') to walk all elements // - Checks getComputedStyle(el).cursor === 'pointer' // - Checks onclick attribute/handler and tabindex // - Skips interactiveTags (a, button, input, select, textarea, details, summary) // - Skips elements with interactive ARIA roles // - Deduplicates inherited cursor:pointer from parent // - Skips empty text and zero-size elements // - Tags each matched element with data-__ab-ci for batch backendNodeId resolution let js = r#" (function() { var results = []; if (!document.body) return results; var interactiveRoles = { 'button':1, 'link':1, 'textbox':1, 'checkbox':1, 'radio':1, 'combobox':1, 'listbox':1, 'menuitem':1, 'menuitemcheckbox':1, 'menuitemradio':1, 'option':1, 'searchbox':1, 'slider':1, 'spinbutton':1, 'switch':1, 'tab':1, 'treeitem':1 }; var interactiveTags = { 'a':1, 'button':1, 'input':1, 'select':1, 'textarea':1, 'details':1, 'summary':1 }; var allElements = document.body.querySelectorAll('*'); for (var i = 0; i < allElements.length; i++) { var el = allElements[i]; if (el.closest && el.closest('[hidden], [aria-hidden="true"]')) continue; var tagName = el.tagName.toLowerCase(); if (interactiveTags[tagName]) continue; var role = el.getAttribute('role'); if (role && interactiveRoles[role.toLowerCase()]) continue; var computedStyle = getComputedStyle(el); var hasCursorPointer = computedStyle.cursor === 'pointer'; var hasOnClick = el.hasAttribute('onclick') || el.onclick !== null; var tabIndex = el.getAttribute('tabindex'); var hasTabIndex = tabIndex !== null && tabIndex !== '-1'; var ce = el.getAttribute('contenteditable'); var isEditable = ce === '' || ce === 'true'; if (!hasCursorPointer && !hasOnClick && !hasTabIndex && !isEditable) continue; // Skip elements that only inherit cursor:pointer from an ancestor if (hasCursorPointer && !hasOnClick && !hasTabIndex && !isEditable) { var parent = el.parentElement; if (parent && getComputedStyle(parent).cursor === 'pointer') continue; } var text = (el.textContent || '').trim().slice(0, 100); var rect = el.getBoundingClientRect(); if (rect.width === 0 || rect.height === 0) continue; // Detect hidden radio/checkbox inputs inside this element (common pattern: //