use std::collections::HashMap; use serde_json::Value; use super::adaptive::{self, ElementFingerprint}; use super::cdp::client::CdpClient; use super::cdp::types::*; #[derive(Debug, Clone)] pub struct RefEntry { pub backend_node_id: Option, pub role: String, pub name: String, pub nth: Option, pub selector: Option, pub frame_id: Option, /// AX fingerprint captured at snapshot time, used by adaptive relocation when /// the node is gone and the role/name/nth re-query also fails. pub fingerprint: Option, } pub struct RefMap { map: HashMap, next_ref: usize, } impl RefMap { pub fn new() -> Self { Self { map: HashMap::new(), next_ref: 1, } } pub fn add( &mut self, ref_id: String, backend_node_id: Option, role: &str, name: &str, nth: Option, ) { self.add_with_frame(ref_id, backend_node_id, role, name, nth, None); } pub fn add_with_frame( &mut self, ref_id: String, backend_node_id: Option, role: &str, name: &str, nth: Option, frame_id: Option<&str>, ) { self.map.insert( ref_id, RefEntry { backend_node_id, role: role.to_string(), name: name.to_string(), nth, selector: None, frame_id: frame_id.map(|s| s.to_string()), fingerprint: None, }, ); } /// Attach an AX fingerprint to an existing ref (set during snapshot, used by /// adaptive relocation). No-op if the ref is unknown. pub fn set_fingerprint(&mut self, ref_id: &str, fingerprint: ElementFingerprint) { if let Some(entry) = self.map.get_mut(ref_id) { entry.fingerprint = Some(fingerprint); } } pub fn add_selector( &mut self, ref_id: String, selector: String, role: &str, name: &str, nth: Option, ) { self.map.insert( ref_id, RefEntry { backend_node_id: None, role: role.to_string(), name: name.to_string(), nth, selector: Some(selector), frame_id: None, fingerprint: None, }, ); } pub fn get(&self, ref_id: &str) -> Option<&RefEntry> { self.map.get(ref_id) } pub fn entries_sorted(&self) -> Vec<(String, RefEntry)> { let mut entries = self .map .iter() .map(|(ref_id, entry)| (ref_id.clone(), entry.clone())) .collect::>(); entries.sort_by_key(|(ref_id, _)| { ref_id .strip_prefix('e') .and_then(|n| n.parse::().ok()) .unwrap_or(usize::MAX) }); entries } pub fn remove(&mut self, ref_id: &str) { self.map.remove(ref_id); } pub fn clear(&mut self) { self.map.clear(); self.next_ref = 1; } pub fn next_ref_num(&self) -> usize { self.next_ref } pub fn set_next_ref_num(&mut self, n: usize) { self.next_ref = n; } } pub fn parse_ref(input: &str) -> Option { let trimmed = input.trim(); if let Some(stripped) = trimmed.strip_prefix('@') { if stripped.starts_with('e') && stripped[1..].chars().all(|c| c.is_ascii_digit()) { return Some(stripped.to_string()); } } if let Some(stripped) = trimmed.strip_prefix("ref=") { if stripped.starts_with('e') && stripped[1..].chars().all(|c| c.is_ascii_digit()) { return Some(stripped.to_string()); } } if trimmed.starts_with('e') && trimmed.len() > 1 && trimmed[1..].chars().all(|c| c.is_ascii_digit()) { return Some(trimmed.to_string()); } None } /// When a saved `@ref`'s node is gone and the role/name/nth re-query also failed, /// try to relocate the element by AX fingerprint similarity. Returns the chosen /// backend node id only when confident (high score + clear margin over the /// runner-up). Opt out with `AGENT_BROWSER_ADAPTIVE_REF=0`. async fn relocate_stale_ref( client: &CdpClient, ref_id: &str, entry: &RefEntry, session_id: &str, iframe_sessions: &HashMap, ) -> Option { if std::env::var("AGENT_BROWSER_ADAPTIVE_REF").as_deref() == Ok("0") { return None; } let baseline = entry.fingerprint.as_ref()?; let candidates = super::snapshot::collect_current_fingerprints( client, session_id, entry.frame_id.as_deref(), iframe_sessions, ) .await .ok()?; match adaptive::pick_best( baseline, &candidates, adaptive::ADAPTIVE_THRESHOLD, adaptive::ADAPTIVE_MARGIN, ) { Ok(reloc) => { eprintln!( "[adaptive] relocated {ref_id} ({} \"{}\") score={:.2} second={:.2} -> backendNodeId {}", entry.role, entry.name, reloc.score, reloc.second_score, reloc.backend_node_id ); Some(reloc.backend_node_id) } Err(_) => None, } } /// Resolve a `@ref` or CSS selector to a click point. Returns /// `(centre_x, centre_y, width, height, session_id)`. Width/height come from the /// element's box model and feed humanize's in-bounds landing jitter; the CSS /// selector path returns zero size (→ land on centre, no jitter). pub async fn resolve_element_center( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(f64, f64, f64, f64, String), String> { if let Some(ref_id) = parse_ref(selector_or_ref) { let entry = ref_map .get(&ref_id) .ok_or_else(|| format!("Unknown ref: {}", ref_id))?; let effective_session_id = resolve_frame_session(entry.frame_id.as_deref(), session_id, iframe_sessions); // Try cached backend_node_id first (fast path) if let Some(backend_node_id) = entry.backend_node_id { let mut active_id = backend_node_id; // Identity check: React often re-uses the same DOM node when // re-rendering — backendNodeId stays the same but accessibleName // / role changes. Without this verification, `click @e20` (saved // when the button said "Add post") happily clicks the *same* // node that now says "Post all", silently submitting the thread. // // On mismatch, try adaptive fingerprint relocation before failing: // a confident high-score/high-margin match is a stronger identity // signal than role+name, and lets a moved+renamed element still // resolve. If relocation isn't confident, surface the original // identity error. Set AGENT_BROWSER_VERIFY_REF=0 to skip the check // (and thus relocation) entirely. if std::env::var("AGENT_BROWSER_VERIFY_REF").as_deref() != Ok("0") { if let Err(e) = verify_ref_identity( client, effective_session_id, backend_node_id, &ref_id, &entry.role, &entry.name, ) .await { match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) .await { Some(id) => active_id = id, None => return Err(e), } } } let result: Result = client .send_command_typed( "DOM.getBoxModel", &DomGetBoxModelParams { backend_node_id: Some(active_id), node_id: None, object_id: None, }, Some(effective_session_id), ) .await; if let Ok(r) = result { let (x, y, w, h) = box_model_dims(&r.model); // Occlusion check: a transient overlay (X.com's "click // outside to close" mask, modal backdrop, sticky banner, // etc.) can land on top of our target between snapshot // and click. Coordinates are correct, but // `document.elementFromPoint(x, y)` returns the overlay // — and the click goes to the overlay's handler, not // ours. Catch it here so the user gets "occluded by // DIV[testid=mask]" instead of "modal silently closed + // thread submitted by accident". // // Set AGENT_BROWSER_VERIFY_CLICK_TARGET=0 to skip. if std::env::var("AGENT_BROWSER_VERIFY_CLICK_TARGET").as_deref() != Ok("0") { verify_click_target(client, effective_session_id, active_id, &ref_id, x, y) .await?; } return Ok((x, y, w, h, effective_session_id.to_string())); } // backend_node_id is stale; re-query the accessibility tree below } // Fallback: re-query the accessibility tree to find a fresh node by role/name. // If that fails, try adaptive fingerprint relocation before giving up. let fresh_id = match find_node_id_by_role_name( client, session_id, &entry.role, &entry.name, entry.nth, entry.frame_id.as_deref(), iframe_sessions, ) .await { Ok(id) => id, Err(e) => match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) .await { Some(id) => id, None => return Err(e), }, }; let result: DomGetBoxModelResult = client .send_command_typed( "DOM.getBoxModel", &DomGetBoxModelParams { backend_node_id: Some(fresh_id), node_id: None, object_id: None, }, Some(effective_session_id), ) .await?; let (x, y, w, h) = box_model_dims(&result.model); return Ok((x, y, w, h, effective_session_id.to_string())); } // CSS selector let (x, y) = resolve_by_selector(client, session_id, selector_or_ref).await?; // No box model on the CSS-selector fast path → zero size → land on centre. Ok((x, y, 0.0, 0.0, session_id.to_string())) } pub async fn resolve_element_object_id( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(String, String), String> { if let Some(ref_id) = parse_ref(selector_or_ref) { let entry = ref_map .get(&ref_id) .ok_or_else(|| format!("Unknown ref: {}", ref_id))?; let effective_session_id = resolve_frame_session(entry.frame_id.as_deref(), session_id, iframe_sessions); // Try cached backend_node_id first (fast path) if let Some(backend_node_id) = entry.backend_node_id { let mut active_id = backend_node_id; // Same identity guard as resolve_element_center — see that // function for why React DOM-node-reuse breaks ref-based // interactions if we skip this, and why a confident adaptive // relocation is allowed to override an identity mismatch. if std::env::var("AGENT_BROWSER_VERIFY_REF").as_deref() != Ok("0") { if let Err(e) = verify_ref_identity( client, effective_session_id, backend_node_id, &ref_id, &entry.role, &entry.name, ) .await { match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) .await { Some(id) => active_id = id, None => return Err(e), } } } let result: Result = client .send_command_typed( "DOM.resolveNode", &DomResolveNodeParams { backend_node_id: Some(active_id), node_id: None, object_group: Some("chrome-use".to_string()), }, Some(effective_session_id), ) .await; if let Ok(r) = result { if let Some(object_id) = r.object.object_id { return Ok((object_id, effective_session_id.to_string())); } } // backend_node_id is stale; re-query the accessibility tree below } // Fallback: re-query the accessibility tree to find a fresh node by role/name. // If that fails, try adaptive fingerprint relocation before giving up. let fresh_id = match find_node_id_by_role_name( client, session_id, &entry.role, &entry.name, entry.nth, entry.frame_id.as_deref(), iframe_sessions, ) .await { Ok(id) => id, Err(e) => match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) .await { Some(id) => id, None => return Err(e), }, }; let result: DomResolveNodeResult = client .send_command_typed( "DOM.resolveNode", &DomResolveNodeParams { backend_node_id: Some(fresh_id), node_id: None, object_group: Some("chrome-use".to_string()), }, Some(effective_session_id), ) .await?; let object_id = result .object .object_id .ok_or_else(|| format!("No objectId for ref {}", ref_id))?; return Ok((object_id, effective_session_id.to_string())); } // Selector fallback (CSS or XPath) let js = build_find_element_js(selector_or_ref); let result: EvaluateResult = client .send_command_typed( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(false), await_promise: Some(false), }, Some(session_id), ) .await?; // A syntactically-invalid selector makes `document.querySelector` THROW. // With returnByValue:false, Runtime.evaluate then returns the thrown // DOMException as a remote object *with* an objectId — which would otherwise // be mistaken for "the element" and silently no-op a `.click()` on it. Treat // any thrown exception as a hard error so a typo'd selector fails loudly. if let Some(ex) = result.exception_details { return Err(format!( "Invalid selector '{}': {}", selector_or_ref, ex.text )); } let object_id = result .result .object_id .ok_or_else(|| format!("Element not found: {}", selector_or_ref))?; Ok((object_id, session_id.to_string())) } /// Determine which CDP session and parameters to use for an AX tree query. /// Cross-origin iframes have a dedicated session (no frameId needed); /// same-origin iframes use the parent session with a frameId parameter. pub(super) fn resolve_ax_session<'a>( frame_id: Option<&str>, session_id: &'a str, iframe_sessions: &'a HashMap, ) -> (serde_json::Value, &'a str) { if let Some(frame_id) = frame_id { if let Some(iframe_sid) = iframe_sessions.get(frame_id) { (serde_json::json!({}), iframe_sid.as_str()) } else { (serde_json::json!({ "frameId": frame_id }), session_id) } } else { (serde_json::json!({}), session_id) } } /// Resolve the effective CDP session for an element's frame. /// If the element's frame_id has a dedicated cross-origin iframe session, return it. /// Otherwise, return the parent session. fn resolve_frame_session<'a>( frame_id: Option<&str>, session_id: &'a str, iframe_sessions: &'a HashMap, ) -> &'a str { frame_id .and_then(|fid| iframe_sessions.get(fid)) .map(|s| s.as_str()) .unwrap_or(session_id) } /// Verify that the cached backendNodeId still has the same accessible role /// and name it had when the snapshot ran. Catches the case where React (or /// any reconciler) reused the DOM node for a different component instance /// — same physical node, different semantics. /// /// On mismatch, returns an actionable error naming both the snapshot label /// and the current label so the agent can re-snapshot intelligently. /// On any CDP failure (e.g. node deleted), returns Ok(()) so the caller's /// existing fallback (`find_node_id_by_role_name`) takes over. async fn verify_ref_identity( client: &CdpClient, session_id: &str, backend_node_id: i64, ref_id: &str, expected_role: &str, expected_name: &str, ) -> Result<(), String> { let params = serde_json::json!({ "backendNodeId": backend_node_id, "fetchRelatives": false, }); // Tight 1s timeout: this is a defensive guard, not a critical path. // The default 30s CDP timeout was the dominant factor in the // "click hangs 5+ minutes" report — three CDP calls (verify + // resolveNode + paint-settle) at 30s each, multiplied by parallel // click invocations queueing on the daemon, totalled multi-minute // user-visible hangs. Cap our own helper so a stuck AX query // doesn't make `click` worse than the no-guard version was. let resp: Result = match tokio::time::timeout( std::time::Duration::from_secs(1), client.send_command_typed("Accessibility.getPartialAXTree", ¶ms, Some(session_id)), ) .await { Ok(r) => r, // Timeout: skip identity verification rather than block the click. Err(_) => return Ok(()), }; let Ok(tree) = resp else { // Node likely gone; let the box-model call fail and trigger fallback. return Ok(()); }; // Find the AXNode for our backendNodeId. fetchRelatives=false still // returns ancestors; the target node has the matching backendNodeId. let Some(node) = tree .nodes .iter() .find(|n| n.backend_d_o_m_node_id == Some(backend_node_id)) else { return Ok(()); }; let actual_role = extract_ax_string(&node.role); let actual_name = extract_ax_string(&node.name); if actual_role == expected_role && actual_name == expected_name { return Ok(()); } Err(format!( "Ref {} no longer matches its snapshot. Was [{} \"{}\"], now [{} \"{}\"].\n\ The DOM mutated between snapshot and interaction (typical with React/Vue \ reusing nodes during re-render). Fix: take a fresh `snapshot` and re-target \ with the new ref. For SPAs where refs churn every interaction, drive the \ element directly with `eval` (e.g. `eval \"document.querySelector(...).click()\"`), \ which doesn't depend on refs.\n\ (Last resort: AGENT_BROWSER_VERIFY_REF=0 disables this safety check — only \ if you accept clicks may land on a re-rendered/wrong node.)", ref_id, expected_role, expected_name, actual_role, actual_name, )) } /// At the moment we'd dispatch the click, ask the page itself which element /// occupies (x, y). If it's not our target (and not a descendant or /// ancestor), an overlay has appeared between snapshot and click — we'd /// silently click the overlay otherwise. Returns Err with details about /// the occluding element so the caller can wait + re-snapshot. /// /// Implemented as a single Runtime.callFunctionOn: resolve the cached /// backendNodeId to a remote object, then run a function on it that /// compares with elementFromPoint. The function returns null when the /// click is safe and a JSON string with diagnostic info when it isn't. async fn verify_click_target( client: &CdpClient, session_id: &str, backend_node_id: i64, ref_id: &str, x: f64, y: f64, ) -> Result<(), String> { use serde::Deserialize; // Resolve once. backendNodeId is stable across renders; only the // element under (x, y) is what changes when an overlay flickers. let resolve_params = DomResolveNodeParams { backend_node_id: Some(backend_node_id), node_id: None, object_group: Some("chrome-use-occlusion".to_string()), }; let resolve_fut = client.send_command_typed::<_, serde_json::Value>( "DOM.resolveNode", &resolve_params, Some(session_id), ); let Ok(resolve_resp) = tokio::time::timeout(std::time::Duration::from_millis(500), resolve_fut).await else { return Ok(()); }; let Ok(resolved) = resolve_resp else { return Ok(()); }; let Some(object_id) = resolved .get("object") .and_then(|o| o.get("objectId")) .and_then(|v| v.as_str()) else { return Ok(()); }; // Auto-retry on transient occlusion. Many real-world overlays // (modal backdrops, focus rings, click-outside masks) blink in for // a frame or two during state transitions and clear on their own. // Without retries the user gets an "occluded" error and has to // wrap every click in their own retry loop. With retries the // common case is invisible — only persistent overlays surface. // // AGENT_BROWSER_OCCLUSION_RETRIES (default 3, 0 disables) // AGENT_BROWSER_OCCLUSION_RETRY_DELAY_MS (default 200) let max_retries: u32 = std::env::var("AGENT_BROWSER_OCCLUSION_RETRIES") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(3); let retry_delay_ms: u64 = std::env::var("AGENT_BROWSER_OCCLUSION_RETRY_DELAY_MS") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(200); #[derive(Deserialize)] struct Occluder { tag: Option, testid: Option, role: Option, #[serde(rename = "ariaLabel")] aria_label: Option, text: Option, reason: Option, } // function(x, y) { ... } where `this` is the target element. // Return null → click is safe. // Return JSON → describes the occluding element. let function_decl = "function(x, y) { \ const at = document.elementFromPoint(x, y); \ if (!at) return JSON.stringify({reason:'no-element-at-point'}); \ if (at === this || this.contains(at) || at.contains(this)) return null; \ return JSON.stringify({ \ tag: at.tagName, \ testid: (at.dataset && at.dataset.testid) || null, \ role: at.getAttribute('role'), \ ariaLabel: at.getAttribute('aria-label'), \ text: ((at.textContent||'').trim().slice(0, 60)) \ }); \ }"; let mut last_occ: Option = None; for attempt in 0..=max_retries { if attempt > 0 { tokio::time::sleep(std::time::Duration::from_millis(retry_delay_ms)).await; } let call_params = serde_json::json!({ "objectId": object_id, "functionDeclaration": function_decl, "arguments": [{"value": x}, {"value": y}], "returnByValue": true, }); let call_fut = client.send_command_typed::<_, serde_json::Value>( "Runtime.callFunctionOn", &call_params, Some(session_id), ); let Ok(call_resp) = tokio::time::timeout(std::time::Duration::from_millis(500), call_fut).await else { return Ok(()); // probe itself stalled — fall through to click }; let Ok(call_result) = call_resp else { return Ok(()); }; let value = call_result.get("result").and_then(|r| r.get("value")); let json_str = match value { Some(serde_json::Value::String(s)) => s.clone(), // null / undefined → element at point IS our target. Safe. _ => return Ok(()), }; let occ: Occluder = match serde_json::from_str(&json_str) { Ok(v) => v, Err(_) => return Ok(()), }; last_occ = Some(occ); } // All retries exhausted — overlay is sticky. Build the descriptive error. let occ = last_occ.expect("loop ran at least once"); if let Some(reason) = occ.reason { return Err(format!( "Ref {} cannot be clicked at its computed position: {}. \ The element may have moved off-screen — re-run snapshot.", ref_id, reason )); } let mut desc = occ.tag.unwrap_or_else(|| "unknown".to_string()); if let Some(t) = occ.testid { desc.push_str(&format!("[testid={}]", t)); } if let Some(r) = occ.role { desc.push_str(&format!("[role={}]", r)); } if let Some(a) = occ.aria_label { desc.push_str(&format!("[aria-label=\"{}\"]", a)); } if let Some(t) = occ.text { if !t.is_empty() { desc.push_str(&format!(" text=\"{}\"", t)); } } let waited_ms = (max_retries as u64) * retry_delay_ms; Err(format!( "Ref {} is occluded by {} at the click point (still occluded after \ {} retries / {}ms). A persistent overlay is in the way — \ re-run snapshot, dismiss the overlay, or set \ AGENT_BROWSER_VERIFY_CLICK_TARGET=0 to bypass.", ref_id, desc, max_retries, waited_ms, )) } /// Re-query the accessibility tree to find a node matching role+name+nth, /// returning its fresh backendDOMNodeId. This uses the same data source /// (Accessibility.getFullAXTree) that built the ref map during snapshot, /// so role/name matching is guaranteed to be consistent. async fn find_node_id_by_role_name( client: &CdpClient, session_id: &str, role: &str, name: &str, nth: Option, frame_id: Option<&str>, iframe_sessions: &HashMap, ) -> Result { let (ax_params, effective_session_id) = resolve_ax_session(frame_id, session_id, iframe_sessions); let ax_tree: GetFullAXTreeResult = client .send_command_typed( "Accessibility.getFullAXTree", &ax_params, Some(effective_session_id), ) .await?; let nth_index = nth.unwrap_or(0); let mut match_count: usize = 0; for node in &ax_tree.nodes { if node.ignored.unwrap_or(false) { continue; } let node_role = extract_ax_string(&node.role); let node_name = extract_ax_string(&node.name); if node_role == role && node_name == name { if match_count == nth_index { return node.backend_d_o_m_node_id.ok_or_else(|| { format!( "AX node has no backendDOMNodeId for role={} name={}", role, name ) }); } match_count += 1; } } Err(format!( "Could not locate element with role={} name={}", role, name )) } pub(super) fn extract_ax_string(value: &Option) -> String { match value { Some(v) => match &v.value { Some(Value::String(s)) => s.clone(), Some(Value::Number(n)) => n.to_string(), Some(Value::Bool(b)) => b.to_string(), _ => String::new(), }, None => String::new(), } } /// Build a JS expression that finds a DOM element by CSS selector or XPath. fn build_find_element_js(selector: &str) -> String { if let Some(xpath) = selector.strip_prefix("xpath=") { return format!( "document.evaluate({}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue", serde_json::to_string(xpath).unwrap_or_default() ); } // Bare string (or explicit `text=`): try CSS first, then fall back to // matching an interactive element by its VISIBLE TEXT. snapshot exposes // buttons/links by their name, so `click "購入手続きへ"` should resolve by // that label — previously it was fed straight to `querySelector` as CSS and // failed as an invalid selector even though the button was right there // (issue #24-B). CSS still wins when it matches, so existing selectors are // unaffected; nested/non-ASCII labels now resolve too. let text_only = selector.strip_prefix("text="); let force_text = text_only.is_some(); let sel_json = serde_json::to_string(selector).unwrap_or_default(); let want_json = serde_json::to_string(text_only.unwrap_or(selector)).unwrap_or_default(); format!( r#"(() => {{ const sel = {sel}; const css = {force_text} ? null : (() => {{ try {{ return document.querySelector(sel); }} catch (_e) {{ return null; }} }})(); if (css) return css; const norm = s => (s == null ? '' : String(s)).replace(/\s+/g, ' ').trim(); const w = norm({want}); if (!w) return null; const wl = w.toLowerCase(); const interactive = Array.from(document.querySelectorAll( 'button,a,[role=button],[role=link],[role=menuitem],[role=tab],[role=option],input[type=submit],input[type=button],input[type=reset],summary,label,[onclick]')); const textOf = e => norm(e.innerText || e.textContent) || norm(e.value) || norm(e.getAttribute && e.getAttribute('aria-label')) || norm(e.getAttribute && e.getAttribute('title')); let hit = interactive.find(e => textOf(e) === w) || interactive.find(e => textOf(e).toLowerCase().includes(wl)); if (hit) return hit; const leaves = Array.from(document.querySelectorAll('*')).filter(e => !e.children.length); return leaves.find(e => norm(e.textContent) === w) || leaves.find(e => norm(e.textContent).toLowerCase().includes(wl)) || null; }})()"#, sel = sel_json, want = want_json, force_text = force_text ) } /// Build a JS expression that counts matching DOM elements by CSS selector or XPath. fn build_count_elements_js(selector: &str) -> String { if let Some(xpath) = selector.strip_prefix("xpath=") { format!( "document.evaluate({}, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null).snapshotLength", serde_json::to_string(xpath).unwrap_or_default() ) } else { format!( "document.querySelectorAll({}).length", serde_json::to_string(selector).unwrap_or_default() ) } } fn build_selector_js(selector: &str) -> String { let find_expr = build_find_element_js(selector); format!( r#"(() => {{ const el = {find_expr}; if (!el) return null; const rect = el.getBoundingClientRect(); return {{ x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }}; }})()"#, ) } async fn resolve_by_selector( client: &CdpClient, session_id: &str, selector: &str, ) -> Result<(f64, f64), String> { let js = build_selector_js(selector); let result: EvaluateResult = client .send_command_typed( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(true), await_promise: Some(false), }, Some(session_id), ) .await?; // A syntactically-invalid CSS selector makes querySelector throw — surface // that as "invalid selector" rather than a misleading "element not found". if let Some(ex) = result.exception_details { return Err(format!("Invalid selector '{}': {}", selector, ex.text)); } let val = result.result.value.unwrap_or(Value::Null); let x = val.get("x").and_then(|v| v.as_f64()); let y = val.get("y").and_then(|v| v.as_f64()); match (x, y) { (Some(x), Some(y)) => Ok((x, y)), _ => Err(format!("Element not found: {}", selector)), } } fn box_model_center(model: &BoxModel) -> (f64, f64) { // content quad: [x1,y1, x2,y2, x3,y3, x4,y4] if model.content.len() >= 8 { let x = (model.content[0] + model.content[2] + model.content[4] + model.content[6]) / 4.0; let y = (model.content[1] + model.content[3] + model.content[5] + model.content[7]) / 4.0; (x, y) } else { (0.0, 0.0) } } /// Centre plus width/height of the content box, derived from the quad's /// bounding extent. Width/height feed humanize's in-bounds landing jitter; a /// degenerate quad yields zero size, which the jitter treats as "land on /// centre" (no jitter). fn box_model_dims(model: &BoxModel) -> (f64, f64, f64, f64) { let (cx, cy) = box_model_center(model); if model.content.len() >= 8 { let xs = [ model.content[0], model.content[2], model.content[4], model.content[6], ]; let ys = [ model.content[1], model.content[3], model.content[5], model.content[7], ]; let w = xs.iter().cloned().fold(f64::MIN, f64::max) - xs.iter().cloned().fold(f64::MAX, f64::min); let h = ys.iter().cloned().fold(f64::MIN, f64::max) - ys.iter().cloned().fold(f64::MAX, f64::min); (cx, cy, w.max(0.0), h.max(0.0)) } else { (cx, cy, 0.0, 0.0) } } pub async fn get_element_text( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { return this.innerText || this.textContent || ''; }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result .result .value .and_then(|v| v.as_str().map(|s| s.to_string())) .unwrap_or_default()) } pub async fn get_element_attribute( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, attribute: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: format!( "function() {{ return this.getAttribute({}); }}", serde_json::to_string(attribute).unwrap_or_default() ), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result.result.value.unwrap_or(Value::Null)) } pub async fn is_element_visible( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { const rect = this.getBoundingClientRect(); const style = window.getComputedStyle(this); return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none' && parseFloat(style.opacity) > 0; }"# .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result .result .value .and_then(|v| v.as_bool()) .unwrap_or(false)) } pub async fn is_element_enabled( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { return !this.disabled; }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result .result .value .and_then(|v| v.as_bool()) .unwrap_or(true)) } pub async fn is_element_checked( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; // Mirrors Playwright's getChecked() with follow-label retargeting: // 1. If element is a native checkbox/radio input, return .checked // 2. If element has an ARIA checked role, return aria-checked // 3. Follow label → input association (label.control) // 4. Check for nested checkbox/radio input as last resort let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { var el = this; // Native checkbox/radio input var tag = el.tagName && el.tagName.toUpperCase(); if (tag === 'INPUT' && (el.type === 'checkbox' || el.type === 'radio')) { return el.checked; } // ARIA role-based checked state var role = el.getAttribute && el.getAttribute('role'); var ariaCheckedRoles = ['checkbox','radio','switch','menuitemcheckbox','menuitemradio','option','treeitem']; if (role && ariaCheckedRoles.indexOf(role) !== -1) { return el.getAttribute('aria-checked') === 'true'; } // Follow label association (Playwright follow-label retarget) var label = el; if (tag !== 'LABEL') { label = el.closest && el.closest('label'); } if (label && label.tagName && label.tagName.toUpperCase() === 'LABEL' && label.control) { var ctrl = label.control; if (ctrl.type === 'checkbox' || ctrl.type === 'radio') { return ctrl.checked; } } // Check for nested native input var input = el.querySelector && el.querySelector('input[type="checkbox"], input[type="radio"]'); if (input) return input.checked; return false; }"#.to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result .result .value .and_then(|v| v.as_bool()) .unwrap_or(false)) } pub async fn get_element_inner_text( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { return this.innerText || ''; }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result .result .value .and_then(|v| v.as_str().map(|s| s.to_string())) .unwrap_or_default()) } pub async fn get_element_inner_html( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { return this.innerHTML || ''; }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result .result .value .and_then(|v| v.as_str().map(|s| s.to_string())) .unwrap_or_default()) } pub async fn get_element_input_value( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { return typeof this.value === 'string' ? this.value : ''; }" .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result .result .value .and_then(|v| v.as_str().map(|s| s.to_string())) .unwrap_or_default()) } pub async fn set_element_value( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, value: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let js = format!( "function() {{ this.value = {}; this.dispatchEvent(new Event('input', {{bubbles: true}})); this.dispatchEvent(new Event('change', {{bubbles: true}})); }}", serde_json::to_string(value).unwrap_or_default() ); client .send_command_typed::<_, EvaluateResult>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js, object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(()) } pub async fn get_element_bounding_box( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { const r = this.getBoundingClientRect(); return { x: r.x, y: r.y, width: r.width, height: r.height }; }"# .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; result .result .value .ok_or_else(|| format!("Could not get bounding box for: {}", selector_or_ref)) } pub async fn get_element_count( client: &CdpClient, session_id: &str, selector: &str, ) -> Result { let js = build_count_elements_js(selector); let result: EvaluateResult = client .send_command_typed( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(true), await_promise: Some(false), }, Some(session_id), ) .await?; Ok(result.result.value.and_then(|v| v.as_i64()).unwrap_or(0)) } pub async fn get_element_styles( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, properties: Option>, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let js = match properties { Some(props) => { let props_json = serde_json::to_string(&props).unwrap_or("[]".to_string()); format!( r#"function() {{ const s = window.getComputedStyle(this); const props = {}; const result = {{}}; for (const p of props) result[p] = s.getPropertyValue(p); return result; }}"#, props_json ) } None => r#"function() { const s = window.getComputedStyle(this); const result = {}; for (let i = 0; i < s.length; i++) { const p = s[i]; result[p] = s.getPropertyValue(p); } return result; }"# .to_string(), }; let result: EvaluateResult = client .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js, object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(result.result.value.unwrap_or(Value::Null)) } #[cfg(test)] mod tests { use super::*; #[test] fn test_parse_ref_at_prefix() { assert_eq!(parse_ref("@e1"), Some("e1".to_string())); assert_eq!(parse_ref("@e123"), Some("e123".to_string())); } #[test] fn test_parse_ref_equals_prefix() { assert_eq!(parse_ref("ref=e1"), Some("e1".to_string())); } #[test] fn test_parse_ref_bare() { assert_eq!(parse_ref("e1"), Some("e1".to_string())); assert_eq!(parse_ref("e42"), Some("e42".to_string())); } #[test] fn test_parse_ref_invalid() { assert_eq!(parse_ref("button"), None); assert_eq!(parse_ref("e"), None); assert_eq!(parse_ref("1"), None); assert_eq!(parse_ref(""), None); } #[test] fn test_ref_map_basic() { let mut map = RefMap::new(); map.add("e1".to_string(), Some(42), "button", "Submit", None); assert!(map.get("e1").is_some()); assert_eq!(map.get("e1").unwrap().role, "button"); assert!(map.get("e2").is_none()); } #[test] fn test_build_selector_js_css() { let js = build_selector_js("#submit-btn"); // CSS is now tried via a `sel` variable, with a visible-text fallback // appended (issue #24-B). It must still use querySelector (not xpath). assert!(js.contains("const sel = \"#submit-btn\"")); assert!(js.contains("document.querySelector(sel)")); assert!(!js.contains("document.evaluate")); } #[test] fn test_build_find_element_js_text_fallback() { // A bare label gets a text-matching fallback so `click "購入手続きへ"` // resolves by visible text, not just CSS (issue #24-B). let js = build_find_element_js("購入手続きへ"); assert!(js.contains("購入手続きへ")); assert!(js.contains("interactive")); // the text-match branch assert!(js.contains("textOf")); // `text=` forces the text path (skips CSS). let forced = build_find_element_js("text=Buy now"); assert!(forced.contains("true ? null")); // force_text => css skipped // xpath is unchanged. let xp = build_find_element_js("xpath=//button"); assert!(xp.contains("document.evaluate")); assert!(!xp.contains("interactive")); } #[test] fn test_build_selector_js_xpath() { let js = build_selector_js("xpath=//button[@id='ok']"); assert!(js.contains("document.evaluate(\"//button[@id='ok']\", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)")); assert!(!js.contains("document.querySelector")); } #[test] fn test_build_selector_js_xpath_empty() { let js = build_selector_js("xpath="); assert!(js.contains("document.evaluate")); } #[test] fn test_build_selector_js_not_xpath_prefix() { // "xpath" without "=" should be treated as CSS selector let js = build_selector_js("xpath//div"); assert!(js.contains("document.querySelector")); } #[test] fn test_build_count_elements_js_css() { let js = build_count_elements_js(".item"); assert!(js.contains("document.querySelectorAll(\".item\").length")); assert!(!js.contains("document.evaluate")); } #[test] fn test_build_count_elements_js_xpath() { let js = build_count_elements_js("xpath=//li"); assert!(js.contains("document.evaluate(\"//li\", document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null).snapshotLength")); assert!(!js.contains("querySelectorAll")); } #[test] fn test_box_model_center() { let model = BoxModel { content: vec![10.0, 20.0, 110.0, 20.0, 110.0, 60.0, 10.0, 60.0], padding: vec![], border: vec![], margin: vec![], width: 100, height: 40, }; let (x, y) = box_model_center(&model); assert!((x - 60.0).abs() < 0.01); assert!((y - 40.0).abs() < 0.01); } // ----------------------------------------------------------------------- // resolve_frame_session tests (Issue #925) // Cross-origin iframe elements must resolve to the dedicated session. // ----------------------------------------------------------------------- #[test] fn test_cross_origin_element_uses_dedicated_session() { let mut iframe_sessions = HashMap::new(); iframe_sessions.insert( "cross-origin-frame".to_string(), "iframe-session".to_string(), ); let session = resolve_frame_session( Some("cross-origin-frame"), "parent-session", &iframe_sessions, ); assert_eq!(session, "iframe-session"); } #[test] fn test_same_origin_element_uses_parent_session() { let iframe_sessions = HashMap::new(); let session = resolve_frame_session( Some("same-origin-frame"), "parent-session", &iframe_sessions, ); assert_eq!(session, "parent-session"); } #[test] fn test_main_frame_element_uses_parent_session() { let iframe_sessions = HashMap::new(); let session = resolve_frame_session(None, "parent-session", &iframe_sessions); assert_eq!(session, "parent-session"); } }