From 8b55c553e602e53979be141e5d1b47dc450079d1 Mon Sep 17 00:00:00 2001 From: leeguooooo Date: Thu, 4 Jun 2026 14:10:50 +0900 Subject: [PATCH] feat(adaptive): relocate stale @refs by AX fingerprint similarity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Borrow Scrapling's adaptive element finding, adapted to this project's in-session AX-ref model. When a saved @ref's node is gone (or its identity no longer matches) and the role/name/nth re-query also fails, score the current page's candidate elements against an AX fingerprint captured at snapshot time and relocate to the best match. - New `adaptive` module: pure, browser-free scoring (role, accessible name via Levenshtein, AX properties, ancestor-role LCS, parent/sibling) plus pick_best with a high absolute threshold (0.70) AND a clear margin (0.15) over the runner-up — so ambiguous twins are refused rather than mis-clicked, matching the existing "fail loudly over wrong click" posture. - Fingerprint captured during the existing AX-tree snapshot walk — no extra CDP round-trips. TreeNode is AX-only (no DOM tag/attrs), so we use AX role as the type and a few discriminating AX properties (value/url/level/checked); DOM id/class would have cost an N×describeNode storm per snapshot. - Wired into both resolve_element_center and resolve_element_object_id: on a verify-identity mismatch or a stale-node fallback miss, relocation is tried before erroring. A confident match overrides the identity guard; otherwise the original error is surfaced. Opt out with AGENT_BROWSER_ADAPTIVE_REF=0. README documents the new tuning knobs. Adds 9 unit tests; full suite 760 passed. --- README.md | 15 +- cli/src/native/adaptive.rs | 366 +++++++++++++++++++++++++++++++++++++ cli/src/native/element.rs | 130 ++++++++++--- cli/src/native/mod.rs | 2 + cli/src/native/snapshot.rs | 118 ++++++++++++ 5 files changed, 606 insertions(+), 25 deletions(-) create mode 100644 cli/src/native/adaptive.rs diff --git a/README.md b/README.md index f2ab4a8..229bf36 100644 --- a/README.md +++ b/README.md @@ -115,9 +115,10 @@ In CI environments, standalone mode is used automatically. ## Anti-detection -When connected to your real Chrome, we inject **zero** JavaScript patches. Your browser's fingerprint is completely genuine. +When connected to your real Chrome, we inject **zero** JavaScript patches. Your browser's fingerprint is completely genuine. The guiding rule is **native CDP/Chrome overrides over JS lies** — a re-defined getter is itself detectable; a native override isn't. -The only thing we do is call `Emulation.setAutomationOverride` via CDP to set `navigator.webdriver = false` at the native Chrome level — undetectable by lie-detection systems like CreepJS. +- `navigator.webdriver = false` via `Emulation.setAutomationOverride` (native, undetectable by CreepJS-style lie tests). +- **`Runtime.enable` is left OFF by default.** A live `Runtime` domain is a detectable CDP signal (the patchright/rebrowser "runtime leak") — even when attached to your real Chrome. We only enable it when you opt into console/error capture (see below). `click`, `fill`, `eval`, etc. work without it. **Test results (connected to real Chrome):** @@ -129,6 +130,16 @@ The only thing we do is call `Emulation.setAutomationOverride` via CDP to set `n When using `--launch` mode (standalone browser), a full suite of 32 stealth patches is applied for headless Chrome. +### Tuning knobs (environment variables) + +| Variable | Default | Effect | +|---|---|---| +| `AGENT_BROWSER_CAPTURE_CONSOLE` | off | Enable `Runtime` domain so `console` / `errors` capture page output. Off keeps the stealthiest profile. | +| `AGENT_BROWSER_TIMEZONE` | unset | `--launch` only. An IANA id (e.g. `Asia/Tokyo`) sets the timezone natively (Intl + Date follow, no JS lie) to match a proxy; `auto` derives one from the locale. | +| `AGENT_BROWSER_BLOCK_WEBRTC` | auto | `--launch` only. Auto-forces WebRTC through the proxy when one is set (no real-IP leak). `1` hides the local IP without a proxy; `0` opts out. | +| `AGENT_BROWSER_HIDE_CANVAS` | off | `--launch` only. Adds session-stable canvas/audio fingerprint noise. Off by default (noise is itself a "lie"). | +| `AGENT_BROWSER_ADAPTIVE_REF` | on | When a saved `@ref` moves and the role/name re-query fails, relocate it by fingerprint similarity (high score + clear margin required, else it fails loudly). `0` disables. | + ## Differences from upstream Based on [agent-browser v0.27.0](https://github.com/vercel-labs/agent-browser). Changes: diff --git a/cli/src/native/adaptive.rs b/cli/src/native/adaptive.rs new file mode 100644 index 0000000..e90ec55 --- /dev/null +++ b/cli/src/native/adaptive.rs @@ -0,0 +1,366 @@ +//! Adaptive @ref relocation. +//! +//! When a saved `@ref`'s DOM node is gone (stale `backendNodeId`) and the +//! role/name/nth re-query also fails, we score the current page's candidate +//! elements against the ref's stored [`ElementFingerprint`] and relocate to the +//! best match — but ONLY when confident: the best candidate must clear a high +//! absolute threshold AND beat the runner-up by a clear margin. This matches the +//! project's "fail loudly rather than mis-click" posture (see the identity and +//! occlusion guards in `element.rs`). +//! +//! Everything in this module is pure and browser-free so the scoring can be +//! unit-tested directly. + +use std::collections::BTreeMap; + +/// Minimum absolute similarity (0..1) for a relocation candidate to be accepted. +pub const ADAPTIVE_THRESHOLD: f64 = 0.70; +/// Minimum gap between the best and second-best candidate to avoid ambiguity. +pub const ADAPTIVE_MARGIN: f64 = 0.15; + +/// A structural/semantic fingerprint of an element, captured at snapshot time so +/// a moved element can be re-identified after the page mutates. +/// +/// Populated purely from the accessibility tree we already walk (`TreeNode`), so +/// capturing it costs no extra CDP round-trips — `TreeNode` has no DOM tag or +/// attributes (those would need an N×`DOM.describeNode` storm per snapshot), so +/// `tag` holds the AX **role** and `attrs` holds discriminating AX properties +/// (value/url/level/checked), not DOM `id`/`class`. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ElementFingerprint { + /// AX role, e.g. "button" (used where a DOM tag would otherwise go). + pub tag: String, + /// Accessible name / visible text — the dominant identity signal. + pub text: String, + /// Discriminating AX properties: value, url, level, checked. Keyed by name. + pub attrs: BTreeMap, + /// Ancestor role signatures from nearest to farthest, e.g. "form" / "list". + pub ancestors: Vec, + /// Parent role. + pub parent_tag: String, + /// Parent accessible name / text. + pub parent_text: String, + /// Index among same-role siblings. + pub sibling_index: u32, + /// Count of same-role siblings. + pub sibling_count: u32, +} + +/// Component weights. They sum to 1.0 so the total score lands in 0..1. +/// Tuned for AX-derived fingerprints: the accessible name dominates, with role +/// and tree structure carrying disambiguation when the name has changed (which +/// is exactly when the exact role+name+nth fallback failed and we got here). +const W_TAG: f64 = 0.20; +const W_TEXT: f64 = 0.40; +const W_ATTRS: f64 = 0.10; +const W_ANCESTORS: f64 = 0.20; +const W_PARENT_SIBLING: f64 = 0.10; + +/// Per-attribute importance for the attribute-overlap score. Strong identity +/// signals (a link's url) outweigh weak ones (heading level). +fn attr_weight(name: &str) -> f64 { + match name { + "url" | "value" => 3.0, + "checked" => 2.0, + _ => 1.0, + } +} + +/// Levenshtein-based string similarity in 0..1 (1.0 = identical). Two empty +/// strings are treated as a perfect match (consistent absence of text). +pub fn string_similarity(a: &str, b: &str) -> f64 { + if a == b { + return 1.0; + } + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + let max_len = a.len().max(b.len()); + if max_len == 0 { + return 1.0; + } + let dist = levenshtein(&a, &b); + 1.0 - (dist as f64 / max_len as f64) +} + +fn levenshtein(a: &[char], b: &[char]) -> usize { + if a.is_empty() { + return b.len(); + } + if b.is_empty() { + return a.len(); + } + let mut prev: Vec = (0..=b.len()).collect(); + let mut cur = vec![0usize; b.len() + 1]; + for (i, &ca) in a.iter().enumerate() { + cur[0] = i + 1; + for (j, &cb) in b.iter().enumerate() { + let cost = if ca == cb { 0 } else { 1 }; + cur[j + 1] = (prev[j + 1] + 1).min(cur[j] + 1).min(prev[j] + cost); + } + std::mem::swap(&mut prev, &mut cur); + } + prev[b.len()] +} + +/// Jaccard similarity over whitespace-separated tokens (used for `class`). +fn token_jaccard(a: &str, b: &str) -> f64 { + let sa: std::collections::BTreeSet<&str> = a.split_whitespace().collect(); + let sb: std::collections::BTreeSet<&str> = b.split_whitespace().collect(); + if sa.is_empty() && sb.is_empty() { + return 1.0; + } + let inter = sa.intersection(&sb).count() as f64; + let union = sa.union(&sb).count() as f64; + if union == 0.0 { + 1.0 + } else { + inter / union + } +} + +/// Length-ratio of the longest common subsequence over two ancestor sequences. +fn lcs_ratio(a: &[String], b: &[String]) -> f64 { + if a.is_empty() && b.is_empty() { + return 1.0; + } + if a.is_empty() || b.is_empty() { + return 0.0; + } + let mut dp = vec![vec![0usize; b.len() + 1]; a.len() + 1]; + for i in 0..a.len() { + for j in 0..b.len() { + dp[i + 1][j + 1] = if a[i] == b[j] { + dp[i][j] + 1 + } else { + dp[i][j + 1].max(dp[i + 1][j]) + }; + } + } + let lcs = dp[a.len()][b.len()] as f64; + (2.0 * lcs) / (a.len() + b.len()) as f64 +} + +fn attr_score(base: &BTreeMap, cand: &BTreeMap) -> f64 { + let mut names: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + names.extend(base.keys().map(|s| s.as_str())); + names.extend(cand.keys().map(|s| s.as_str())); + if names.is_empty() { + return 1.0; // no attributes on either side — neutral + } + let mut total = 0.0; + let mut got = 0.0; + for name in names { + let w = attr_weight(name); + total += w; + match (base.get(name), cand.get(name)) { + (Some(a), Some(b)) => { + if name == "class" { + got += w * token_jaccard(a, b); + } else if a == b { + got += w; + } + } + _ => {} // present on only one side → no credit + } + } + if total == 0.0 { + 1.0 + } else { + got / total + } +} + +fn parent_sibling_score(base: &ElementFingerprint, cand: &ElementFingerprint) -> f64 { + // Split the 0.10 budget: parent tag 0.4, parent text 0.3, sibling pos 0.3. + let parent_tag = if base.parent_tag == cand.parent_tag { + 1.0 + } else { + 0.0 + }; + let parent_text = string_similarity(&base.parent_text, &cand.parent_text); + let span = base.sibling_count.max(1) as f64; + let delta = (base.sibling_index as i64 - cand.sibling_index as i64).unsigned_abs() as f64; + let sibling = 1.0 - (delta / span).min(1.0); + 0.4 * parent_tag + 0.3 * parent_text + 0.3 * sibling +} + +/// Similarity score in 0..1 between a stored baseline and a candidate element. +pub fn score(base: &ElementFingerprint, cand: &ElementFingerprint) -> f64 { + let tag = if base.tag == cand.tag { 1.0 } else { 0.0 }; + let text = string_similarity(&base.text, &cand.text); + let attrs = attr_score(&base.attrs, &cand.attrs); + let ancestors = lcs_ratio(&base.ancestors, &cand.ancestors); + let parent_sibling = parent_sibling_score(base, cand); + + W_TAG * tag + + W_TEXT * text + + W_ATTRS * attrs + + W_ANCESTORS * ancestors + + W_PARENT_SIBLING * parent_sibling +} + +/// Why a relocation was rejected. +#[derive(Debug, Clone, PartialEq)] +pub enum RejectReason { + /// No candidates to score. + NoCandidates, + /// Best score below [`ADAPTIVE_THRESHOLD`]. + LowScore { best: f64 }, + /// Best score too close to the runner-up (below [`ADAPTIVE_MARGIN`]). + Ambiguous { best: f64, second: f64 }, +} + +/// A successful relocation decision. +#[derive(Debug, Clone, PartialEq)] +pub struct Relocation { + /// Chosen candidate's backend node id. + pub backend_node_id: i64, + /// Winning score. + pub score: f64, + /// Runner-up score (0.0 when there was only one candidate). + pub second_score: f64, +} + +/// Pick the best candidate, accepting only when confident. `candidates` is a +/// list of `(backend_node_id, fingerprint)` for the current page. +pub fn pick_best( + base: &ElementFingerprint, + candidates: &[(i64, ElementFingerprint)], + threshold: f64, + margin: f64, +) -> Result { + if candidates.is_empty() { + return Err(RejectReason::NoCandidates); + } + let mut scored: Vec<(i64, f64)> = candidates + .iter() + .map(|(id, fp)| (*id, score(base, fp))) + .collect(); + // Highest score first; stable enough for deterministic ties. + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + let (best_id, best) = scored[0]; + let second = scored.get(1).map(|(_, s)| *s).unwrap_or(0.0); + + if best < threshold { + return Err(RejectReason::LowScore { best }); + } + if best - second < margin { + return Err(RejectReason::Ambiguous { best, second }); + } + Ok(Relocation { + backend_node_id: best_id, + score: best, + second_score: second, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fp(tag: &str, text: &str, attrs: &[(&str, &str)]) -> ElementFingerprint { + ElementFingerprint { + tag: tag.to_string(), + text: text.to_string(), + attrs: attrs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ..Default::default() + } + } + + #[test] + fn identical_fingerprints_score_one() { + let a = fp("button", "Submit", &[("id", "go"), ("class", "btn primary")]); + assert!((score(&a, &a) - 1.0).abs() < 1e-9); + } + + #[test] + fn different_tag_caps_score_below_threshold() { + let a = fp("button", "Submit", &[("id", "go")]); + let b = fp("a", "Submit", &[("id", "go")]); + // Same text + same attrs but different role: must lose the role weight + // (W_TAG = 0.20), landing around 0.80 and below a perfect match. + let s = score(&a, &b); + assert!(s < 0.85 && s > 0.75, "got {s}"); + } + + #[test] + fn string_similarity_basics() { + assert_eq!(string_similarity("abc", "abc"), 1.0); + assert_eq!(string_similarity("", ""), 1.0); + assert!(string_similarity("Submit", "Submit now") > 0.5); + assert!(string_similarity("Add post", "Post all") < 0.6); + } + + #[test] + fn class_uses_token_overlap() { + let a = fp("div", "", &[("class", "card primary big")]); + let b = fp("div", "", &[("class", "card primary")]); + // partial class overlap should still score high (tag+text match, attrs partial) + let s = score(&a, &b); + assert!(s > 0.85, "got {s}"); + } + + #[test] + fn ancestors_lcs() { + let mut a = fp("button", "OK", &[]); + let mut b = fp("button", "OK", &[]); + a.ancestors = vec!["form#f".into(), "div.col".into(), "body".into()]; + // b wrapped in an extra div — DOM path changed but mostly preserved + b.ancestors = vec!["form#f".into(), "div.wrap".into(), "div.col".into(), "body".into()]; + let s = score(&a, &b); + assert!(s > 0.85, "got {s}"); + } + + #[test] + fn pick_best_accepts_clear_winner() { + let base = fp("button", "Submit", &[("id", "go")]); + let winner = fp("button", "Submit", &[("id", "go")]); + let other = fp("a", "Home", &[("href", "/")]); + let out = pick_best( + &base, + &[(10, other), (20, winner)], + ADAPTIVE_THRESHOLD, + ADAPTIVE_MARGIN, + ) + .expect("should accept"); + assert_eq!(out.backend_node_id, 20); + assert!(out.score > out.second_score); + } + + #[test] + fn pick_best_rejects_ambiguous_twins() { + let base = fp("button", "Delete", &[("class", "btn danger")]); + // Two near-identical delete buttons — must refuse to guess. + let twin_a = fp("button", "Delete", &[("class", "btn danger")]); + let twin_b = fp("button", "Delete", &[("class", "btn danger")]); + let err = pick_best( + &base, + &[(1, twin_a), (2, twin_b)], + ADAPTIVE_THRESHOLD, + ADAPTIVE_MARGIN, + ) + .unwrap_err(); + assert!(matches!(err, RejectReason::Ambiguous { .. }), "got {err:?}"); + } + + #[test] + fn pick_best_rejects_low_score() { + let base = fp("button", "Submit order", &[("id", "checkout")]); + let junk = fp("span", "unrelated footer text", &[("class", "muted")]); + let err = pick_best(&base, &[(1, junk)], ADAPTIVE_THRESHOLD, ADAPTIVE_MARGIN).unwrap_err(); + assert!(matches!(err, RejectReason::LowScore { .. }), "got {err:?}"); + } + + #[test] + fn pick_best_no_candidates() { + let base = fp("button", "x", &[]); + assert_eq!( + pick_best(&base, &[], ADAPTIVE_THRESHOLD, ADAPTIVE_MARGIN).unwrap_err(), + RejectReason::NoCandidates + ); + } +} diff --git a/cli/src/native/element.rs b/cli/src/native/element.rs index 965e467..3bf3277 100644 --- a/cli/src/native/element.rs +++ b/cli/src/native/element.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use serde_json::Value; +use super::adaptive::{self, ElementFingerprint}; use super::cdp::client::CdpClient; use super::cdp::types::*; @@ -13,6 +14,9 @@ pub struct RefEntry { pub nth: Option, pub selector: Option, pub frame_id: Option, + /// AX fingerprint captured at snapshot time, used by adaptive relocation when + /// the node is gone and the role/name/nth re-query also fails. + pub fingerprint: Option, } pub struct RefMap { @@ -57,10 +61,19 @@ impl RefMap { nth, selector: None, frame_id: frame_id.map(|s| s.to_string()), + fingerprint: None, }, ); } + /// Attach an AX fingerprint to an existing ref (set during snapshot, used by + /// adaptive relocation). No-op if the ref is unknown. + pub fn set_fingerprint(&mut self, ref_id: &str, fingerprint: ElementFingerprint) { + if let Some(entry) = self.map.get_mut(ref_id) { + entry.fingerprint = Some(fingerprint); + } + } + pub fn add_selector( &mut self, ref_id: String, @@ -78,6 +91,7 @@ impl RefMap { nth, selector: Some(selector), frame_id: None, + fingerprint: None, }, ); } @@ -146,6 +160,46 @@ pub fn parse_ref(input: &str) -> Option { None } +/// When a saved `@ref`'s node is gone and the role/name/nth re-query also failed, +/// try to relocate the element by AX fingerprint similarity. Returns the chosen +/// backend node id only when confident (high score + clear margin over the +/// runner-up). Opt out with `AGENT_BROWSER_ADAPTIVE_REF=0`. +async fn relocate_stale_ref( + client: &CdpClient, + ref_id: &str, + entry: &RefEntry, + session_id: &str, + iframe_sessions: &HashMap, +) -> Option { + if std::env::var("AGENT_BROWSER_ADAPTIVE_REF").as_deref() == Ok("0") { + return None; + } + let baseline = entry.fingerprint.as_ref()?; + let candidates = super::snapshot::collect_current_fingerprints( + client, + session_id, + entry.frame_id.as_deref(), + iframe_sessions, + ) + .await + .ok()?; + match adaptive::pick_best( + baseline, + &candidates, + adaptive::ADAPTIVE_THRESHOLD, + adaptive::ADAPTIVE_MARGIN, + ) { + Ok(reloc) => { + eprintln!( + "[adaptive] relocated {ref_id} ({} \"{}\") score={:.2} second={:.2} -> backendNodeId {}", + entry.role, entry.name, reloc.score, reloc.second_score, reloc.backend_node_id + ); + Some(reloc.backend_node_id) + } + Err(_) => None, + } +} + pub async fn resolve_element_center( client: &CdpClient, session_id: &str, @@ -163,15 +217,19 @@ pub async fn resolve_element_center( // Try cached backend_node_id first (fast path) if let Some(backend_node_id) = entry.backend_node_id { + let mut active_id = backend_node_id; // Identity check: React often re-uses the same DOM node when // re-rendering — backendNodeId stays the same but accessibleName // / role changes. Without this verification, `click @e20` (saved // when the button said "Add post") happily clicks the *same* // node that now says "Post all", silently submitting the thread. // - // Set AGENT_BROWSER_VERIFY_REF=0 to skip (saves one CDP - // roundtrip per ref-based interaction; only safe if you know - // the page is static between snapshot and click). + // On mismatch, try adaptive fingerprint relocation before failing: + // a confident high-score/high-margin match is a stronger identity + // signal than role+name, and lets a moved+renamed element still + // resolve. If relocation isn't confident, surface the original + // identity error. Set AGENT_BROWSER_VERIFY_REF=0 to skip the check + // (and thus relocation) entirely. if std::env::var("AGENT_BROWSER_VERIFY_REF").as_deref() != Ok("0") { if let Err(e) = verify_ref_identity( client, @@ -183,7 +241,12 @@ pub async fn resolve_element_center( ) .await { - return Err(e); + match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) + .await + { + Some(id) => active_id = id, + None => return Err(e), + } } } @@ -191,7 +254,7 @@ pub async fn resolve_element_center( .send_command_typed( "DOM.getBoxModel", &DomGetBoxModelParams { - backend_node_id: Some(backend_node_id), + backend_node_id: Some(active_id), node_id: None, object_id: None, }, @@ -213,15 +276,9 @@ pub async fn resolve_element_center( // // Set AGENT_BROWSER_VERIFY_CLICK_TARGET=0 to skip. if std::env::var("AGENT_BROWSER_VERIFY_CLICK_TARGET").as_deref() != Ok("0") { - if let Err(e) = verify_click_target( - client, - effective_session_id, - backend_node_id, - &ref_id, - x, - y, - ) - .await + if let Err(e) = + verify_click_target(client, effective_session_id, active_id, &ref_id, x, y) + .await { return Err(e); } @@ -231,8 +288,9 @@ pub async fn resolve_element_center( // backend_node_id is stale; re-query the accessibility tree below } - // Fallback: re-query the accessibility tree to find a fresh node by role/name - let fresh_id = find_node_id_by_role_name( + // Fallback: re-query the accessibility tree to find a fresh node by role/name. + // If that fails, try adaptive fingerprint relocation before giving up. + let fresh_id = match find_node_id_by_role_name( client, session_id, &entry.role, @@ -241,7 +299,16 @@ pub async fn resolve_element_center( entry.frame_id.as_deref(), iframe_sessions, ) - .await?; + .await + { + Ok(id) => id, + Err(e) => match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) + .await + { + Some(id) => id, + None => return Err(e), + }, + }; let result: DomGetBoxModelResult = client .send_command_typed( "DOM.getBoxModel", @@ -279,9 +346,11 @@ pub async fn resolve_element_object_id( // Try cached backend_node_id first (fast path) if let Some(backend_node_id) = entry.backend_node_id { + let mut active_id = backend_node_id; // Same identity guard as resolve_element_center — see that // function for why React DOM-node-reuse breaks ref-based - // interactions if we skip this. + // interactions if we skip this, and why a confident adaptive + // relocation is allowed to override an identity mismatch. if std::env::var("AGENT_BROWSER_VERIFY_REF").as_deref() != Ok("0") { if let Err(e) = verify_ref_identity( client, @@ -293,7 +362,12 @@ pub async fn resolve_element_object_id( ) .await { - return Err(e); + match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) + .await + { + Some(id) => active_id = id, + None => return Err(e), + } } } @@ -301,7 +375,7 @@ pub async fn resolve_element_object_id( .send_command_typed( "DOM.resolveNode", &DomResolveNodeParams { - backend_node_id: Some(backend_node_id), + backend_node_id: Some(active_id), node_id: None, object_group: Some("agent-browser".to_string()), }, @@ -317,8 +391,9 @@ pub async fn resolve_element_object_id( // backend_node_id is stale; re-query the accessibility tree below } - // Fallback: re-query the accessibility tree to find a fresh node by role/name - let fresh_id = find_node_id_by_role_name( + // Fallback: re-query the accessibility tree to find a fresh node by role/name. + // If that fails, try adaptive fingerprint relocation before giving up. + let fresh_id = match find_node_id_by_role_name( client, session_id, &entry.role, @@ -327,7 +402,16 @@ pub async fn resolve_element_object_id( entry.frame_id.as_deref(), iframe_sessions, ) - .await?; + .await + { + Ok(id) => id, + Err(e) => match relocate_stale_ref(client, &ref_id, entry, session_id, iframe_sessions) + .await + { + Some(id) => id, + None => return Err(e), + }, + }; let result: DomResolveNodeResult = client .send_command_typed( "DOM.resolveNode", diff --git a/cli/src/native/mod.rs b/cli/src/native/mod.rs index 94da739..36dd81a 100644 --- a/cli/src/native/mod.rs +++ b/cli/src/native/mod.rs @@ -1,6 +1,8 @@ #[allow(dead_code)] pub mod actions; #[allow(dead_code)] +pub mod adaptive; +#[allow(dead_code)] pub mod auth; #[allow(dead_code)] pub mod browser; diff --git a/cli/src/native/snapshot.rs b/cli/src/native/snapshot.rs index 8aec758..886387d 100644 --- a/cli/src/native/snapshot.rs +++ b/cli/src/native/snapshot.rs @@ -6,6 +6,7 @@ use super::cdp::client::CdpClient; use super::cdp::types::{ AXNode, AXProperty, AXValue, EvaluateParams, EvaluateResult, GetFullAXTreeResult, }; +use super::adaptive::ElementFingerprint; use super::element::{resolve_ax_session, RefMap}; const INTERACTIVE_ROLES: &[&str] = &[ @@ -148,6 +149,122 @@ impl TreeNode { } } +/// Build an AX fingerprint for a tree node, used by adaptive @ref relocation. +/// Pulls only data already in the AX tree (no extra CDP calls): role as `tag`, +/// accessible name as `text`, a few discriminating AX properties as `attrs`, and +/// the ancestor/parent/sibling structure from the tree links. +fn build_ax_fingerprint(tree_nodes: &[TreeNode], idx: usize) -> ElementFingerprint { + let node = &tree_nodes[idx]; + + let mut attrs = std::collections::BTreeMap::new(); + if let Some(v) = &node.value_text { + if !v.is_empty() { + attrs.insert("value".to_string(), v.clone()); + } + } + if let Some(u) = &node.url { + if !u.is_empty() { + attrs.insert("url".to_string(), u.clone()); + } + } + if let Some(l) = node.level { + attrs.insert("level".to_string(), l.to_string()); + } + if let Some(c) = &node.checked { + attrs.insert("checked".to_string(), c.clone()); + } + + // Ancestor roles, nearest first, capped to keep the signature stable. + let mut ancestors = Vec::new(); + let mut cur = node.parent_idx; + while let Some(pidx) = cur { + if ancestors.len() >= 6 { + break; + } + let role = tree_nodes[pidx].role.clone(); + if !role.is_empty() { + ancestors.push(role); + } + cur = tree_nodes[pidx].parent_idx; + } + + let (parent_tag, parent_text) = node + .parent_idx + .map(|pidx| (tree_nodes[pidx].role.clone(), tree_nodes[pidx].name.clone())) + .unwrap_or_default(); + + // Position among same-role siblings under the same parent. + let (sibling_index, sibling_count) = match node.parent_idx { + Some(pidx) => { + let mut count = 0u32; + let mut index = 0u32; + for &child in &tree_nodes[pidx].children { + if tree_nodes[child].role == node.role { + if child == idx { + index = count; + } + count += 1; + } + } + (index, count) + } + None => (0, 0), + }; + + ElementFingerprint { + tag: node.role.clone(), + text: node.name.clone(), + attrs, + ancestors, + parent_tag, + parent_text, + sibling_index, + sibling_count, + } +} + +/// Collect AX fingerprints for every node that has a backend node id, used as the +/// candidate set when relocating a stale @ref. Reuses the same extraction as the +/// baseline so the two are scored in the same space. +pub(super) fn collect_fingerprints(tree_nodes: &[TreeNode]) -> Vec<(i64, ElementFingerprint)> { + tree_nodes + .iter() + .enumerate() + .filter_map(|(idx, n)| { + n.backend_node_id + .map(|bid| (bid, build_ax_fingerprint(tree_nodes, idx))) + }) + .collect() +} + +/// Fetch a fresh AX tree for the given frame and return `(backend_node_id, +/// fingerprint)` for every node — the candidate set for adaptive @ref +/// relocation. One `getFullAXTree` call, no per-element work. +pub(super) async fn collect_current_fingerprints( + client: &CdpClient, + session_id: &str, + frame_id: Option<&str>, + iframe_sessions: &HashMap, +) -> Result, String> { + let (ax_params, effective_session_id) = + resolve_ax_session(frame_id, session_id, iframe_sessions); + let _ = client + .send_command_no_params("DOM.enable", Some(effective_session_id)) + .await; + let _ = client + .send_command_no_params("Accessibility.enable", Some(effective_session_id)) + .await; + let ax_tree: GetFullAXTreeResult = client + .send_command_typed( + "Accessibility.getFullAXTree", + &ax_params, + Some(effective_session_id), + ) + .await?; + let (tree_nodes, _roots) = build_tree(&ax_tree.nodes); + Ok(collect_fingerprints(&tree_nodes)) +} + /// The type of a hidden form input found inside a cursor-interactive element. #[derive(Clone, Copy)] enum HiddenInputKind { @@ -397,6 +514,7 @@ pub async fn take_snapshot( actual_nth, frame_id, ); + ref_map.set_fingerprint(&ref_id, build_ax_fingerprint(&tree_nodes, *idx)); tree_nodes[*idx].has_ref = true; tree_nodes[*idx].ref_id = Some(ref_id);