diff --git a/README.md b/README.md index 229bf36..b6ac856 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ When using `--launch` mode (standalone browser), a full suite of 32 stealth patc | `AGENT_BROWSER_BLOCK_WEBRTC` | auto | `--launch` only. Auto-forces WebRTC through the proxy when one is set (no real-IP leak). `1` hides the local IP without a proxy; `0` opts out. | | `AGENT_BROWSER_HIDE_CANVAS` | off | `--launch` only. Adds session-stable canvas/audio fingerprint noise. Off by default (noise is itself a "lie"). | | `AGENT_BROWSER_ADAPTIVE_REF` | on | When a saved `@ref` moves and the role/name re-query fails, relocate it by fingerprint similarity (high score + clear margin required, else it fails loudly). `0` disables. | +| `AGENT_BROWSER_CLICK_MODE` | _(auto)_ | Click strategy. Default scrolls the target into view, dispatches a coordinate click, and falls back to a DOM `.click()` if a floating layer occludes the point. `dom` always uses `.click()` (best for autocomplete/menu items that close on blur); `coord` is strict coordinate-only (hard-fail on occlusion). | ## Differences from upstream diff --git a/cli/src/native/interaction.rs b/cli/src/native/interaction.rs index 26758e6..9ce4f17 100644 --- a/cli/src/native/interaction.rs +++ b/cli/src/native/interaction.rs @@ -15,7 +15,111 @@ pub async fn click( click_count: i32, iframe_sessions: &HashMap, ) -> Result<(), String> { - let (x, y, effective_session_id) = resolve_element_center( + // AGENT_BROWSER_CLICK_MODE: "" (default) = coordinate click with a DOM + // fallback; "coord" = strict coordinate only (no fallback); "dom" = always + // dispatch through the DOM. + let mode = std::env::var("AGENT_BROWSER_CLICK_MODE").unwrap_or_default(); + + // (A) Scroll the target into view first so the computed coordinates land + // inside the viewport. Without this, an element below the fold (or revealed + // after scroll/popup) yields off-viewport coordinates and the click lands on + // whatever currently occupies that point. Best-effort: ignore failures. + scroll_into_view_if_needed(client, session_id, ref_map, selector_or_ref, iframe_sessions).await; + + if mode == "dom" { + return dom_click(client, session_id, ref_map, selector_or_ref, iframe_sessions).await; + } + + let resolved = resolve_element_center( + client, + session_id, + ref_map, + selector_or_ref, + iframe_sessions, + ) + .await; + + match resolved { + Ok((x, y, effective_session_id)) => { + dispatch_click(client, &effective_session_id, x, y, button, click_count).await + } + Err(e) => { + // (B) The coordinate path failed — typically a persistent overlay + // failing the occlusion guard, or coordinates that won't resolve. + // Fall back to a DOM-dispatched `.click()` on the intended element, + // which targets the element directly instead of a screen point. + // Skipped for strict "coord" mode and for non-left / multi-clicks + // (a DOM `.click()` can't express right/middle/double semantics). + if mode == "coord" || button != "left" || click_count != 1 { + return Err(e); + } + eprintln!( + "[click] coordinate click failed ({e}); falling back to DOM dispatch \ + (set AGENT_BROWSER_CLICK_MODE=coord to disable)" + ); + dom_click(client, session_id, ref_map, selector_or_ref, iframe_sessions) + .await + .map_err(|dom_err| format!("{e}\n(DOM-dispatch fallback also failed: {dom_err})")) + } + } +} + +/// Best-effort scroll-into-view before a coordinate click. Uses Chrome's +/// `scrollIntoViewIfNeeded` (only scrolls when not already fully visible), +/// falling back to centered `scrollIntoView`. Resolution failures are ignored — +/// the subsequent resolve will surface a real "not found" error. +async fn scroll_into_view_if_needed( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, + selector_or_ref: &str, + iframe_sessions: &HashMap, +) { + let Ok((object_id, effective_session_id)) = resolve_element_object_id( + client, + session_id, + ref_map, + selector_or_ref, + iframe_sessions, + ) + .await + else { + return; + }; + let js = "function() { try { \ + if (typeof this.scrollIntoViewIfNeeded === 'function') { this.scrollIntoViewIfNeeded(true); } \ + else { this.scrollIntoView({ block: 'center', inline: 'center' }); } \ + } catch (e) {} }"; + let _ = client + .send_command_typed::<_, Value>( + "Runtime.callFunctionOn", + &CallFunctionOnParams { + function_declaration: js.to_string(), + object_id: Some(object_id), + arguments: None, + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(&effective_session_id), + ) + .await; + // Let the scroll settle so the following getBoxModel sees final coordinates. + wait_for_paint_settled(client, &effective_session_id).await; +} + +/// Dispatch a click through the DOM (`element.click()`) instead of via screen +/// coordinates. Targets the intended element directly, so it works when a +/// floating layer occludes the click point or the element sits in a portal that +/// confuses `elementFromPoint`. Used as the fallback for `click` and when +/// `AGENT_BROWSER_CLICK_MODE=dom`. +async fn dom_click( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, + selector_or_ref: &str, + iframe_sessions: &HashMap, +) -> Result<(), String> { + let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, @@ -23,7 +127,21 @@ pub async fn click( iframe_sessions, ) .await?; - dispatch_click(client, &effective_session_id, x, y, button, click_count).await + client + .send_command_typed::<_, Value>( + "Runtime.callFunctionOn", + &CallFunctionOnParams { + function_declaration: "function() { this.click(); }".to_string(), + object_id: Some(object_id), + arguments: None, + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(&effective_session_id), + ) + .await?; + wait_for_paint_settled(client, &effective_session_id).await; + Ok(()) } pub async fn dblclick( diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index f752dab..90fe80e 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -336,6 +336,25 @@ agent-browser profiler start # Start Chrome DevTools profiling agent-browser profiler stop trace.json # Stop and save profile ``` +### Debugging forms / hidden state with `eval` + +The a11y `snapshot` shows visible, interactive elements — it does **not** show +hidden inputs or a control's actual submitted value. When a form "looks filled" +but submit-validation rejects it, go straight to the DOM with `eval` instead of +guessing from the snapshot. This is usually the fastest way to find the real +problem (e.g. a hidden `point_choice=none` that the visible UI never exposes): + +```bash +# Dump every field's name → value, including hidden inputs and unchecked radios +agent-browser eval "JSON.stringify([...document.forms[0].elements].map(e=>({name:e.name,type:e.type,value:e.value,checked:e.checked})).filter(e=>e.name))" + +# Inspect one hidden field directly +agent-browser eval "document.querySelector('[name=point_choice]')?.value" + +# Why won't it submit? Ask the browser's own validity API +agent-browser eval "[...document.forms[0].elements].filter(e=>!e.validity?.valid).map(e=>e.name+': '+e.validationMessage)" +``` + ## React / Web Vitals Requires `--enable react-devtools` at launch for the `react ...` commands. @@ -391,8 +410,22 @@ AGENT_BROWSER_HIDE_SCROLLBARS="false" # Keep native scrollbars visible in AGENT_BROWSER_PROVIDER="browserbase" # Cloud browser provider AGENT_BROWSER_STREAM_PORT="9223" # Override WebSocket streaming port (default: OS-assigned) AGENT_BROWSER_HOME="/path/to/agent-browser" # Custom install location +AGENT_BROWSER_CLICK_MODE="dom" # Click strategy: "" (default: scroll-in + coordinate + # click, DOM-dispatch fallback), "coord" (strict + # coordinate only), "dom" (always element.click()) ``` +### Click reliability + +`click` auto-scrolls the target into view first, then dispatches a coordinate +click. If that fails (a floating layer fails the occlusion guard, or the point +won't resolve) it falls back to a DOM-dispatched `.click()` on the intended +element. If a click *reports success but the page didn't react* — common for +autocomplete/menu `
  • ` items that close on the input's blur — retry that one +with `AGENT_BROWSER_CLICK_MODE=dom` (a DOM dispatch doesn't move focus the way a +real pointer press does, so the item still selects). `=coord` disables the +fallback when you specifically want a hard failure on occlusion. + ### Stealth / anti-detection knobs (fork) ```bash