fix(click): occlusion guard for selector clicks — no more silent false success

A coordinate click resolved from a CSS selector (incl. the getByText/find path's
located node) skipped the occlusion check that @ref clicks already get, so an
overlay on top made the click land on the overlay while still reporting ✓ Done —
the worst failure mode for an agent (Hermes #1, issue #2/#3). Now: if the click
point doesn't hit the target (elementFromPoint isn't the element / a descendant /
an ancestor wrapper), dispatch through the DOM instead, which fires the real
handler. Best-effort probe (a flaky check never blocks the normal path); skipped
for strict CLICK_MODE=coord and non-left/multi-clicks.

Verified: occluded button click hits 0→1 (was silent ✓Done); normal click
unaffected.
This commit is contained in:
leeguooooo
2026-06-11 22:34:12 +09:00
parent 6ecda4d706
commit b92757412d
+64 -1
View File
@@ -4,7 +4,7 @@ use serde_json::Value;
use super::cdp::client::CdpClient;
use super::cdp::types::*;
use super::element::{resolve_element_center, resolve_element_object_id, RefMap};
use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap};
use super::humanize;
pub async fn click(
@@ -56,6 +56,33 @@ pub async fn click(
match resolved {
Ok((cx, cy, w, h, effective_session_id)) => {
// Occlusion guard for the CSS-selector path. `@ref` clicks are already
// occlusion-checked in resolve_element_center, but a plain selector
// resolves to coordinates without that check — so an overlay (modal
// backdrop, sticky banner, the getByText located node sitting under a
// full-screen layer) would make the coordinate click land on the
// overlay and still report success. If the click point doesn't hit the
// target, dispatch through the DOM instead (targets the element
// directly). Skipped for strict `coord` mode and non-left/multi-clicks.
if mode != "coord"
&& button == "left"
&& click_count == 1
&& parse_ref(selector_or_ref).is_none()
&& point_misses_element(client, &effective_session_id, selector_or_ref).await
{
eprintln!(
"[click] target occluded at its click point; dispatching through \
the DOM (set AGENT_BROWSER_CLICK_MODE=coord to disable)"
);
return dom_click(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await;
}
// Land on a jittered point inside the element rather than its exact
// centre (Fast/Human). Zero size or Off → exact centre.
let (tx, ty) = humanize::landing_point(
@@ -92,6 +119,42 @@ pub async fn click(
}
}
/// True if a coordinate click at the selector's centre would land on something
/// OTHER than the element (an overlay on top), i.e. the element is occluded.
/// `false` when not occluded, the element is missing, or the probe fails (so we
/// never block a click on a flaky probe — the normal coordinate path runs).
async fn point_misses_element(client: &CdpClient, session_id: &str, selector: &str) -> bool {
let js = format!(
r#"(() => {{
const el = document.querySelector({sel});
if (!el) return false;
const r = el.getBoundingClientRect();
if (r.width === 0 || r.height === 0) return false;
const hit = document.elementFromPoint(r.left + r.width / 2, r.top + r.height / 2);
if (!hit) return false;
// Not occluded if the hit is the element, a descendant, or an ancestor
// wrapper (clicking those still reaches the element's handlers).
return !(hit === el || el.contains(hit) || hit.contains(el));
}})()"#,
sel = serde_json::to_string(selector).unwrap_or_default()
);
match client
.send_command_typed::<_, EvaluateResult>(
"Runtime.evaluate",
&EvaluateParams {
expression: js,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
)
.await
{
Ok(r) => r.result.value.and_then(|v| v.as_bool()).unwrap_or(false),
Err(_) => false,
}
}
/// Best-effort scroll-into-view before a coordinate click. Uses Chrome's
/// `scrollIntoViewIfNeeded` (only scrolls when not already fully visible),
/// falling back to centered `scrollIntoView`. Resolution failures are ignored —