use std::collections::HashMap; use serde_json::Value; use super::cdp::client::CdpClient; use super::cdp::types::*; use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap}; use super::humanize; /// Whether a pointer interaction should be DOM-dispatched (invoke the event on /// the element in its own session) rather than dispatched at a viewport /// coordinate via `Input.dispatchMouseEvent`. True when the target is inside an /// iframe (an OOPIF element's box can't be mapped to a top-viewport point) or we /// drive over the extension relay (a coordinate Input event isn't confined to the /// target tab on a busy real Chrome — it drifts onto the foreground tab; issues /// #31/#36). DOM-dispatch always hits the right element in the right tab. fn prefer_dom_dispatch(ref_map: &RefMap, selector_or_ref: &str) -> bool { ref_map.ref_is_in_iframe(selector_or_ref) || crate::connect::relay_url().is_some() } pub async fn click( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, button: &str, click_count: i32, iframe_sessions: &HashMap, ) -> Result<(), String> { // AGENT_BROWSER_CLICK_MODE: "" (default) = coordinate click with a DOM // fallback; "coord" = strict coordinate only (no fallback); "dom" = always // dispatch through the DOM. let mode = std::env::var("AGENT_BROWSER_CLICK_MODE").unwrap_or_default(); // (A) Scroll the target into view first so the computed coordinates land // inside the viewport. Without this, an element below the fold (or revealed // after scroll/popup) yields off-viewport coordinates and the click lands on // whatever currently occupies that point. Best-effort: ignore failures. scroll_into_view_if_needed( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; if mode == "dom" { return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } // An element INSIDE an iframe needs a TRUSTED activation: a DOM `.click()` is // `isTrusted:false`, which security-sensitive embedded forms reject — Google // Payments' enabled `保存` button silently no-ops on a synthetic click (issue // #39). A coordinate `Input.dispatchMouseEvent` can't help either: `getBoxModel` // for a sub-frame node returns frame-local coordinates that don't compose the // iframe's offset, so the click lands in the wrong place. The frame-agnostic // trusted path is keyboard activation — focus the element in its own frame, then // dispatch a real Enter on the page session; Chrome routes the key to the // focused element regardless of frame (same as `type --focused`), and Enter on a // focused button/link fires a trusted `click`. `coord` mode opts out. let in_iframe = ref_map.ref_is_in_iframe(selector_or_ref); if mode != "coord" && button == "left" && click_count == 1 && in_iframe { return dom_activate( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } // On the relay (the user's real Chrome) a TOP-document coordinate click used to // drift onto the foreground tab; that root cause is fixed (#5: the agent drives // its own pinned tab), but DOM-dispatch stays the conservative default here. if mode != "coord" && button == "left" && click_count == 1 && crate::connect::relay_url().is_some() { return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } let resolved = resolve_element_center( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; match resolved { Ok((cx, cy, w, h, effective_session_id)) => { // Occlusion guard for the CSS-selector path. `@ref` clicks are already // occlusion-checked in resolve_element_center, but a plain selector // resolves to coordinates without that check — so an overlay (modal // backdrop, sticky banner, the getByText located node sitting under a // full-screen layer) would make the coordinate click land on the // overlay and still report success. If the click point doesn't hit the // target, dispatch through the DOM instead (targets the element // directly). Skipped for strict `coord` mode and non-left/multi-clicks. if mode != "coord" && button == "left" && click_count == 1 && parse_ref(selector_or_ref).is_none() && point_misses_element(client, &effective_session_id, selector_or_ref).await { eprintln!( "[click] target occluded at its click point; dispatching through \ the DOM (set AGENT_BROWSER_CLICK_MODE=coord to disable)" ); return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } // Land on a jittered point inside the element rather than its exact // centre (Fast/Human). Zero size or Off → exact centre. let (tx, ty) = humanize::landing_point( (cx - w / 2.0, cy - h / 2.0, w, h), humanize::active_level(), humanize::next_seed(), ); dispatch_click(client, &effective_session_id, tx, ty, button, click_count).await } Err(e) => { // (B) The coordinate path failed — typically a persistent overlay // failing the occlusion guard, or coordinates that won't resolve. // Fall back to a DOM-dispatched `.click()` on the intended element, // which targets the element directly instead of a screen point. // Skipped for strict "coord" mode and for non-left / multi-clicks // (a DOM `.click()` can't express right/middle/double semantics). if mode == "coord" || button != "left" || click_count != 1 { return Err(e); } eprintln!( "[click] coordinate click failed ({e}); falling back to DOM dispatch \ (set AGENT_BROWSER_CLICK_MODE=coord to disable)" ); dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await .map_err(|dom_err| format!("{e}\n(DOM-dispatch fallback also failed: {dom_err})")) } } } /// True if a coordinate click at the selector's centre would land on something /// OTHER than the element (an overlay on top), i.e. the element is occluded. /// `false` when not occluded, the element is missing, or the probe fails (so we /// never block a click on a flaky probe — the normal coordinate path runs). async fn point_misses_element(client: &CdpClient, session_id: &str, selector: &str) -> bool { let js = format!( r#"(() => {{ const el = document.querySelector({sel}); if (!el) return false; const r = el.getBoundingClientRect(); if (r.width === 0 || r.height === 0) return false; const hit = document.elementFromPoint(r.left + r.width / 2, r.top + r.height / 2); if (!hit) return false; // Not occluded if the hit is the element, a descendant, or an ancestor // wrapper (clicking those still reaches the element's handlers). return !(hit === el || el.contains(hit) || hit.contains(el)); }})()"#, sel = serde_json::to_string(selector).unwrap_or_default() ); match client .send_command_typed::<_, EvaluateResult>( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(true), await_promise: Some(false), }, Some(session_id), ) .await { Ok(r) => r.result.value.and_then(|v| v.as_bool()).unwrap_or(false), Err(_) => false, } } /// Best-effort scroll-into-view before a coordinate click. Uses Chrome's /// `scrollIntoViewIfNeeded` (only scrolls when not already fully visible), /// falling back to centered `scrollIntoView`. Resolution failures are ignored — /// the subsequent resolve will surface a real "not found" error. async fn scroll_into_view_if_needed( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) { let Ok((object_id, effective_session_id)) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await else { return; }; let js = "function() { try { \ if (typeof this.scrollIntoViewIfNeeded === 'function') { this.scrollIntoViewIfNeeded(true); } \ else { this.scrollIntoView({ block: 'center', inline: 'center' }); } \ } catch (e) {} }"; let _ = client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js.to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await; // Let the scroll settle so the following getBoxModel sees final coordinates. wait_for_paint_settled(client, &effective_session_id).await; } /// Dispatch a click through the DOM (`element.click()`) instead of via screen /// coordinates. Targets the intended element directly, so it works when a /// floating layer occludes the click point or the element sits in a portal that /// confuses `elementFromPoint`. Used as the fallback for `click` and when /// `AGENT_BROWSER_CLICK_MODE=dom`. async fn dom_click( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { this.click(); }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; wait_for_paint_settled(client, &effective_session_id).await; Ok(()) } /// Trusted activation of an element inside an iframe (issue #39). Focuses the /// element in its own frame session, then dispatches a real Enter/Space on the /// page session — Chrome routes the key to the focused element across frames, and /// Enter/Space on a focused button/link/checkbox fires a `click` with /// `isTrusted: true`, which security-sensitive embedded forms (Google Payments /// `保存`) require. Non-activatable roles (a `div[onclick]`) can't be keyboard- /// activated, so they fall back to a DOM `.click()`. async fn dom_activate( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let role = parse_ref(selector_or_ref) .and_then(|r| ref_map.get(&r).map(|e| e.role.clone())) .unwrap_or_default(); // Space toggles checkbox-like controls; Enter activates buttons/links/menus. let key = match role.as_str() { "checkbox" | "radio" | "switch" | "option" | "menuitemcheckbox" | "menuitemradio" => { Some("space") } "button" | "link" | "menuitem" | "tab" | "treeitem" => Some("enter"), _ => None, }; let Some(key) = key else { // Not keyboard-activatable — best effort via DOM .click() (untrusted). return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; }; let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; // Focus the element in its OWN frame session so the keystroke lands on it. client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { this.focus(); }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; // Trusted key on the page session — routed to the focused (in-frame) element. press_key(client, session_id, key).await?; wait_for_paint_settled(client, &effective_session_id).await; Ok(()) } /// DOM-dispatch a double-click on the element in its own session (no coordinates) /// — the relay/iframe-safe counterpart to a coordinate dblclick. Fires the full /// click,click,dblclick sequence so handlers bound to any of them respond. async fn dom_dblclick( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { const opts = { bubbles: true, cancelable: true, view: window }; this.dispatchEvent(new MouseEvent('click', opts)); this.dispatchEvent(new MouseEvent('click', { ...opts, detail: 2 })); this.dispatchEvent(new MouseEvent('dblclick', opts)); }"# .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; wait_for_paint_settled(client, &effective_session_id).await; Ok(()) } pub async fn dblclick( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { // Same relay/iframe drift hazard as a single click — DOM-dispatch the // double-click there instead of a coordinate one (issues #31/#36). if std::env::var("AGENT_BROWSER_CLICK_MODE").as_deref() != Ok("coord") && prefer_dom_dispatch(ref_map, selector_or_ref) { return dom_dblclick( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } click( client, session_id, ref_map, selector_or_ref, "left", 2, iframe_sessions, ) .await } /// DOM-dispatch a hover (pointer/mouse enter+move) on the element in its own /// session — reaches OOPIF elements and never drifts to the foreground tab over /// the relay, unlike a coordinate `mouseMoved` (issues #31/#36). async fn dom_hover( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { const r = this.getBoundingClientRect(); const cx = r.left + r.width / 2, cy = r.top + r.height / 2; const base = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy }; this.dispatchEvent(new PointerEvent('pointerover', base)); this.dispatchEvent(new PointerEvent('pointerenter', { ...base, bubbles: false })); this.dispatchEvent(new MouseEvent('mouseover', base)); this.dispatchEvent(new MouseEvent('mouseenter', { ...base, bubbles: false })); this.dispatchEvent(new MouseEvent('mousemove', base)); }"# .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(()) } pub async fn hover( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { // Coordinate `mouseMoved` drifts to the foreground tab over the relay and // can't reach an OOPIF — DOM-dispatch the hover there (issues #31/#36). if prefer_dom_dispatch(ref_map, selector_or_ref) { return dom_hover( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } let (x, y, _w, _h, effective_session_id) = resolve_element_center( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Input.dispatchMouseEvent", &DispatchMouseEventParams { event_type: "mouseMoved".to_string(), x, y, button: None, buttons: None, click_count: None, delta_x: None, delta_y: None, modifiers: None, }, Some(&effective_session_id), ) .await?; Ok(()) } /// DOM-dispatch an HTML5 drag-and-drop from `source` to `target` in their shared /// session — the relay/iframe-safe counterpart to the coordinate drag, which /// drifts to the foreground tab over the relay and can't reach an OOPIF (issues /// #31/#36). Covers HTML5 DnD (sortable lists, file/card boards); pointer-driven /// drag (canvas, sliders) still needs the coordinate path. Errors if source and /// target live in different frames — a synthetic cross-frame DnD isn't reliable. pub async fn dom_drag( client: &CdpClient, session_id: &str, ref_map: &RefMap, source: &str, target: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (src_obj, src_session) = resolve_element_object_id(client, session_id, ref_map, source, iframe_sessions).await?; let (tgt_obj, tgt_session) = resolve_element_object_id(client, session_id, ref_map, target, iframe_sessions).await?; if src_session != tgt_session { return Err( "drag source and target are in different frames; cross-frame drag-and-drop over the \ relay isn't supported — drag within a single frame, or use a launched browser with \ AGENT_BROWSER_CLICK_MODE=coord" .to_string(), ); } client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function(target) { const dt = new DataTransfer(); const ev = (type, el) => el.dispatchEvent( new DragEvent(type, { bubbles: true, cancelable: true, dataTransfer: dt })); ev('dragstart', this); ev('drag', this); ev('dragenter', target); ev('dragover', target); ev('drop', target); ev('dragend', this); }"# .to_string(), object_id: Some(src_obj), arguments: Some(vec![CallArgument { value: None, object_id: Some(tgt_obj), }]), return_by_value: Some(true), await_promise: Some(false), }, Some(&src_session), ) .await?; wait_for_paint_settled(client, &src_session).await; Ok(()) } pub async fn fill( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, value: &str, iframe_sessions: &HashMap, ) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; // Emulate a real edit so framework-controlled inputs (React/Vue) and // site-side listeners actually see the change (issue #25): set the value // through the element's PROTOTYPE setter (which React's _valueTracker hooks), // then dispatch input → change → blur/focusout. Beyond plain inputs, detect // rich editors and use their own API/events (issue #41): CodeMirror 5 and // Monaco have a model that `.value`/`textContent` can't touch; ProseMirror / // contenteditable need `execCommand('insertText')` so beforeinput/input fire // (a raw `textContent =` corrupts PM's doc and skips React composers). // Returns the engine used so the caller can report it. `type ` // remains for sites that need per-keystroke events. let fill_js = format!( r#"function() {{ const el = this; const v = {val}; try {{ el.focus(); }} catch (e) {{}} const tag = el.tagName; const fire = (type, ctor) => el.dispatchEvent(new (ctor || Event)(type, {{ bubbles: true }})); // CodeMirror 5: a hidden