use std::collections::HashMap; use serde_json::Value; use super::cdp::client::CdpClient; use super::cdp::types::*; use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap}; use super::humanize; pub async fn click( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, button: &str, click_count: i32, iframe_sessions: &HashMap, ) -> Result<(), String> { // AGENT_BROWSER_CLICK_MODE: "" (default) = coordinate click with a DOM // fallback; "coord" = strict coordinate only (no fallback); "dom" = always // dispatch through the DOM. let mode = std::env::var("AGENT_BROWSER_CLICK_MODE").unwrap_or_default(); // (A) Scroll the target into view first so the computed coordinates land // inside the viewport. Without this, an element below the fold (or revealed // after scroll/popup) yields off-viewport coordinates and the click lands on // whatever currently occupies that point. Best-effort: ignore failures. scroll_into_view_if_needed( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; if mode == "dom" { return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } let resolved = resolve_element_center( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; match resolved { Ok((cx, cy, w, h, effective_session_id)) => { // Occlusion guard for the CSS-selector path. `@ref` clicks are already // occlusion-checked in resolve_element_center, but a plain selector // resolves to coordinates without that check — so an overlay (modal // backdrop, sticky banner, the getByText located node sitting under a // full-screen layer) would make the coordinate click land on the // overlay and still report success. If the click point doesn't hit the // target, dispatch through the DOM instead (targets the element // directly). Skipped for strict `coord` mode and non-left/multi-clicks. if mode != "coord" && button == "left" && click_count == 1 && parse_ref(selector_or_ref).is_none() && point_misses_element(client, &effective_session_id, selector_or_ref).await { eprintln!( "[click] target occluded at its click point; dispatching through \ the DOM (set AGENT_BROWSER_CLICK_MODE=coord to disable)" ); return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } // Land on a jittered point inside the element rather than its exact // centre (Fast/Human). Zero size or Off → exact centre. let (tx, ty) = humanize::landing_point( (cx - w / 2.0, cy - h / 2.0, w, h), humanize::active_level(), humanize::next_seed(), ); dispatch_click(client, &effective_session_id, tx, ty, button, click_count).await } Err(e) => { // (B) The coordinate path failed — typically a persistent overlay // failing the occlusion guard, or coordinates that won't resolve. // Fall back to a DOM-dispatched `.click()` on the intended element, // which targets the element directly instead of a screen point. // Skipped for strict "coord" mode and for non-left / multi-clicks // (a DOM `.click()` can't express right/middle/double semantics). if mode == "coord" || button != "left" || click_count != 1 { return Err(e); } eprintln!( "[click] coordinate click failed ({e}); falling back to DOM dispatch \ (set AGENT_BROWSER_CLICK_MODE=coord to disable)" ); dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await .map_err(|dom_err| format!("{e}\n(DOM-dispatch fallback also failed: {dom_err})")) } } } /// True if a coordinate click at the selector's centre would land on something /// OTHER than the element (an overlay on top), i.e. the element is occluded. /// `false` when not occluded, the element is missing, or the probe fails (so we /// never block a click on a flaky probe — the normal coordinate path runs). async fn point_misses_element(client: &CdpClient, session_id: &str, selector: &str) -> bool { let js = format!( r#"(() => {{ const el = document.querySelector({sel}); if (!el) return false; const r = el.getBoundingClientRect(); if (r.width === 0 || r.height === 0) return false; const hit = document.elementFromPoint(r.left + r.width / 2, r.top + r.height / 2); if (!hit) return false; // Not occluded if the hit is the element, a descendant, or an ancestor // wrapper (clicking those still reaches the element's handlers). return !(hit === el || el.contains(hit) || hit.contains(el)); }})()"#, sel = serde_json::to_string(selector).unwrap_or_default() ); match client .send_command_typed::<_, EvaluateResult>( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(true), await_promise: Some(false), }, Some(session_id), ) .await { Ok(r) => r.result.value.and_then(|v| v.as_bool()).unwrap_or(false), Err(_) => false, } } /// Best-effort scroll-into-view before a coordinate click. Uses Chrome's /// `scrollIntoViewIfNeeded` (only scrolls when not already fully visible), /// falling back to centered `scrollIntoView`. Resolution failures are ignored — /// the subsequent resolve will surface a real "not found" error. async fn scroll_into_view_if_needed( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) { let Ok((object_id, effective_session_id)) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await else { return; }; let js = "function() { try { \ if (typeof this.scrollIntoViewIfNeeded === 'function') { this.scrollIntoViewIfNeeded(true); } \ else { this.scrollIntoView({ block: 'center', inline: 'center' }); } \ } catch (e) {} }"; let _ = client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js.to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await; // Let the scroll settle so the following getBoxModel sees final coordinates. wait_for_paint_settled(client, &effective_session_id).await; } /// Dispatch a click through the DOM (`element.click()`) instead of via screen /// coordinates. Targets the intended element directly, so it works when a /// floating layer occludes the click point or the element sits in a portal that /// confuses `elementFromPoint`. Used as the fallback for `click` and when /// `AGENT_BROWSER_CLICK_MODE=dom`. async fn dom_click( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { this.click(); }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; wait_for_paint_settled(client, &effective_session_id).await; Ok(()) } pub async fn dblclick( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { click( client, session_id, ref_map, selector_or_ref, "left", 2, iframe_sessions, ) .await } pub async fn hover( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (x, y, _w, _h, effective_session_id) = resolve_element_center( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Input.dispatchMouseEvent", &DispatchMouseEventParams { event_type: "mouseMoved".to_string(), x, y, button: None, buttons: None, click_count: None, delta_x: None, delta_y: None, modifiers: None, }, Some(&effective_session_id), ) .await?; Ok(()) } pub async fn fill( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, value: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; // Focus the element client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { this.focus(); }".to_string(), object_id: Some(object_id.clone()), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; // Select all + delete to clear client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { this.select && this.select(); this.value = ''; this.dispatchEvent(new Event('input', { bubbles: true })); }"# .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; // Insert text (keyboard input dispatched at page level, use parent session_id) client .send_command_typed::<_, Value>( "Input.insertText", &InsertTextParams { text: value.to_string(), }, Some(session_id), ) .await?; Ok(()) } #[allow(clippy::too_many_arguments)] pub async fn type_text( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, text: &str, clear: bool, delay_ms: Option, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; // Focus client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { this.focus(); }".to_string(), object_id: Some(object_id.clone()), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; if clear { client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { this.select && this.select(); this.value = ''; this.dispatchEvent(new Event('input', { bubbles: true })); }"# .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; } type_text_into_active_context(client, session_id, text, delay_ms).await } pub async fn type_text_into_active_context( client: &CdpClient, session_id: &str, text: &str, delay_ms: Option, ) -> Result<(), String> { // Per-character timing: an explicit `delay_ms` wins (caller asked for a // fixed cadence); otherwise fall back to humanize — variable, human-like // inter-keystroke gaps at Fast/Human, all-zero (instant) at Off. let chars: Vec = text.chars().collect(); let cadence: Vec = match delay_ms { Some(d) => vec![std::time::Duration::from_millis(d); chars.len()], None => { humanize::keystroke_delays(chars.len(), humanize::active_level(), humanize::next_seed()) } }; for (i, ch) in chars.into_iter().enumerate() { if matches!(ch, '\n' | '\r' | '\t') { let (key, code, key_code) = char_to_key_info(ch); let text_str = key_text(&key); client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyDown".to_string(), key: Some(key.clone()), code: Some(code.clone()), text: text_str.clone(), unmodified_text: text_str, windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers: None, }, Some(session_id), ) .await?; client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyUp".to_string(), key: Some(key), code: Some(code), text: None, unmodified_text: None, windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers: None, }, Some(session_id), ) .await?; } else { // VS Code/Electron webviews reject repeated dispatchKeyEvent calls // carrying printable `text`. Insert printable characters directly // and reserve key events for controls like Enter and Tab. client .send_command_typed::<_, Value>( "Input.insertText", &InsertTextParams { text: ch.to_string(), }, Some(session_id), ) .await?; } let gap = cadence[i]; if !gap.is_zero() { tokio::time::sleep(gap).await; } } Ok(()) } pub async fn press_key(client: &CdpClient, session_id: &str, key: &str) -> Result<(), String> { press_key_with_modifiers(client, session_id, key, None).await } /// Dispatch a keyDown+keyUp sequence for `key` with an optional CDP modifier bitmask. /// /// Modifier values follow the CDP `Input.dispatchKeyEvent` spec: /// 1 = Alt, 2 = Control, 4 = Meta (Cmd), 8 = Shift. /// /// Callers that need a platform-appropriate modifier (e.g. Cmd on macOS, /// Ctrl elsewhere) must choose the value themselves -- see `cfg!(target_os)`. pub async fn press_key_with_modifiers( client: &CdpClient, session_id: &str, key: &str, modifiers: Option, ) -> Result<(), String> { let (key_name, code, key_code) = named_key_info(key); // Suppress text insertion when Control (2) or Meta (4) modifiers are active, // since these are command chords (e.g. Ctrl+A = select-all), not text input. let has_command_modifier = modifiers.is_some_and(|m| m & (2 | 4) != 0); let text = if has_command_modifier { None } else { key_text(&key_name) }; client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyDown".to_string(), key: Some(key_name.clone()), code: Some(code.clone()), text: text.clone(), unmodified_text: text.clone(), windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers, }, Some(session_id), ) .await?; client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyUp".to_string(), key: Some(key_name), code: Some(code), text: None, unmodified_text: None, windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers, }, Some(session_id), ) .await?; Ok(()) } pub async fn scroll( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: Option<&str>, delta_x: f64, delta_y: f64, iframe_sessions: &HashMap, ) -> Result<(), String> { if let Some(sel) = selector_or_ref { let (object_id, effective_session_id) = resolve_element_object_id(client, session_id, ref_map, sel, iframe_sessions).await?; let js = "function(dx, dy) { this.scrollBy(dx, dy); }".to_string(); client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js, object_id: Some(object_id), arguments: Some(vec![ CallArgument { value: Some(serde_json::json!(delta_x)), object_id: None, }, CallArgument { value: Some(serde_json::json!(delta_y)), object_id: None, }, ]), return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; } else { let js = format!("window.scrollBy({}, {})", delta_x, delta_y); client .send_command_typed::<_, Value>( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(true), await_promise: Some(false), }, Some(session_id), ) .await?; } Ok(()) } pub async fn select_option( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, values: &[String], iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let js = r#"function(vals) { const options = Array.from(this.options); for (const opt of options) { opt.selected = vals.includes(opt.value) || vals.includes(opt.textContent.trim()); } this.dispatchEvent(new Event('change', { bubbles: true })); }"# .to_string(); client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js, object_id: Some(object_id), arguments: Some(vec![CallArgument { value: Some(serde_json::json!(values)), object_id: None, }]), return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(()) } pub async fn check( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let is_checked = super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; if !is_checked { click( client, session_id, ref_map, selector_or_ref, "left", 1, iframe_sessions, ) .await?; // Verify the click changed the state (Playwright parity: _setChecked re-checks). // If the coordinate-based click missed (e.g. hidden input, overlay), retry // with a JS .click() on the element and its associated input. if !super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await? { js_click_checkbox( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; } } Ok(()) } pub async fn uncheck( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let is_checked = super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; if is_checked { click( client, session_id, ref_map, selector_or_ref, "left", 1, iframe_sessions, ) .await?; // Same verify-and-retry as check(). if super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await? { js_click_checkbox( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; } } Ok(()) } /// Fallback for when the coordinate-based CDP click did not toggle the /// checkbox/radio state. This mirrors how Playwright dispatches clicks /// through the DOM rather than via raw Input.dispatchMouseEvent coordinates. /// /// Uses the same follow-label resolution as `is_element_checked`: /// 1. If the element is a native input → `.click()` it directly. /// 2. If the element is inside a `