use std::collections::HashMap; use serde_json::Value; use super::cdp::client::CdpClient; use super::cdp::types::*; use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap}; use super::humanize; pub async fn click( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, button: &str, click_count: i32, iframe_sessions: &HashMap, ) -> Result<(), String> { // AGENT_BROWSER_CLICK_MODE: "" (default) = coordinate click with a DOM // fallback; "coord" = strict coordinate only (no fallback); "dom" = always // dispatch through the DOM. let mode = std::env::var("AGENT_BROWSER_CLICK_MODE").unwrap_or_default(); // (A) Scroll the target into view first so the computed coordinates land // inside the viewport. Without this, an element below the fold (or revealed // after scroll/popup) yields off-viewport coordinates and the click lands on // whatever currently occupies that point. Best-effort: ignore failures. scroll_into_view_if_needed( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; if mode == "dom" { return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } // Over the extension relay we drive the user's real, in-use Chrome, where a // coordinate `Input.dispatchMouseEvent` is NOT reliably confined to our target // tab — it can be delivered to whatever tab is in the foreground, and an OOPIF // element's box can't be mapped to a top-viewport point at all. This twice // opened an unrelated tab on the user's busy Chrome (issues #31/#36). So on the // relay, never use coordinates for a normal left click: DOM-dispatch invokes // the element's click in its own (frame) session, always hitting the right // element in the right tab. Double/right clicks still need true pointer // semantics, and `coord` mode is an explicit opt-out. if mode != "coord" && button == "left" && click_count == 1 { let in_iframe = parse_ref(selector_or_ref) .and_then(|r| ref_map.get(&r).map(|e| e.frame_id.is_some())) .unwrap_or(false); if in_iframe || crate::connect::relay_url().is_some() { return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } } let resolved = resolve_element_center( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; match resolved { Ok((cx, cy, w, h, effective_session_id)) => { // Occlusion guard for the CSS-selector path. `@ref` clicks are already // occlusion-checked in resolve_element_center, but a plain selector // resolves to coordinates without that check — so an overlay (modal // backdrop, sticky banner, the getByText located node sitting under a // full-screen layer) would make the coordinate click land on the // overlay and still report success. If the click point doesn't hit the // target, dispatch through the DOM instead (targets the element // directly). Skipped for strict `coord` mode and non-left/multi-clicks. if mode != "coord" && button == "left" && click_count == 1 && parse_ref(selector_or_ref).is_none() && point_misses_element(client, &effective_session_id, selector_or_ref).await { eprintln!( "[click] target occluded at its click point; dispatching through \ the DOM (set AGENT_BROWSER_CLICK_MODE=coord to disable)" ); return dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await; } // Land on a jittered point inside the element rather than its exact // centre (Fast/Human). Zero size or Off → exact centre. let (tx, ty) = humanize::landing_point( (cx - w / 2.0, cy - h / 2.0, w, h), humanize::active_level(), humanize::next_seed(), ); dispatch_click(client, &effective_session_id, tx, ty, button, click_count).await } Err(e) => { // (B) The coordinate path failed — typically a persistent overlay // failing the occlusion guard, or coordinates that won't resolve. // Fall back to a DOM-dispatched `.click()` on the intended element, // which targets the element directly instead of a screen point. // Skipped for strict "coord" mode and for non-left / multi-clicks // (a DOM `.click()` can't express right/middle/double semantics). if mode == "coord" || button != "left" || click_count != 1 { return Err(e); } eprintln!( "[click] coordinate click failed ({e}); falling back to DOM dispatch \ (set AGENT_BROWSER_CLICK_MODE=coord to disable)" ); dom_click( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await .map_err(|dom_err| format!("{e}\n(DOM-dispatch fallback also failed: {dom_err})")) } } } /// True if a coordinate click at the selector's centre would land on something /// OTHER than the element (an overlay on top), i.e. the element is occluded. /// `false` when not occluded, the element is missing, or the probe fails (so we /// never block a click on a flaky probe — the normal coordinate path runs). async fn point_misses_element(client: &CdpClient, session_id: &str, selector: &str) -> bool { let js = format!( r#"(() => {{ const el = document.querySelector({sel}); if (!el) return false; const r = el.getBoundingClientRect(); if (r.width === 0 || r.height === 0) return false; const hit = document.elementFromPoint(r.left + r.width / 2, r.top + r.height / 2); if (!hit) return false; // Not occluded if the hit is the element, a descendant, or an ancestor // wrapper (clicking those still reaches the element's handlers). return !(hit === el || el.contains(hit) || hit.contains(el)); }})()"#, sel = serde_json::to_string(selector).unwrap_or_default() ); match client .send_command_typed::<_, EvaluateResult>( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(true), await_promise: Some(false), }, Some(session_id), ) .await { Ok(r) => r.result.value.and_then(|v| v.as_bool()).unwrap_or(false), Err(_) => false, } } /// Best-effort scroll-into-view before a coordinate click. Uses Chrome's /// `scrollIntoViewIfNeeded` (only scrolls when not already fully visible), /// falling back to centered `scrollIntoView`. Resolution failures are ignored — /// the subsequent resolve will surface a real "not found" error. async fn scroll_into_view_if_needed( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) { let Ok((object_id, effective_session_id)) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await else { return; }; let js = "function() { try { \ if (typeof this.scrollIntoViewIfNeeded === 'function') { this.scrollIntoViewIfNeeded(true); } \ else { this.scrollIntoView({ block: 'center', inline: 'center' }); } \ } catch (e) {} }"; let _ = client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js.to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await; // Let the scroll settle so the following getBoxModel sees final coordinates. wait_for_paint_settled(client, &effective_session_id).await; } /// Dispatch a click through the DOM (`element.click()`) instead of via screen /// coordinates. Targets the intended element directly, so it works when a /// floating layer occludes the click point or the element sits in a portal that /// confuses `elementFromPoint`. Used as the fallback for `click` and when /// `AGENT_BROWSER_CLICK_MODE=dom`. async fn dom_click( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { this.click(); }".to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; wait_for_paint_settled(client, &effective_session_id).await; Ok(()) } pub async fn dblclick( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { click( client, session_id, ref_map, selector_or_ref, "left", 2, iframe_sessions, ) .await } pub async fn hover( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (x, y, _w, _h, effective_session_id) = resolve_element_center( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; client .send_command_typed::<_, Value>( "Input.dispatchMouseEvent", &DispatchMouseEventParams { event_type: "mouseMoved".to_string(), x, y, button: None, buttons: None, click_count: None, delta_x: None, delta_y: None, modifiers: None, }, Some(&effective_session_id), ) .await?; Ok(()) } pub async fn fill( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, value: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; // Emulate a real edit so framework-controlled inputs (React/Vue) and // site-side listeners actually see the change (issue #25): the old path set // `this.value` directly and used Input.insertText, which left React's // internal value-tracker out of sync and never fired change/blur — so // dependent logic (e.g. Mercari's postal-code → 都道府県 autocomplete) never // ran even though the value was visible. Set the value through the element's // PROTOTYPE setter (which React's _valueTracker hooks), then dispatch // input → change → blur/focusout. `type ` remains for sites that // need per-keystroke events. let fill_js = format!( r#"function() {{ const el = this; const v = {val}; try {{ el.focus(); }} catch (e) {{}} const tag = el.tagName; const fire = (type, ctor) => el.dispatchEvent(new (ctor || Event)(type, {{ bubbles: true }})); if (tag === 'SELECT') {{ el.value = v; fire('input'); fire('change'); return true; }} if (el.isContentEditable) {{ el.textContent = v; fire('input', window.InputEvent || Event); fire('change'); try {{ el.blur(); }} catch (e) {{}} fire('focusout'); return true; }} const proto = tag === 'TEXTAREA' ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype; const desc = Object.getOwnPropertyDescriptor(proto, 'value'); const set = desc && desc.set ? (x) => desc.set.call(el, x) : (x) => {{ el.value = x; }}; set(''); // reset the framework tracker fire('input', window.InputEvent || Event); set(v); // native setter → React/Vue registers fire('input', window.InputEvent || Event); fire('change'); try {{ el.blur(); }} catch (e) {{}} fire('focusout'); // blur-triggered lookups/validation return true; }}"#, val = serde_json::to_string(value).unwrap_or_default() ); client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: fill_js, object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(()) } #[allow(clippy::too_many_arguments)] pub async fn type_text( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, text: &str, clear: bool, delay_ms: Option, iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; // Focus client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: "function() { this.focus(); }".to_string(), object_id: Some(object_id.clone()), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; if clear { client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: r#"function() { this.select && this.select(); this.value = ''; this.dispatchEvent(new Event('input', { bubbles: true })); }"# .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; } type_text_into_active_context(client, session_id, text, delay_ms).await } pub async fn type_text_into_active_context( client: &CdpClient, session_id: &str, text: &str, delay_ms: Option, ) -> Result<(), String> { // Per-character timing: an explicit `delay_ms` wins (caller asked for a // fixed cadence); otherwise fall back to humanize — variable, human-like // inter-keystroke gaps at Fast/Human, all-zero (instant) at Off. let chars: Vec = text.chars().collect(); let cadence: Vec = match delay_ms { Some(d) => vec![std::time::Duration::from_millis(d); chars.len()], None => { humanize::keystroke_delays(chars.len(), humanize::active_level(), humanize::next_seed()) } }; for (i, ch) in chars.into_iter().enumerate() { if matches!(ch, '\n' | '\r' | '\t') { let (key, code, key_code) = char_to_key_info(ch); let text_str = key_text(&key); client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyDown".to_string(), key: Some(key.clone()), code: Some(code.clone()), text: text_str.clone(), unmodified_text: text_str, windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers: None, }, Some(session_id), ) .await?; client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyUp".to_string(), key: Some(key), code: Some(code), text: None, unmodified_text: None, windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers: None, }, Some(session_id), ) .await?; } else { // VS Code/Electron webviews reject repeated dispatchKeyEvent calls // carrying printable `text`. Insert printable characters directly // and reserve key events for controls like Enter and Tab. client .send_command_typed::<_, Value>( "Input.insertText", &InsertTextParams { text: ch.to_string(), }, Some(session_id), ) .await?; } let gap = cadence[i]; if !gap.is_zero() { tokio::time::sleep(gap).await; } } Ok(()) } pub async fn press_key(client: &CdpClient, session_id: &str, key: &str) -> Result<(), String> { press_key_with_modifiers(client, session_id, key, None).await } /// Dispatch a keyDown+keyUp sequence for `key` with an optional CDP modifier bitmask. /// /// Modifier values follow the CDP `Input.dispatchKeyEvent` spec: /// 1 = Alt, 2 = Control, 4 = Meta (Cmd), 8 = Shift. /// /// Callers that need a platform-appropriate modifier (e.g. Cmd on macOS, /// Ctrl elsewhere) must choose the value themselves -- see `cfg!(target_os)`. pub async fn press_key_with_modifiers( client: &CdpClient, session_id: &str, key: &str, modifiers: Option, ) -> Result<(), String> { let (key_name, code, key_code) = named_key_info(key); // Suppress text insertion when Control (2) or Meta (4) modifiers are active, // since these are command chords (e.g. Ctrl+A = select-all), not text input. let has_command_modifier = modifiers.is_some_and(|m| m & (2 | 4) != 0); let text = if has_command_modifier { None } else { key_text(&key_name) }; client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyDown".to_string(), key: Some(key_name.clone()), code: Some(code.clone()), text: text.clone(), unmodified_text: text.clone(), windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers, }, Some(session_id), ) .await?; client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: "keyUp".to_string(), key: Some(key_name), code: Some(code), text: None, unmodified_text: None, windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers, }, Some(session_id), ) .await?; Ok(()) } /// Dispatch a SINGLE key event (`keyDown` or `keyUp`) carrying the full key /// descriptor — `key`, `code`, `windowsVirtualKeyCode`/`nativeVirtualKeyCode`, /// and (on key-down) printable `text`. Powers the `keydown`/`keyup` commands. /// /// The previous implementation sent only `{key}`, so games and shortcut handlers /// that read `event.code` (e.g. `"KeyD"`, `"ArrowRight"`) or `event.keyCode` saw /// nothing — a held key set no movement flag and did nothing (dogfood: holding a /// direction in a canvas platformer barely nudged the player). Sending the same /// descriptor `press` uses makes hold-to-move work regardless of which field the /// page keys off. pub async fn dispatch_single_key( client: &CdpClient, session_id: &str, key: &str, event_type: &str, ) -> Result<(), String> { let (key_name, code, key_code) = named_key_info(key); // Printable text is only meaningful on key-down; key-up never inserts. let text = if event_type == "keyDown" { key_text(&key_name) } else { None }; client .send_command_typed::<_, Value>( "Input.dispatchKeyEvent", &DispatchKeyEventParams { event_type: event_type.to_string(), key: Some(key_name), code: Some(code), text: text.clone(), unmodified_text: text, windows_virtual_key_code: Some(key_code), native_virtual_key_code: Some(key_code), modifiers: None, }, Some(session_id), ) .await?; Ok(()) } pub async fn scroll( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: Option<&str>, delta_x: f64, delta_y: f64, iframe_sessions: &HashMap, ) -> Result<(), String> { if let Some(sel) = selector_or_ref { let (object_id, effective_session_id) = resolve_element_object_id(client, session_id, ref_map, sel, iframe_sessions).await?; let js = "function(dx, dy) { this.scrollBy(dx, dy); }".to_string(); client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js, object_id: Some(object_id), arguments: Some(vec![ CallArgument { value: Some(serde_json::json!(delta_x)), object_id: None, }, CallArgument { value: Some(serde_json::json!(delta_y)), object_id: None, }, ]), return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; } else { let js = format!("window.scrollBy({}, {})", delta_x, delta_y); client .send_command_typed::<_, Value>( "Runtime.evaluate", &EvaluateParams { expression: js, return_by_value: Some(true), await_promise: Some(false), }, Some(session_id), ) .await?; } Ok(()) } pub async fn select_option( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, values: &[String], iframe_sessions: &HashMap, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; let js = r#"function(vals) { const options = Array.from(this.options); for (const opt of options) { opt.selected = vals.includes(opt.value) || vals.includes(opt.textContent.trim()); } this.dispatchEvent(new Event('change', { bubbles: true })); }"# .to_string(); client .send_command_typed::<_, Value>( "Runtime.callFunctionOn", &CallFunctionOnParams { function_declaration: js, object_id: Some(object_id), arguments: Some(vec![CallArgument { value: Some(serde_json::json!(values)), object_id: None, }]), return_by_value: Some(true), await_promise: Some(false), }, Some(&effective_session_id), ) .await?; Ok(()) } pub async fn check( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let is_checked = super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; if !is_checked { click( client, session_id, ref_map, selector_or_ref, "left", 1, iframe_sessions, ) .await?; // Verify the click changed the state (Playwright parity: _setChecked re-checks). // If the coordinate-based click missed (e.g. hidden input, overlay), retry // with a JS .click() on the element and its associated input. if !super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await? { js_click_checkbox( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; } } Ok(()) } pub async fn uncheck( client: &CdpClient, session_id: &str, ref_map: &RefMap, selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { let is_checked = super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; if is_checked { click( client, session_id, ref_map, selector_or_ref, "left", 1, iframe_sessions, ) .await?; // Same verify-and-retry as check(). if super::element::is_element_checked( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await? { js_click_checkbox( client, session_id, ref_map, selector_or_ref, iframe_sessions, ) .await?; } } Ok(()) } /// Fallback for when the coordinate-based CDP click did not toggle the /// checkbox/radio state. This mirrors how Playwright dispatches clicks /// through the DOM rather than via raw Input.dispatchMouseEvent coordinates. /// /// Uses the same follow-label resolution as `is_element_checked`: /// 1. If the element is a native input → `.click()` it directly. /// 2. If the element is inside a `