diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 355d623..de95c9e 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -5513,19 +5513,29 @@ async fn handle_wheel(cmd: &Value, state: &DaemonState) -> Result let delta_x = cmd.get("deltaX").and_then(|v| v.as_f64()).unwrap_or(0.0); let delta_y = cmd.get("deltaY").and_then(|v| v.as_f64()).unwrap_or(0.0); - mgr.client - .send_command( - "Input.dispatchMouseEvent", - Some(json!({ - "type": "mouseWheel", - "x": x, - "y": y, - "deltaX": delta_x, - "deltaY": delta_y, - })), - Some(&session_id), - ) - .await?; + // Humanize: at Off this is one instant wheel event (unchanged); at + // Fast/Human the scroll is split into eased, slightly-jittered segments so + // it ramps and settles like a real wheel/trackpad flick. + let level = humanize::active_level(); + let seed = humanize::next_seed(); + for (dx, dy, delay) in humanize::scroll_segments(delta_x, delta_y, level, seed) { + mgr.client + .send_command( + "Input.dispatchMouseEvent", + Some(json!({ + "type": "mouseWheel", + "x": x, + "y": y, + "deltaX": dx, + "deltaY": dy, + })), + Some(&session_id), + ) + .await?; + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + } Ok(json!({ "scrolled": true, "deltaX": delta_x, "deltaY": delta_y })) } @@ -6522,7 +6532,7 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result Result Result = + if matches!(level, humanize::HumanizeLevel::Off) { + (1..=10) + .map(|i| { + let cx = sx + (tx - sx) * (i as f64) / 10.0; + let cy = sy + (ty - sy) * (i as f64) / 10.0; + (cx, cy, std::time::Duration::from_millis(10)) + }) + .collect() + } else { + humanize::move_path((sx, sy), (tx, ty), level, humanize::next_seed()) + .into_iter() + .map(|s| (s.x, s.y, s.delay)) + .collect() + }; + for (cx, cy, delay) in drag_path { mgr.client .send_command( "Input.dispatchMouseEvent", @@ -6568,7 +6592,9 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result, -) -> Result<(f64, f64, String), String> { +) -> Result<(f64, f64, f64, f64, String), String> { if let Some(ref_id) = parse_ref(selector_or_ref) { let entry = ref_map .get(&ref_id) @@ -263,7 +267,7 @@ pub async fn resolve_element_center( .await; if let Ok(r) = result { - let (x, y) = box_model_center(&r.model); + let (x, y, w, h) = box_model_dims(&r.model); // Occlusion check: a transient overlay (X.com's "click // outside to close" mask, modal backdrop, sticky banner, // etc.) can land on top of our target between snapshot @@ -279,7 +283,7 @@ pub async fn resolve_element_center( verify_click_target(client, effective_session_id, active_id, &ref_id, x, y) .await?; } - return Ok((x, y, effective_session_id.to_string())); + return Ok((x, y, w, h, effective_session_id.to_string())); } // backend_node_id is stale; re-query the accessibility tree below } @@ -316,13 +320,14 @@ pub async fn resolve_element_center( Some(effective_session_id), ) .await?; - let (x, y) = box_model_center(&result.model); - return Ok((x, y, effective_session_id.to_string())); + let (x, y, w, h) = box_model_dims(&result.model); + return Ok((x, y, w, h, effective_session_id.to_string())); } // CSS selector let (x, y) = resolve_by_selector(client, session_id, selector_or_ref).await?; - Ok((x, y, session_id.to_string())) + // No box model on the CSS-selector fast path → zero size → land on centre. + Ok((x, y, 0.0, 0.0, session_id.to_string())) } pub async fn resolve_element_object_id( @@ -872,6 +877,35 @@ fn box_model_center(model: &BoxModel) -> (f64, f64) { } } +/// Centre plus width/height of the content box, derived from the quad's +/// bounding extent. Width/height feed humanize's in-bounds landing jitter; a +/// degenerate quad yields zero size, which the jitter treats as "land on +/// centre" (no jitter). +fn box_model_dims(model: &BoxModel) -> (f64, f64, f64, f64) { + let (cx, cy) = box_model_center(model); + if model.content.len() >= 8 { + let xs = [ + model.content[0], + model.content[2], + model.content[4], + model.content[6], + ]; + let ys = [ + model.content[1], + model.content[3], + model.content[5], + model.content[7], + ]; + let w = xs.iter().cloned().fold(f64::MIN, f64::max) + - xs.iter().cloned().fold(f64::MAX, f64::min); + let h = ys.iter().cloned().fold(f64::MIN, f64::max) + - ys.iter().cloned().fold(f64::MAX, f64::min); + (cx, cy, w.max(0.0), h.max(0.0)) + } else { + (cx, cy, 0.0, 0.0) + } +} + pub async fn get_element_text( client: &CdpClient, session_id: &str, diff --git a/cli/src/native/humanize.rs b/cli/src/native/humanize.rs index b08f30e..e09b11b 100644 --- a/cli/src/native/humanize.rs +++ b/cli/src/native/humanize.rs @@ -277,6 +277,41 @@ pub fn move_path( out } +/// Split a wheel scroll of (`total_dx`, `total_dy`) into eased segments. `Off` +/// returns a single instant segment (today's one-shot scroll); `Fast`/`Human` +/// break it into several accelerate-then-decelerate chunks with small, +/// jittered inter-segment delays, the way a trackpad/wheel flick actually +/// lands. The segment deltas always sum to the requested total. +pub fn scroll_segments( + total_dx: f64, + total_dy: f64, + level: HumanizeLevel, + seed: u64, +) -> Vec<(f64, f64, Duration)> { + if level.is_off() { + return vec![(total_dx, total_dy, Duration::ZERO)]; + } + let (segs, base_ms) = match level { + HumanizeLevel::Fast => (4usize, 18.0), + _ => (9usize, 28.0), + }; + let mut rng = Rng::new(seed); + let mut out = Vec::with_capacity(segs); + let mut prev = 0.0; + for i in 1..=segs { + let f = ease(i as f64 / segs as f64); + let frac = f - prev; + prev = f; + let jitter = 1.0 + 0.3 * rng.signed(); + out.push(( + total_dx * frac, + total_dy * frac, + Duration::from_millis((base_ms * jitter).max(4.0) as u64), + )); + } + out +} + /// Dwell between `mousePressed` and `mouseReleased` (a real click isn't /// instantaneous). Zero for `Off`. pub fn press_dwell(level: HumanizeLevel, seed: u64) -> Duration { @@ -435,6 +470,23 @@ mod tests { assert!(human.iter().all(|d| *d >= Duration::from_millis(8))); } + #[test] + fn scroll_segments_sum_to_total_and_single_when_off() { + let off = scroll_segments(0.0, 600.0, HumanizeLevel::Off, 1); + assert_eq!(off.len(), 1); + assert_eq!((off[0].0, off[0].1), (0.0, 600.0)); + assert_eq!(off[0].2, Duration::ZERO); + + let human = scroll_segments(0.0, 600.0, HumanizeLevel::Human, 5); + assert!(human.len() >= 5); + let total_dy: f64 = human.iter().map(|s| s.1).sum(); + assert!( + (total_dy - 600.0).abs() < 1e-6, + "segments must sum to total" + ); + assert!(human.iter().all(|s| s.2 >= Duration::from_millis(4))); + } + #[test] fn detect_escalates_on_known_vendor_else_baseline() { let mut s = DetectSignals::default(); diff --git a/cli/src/native/interaction.rs b/cli/src/native/interaction.rs index ed701e9..2ffa46f 100644 --- a/cli/src/native/interaction.rs +++ b/cli/src/native/interaction.rs @@ -55,8 +55,15 @@ pub async fn click( .await; match resolved { - Ok((x, y, effective_session_id)) => { - dispatch_click(client, &effective_session_id, x, y, button, click_count).await + Ok((cx, cy, w, h, effective_session_id)) => { + // Land on a jittered point inside the element rather than its exact + // centre (Fast/Human). Zero size or Off → exact centre. + let (tx, ty) = humanize::landing_point( + (cx - w / 2.0, cy - h / 2.0, w, h), + humanize::active_level(), + humanize::next_seed(), + ); + dispatch_click(client, &effective_session_id, tx, ty, button, click_count).await } Err(e) => { // (B) The coordinate path failed — typically a persistent overlay @@ -191,7 +198,7 @@ pub async fn hover( selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { - let (x, y, effective_session_id) = resolve_element_center( + let (x, y, _w, _h, effective_session_id) = resolve_element_center( client, session_id, ref_map, @@ -999,7 +1006,7 @@ pub async fn tap_touch( selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { - let (x, y, effective_session_id) = resolve_element_center( + let (x, y, _w, _h, effective_session_id) = resolve_element_center( client, session_id, ref_map,