feat(stealth): in-bbox landing jitter + eased wheel/drag (humanize v4)
Completes the humanize suite: - Clicks land on a jittered point inside the element's box (Fast/Human) instead of its exact centre. `resolve_element_center` now also returns the element width/height (box_model_dims); the CSS-selector path reports zero size → land on centre (no jitter, no regression). Jitter is clamped to the inner box so the click never misses. - Wheel scrolls split into eased, jittered segments (humanize::scroll_segments, unit-tested) instead of one instant jump. - Drag follows the curved trajectory at Fast/Human (linear 10-step at Off). Off is unchanged throughout. 9/9 unit tests; verified headless — jittered click still lands (→ iana.org), segmented scroll moves the page.
This commit is contained in:
+48
-22
@@ -5513,19 +5513,29 @@ async fn handle_wheel(cmd: &Value, state: &DaemonState) -> Result<Value, String>
|
|||||||
let delta_x = cmd.get("deltaX").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
let delta_x = cmd.get("deltaX").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
let delta_y = cmd.get("deltaY").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
let delta_y = cmd.get("deltaY").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
|
||||||
mgr.client
|
// Humanize: at Off this is one instant wheel event (unchanged); at
|
||||||
.send_command(
|
// Fast/Human the scroll is split into eased, slightly-jittered segments so
|
||||||
"Input.dispatchMouseEvent",
|
// it ramps and settles like a real wheel/trackpad flick.
|
||||||
Some(json!({
|
let level = humanize::active_level();
|
||||||
"type": "mouseWheel",
|
let seed = humanize::next_seed();
|
||||||
"x": x,
|
for (dx, dy, delay) in humanize::scroll_segments(delta_x, delta_y, level, seed) {
|
||||||
"y": y,
|
mgr.client
|
||||||
"deltaX": delta_x,
|
.send_command(
|
||||||
"deltaY": delta_y,
|
"Input.dispatchMouseEvent",
|
||||||
})),
|
Some(json!({
|
||||||
Some(&session_id),
|
"type": "mouseWheel",
|
||||||
)
|
"x": x,
|
||||||
.await?;
|
"y": y,
|
||||||
|
"deltaX": dx,
|
||||||
|
"deltaY": dy,
|
||||||
|
})),
|
||||||
|
Some(&session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if !delay.is_zero() {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok(json!({ "scrolled": true, "deltaX": delta_x, "deltaY": delta_y }))
|
Ok(json!({ "scrolled": true, "deltaX": delta_x, "deltaY": delta_y }))
|
||||||
}
|
}
|
||||||
@@ -6522,7 +6532,7 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'target' parameter")?;
|
.ok_or("Missing 'target' parameter")?;
|
||||||
|
|
||||||
let (sx, sy, source_session_id) = super::element::resolve_element_center(
|
let (sx, sy, _, _, source_session_id) = super::element::resolve_element_center(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
&session_id,
|
&session_id,
|
||||||
&state.ref_map,
|
&state.ref_map,
|
||||||
@@ -6530,7 +6540,7 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
&state.iframe_sessions,
|
&state.iframe_sessions,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let (tx, ty, target_session_id) = super::element::resolve_element_center(
|
let (tx, ty, _, _, target_session_id) = super::element::resolve_element_center(
|
||||||
&mgr.client,
|
&mgr.client,
|
||||||
&session_id,
|
&session_id,
|
||||||
&state.ref_map,
|
&state.ref_map,
|
||||||
@@ -6555,12 +6565,26 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Move in steps to target, keeping the left button held (buttons: 1) so
|
// Move to the target with the left button held (buttons: 1) so the browser
|
||||||
// that the browser sees a drag rather than a plain pointer move.
|
// sees a drag. At Off this is the original linear 10-step path; at
|
||||||
let steps = 10;
|
// Fast/Human it follows humanize's curved, decelerating trajectory.
|
||||||
for i in 1..=steps {
|
let level = humanize::active_level();
|
||||||
let cx = sx + (tx - sx) * (i as f64) / (steps as f64);
|
let drag_path: Vec<(f64, f64, std::time::Duration)> =
|
||||||
let cy = sy + (ty - sy) * (i as f64) / (steps as f64);
|
if matches!(level, humanize::HumanizeLevel::Off) {
|
||||||
|
(1..=10)
|
||||||
|
.map(|i| {
|
||||||
|
let cx = sx + (tx - sx) * (i as f64) / 10.0;
|
||||||
|
let cy = sy + (ty - sy) * (i as f64) / 10.0;
|
||||||
|
(cx, cy, std::time::Duration::from_millis(10))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
} else {
|
||||||
|
humanize::move_path((sx, sy), (tx, ty), level, humanize::next_seed())
|
||||||
|
.into_iter()
|
||||||
|
.map(|s| (s.x, s.y, s.delay))
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
for (cx, cy, delay) in drag_path {
|
||||||
mgr.client
|
mgr.client
|
||||||
.send_command(
|
.send_command(
|
||||||
"Input.dispatchMouseEvent",
|
"Input.dispatchMouseEvent",
|
||||||
@@ -6568,7 +6592,9 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
|||||||
Some(&target_session_id),
|
Some(&target_session_id),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
|
if !delay.is_zero() {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mouse up at target
|
// Mouse up at target
|
||||||
|
|||||||
@@ -200,13 +200,17 @@ async fn relocate_stale_ref(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolve a `@ref` or CSS selector to a click point. Returns
|
||||||
|
/// `(centre_x, centre_y, width, height, session_id)`. Width/height come from the
|
||||||
|
/// element's box model and feed humanize's in-bounds landing jitter; the CSS
|
||||||
|
/// selector path returns zero size (→ land on centre, no jitter).
|
||||||
pub async fn resolve_element_center(
|
pub async fn resolve_element_center(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
ref_map: &RefMap,
|
ref_map: &RefMap,
|
||||||
selector_or_ref: &str,
|
selector_or_ref: &str,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
) -> Result<(f64, f64, String), String> {
|
) -> Result<(f64, f64, f64, f64, String), String> {
|
||||||
if let Some(ref_id) = parse_ref(selector_or_ref) {
|
if let Some(ref_id) = parse_ref(selector_or_ref) {
|
||||||
let entry = ref_map
|
let entry = ref_map
|
||||||
.get(&ref_id)
|
.get(&ref_id)
|
||||||
@@ -263,7 +267,7 @@ pub async fn resolve_element_center(
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
if let Ok(r) = result {
|
if let Ok(r) = result {
|
||||||
let (x, y) = box_model_center(&r.model);
|
let (x, y, w, h) = box_model_dims(&r.model);
|
||||||
// Occlusion check: a transient overlay (X.com's "click
|
// Occlusion check: a transient overlay (X.com's "click
|
||||||
// outside to close" mask, modal backdrop, sticky banner,
|
// outside to close" mask, modal backdrop, sticky banner,
|
||||||
// etc.) can land on top of our target between snapshot
|
// etc.) can land on top of our target between snapshot
|
||||||
@@ -279,7 +283,7 @@ pub async fn resolve_element_center(
|
|||||||
verify_click_target(client, effective_session_id, active_id, &ref_id, x, y)
|
verify_click_target(client, effective_session_id, active_id, &ref_id, x, y)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
return Ok((x, y, effective_session_id.to_string()));
|
return Ok((x, y, w, h, effective_session_id.to_string()));
|
||||||
}
|
}
|
||||||
// backend_node_id is stale; re-query the accessibility tree below
|
// backend_node_id is stale; re-query the accessibility tree below
|
||||||
}
|
}
|
||||||
@@ -316,13 +320,14 @@ pub async fn resolve_element_center(
|
|||||||
Some(effective_session_id),
|
Some(effective_session_id),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let (x, y) = box_model_center(&result.model);
|
let (x, y, w, h) = box_model_dims(&result.model);
|
||||||
return Ok((x, y, effective_session_id.to_string()));
|
return Ok((x, y, w, h, effective_session_id.to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// CSS selector
|
// CSS selector
|
||||||
let (x, y) = resolve_by_selector(client, session_id, selector_or_ref).await?;
|
let (x, y) = resolve_by_selector(client, session_id, selector_or_ref).await?;
|
||||||
Ok((x, y, session_id.to_string()))
|
// No box model on the CSS-selector fast path → zero size → land on centre.
|
||||||
|
Ok((x, y, 0.0, 0.0, session_id.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn resolve_element_object_id(
|
pub async fn resolve_element_object_id(
|
||||||
@@ -872,6 +877,35 @@ fn box_model_center(model: &BoxModel) -> (f64, f64) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Centre plus width/height of the content box, derived from the quad's
|
||||||
|
/// bounding extent. Width/height feed humanize's in-bounds landing jitter; a
|
||||||
|
/// degenerate quad yields zero size, which the jitter treats as "land on
|
||||||
|
/// centre" (no jitter).
|
||||||
|
fn box_model_dims(model: &BoxModel) -> (f64, f64, f64, f64) {
|
||||||
|
let (cx, cy) = box_model_center(model);
|
||||||
|
if model.content.len() >= 8 {
|
||||||
|
let xs = [
|
||||||
|
model.content[0],
|
||||||
|
model.content[2],
|
||||||
|
model.content[4],
|
||||||
|
model.content[6],
|
||||||
|
];
|
||||||
|
let ys = [
|
||||||
|
model.content[1],
|
||||||
|
model.content[3],
|
||||||
|
model.content[5],
|
||||||
|
model.content[7],
|
||||||
|
];
|
||||||
|
let w = xs.iter().cloned().fold(f64::MIN, f64::max)
|
||||||
|
- xs.iter().cloned().fold(f64::MAX, f64::min);
|
||||||
|
let h = ys.iter().cloned().fold(f64::MIN, f64::max)
|
||||||
|
- ys.iter().cloned().fold(f64::MAX, f64::min);
|
||||||
|
(cx, cy, w.max(0.0), h.max(0.0))
|
||||||
|
} else {
|
||||||
|
(cx, cy, 0.0, 0.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get_element_text(
|
pub async fn get_element_text(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
|
|||||||
@@ -277,6 +277,41 @@ pub fn move_path(
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Split a wheel scroll of (`total_dx`, `total_dy`) into eased segments. `Off`
|
||||||
|
/// returns a single instant segment (today's one-shot scroll); `Fast`/`Human`
|
||||||
|
/// break it into several accelerate-then-decelerate chunks with small,
|
||||||
|
/// jittered inter-segment delays, the way a trackpad/wheel flick actually
|
||||||
|
/// lands. The segment deltas always sum to the requested total.
|
||||||
|
pub fn scroll_segments(
|
||||||
|
total_dx: f64,
|
||||||
|
total_dy: f64,
|
||||||
|
level: HumanizeLevel,
|
||||||
|
seed: u64,
|
||||||
|
) -> Vec<(f64, f64, Duration)> {
|
||||||
|
if level.is_off() {
|
||||||
|
return vec![(total_dx, total_dy, Duration::ZERO)];
|
||||||
|
}
|
||||||
|
let (segs, base_ms) = match level {
|
||||||
|
HumanizeLevel::Fast => (4usize, 18.0),
|
||||||
|
_ => (9usize, 28.0),
|
||||||
|
};
|
||||||
|
let mut rng = Rng::new(seed);
|
||||||
|
let mut out = Vec::with_capacity(segs);
|
||||||
|
let mut prev = 0.0;
|
||||||
|
for i in 1..=segs {
|
||||||
|
let f = ease(i as f64 / segs as f64);
|
||||||
|
let frac = f - prev;
|
||||||
|
prev = f;
|
||||||
|
let jitter = 1.0 + 0.3 * rng.signed();
|
||||||
|
out.push((
|
||||||
|
total_dx * frac,
|
||||||
|
total_dy * frac,
|
||||||
|
Duration::from_millis((base_ms * jitter).max(4.0) as u64),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
/// Dwell between `mousePressed` and `mouseReleased` (a real click isn't
|
/// Dwell between `mousePressed` and `mouseReleased` (a real click isn't
|
||||||
/// instantaneous). Zero for `Off`.
|
/// instantaneous). Zero for `Off`.
|
||||||
pub fn press_dwell(level: HumanizeLevel, seed: u64) -> Duration {
|
pub fn press_dwell(level: HumanizeLevel, seed: u64) -> Duration {
|
||||||
@@ -435,6 +470,23 @@ mod tests {
|
|||||||
assert!(human.iter().all(|d| *d >= Duration::from_millis(8)));
|
assert!(human.iter().all(|d| *d >= Duration::from_millis(8)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scroll_segments_sum_to_total_and_single_when_off() {
|
||||||
|
let off = scroll_segments(0.0, 600.0, HumanizeLevel::Off, 1);
|
||||||
|
assert_eq!(off.len(), 1);
|
||||||
|
assert_eq!((off[0].0, off[0].1), (0.0, 600.0));
|
||||||
|
assert_eq!(off[0].2, Duration::ZERO);
|
||||||
|
|
||||||
|
let human = scroll_segments(0.0, 600.0, HumanizeLevel::Human, 5);
|
||||||
|
assert!(human.len() >= 5);
|
||||||
|
let total_dy: f64 = human.iter().map(|s| s.1).sum();
|
||||||
|
assert!(
|
||||||
|
(total_dy - 600.0).abs() < 1e-6,
|
||||||
|
"segments must sum to total"
|
||||||
|
);
|
||||||
|
assert!(human.iter().all(|s| s.2 >= Duration::from_millis(4)));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn detect_escalates_on_known_vendor_else_baseline() {
|
fn detect_escalates_on_known_vendor_else_baseline() {
|
||||||
let mut s = DetectSignals::default();
|
let mut s = DetectSignals::default();
|
||||||
|
|||||||
@@ -55,8 +55,15 @@ pub async fn click(
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
match resolved {
|
match resolved {
|
||||||
Ok((x, y, effective_session_id)) => {
|
Ok((cx, cy, w, h, effective_session_id)) => {
|
||||||
dispatch_click(client, &effective_session_id, x, y, button, click_count).await
|
// Land on a jittered point inside the element rather than its exact
|
||||||
|
// centre (Fast/Human). Zero size or Off → exact centre.
|
||||||
|
let (tx, ty) = humanize::landing_point(
|
||||||
|
(cx - w / 2.0, cy - h / 2.0, w, h),
|
||||||
|
humanize::active_level(),
|
||||||
|
humanize::next_seed(),
|
||||||
|
);
|
||||||
|
dispatch_click(client, &effective_session_id, tx, ty, button, click_count).await
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// (B) The coordinate path failed — typically a persistent overlay
|
// (B) The coordinate path failed — typically a persistent overlay
|
||||||
@@ -191,7 +198,7 @@ pub async fn hover(
|
|||||||
selector_or_ref: &str,
|
selector_or_ref: &str,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let (x, y, effective_session_id) = resolve_element_center(
|
let (x, y, _w, _h, effective_session_id) = resolve_element_center(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
ref_map,
|
ref_map,
|
||||||
@@ -999,7 +1006,7 @@ pub async fn tap_touch(
|
|||||||
selector_or_ref: &str,
|
selector_or_ref: &str,
|
||||||
iframe_sessions: &HashMap<String, String>,
|
iframe_sessions: &HashMap<String, String>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let (x, y, effective_session_id) = resolve_element_center(
|
let (x, y, _w, _h, effective_session_id) = resolve_element_center(
|
||||||
client,
|
client,
|
||||||
session_id,
|
session_id,
|
||||||
ref_map,
|
ref_map,
|
||||||
|
|||||||
Reference in New Issue
Block a user