diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index bebffc2..06b8476 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -13,7 +13,8 @@ use super::cdp::chrome::LaunchOptions; use super::cdp::client::CdpClient; use super::cdp::types::{ AttachToTargetParams, AttachToTargetResult, CdpEvent, ConsoleApiCalledEvent, - CreateTargetResult, ExceptionThrownEvent, TargetCreatedEvent, TargetDestroyedEvent, + CreateTargetResult, DispatchMouseEventParams, ExceptionThrownEvent, TargetCreatedEvent, + TargetDestroyedEvent, }; use super::cookies; use super::diff; @@ -104,6 +105,13 @@ pub enum BackendType { WebDriver, } +#[derive(Debug, Clone, Copy, Default)] +pub struct MouseState { + pub x: f64, + pub y: f64, + pub buttons: i32, +} + pub struct DaemonState { pub browser: Option, pub appium: Option, @@ -129,6 +137,7 @@ pub struct DaemonState { pub tracked_requests: Vec, pub request_tracking: bool, pub active_frame_id: Option, + pub mouse_state: MouseState, /// Shared slot for stream server to receive CDP client when browser launches. pub stream_client: Option>>>>, /// Stream server instance kept alive so the broadcast channel remains open. @@ -165,11 +174,16 @@ impl DaemonState { tracked_requests: Vec::new(), request_tracking: false, active_frame_id: None, + mouse_state: MouseState::default(), stream_client: None, stream_server: None, } } + fn reset_input_state(&mut self) { + self.mouse_state = MouseState::default(); + } + /// Create state with an optional stream client slot and server instance /// (for daemon startup with stream server). pub fn new_with_stream( @@ -708,6 +722,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value { let _ = mgr.close().await; } state.browser = None; + state.reset_input_state(); state.update_stream_client().await; } if let Err(e) = auto_launch(state).await { @@ -919,6 +934,7 @@ async fn auto_launch(state: &mut DaemonState) -> Result<(), String> { if let Ok(cdp) = env::var("AGENT_BROWSER_CDP") { let mgr = BrowserManager::connect_cdp(&cdp).await?; + state.reset_input_state(); state.browser = Some(mgr); state.subscribe_to_browser_events(); state.update_stream_client().await; @@ -927,6 +943,7 @@ async fn auto_launch(state: &mut DaemonState) -> Result<(), String> { } if env::var("AGENT_BROWSER_AUTO_CONNECT").is_ok() { + state.reset_input_state(); state.browser = Some(connect_auto_with_fresh_tab().await?); state.subscribe_to_browser_events(); state.update_stream_client().await; @@ -935,6 +952,7 @@ async fn auto_launch(state: &mut DaemonState) -> Result<(), String> { } let mgr = BrowserManager::launch(options, engine.as_deref()).await?; + state.reset_input_state(); state.browser = Some(mgr); state.subscribe_to_browser_events(); state.update_stream_client().await; @@ -1041,6 +1059,7 @@ async fn handle_launch(cmd: &Value, state: &mut DaemonState) -> Result Result Result Result Result { + state.reset_input_state(); state.browser = Some(mgr); state.subscribe_to_browser_events(); state.update_stream_client().await; @@ -1195,6 +1218,7 @@ async fn handle_launch(cmd: &Value, state: &mut DaemonState) -> Result Result Result Result { mgr.close().await?; } state.browser = None; + state.reset_input_state(); state.update_stream_client().await; // Close WebDriver sessions @@ -5643,28 +5670,110 @@ async fn handle_device_list() -> Result { // Input event handlers // --------------------------------------------------------------------------- -async fn handle_input_mouse(cmd: &Value, state: &DaemonState) -> Result { +fn mouse_button_mask(button: &str) -> i32 { + match button { + "left" => 1, + "right" => 2, + "middle" => 4, + "back" => 8, + "forward" => 16, + _ => 0, + } +} + +fn primary_button_from_mask(buttons: i32) -> &'static str { + if buttons & 1 != 0 { + "left" + } else if buttons & 2 != 0 { + "right" + } else if buttons & 4 != 0 { + "middle" + } else if buttons & 8 != 0 { + "back" + } else if buttons & 16 != 0 { + "forward" + } else { + "none" + } +} + +#[allow(clippy::too_many_arguments)] +fn build_mouse_event_params( + mouse_state: &mut MouseState, + event_type: &str, + x: Option, + y: Option, + button: Option<&str>, + buttons: Option, + click_count: Option, + delta_x: Option, + delta_y: Option, + modifiers: Option, +) -> DispatchMouseEventParams { + let x = x.unwrap_or(mouse_state.x); + let y = y.unwrap_or(mouse_state.y); + mouse_state.x = x; + mouse_state.y = y; + + let mut next_buttons = buttons.unwrap_or(mouse_state.buttons); + if buttons.is_none() { + match event_type { + "mousePressed" => { + next_buttons |= mouse_button_mask(button.unwrap_or("left")); + } + "mouseReleased" => { + next_buttons &= !mouse_button_mask(button.unwrap_or("left")); + } + _ => {} + } + } + mouse_state.buttons = next_buttons; + + DispatchMouseEventParams { + event_type: event_type.to_string(), + x, + y, + button: Some( + button + .unwrap_or(primary_button_from_mask(next_buttons)) + .to_string(), + ), + buttons: Some(next_buttons), + click_count, + delta_x, + delta_y, + modifiers, + } +} + +async fn handle_input_mouse(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); let event_type = cmd .get("type") .and_then(|v| v.as_str()) .unwrap_or("mouseMoved"); - let x = cmd.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let y = cmd.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); + let params = build_mouse_event_params( + &mut state.mouse_state, + event_type, + cmd.get("x").and_then(|v| v.as_f64()), + cmd.get("y").and_then(|v| v.as_f64()), + cmd.get("button").and_then(|v| v.as_str()), + cmd.get("buttons") + .and_then(|v| v.as_i64()) + .map(|v| v as i32), + cmd.get("clickCount") + .and_then(|v| v.as_i64()) + .map(|v| v as i32), + cmd.get("deltaX").and_then(|v| v.as_f64()), + cmd.get("deltaY").and_then(|v| v.as_f64()), + cmd.get("modifiers") + .and_then(|v| v.as_i64()) + .map(|v| v as i32), + ); mgr.client - .send_command( - "Input.dispatchMouseEvent", - Some(json!({ - "type": event_type, "x": x, "y": y, - "button": cmd.get("button").and_then(|v| v.as_str()).unwrap_or("none"), - "clickCount": cmd.get("clickCount").and_then(|v| v.as_i64()).unwrap_or(0), - "deltaX": cmd.get("deltaX").and_then(|v| v.as_f64()).unwrap_or(0.0), - "deltaY": cmd.get("deltaY").and_then(|v| v.as_f64()).unwrap_or(0.0), - })), - Some(&session_id), - ) + .send_command_typed::<_, Value>("Input.dispatchMouseEvent", ¶ms, Some(&session_id)) .await?; Ok(json!({ "dispatched": event_type })) } @@ -5765,48 +5874,72 @@ async fn handle_inserttext(cmd: &Value, state: &DaemonState) -> Result Result { +async fn handle_mousemove(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); let x = cmd.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); let y = cmd.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); + let params = build_mouse_event_params( + &mut state.mouse_state, + "mouseMoved", + Some(x), + Some(y), + None, + None, + None, + None, + None, + None, + ); mgr.client - .send_command( - "Input.dispatchMouseEvent", - Some(json!({ "type": "mouseMoved", "x": x, "y": y })), - Some(&session_id), - ) + .send_command_typed::<_, Value>("Input.dispatchMouseEvent", ¶ms, Some(&session_id)) .await?; Ok(json!({ "moved": true })) } -async fn handle_mousedown(cmd: &Value, state: &DaemonState) -> Result { +async fn handle_mousedown(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); let button = cmd.get("button").and_then(|v| v.as_str()).unwrap_or("left"); + let params = build_mouse_event_params( + &mut state.mouse_state, + "mousePressed", + None, + None, + Some(button), + None, + Some(1), + None, + None, + None, + ); mgr.client - .send_command( - "Input.dispatchMouseEvent", - Some(json!({ "type": "mousePressed", "x": 0, "y": 0, "button": button, "clickCount": 1 })), - Some(&session_id), - ) + .send_command_typed::<_, Value>("Input.dispatchMouseEvent", ¶ms, Some(&session_id)) .await?; Ok(json!({ "pressed": true })) } -async fn handle_mouseup(cmd: &Value, state: &DaemonState) -> Result { +async fn handle_mouseup(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); let button = cmd.get("button").and_then(|v| v.as_str()).unwrap_or("left"); + let params = build_mouse_event_params( + &mut state.mouse_state, + "mouseReleased", + None, + None, + Some(button), + None, + Some(1), + None, + None, + None, + ); mgr.client - .send_command( - "Input.dispatchMouseEvent", - Some(json!({ "type": "mouseReleased", "x": 0, "y": 0, "button": button, "clickCount": 1 })), - Some(&session_id), - ) + .send_command_typed::<_, Value>("Input.dispatchMouseEvent", ¶ms, Some(&session_id)) .await?; Ok(json!({ "released": true })) } @@ -5862,6 +5995,96 @@ mod tests { assert_eq!(state.session_id, "default"); assert!(!state.tracing_state.active); assert!(!state.recording_state.active); + assert_eq!(state.mouse_state.x, 0.0); + assert_eq!(state.mouse_state.y, 0.0); + assert_eq!(state.mouse_state.buttons, 0); + } + + #[test] + fn test_mouse_event_params_preserve_position_and_buttons() { + let mut mouse_state = MouseState::default(); + + let move_params = build_mouse_event_params( + &mut mouse_state, + "mouseMoved", + Some(120.0), + Some(240.0), + None, + None, + None, + None, + None, + None, + ); + assert_eq!(move_params.x, 120.0); + assert_eq!(move_params.y, 240.0); + assert_eq!(move_params.buttons, Some(0)); + + let down_params = build_mouse_event_params( + &mut mouse_state, + "mousePressed", + None, + None, + Some("left"), + None, + Some(1), + None, + None, + None, + ); + assert_eq!(down_params.x, 120.0); + assert_eq!(down_params.y, 240.0); + assert_eq!(down_params.button.as_deref(), Some("left")); + assert_eq!(down_params.buttons, Some(1)); + assert_eq!(mouse_state.buttons, 1); + + let drag_move_params = build_mouse_event_params( + &mut mouse_state, + "mouseMoved", + Some(150.0), + Some(260.0), + None, + None, + None, + None, + None, + None, + ); + assert_eq!(drag_move_params.buttons, Some(1)); + assert_eq!(drag_move_params.button.as_deref(), Some("left")); + assert_eq!(mouse_state.x, 150.0); + assert_eq!(mouse_state.y, 260.0); + + let up_params = build_mouse_event_params( + &mut mouse_state, + "mouseReleased", + None, + None, + Some("left"), + None, + Some(1), + None, + None, + None, + ); + assert_eq!(up_params.x, 150.0); + assert_eq!(up_params.y, 260.0); + assert_eq!(up_params.buttons, Some(0)); + assert_eq!(mouse_state.buttons, 0); + } + + #[test] + fn test_reset_input_state_clears_mouse_state() { + let mut state = DaemonState::new(); + state.mouse_state.x = 12.0; + state.mouse_state.y = 34.0; + state.mouse_state.buttons = 1; + + state.reset_input_state(); + + assert_eq!(state.mouse_state.x, 0.0); + assert_eq!(state.mouse_state.y, 0.0); + assert_eq!(state.mouse_state.buttons, 0); } #[test] diff --git a/cli/src/native/e2e_tests.rs b/cli/src/native/e2e_tests.rs index d18c8d9..bafe294 100644 --- a/cli/src/native/e2e_tests.rs +++ b/cli/src/native/e2e_tests.rs @@ -7,6 +7,7 @@ //! Run serially to avoid Chrome instance contention: //! cargo test e2e -- --ignored --test-threads=1 +use base64::{engine::general_purpose::STANDARD, Engine}; use serde_json::{json, Value}; use super::actions::{execute_command, DaemonState}; @@ -24,6 +25,22 @@ fn get_data(resp: &Value) -> &Value { resp.get("data").expect("Missing 'data' in response") } +fn native_test_fixture_html(name: &str) -> &'static str { + match name { + "drag_probe" => include_str!("test_fixtures/drag_probe.html"), + "html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"), + "pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"), + _ => panic!("Unknown native test fixture: {}", name), + } +} + +fn native_test_fixture_url(name: &str) -> String { + format!( + "data:text/html;base64,{}", + STANDARD.encode(native_test_fixture_html(name)) + ) +} + // --------------------------------------------------------------------------- // Core: launch, navigate, evaluate, url, title, close // --------------------------------------------------------------------------- @@ -1152,6 +1169,245 @@ async fn e2e_hover_scroll_press() { assert_success(&resp); } +// --------------------------------------------------------------------------- +// Raw mouse regressions +// --------------------------------------------------------------------------- + +#[tokio::test] +#[ignore] +async fn e2e_mouse_down_move_up_preserves_drag_state() { + let mut state = DaemonState::new(); + + let resp = execute_command( + &json!({ "id": "1", "action": "launch", "headless": true }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ + "id": "2", + "action": "navigate", + "url": native_test_fixture_url("drag_probe") + }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ + "id": "3", + "action": "evaluate", + "script": r#"(() => { + const rect = document.getElementById('target').getBoundingClientRect(); + return { + left: Math.round(rect.left), + top: Math.round(rect.top), + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2) + }; + })()"# + }), + &mut state, + ) + .await; + assert_success(&resp); + let start = &get_data(&resp)["result"]; + let initial_left = start["left"] + .as_i64() + .expect("target left should be numeric"); + let initial_top = start["top"].as_i64().expect("target top should be numeric"); + let start_x = start["x"].as_i64().expect("target x should be numeric"); + let start_y = start["y"].as_i64().expect("target y should be numeric"); + let end_x = start_x + 80; + let end_y = start_y + 60; + + let resp = execute_command( + &json!({ "id": "4", "action": "mousemove", "x": start_x, "y": start_y }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "5", "action": "mousedown", "button": "left" }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "6", "action": "mousemove", "x": end_x, "y": end_y }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "7", "action": "mouseup", "button": "left" }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "8", "action": "evaluate", "script": "window.__dragProbe" }), + &mut state, + ) + .await; + assert_success(&resp); + + let probe = &get_data(&resp)["result"]; + assert_eq!(probe["finalLeft"].as_i64(), Some(initial_left + 80)); + assert_eq!(probe["finalTop"].as_i64(), Some(initial_top + 60)); + + let events = probe["events"] + .as_array() + .expect("drag probe should expose events"); + assert!( + events.iter().any(|event| { + event["type"] == "mousedown" + && event["x"].as_f64() == Some(start_x as f64) + && event["y"].as_f64() == Some(start_y as f64) + && event["buttons"].as_i64() == Some(1) + }), + "Expected a non-zero mousedown event in drag probe" + ); + assert!( + events.iter().any(|event| { + event["type"] == "mousemove" + && event["x"].as_f64() == Some(end_x as f64) + && event["y"].as_f64() == Some(end_y as f64) + && event["buttons"].as_i64() == Some(1) + }), + "Expected a drag mousemove with the button still pressed" + ); + assert!( + events.iter().any(|event| { + event["type"] == "mouseup" + && event["x"].as_f64() == Some(end_x as f64) + && event["y"].as_f64() == Some(end_y as f64) + && event["buttons"].as_i64() == Some(0) + }), + "Expected mouseup at the last drag position" + ); + + let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await; + assert_success(&resp); +} + +#[tokio::test] +#[ignore] +async fn e2e_mouse_drag_reaches_pointer_capture_target() { + let mut state = DaemonState::new(); + + let resp = execute_command( + &json!({ "id": "1", "action": "launch", "headless": true }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ + "id": "2", + "action": "navigate", + "url": native_test_fixture_url("pointer_capture_probe") + }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ + "id": "3", + "action": "evaluate", + "script": r#"(() => { + const rect = document.getElementById('handle').getBoundingClientRect(); + return { + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2) + }; + })()"# + }), + &mut state, + ) + .await; + assert_success(&resp); + + let start = &get_data(&resp)["result"]; + let start_x = start["x"].as_i64().expect("handle x should be numeric"); + let start_y = start["y"].as_i64().expect("handle y should be numeric"); + let end_x = start_x + 80; + let end_y = start_y + 60; + + let resp = execute_command( + &json!({ "id": "4", "action": "mousemove", "x": start_x, "y": start_y }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "5", "action": "mousedown", "button": "left" }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "6", "action": "mousemove", "x": end_x, "y": end_y }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "7", "action": "mouseup", "button": "left" }), + &mut state, + ) + .await; + assert_success(&resp); + + let resp = execute_command( + &json!({ "id": "8", "action": "evaluate", "script": "window.__pointerCaptureProbe" }), + &mut state, + ) + .await; + assert_success(&resp); + + let probe = &get_data(&resp)["result"]; + assert_eq!(probe["moved"].as_bool(), Some(true)); + + let events = probe["events"] + .as_array() + .expect("pointer capture probe should expose events"); + assert!( + events.iter().any(|event| { + event["type"] == "pointermove" + && event["phase"] == "drag" + && event["hasCapture"].as_bool() == Some(true) + && event["x"].as_f64() == Some(end_x as f64) + && event["y"].as_f64() == Some(end_y as f64) + }), + "Expected pointermove with capture during the drag" + ); + assert!( + events.iter().any(|event| { + event["type"] == "pointerup" + && event["phase"] == "up" + && event["hadCapture"].as_bool() == Some(true) + }), + "Expected pointerup to observe an active pointer capture" + ); + + let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await; + assert_success(&resp); +} + // --------------------------------------------------------------------------- // State save/load, state management // --------------------------------------------------------------------------- diff --git a/cli/src/native/test_fixtures/drag_probe.html b/cli/src/native/test_fixtures/drag_probe.html new file mode 100644 index 0000000..5ba3569 --- /dev/null +++ b/cli/src/native/test_fixtures/drag_probe.html @@ -0,0 +1,135 @@ + + + + + Drag Probe + + + +
+
drag me
+
+ 

+  
+
+
diff --git a/cli/src/native/test_fixtures/html5_drag_probe.html b/cli/src/native/test_fixtures/html5_drag_probe.html
new file mode 100644
index 0000000..3f914f1
--- /dev/null
+++ b/cli/src/native/test_fixtures/html5_drag_probe.html
@@ -0,0 +1,91 @@
+
+
+
+  
+  HTML5 Drag Probe
+  
+
+
+  
drag source

+  
drop zone

+  

+  
+
+
diff --git a/cli/src/native/test_fixtures/pointer_capture_probe.html b/cli/src/native/test_fixtures/pointer_capture_probe.html
new file mode 100644
index 0000000..b773e44
--- /dev/null
+++ b/cli/src/native/test_fixtures/pointer_capture_probe.html
@@ -0,0 +1,113 @@
+
+
+
+  
+  Pointer Capture Probe
+  
+
+
+  

+    

+  

+  

+  
+
+