fix(relay): DOM-dispatch hover/dblclick/drag; deeper iframe snapshot; key-events typing (#37)
Follow-up to #36 — make the whole interaction surface reach cross-origin OOPIFs and stop coordinate events drifting onto the user's foreground tab over the relay. - hover/dblclick/drag now DOM-dispatch over the relay or into an iframe (like click already did): a coordinate Input event isn't confined to the target tab on a busy real Chrome and can't map an OOPIF element's box to a top-viewport point. drag does an HTML5 DnD in the element's frame; cross-frame drag errors loudly instead of drifting. - snapshot recurses iframes to MAX_IFRAME_DEPTH (3) instead of one level, so refs inside nested payment/checkout widgets get a frame_id and resolve into the right frame. - relay tab adoption merges several Target.getTargets snapshots — a single flaky relay snapshot was dropping live tabs (a driven tab vanished after restart). - `type --key-events` (alias --keys) sends real per-character keyDown/keyUp instead of Input.insertText, so autocomplete/combobox widgets that ignore the insertText input event fire (Google address postal lookup; commits Angular reactive forms so Save enables). - SKILL: hard "snapshot-first, never default to screenshot+coordinates" rule; snapshot -i pierces cross-origin iframes since v1.5.12; cross-origin iframe driving guidance (#37).
This commit is contained in:
@@ -3225,6 +3225,14 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
let session_id = mgr.active_session_id()?.to_string();
|
||||
|
||||
// `--key-events`: dispatch real per-character keyDown/keyUp instead of
|
||||
// Input.insertText, so autocomplete/combobox widgets that only react to key
|
||||
// events fire (e.g. Google's address postal-code lookup) (issue #4/#36).
|
||||
let key_events = cmd
|
||||
.get("keyEvents")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
// `type --focused <text>`: type into the currently-focused element without a
|
||||
// selector (custom widgets that move focus to a hidden input on open).
|
||||
if cmd
|
||||
@@ -3236,7 +3244,8 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
||||
.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Missing 'text' parameter")?;
|
||||
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None).await?;
|
||||
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None, key_events)
|
||||
.await?;
|
||||
return Ok(json!({ "typed": text, "focused": true }));
|
||||
}
|
||||
|
||||
@@ -3260,6 +3269,7 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
||||
clear,
|
||||
delay,
|
||||
&state.iframe_sessions,
|
||||
key_events,
|
||||
)
|
||||
.await?;
|
||||
Ok(json!({ "typed": text }))
|
||||
@@ -4722,8 +4732,18 @@ async fn handle_keyboard(cmd: &Value, state: &DaemonState) -> Result<Value, Stri
|
||||
.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Missing 'text' parameter")?;
|
||||
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None)
|
||||
.await?;
|
||||
let key_events = cmd
|
||||
.get("keyEvents")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
interaction::type_text_into_active_context(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
text,
|
||||
None,
|
||||
key_events,
|
||||
)
|
||||
.await?;
|
||||
return Ok(json!({ "typed": text }));
|
||||
}
|
||||
Some("insertText") => {
|
||||
@@ -7252,6 +7272,28 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Missing 'target' parameter")?;
|
||||
|
||||
// Over the relay (or into an iframe) a coordinate drag drifts to the
|
||||
// foreground tab and can't reach an OOPIF — DOM-dispatch an HTML5 drag in the
|
||||
// element's own session instead (issues #31/#36). `coord` mode forces the
|
||||
// coordinate path for pointer-driven drags (canvas/sliders) on a launched
|
||||
// browser.
|
||||
if std::env::var("AGENT_BROWSER_CLICK_MODE").as_deref() != Ok("coord")
|
||||
&& (crate::connect::relay_url().is_some()
|
||||
|| state.ref_map.ref_is_in_iframe(source)
|
||||
|| state.ref_map.ref_is_in_iframe(target))
|
||||
{
|
||||
super::interaction::dom_drag(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&state.ref_map,
|
||||
source,
|
||||
target,
|
||||
&state.iframe_sessions,
|
||||
)
|
||||
.await?;
|
||||
return Ok(json!({ "dragged": { "source": source, "target": target }, "via": "dom" }));
|
||||
}
|
||||
|
||||
let (sx, sy, _, _, source_session_id) = super::element::resolve_element_center(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
|
||||
Reference in New Issue
Block a user