fix(relay): DOM-dispatch hover/dblclick/drag; deeper iframe snapshot; key-events typing (#37)

Follow-up to #36 — make the whole interaction surface reach cross-origin OOPIFs
and stop coordinate events drifting onto the user's foreground tab over the relay.

- hover/dblclick/drag now DOM-dispatch over the relay or into an iframe (like
  click already did): a coordinate Input event isn't confined to the target tab
  on a busy real Chrome and can't map an OOPIF element's box to a top-viewport
  point. drag does an HTML5 DnD in the element's frame; cross-frame drag errors
  loudly instead of drifting.
- snapshot recurses iframes to MAX_IFRAME_DEPTH (3) instead of one level, so refs
  inside nested payment/checkout widgets get a frame_id and resolve into the
  right frame.
- relay tab adoption merges several Target.getTargets snapshots — a single flaky
  relay snapshot was dropping live tabs (a driven tab vanished after restart).
- `type --key-events` (alias --keys) sends real per-character keyDown/keyUp
  instead of Input.insertText, so autocomplete/combobox widgets that ignore the
  insertText input event fire (Google address postal lookup; commits Angular
  reactive forms so Save enables).
- SKILL: hard "snapshot-first, never default to screenshot+coordinates" rule;
  snapshot -i pierces cross-origin iframes since v1.5.12; cross-origin iframe
  driving guidance (#37).
This commit is contained in:
leeguooooo
2026-06-17 00:58:12 +09:00
parent 70ab38d35f
commit 9f24e66033
10 changed files with 420 additions and 42 deletions
+30 -3
View File
@@ -525,20 +525,29 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": rest[1..].join(" ") }))
}
"type" => {
// `--key-events` (alias `--keys`): send real per-character keystrokes
// instead of Input.insertText, so autocomplete/combobox widgets that
// only react to key events fire (e.g. Google address postal lookup).
let key_events = rest.iter().any(|a| *a == "--key-events" || *a == "--keys");
let rest: Vec<&str> = rest
.iter()
.copied()
.filter(|a| *a != "--key-events" && *a != "--keys")
.collect();
// `type --focused <text>` types into whatever element currently has
// focus (no selector) — for custom widgets that move focus to a hidden
// input after you open them.
if rest.first() == Some(&"--focused") {
return Ok(json!({
"id": id, "action": "type", "focused": true,
"text": rest[1..].join(" "),
"text": rest[1..].join(" "), "keyEvents": key_events,
}));
}
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
context: "type".to_string(),
usage: "type <selector> <text> (or: type --focused <text>)",
usage: "type <selector> <text> (or: type --focused <text>) [--key-events]",
})?;
Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" ") }))
Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" "), "keyEvents": key_events }))
}
"pick" => {
// pick <selector|@ref> --option "<text>" — atomic combobox select:
@@ -4112,6 +4121,24 @@ mod tests {
assert_eq!(cmd["action"], "type");
assert_eq!(cmd["selector"], "#input");
assert_eq!(cmd["text"], "some text");
assert_eq!(cmd["keyEvents"], false);
}
#[test]
fn test_type_key_events() {
// --key-events sends real keystrokes (for autocomplete/combobox) and must
// not be swallowed into the typed text.
let cmd = parse_command(&args("type #postal 201-0001 --key-events"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "type");
assert_eq!(cmd["selector"], "#postal");
assert_eq!(cmd["text"], "201-0001");
assert_eq!(cmd["keyEvents"], true);
let focused =
parse_command(&args("type --focused 201-0001 --keys"), &default_flags()).unwrap();
assert_eq!(focused["focused"], true);
assert_eq!(focused["text"], "201-0001");
assert_eq!(focused["keyEvents"], true);
}
#[test]