From 9f24e66033e0a419a289a47b2f47ba5510b451d8 Mon Sep 17 00:00:00 2001 From: leeguooooo Date: Wed, 17 Jun 2026 00:58:12 +0900 Subject: [PATCH] fix(relay): DOM-dispatch hover/dblclick/drag; deeper iframe snapshot; key-events typing (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #36 — make the whole interaction surface reach cross-origin OOPIFs and stop coordinate events drifting onto the user's foreground tab over the relay. - hover/dblclick/drag now DOM-dispatch over the relay or into an iframe (like click already did): a coordinate Input event isn't confined to the target tab on a busy real Chrome and can't map an OOPIF element's box to a top-viewport point. drag does an HTML5 DnD in the element's frame; cross-frame drag errors loudly instead of drifting. - snapshot recurses iframes to MAX_IFRAME_DEPTH (3) instead of one level, so refs inside nested payment/checkout widgets get a frame_id and resolve into the right frame. - relay tab adoption merges several Target.getTargets snapshots — a single flaky relay snapshot was dropping live tabs (a driven tab vanished after restart). - `type --key-events` (alias --keys) sends real per-character keyDown/keyUp instead of Input.insertText, so autocomplete/combobox widgets that ignore the insertText input event fire (Google address postal lookup; commits Angular reactive forms so Save enables). - SKILL: hard "snapshot-first, never default to screenshot+coordinates" rule; snapshot -i pierces cross-origin iframes since v1.5.12; cross-origin iframe driving guidance (#37). --- cli/Cargo.lock | 2 +- cli/Cargo.toml | 2 +- cli/src/commands.rs | 33 ++++- cli/src/native/actions.rs | 48 ++++++- cli/src/native/browser.rs | 69 ++++++++-- cli/src/native/element.rs | 9 ++ cli/src/native/interaction.rs | 232 +++++++++++++++++++++++++++++++--- cli/src/native/snapshot.rs | 32 ++++- cli/src/output.rs | 7 + skill-data/core/SKILL.md | 28 ++++ 10 files changed, 420 insertions(+), 42 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 36e1720..03cb963 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrome-use" -version = "1.5.11" +version = "1.5.12" dependencies = [ "aes", "aes-gcm", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 982d14e..0d2cddd 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chrome-use" -version = "1.5.11" +version = "1.5.12" edition = "2021" description = "Fast browser automation CLI for AI agents" license = "Apache-2.0" diff --git a/cli/src/commands.rs b/cli/src/commands.rs index faa6c5d..41e23c2 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -525,20 +525,29 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result { + // `--key-events` (alias `--keys`): send real per-character keystrokes + // instead of Input.insertText, so autocomplete/combobox widgets that + // only react to key events fire (e.g. Google address postal lookup). + let key_events = rest.iter().any(|a| *a == "--key-events" || *a == "--keys"); + let rest: Vec<&str> = rest + .iter() + .copied() + .filter(|a| *a != "--key-events" && *a != "--keys") + .collect(); // `type --focused ` types into whatever element currently has // focus (no selector) — for custom widgets that move focus to a hidden // input after you open them. if rest.first() == Some(&"--focused") { return Ok(json!({ "id": id, "action": "type", "focused": true, - "text": rest[1..].join(" "), + "text": rest[1..].join(" "), "keyEvents": key_events, })); } let sel = rest.first().ok_or_else(|| ParseError::MissingArguments { context: "type".to_string(), - usage: "type (or: type --focused )", + usage: "type (or: type --focused ) [--key-events]", })?; - Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" ") })) + Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" "), "keyEvents": key_events })) } "pick" => { // pick --option "" — atomic combobox select: @@ -4112,6 +4121,24 @@ mod tests { assert_eq!(cmd["action"], "type"); assert_eq!(cmd["selector"], "#input"); assert_eq!(cmd["text"], "some text"); + assert_eq!(cmd["keyEvents"], false); + } + + #[test] + fn test_type_key_events() { + // --key-events sends real keystrokes (for autocomplete/combobox) and must + // not be swallowed into the typed text. + let cmd = parse_command(&args("type #postal 201-0001 --key-events"), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "type"); + assert_eq!(cmd["selector"], "#postal"); + assert_eq!(cmd["text"], "201-0001"); + assert_eq!(cmd["keyEvents"], true); + + let focused = + parse_command(&args("type --focused 201-0001 --keys"), &default_flags()).unwrap(); + assert_eq!(focused["focused"], true); + assert_eq!(focused["text"], "201-0001"); + assert_eq!(focused["keyEvents"], true); } #[test] diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index b622568..7c9c9c8 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -3225,6 +3225,14 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result`: type into the currently-focused element without a // selector (custom widgets that move focus to a hidden input on open). if cmd @@ -3236,7 +3244,8 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result Result Result { @@ -7252,6 +7272,28 @@ async fn handle_drag(cmd: &Value, state: &mut DaemonState) -> Result Result, String> { + let rounds = if crate::connect::relay_url().is_some() { + 3 + } else { + 1 + }; + let mut by_id: HashMap = HashMap::new(); + let mut any_ok = false; + for i in 0..rounds { + if i > 0 { + tokio::time::sleep(Duration::from_millis(150)).await; + } + match self + .client + .send_command_typed::<_, GetTargetsResult>("Target.getTargets", &json!({}), None) + .await + { + Ok(result) => { + any_ok = true; + for t in result.target_infos.into_iter().filter(should_track_target) { + by_id.entry(t.target_id.clone()).or_insert(t); + } + } + Err(e) if i == rounds - 1 && !any_ok => return Err(e), + Err(_) => {} + } + } + Ok(by_id.into_values().collect()) + } + async fn discover_and_attach_targets(&mut self) -> Result<(), String> { self.client .send_command_typed::<_, Value>( @@ -734,16 +771,7 @@ impl BrowserManager { ) .await?; - let result: GetTargetsResult = self - .client - .send_command_typed("Target.getTargets", &json!({}), None) - .await?; - - let page_targets: Vec = result - .target_infos - .into_iter() - .filter(should_track_target) - .collect(); + let page_targets: Vec = self.collect_page_targets().await?; if page_targets.is_empty() { // Create a new tab @@ -816,10 +844,23 @@ impl BrowserManager { }); } - self.active_page_index = 0; - self.pin_active_target(); - let session_id = self.pages[0].session_id.clone(); - self.enable_domains(&session_id).await?; + if self.agent_group().is_some() { + // Relay: the adopted tabs above are the USER's, in their real + // Chrome. NEVER make one of them the agent's working tab — that is + // how commands drifted onto whatever page the user was viewing + // between steps (eval/click/get landed on the user's foreground + // tab; #35). Open our own dedicated background tab in the session's + // group and pin THAT as active. The user's tabs stay adopted (so + // `tab list` / explicit `tab switch` can reach them) but are never + // auto-selected — the agent only ever drives a tab it owns. + self.tab_new(None, None).await?; + } else { + // A browser we launched: every tab is ours, so the first is fine. + self.active_page_index = 0; + self.pin_active_target(); + let session_id = self.pages[0].session_id.clone(); + self.enable_domains(&session_id).await?; + } } Ok(()) diff --git a/cli/src/native/element.rs b/cli/src/native/element.rs index 10b356c..500ef10 100644 --- a/cli/src/native/element.rs +++ b/cli/src/native/element.rs @@ -100,6 +100,15 @@ impl RefMap { self.map.get(ref_id) } + /// Whether `selector_or_ref` is a `@ref` whose snapshot entry lives inside an + /// iframe (has a `frame_id`). Pointer interactions use this to choose + /// DOM-dispatch over coordinates for OOPIF elements (issue #36). + pub fn ref_is_in_iframe(&self, selector_or_ref: &str) -> bool { + parse_ref(selector_or_ref) + .and_then(|r| self.map.get(&r).map(|e| e.frame_id.is_some())) + .unwrap_or(false) + } + pub fn entries_sorted(&self) -> Vec<(String, RefEntry)> { let mut entries = self .map diff --git a/cli/src/native/interaction.rs b/cli/src/native/interaction.rs index ece80ae..46fb0f6 100644 --- a/cli/src/native/interaction.rs +++ b/cli/src/native/interaction.rs @@ -7,6 +7,17 @@ use super::cdp::types::*; use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap}; use super::humanize; +/// Whether a pointer interaction should be DOM-dispatched (invoke the event on +/// the element in its own session) rather than dispatched at a viewport +/// coordinate via `Input.dispatchMouseEvent`. True when the target is inside an +/// iframe (an OOPIF element's box can't be mapped to a top-viewport point) or we +/// drive over the extension relay (a coordinate Input event isn't confined to the +/// target tab on a busy real Chrome — it drifts onto the foreground tab; issues +/// #31/#36). DOM-dispatch always hits the right element in the right tab. +fn prefer_dom_dispatch(ref_map: &RefMap, selector_or_ref: &str) -> bool { + ref_map.ref_is_in_iframe(selector_or_ref) || crate::connect::relay_url().is_some() +} + pub async fn click( client: &CdpClient, session_id: &str, @@ -54,20 +65,15 @@ pub async fn click( // the element's click in its own (frame) session, always hitting the right // element in the right tab. Double/right clicks still need true pointer // semantics, and `coord` mode is an explicit opt-out. - if mode != "coord" && button == "left" && click_count == 1 { - let in_iframe = parse_ref(selector_or_ref) - .and_then(|r| ref_map.get(&r).map(|e| e.frame_id.is_some())) - .unwrap_or(false); - if in_iframe || crate::connect::relay_url().is_some() { - return dom_click( - client, - session_id, - ref_map, - selector_or_ref, - iframe_sessions, - ) - .await; - } + if mode != "coord" && button == "left" && click_count == 1 && prefer_dom_dispatch(ref_map, selector_or_ref) { + return dom_click( + client, + session_id, + ref_map, + selector_or_ref, + iframe_sessions, + ) + .await; } let resolved = resolve_element_center( @@ -260,6 +266,47 @@ async fn dom_click( Ok(()) } +/// DOM-dispatch a double-click on the element in its own session (no coordinates) +/// — the relay/iframe-safe counterpart to a coordinate dblclick. Fires the full +/// click,click,dblclick sequence so handlers bound to any of them respond. +async fn dom_dblclick( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, + selector_or_ref: &str, + iframe_sessions: &HashMap, +) -> Result<(), String> { + let (object_id, effective_session_id) = resolve_element_object_id( + client, + session_id, + ref_map, + selector_or_ref, + iframe_sessions, + ) + .await?; + client + .send_command_typed::<_, Value>( + "Runtime.callFunctionOn", + &CallFunctionOnParams { + function_declaration: r#"function() { + const opts = { bubbles: true, cancelable: true, view: window }; + this.dispatchEvent(new MouseEvent('click', opts)); + this.dispatchEvent(new MouseEvent('click', { ...opts, detail: 2 })); + this.dispatchEvent(new MouseEvent('dblclick', opts)); + }"# + .to_string(), + object_id: Some(object_id), + arguments: None, + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(&effective_session_id), + ) + .await?; + wait_for_paint_settled(client, &effective_session_id).await; + Ok(()) +} + pub async fn dblclick( client: &CdpClient, session_id: &str, @@ -267,6 +314,13 @@ pub async fn dblclick( selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { + // Same relay/iframe drift hazard as a single click — DOM-dispatch the + // double-click there instead of a coordinate one (issues #31/#36). + if std::env::var("AGENT_BROWSER_CLICK_MODE").as_deref() != Ok("coord") + && prefer_dom_dispatch(ref_map, selector_or_ref) + { + return dom_dblclick(client, session_id, ref_map, selector_or_ref, iframe_sessions).await; + } click( client, session_id, @@ -279,6 +333,50 @@ pub async fn dblclick( .await } +/// DOM-dispatch a hover (pointer/mouse enter+move) on the element in its own +/// session — reaches OOPIF elements and never drifts to the foreground tab over +/// the relay, unlike a coordinate `mouseMoved` (issues #31/#36). +async fn dom_hover( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, + selector_or_ref: &str, + iframe_sessions: &HashMap, +) -> Result<(), String> { + let (object_id, effective_session_id) = resolve_element_object_id( + client, + session_id, + ref_map, + selector_or_ref, + iframe_sessions, + ) + .await?; + client + .send_command_typed::<_, Value>( + "Runtime.callFunctionOn", + &CallFunctionOnParams { + function_declaration: r#"function() { + const r = this.getBoundingClientRect(); + const cx = r.left + r.width / 2, cy = r.top + r.height / 2; + const base = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy }; + this.dispatchEvent(new PointerEvent('pointerover', base)); + this.dispatchEvent(new PointerEvent('pointerenter', { ...base, bubbles: false })); + this.dispatchEvent(new MouseEvent('mouseover', base)); + this.dispatchEvent(new MouseEvent('mouseenter', { ...base, bubbles: false })); + this.dispatchEvent(new MouseEvent('mousemove', base)); + }"# + .to_string(), + object_id: Some(object_id), + arguments: None, + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(&effective_session_id), + ) + .await?; + Ok(()) +} + pub async fn hover( client: &CdpClient, session_id: &str, @@ -286,6 +384,11 @@ pub async fn hover( selector_or_ref: &str, iframe_sessions: &HashMap, ) -> Result<(), String> { + // Coordinate `mouseMoved` drifts to the foreground tab over the relay and + // can't reach an OOPIF — DOM-dispatch the hover there (issues #31/#36). + if prefer_dom_dispatch(ref_map, selector_or_ref) { + return dom_hover(client, session_id, ref_map, selector_or_ref, iframe_sessions).await; + } let (x, y, _w, _h, effective_session_id) = resolve_element_center( client, session_id, @@ -314,6 +417,63 @@ pub async fn hover( Ok(()) } +/// DOM-dispatch an HTML5 drag-and-drop from `source` to `target` in their shared +/// session — the relay/iframe-safe counterpart to the coordinate drag, which +/// drifts to the foreground tab over the relay and can't reach an OOPIF (issues +/// #31/#36). Covers HTML5 DnD (sortable lists, file/card boards); pointer-driven +/// drag (canvas, sliders) still needs the coordinate path. Errors if source and +/// target live in different frames — a synthetic cross-frame DnD isn't reliable. +pub async fn dom_drag( + client: &CdpClient, + session_id: &str, + ref_map: &RefMap, + source: &str, + target: &str, + iframe_sessions: &HashMap, +) -> Result<(), String> { + let (src_obj, src_session) = + resolve_element_object_id(client, session_id, ref_map, source, iframe_sessions).await?; + let (tgt_obj, tgt_session) = + resolve_element_object_id(client, session_id, ref_map, target, iframe_sessions).await?; + if src_session != tgt_session { + return Err( + "drag source and target are in different frames; cross-frame drag-and-drop over the \ + relay isn't supported — drag within a single frame, or use a launched browser with \ + AGENT_BROWSER_CLICK_MODE=coord" + .to_string(), + ); + } + client + .send_command_typed::<_, Value>( + "Runtime.callFunctionOn", + &CallFunctionOnParams { + function_declaration: r#"function(target) { + const dt = new DataTransfer(); + const ev = (type, el) => el.dispatchEvent( + new DragEvent(type, { bubbles: true, cancelable: true, dataTransfer: dt })); + ev('dragstart', this); + ev('drag', this); + ev('dragenter', target); + ev('dragover', target); + ev('drop', target); + ev('dragend', this); + }"# + .to_string(), + object_id: Some(src_obj), + arguments: Some(vec![CallArgument { + value: None, + object_id: Some(tgt_obj), + }]), + return_by_value: Some(true), + await_promise: Some(false), + }, + Some(&src_session), + ) + .await?; + wait_for_paint_settled(client, &src_session).await; + Ok(()) +} + pub async fn fill( client: &CdpClient, session_id: &str, @@ -397,6 +557,7 @@ pub async fn type_text( clear: bool, delay_ms: Option, iframe_sessions: &HashMap, + key_events: bool, ) -> Result<(), String> { let (object_id, effective_session_id) = resolve_element_object_id( client, @@ -443,7 +604,7 @@ pub async fn type_text( .await?; } - type_text_into_active_context(client, session_id, text, delay_ms).await + type_text_into_active_context(client, session_id, text, delay_ms, key_events).await } pub async fn type_text_into_active_context( @@ -451,6 +612,7 @@ pub async fn type_text_into_active_context( session_id: &str, text: &str, delay_ms: Option, + key_events: bool, ) -> Result<(), String> { // Per-character timing: an explicit `delay_ms` wins (caller asked for a // fixed cadence); otherwise fall back to humanize — variable, human-like @@ -500,6 +662,46 @@ pub async fn type_text_into_active_context( Some(session_id), ) .await?; + } else if key_events { + // Real keystrokes (keyDown+keyUp carrying `text`) for autocomplete / + // combobox widgets that only react to key events and ignore the + // `input` that `Input.insertText` fires — e.g. Google's address + // postal-code → city/prefecture lookup (issue #36 / #4). The keyDown's + // `text` still inserts the character, so the field also fills. + let (key, code, key_code) = char_to_key_info(ch); + let s = ch.to_string(); + client + .send_command_typed::<_, Value>( + "Input.dispatchKeyEvent", + &DispatchKeyEventParams { + event_type: "keyDown".to_string(), + key: Some(key.clone()), + code: Some(code.clone()), + text: Some(s.clone()), + unmodified_text: Some(s), + windows_virtual_key_code: Some(key_code), + native_virtual_key_code: Some(key_code), + modifiers: None, + }, + Some(session_id), + ) + .await?; + client + .send_command_typed::<_, Value>( + "Input.dispatchKeyEvent", + &DispatchKeyEventParams { + event_type: "keyUp".to_string(), + key: Some(key), + code: Some(code), + text: None, + unmodified_text: None, + windows_virtual_key_code: Some(key_code), + native_virtual_key_code: Some(key_code), + modifiers: None, + }, + Some(session_id), + ) + .await?; } else { // VS Code/Electron webviews reject repeated dispatchKeyEvent calls // carrying printable `text`. Insert printable characters directly diff --git a/cli/src/native/snapshot.rs b/cli/src/native/snapshot.rs index 142d99a..09085d1 100644 --- a/cli/src/native/snapshot.rs +++ b/cli/src/native/snapshot.rs @@ -330,6 +330,13 @@ impl RoleNameTracker { } } +/// Max iframe nesting depth `take_snapshot` expands. Embedded payment/checkout +/// widgets nest a few frames deep (e.g. AdSense → payments.google.com → an inner +/// form frame); expanding past the first level is what gives those inner refs a +/// `frame_id` so clicks resolve into the right frame (issue #36). Capped to keep +/// a pathological frame tree from blowing up the snapshot. +const MAX_IFRAME_DEPTH: usize = 3; + pub async fn take_snapshot( client: &CdpClient, session_id: &str, @@ -337,6 +344,19 @@ pub async fn take_snapshot( ref_map: &mut RefMap, frame_id: Option<&str>, iframe_sessions: &HashMap, +) -> Result { + take_snapshot_at_depth(client, session_id, options, ref_map, frame_id, iframe_sessions, 0).await +} + +#[allow(clippy::too_many_arguments)] +async fn take_snapshot_at_depth( + client: &CdpClient, + session_id: &str, + options: &SnapshotOptions, + ref_map: &mut RefMap, + frame_id: Option<&str>, + iframe_sessions: &HashMap, + depth: usize, ) -> Result { client .send_command_no_params("DOM.enable", Some(session_id)) @@ -606,10 +626,11 @@ pub async fn take_snapshot( } // Recurse into child iframes: for each Iframe node with a backend_node_id, - // resolve the child frame ID and take a snapshot of its content. - // We only recurse from the main frame (frame_id == None) to avoid - // unbounded depth; nested iframes within iframes are not expanded. - if frame_id.is_none() { + // resolve the child frame ID and snapshot its content. Recurse to + // MAX_IFRAME_DEPTH (not just the main frame) so refs inside nested + // payment/checkout widgets get a `frame_id` and clicks resolve into the right + // frame (issue #36); the cap bounds a pathological frame tree. + if depth < MAX_IFRAME_DEPTH { let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); // (ref_id, child_snapshot) for node in tree_nodes.iter() { if node.role != "Iframe" || !node.has_ref { @@ -622,13 +643,14 @@ pub async fn take_snapshot( if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await { // Snapshot the child frame; errors are silently ignored // (e.g. cross-origin iframes) - if let Ok(child_text) = Box::pin(take_snapshot( + if let Ok(child_text) = Box::pin(take_snapshot_at_depth( client, session_id, options, ref_map, Some(&child_fid), iframe_sessions, + depth + 1, )) .await { diff --git a/cli/src/output.rs b/cli/src/output.rs index 999c79d..1aec9ea 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -1464,6 +1464,12 @@ Usage: chrome-use type Types text into the specified element character by character. Unlike fill, this does not clear existing content first. +Options: + --key-events Send real per-character keyDown/keyUp instead of + (alias --keys) Input.insertText. Use for autocomplete / combobox fields + that only react to key events — e.g. a postal-code box + that auto-fills city/prefecture, or Google Places. + Global Options: --json Output as JSON --session Use specific session @@ -1471,6 +1477,7 @@ Global Options: Examples: chrome-use type "#search" "hello" chrome-use type @e2 "additional text" + chrome-use type @e5 "201-0001" --key-events # trigger the address autocomplete See Also: For typing into contenteditable editors (Lexical, ProseMirror, etc.) diff --git a/skill-data/core/SKILL.md b/skill-data/core/SKILL.md index 3371b05..d2075bd 100644 --- a/skill-data/core/SKILL.md +++ b/skill-data/core/SKILL.md @@ -36,6 +36,18 @@ Refs (`@e1`, `@e2`, ...) are assigned fresh on every snapshot. They become submits, dynamic re-renders, dialog opens. Always re-snapshot before your next ref interaction. +> **Snapshot-first, always. Never default to `screenshot` + coordinate clicking +> for form fields or buttons.** Run `snapshot -i` and act on `@refs`. Use +> coordinates only for canvas/WebGL, or when `snapshot` genuinely returns nothing +> for your target. This holds **even inside cross-origin embedded iframes** — +> since v1.5.12 `snapshot -i` pierces out-of-process iframes (Google Payments, +> Stripe, embedded checkout/KYC) and lists their elements with refs, so +> `click @e` / `type @e` / `fill @e` work directly. A screenshot is for a genuine +> *visual* check you report to the user — not your own input. (Full-page +> screenshots of a real retina Chrome are often too large for the image reader +> anyway.) Driving off pixels on the relay also risks a coordinate event drifting +> onto the user's foreground tab — refs never do. See issue #37. + ## Before you automate: pick the cheapest tool Driving a browser is the heavy option. chrome-use earns its keep when you @@ -264,6 +276,10 @@ chrome-use hover @e1 # hover chrome-use focus @e1 # focus (useful before keyboard input) chrome-use fill @e2 "hello" # clear then type chrome-use type @e2 " world" # type without clearing +chrome-use type @e5 "201-0001" --key-events # real keystrokes (not insertText) — + # use for autocomplete/combobox fields that + # only react to key events (e.g. a postal box + # that auto-fills city/prefecture, Google Places) chrome-use press Enter # press a key at current focus (down+up) chrome-use press Control+a # key combination chrome-use keydown d # HOLD a key down (no auto-release) @@ -295,6 +311,18 @@ chrome-use scrollintoview @e1 # scroll element into view chrome-use drag @e1 @e2 # drag and drop ``` +**Cross-origin iframes (embedded payment / checkout / KYC widgets — Google +Payments, Stripe, etc.) — drive them by ref, never by screenshot.** `snapshot -i` +pierces these out-of-process iframes and lists their elements by `@ref` +(including input values); `get text --all-frames` reads their text. Then just act +on the refs: `click @e`, `type @e`, `hover @e`, `dblclick @e`, `drag @a @b` all +work into the iframe. Over the extension relay these are dispatched through the +DOM (in the element's own frame), so they hit the right element in the right tab +— a coordinate click/scroll there can drift onto whatever tab is in the +foreground, so prefer refs. For below-the-fold content in such a frame, scroll it +with `scroll down N --at x,y` (a pixel over the frame) or `--frame n`. For a +postal/autocomplete box inside the frame, `type @e "…" --key-events`. + ### When refs don't work or you don't want to snapshot Use semantic locators: