From a83d1b1df96be1639cfe8a02901f372a1c886d63 Mon Sep 17 00:00:00 2001 From: leeguooooo Date: Wed, 17 Jun 2026 17:49:38 +0900 Subject: [PATCH] feat: rich-editor fill, box centers, screenshot downscale, disabled+docs (#41-#45) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dogfooding backlog from this session's embedded-form/editor work. #41 fill on rich editors: detect CodeMirror 5 / Monaco / ProseMirror / contenteditable and set via their own API or execCommand('insertText') so beforeinput/input fire (a raw .value/textContent write no-op'd juejin's CodeMirror and skipped React composers). Response echoes the `engine` used. `fill --file ` / `--stdin` set large multiline text without shell-escaping. `get value` now reads CodeMirror/Monaco/contenteditable too. #42 screenshot --max-width/--max-height/--scale, plus a default 2000px longest-edge cap (AGENT_BROWSER_SCREENSHOT_MAX_EDGE; 0 disables) so retina full-page shots fit an agent's image reader and --scale 0.5 makes screenshot px line up with click px. Annotated shots are never downscaled. #43 `box @ref` (already a top-level alias of `get box`) now also returns centerX/centerY/inViewport in CSS px — feed straight into `click x y` when a ref-click no-ops (e.g. a button in a cross-origin iframe). #44 no code change needed — disabled elements already list as `button "Save" [disabled, ref=eN]`; the reporter's missing button was DOM-gated on validity. Added a skill note: `find text` can't reach into a cross-origin iframe — target those by snapshot @ref. #45 core skill now distinguishes screenshot-to-locate (discouraged) from screenshot-to-capture a reusable image asset via `screenshot [--clip] ` (encouraged), so agents stop over-reading the prohibition. #40 (group-scoped relay) stays deferred — needs an ab-connect extension change. Verified live: fill --file round-trips multiline+CJK+backticks; contenteditable engine=contenteditable + get value reads it back; box gives centerX/centerY/ inViewport; screenshot of retina example.com → 2000px; disabled button shows [disabled]. 856 tests pass. --- cli/Cargo.lock | 2 +- cli/Cargo.toml | 2 +- cli/src/commands.rs | 86 ++++++++++++++++++++++++++++++++- cli/src/native/actions.rs | 89 ++++++++++++++++++++++++++++++++++- cli/src/native/element.rs | 34 +++++++++++-- cli/src/native/interaction.rs | 69 ++++++++++++++++++++------- cli/src/output.rs | 46 ++++++++++++++++-- package.json | 2 +- skill-data/core/SKILL.md | 18 +++++++ 9 files changed, 315 insertions(+), 33 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 5fcafe9..880aaee 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrome-use" -version = "1.5.20" +version = "1.5.21" dependencies = [ "aes", "aes-gcm", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ed944f7..1e425ac 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chrome-use" -version = "1.5.20" +version = "1.5.21" edition = "2021" description = "Fast browser automation CLI for AI agents" license = "Apache-2.0" diff --git a/cli/src/commands.rs b/cli/src/commands.rs index 1515fa2..dfb5cdb 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -79,6 +79,7 @@ const KNOWN_COMMANDS: &[&str] = &[ "dialog", "upload", "site", + "box", ]; /// Levenshtein distance, capped — small inputs only (command names). @@ -560,9 +561,37 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result { let sel = rest.first().ok_or_else(|| ParseError::MissingArguments { context: "fill".to_string(), - usage: "fill ", + usage: "fill | fill --file | fill --stdin", })?; - Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": rest[1..].join(" ") })) + // Large/multiline content without shell-escaping hell (issue #41): + // `fill --file ` reads the value from a UTF-8 file, and + // `fill --stdin` reads it from stdin — sent verbatim, so backticks, + // quotes, newlines and non-ASCII pass through untouched. + let value = match rest.get(1).copied() { + Some("--file") => { + let path = rest.get(2).ok_or(ParseError::InvalidValue { + message: "fill --file requires a path".to_string(), + usage: "fill --file ", + })?; + std::fs::read_to_string(path).map_err(|e| ParseError::InvalidValue { + message: format!("fill --file: cannot read {path}: {e}"), + usage: "fill --file ", + })? + } + Some("--stdin") => { + use std::io::Read; + let mut buf = String::new(); + io::stdin() + .read_to_string(&mut buf) + .map_err(|e| ParseError::InvalidValue { + message: format!("fill --stdin: {e}"), + usage: "fill --stdin", + })?; + buf + } + _ => rest[1..].join(" "), + }; + Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": value })) } "type" => { // `--key-events` (alias `--keys`): send real per-character keystrokes @@ -1006,11 +1035,55 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result = None; + let mut max_width: Option = None; + let mut max_height: Option = None; + let mut scale: Option = None; let mut positional: Vec<&str> = Vec::new(); let mut i = 0; + // Parse a numeric value for a downscale flag (issue #42). + let parse_num = |i: &mut usize, flag: &str| -> Result { + let v = rest + .get(*i + 1) + .ok_or_else(|| ParseError::MissingArguments { + context: format!("screenshot {flag}"), + usage: "screenshot [--max-width ] [--max-height ] [--scale <0..1>]", + })?; + *i += 1; + Ok(v.to_string()) + }; while i < rest.len() { match rest[i] { "--full" | "-f" => full_page = true, + // Downscale the saved image so retina/full-page shots fit an + // agent's image reader and screenshot px line up with click px (#42). + "--max-width" => { + let v = parse_num(&mut i, "--max-width")?; + max_width = Some(v.parse().map_err(|_| ParseError::InvalidValue { + message: format!("--max-width expects a number, got '{v}'"), + usage: "screenshot --max-width ", + })?); + } + "--max-height" => { + let v = parse_num(&mut i, "--max-height")?; + max_height = Some(v.parse().map_err(|_| ParseError::InvalidValue { + message: format!("--max-height expects a number, got '{v}'"), + usage: "screenshot --max-height ", + })?); + } + "--scale" => { + let v = parse_num(&mut i, "--scale")?; + let s: f64 = v.parse().map_err(|_| ParseError::InvalidValue { + message: format!("--scale expects a number like 0.5, got '{v}'"), + usage: "screenshot --scale <0..1>", + })?; + if s <= 0.0 || s > 1.0 { + return Err(ParseError::InvalidValue { + message: format!("--scale must be in (0, 1], got '{v}'"), + usage: "screenshot --scale <0..1>", + }); + } + scale = Some(s); + } // `--clip x,y,w,h` captures a pixel region (issue #34). "--clip" => { let raw = rest @@ -1073,6 +1146,15 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result Result = None; + if !annotate { + let scale = cmd.get("scale").and_then(|v| v.as_f64()); + let max_w = cmd + .get("maxWidth") + .and_then(|v| v.as_u64()) + .map(|v| v as u32); + let max_h = cmd + .get("maxHeight") + .and_then(|v| v.as_u64()) + .map(|v| v as u32); + let default_edge = if scale.is_none() && max_w.is_none() && max_h.is_none() { + std::env::var("AGENT_BROWSER_SCREENSHOT_MAX_EDGE") + .ok() + .and_then(|s| s.parse::().ok()) + .or(Some(2000)) + .filter(|&e| e > 0) + } else { + None + }; + resized = downscale_screenshot(&result.path, scale, max_w, max_h, default_edge); + } + let mut response = json!({ "path": absolutize_saved_path(&result.path) }); + if let Some((w, h)) = resized { + response["width"] = json!(w); + response["height"] = json!(h); + response["resized"] = json!(true); + } if !result.annotations.is_empty() { response["annotations"] = serde_json::to_value(&result.annotations) .map_err(|e| format!("Failed to serialize annotations: {}", e))?; @@ -3130,6 +3163,56 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result, + max_w: Option, + max_h: Option, + default_edge: Option, +) -> Option<(u32, u32)> { + let img = image::open(path).ok()?; + let (w, h) = (img.width(), img.height()); + if w == 0 || h == 0 { + return None; + } + + // Collect candidate scale factors (≤ 1.0); the smallest wins. + let mut factor = 1.0f64; + if let Some(s) = scale { + factor = factor.min(s); + } + if let Some(mw) = max_w { + if w > mw { + factor = factor.min(mw as f64 / w as f64); + } + } + if let Some(mh) = max_h { + if h > mh { + factor = factor.min(mh as f64 / h as f64); + } + } + if let Some(edge) = default_edge { + let longest = w.max(h); + if longest > edge { + factor = factor.min(edge as f64 / longest as f64); + } + } + + if factor >= 1.0 { + return None; // already within bounds — never upscale + } + let nw = ((w as f64 * factor).round() as u32).max(1); + let nh = ((h as f64 * factor).round() as u32).max(1); + let resized = img.resize(nw, nh, image::imageops::FilterType::Lanczos3); + resized.save(path).ok()?; + Some((resized.width(), resized.height())) +} + async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result { // First-class coordinate click (issue #8.4): click a raw viewport point with // no element resolution. Parsed from `click ` / `click --coords x,y`. @@ -3288,7 +3371,7 @@ async fn handle_fill(cmd: &Value, state: &mut DaemonState) -> Result Result Result { diff --git a/cli/src/native/element.rs b/cli/src/native/element.rs index d823b4b..4d6736b 100644 --- a/cli/src/native/element.rs +++ b/cli/src/native/element.rs @@ -1529,9 +1529,27 @@ pub async fn get_element_input_value( .send_command_typed( "Runtime.callFunctionOn", &CallFunctionOnParams { - function_declaration: - "function() { return typeof this.value === 'string' ? this.value : ''; }" - .to_string(), + // Read rich-editor content too (issue #41): CodeMirror 5 / Monaco + // keep their text in a model, not `.value`; contenteditable keeps + // it as innerText. Falls back to `.value` for plain inputs. + function_declaration: r#"function() { + const el = this; + const cm5 = el.closest && el.closest('.CodeMirror'); + if (cm5 && cm5.CodeMirror) return cm5.CodeMirror.getValue(); + if (window.monaco && monaco.editor) { + try { + const eds = monaco.editor.getEditors ? monaco.editor.getEditors() : []; + const ed = eds.find(e => e.getDomNode && e.getDomNode().contains(el)) || eds[0]; + if (ed) return ed.getValue(); + const m = monaco.editor.getModels ? monaco.editor.getModels() : []; + if (m[0]) return m[0].getValue(); + } catch (e) {} + } + if (typeof el.value === 'string') return el.value; + if (el.isContentEditable) return el.innerText; + return ''; + }"# + .to_string(), object_id: Some(object_id), arguments: None, return_by_value: Some(true), @@ -1609,7 +1627,15 @@ pub async fn get_element_bounding_box( &CallFunctionOnParams { function_declaration: r#"function() { const r = this.getBoundingClientRect(); - return { x: r.x, y: r.y, width: r.width, height: r.height }; + const inViewport = r.bottom > 0 && r.right > 0 + && r.top < (innerHeight || document.documentElement.clientHeight) + && r.left < (innerWidth || document.documentElement.clientWidth); + return { + x: r.x, y: r.y, width: r.width, height: r.height, + centerX: Math.round(r.x + r.width / 2), + centerY: Math.round(r.y + r.height / 2), + inViewport, + }; }"# .to_string(), object_id: Some(object_id), diff --git a/cli/src/native/interaction.rs b/cli/src/native/interaction.rs index 546c933..c912f3e 100644 --- a/cli/src/native/interaction.rs +++ b/cli/src/native/interaction.rs @@ -579,7 +579,7 @@ pub async fn fill( selector_or_ref: &str, value: &str, iframe_sessions: &HashMap, -) -> Result<(), String> { +) -> Result { let (object_id, effective_session_id) = resolve_element_object_id( client, session_id, @@ -590,14 +590,15 @@ pub async fn fill( .await?; // Emulate a real edit so framework-controlled inputs (React/Vue) and - // site-side listeners actually see the change (issue #25): the old path set - // `this.value` directly and used Input.insertText, which left React's - // internal value-tracker out of sync and never fired change/blur — so - // dependent logic (e.g. Mercari's postal-code → 都道府県 autocomplete) never - // ran even though the value was visible. Set the value through the element's - // PROTOTYPE setter (which React's _valueTracker hooks), then dispatch - // input → change → blur/focusout. `type ` remains for sites that - // need per-keystroke events. + // site-side listeners actually see the change (issue #25): set the value + // through the element's PROTOTYPE setter (which React's _valueTracker hooks), + // then dispatch input → change → blur/focusout. Beyond plain inputs, detect + // rich editors and use their own API/events (issue #41): CodeMirror 5 and + // Monaco have a model that `.value`/`textContent` can't touch; ProseMirror / + // contenteditable need `execCommand('insertText')` so beforeinput/input fire + // (a raw `textContent =` corrupts PM's doc and skips React composers). + // Returns the engine used so the caller can report it. `type ` + // remains for sites that need per-keystroke events. let fill_js = format!( r#"function() {{ const el = this; @@ -605,13 +606,43 @@ pub async fn fill( try {{ el.focus(); }} catch (e) {{}} const tag = el.tagName; const fire = (type, ctor) => el.dispatchEvent(new (ctor || Event)(type, {{ bubbles: true }})); - if (tag === 'SELECT') {{ - el.value = v; fire('input'); fire('change'); return true; + + // CodeMirror 5: a hidden