feat: rich-editor fill, box centers, screenshot downscale, disabled+docs (#41-#45)
Release binaries / Build macOS ARM64 (push) Has been cancelled
Release binaries / Build macOS x64 (push) Has been cancelled
Release binaries / Build Linux ARM64 (push) Has been cancelled
Release binaries / Build Linux musl ARM64 (push) Has been cancelled
Release binaries / Build Linux musl x64 (push) Has been cancelled
Release binaries / Build Linux x64 (push) Has been cancelled
Release binaries / Build Windows x64 (push) Has been cancelled
Release binaries / Attach binaries to GitHub Release (push) Has been cancelled

Dogfooding backlog from this session's embedded-form/editor work.

#41 fill on rich editors: detect CodeMirror 5 / Monaco / ProseMirror /
contenteditable and set via their own API or execCommand('insertText') so
beforeinput/input fire (a raw .value/textContent write no-op'd juejin's
CodeMirror and skipped React composers). Response echoes the `engine` used.
`fill <sel> --file <path>` / `--stdin` set large multiline text without
shell-escaping. `get value` now reads CodeMirror/Monaco/contenteditable too.

#42 screenshot --max-width/--max-height/--scale, plus a default 2000px
longest-edge cap (AGENT_BROWSER_SCREENSHOT_MAX_EDGE; 0 disables) so retina
full-page shots fit an agent's image reader and --scale 0.5 makes screenshot
px line up with click px. Annotated shots are never downscaled.

#43 `box @ref` (already a top-level alias of `get box`) now also returns
centerX/centerY/inViewport in CSS px — feed straight into `click x y` when a
ref-click no-ops (e.g. a button in a cross-origin iframe).

#44 no code change needed — disabled elements already list as
`button "Save" [disabled, ref=eN]`; the reporter's missing button was
DOM-gated on validity. Added a skill note: `find text` can't reach into a
cross-origin iframe — target those by snapshot @ref.

#45 core skill now distinguishes screenshot-to-locate (discouraged) from
screenshot-to-capture a reusable image asset via `screenshot [--clip] <file>`
(encouraged), so agents stop over-reading the prohibition.

#40 (group-scoped relay) stays deferred — needs an ab-connect extension change.

Verified live: fill --file round-trips multiline+CJK+backticks; contenteditable
engine=contenteditable + get value reads it back; box gives centerX/centerY/
inViewport; screenshot of retina example.com → 2000px; disabled button shows
[disabled]. 856 tests pass.
This commit is contained in:
leeguooooo
2026-06-17 17:49:38 +09:00
parent 50b27ac0e0
commit a83d1b1df9
9 changed files with 315 additions and 33 deletions
+87 -2
View File
@@ -3114,7 +3114,40 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
)
.await?;
// Downscale the saved image so retina/full-page shots fit an agent's image
// reader and screenshot pixels line up with `click x y` CSS px (issue #42).
// Explicit --scale / --max-width / --max-height win; otherwise a default cap
// (2000px longest edge, AGENT_BROWSER_SCREENSHOT_MAX_EDGE overrides, 0 = off)
// applies. Annotated shots are left untouched so ref overlays stay aligned.
let mut resized: Option<(u32, u32)> = None;
if !annotate {
let scale = cmd.get("scale").and_then(|v| v.as_f64());
let max_w = cmd
.get("maxWidth")
.and_then(|v| v.as_u64())
.map(|v| v as u32);
let max_h = cmd
.get("maxHeight")
.and_then(|v| v.as_u64())
.map(|v| v as u32);
let default_edge = if scale.is_none() && max_w.is_none() && max_h.is_none() {
std::env::var("AGENT_BROWSER_SCREENSHOT_MAX_EDGE")
.ok()
.and_then(|s| s.parse::<u32>().ok())
.or(Some(2000))
.filter(|&e| e > 0)
} else {
None
};
resized = downscale_screenshot(&result.path, scale, max_w, max_h, default_edge);
}
let mut response = json!({ "path": absolutize_saved_path(&result.path) });
if let Some((w, h)) = resized {
response["width"] = json!(w);
response["height"] = json!(h);
response["resized"] = json!(true);
}
if !result.annotations.is_empty() {
response["annotations"] = serde_json::to_value(&result.annotations)
.map_err(|e| format!("Failed to serialize annotations: {}", e))?;
@@ -3130,6 +3163,56 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
Ok(response)
}
/// Downscale a saved screenshot in place (issue #42). Resolves the target longest
/// edge from `scale` (fraction of current), explicit `max_w`/`max_h` caps, or a
/// `default_edge` cap — whichever yields the smaller image. Only ever shrinks;
/// no-op (returns None) if the image is already within bounds or can't be read.
/// Returns the new (width, height) when it actually resized.
fn downscale_screenshot(
path: &str,
scale: Option<f64>,
max_w: Option<u32>,
max_h: Option<u32>,
default_edge: Option<u32>,
) -> Option<(u32, u32)> {
let img = image::open(path).ok()?;
let (w, h) = (img.width(), img.height());
if w == 0 || h == 0 {
return None;
}
// Collect candidate scale factors (≤ 1.0); the smallest wins.
let mut factor = 1.0f64;
if let Some(s) = scale {
factor = factor.min(s);
}
if let Some(mw) = max_w {
if w > mw {
factor = factor.min(mw as f64 / w as f64);
}
}
if let Some(mh) = max_h {
if h > mh {
factor = factor.min(mh as f64 / h as f64);
}
}
if let Some(edge) = default_edge {
let longest = w.max(h);
if longest > edge {
factor = factor.min(edge as f64 / longest as f64);
}
}
if factor >= 1.0 {
return None; // already within bounds — never upscale
}
let nw = ((w as f64 * factor).round() as u32).max(1);
let nh = ((h as f64 * factor).round() as u32).max(1);
let resized = img.resize(nw, nh, image::imageops::FilterType::Lanczos3);
resized.save(path).ok()?;
Some((resized.width(), resized.height()))
}
async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
// First-class coordinate click (issue #8.4): click a raw viewport point with
// no element resolution. Parsed from `click <x> <y>` / `click --coords x,y`.
@@ -3288,7 +3371,7 @@ async fn handle_fill(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
let session_id = mgr.active_session_id()?.to_string();
interaction::fill(
let engine = interaction::fill(
&mgr.client,
&session_id,
&state.ref_map,
@@ -3297,7 +3380,9 @@ async fn handle_fill(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
&state.iframe_sessions,
)
.await?;
Ok(json!({ "filled": selector }))
// Echo the input path used (input/contenteditable/codemirror5/monaco/select)
// so the agent can confirm a rich editor was handled, not silently no-op'd (#41).
Ok(json!({ "filled": selector, "engine": engine }))
}
async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
+30 -4
View File
@@ -1529,9 +1529,27 @@ pub async fn get_element_input_value(
.send_command_typed(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration:
"function() { return typeof this.value === 'string' ? this.value : ''; }"
.to_string(),
// Read rich-editor content too (issue #41): CodeMirror 5 / Monaco
// keep their text in a model, not `.value`; contenteditable keeps
// it as innerText. Falls back to `.value` for plain inputs.
function_declaration: r#"function() {
const el = this;
const cm5 = el.closest && el.closest('.CodeMirror');
if (cm5 && cm5.CodeMirror) return cm5.CodeMirror.getValue();
if (window.monaco && monaco.editor) {
try {
const eds = monaco.editor.getEditors ? monaco.editor.getEditors() : [];
const ed = eds.find(e => e.getDomNode && e.getDomNode().contains(el)) || eds[0];
if (ed) return ed.getValue();
const m = monaco.editor.getModels ? monaco.editor.getModels() : [];
if (m[0]) return m[0].getValue();
} catch (e) {}
}
if (typeof el.value === 'string') return el.value;
if (el.isContentEditable) return el.innerText;
return '';
}"#
.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
@@ -1609,7 +1627,15 @@ pub async fn get_element_bounding_box(
&CallFunctionOnParams {
function_declaration: r#"function() {
const r = this.getBoundingClientRect();
return { x: r.x, y: r.y, width: r.width, height: r.height };
const inViewport = r.bottom > 0 && r.right > 0
&& r.top < (innerHeight || document.documentElement.clientHeight)
&& r.left < (innerWidth || document.documentElement.clientWidth);
return {
x: r.x, y: r.y, width: r.width, height: r.height,
centerX: Math.round(r.x + r.width / 2),
centerY: Math.round(r.y + r.height / 2),
inViewport,
};
}"#
.to_string(),
object_id: Some(object_id),
+52 -17
View File
@@ -579,7 +579,7 @@ pub async fn fill(
selector_or_ref: &str,
value: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
) -> Result<String, String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
@@ -590,14 +590,15 @@ pub async fn fill(
.await?;
// Emulate a real edit so framework-controlled inputs (React/Vue) and
// site-side listeners actually see the change (issue #25): the old path set
// `this.value` directly and used Input.insertText, which left React's
// internal value-tracker out of sync and never fired change/blur — so
// dependent logic (e.g. Mercari's postal-code → 都道府県 autocomplete) never
// ran even though the value was visible. Set the value through the element's
// PROTOTYPE setter (which React's _valueTracker hooks), then dispatch
// input → change → blur/focusout. `type <sel> <text>` remains for sites that
// need per-keystroke events.
// site-side listeners actually see the change (issue #25): set the value
// through the element's PROTOTYPE setter (which React's _valueTracker hooks),
// then dispatch input → changeblur/focusout. Beyond plain inputs, detect
// rich editors and use their own API/events (issue #41): CodeMirror 5 and
// Monaco have a model that `.value`/`textContent` can't touch; ProseMirror /
// contenteditable need `execCommand('insertText')` so beforeinput/input fire
// (a raw `textContent =` corrupts PM's doc and skips React composers).
// Returns the engine used so the caller can report it. `type <sel> <text>`
// remains for sites that need per-keystroke events.
let fill_js = format!(
r#"function() {{
const el = this;
@@ -605,13 +606,43 @@ pub async fn fill(
try {{ el.focus(); }} catch (e) {{}}
const tag = el.tagName;
const fire = (type, ctor) => el.dispatchEvent(new (ctor || Event)(type, {{ bubbles: true }}));
if (tag === 'SELECT') {{
el.value = v; fire('input'); fire('change'); return true;
// CodeMirror 5: a hidden <textarea> inside .CodeMirror with a live instance.
const cm5 = el.closest && el.closest('.CodeMirror');
if (cm5 && cm5.CodeMirror) {{ cm5.CodeMirror.setValue(v); return 'codemirror5'; }}
// Monaco: global `monaco`; prefer the editor whose DOM contains el.
if (window.monaco && monaco.editor) {{
try {{
const eds = monaco.editor.getEditors ? monaco.editor.getEditors() : [];
const ed = eds.find(e => e.getDomNode && e.getDomNode().contains(el)) || eds[0];
if (ed) {{ ed.setValue(v); return 'monaco'; }}
const models = monaco.editor.getModels ? monaco.editor.getModels() : [];
if (models[0]) {{ models[0].setValue(v); return 'monaco'; }}
}} catch (e) {{}}
}}
if (tag === 'SELECT') {{ el.value = v; fire('input'); fire('change'); return 'select'; }}
if (el.isContentEditable) {{
el.textContent = v; fire('input', window.InputEvent || Event); fire('change');
try {{ el.blur(); }} catch (e) {{}} fire('focusout'); return true;
// ProseMirror / contenteditable: select-all then insertText fires
// beforeinput/input that PM and React composers listen for.
let ok = false;
try {{
const sel = window.getSelection();
const range = document.createRange();
range.selectNodeContents(el);
sel.removeAllRanges();
sel.addRange(range);
ok = document.execCommand('insertText', false, v);
}} catch (e) {{}}
if (!ok) {{ el.textContent = v; fire('input', window.InputEvent || Event); }}
fire('change');
try {{ el.blur(); }} catch (e) {{}}
fire('focusout');
return ok ? 'contenteditable' : 'contenteditable-fallback';
}}
const proto = tag === 'TEXTAREA' ? window.HTMLTextAreaElement.prototype
: window.HTMLInputElement.prototype;
const desc = Object.getOwnPropertyDescriptor(proto, 'value');
@@ -623,13 +654,13 @@ pub async fn fill(
fire('change');
try {{ el.blur(); }} catch (e) {{}}
fire('focusout'); // blur-triggered lookups/validation
return true;
return 'input';
}}"#,
val = serde_json::to_string(value).unwrap_or_default()
);
client
.send_command_typed::<_, Value>(
let result: EvaluateResult = client
.send_command_typed(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: fill_js,
@@ -642,7 +673,11 @@ pub async fn fill(
)
.await?;
Ok(())
Ok(result
.result
.value
.and_then(|v| v.as_str().map(String::from))
.unwrap_or_else(|| "input".to_string()))
}
#[allow(clippy::too_many_arguments)]