feat: rich-editor fill, box centers, screenshot downscale, disabled+docs (#41-#45)
Release binaries / Build macOS ARM64 (push) Has been cancelled
Release binaries / Build macOS x64 (push) Has been cancelled
Release binaries / Build Linux ARM64 (push) Has been cancelled
Release binaries / Build Linux musl ARM64 (push) Has been cancelled
Release binaries / Build Linux musl x64 (push) Has been cancelled
Release binaries / Build Linux x64 (push) Has been cancelled
Release binaries / Build Windows x64 (push) Has been cancelled
Release binaries / Attach binaries to GitHub Release (push) Has been cancelled
Release binaries / Build macOS ARM64 (push) Has been cancelled
Release binaries / Build macOS x64 (push) Has been cancelled
Release binaries / Build Linux ARM64 (push) Has been cancelled
Release binaries / Build Linux musl ARM64 (push) Has been cancelled
Release binaries / Build Linux musl x64 (push) Has been cancelled
Release binaries / Build Linux x64 (push) Has been cancelled
Release binaries / Build Windows x64 (push) Has been cancelled
Release binaries / Attach binaries to GitHub Release (push) Has been cancelled
Dogfooding backlog from this session's embedded-form/editor work. #41 fill on rich editors: detect CodeMirror 5 / Monaco / ProseMirror / contenteditable and set via their own API or execCommand('insertText') so beforeinput/input fire (a raw .value/textContent write no-op'd juejin's CodeMirror and skipped React composers). Response echoes the `engine` used. `fill <sel> --file <path>` / `--stdin` set large multiline text without shell-escaping. `get value` now reads CodeMirror/Monaco/contenteditable too. #42 screenshot --max-width/--max-height/--scale, plus a default 2000px longest-edge cap (AGENT_BROWSER_SCREENSHOT_MAX_EDGE; 0 disables) so retina full-page shots fit an agent's image reader and --scale 0.5 makes screenshot px line up with click px. Annotated shots are never downscaled. #43 `box @ref` (already a top-level alias of `get box`) now also returns centerX/centerY/inViewport in CSS px — feed straight into `click x y` when a ref-click no-ops (e.g. a button in a cross-origin iframe). #44 no code change needed — disabled elements already list as `button "Save" [disabled, ref=eN]`; the reporter's missing button was DOM-gated on validity. Added a skill note: `find text` can't reach into a cross-origin iframe — target those by snapshot @ref. #45 core skill now distinguishes screenshot-to-locate (discouraged) from screenshot-to-capture a reusable image asset via `screenshot [--clip] <file>` (encouraged), so agents stop over-reading the prohibition. #40 (group-scoped relay) stays deferred — needs an ab-connect extension change. Verified live: fill --file round-trips multiline+CJK+backticks; contenteditable engine=contenteditable + get value reads it back; box gives centerX/centerY/ inViewport; screenshot of retina example.com → 2000px; disabled button shows [disabled]. 856 tests pass.
This commit is contained in:
@@ -3114,7 +3114,40 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Downscale the saved image so retina/full-page shots fit an agent's image
|
||||
// reader and screenshot pixels line up with `click x y` CSS px (issue #42).
|
||||
// Explicit --scale / --max-width / --max-height win; otherwise a default cap
|
||||
// (2000px longest edge, AGENT_BROWSER_SCREENSHOT_MAX_EDGE overrides, 0 = off)
|
||||
// applies. Annotated shots are left untouched so ref overlays stay aligned.
|
||||
let mut resized: Option<(u32, u32)> = None;
|
||||
if !annotate {
|
||||
let scale = cmd.get("scale").and_then(|v| v.as_f64());
|
||||
let max_w = cmd
|
||||
.get("maxWidth")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|v| v as u32);
|
||||
let max_h = cmd
|
||||
.get("maxHeight")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|v| v as u32);
|
||||
let default_edge = if scale.is_none() && max_w.is_none() && max_h.is_none() {
|
||||
std::env::var("AGENT_BROWSER_SCREENSHOT_MAX_EDGE")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<u32>().ok())
|
||||
.or(Some(2000))
|
||||
.filter(|&e| e > 0)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
resized = downscale_screenshot(&result.path, scale, max_w, max_h, default_edge);
|
||||
}
|
||||
|
||||
let mut response = json!({ "path": absolutize_saved_path(&result.path) });
|
||||
if let Some((w, h)) = resized {
|
||||
response["width"] = json!(w);
|
||||
response["height"] = json!(h);
|
||||
response["resized"] = json!(true);
|
||||
}
|
||||
if !result.annotations.is_empty() {
|
||||
response["annotations"] = serde_json::to_value(&result.annotations)
|
||||
.map_err(|e| format!("Failed to serialize annotations: {}", e))?;
|
||||
@@ -3130,6 +3163,56 @@ async fn handle_screenshot(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Downscale a saved screenshot in place (issue #42). Resolves the target longest
|
||||
/// edge from `scale` (fraction of current), explicit `max_w`/`max_h` caps, or a
|
||||
/// `default_edge` cap — whichever yields the smaller image. Only ever shrinks;
|
||||
/// no-op (returns None) if the image is already within bounds or can't be read.
|
||||
/// Returns the new (width, height) when it actually resized.
|
||||
fn downscale_screenshot(
|
||||
path: &str,
|
||||
scale: Option<f64>,
|
||||
max_w: Option<u32>,
|
||||
max_h: Option<u32>,
|
||||
default_edge: Option<u32>,
|
||||
) -> Option<(u32, u32)> {
|
||||
let img = image::open(path).ok()?;
|
||||
let (w, h) = (img.width(), img.height());
|
||||
if w == 0 || h == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Collect candidate scale factors (≤ 1.0); the smallest wins.
|
||||
let mut factor = 1.0f64;
|
||||
if let Some(s) = scale {
|
||||
factor = factor.min(s);
|
||||
}
|
||||
if let Some(mw) = max_w {
|
||||
if w > mw {
|
||||
factor = factor.min(mw as f64 / w as f64);
|
||||
}
|
||||
}
|
||||
if let Some(mh) = max_h {
|
||||
if h > mh {
|
||||
factor = factor.min(mh as f64 / h as f64);
|
||||
}
|
||||
}
|
||||
if let Some(edge) = default_edge {
|
||||
let longest = w.max(h);
|
||||
if longest > edge {
|
||||
factor = factor.min(edge as f64 / longest as f64);
|
||||
}
|
||||
}
|
||||
|
||||
if factor >= 1.0 {
|
||||
return None; // already within bounds — never upscale
|
||||
}
|
||||
let nw = ((w as f64 * factor).round() as u32).max(1);
|
||||
let nh = ((h as f64 * factor).round() as u32).max(1);
|
||||
let resized = img.resize(nw, nh, image::imageops::FilterType::Lanczos3);
|
||||
resized.save(path).ok()?;
|
||||
Some((resized.width(), resized.height()))
|
||||
}
|
||||
|
||||
async fn handle_click(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||
// First-class coordinate click (issue #8.4): click a raw viewport point with
|
||||
// no element resolution. Parsed from `click <x> <y>` / `click --coords x,y`.
|
||||
@@ -3288,7 +3371,7 @@ async fn handle_fill(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
let session_id = mgr.active_session_id()?.to_string();
|
||||
|
||||
interaction::fill(
|
||||
let engine = interaction::fill(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&state.ref_map,
|
||||
@@ -3297,7 +3380,9 @@ async fn handle_fill(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
||||
&state.iframe_sessions,
|
||||
)
|
||||
.await?;
|
||||
Ok(json!({ "filled": selector }))
|
||||
// Echo the input path used (input/contenteditable/codemirror5/monaco/select)
|
||||
// so the agent can confirm a rich editor was handled, not silently no-op'd (#41).
|
||||
Ok(json!({ "filled": selector, "engine": engine }))
|
||||
}
|
||||
|
||||
async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||
|
||||
@@ -1529,9 +1529,27 @@ pub async fn get_element_input_value(
|
||||
.send_command_typed(
|
||||
"Runtime.callFunctionOn",
|
||||
&CallFunctionOnParams {
|
||||
function_declaration:
|
||||
"function() { return typeof this.value === 'string' ? this.value : ''; }"
|
||||
.to_string(),
|
||||
// Read rich-editor content too (issue #41): CodeMirror 5 / Monaco
|
||||
// keep their text in a model, not `.value`; contenteditable keeps
|
||||
// it as innerText. Falls back to `.value` for plain inputs.
|
||||
function_declaration: r#"function() {
|
||||
const el = this;
|
||||
const cm5 = el.closest && el.closest('.CodeMirror');
|
||||
if (cm5 && cm5.CodeMirror) return cm5.CodeMirror.getValue();
|
||||
if (window.monaco && monaco.editor) {
|
||||
try {
|
||||
const eds = monaco.editor.getEditors ? monaco.editor.getEditors() : [];
|
||||
const ed = eds.find(e => e.getDomNode && e.getDomNode().contains(el)) || eds[0];
|
||||
if (ed) return ed.getValue();
|
||||
const m = monaco.editor.getModels ? monaco.editor.getModels() : [];
|
||||
if (m[0]) return m[0].getValue();
|
||||
} catch (e) {}
|
||||
}
|
||||
if (typeof el.value === 'string') return el.value;
|
||||
if (el.isContentEditable) return el.innerText;
|
||||
return '';
|
||||
}"#
|
||||
.to_string(),
|
||||
object_id: Some(object_id),
|
||||
arguments: None,
|
||||
return_by_value: Some(true),
|
||||
@@ -1609,7 +1627,15 @@ pub async fn get_element_bounding_box(
|
||||
&CallFunctionOnParams {
|
||||
function_declaration: r#"function() {
|
||||
const r = this.getBoundingClientRect();
|
||||
return { x: r.x, y: r.y, width: r.width, height: r.height };
|
||||
const inViewport = r.bottom > 0 && r.right > 0
|
||||
&& r.top < (innerHeight || document.documentElement.clientHeight)
|
||||
&& r.left < (innerWidth || document.documentElement.clientWidth);
|
||||
return {
|
||||
x: r.x, y: r.y, width: r.width, height: r.height,
|
||||
centerX: Math.round(r.x + r.width / 2),
|
||||
centerY: Math.round(r.y + r.height / 2),
|
||||
inViewport,
|
||||
};
|
||||
}"#
|
||||
.to_string(),
|
||||
object_id: Some(object_id),
|
||||
|
||||
@@ -579,7 +579,7 @@ pub async fn fill(
|
||||
selector_or_ref: &str,
|
||||
value: &str,
|
||||
iframe_sessions: &HashMap<String, String>,
|
||||
) -> Result<(), String> {
|
||||
) -> Result<String, String> {
|
||||
let (object_id, effective_session_id) = resolve_element_object_id(
|
||||
client,
|
||||
session_id,
|
||||
@@ -590,14 +590,15 @@ pub async fn fill(
|
||||
.await?;
|
||||
|
||||
// Emulate a real edit so framework-controlled inputs (React/Vue) and
|
||||
// site-side listeners actually see the change (issue #25): the old path set
|
||||
// `this.value` directly and used Input.insertText, which left React's
|
||||
// internal value-tracker out of sync and never fired change/blur — so
|
||||
// dependent logic (e.g. Mercari's postal-code → 都道府県 autocomplete) never
|
||||
// ran even though the value was visible. Set the value through the element's
|
||||
// PROTOTYPE setter (which React's _valueTracker hooks), then dispatch
|
||||
// input → change → blur/focusout. `type <sel> <text>` remains for sites that
|
||||
// need per-keystroke events.
|
||||
// site-side listeners actually see the change (issue #25): set the value
|
||||
// through the element's PROTOTYPE setter (which React's _valueTracker hooks),
|
||||
// then dispatch input → change → blur/focusout. Beyond plain inputs, detect
|
||||
// rich editors and use their own API/events (issue #41): CodeMirror 5 and
|
||||
// Monaco have a model that `.value`/`textContent` can't touch; ProseMirror /
|
||||
// contenteditable need `execCommand('insertText')` so beforeinput/input fire
|
||||
// (a raw `textContent =` corrupts PM's doc and skips React composers).
|
||||
// Returns the engine used so the caller can report it. `type <sel> <text>`
|
||||
// remains for sites that need per-keystroke events.
|
||||
let fill_js = format!(
|
||||
r#"function() {{
|
||||
const el = this;
|
||||
@@ -605,13 +606,43 @@ pub async fn fill(
|
||||
try {{ el.focus(); }} catch (e) {{}}
|
||||
const tag = el.tagName;
|
||||
const fire = (type, ctor) => el.dispatchEvent(new (ctor || Event)(type, {{ bubbles: true }}));
|
||||
if (tag === 'SELECT') {{
|
||||
el.value = v; fire('input'); fire('change'); return true;
|
||||
|
||||
// CodeMirror 5: a hidden <textarea> inside .CodeMirror with a live instance.
|
||||
const cm5 = el.closest && el.closest('.CodeMirror');
|
||||
if (cm5 && cm5.CodeMirror) {{ cm5.CodeMirror.setValue(v); return 'codemirror5'; }}
|
||||
|
||||
// Monaco: global `monaco`; prefer the editor whose DOM contains el.
|
||||
if (window.monaco && monaco.editor) {{
|
||||
try {{
|
||||
const eds = monaco.editor.getEditors ? monaco.editor.getEditors() : [];
|
||||
const ed = eds.find(e => e.getDomNode && e.getDomNode().contains(el)) || eds[0];
|
||||
if (ed) {{ ed.setValue(v); return 'monaco'; }}
|
||||
const models = monaco.editor.getModels ? monaco.editor.getModels() : [];
|
||||
if (models[0]) {{ models[0].setValue(v); return 'monaco'; }}
|
||||
}} catch (e) {{}}
|
||||
}}
|
||||
|
||||
if (tag === 'SELECT') {{ el.value = v; fire('input'); fire('change'); return 'select'; }}
|
||||
|
||||
if (el.isContentEditable) {{
|
||||
el.textContent = v; fire('input', window.InputEvent || Event); fire('change');
|
||||
try {{ el.blur(); }} catch (e) {{}} fire('focusout'); return true;
|
||||
// ProseMirror / contenteditable: select-all then insertText fires
|
||||
// beforeinput/input that PM and React composers listen for.
|
||||
let ok = false;
|
||||
try {{
|
||||
const sel = window.getSelection();
|
||||
const range = document.createRange();
|
||||
range.selectNodeContents(el);
|
||||
sel.removeAllRanges();
|
||||
sel.addRange(range);
|
||||
ok = document.execCommand('insertText', false, v);
|
||||
}} catch (e) {{}}
|
||||
if (!ok) {{ el.textContent = v; fire('input', window.InputEvent || Event); }}
|
||||
fire('change');
|
||||
try {{ el.blur(); }} catch (e) {{}}
|
||||
fire('focusout');
|
||||
return ok ? 'contenteditable' : 'contenteditable-fallback';
|
||||
}}
|
||||
|
||||
const proto = tag === 'TEXTAREA' ? window.HTMLTextAreaElement.prototype
|
||||
: window.HTMLInputElement.prototype;
|
||||
const desc = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||
@@ -623,13 +654,13 @@ pub async fn fill(
|
||||
fire('change');
|
||||
try {{ el.blur(); }} catch (e) {{}}
|
||||
fire('focusout'); // blur-triggered lookups/validation
|
||||
return true;
|
||||
return 'input';
|
||||
}}"#,
|
||||
val = serde_json::to_string(value).unwrap_or_default()
|
||||
);
|
||||
|
||||
client
|
||||
.send_command_typed::<_, Value>(
|
||||
let result: EvaluateResult = client
|
||||
.send_command_typed(
|
||||
"Runtime.callFunctionOn",
|
||||
&CallFunctionOnParams {
|
||||
function_declaration: fill_js,
|
||||
@@ -642,7 +673,11 @@ pub async fn fill(
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
Ok(result
|
||||
.result
|
||||
.value
|
||||
.and_then(|v| v.as_str().map(String::from))
|
||||
.unwrap_or_else(|| "input".to_string()))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
|
||||
Reference in New Issue
Block a user