chore(release): 0.27.0-fork.46 — cap --annotate legend

Dense pages produced hundreds of legend lines on stdout (Hermes: HN dumped 320).
Print the first 40 with a "… and N more" summary; every marker is still drawn in
the image, and --json still returns the full list.

cli/Cargo.lock

@@ -45,7 +45,7 @@ dependencies = [
 
 [[package]]
 name = "agent-browser-stealth"
-version = "0.27.0-fork.42"
+version = "0.27.0-fork.46"
 dependencies = [
  "aes-gcm",
  "async-trait",

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "agent-browser-stealth"
-version = "0.27.0-fork.42"
+version = "0.27.0-fork.46"
 edition = "2021"
 description = "Fast browser automation CLI for AI agents"
 license = "Apache-2.0"

cli/src/commands.rs

@@ -403,12 +403,48 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
             Ok(json!({ "id": id, "action": "fill", "selector": sel, "value": rest[1..].join(" ") }))
         }
         "type" => {
+            // `type --focused <text>` types into whatever element currently has
+            // focus (no selector) — for custom widgets that move focus to a hidden
+            // input after you open them.
+            if rest.first() == Some(&"--focused") {
+                return Ok(json!({
+                    "id": id, "action": "type", "focused": true,
+                    "text": rest[1..].join(" "),
+                }));
+            }
             let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
                 context: "type".to_string(),
-                usage: "type <selector> <text>",
+                usage: "type <selector> <text>   (or: type --focused <text>)",
             })?;
             Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" ") }))
         }
+        "pick" => {
+            // pick <selector|@ref> --option "<text>"  — atomic combobox select:
+            // open the control, wait for options (incl. portal menus), match by
+            // text, fire the right event sequence, verify. Covers native <select>,
+            // ARIA combobox/listbox, and react-select.
+            let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
+                context: "pick".to_string(),
+                usage: "pick <selector> --option \"<text>\"",
+            })?;
+            let opt_pos = rest.iter().position(|a| *a == "--option" || *a == "-o");
+            let option = match opt_pos {
+                Some(p) => rest[p + 1..].join(" "),
+                None => {
+                    return Err(ParseError::MissingArguments {
+                        context: "pick".to_string(),
+                        usage: "pick <selector> --option \"<text>\"",
+                    })
+                }
+            };
+            if option.is_empty() {
+                return Err(ParseError::MissingArguments {
+                    context: "pick".to_string(),
+                    usage: "pick <selector> --option \"<text>\"",
+                });
+            }
+            Ok(json!({ "id": id, "action": "pick", "selector": sel, "option": option }))
+        }
         "hover" => {
             let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
                 context: "hover".to_string(),
@@ -841,17 +877,31 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
 
         // === Eval ===
         "eval" => {
-            // Check for flags: -b/--base64 or --stdin
-            let (is_base64, is_stdin, script_parts): (bool, bool, &[&str]) =
+            // Check for flags: -b/--base64, --stdin, or --file <path>
+            let (is_base64, is_stdin, is_file, script_parts): (bool, bool, bool, &[&str]) =
                 if rest.first() == Some(&"-b") || rest.first() == Some(&"--base64") {
-                    (true, false, &rest[1..])
+                    (true, false, false, &rest[1..])
                 } else if rest.first() == Some(&"--stdin") {
-                    (false, true, &rest[1..])
+                    (false, true, false, &rest[1..])
+                } else if rest.first() == Some(&"--file") {
+                    (false, false, true, &rest[1..])
                 } else {
-                    (false, false, rest.as_slice())
+                    (false, false, false, rest.as_slice())
                 };
 
-            let script = if is_stdin {
+            let script = if is_file {
+                // Read the script from a file. Avoids shell-mangling of inline JS
+                // (non-ASCII identifiers/strings, quotes, large scripts) — the file
+                // is read as UTF-8 and sent verbatim.
+                let path = script_parts.first().ok_or(ParseError::InvalidValue {
+                    message: "eval --file requires a path".to_string(),
+                    usage: "eval --file <path>",
+                })?;
+                std::fs::read_to_string(path).map_err(|e| ParseError::InvalidValue {
+                    message: format!("eval --file: cannot read {path}: {e}"),
+                    usage: "eval --file <path>",
+                })?
+            } else if is_stdin {
                 // Read script from stdin
                 let stdin = io::stdin();
                 let lines: Vec<String> = stdin

cli/src/connection.rs

@@ -821,7 +821,33 @@ fn connect(session: &str) -> Result<Connection, String> {
     }
 }
 
-pub fn send_command(cmd: Value, session: &str) -> Result<Response, String> {
+pub fn send_command(mut cmd: Value, session: &str) -> Result<Response, String> {
+    // Forward per-invocation env to the daemon. The daemon's environment is
+    // frozen at spawn, so settings like AGENT_BROWSER_CLICK_MODE /
+    // AGENT_BROWSER_HUMANIZE (incl. the --humanize flag, which sets the latter)
+    // are otherwise silently ignored on an already-running daemon. Carry them in
+    // the envelope so they apply to THIS command.
+    if let Some(obj) = cmd.as_object_mut() {
+        if let Ok(m) = std::env::var("AGENT_BROWSER_CLICK_MODE") {
+            obj.insert("_clickMode".to_string(), Value::String(m));
+        }
+        if let Ok(h) = std::env::var("AGENT_BROWSER_HUMANIZE") {
+            // Only forward a recognized level; warn once (like the --humanize flag
+            // does) when the env var is set to garbage, instead of silently
+            // ignoring it.
+            if crate::native::humanize::HumanizeLevel::parse(&h).is_some() {
+                obj.insert("_humanize".to_string(), Value::String(h));
+            } else {
+                static WARNED: std::sync::Once = std::sync::Once::new();
+                WARNED.call_once(|| {
+                    eprintln!(
+                        "warning: AGENT_BROWSER_HUMANIZE must be off|fast|human, got {h:?} (ignored)"
+                    );
+                });
+            }
+        }
+    }
+
     // Retry logic for transient errors (EAGAIN/EWOULDBLOCK/connection issues)
     const MAX_RETRIES: u32 = 5;
     const RETRY_DELAY_MS: u64 = 200;

cli/src/native/actions.rs

@@ -1160,6 +1160,23 @@ impl Drop for DaemonState {
 
 pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
     let action = cmd.get("action").and_then(|v| v.as_str()).unwrap_or("");
+
+    // Apply per-invocation overrides the client forwarded (the daemon's own env
+    // is frozen at spawn). CLICK_MODE is read fresh from the process env by
+    // interaction::click, so mirror it here — set when this command provided it,
+    // clear otherwise, so a value from an earlier command never leaks forward.
+    match cmd.get("_clickMode").and_then(|v| v.as_str()) {
+        Some(m) if !m.is_empty() => std::env::set_var("AGENT_BROWSER_CLICK_MODE", m),
+        _ => std::env::remove_var("AGENT_BROWSER_CLICK_MODE"),
+    }
+    // Humanize: set the session level from the client's --humanize / env. Only
+    // set when provided (don't clear — the adaptive per-navigation detector also
+    // owns this level between explicit overrides).
+    if let Some(h) = cmd.get("_humanize").and_then(|v| v.as_str()) {
+        if let Some(level) = super::humanize::HumanizeLevel::parse(h) {
+            super::humanize::set_detected_level(level);
+        }
+    }
     let id = cmd
         .get("id")
         .and_then(|v| v.as_str())
@@ -1306,6 +1323,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
         "fill" => handle_fill(cmd, state).await,
         "type" => handle_type(cmd, state).await,
         "press" => handle_press(cmd, state).await,
+        "pick" => handle_pick(cmd, state).await,
         "hover" => handle_hover(cmd, state).await,
         "scroll" => handle_scroll(cmd, state).await,
         "select" => handle_select(cmd, state).await,
@@ -3014,6 +3032,22 @@ async fn handle_fill(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
 async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
     let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
     let session_id = mgr.active_session_id()?.to_string();
+
+    // `type --focused <text>`: type into the currently-focused element without a
+    // selector (custom widgets that move focus to a hidden input on open).
+    if cmd
+        .get("focused")
+        .and_then(|v| v.as_bool())
+        .unwrap_or(false)
+    {
+        let text = cmd
+            .get("text")
+            .and_then(|v| v.as_str())
+            .ok_or("Missing 'text' parameter")?;
+        interaction::type_text_into_active_context(&mgr.client, &session_id, text, None).await?;
+        return Ok(json!({ "typed": text, "focused": true }));
+    }
+
     let selector = cmd
         .get("selector")
         .and_then(|v| v.as_str())
@@ -3039,6 +3073,103 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
     Ok(json!({ "typed": text }))
 }
 
+/// Atomic combobox select: `pick <selector> --option "<text>"`. Opens the control
+/// (so a portal-rendered menu mounts), polls for the option by visible text, then
+/// fires the full pointer/mouse event sequence on it — covering native `<select>`,
+/// ARIA combobox/listbox, and react-select, which a bare `click`+`press Enter`
+/// can't do reliably. Runs as one in-page async routine so the open→render→pick
+/// dance happens without round-trips that let the menu collapse between commands.
+async fn handle_pick(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
+    let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
+    let session_id = mgr.active_session_id()?.to_string();
+    let selector = cmd
+        .get("selector")
+        .and_then(|v| v.as_str())
+        .ok_or("Missing 'selector' parameter")?;
+    let option = cmd
+        .get("option")
+        .and_then(|v| v.as_str())
+        .ok_or("Missing 'option' parameter")?;
+
+    let (object_id, effective_session_id) = super::element::resolve_element_object_id(
+        &mgr.client,
+        &session_id,
+        &state.ref_map,
+        selector,
+        &state.iframe_sessions,
+    )
+    .await?;
+
+    let func = format!(
+        r#"async function() {{
+            const want = {opt};
+            const norm = s => (s || '').replace(/\s+/g, ' ').trim();
+            const matches = el => norm(el.textContent).toLowerCase().includes(want.toLowerCase());
+            const el = this;
+            const fire = (n, t) => n.dispatchEvent(new MouseEvent(t, {{ bubbles: true, cancelable: true, view: window }}));
+
+            // Native <select>: set the matching option and dispatch input/change.
+            if (el.tagName === 'SELECT') {{
+                const opt = [...el.options].find(matches);
+                if (!opt) return {{ ok: false, error: 'no <option> matched ' + JSON.stringify(want) }};
+                el.value = opt.value;
+                el.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                el.dispatchEvent(new Event('change', {{ bubbles: true }}));
+                return {{ ok: true, picked: norm(opt.textContent), value: el.value, kind: 'select' }};
+            }}
+
+            // Custom widget: open it.
+            (el.focus && el.focus());
+            ['pointerdown', 'mousedown', 'mouseup', 'click'].forEach(t => fire(el, t));
+
+            // Poll for the option to render anywhere in the document (portals
+            // mount the menu outside the trigger), then click it.
+            const sel = '[role=option], [role=listbox] [role=option], li[role=option], [class*=option], [class*=item]';
+            const find = () => [...document.querySelectorAll(sel)].find(o => o.offsetParent !== null && matches(o));
+            const deadline = Date.now() + 2500;
+            let opt = find();
+            while (!opt && Date.now() < deadline) {{
+                await new Promise(r => setTimeout(r, 80));
+                opt = find();
+            }}
+            if (!opt) return {{ ok: false, error: 'option ' + JSON.stringify(want) + ' did not appear after opening the control' }};
+            (opt.scrollIntoView && opt.scrollIntoView({{ block: 'center' }}));
+            ['pointermove', 'pointerover', 'mouseover', 'pointerdown', 'mousedown', 'mouseup', 'click'].forEach(t => fire(opt, t));
+            return {{ ok: true, picked: norm(opt.textContent), kind: 'custom' }};
+        }}"#,
+        opt = serde_json::to_string(option).unwrap_or_default(),
+    );
+
+    let result: super::cdp::types::EvaluateResult = mgr
+        .client
+        .send_command_typed(
+            "Runtime.callFunctionOn",
+            &super::cdp::types::CallFunctionOnParams {
+                function_declaration: func,
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(true),
+            },
+            Some(&effective_session_id),
+        )
+        .await?;
+
+    if let Some(ref ex) = result.exception_details {
+        return Err(format!("pick failed: {}", ex.text));
+    }
+    let val = result.result.value.unwrap_or(Value::Null);
+    if val.get("ok").and_then(|v| v.as_bool()).unwrap_or(false) {
+        Ok(json!({ "picked": val.get("picked"), "selector": selector }))
+    } else {
+        Err(val
+            .get("error")
+            .and_then(|v| v.as_str())
+            .unwrap_or("pick failed")
+            .to_string())
+    }
+}
+
 async fn handle_press(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
     let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
     let session_id = mgr.active_session_id()?.to_string();

cli/src/native/interaction.rs

@@ -4,7 +4,7 @@ use serde_json::Value;
 
 use super::cdp::client::CdpClient;
 use super::cdp::types::*;
-use super::element::{resolve_element_center, resolve_element_object_id, RefMap};
+use super::element::{parse_ref, resolve_element_center, resolve_element_object_id, RefMap};
 use super::humanize;
 
 pub async fn click(
@@ -56,6 +56,33 @@ pub async fn click(
 
     match resolved {
         Ok((cx, cy, w, h, effective_session_id)) => {
+            // Occlusion guard for the CSS-selector path. `@ref` clicks are already
+            // occlusion-checked in resolve_element_center, but a plain selector
+            // resolves to coordinates without that check — so an overlay (modal
+            // backdrop, sticky banner, the getByText located node sitting under a
+            // full-screen layer) would make the coordinate click land on the
+            // overlay and still report success. If the click point doesn't hit the
+            // target, dispatch through the DOM instead (targets the element
+            // directly). Skipped for strict `coord` mode and non-left/multi-clicks.
+            if mode != "coord"
+                && button == "left"
+                && click_count == 1
+                && parse_ref(selector_or_ref).is_none()
+                && point_misses_element(client, &effective_session_id, selector_or_ref).await
+            {
+                eprintln!(
+                    "[click] target occluded at its click point; dispatching through \
+                     the DOM (set AGENT_BROWSER_CLICK_MODE=coord to disable)"
+                );
+                return dom_click(
+                    client,
+                    session_id,
+                    ref_map,
+                    selector_or_ref,
+                    iframe_sessions,
+                )
+                .await;
+            }
             // Land on a jittered point inside the element rather than its exact
             // centre (Fast/Human). Zero size or Off → exact centre.
             let (tx, ty) = humanize::landing_point(
@@ -92,6 +119,42 @@ pub async fn click(
     }
 }
 
+/// True if a coordinate click at the selector's centre would land on something
+/// OTHER than the element (an overlay on top), i.e. the element is occluded.
+/// `false` when not occluded, the element is missing, or the probe fails (so we
+/// never block a click on a flaky probe — the normal coordinate path runs).
+async fn point_misses_element(client: &CdpClient, session_id: &str, selector: &str) -> bool {
+    let js = format!(
+        r#"(() => {{
+            const el = document.querySelector({sel});
+            if (!el) return false;
+            const r = el.getBoundingClientRect();
+            if (r.width === 0 || r.height === 0) return false;
+            const hit = document.elementFromPoint(r.left + r.width / 2, r.top + r.height / 2);
+            if (!hit) return false;
+            // Not occluded if the hit is the element, a descendant, or an ancestor
+            // wrapper (clicking those still reaches the element's handlers).
+            return !(hit === el || el.contains(hit) || hit.contains(el));
+        }})()"#,
+        sel = serde_json::to_string(selector).unwrap_or_default()
+    );
+    match client
+        .send_command_typed::<_, EvaluateResult>(
+            "Runtime.evaluate",
+            &EvaluateParams {
+                expression: js,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await
+    {
+        Ok(r) => r.result.value.and_then(|v| v.as_bool()).unwrap_or(false),
+        Err(_) => false,
+    }
+}
+
 /// Best-effort scroll-into-view before a coordinate click. Uses Chrome's
 /// `scrollIntoViewIfNeeded` (only scrolls when not already fully visible),
 /// falling back to centered `scrollIntoView`. Resolution failures are ignored —

cli/src/output.rs

@@ -130,6 +130,20 @@ fn format_stream_status_text(action: Option<&str>, data: &serde_json::Value) ->
     }
 }
 
+/// Shorten an over-long string by keeping its head and tail and eliding the
+/// middle, with a char count. Used so multi-KB URLs (JWT/OTP login links) don't
+/// flood `tab list`.
+fn truncate_middle(s: &str, max: usize) -> String {
+    let n = s.chars().count();
+    if n <= max {
+        return s.to_string();
+    }
+    let keep = max.saturating_sub(1) / 2;
+    let head: String = s.chars().take(keep).collect();
+    let tail: String = s.chars().skip(n - keep).collect();
+    format!("{head}…{tail} [{n} chars]")
+}
+
 pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &OutputOptions) {
     if opts.json {
         if opts.content_boundaries {
@@ -344,6 +358,16 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
         }
         // Eval result
         if let Some(result) = data.get("result") {
+            // Surface which page the eval actually ran on — to stderr, so it
+            // never corrupts the parsed value on stdout. Lets an agent catch tab
+            // drift (commands landing on the wrong tab) before trusting a result,
+            // e.g. a logged-in `fetch` that hit the wrong origin. (In
+            // content-boundaries mode the origin is already in the banner.)
+            if !opts.content_boundaries {
+                if let Some(o) = origin.filter(|o| !o.is_empty()) {
+                    eprintln!("eval @ {o}");
+                }
+            }
             let formatted = serde_json::to_string_pretty(result).unwrap_or_default();
             print_with_boundaries(&formatted, origin, opts);
             return;
@@ -422,6 +446,9 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
                     .and_then(|v| v.as_str())
                     .unwrap_or("Untitled");
                 let url = tab.get("url").and_then(|v| v.as_str()).unwrap_or("");
+                // Truncate very long URLs (e.g. multi-KB JWT/OTP login links) so
+                // the list stays readable instead of flooding the terminal.
+                let url = truncate_middle(url, 120);
                 let active = tab.get("active").and_then(|v| v.as_bool()).unwrap_or(false);
                 let marker = if active {
                     color::cyan("→")
@@ -734,7 +761,12 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
                         color::green(path)
                     );
                     if let Some(annotations) = data.get("annotations").and_then(|v| v.as_array()) {
-                        for ann in annotations {
+                        // Cap the printed legend on dense pages (it can be
+                        // hundreds of lines and flood the terminal). The image
+                        // still shows every marker; --json returns the full list.
+                        const LEGEND_CAP: usize = 40;
+                        let total = annotations.len();
+                        for ann in annotations.iter().take(LEGEND_CAP) {
                             let num = ann.get("number").and_then(|n| n.as_u64()).unwrap_or(0);
                             let ref_id = ann.get("ref").and_then(|r| r.as_str()).unwrap_or("");
                             let role = ann.get("role").and_then(|r| r.as_str()).unwrap_or("");
@@ -756,6 +788,15 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
                                 );
                             }
                         }
+                        if total > LEGEND_CAP {
+                            println!(
+                                "   {}",
+                                color::dim(&format!(
+                                    "… and {} more markers (shown in the image; --json for the full list)",
+                                    total - LEGEND_CAP
+                                ))
+                            );
+                        }
                     }
                 }
                 "pdf" => println!(

package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-browser-stealth",
-  "version": "0.27.0-fork.42",
+  "version": "0.27.0-fork.46",
   "description": "Browser automation CLI for AI agents — stealth fork with anti-detection",
   "type": "module",
   "packageManager": "pnpm@11.1.3",

skill-data/core/SKILL.md

@@ -182,14 +182,17 @@ Snapshot output looks like:
 Page: Example - Log in
 URL: https://example.com/login
 
-@e1 [heading] "Log in"
-@e2 [form]
-  @e3 [input type="email"] placeholder="Email"
-  @e4 [input type="password"] placeholder="Password"
-  @e5 [button type="submit"] "Continue"
-  @e6 [link] "Forgot password?"
+- heading "Log in" [level=1, ref=e1]
+- textbox "Email" [ref=e2]
+- textbox "Password" [ref=e3]
+- button "Continue" [ref=e4]
+- link "Forgot password?" [ref=e5]
 ```
 
+Each line is `- <role> "<accessible name>" [<attrs>, ref=eN]`, indented by nesting
+depth. You pass the ref to commands as `@eN` (e.g. `click @e4`). Refs are
+assigned fresh on every snapshot.
+
 For unstructured reading (no refs needed):
 
 ```bash
@@ -216,8 +219,16 @@ agent-browser press Enter                 # press a key at current focus
 agent-browser press Control+a             # key combination
 agent-browser check @e3                   # check checkbox
 agent-browser uncheck @e3                 # uncheck
-agent-browser select @e4 "option-value"   # select dropdown option
+agent-browser select @e4 "option-value"   # native <select> only
 agent-browser select @e4 "a" "b"          # select multiple
+agent-browser pick @e4 --option "Europe"  # ANY combobox (react-select / ARIA /
+                                          # native): opens it, waits for the menu
+                                          # (incl. portal-rendered), matches by
+                                          # visible text, fires the right events,
+                                          # and ERRORS if the option never shows
+                                          # (no silent no-op). Use this for custom
+                                          # dropdowns where `select` returns ✓ but
+                                          # changes nothing.
 agent-browser upload @e5 file1.pdf        # upload file(s)
 agent-browser scroll down 500             # scroll page (up/down/left/right)
 agent-browser scrollintoview @e1          # scroll element into view
@@ -386,9 +397,16 @@ Array.from(rows).map(r => ({
 EOF
 ```
 
-Prefer `eval --stdin` (heredoc) or `eval -b <base64>` for any JS with
-quotes or special characters. Inline `agent-browser eval "..."` works
-only for simple expressions.
+Prefer `eval --stdin` (heredoc), `eval --file <path>`, or `eval -b <base64>`
+for any JS with quotes, **non-ASCII identifiers/strings (e.g. Chinese)**, or
+large scripts — inline `agent-browser eval "..."` is shell-mangled and works
+only for simple ASCII expressions.
+
+**`eval` runs in the page's MAIN world and state persists across calls**, so a
+top-level `const x`/`let x`/`var x` in one call collides with the next
+(`SyntaxError: Identifier 'x' has already been declared`). Either use unique
+names, assign to `window.x`, or wrap the body in an IIFE
+(`(() => { const x = …; return x; })()`).
 
 ### Screenshot