Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
649fa4ce94 | ||
|
|
3ac69e822a | ||
|
|
d7a0ed85f9 | ||
|
|
9eaa5495ae | ||
|
|
68734fcb36 | ||
|
|
4b33dbadb4 | ||
|
|
fc73ee6c90 |
Generated
+1
-1
@@ -45,7 +45,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "agent-browser-stealth"
|
||||
version = "0.27.0-fork.44"
|
||||
version = "0.27.0-fork.47"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"async-trait",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "agent-browser-stealth"
|
||||
version = "0.27.0-fork.44"
|
||||
version = "0.27.0-fork.47"
|
||||
edition = "2021"
|
||||
description = "Fast browser automation CLI for AI agents"
|
||||
license = "Apache-2.0"
|
||||
|
||||
@@ -418,6 +418,33 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
||||
})?;
|
||||
Ok(json!({ "id": id, "action": "type", "selector": sel, "text": rest[1..].join(" ") }))
|
||||
}
|
||||
"pick" => {
|
||||
// pick <selector|@ref> --option "<text>" — atomic combobox select:
|
||||
// open the control, wait for options (incl. portal menus), match by
|
||||
// text, fire the right event sequence, verify. Covers native <select>,
|
||||
// ARIA combobox/listbox, and react-select.
|
||||
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "pick".to_string(),
|
||||
usage: "pick <selector> --option \"<text>\"",
|
||||
})?;
|
||||
let opt_pos = rest.iter().position(|a| *a == "--option" || *a == "-o");
|
||||
let option = match opt_pos {
|
||||
Some(p) => rest[p + 1..].join(" "),
|
||||
None => {
|
||||
return Err(ParseError::MissingArguments {
|
||||
context: "pick".to_string(),
|
||||
usage: "pick <selector> --option \"<text>\"",
|
||||
})
|
||||
}
|
||||
};
|
||||
if option.is_empty() {
|
||||
return Err(ParseError::MissingArguments {
|
||||
context: "pick".to_string(),
|
||||
usage: "pick <selector> --option \"<text>\"",
|
||||
});
|
||||
}
|
||||
Ok(json!({ "id": id, "action": "pick", "selector": sel, "option": option }))
|
||||
}
|
||||
"hover" => {
|
||||
let sel = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "hover".to_string(),
|
||||
|
||||
@@ -1323,6 +1323,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
|
||||
"fill" => handle_fill(cmd, state).await,
|
||||
"type" => handle_type(cmd, state).await,
|
||||
"press" => handle_press(cmd, state).await,
|
||||
"pick" => handle_pick(cmd, state).await,
|
||||
"hover" => handle_hover(cmd, state).await,
|
||||
"scroll" => handle_scroll(cmd, state).await,
|
||||
"select" => handle_select(cmd, state).await,
|
||||
@@ -3072,6 +3073,103 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
|
||||
Ok(json!({ "typed": text }))
|
||||
}
|
||||
|
||||
/// Atomic combobox select: `pick <selector> --option "<text>"`. Opens the control
|
||||
/// (so a portal-rendered menu mounts), polls for the option by visible text, then
|
||||
/// fires the full pointer/mouse event sequence on it — covering native `<select>`,
|
||||
/// ARIA combobox/listbox, and react-select, which a bare `click`+`press Enter`
|
||||
/// can't do reliably. Runs as one in-page async routine so the open→render→pick
|
||||
/// dance happens without round-trips that let the menu collapse between commands.
|
||||
async fn handle_pick(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
let session_id = mgr.active_session_id()?.to_string();
|
||||
let selector = cmd
|
||||
.get("selector")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Missing 'selector' parameter")?;
|
||||
let option = cmd
|
||||
.get("option")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Missing 'option' parameter")?;
|
||||
|
||||
let (object_id, effective_session_id) = super::element::resolve_element_object_id(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&state.ref_map,
|
||||
selector,
|
||||
&state.iframe_sessions,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let func = format!(
|
||||
r#"async function() {{
|
||||
const want = {opt};
|
||||
const norm = s => (s || '').replace(/\s+/g, ' ').trim();
|
||||
const matches = el => norm(el.textContent).toLowerCase().includes(want.toLowerCase());
|
||||
const el = this;
|
||||
const fire = (n, t) => n.dispatchEvent(new MouseEvent(t, {{ bubbles: true, cancelable: true, view: window }}));
|
||||
|
||||
// Native <select>: set the matching option and dispatch input/change.
|
||||
if (el.tagName === 'SELECT') {{
|
||||
const opt = [...el.options].find(matches);
|
||||
if (!opt) return {{ ok: false, error: 'no <option> matched ' + JSON.stringify(want) }};
|
||||
el.value = opt.value;
|
||||
el.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
el.dispatchEvent(new Event('change', {{ bubbles: true }}));
|
||||
return {{ ok: true, picked: norm(opt.textContent), value: el.value, kind: 'select' }};
|
||||
}}
|
||||
|
||||
// Custom widget: open it.
|
||||
(el.focus && el.focus());
|
||||
['pointerdown', 'mousedown', 'mouseup', 'click'].forEach(t => fire(el, t));
|
||||
|
||||
// Poll for the option to render anywhere in the document (portals
|
||||
// mount the menu outside the trigger), then click it.
|
||||
const sel = '[role=option], [role=listbox] [role=option], li[role=option], [class*=option], [class*=item]';
|
||||
const find = () => [...document.querySelectorAll(sel)].find(o => o.offsetParent !== null && matches(o));
|
||||
const deadline = Date.now() + 2500;
|
||||
let opt = find();
|
||||
while (!opt && Date.now() < deadline) {{
|
||||
await new Promise(r => setTimeout(r, 80));
|
||||
opt = find();
|
||||
}}
|
||||
if (!opt) return {{ ok: false, error: 'option ' + JSON.stringify(want) + ' did not appear after opening the control' }};
|
||||
(opt.scrollIntoView && opt.scrollIntoView({{ block: 'center' }}));
|
||||
['pointermove', 'pointerover', 'mouseover', 'pointerdown', 'mousedown', 'mouseup', 'click'].forEach(t => fire(opt, t));
|
||||
return {{ ok: true, picked: norm(opt.textContent), kind: 'custom' }};
|
||||
}}"#,
|
||||
opt = serde_json::to_string(option).unwrap_or_default(),
|
||||
);
|
||||
|
||||
let result: super::cdp::types::EvaluateResult = mgr
|
||||
.client
|
||||
.send_command_typed(
|
||||
"Runtime.callFunctionOn",
|
||||
&super::cdp::types::CallFunctionOnParams {
|
||||
function_declaration: func,
|
||||
object_id: Some(object_id),
|
||||
arguments: None,
|
||||
return_by_value: Some(true),
|
||||
await_promise: Some(true),
|
||||
},
|
||||
Some(&effective_session_id),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Some(ref ex) = result.exception_details {
|
||||
return Err(format!("pick failed: {}", ex.text));
|
||||
}
|
||||
let val = result.result.value.unwrap_or(Value::Null);
|
||||
if val.get("ok").and_then(|v| v.as_bool()).unwrap_or(false) {
|
||||
Ok(json!({ "picked": val.get("picked"), "selector": selector }))
|
||||
} else {
|
||||
Err(val
|
||||
.get("error")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("pick failed")
|
||||
.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_press(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
|
||||
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
|
||||
let session_id = mgr.active_session_id()?.to_string();
|
||||
|
||||
@@ -315,6 +315,13 @@ pub struct BrowserManager {
|
||||
/// browser after it ends. Only ever holds tabs we created — never the user's
|
||||
/// existing tabs or other sessions' tabs — so closing them is always safe.
|
||||
created_targets: HashSet<String>,
|
||||
/// The session's *intended* active tab, pinned by stable target_id rather
|
||||
/// than the fragile `active_page_index`. Set on every explicit open / tab new
|
||||
/// / tab switch. `active_session_id` resolves through this so a foreign tab
|
||||
/// opening (passive discovery), a tab closing, or list reordering can't drift
|
||||
/// the session's commands onto the wrong page — the wrong-origin-fetch hazard
|
||||
/// in the dogfood reports. Falls back to the index if the pinned tab is gone.
|
||||
active_target_id: Option<String>,
|
||||
next_tab_id: u32,
|
||||
/// Whether to enable the CDP `Runtime` domain (console / error / exception capture).
|
||||
/// OFF by default for stealth: a live `Runtime.enable` is a detectable CDP signal
|
||||
@@ -440,6 +447,7 @@ impl BrowserManager {
|
||||
ignore_https_errors,
|
||||
visited_origins: HashSet::new(),
|
||||
created_targets: HashSet::new(),
|
||||
active_target_id: None,
|
||||
next_tab_id: 1,
|
||||
capture_console: console_capture_enabled(),
|
||||
};
|
||||
@@ -531,6 +539,7 @@ impl BrowserManager {
|
||||
ignore_https_errors: false,
|
||||
visited_origins: HashSet::new(),
|
||||
created_targets: HashSet::new(),
|
||||
active_target_id: None,
|
||||
next_tab_id: 1,
|
||||
capture_console: console_capture_enabled(),
|
||||
};
|
||||
@@ -547,6 +556,7 @@ impl BrowserManager {
|
||||
target_type: "page".to_string(),
|
||||
});
|
||||
manager.active_page_index = 0;
|
||||
manager.pin_active_target();
|
||||
manager.enable_domains_direct().await?;
|
||||
} else {
|
||||
manager.discover_and_attach_targets().await?;
|
||||
@@ -621,6 +631,7 @@ impl BrowserManager {
|
||||
target_type: "page".to_string(),
|
||||
});
|
||||
self.active_page_index = 0;
|
||||
self.pin_active_target();
|
||||
self.enable_domains(&attach_result.session_id).await?;
|
||||
} else {
|
||||
for target in &page_targets {
|
||||
@@ -650,6 +661,7 @@ impl BrowserManager {
|
||||
}
|
||||
|
||||
self.active_page_index = 0;
|
||||
self.pin_active_target();
|
||||
let session_id = self.pages[0].session_id.clone();
|
||||
self.enable_domains(&session_id).await?;
|
||||
}
|
||||
@@ -736,9 +748,31 @@ impl BrowserManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Index of the session's active page, resolved through the pinned
|
||||
/// `active_target_id` (stable across reorder/removal/passive discovery) and
|
||||
/// falling back to `active_page_index` when nothing is pinned or the pin is
|
||||
/// gone. This is what keeps commands on the tab the agent actually opened.
|
||||
fn resolved_active_index(&self) -> usize {
|
||||
if let Some(tid) = &self.active_target_id {
|
||||
if let Some(i) = self.pages.iter().position(|p| &p.target_id == tid) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
self.active_page_index
|
||||
}
|
||||
|
||||
/// Pin the current active page by target_id so later commands stick to it.
|
||||
/// Call after any explicit open / tab new / tab switch.
|
||||
fn pin_active_target(&mut self) {
|
||||
self.active_target_id = self
|
||||
.pages
|
||||
.get(self.active_page_index)
|
||||
.map(|p| p.target_id.clone());
|
||||
}
|
||||
|
||||
pub fn active_session_id(&self) -> Result<&str, String> {
|
||||
self.pages
|
||||
.get(self.active_page_index)
|
||||
.get(self.resolved_active_index())
|
||||
.map(|p| p.session_id.as_str())
|
||||
.ok_or_else(|| "No active page".to_string())
|
||||
}
|
||||
@@ -991,7 +1025,7 @@ impl BrowserManager {
|
||||
|
||||
pub fn active_target_id(&self) -> Result<&str, String> {
|
||||
self.pages
|
||||
.get(self.active_page_index)
|
||||
.get(self.resolved_active_index())
|
||||
.map(|p| p.target_id.as_str())
|
||||
.ok_or_else(|| "No active page".to_string())
|
||||
}
|
||||
@@ -1219,6 +1253,7 @@ impl BrowserManager {
|
||||
target_type: "page".to_string(),
|
||||
});
|
||||
self.active_page_index = index;
|
||||
self.pin_active_target();
|
||||
|
||||
Ok(json!({
|
||||
"tabId": format_tab_id(tab_id),
|
||||
@@ -1238,6 +1273,7 @@ impl BrowserManager {
|
||||
}
|
||||
|
||||
self.active_page_index = index;
|
||||
self.pin_active_target();
|
||||
let session_id = self.pages[index].session_id.clone();
|
||||
self.enable_domains(&session_id).await?;
|
||||
|
||||
@@ -1581,6 +1617,7 @@ impl BrowserManager {
|
||||
let index = self.pages.len();
|
||||
self.pages.push(page);
|
||||
self.active_page_index = index;
|
||||
self.pin_active_target();
|
||||
}
|
||||
|
||||
/// Add a passively-discovered page WITHOUT changing the active tab.
|
||||
@@ -1783,6 +1820,7 @@ async fn initialize_lightpanda_manager(
|
||||
ignore_https_errors: false,
|
||||
visited_origins: HashSet::new(),
|
||||
created_targets: HashSet::new(),
|
||||
active_target_id: None,
|
||||
next_tab_id: 1,
|
||||
capture_console: console_capture_enabled(),
|
||||
};
|
||||
|
||||
@@ -556,8 +556,12 @@ async fn verify_ref_identity(
|
||||
Err(format!(
|
||||
"Ref {} no longer matches its snapshot. Was [{} \"{}\"], now [{} \"{}\"].\n\
|
||||
The DOM mutated between snapshot and interaction (typical with React/Vue \
|
||||
reusing nodes during re-render). Take a fresh snapshot, then re-target.\n\
|
||||
To bypass this guard set AGENT_BROWSER_VERIFY_REF=0.",
|
||||
reusing nodes during re-render). Fix: take a fresh `snapshot` and re-target \
|
||||
with the new ref. For SPAs where refs churn every interaction, drive the \
|
||||
element directly with `eval` (e.g. `eval \"document.querySelector(...).click()\"`), \
|
||||
which doesn't depend on refs.\n\
|
||||
(Last resort: AGENT_BROWSER_VERIFY_REF=0 disables this safety check — only \
|
||||
if you accept clicks may land on a re-rendered/wrong node.)",
|
||||
ref_id, expected_role, expected_name, actual_role, actual_name,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1305,6 +1305,39 @@ fn render_tree(
|
||||
}
|
||||
}
|
||||
|
||||
/// True if a snapshot line names an interactive ARIA role. Compaction keeps
|
||||
/// these even without a `ref=`/`": "` marker, so a clickable control never gets
|
||||
/// dropped from `-c` output (the dogfood reports saw a button present in the full
|
||||
/// snapshot vanish from compact, leaving the agent clicking an empty ref).
|
||||
fn is_interactive_line(line: &str) -> bool {
|
||||
const ROLES: &[&str] = &[
|
||||
"button",
|
||||
"link",
|
||||
"textbox",
|
||||
"checkbox",
|
||||
"radio",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"switch",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"searchbox",
|
||||
"tab ",
|
||||
"clickable",
|
||||
"focusable",
|
||||
"editable",
|
||||
];
|
||||
let t = line.trim_start();
|
||||
// Lines look like `- button "Label" [ref=e1]`; match the role token after the
|
||||
// leading "- " marker.
|
||||
let t = t.strip_prefix("- ").unwrap_or(t);
|
||||
ROLES.iter().any(|r| t.starts_with(r))
|
||||
}
|
||||
|
||||
fn compact_tree(tree: &str, interactive: bool) -> String {
|
||||
let lines: Vec<&str> = tree.lines().collect();
|
||||
if lines.is_empty() {
|
||||
@@ -1314,7 +1347,7 @@ fn compact_tree(tree: &str, interactive: bool) -> String {
|
||||
let mut keep = vec![false; lines.len()];
|
||||
|
||||
for (i, line) in lines.iter().enumerate() {
|
||||
if line.contains("ref=") || line.contains(": ") {
|
||||
if line.contains("ref=") || line.contains(": ") || is_interactive_line(line) {
|
||||
keep[i] = true;
|
||||
// Mark ancestors
|
||||
let my_indent = count_indent(line);
|
||||
|
||||
+15
-1
@@ -761,7 +761,12 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
||||
color::green(path)
|
||||
);
|
||||
if let Some(annotations) = data.get("annotations").and_then(|v| v.as_array()) {
|
||||
for ann in annotations {
|
||||
// Cap the printed legend on dense pages (it can be
|
||||
// hundreds of lines and flood the terminal). The image
|
||||
// still shows every marker; --json returns the full list.
|
||||
const LEGEND_CAP: usize = 40;
|
||||
let total = annotations.len();
|
||||
for ann in annotations.iter().take(LEGEND_CAP) {
|
||||
let num = ann.get("number").and_then(|n| n.as_u64()).unwrap_or(0);
|
||||
let ref_id = ann.get("ref").and_then(|r| r.as_str()).unwrap_or("");
|
||||
let role = ann.get("role").and_then(|r| r.as_str()).unwrap_or("");
|
||||
@@ -783,6 +788,15 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
||||
);
|
||||
}
|
||||
}
|
||||
if total > LEGEND_CAP {
|
||||
println!(
|
||||
" {}",
|
||||
color::dim(&format!(
|
||||
"… and {} more markers (shown in the image; --json for the full list)",
|
||||
total - LEGEND_CAP
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
"pdf" => println!(
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "agent-browser-stealth",
|
||||
"version": "0.27.0-fork.44",
|
||||
"version": "0.27.0-fork.47",
|
||||
"description": "Browser automation CLI for AI agents — stealth fork with anti-detection",
|
||||
"type": "module",
|
||||
"packageManager": "pnpm@11.1.3",
|
||||
|
||||
@@ -219,8 +219,16 @@ agent-browser press Enter # press a key at current focus
|
||||
agent-browser press Control+a # key combination
|
||||
agent-browser check @e3 # check checkbox
|
||||
agent-browser uncheck @e3 # uncheck
|
||||
agent-browser select @e4 "option-value" # select dropdown option
|
||||
agent-browser select @e4 "option-value" # native <select> only
|
||||
agent-browser select @e4 "a" "b" # select multiple
|
||||
agent-browser pick @e4 --option "Europe" # ANY combobox (react-select / ARIA /
|
||||
# native): opens it, waits for the menu
|
||||
# (incl. portal-rendered), matches by
|
||||
# visible text, fires the right events,
|
||||
# and ERRORS if the option never shows
|
||||
# (no silent no-op). Use this for custom
|
||||
# dropdowns where `select` returns ✓ but
|
||||
# changes nothing.
|
||||
agent-browser upload @e5 file1.pdf # upload file(s)
|
||||
agent-browser scroll down 500 # scroll page (up/down/left/right)
|
||||
agent-browser scrollintoview @e1 # scroll element into view
|
||||
|
||||
Reference in New Issue
Block a user