fix: make auth login selector targeting more reliable (#945)
Navigate with load, then wait for username/password/submit selectors using the default action timeout. This avoids networkidle hangs on pages with continuous background requests.
This commit is contained in:
Kevin Kipp
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"agent-browser": patch
|
||||
---
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **Auth login readiness** - `agent-browser auth login` now navigates with `load`, waits for usable login form selectors, and uses staged username detection (targeted email/username selectors first, then broad text-input fallback). This reduces SPA timing failures, avoids false matches on unrelated text fields, and prevents `networkidle` hangs on pages with continuous background requests.
|
||||
@@ -486,7 +486,7 @@ agent-browser --session-name secure open example.com
|
||||
|
||||
agent-browser includes security features for safe AI agent deployments. All features are opt-in -- existing workflows are unaffected until you explicitly enable a feature:
|
||||
|
||||
- **Authentication Vault** -- Store credentials locally (always encrypted), reference by name. The LLM never sees passwords. A key is auto-generated at `~/.agent-browser/.encryption-key` if `AGENT_BROWSER_ENCRYPTION_KEY` is not set: `echo "pass" | agent-browser auth save github --url https://github.com/login --username user --password-stdin` then `agent-browser auth login github`
|
||||
- **Authentication Vault** -- Store credentials locally (always encrypted), reference by name. The LLM never sees passwords. `auth login` navigates with `load` and then waits for login form selectors to appear (SPA-friendly, timeout follows the default action timeout). A key is auto-generated at `~/.agent-browser/.encryption-key` if `AGENT_BROWSER_ENCRYPTION_KEY` is not set: `echo "pass" | agent-browser auth save github --url https://github.com/login --username user --password-stdin` then `agent-browser auth login github`
|
||||
- **Content Boundary Markers** -- Wrap page output in delimiters so LLMs can distinguish tool output from untrusted content: `--content-boundaries`
|
||||
- **Domain Allowlist** -- Restrict navigation to trusted domains (wildcards like `*.example.com` also match the bare domain): `--allowed-domains "example.com,*.example.com"`. Sub-resource requests (scripts, images, fetch) and WebSocket/EventSource connections to non-allowed domains are also blocked. Include any CDN domains your target pages depend on (e.g., `*.cdn.example.com`).
|
||||
- **Action Policy** -- Gate destructive actions with a static policy file: `--action-policy ./policy.json`
|
||||
|
||||
+185
-31
@@ -37,6 +37,23 @@ use super::webdriver::backend::{BrowserBackend, WebDriverBackend, WEBDRIVER_UNSU
|
||||
use super::webdriver::ios;
|
||||
use super::webdriver::safari;
|
||||
|
||||
/// Wait strategy used by `auth_login` when navigating to the login page.
|
||||
///
|
||||
/// We intentionally use `Load` (instead of `NetworkIdle`) because many modern
|
||||
/// apps keep background requests active indefinitely (polling, analytics,
|
||||
/// websockets), which can prevent network-idle from ever resolving.
|
||||
///
|
||||
/// After navigation completes, `auth_login` explicitly waits for form selectors
|
||||
/// to appear before filling/clicking.
|
||||
pub const AUTH_LOGIN_WAIT_UNTIL: WaitUntil = WaitUntil::Load;
|
||||
|
||||
/// Poll interval used while waiting for auth form selectors to appear.
|
||||
const AUTH_LOGIN_SELECTOR_POLL_INTERVAL_MS: u64 = 100;
|
||||
|
||||
/// Time spent trying targeted username selectors before broad text-input
|
||||
/// fallback selectors are allowed.
|
||||
const AUTH_LOGIN_PREFERRED_SELECTOR_WINDOW_MS: u64 = 5_000;
|
||||
|
||||
pub struct PendingConfirmation {
|
||||
pub action: String,
|
||||
pub cmd: Value,
|
||||
@@ -5435,6 +5452,80 @@ async fn handle_http_credentials(cmd: &Value, state: &DaemonState) -> Result<Val
|
||||
// Auth handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Wait for any selector in `selectors` to appear and return the first match.
|
||||
///
|
||||
/// This is used by `auth_login` auto-detection so SPA login forms can render
|
||||
/// after initial navigation without requiring global network-idle.
|
||||
async fn wait_for_any_selector(
|
||||
client: &super::cdp::client::CdpClient,
|
||||
session_id: &str,
|
||||
selectors: &[&str],
|
||||
timeout_ms: u64,
|
||||
) -> Result<String, String> {
|
||||
let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(timeout_ms);
|
||||
|
||||
loop {
|
||||
for selector in selectors {
|
||||
let expression = format!(
|
||||
r#"(() => {{
|
||||
const el = document.querySelector({sel});
|
||||
if (!el) return false;
|
||||
|
||||
const r = el.getBoundingClientRect();
|
||||
const s = window.getComputedStyle(el);
|
||||
const opacity = parseFloat(s.opacity || '1');
|
||||
const isVisible =
|
||||
r.width > 0 &&
|
||||
r.height > 0 &&
|
||||
s.visibility !== 'hidden' &&
|
||||
s.display !== 'none' &&
|
||||
(!Number.isFinite(opacity) || opacity > 0);
|
||||
|
||||
if (!isVisible) return false;
|
||||
if (el.matches(':disabled')) return false;
|
||||
|
||||
if (el instanceof HTMLInputElement && el.type === 'hidden') return false;
|
||||
if ((el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) && el.readOnly) return false;
|
||||
|
||||
return true;
|
||||
}})()"#,
|
||||
sel = serde_json::to_string(selector).unwrap_or_default()
|
||||
);
|
||||
|
||||
let result: super::cdp::types::EvaluateResult = client
|
||||
.send_command_typed(
|
||||
"Runtime.evaluate",
|
||||
&super::cdp::types::EvaluateParams {
|
||||
expression,
|
||||
return_by_value: Some(true),
|
||||
await_promise: Some(true),
|
||||
},
|
||||
Some(session_id),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if result
|
||||
.result
|
||||
.value
|
||||
.as_ref()
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Ok((*selector).to_string());
|
||||
}
|
||||
}
|
||||
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("Wait timed out after {}ms", timeout_ms));
|
||||
}
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(
|
||||
AUTH_LOGIN_SELECTOR_POLL_INTERVAL_MS,
|
||||
))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_auth_save(cmd: &Value) -> Result<Value, String> {
|
||||
let name = cmd
|
||||
.get("name")
|
||||
@@ -5480,17 +5571,30 @@ async fn handle_auth_login(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
||||
let password = cred.password;
|
||||
|
||||
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
|
||||
mgr.navigate(&url, WaitUntil::Load).await?;
|
||||
mgr.navigate(&url, AUTH_LOGIN_WAIT_UNTIL).await?;
|
||||
|
||||
let session_id = mgr.active_session_id()?.to_string();
|
||||
let auth_timeout_ms = mgr.default_timeout_ms();
|
||||
|
||||
let auto_user_selectors = [
|
||||
let preferred_user_selectors = [
|
||||
"input[type=email]",
|
||||
"input[name=email]",
|
||||
"input[type=text][name*=user]",
|
||||
"input[id*=user]",
|
||||
"input[type=text]",
|
||||
"input[id=email]",
|
||||
"input[autocomplete=email]",
|
||||
"input[autocomplete=username]",
|
||||
"input[name=username]",
|
||||
"input[name*=email i]",
|
||||
"input[name*=user i]",
|
||||
"input[id*=email i]",
|
||||
"input[id*=user i]",
|
||||
"input[type=text][name*=email i]",
|
||||
"input[type=text][name*=user i]",
|
||||
"input[type=text][id*=email i]",
|
||||
"input[type=text][id*=user i]",
|
||||
"input[type=text][autocomplete=email]",
|
||||
"input[type=text][autocomplete=username]",
|
||||
];
|
||||
let fallback_user_selectors = ["input[type=text]", "input:not([type])"];
|
||||
let auto_submit_selectors = [
|
||||
"button[type=submit]",
|
||||
"input[type=submit]",
|
||||
@@ -5515,22 +5619,50 @@ async fn handle_auth_login(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
||||
|
||||
// Find and fill username
|
||||
let user_sel = if let Some(s) = username_sel {
|
||||
wait_for_selector(&mgr.client, &session_id, &s, "visible", auth_timeout_ms)
|
||||
.await
|
||||
.map_err(|_| format!("Timed out waiting for username selector '{}'", s))?;
|
||||
s
|
||||
} else {
|
||||
let mut found = None;
|
||||
for sel in &auto_user_selectors {
|
||||
let js = format!(
|
||||
"!!document.querySelector({})",
|
||||
serde_json::to_string(sel).unwrap_or_default()
|
||||
);
|
||||
if let Ok(val) = mgr.evaluate(&js, None).await {
|
||||
if val.as_bool().unwrap_or(false) {
|
||||
found = Some(sel.to_string());
|
||||
break;
|
||||
let preferred_window_ms = auth_timeout_ms.min(AUTH_LOGIN_PREFERRED_SELECTOR_WINDOW_MS);
|
||||
let fallback_window_ms = auth_timeout_ms.saturating_sub(preferred_window_ms);
|
||||
|
||||
match wait_for_any_selector(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&preferred_user_selectors,
|
||||
preferred_window_ms,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(selector) => selector,
|
||||
Err(_) => {
|
||||
if fallback_window_ms == 0 {
|
||||
return Err(format!(
|
||||
"Timed out waiting for username field (preferred selectors for {}ms: {})",
|
||||
preferred_window_ms,
|
||||
preferred_user_selectors.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
wait_for_any_selector(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&fallback_user_selectors,
|
||||
fallback_window_ms,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
format!(
|
||||
"Timed out waiting for username field (preferred selectors for {}ms: {}; fallback selectors for {}ms: {})",
|
||||
preferred_window_ms,
|
||||
preferred_user_selectors.join(", "),
|
||||
fallback_window_ms,
|
||||
fallback_user_selectors.join(", ")
|
||||
)
|
||||
})?
|
||||
}
|
||||
}
|
||||
}
|
||||
found.ok_or("Could not find username field")?
|
||||
};
|
||||
interaction::fill(
|
||||
&mgr.client,
|
||||
@@ -5543,6 +5675,15 @@ async fn handle_auth_login(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
||||
|
||||
// Find and fill password
|
||||
let pass_sel = password_sel.unwrap_or_else(|| "input[type=password]".to_string());
|
||||
wait_for_selector(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&pass_sel,
|
||||
"visible",
|
||||
auth_timeout_ms,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| format!("Timed out waiting for password selector '{}'", pass_sel))?;
|
||||
interaction::fill(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
@@ -5554,22 +5695,24 @@ async fn handle_auth_login(cmd: &Value, state: &mut DaemonState) -> Result<Value
|
||||
|
||||
// Find and click submit
|
||||
let sub_sel = if let Some(s) = submit_sel {
|
||||
wait_for_selector(&mgr.client, &session_id, &s, "visible", auth_timeout_ms)
|
||||
.await
|
||||
.map_err(|_| format!("Timed out waiting for submit selector '{}'", s))?;
|
||||
s
|
||||
} else {
|
||||
let mut found = None;
|
||||
for sel in &auto_submit_selectors {
|
||||
let js = format!(
|
||||
"!!document.querySelector({})",
|
||||
serde_json::to_string(sel).unwrap_or_default()
|
||||
);
|
||||
if let Ok(val) = mgr.evaluate(&js, None).await {
|
||||
if val.as_bool().unwrap_or(false) {
|
||||
found = Some(sel.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
found.ok_or("Could not find submit button")?
|
||||
wait_for_any_selector(
|
||||
&mgr.client,
|
||||
&session_id,
|
||||
&auto_submit_selectors,
|
||||
auth_timeout_ms,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
format!(
|
||||
"Timed out waiting for submit button (tried selectors: {})",
|
||||
auto_submit_selectors.join(", ")
|
||||
)
|
||||
})?
|
||||
};
|
||||
interaction::click(
|
||||
&mgr.client,
|
||||
@@ -6656,4 +6799,15 @@ mod tests {
|
||||
"Should not add a second wildcard when routes already contain one"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auth_login_waits_for_load_event() {
|
||||
use super::super::browser::WaitUntil;
|
||||
assert_eq!(
|
||||
super::AUTH_LOGIN_WAIT_UNTIL,
|
||||
WaitUntil::Load,
|
||||
"auth_login should navigate with Load and then wait for form \
|
||||
selectors explicitly"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,7 +129,7 @@ pub struct PageInfo {
|
||||
pub target_type: String, // "page" or "webview"
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum WaitUntil {
|
||||
Load,
|
||||
DomContentLoaded,
|
||||
@@ -592,6 +592,10 @@ impl BrowserManager {
|
||||
!self.pages.is_empty()
|
||||
}
|
||||
|
||||
pub fn default_timeout_ms(&self) -> u64 {
|
||||
self.default_timeout_ms
|
||||
}
|
||||
|
||||
/// Checks if the CDP connection is alive by sending a simple command.
|
||||
/// Returns false if the command times out or fails.
|
||||
pub async fn is_connection_alive(&self) -> bool {
|
||||
|
||||
@@ -2614,6 +2614,149 @@ async fn start_echo_server() -> (String, tokio::task::JoinHandle<()>) {
|
||||
(base_url, handle)
|
||||
}
|
||||
|
||||
/// Starts a tiny HTTP server that serves a delayed-render login form.
|
||||
///
|
||||
/// The page continuously fetches `/ping` so `networkidle` is hard to reach,
|
||||
/// while the login form itself appears after `render_delay_ms`.
|
||||
async fn start_delayed_login_server(
|
||||
render_delay_ms: u64,
|
||||
ping_interval_ms: u64,
|
||||
) -> (String, tokio::task::JoinHandle<()>) {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let base_url = format!("http://127.0.0.1:{}", port);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
// Serve enough requests for navigation + many background /ping calls.
|
||||
for _ in 0..1000 {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
break;
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 8192];
|
||||
let n = stream.read(&mut buf).await.unwrap_or(0);
|
||||
let request = String::from_utf8_lossy(&buf[..n]);
|
||||
let request_line = request.lines().next().unwrap_or_default();
|
||||
let path = request_line.split_whitespace().nth(1).unwrap_or("/");
|
||||
|
||||
let (status, content_type, body) = if path.starts_with("/ping") {
|
||||
("204 No Content", "text/plain", String::new())
|
||||
} else {
|
||||
let html = format!(
|
||||
r#"<!doctype html>
|
||||
<html>
|
||||
<head><meta charset="utf-8"><title>Delayed Login</title></head>
|
||||
<body>
|
||||
<input id="search" type="text" name="search" />
|
||||
<div id="root">loading...</div>
|
||||
<script>
|
||||
setInterval(() => {{
|
||||
fetch('/ping?ts=' + Date.now()).catch(() => {{}});
|
||||
}}, {ping_interval_ms});
|
||||
|
||||
setTimeout(() => {{
|
||||
const root = document.getElementById('root');
|
||||
root.innerHTML = `
|
||||
<form id="login-form" onsubmit="event.preventDefault(); window.__submitted = true;">
|
||||
<input type="email" name="email" />
|
||||
<input type="password" name="password" />
|
||||
<button type="submit">Sign in</button>
|
||||
</form>
|
||||
`;
|
||||
}}, {render_delay_ms});
|
||||
</script>
|
||||
</body>
|
||||
</html>"#,
|
||||
);
|
||||
("200 OK", "text/html", html)
|
||||
};
|
||||
|
||||
let response = format!(
|
||||
"HTTP/1.1 {}\r\nContent-Type: {}\r\nAccess-Control-Allow-Origin: *\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||
status,
|
||||
content_type,
|
||||
body.len(),
|
||||
body,
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
let _ = stream.flush().await;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
(base_url, handle)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn e2e_auth_login_waits_for_delayed_spa_form_render() {
|
||||
let (base_url, _server) = start_delayed_login_server(1200, 100).await;
|
||||
let mut state = DaemonState::new();
|
||||
|
||||
let profile_name = format!(
|
||||
"e2e-auth-login-spa-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_else(|_| std::time::Duration::from_secs(0))
|
||||
.as_millis()
|
||||
);
|
||||
|
||||
let launch = execute_command(
|
||||
&json!({ "id": "1", "action": "launch", "headless": true }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&launch);
|
||||
|
||||
let save = execute_command(
|
||||
&json!({
|
||||
"id": "2",
|
||||
"action": "auth_save",
|
||||
"name": profile_name.clone(),
|
||||
"url": format!("{}/login", base_url),
|
||||
"username": "user@example.com",
|
||||
"password": "super-secret",
|
||||
}),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&save);
|
||||
|
||||
let login = execute_command(
|
||||
&json!({ "id": "3", "action": "auth_login", "name": profile_name.clone() }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&login);
|
||||
assert_eq!(get_data(&login)["loggedIn"], true);
|
||||
|
||||
let verify = execute_command(
|
||||
&json!({
|
||||
"id": "4",
|
||||
"action": "evaluate",
|
||||
"script": "({ user: document.querySelector('input[type=email]')?.value ?? '', pass: document.querySelector('input[type=password]')?.value ?? '', search: document.querySelector('#search')?.value ?? '', submitted: !!window.__submitted })",
|
||||
}),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
assert_success(&verify);
|
||||
let result = &get_data(&verify)["result"];
|
||||
assert_eq!(result["user"], "user@example.com");
|
||||
assert_eq!(result["pass"], "super-secret");
|
||||
assert_eq!(result["search"], "");
|
||||
assert_eq!(result["submitted"], true);
|
||||
|
||||
let _ = execute_command(
|
||||
&json!({ "id": "5", "action": "auth_delete", "name": profile_name }),
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
|
||||
let close = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
|
||||
assert_success(&close);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Origin-scoped --headers tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
+6
-2
@@ -1914,7 +1914,7 @@ Usage: agent-browser auth <subcommand> [args]
|
||||
|
||||
Subcommands:
|
||||
save <name> Save credentials for a login profile
|
||||
login <name> Login using saved credentials
|
||||
login <name> Login using saved credentials (waits for form fields)
|
||||
list List saved profiles (names and URLs only)
|
||||
show <name> Show profile metadata (no passwords)
|
||||
delete <name> Delete a saved profile
|
||||
@@ -1928,6 +1928,10 @@ Save Options:
|
||||
--password-selector <s> Custom CSS selector for password field
|
||||
--submit-selector <s> Custom CSS selector for submit button
|
||||
|
||||
Login behavior:
|
||||
auth login waits for form selectors to appear before filling/clicking.
|
||||
Selector wait timeout follows the default action timeout.
|
||||
|
||||
Global Options:
|
||||
--json Output as JSON
|
||||
--session <name> Use specific session
|
||||
@@ -2572,7 +2576,7 @@ Batch:
|
||||
|
||||
Auth Vault:
|
||||
auth save <name> [opts] Save auth profile (--url, --username, --password/--password-stdin)
|
||||
auth login <name> Login using saved credentials
|
||||
auth login <name> Login using saved credentials (waits for form fields)
|
||||
auth list List saved auth profiles
|
||||
auth show <name> Show auth profile metadata
|
||||
auth delete <name> Delete auth profile
|
||||
|
||||
@@ -259,6 +259,8 @@ Save options:
|
||||
- `--password-selector <sel>` -- custom CSS selector for password field
|
||||
- `--submit-selector <sel>` -- custom CSS selector for submit button
|
||||
|
||||
`auth login` navigates with `load` and then waits for the username/password/submit selectors to appear before interacting. This improves reliability on SPA login pages where fields render after initial page load.
|
||||
|
||||
```bash
|
||||
echo "pass" | agent-browser auth save github --url https://github.com/login --username user --password-stdin
|
||||
agent-browser auth login github
|
||||
|
||||
@@ -47,6 +47,8 @@ agent-browser auth show github
|
||||
agent-browser auth delete github
|
||||
```
|
||||
|
||||
`auth login` navigates with the `load` lifecycle event and then waits for form selectors to appear before filling/clicking. This makes delayed SPA login pages more reliable while avoiding `networkidle` hangs on pages with long-lived background requests.
|
||||
|
||||
Custom selectors can be specified if auto-detection fails:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -90,6 +90,8 @@ echo "$PASSWORD" | agent-browser auth save myapp --url https://app.example.com/l
|
||||
agent-browser auth login myapp
|
||||
```
|
||||
|
||||
`auth login` navigates with `load` and then waits for login form selectors to appear before filling/clicking, which is more reliable on delayed SPA login screens.
|
||||
|
||||
**Option 5: State file (manual save/load)**
|
||||
|
||||
```bash
|
||||
@@ -230,6 +232,8 @@ agent-browser auth show github
|
||||
agent-browser auth delete github
|
||||
```
|
||||
|
||||
`auth login` waits for username/password/submit selectors before interacting, with a timeout tied to the default action timeout.
|
||||
|
||||
### Authentication with State Persistence
|
||||
|
||||
```bash
|
||||
|
||||
Reference in New Issue
Block a user