From cd1f255129e1fcac0ff4e43eb1f3932005bd332c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 24 Mar 2026 12:52:39 -0500 Subject: [PATCH] fix: handle proxy authentication via CDP Fetch.authRequired (#1000) * fix: handle proxy authentication via CDP Fetch.authRequired Chrome's --proxy-server flag does not support credentials embedded in the URL. When a proxy requires authentication, Chrome receives a 407 from the proxy but has no way to respond with credentials, resulting in net::ERR_INVALID_AUTH_CREDENTIALS. Fix by: 1. Parsing credentials from the proxy URL (already done by parse_proxy) 2. Storing them in DaemonState.proxy_credentials 3. Enabling Fetch.enable with handleAuthRequests: true 4. Responding to Fetch.authRequired events with Fetch.continueWithAuth 5. Passing only the server URL (without credentials) to --proxy-server 6. Forwarding credentials to the daemon via dedicated env vars Also adds fallback to standard proxy env vars (HTTP_PROXY, HTTPS_PROXY, ALL_PROXY, NO_PROXY) when AGENT_BROWSER_PROXY is not set. Fixes #990 * refactor: use typed struct for parse_proxy, fix double Fetch.enable and username-only auth - Replace serde_json::Value return from parse_proxy with a typed ParsedProxy struct - Fix double Fetch.enable call when both proxy auth and domain filter are active (the second call could overwrite handleAuthRequests from the first) - Allow username-only proxy auth (some proxies don't require a password) - Handle empty username/password in parse_proxy as None instead of Some("") - Use install_domain_filter_fetch in auto_launch for consistency - Update unit tests to use typed struct fields --------- Co-authored-by: ctate <366502+ctate@users.noreply.github.com> --- cli/src/connection.rs | 8 ++ cli/src/flags.rs | 14 ++- cli/src/main.rs | 124 ++++++++++++++++++-------- cli/src/native/actions.rs | 167 +++++++++++++++++++++++++++++------ cli/src/native/cdp/chrome.rs | 4 + cli/src/native/network.rs | 18 ++-- cli/src/output.rs | 9 +- 7 files changed, 266 insertions(+), 78 deletions(-) diff --git a/cli/src/connection.rs b/cli/src/connection.rs index 2665d38..bfcf2fd 100644 --- a/cli/src/connection.rs +++ b/cli/src/connection.rs @@ -188,6 +188,8 @@ pub struct DaemonOptions<'a> { pub user_agent: Option<&'a str>, pub proxy: Option<&'a str>, pub proxy_bypass: Option<&'a str>, + pub proxy_username: Option<&'a str>, + pub proxy_password: Option<&'a str>, pub ignore_https_errors: bool, pub allow_file_access: bool, pub profile: Option<&'a str>, @@ -233,6 +235,12 @@ fn apply_daemon_env(cmd: &mut Command, session: &str, opts: &DaemonOptions) { if let Some(pb) = opts.proxy_bypass { cmd.env("AGENT_BROWSER_PROXY_BYPASS", pb); } + if let Some(pu) = opts.proxy_username { + cmd.env("AGENT_BROWSER_PROXY_USERNAME", pu); + } + if let Some(pp) = opts.proxy_password { + cmd.env("AGENT_BROWSER_PROXY_PASSWORD", pp); + } if opts.ignore_https_errors { cmd.env("AGENT_BROWSER_IGNORE_HTTPS_ERRORS", "1"); } diff --git a/cli/src/flags.rs b/cli/src/flags.rs index 8eae1be..501127a 100644 --- a/cli/src/flags.rs +++ b/cli/src/flags.rs @@ -353,10 +353,20 @@ pub fn parse_flags(args: &[String]) -> Flags { extensions, profile: env::var("AGENT_BROWSER_PROFILE").ok().or(config.profile), state: env::var("AGENT_BROWSER_STATE").ok().or(config.state), - proxy: env::var("AGENT_BROWSER_PROXY").ok().or(config.proxy), + proxy: env::var("AGENT_BROWSER_PROXY") + .ok() + .or(config.proxy) + .or_else(|| env::var("HTTP_PROXY").ok()) + .or_else(|| env::var("http_proxy").ok()) + .or_else(|| env::var("HTTPS_PROXY").ok()) + .or_else(|| env::var("https_proxy").ok()) + .or_else(|| env::var("ALL_PROXY").ok()) + .or_else(|| env::var("all_proxy").ok()), proxy_bypass: env::var("AGENT_BROWSER_PROXY_BYPASS") .ok() - .or(config.proxy_bypass), + .or(config.proxy_bypass) + .or_else(|| env::var("NO_PROXY").ok()) + .or_else(|| env::var("no_proxy").ok()), args: env::var("AGENT_BROWSER_ARGS").ok().or(config.args), user_agent: env::var("AGENT_BROWSER_USER_AGENT") .ok() diff --git a/cli/src/main.rs b/cli/src/main.rs index a059dea..ab01c45 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -54,34 +54,67 @@ fn print_json_error_with_type(message: impl AsRef, error_type: &str) { })); } -fn parse_proxy(proxy_str: &str) -> serde_json::Value { +struct ParsedProxy { + server: String, + username: Option, + password: Option, +} + +fn parse_proxy(proxy_str: &str) -> ParsedProxy { let Some(protocol_end) = proxy_str.find("://") else { - return json!({ "server": proxy_str }); + return ParsedProxy { + server: proxy_str.to_string(), + username: None, + password: None, + }; }; let protocol = &proxy_str[..protocol_end + 3]; let rest = &proxy_str[protocol_end + 3..]; let Some(at_pos) = rest.rfind('@') else { - return json!({ "server": proxy_str }); + return ParsedProxy { + server: proxy_str.to_string(), + username: None, + password: None, + }; }; let creds = &rest[..at_pos]; let server_part = &rest[at_pos + 1..]; let server = format!("{}{}", protocol, server_part); - let Some(colon_pos) = creds.find(':') else { - return json!({ - "server": server, - "username": creds, - "password": "" - }); + let (username, password) = match creds.find(':') { + Some(colon_pos) => { + let u = &creds[..colon_pos]; + let p = &creds[colon_pos + 1..]; + ( + if u.is_empty() { + None + } else { + Some(u.to_string()) + }, + if p.is_empty() { + None + } else { + Some(p.to_string()) + }, + ) + } + None => ( + if creds.is_empty() { + None + } else { + Some(creds.to_string()) + }, + None, + ), }; - json!({ - "server": server, - "username": &creds[..colon_pos], - "password": &creds[colon_pos + 1..] - }) + ParsedProxy { + server, + username, + password, + } } fn run_session(args: &[String], session: &str, json_mode: bool) { @@ -330,6 +363,13 @@ fn main() { return; } + // Parse proxy URL to separate server from credentials for the daemon. + let (proxy_server, proxy_username, proxy_password) = if let Some(ref proxy_str) = flags.proxy { + let parsed = parse_proxy(proxy_str); + (Some(parsed.server), parsed.username, parsed.password) + } else { + (None, None, None) + }; let daemon_opts = DaemonOptions { headed: flags.headed, debug: flags.debug, @@ -337,8 +377,10 @@ fn main() { extensions: &flags.extensions, args: flags.args.as_deref(), user_agent: flags.user_agent.as_deref(), - proxy: flags.proxy.as_deref(), + proxy: proxy_server.as_deref(), proxy_bypass: flags.proxy_bypass.as_deref(), + proxy_username: proxy_username.as_deref(), + proxy_password: proxy_password.as_deref(), ignore_https_errors: flags.ignore_https_errors, allow_file_access: flags.allow_file_access, profile: flags.profile.as_deref(), @@ -694,12 +736,16 @@ fn main() { } if let Some(ref proxy_str) = flags.proxy { - let mut proxy_obj = parse_proxy(proxy_str); - // Add bypass if specified + let parsed = parse_proxy(proxy_str); + let mut proxy_obj = json!({ "server": parsed.server }); + if let Some(ref username) = parsed.username { + proxy_obj["username"] = json!(username); + } + if let Some(ref password) = parsed.password { + proxy_obj["password"] = json!(password); + } if let Some(ref bypass) = flags.proxy_bypass { - if let Some(obj) = proxy_obj.as_object_mut() { - obj.insert("bypass".to_string(), json!(bypass)); - } + proxy_obj["bypass"] = json!(bypass); } cmd_obj.insert("proxy".to_string(), proxy_obj); } @@ -1007,55 +1053,55 @@ mod tests { #[test] fn test_parse_proxy_simple() { let result = parse_proxy("http://proxy.com:8080"); - assert_eq!(result["server"], "http://proxy.com:8080"); - assert!(result.get("username").is_none()); - assert!(result.get("password").is_none()); + assert_eq!(result.server, "http://proxy.com:8080"); + assert!(result.username.is_none()); + assert!(result.password.is_none()); } #[test] fn test_parse_proxy_with_auth() { let result = parse_proxy("http://user:pass@proxy.com:8080"); - assert_eq!(result["server"], "http://proxy.com:8080"); - assert_eq!(result["username"], "user"); - assert_eq!(result["password"], "pass"); + assert_eq!(result.server, "http://proxy.com:8080"); + assert_eq!(result.username.as_deref(), Some("user")); + assert_eq!(result.password.as_deref(), Some("pass")); } #[test] fn test_parse_proxy_username_only() { let result = parse_proxy("http://user@proxy.com:8080"); - assert_eq!(result["server"], "http://proxy.com:8080"); - assert_eq!(result["username"], "user"); - assert_eq!(result["password"], ""); + assert_eq!(result.server, "http://proxy.com:8080"); + assert_eq!(result.username.as_deref(), Some("user")); + assert!(result.password.is_none()); } #[test] fn test_parse_proxy_no_protocol() { let result = parse_proxy("proxy.com:8080"); - assert_eq!(result["server"], "proxy.com:8080"); - assert!(result.get("username").is_none()); + assert_eq!(result.server, "proxy.com:8080"); + assert!(result.username.is_none()); } #[test] fn test_parse_proxy_socks5() { let result = parse_proxy("socks5://proxy.com:1080"); - assert_eq!(result["server"], "socks5://proxy.com:1080"); - assert!(result.get("username").is_none()); + assert_eq!(result.server, "socks5://proxy.com:1080"); + assert!(result.username.is_none()); } #[test] fn test_parse_proxy_socks5_with_auth() { let result = parse_proxy("socks5://admin:secret@proxy.com:1080"); - assert_eq!(result["server"], "socks5://proxy.com:1080"); - assert_eq!(result["username"], "admin"); - assert_eq!(result["password"], "secret"); + assert_eq!(result.server, "socks5://proxy.com:1080"); + assert_eq!(result.username.as_deref(), Some("admin")); + assert_eq!(result.password.as_deref(), Some("secret")); } #[test] fn test_parse_proxy_complex_password() { let result = parse_proxy("http://user:p@ss:w0rd@proxy.com:8080"); - assert_eq!(result["server"], "http://proxy.com:8080"); - assert_eq!(result["username"], "user"); - assert_eq!(result["password"], "p@ss:w0rd"); + assert_eq!(result.server, "http://proxy.com:8080"); + assert_eq!(result.username.as_deref(), Some("user")); + assert_eq!(result.password.as_deref(), Some("p@ss:w0rd")); } #[test] diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 114ca74..e50db6a 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -195,6 +195,9 @@ pub struct DaemonState { /// Key is the origin (scheme + host + port), value is the headers map. /// Wrapped in Arc> so the background Fetch handler can read it. pub origin_headers: Arc>>>, + /// Proxy authentication credentials (username, password) for handling + /// Fetch.authRequired events from authenticated proxies. + pub proxy_credentials: Arc>>, /// Background task that processes Fetch.requestPaused events in real-time, /// handling domain filtering, route interception, and origin-scoped headers /// without deadlocking navigation/evaluate. @@ -242,6 +245,7 @@ impl DaemonState { active_frame_id: None, iframe_sessions: HashMap::new(), origin_headers: Arc::new(RwLock::new(HashMap::new())), + proxy_credentials: Arc::new(RwLock::new(None)), fetch_handler_task: None, mouse_state: MouseState::default(), pending_dialog: None, @@ -272,8 +276,9 @@ impl DaemonState { } } - /// Start the background task that processes all Fetch.requestPaused events - /// in real-time (domain filtering, route interception, origin-scoped headers). + /// Start the background task that processes Fetch.requestPaused and + /// Fetch.authRequired events in real-time (domain filtering, route + /// interception, origin-scoped headers, proxy authentication). /// Must be called after the browser is set and events are subscribed. fn start_fetch_handler(&mut self) { // Abort any existing handler. @@ -290,10 +295,50 @@ impl DaemonState { let domain_filter = self.domain_filter.clone(); let routes = self.routes.clone(); let origin_headers = self.origin_headers.clone(); + let proxy_credentials = self.proxy_credentials.clone(); self.fetch_handler_task = Some(tokio::spawn(async move { loop { match rx.recv().await { + Ok(event) if event.method == "Fetch.authRequired" => { + let request_id = event + .params + .get("requestId") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let sid = event.session_id.clone().unwrap_or_default(); + let creds = proxy_credentials.read().await; + if let Some((ref user, ref pass)) = *creds { + let _ = client + .send_command( + "Fetch.continueWithAuth", + Some(json!({ + "requestId": request_id, + "authChallengeResponse": { + "response": "ProvideCredentials", + "username": user, + "password": pass, + } + })), + Some(&sid), + ) + .await; + } else { + let _ = client + .send_command( + "Fetch.continueWithAuth", + Some(json!({ + "requestId": request_id, + "authChallengeResponse": { + "response": "CancelAuth", + } + })), + Some(&sid), + ) + .await; + } + } Ok(event) if event.method == "Fetch.requestPaused" => { let request_id = event .params @@ -854,10 +899,12 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value { // Install domain filter on new pages let df = state.domain_filter.read().await; if let Some(ref filter) = *df { + let has_proxy_creds = state.proxy_credentials.read().await.is_some(); let _ = network::install_domain_filter( &mgr.client, &attach.session_id, &filter.allowed_domains, + has_proxy_creds, ) .await; } @@ -1186,6 +1233,16 @@ async fn auto_launch(state: &mut DaemonState) -> Result<(), String> { let options = launch_options_from_env(); let engine = env::var("AGENT_BROWSER_ENGINE").ok(); + // Store proxy credentials for Fetch.authRequired handling + let has_proxy_auth = options.proxy_username.is_some(); + if has_proxy_auth { + let mut creds = state.proxy_credentials.write().await; + *creds = Some(( + options.proxy_username.clone().unwrap_or_default(), + options.proxy_password.clone().unwrap_or_default(), + )); + } + if let Ok(cdp) = env::var("AGENT_BROWSER_CDP") { let mgr = BrowserManager::connect_cdp(&cdp).await?; state.reset_input_state(); @@ -1213,6 +1270,16 @@ async fn auto_launch(state: &mut DaemonState) -> Result<(), String> { state.subscribe_to_browser_events(); state.start_fetch_handler(); state.update_stream_client().await; + + // Enable Fetch with handleAuthRequests for proxy authentication + if has_proxy_auth { + if let Some(ref mgr) = state.browser { + if let Ok(session_id) = mgr.active_session_id() { + let _ = network::install_domain_filter_fetch(&mgr.client, session_id, true).await; + } + } + } + try_auto_restore_state(state).await; Ok(()) } @@ -1234,6 +1301,8 @@ fn launch_options_from_env() -> LaunchOptions { executable_path: env::var("AGENT_BROWSER_EXECUTABLE_PATH").ok(), proxy: env::var("AGENT_BROWSER_PROXY").ok(), proxy_bypass: env::var("AGENT_BROWSER_PROXY_BYPASS").ok(), + proxy_username: env::var("AGENT_BROWSER_PROXY_USERNAME").ok(), + proxy_password: env::var("AGENT_BROWSER_PROXY_PASSWORD").ok(), profile: env::var("AGENT_BROWSER_PROFILE").ok(), allow_file_access: env::var("AGENT_BROWSER_ALLOW_FILE_ACCESS") .map(|v| v == "1" || v == "true") @@ -1437,6 +1506,18 @@ async fn handle_launch(cmd: &Value, state: &mut DaemonState) -> Result Result Result Result Vec { .collect(); let has_domain_filter = state.domain_filter.read().await.is_some(); let has_origin_headers = !state.origin_headers.read().await.is_empty(); - if (has_domain_filter || has_origin_headers) && !patterns.iter().any(|p| p["urlPattern"] == "*") + let has_proxy_creds = state.proxy_credentials.read().await.is_some(); + if (has_domain_filter || has_origin_headers || has_proxy_creds) + && !patterns.iter().any(|p| p["urlPattern"] == "*") { patterns.push(json!({ "urlPattern": "*" })); } patterns } +/// Build the full Fetch.enable params object, including `handleAuthRequests` +/// when proxy credentials are configured. +async fn build_fetch_enable_params(state: &DaemonState, patterns: Vec) -> Value { + let has_proxy_creds = state.proxy_credentials.read().await.is_some(); + if has_proxy_creds { + json!({ "patterns": patterns, "handleAuthRequests": true }) + } else { + json!({ "patterns": patterns }) + } +} + async fn handle_route(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); @@ -5861,12 +5982,9 @@ async fn handle_route(cmd: &Value, state: &mut DaemonState) -> Result Result, pub proxy: Option, pub proxy_bypass: Option, + pub proxy_username: Option, + pub proxy_password: Option, pub profile: Option, pub args: Vec, pub allow_file_access: bool, @@ -85,6 +87,8 @@ impl Default for LaunchOptions { executable_path: None, proxy: None, proxy_bypass: None, + proxy_username: None, + proxy_password: None, profile: None, args: Vec::new(), allow_file_access: false, diff --git a/cli/src/native/network.rs b/cli/src/native/network.rs index 6827f63..3d5bad4 100644 --- a/cli/src/native/network.rs +++ b/cli/src/native/network.rs @@ -233,15 +233,16 @@ pub async fn install_domain_filter_script( pub async fn install_domain_filter_fetch( client: &CdpClient, session_id: &str, + handle_auth_requests: bool, ) -> Result<(), String> { + let mut params = json!({ + "patterns": [{ "urlPattern": "*" }] + }); + if handle_auth_requests { + params["handleAuthRequests"] = json!(true); + } client - .send_command( - "Fetch.enable", - Some(json!({ - "patterns": [{ "urlPattern": "*" }] - })), - Some(session_id), - ) + .send_command("Fetch.enable", Some(params), Some(session_id)) .await?; Ok(()) } @@ -253,9 +254,10 @@ pub async fn install_domain_filter( client: &CdpClient, session_id: &str, allowed_domains: &[String], + handle_auth_requests: bool, ) -> Result<(), String> { install_domain_filter_script(client, session_id, allowed_domains).await?; - install_domain_filter_fetch(client, session_id).await?; + install_domain_filter_fetch(client, session_id, handle_auth_requests).await?; Ok(()) } diff --git a/cli/src/output.rs b/cli/src/output.rs index 34f6e08..5d70ff2 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -2684,9 +2684,9 @@ Options: --args Browser launch args, comma or newline separated (or AGENT_BROWSER_ARGS) e.g., --args "--no-sandbox,--disable-blink-features=AutomationControlled" --user-agent Custom User-Agent (or AGENT_BROWSER_USER_AGENT) - --proxy Proxy server URL (or AGENT_BROWSER_PROXY) - e.g., --proxy "http://user:pass@127.0.0.1:7890" - --proxy-bypass Bypass proxy for these hosts (or AGENT_BROWSER_PROXY_BYPASS) + --proxy Proxy server URL (or AGENT_BROWSER_PROXY, HTTP_PROXY, HTTPS_PROXY, ALL_PROXY) + Supports authenticated proxies: --proxy "http://user:pass@127.0.0.1:7890" + --proxy-bypass Bypass proxy for these hosts (or AGENT_BROWSER_PROXY_BYPASS, NO_PROXY) e.g., --proxy-bypass "localhost,*.internal.com" --ignore-https-errors Ignore HTTPS certificate errors --allow-file-access Allow file:// URLs to access local files (Chromium only) @@ -2764,6 +2764,9 @@ Environment: AGENT_BROWSER_CONFIRM_ACTIONS Action categories requiring confirmation AGENT_BROWSER_CONFIRM_INTERACTIVE Enable interactive confirmation prompts AGENT_BROWSER_ENGINE Browser engine: chrome (default), lightpanda + HTTP_PROXY / HTTPS_PROXY Standard proxy env vars (fallback if AGENT_BROWSER_PROXY not set) + ALL_PROXY SOCKS proxy (fallback for proxy) + NO_PROXY Bypass proxy for hosts (fallback for proxy-bypass) AGENT_BROWSER_SCREENSHOT_DIR Default screenshot output directory AGENT_BROWSER_SCREENSHOT_QUALITY JPEG quality 0-100 AGENT_BROWSER_SCREENSHOT_FORMAT Screenshot format: png, jpeg