Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d99a223d23 | ||
|
|
4e7e80a596 | ||
|
|
9bf79a4242 | ||
|
|
5b4ffdb2bb | ||
|
|
62e7229b47 | ||
|
|
23ab4ce68f |
Generated
+1
-1
@@ -290,7 +290,7 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.3.0"
|
version = "1.4.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "chrome-use"
|
name = "chrome-use"
|
||||||
version = "1.3.0"
|
version = "1.4.1"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "Fast browser automation CLI for AI agents"
|
description = "Fast browser automation CLI for AI agents"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
@@ -3,6 +3,23 @@ use std::env;
|
|||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
/// Embed the version of the `ab-connect` extension this CLI ships alongside, so
|
||||||
|
/// `doctor` can tell a connected extension "you're older than what this CLI
|
||||||
|
/// expects, update it." Read from the extension manifest at build time so it
|
||||||
|
/// stays in sync with whatever extension version is in the same checkout/release
|
||||||
|
/// (the ext is on its own 0.4.x line, separate from the CLI version). Falls back
|
||||||
|
/// to "unknown" if the manifest can't be read.
|
||||||
|
fn embed_extension_version() {
|
||||||
|
let manifest = Path::new("../extensions/ab-connect/manifest.json");
|
||||||
|
println!("cargo:rerun-if-changed=../extensions/ab-connect/manifest.json");
|
||||||
|
let version = fs::read_to_string(manifest)
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
|
||||||
|
.and_then(|v| v.get("version").and_then(|x| x.as_str()).map(String::from))
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
println!("cargo:rustc-env=AB_CONNECT_VERSION={}", version);
|
||||||
|
}
|
||||||
|
|
||||||
/// Ensure `packages/dashboard/out/` exists so `rust-embed` doesn't fail during
|
/// Ensure `packages/dashboard/out/` exists so `rust-embed` doesn't fail during
|
||||||
/// Rust-only dev builds where the dashboard hasn't been built. The placeholder
|
/// Rust-only dev builds where the dashboard hasn't been built. The placeholder
|
||||||
/// `index.html` is only written when the directory is completely absent.
|
/// `index.html` is only written when the directory is completely absent.
|
||||||
@@ -20,6 +37,7 @@ fn ensure_dashboard_dir() {
|
|||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
ensure_dashboard_dir();
|
ensure_dashboard_dir();
|
||||||
|
embed_extension_version();
|
||||||
|
|
||||||
let protocol_dir = Path::new("cdp-protocol");
|
let protocol_dir = Path::new("cdp-protocol");
|
||||||
let out_dir = env::var("OUT_DIR").unwrap();
|
let out_dir = env::var("OUT_DIR").unwrap();
|
||||||
|
|||||||
@@ -449,6 +449,28 @@ pub fn relay_url() -> Option<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sidecar recording the connected extension's version, written by the host when
|
||||||
|
/// it receives the extension's `hello` (sibling of `relay-cdp-url`). Lets
|
||||||
|
/// `doctor` surface which extension build is live without a CDP round-trip.
|
||||||
|
fn relay_ext_version_path() -> PathBuf {
|
||||||
|
relay_url_path().with_file_name("relay-ext-version")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Version of the connected `ab-connect` extension, if the host learned it from
|
||||||
|
/// the extension's `hello`. `None` when no extension has connected since the
|
||||||
|
/// host started, or the extension predates version reporting.
|
||||||
|
pub fn relay_ext_version() -> Option<String> {
|
||||||
|
let s = std::fs::read_to_string(relay_ext_version_path())
|
||||||
|
.ok()?
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
if s.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Hidden `__nm-host` mode: launched by Chrome for the ab-connect extension.
|
/// Hidden `__nm-host` mode: launched by Chrome for the ab-connect extension.
|
||||||
///
|
///
|
||||||
/// Bridges the extension (native-messaging stdio, envelope protocol) to a local
|
/// Bridges the extension (native-messaging stdio, envelope protocol) to a local
|
||||||
@@ -570,6 +592,15 @@ async fn nm_host_main() {
|
|||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
Err(_) => continue,
|
Err(_) => continue,
|
||||||
};
|
};
|
||||||
|
// Extension version handshake: record it next to the relay URL so
|
||||||
|
// `doctor` can report which extension build is live (and whether it's
|
||||||
|
// behind). Best-effort; the message carries no CDP payload.
|
||||||
|
if v.get("method").and_then(|m| m.as_str()) == Some("hello") {
|
||||||
|
if let Some(ver) = v.get("version").and_then(|x| x.as_str()) {
|
||||||
|
let _ = std::fs::write(relay_ext_version_path(), ver);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let outs = {
|
let outs = {
|
||||||
let mut s = state.lock().await;
|
let mut s = state.lock().await;
|
||||||
s.handle_ext_message(&v, "")
|
s.handle_ext_message(&v, "")
|
||||||
@@ -602,6 +633,7 @@ async fn nm_host_main() {
|
|||||||
}
|
}
|
||||||
nm_log("[nm-host] stdin EOF — Chrome closed the port");
|
nm_log("[nm-host] stdin EOF — Chrome closed the port");
|
||||||
let _ = std::fs::remove_file(relay_url_path());
|
let _ = std::fs::remove_file(relay_url_path());
|
||||||
|
let _ = std::fs::remove_file(relay_ext_version_path());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ mod launch;
|
|||||||
mod network;
|
mod network;
|
||||||
mod providers;
|
mod providers;
|
||||||
mod security;
|
mod security;
|
||||||
|
mod versions;
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
@@ -97,6 +98,7 @@ pub fn run_doctor(opts: DoctorOptions) -> i32 {
|
|||||||
let mut fixed: Vec<String> = Vec::new();
|
let mut fixed: Vec<String> = Vec::new();
|
||||||
|
|
||||||
environment::check(&mut checks);
|
environment::check(&mut checks);
|
||||||
|
versions::check(&mut checks);
|
||||||
chrome::check(&mut checks);
|
chrome::check(&mut checks);
|
||||||
daemon::check(&mut checks);
|
daemon::check(&mut checks);
|
||||||
config::check(&mut checks);
|
config::check(&mut checks);
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
//! Version-coherence checks across all four moving parts: the CLI binary, the
|
||||||
|
//! per-session daemons (covered by `daemon.rs`), the connected `ab-connect`
|
||||||
|
//! extension, and the bundled skill. The extension was previously a black box —
|
||||||
|
//! nothing reported which build was live — so a user could sit on an old
|
||||||
|
//! extension with no signal. The extension now reports its version over the
|
||||||
|
//! relay (`hello`), the host records it, and this surfaces it in one place.
|
||||||
|
|
||||||
|
use super::{Check, Status};
|
||||||
|
use crate::{connect, upgrade};
|
||||||
|
|
||||||
|
pub(super) fn check(checks: &mut Vec<Check>) {
|
||||||
|
let category = "Versions";
|
||||||
|
let cli_version = env!("CARGO_PKG_VERSION");
|
||||||
|
|
||||||
|
// CLI — compare against the latest seen by the background update check.
|
||||||
|
match upgrade::cached_latest_version() {
|
||||||
|
Some(latest) if upgrade::version_is_newer(&latest, cli_version) => {
|
||||||
|
checks.push(
|
||||||
|
Check::new(
|
||||||
|
"versions.cli",
|
||||||
|
category,
|
||||||
|
Status::Warn,
|
||||||
|
format!("CLI {cli_version} (newer available: {latest})"),
|
||||||
|
)
|
||||||
|
.with_fix("chrome-use upgrade".to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.cli",
|
||||||
|
category,
|
||||||
|
Status::Pass,
|
||||||
|
format!("CLI {cli_version}"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extension — the build this CLI shipped alongside (embedded at compile time
|
||||||
|
// from the extension manifest) is what we expect to be running.
|
||||||
|
let expected_ext = env!("AB_CONNECT_VERSION");
|
||||||
|
match connect::relay_ext_version() {
|
||||||
|
Some(ext) if upgrade::version_is_newer(expected_ext, &ext) => {
|
||||||
|
checks.push(
|
||||||
|
Check::new(
|
||||||
|
"versions.extension",
|
||||||
|
category,
|
||||||
|
Status::Warn,
|
||||||
|
format!("extension {ext} is behind the bundled {expected_ext}"),
|
||||||
|
)
|
||||||
|
.with_fix(
|
||||||
|
"update ab-connect in Chrome: chrome://extensions \u{2192} reload \
|
||||||
|
(or wait for the Web Store auto-update)"
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Some(ext) => {
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.extension",
|
||||||
|
category,
|
||||||
|
Status::Pass,
|
||||||
|
format!("extension {ext}"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.extension",
|
||||||
|
category,
|
||||||
|
Status::Info,
|
||||||
|
format!(
|
||||||
|
"extension not connected (or it predates version reporting — \
|
||||||
|
expected {expected_ext})"
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skill — ships inside the same release artifact as the binary, so it's
|
||||||
|
// version-locked here. Copies made elsewhere via `skills add` aren't.
|
||||||
|
checks.push(Check::new(
|
||||||
|
"versions.skill",
|
||||||
|
category,
|
||||||
|
Status::Info,
|
||||||
|
format!(
|
||||||
|
"skills bundled with this CLI ({cli_version}); copies made via `skills add` \
|
||||||
|
elsewhere may be stale — re-run to refresh"
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
+28
-15
@@ -2888,7 +2888,32 @@ async fn handle_snapshot(cmd: &Value, state: &mut DaemonState) -> Result<Value,
|
|||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
Ok(json!({ "snapshot": tree, "origin": url, "refs": refs }))
|
let ref_count = refs.len();
|
||||||
|
let mut out = json!({ "snapshot": tree, "origin": url, "refs": refs });
|
||||||
|
|
||||||
|
// Canvas/WebGL apps (games, map/3D viewers, drawing tools) paint to a
|
||||||
|
// <canvas> and expose almost no accessibility tree, so `snapshot` comes back
|
||||||
|
// near-empty and agents get stuck looking for refs that will never exist
|
||||||
|
// (dogfood: the Dead Cell game). When the tree is sparse but a canvas
|
||||||
|
// dominates the viewport, tell them to switch to the screenshot-driven path.
|
||||||
|
if ref_count < 3 {
|
||||||
|
let canvas_js =
|
||||||
|
"(() => { const c = document.querySelector('canvas'); if (!c) return false; \
|
||||||
|
const r = c.getBoundingClientRect(); \
|
||||||
|
return r.width * r.height > innerWidth * innerHeight * 0.5; })()";
|
||||||
|
if let Ok(v) = mgr.evaluate(canvas_js, None).await {
|
||||||
|
if v.as_bool() == Some(true) {
|
||||||
|
out["note"] = json!(
|
||||||
|
"This page renders to a <canvas> (game / WebGL / editor) and exposes almost no \
|
||||||
|
accessibility tree — refs won't help. Use `screenshot` to see it, coordinate \
|
||||||
|
`click <x> <y>` to interact, and `keydown`/`keyup`/`press` for keyboard \
|
||||||
|
(hold-to-move: `keydown d` … `keyup d`)."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve a (possibly relative) saved-file path to an absolute one so the CLI
|
/// Resolve a (possibly relative) saved-file path to an absolute one so the CLI
|
||||||
@@ -8760,13 +8785,7 @@ async fn handle_keydown(cmd: &Value, state: &DaemonState) -> Result<Value, Strin
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'key' parameter")?;
|
.ok_or("Missing 'key' parameter")?;
|
||||||
|
|
||||||
mgr.client
|
interaction::dispatch_single_key(&mgr.client, &session_id, key, "keyDown").await?;
|
||||||
.send_command(
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
Some(json!({ "type": "keyDown", "key": key })),
|
|
||||||
Some(&session_id),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
Ok(json!({ "keydown": key }))
|
Ok(json!({ "keydown": key }))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -8778,13 +8797,7 @@ async fn handle_keyup(cmd: &Value, state: &DaemonState) -> Result<Value, String>
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.ok_or("Missing 'key' parameter")?;
|
.ok_or("Missing 'key' parameter")?;
|
||||||
|
|
||||||
mgr.client
|
interaction::dispatch_single_key(&mgr.client, &session_id, key, "keyUp").await?;
|
||||||
.send_command(
|
|
||||||
"Input.dispatchKeyEvent",
|
|
||||||
Some(json!({ "type": "keyUp", "key": key })),
|
|
||||||
Some(&session_id),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
Ok(json!({ "keyup": key }))
|
Ok(json!({ "keyup": key }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -166,6 +166,25 @@ fn resolve_active_index(
|
|||||||
active_page_index
|
active_page_index
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the resolved active page is a tab the session created (its target_id
|
||||||
|
/// is in `created_targets`). Pure core of [`BrowserManager::active_is_session_owned`]
|
||||||
|
/// so the relay no-hijack rule is unit-testable without a live browser.
|
||||||
|
fn active_index_is_owned(
|
||||||
|
pages: &[PageInfo],
|
||||||
|
active_target_id: Option<&str>,
|
||||||
|
active_page_index: usize,
|
||||||
|
created_targets: &HashSet<String>,
|
||||||
|
) -> bool {
|
||||||
|
pages
|
||||||
|
.get(resolve_active_index(
|
||||||
|
pages,
|
||||||
|
active_target_id,
|
||||||
|
active_page_index,
|
||||||
|
))
|
||||||
|
.map(|p| created_targets.contains(&p.target_id))
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
/// Converts common error messages into AI-friendly, actionable descriptions.
|
/// Converts common error messages into AI-friendly, actionable descriptions.
|
||||||
pub fn to_ai_friendly_error(error: &str) -> String {
|
pub fn to_ai_friendly_error(error: &str) -> String {
|
||||||
let lower = error.to_lowercase();
|
let lower = error.to_lowercase();
|
||||||
@@ -799,6 +818,20 @@ impl BrowserManager {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the resolved active page is a tab THIS session created (via
|
||||||
|
/// `Target.createTarget` — `tab new`, `ensure_page`, or the first `open`).
|
||||||
|
/// On the shared real browser a fresh session also passively attaches to the
|
||||||
|
/// user's existing tabs; those are NOT owned, and navigating one would
|
||||||
|
/// clobber the user's page. Used to gate `navigate` on the relay.
|
||||||
|
fn active_is_session_owned(&self) -> bool {
|
||||||
|
active_index_is_owned(
|
||||||
|
&self.pages,
|
||||||
|
self.active_target_id.as_deref(),
|
||||||
|
self.active_page_index,
|
||||||
|
&self.created_targets,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Pin the current active page by target_id so later commands stick to it.
|
/// Pin the current active page by target_id so later commands stick to it.
|
||||||
/// Call after any explicit open / tab new / tab switch.
|
/// Call after any explicit open / tab new / tab switch.
|
||||||
fn pin_active_target(&mut self) {
|
fn pin_active_target(&mut self) {
|
||||||
@@ -816,6 +849,18 @@ impl BrowserManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn navigate(&mut self, url: &str, wait_until: WaitUntil) -> Result<Value, String> {
|
pub async fn navigate(&mut self, url: &str, wait_until: WaitUntil) -> Result<Value, String> {
|
||||||
|
// On the shared real browser (extension relay), a fresh session only
|
||||||
|
// passively attached to the user's existing tabs — it doesn't own any. The
|
||||||
|
// pre-fix code made one of those the active tab, so the first `open` then
|
||||||
|
// navigated (clobbered) the user's page: in dogfooding an `open` replaced a
|
||||||
|
// half-filled form with the target site. If the active tab isn't one we
|
||||||
|
// created, open our own tab in this session's group and navigate THAT, so
|
||||||
|
// the user's (and other sessions') tabs are never hijacked. Off the relay
|
||||||
|
// (a browser we launched) reusing the active tab is correct, so this is
|
||||||
|
// gated on `agent_group()`.
|
||||||
|
if self.agent_group().is_some() && !self.active_is_session_owned() {
|
||||||
|
self.tab_new(None, None).await?;
|
||||||
|
}
|
||||||
let session_id = self.active_session_id()?.to_string();
|
let session_id = self.active_session_id()?.to_string();
|
||||||
let mut lifecycle_rx = self.client.subscribe();
|
let mut lifecycle_rx = self.client.subscribe();
|
||||||
|
|
||||||
@@ -2431,6 +2476,35 @@ mod tests {
|
|||||||
assert_eq!(resolve_active_index(&pages, Some("CLOSED"), 1), 1);
|
assert_eq!(resolve_active_index(&pages, Some("CLOSED"), 1), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- issue: `open` must not hijack a user's tab on the relay (dogfood) ---
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn active_not_owned_when_only_user_tabs_discovered() {
|
||||||
|
// A fresh relay session passively attached to the user's tabs but created
|
||||||
|
// none — so navigate must NOT reuse the active tab (it'd clobber the
|
||||||
|
// user's page); it has to open its own first.
|
||||||
|
let pages = vec![page("USER_A"), page("USER_B")];
|
||||||
|
let created = HashSet::new();
|
||||||
|
assert!(!active_index_is_owned(&pages, Some("USER_A"), 0, &created));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn active_owned_when_session_created_the_tab() {
|
||||||
|
let pages = vec![page("USER_A"), page("OURS")];
|
||||||
|
let mut created = HashSet::new();
|
||||||
|
created.insert("OURS".to_string());
|
||||||
|
// Active pinned to the tab we created → safe to navigate it.
|
||||||
|
assert!(active_index_is_owned(&pages, Some("OURS"), 1, &created));
|
||||||
|
// But pinned to the user's tab → not owned, even though we own another.
|
||||||
|
assert!(!active_index_is_owned(&pages, Some("USER_A"), 0, &created));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn active_not_owned_when_no_pages() {
|
||||||
|
let created = HashSet::new();
|
||||||
|
assert!(!active_index_is_owned(&[], None, 0, &created));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolve_active_index_pin_survives_passive_background_tab() {
|
fn resolve_active_index_pin_survives_passive_background_tab() {
|
||||||
// A foreign tab ("Z") gets appended by passive discovery after we pinned
|
// A foreign tab ("Z") gets appended by passive discovery after we pinned
|
||||||
|
|||||||
@@ -557,6 +557,48 @@ pub async fn press_key_with_modifiers(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Dispatch a SINGLE key event (`keyDown` or `keyUp`) carrying the full key
|
||||||
|
/// descriptor — `key`, `code`, `windowsVirtualKeyCode`/`nativeVirtualKeyCode`,
|
||||||
|
/// and (on key-down) printable `text`. Powers the `keydown`/`keyup` commands.
|
||||||
|
///
|
||||||
|
/// The previous implementation sent only `{key}`, so games and shortcut handlers
|
||||||
|
/// that read `event.code` (e.g. `"KeyD"`, `"ArrowRight"`) or `event.keyCode` saw
|
||||||
|
/// nothing — a held key set no movement flag and did nothing (dogfood: holding a
|
||||||
|
/// direction in a canvas platformer barely nudged the player). Sending the same
|
||||||
|
/// descriptor `press` uses makes hold-to-move work regardless of which field the
|
||||||
|
/// page keys off.
|
||||||
|
pub async fn dispatch_single_key(
|
||||||
|
client: &CdpClient,
|
||||||
|
session_id: &str,
|
||||||
|
key: &str,
|
||||||
|
event_type: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (key_name, code, key_code) = named_key_info(key);
|
||||||
|
// Printable text is only meaningful on key-down; key-up never inserts.
|
||||||
|
let text = if event_type == "keyDown" {
|
||||||
|
key_text(&key_name)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
client
|
||||||
|
.send_command_typed::<_, Value>(
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
&DispatchKeyEventParams {
|
||||||
|
event_type: event_type.to_string(),
|
||||||
|
key: Some(key_name),
|
||||||
|
code: Some(code),
|
||||||
|
text: text.clone(),
|
||||||
|
unmodified_text: text,
|
||||||
|
windows_virtual_key_code: Some(key_code),
|
||||||
|
native_virtual_key_code: Some(key_code),
|
||||||
|
modifiers: None,
|
||||||
|
},
|
||||||
|
Some(session_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn scroll(
|
pub async fn scroll(
|
||||||
client: &CdpClient,
|
client: &CdpClient,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
|
|||||||
@@ -297,6 +297,11 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
|
|||||||
// Snapshot
|
// Snapshot
|
||||||
if let Some(snapshot) = data.get("snapshot").and_then(|v| v.as_str()) {
|
if let Some(snapshot) = data.get("snapshot").and_then(|v| v.as_str()) {
|
||||||
print_with_boundaries(snapshot, origin, opts);
|
print_with_boundaries(snapshot, origin, opts);
|
||||||
|
// Canvas-app hint: the tree was near-empty but the page paints to a
|
||||||
|
// <canvas>, so refs are a dead end — point at the screenshot path.
|
||||||
|
if let Some(note) = data.get("note").and_then(|v| v.as_str()) {
|
||||||
|
eprintln!("{}", color::dim(note));
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Title
|
// Title
|
||||||
@@ -3066,6 +3071,9 @@ Core Commands:
|
|||||||
type <sel> <text> Type into element
|
type <sel> <text> Type into element
|
||||||
fill <sel> <text> Clear and fill
|
fill <sel> <text> Clear and fill
|
||||||
press <key> Press key (Enter, Tab, Control+a)
|
press <key> Press key (Enter, Tab, Control+a)
|
||||||
|
keydown <key> Hold a key down (no auto-release) — for games/shortcuts
|
||||||
|
keyup <key> Release a held key. Pair with keydown to hold-to-move:
|
||||||
|
`keydown d` … `keyup d`
|
||||||
keyboard type <text> Type text with real keystrokes (no selector)
|
keyboard type <text> Type text with real keystrokes (no selector)
|
||||||
keyboard inserttext <text> Insert text without key events
|
keyboard inserttext <text> Insert text without key events
|
||||||
hover <sel> Hover element
|
hover <sel> Hover element
|
||||||
|
|||||||
@@ -52,6 +52,27 @@ fn is_newer(latest: &str, current: &str) -> bool {
|
|||||||
matches!((parse_version(latest), parse_version(current)), (Some(l), Some(c)) if l > c)
|
matches!((parse_version(latest), parse_version(current)), (Some(l), Some(c)) if l > c)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Public semver-ish comparison (`latest` strictly newer than `current`), so
|
||||||
|
/// `doctor` can flag a stale extension/CLI without re-implementing parsing.
|
||||||
|
pub fn version_is_newer(latest: &str, current: &str) -> bool {
|
||||||
|
is_newer(latest, current)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The latest CLI version recorded by the background update check, if any.
|
||||||
|
/// `doctor` uses it to show "a newer chrome-use is available" without a network
|
||||||
|
/// call (the `__update-check` worker refreshes the cache out of band).
|
||||||
|
pub fn cached_latest_version() -> Option<String> {
|
||||||
|
std::fs::read_to_string(update_cache_path())
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
|
||||||
|
.and_then(|j| {
|
||||||
|
j.get("latest")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
})
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
/// Hidden `__update-check` subcommand: fetch the latest release tag and cache it.
|
/// Hidden `__update-check` subcommand: fetch the latest release tag and cache it.
|
||||||
/// Spawned detached by [`maybe_notify_update`] so the network call never blocks a
|
/// Spawned detached by [`maybe_notify_update`] so the network call never blocks a
|
||||||
/// real command. Uses `curl` (no extra deps, matches `upgrade`).
|
/// real command. Uses `curl` (no extra deps, matches `upgrade`).
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
@@ -108,6 +108,12 @@ function connectHost() {
|
|||||||
// reconnect. Keep chrome.debugger attached so reconnect is cheap.
|
// reconnect. Keep chrome.debugger attached so reconnect is cheap.
|
||||||
for (const tabId of tabs.keys()) setBadge(tabId, 'connecting')
|
for (const tabId of tabs.keys()) setBadge(tabId, 'connecting')
|
||||||
})
|
})
|
||||||
|
// Report our version so the host can tell the CLI/`doctor` which extension
|
||||||
|
// build is live (otherwise the extension version is a black box — the user
|
||||||
|
// can't tell they're on an old one). Best-effort; ignored by older hosts.
|
||||||
|
try {
|
||||||
|
postToHost({ method: 'hello', version: chrome.runtime.getManifest().version })
|
||||||
|
} catch {}
|
||||||
// Tell the daemon about everything we already have attached, then attach
|
// Tell the daemon about everything we already have attached, then attach
|
||||||
// anything new.
|
// anything new.
|
||||||
reannounceAttachedTabs()
|
reannounceAttachedTabs()
|
||||||
@@ -149,23 +155,56 @@ function tabForTarget(targetId) {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
// Best-effort recovery for a stale `cb-tab-<tabId>` session: the handle is gone
|
// The STABLE Chrome tabId encoded in a `cb-tab-<tabId>` session id (#17), or
|
||||||
// from our maps, but if the underlying Chrome tab still exists and is eligible,
|
// null for any other session shape (child/iframe sessions). The tabId is the
|
||||||
// re-attach to it and return its id so the in-flight command can be retried.
|
// real source of truth: it survives the renderer-process swaps (cross-origin
|
||||||
// Returns null when the tab is genuinely gone (closed / restricted), in which
|
// OAuth/SSO navs) that tear down the page's CDP target — which is why binding to
|
||||||
// case the caller surfaces the stale-session error. (issue #20.1)
|
// it (like claude-in-chrome) rides through the hop that killed the old
|
||||||
|
// target/sessionId binding (issue #23).
|
||||||
|
function tabIdFromSession(sessionId) {
|
||||||
|
const m = /^cb-tab-(\d+)$/.exec(sessionId || '')
|
||||||
|
return m ? Number(m[1]) : null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the debugger is attached to a `cb-tab-<tabId>` session's tab, re-attaching
|
||||||
|
// across the transient window of a process swap (with a couple of short retries).
|
||||||
|
// Returns the tabId on success, or null when the tab is genuinely gone
|
||||||
|
// (closed / restricted). (issues #20.1, #23)
|
||||||
async function recoverSessionTab(sessionId) {
|
async function recoverSessionTab(sessionId) {
|
||||||
const m = /^cb-tab-(\d+)$/.exec(sessionId)
|
const tabId = tabIdFromSession(sessionId)
|
||||||
if (!m) return null
|
if (tabId == null) return null
|
||||||
const tabId = Number(m[1])
|
for (let i = 0; i < 3; i++) {
|
||||||
const tab = await chrome.tabs.get(tabId).catch(() => null)
|
const tab = await chrome.tabs.get(tabId).catch(() => null)
|
||||||
if (!eligible(tab)) return null
|
if (!eligible(tab)) return null
|
||||||
try {
|
try {
|
||||||
await attachTab(tabId)
|
await attachTab(tabId)
|
||||||
} catch {
|
if (tabs.has(tabId)) return tabId
|
||||||
return null
|
} catch {
|
||||||
|
// mid-swap: the tab exists but isn't attachable yet — back off and retry.
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 120 + i * 150))
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send a CDP command to a tab, riding a debugger detach that can happen between
|
||||||
|
// our attach check and the command itself (a renderer-process swap mid-flight).
|
||||||
|
// On a detached-style failure, drop the stale handle, re-attach the stable tab,
|
||||||
|
// and retry once — so a cross-process nav never surfaces as a hard error (#23).
|
||||||
|
async function sendCdpToTab(tabId, method, params) {
|
||||||
|
const dbg = { tabId }
|
||||||
|
try {
|
||||||
|
return await chrome.debugger.sendCommand(dbg, method, params)
|
||||||
|
} catch (e) {
|
||||||
|
const msg = String((e && e.message) || e)
|
||||||
|
if (!/detached|not attached|target.*(closed|gone)|no target|cannot access|frame.*detached/i.test(msg)) {
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
detachTab(tabId, false)
|
||||||
|
const ok = await recoverSessionTab(`cb-tab-${tabId}`)
|
||||||
|
if (!ok) throw e
|
||||||
|
return await chrome.debugger.sendCommand(dbg, method, params)
|
||||||
}
|
}
|
||||||
return tabs.has(tabId) ? tabId : null
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function anyConnectedTab() {
|
function anyConnectedTab() {
|
||||||
@@ -226,24 +265,26 @@ async function handleForwardCdpCommand(msg) {
|
|||||||
// Fail loudly instead so the agent sees an actionable error, not bad data.
|
// Fail loudly instead so the agent sees an actionable error, not bad data.
|
||||||
let tabId
|
let tabId
|
||||||
if (sessionId) {
|
if (sessionId) {
|
||||||
tabId = tabForSession(sessionId)
|
// The stable Chrome tabId encoded in `cb-tab-<tabId>` is the source of truth
|
||||||
if (!tabId) {
|
// (it survives renderer-process swaps; the CDP target/sessionId does not).
|
||||||
// The session's debugger handle is gone, but `cb-tab-<tabId>` encodes the
|
// Resolve via it primarily — don't depend on a session→tab map entry that the
|
||||||
// STABLE Chrome tabId (#17). A cross-process navigation (e.g. an SSO
|
// detach handler may have cleared — and ensure the debugger is attached,
|
||||||
// redirect to another origin), a service-worker restart, or DevTools
|
// re-attaching across a cross-process nav before failing (issues #20.1, #23).
|
||||||
// briefly stealing the debugger all tear the handle down while the tab
|
// `tabForSession` still covers child/iframe sessions that aren't `cb-tab-*`.
|
||||||
// itself lives on. Before failing, try to transparently re-attach to that
|
tabId = tabIdFromSession(sessionId) ?? tabForSession(sessionId)
|
||||||
// same tab and retry — so `open`/`navigate`/`eval` self-heal instead of
|
if (tabId == null) {
|
||||||
// dead-ending the agent (issue #20.1). attachTab re-mints the identical
|
throw new Error(`unknown sessionId ${sessionId} for ${method}`)
|
||||||
// `cb-tab-<tabId>` session, so the daemon's binding stays valid.
|
}
|
||||||
tabId = await recoverSessionTab(sessionId)
|
if (!tabs.has(tabId)) {
|
||||||
if (!tabId) {
|
const recovered = await recoverSessionTab(sessionId)
|
||||||
|
if (!recovered) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`stale sessionId ${sessionId} for ${method}: its tab is gone (closed, ` +
|
`stale sessionId ${sessionId} for ${method}: its tab is gone (closed, ` +
|
||||||
`navigated across processes, or lost after an extension restart). ` +
|
`navigated across processes, or lost after an extension restart). ` +
|
||||||
`Re-attach by re-opening your target URL before retrying.`,
|
`Re-attach by re-opening your target URL before retrying.`,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
tabId = recovered
|
||||||
}
|
}
|
||||||
} else if (typeof params?.targetId === 'string') {
|
} else if (typeof params?.targetId === 'string') {
|
||||||
tabId = tabForTarget(params.targetId)
|
tabId = tabForTarget(params.targetId)
|
||||||
@@ -253,18 +294,17 @@ async function handleForwardCdpCommand(msg) {
|
|||||||
// applies to any attached tab.
|
// applies to any attached tab.
|
||||||
tabId = anyConnectedTab()
|
tabId = anyConnectedTab()
|
||||||
}
|
}
|
||||||
if (!tabId) throw new Error(`no attached tab for ${method}`)
|
if (tabId == null) throw new Error(`no attached tab for ${method}`)
|
||||||
const dbg = { tabId }
|
|
||||||
|
|
||||||
// Re-enabling Runtime can leave a stale state; bounce it (matches upstream).
|
// Re-enabling Runtime can leave a stale state; bounce it (matches upstream).
|
||||||
if (method === 'Runtime.enable') {
|
if (method === 'Runtime.enable') {
|
||||||
try {
|
try {
|
||||||
await chrome.debugger.sendCommand(dbg, 'Runtime.disable')
|
await sendCdpToTab(tabId, 'Runtime.disable', undefined)
|
||||||
await new Promise((r) => setTimeout(r, 30))
|
await new Promise((r) => setTimeout(r, 30))
|
||||||
} catch {}
|
} catch {}
|
||||||
return await chrome.debugger.sendCommand(dbg, 'Runtime.enable', params)
|
return await sendCdpToTab(tabId, 'Runtime.enable', params)
|
||||||
}
|
}
|
||||||
return await chrome.debugger.sendCommand(dbg, method, params)
|
return await sendCdpToTab(tabId, method, params)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- attach / detach ------------------------------------------------------
|
// ---- attach / detach ------------------------------------------------------
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest_version": 3,
|
"manifest_version": 3,
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "0.4.6",
|
"version": "0.4.8",
|
||||||
"description": "Let chrome-use drive your logged-in Chrome \u2014 install once, no token, no per-use confirmation.",
|
"description": "Let chrome-use drive your logged-in Chrome \u2014 install once, no token, no per-use confirmation.",
|
||||||
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6vQIyscGIPYPZdSpPwPL0+0gxUROyRgCpmvCSDoc8XUm4qm97VbKnD9Ijc1lV22lNWZtE78gaRjt6BeSfuMgnBymnhLKjN1gU6AI5QUU0mrJyeHdWKvrKQR5FmsM2A7Xr1ykE2SiiS8zNUS3Y/6O5l+Nva7wrVy6E4a2dkBVQkOsu+DV+nEZvhIyuDY5D5SPXqNwUTWTaglwj5mjvHz36xSwCWlPmrtJ+ED0AUyrb2z4GIOmvk4kqtBVrh/UD058klLo4CkYOnIybB5aV6WYuwarfPY4bF/dLggPem+ewLNTUNBuwrxj/A4nUv0LJTuRO8rR7f8WR9qnRCY0Ic5saQIDAQAB",
|
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6vQIyscGIPYPZdSpPwPL0+0gxUROyRgCpmvCSDoc8XUm4qm97VbKnD9Ijc1lV22lNWZtE78gaRjt6BeSfuMgnBymnhLKjN1gU6AI5QUU0mrJyeHdWKvrKQR5FmsM2A7Xr1ykE2SiiS8zNUS3Y/6O5l+Nva7wrVy6E4a2dkBVQkOsu+DV+nEZvhIyuDY5D5SPXqNwUTWTaglwj5mjvHz36xSwCWlPmrtJ+ED0AUyrb2z4GIOmvk4kqtBVrh/UD058klLo4CkYOnIybB5aV6WYuwarfPY4bF/dLggPem+ewLNTUNBuwrxj/A4nUv0LJTuRO8rR7f8WR9qnRCY0Ic5saQIDAQAB",
|
||||||
"icons": {
|
"icons": {
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "chrome-use",
|
"name": "chrome-use",
|
||||||
"version": "1.3.0",
|
"version": "1.4.1",
|
||||||
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
"description": "chrome-use — drive your real, logged-in Chrome from any AI agent, stealth by default",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"packageManager": "pnpm@11.1.3",
|
"packageManager": "pnpm@11.1.3",
|
||||||
|
|||||||
@@ -226,8 +226,11 @@ chrome-use hover @e1 # hover
|
|||||||
chrome-use focus @e1 # focus (useful before keyboard input)
|
chrome-use focus @e1 # focus (useful before keyboard input)
|
||||||
chrome-use fill @e2 "hello" # clear then type
|
chrome-use fill @e2 "hello" # clear then type
|
||||||
chrome-use type @e2 " world" # type without clearing
|
chrome-use type @e2 " world" # type without clearing
|
||||||
chrome-use press Enter # press a key at current focus
|
chrome-use press Enter # press a key at current focus (down+up)
|
||||||
chrome-use press Control+a # key combination
|
chrome-use press Control+a # key combination
|
||||||
|
chrome-use keydown d # HOLD a key down (no auto-release)
|
||||||
|
chrome-use keyup d # release it — pair them to hold-to-move
|
||||||
|
# in a game: `keydown d; sleep; keyup d`
|
||||||
chrome-use check @e3 # check checkbox
|
chrome-use check @e3 # check checkbox
|
||||||
chrome-use uncheck @e3 # uncheck
|
chrome-use uncheck @e3 # uncheck
|
||||||
chrome-use select @e4 "option-value" # native <select> only
|
chrome-use select @e4 "option-value" # native <select> only
|
||||||
@@ -296,6 +299,23 @@ chrome-use click --coords 449,320 # same, explicit flag
|
|||||||
|
|
||||||
A bare-number argument is always a coordinate, never a selector.
|
A bare-number argument is always a coordinate, never a selector.
|
||||||
|
|
||||||
|
### Canvas / WebGL apps (games, map & 3D viewers, drawing tools)
|
||||||
|
|
||||||
|
These paint everything to a `<canvas>` and expose **almost no accessibility
|
||||||
|
tree**, so `snapshot` comes back near-empty and refs are a dead end. `snapshot`
|
||||||
|
detects this and prints a one-line hint. Drive them the screenshot way:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
||||||
|
# eval/get text return nothing useful)
|
||||||
|
chrome-use click 640 360 # interact by viewport coordinate
|
||||||
|
chrome-use keydown d; sleep 0.6; chrome-use keyup d # hold-to-move
|
||||||
|
chrome-use press Space # discrete actions (jump/attack/confirm)
|
||||||
|
```
|
||||||
|
|
||||||
|
Each command is a ~250ms round-trip, so this is fine for turn-based / canvas
|
||||||
|
*apps* but too slow to play a real-time 60fps action game frame-by-frame.
|
||||||
|
|
||||||
## Waiting (read this)
|
## Waiting (read this)
|
||||||
|
|
||||||
Agents fail more often from bad waits than from bad selectors. Pick the
|
Agents fail more often from bad waits than from bad selectors. Pick the
|
||||||
|
|||||||
Reference in New Issue
Block a user