Compare commits
28
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3142858f30 | ||
|
|
a8dcbb1222 | ||
|
|
a9fcef4579 | ||
|
|
59baf97e51 | ||
|
|
d02ef66c89 | ||
|
|
1689cf9eca | ||
|
|
03a53c9f36 | ||
|
|
b1c0c6a366 | ||
|
|
c88734da89 | ||
|
|
28740acecf | ||
|
|
4112234371 | ||
|
|
5e08e5d077 | ||
|
|
42879c337a | ||
|
|
412ac63b68 | ||
|
|
e6e832d2bc | ||
|
|
b19ca760aa | ||
|
|
e7c4936bc7 | ||
|
|
7aad47d3bd | ||
|
|
e196ed3e35 | ||
|
|
1f31452fea | ||
|
|
3675e6bd7a | ||
|
|
fff9a146bd | ||
|
|
34dcb7195a | ||
|
|
7bdfcf8541 | ||
|
|
6abee37641 | ||
|
|
7b43d408da | ||
|
|
2dc093cd62 | ||
|
|
673e2e266e |
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "agent-browser",
|
||||
"description": "Headless browser automation for AI agents",
|
||||
"owner": {
|
||||
"name": "Vercel",
|
||||
"email": "support@vercel.com"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "agent-browser",
|
||||
"description": "Automates browser interactions for web testing, form filling, screenshots, and data extraction",
|
||||
"source": "./",
|
||||
"strict": false,
|
||||
"skills": ["./skills/agent-browser"],
|
||||
"category": "development"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -5,6 +5,7 @@ Instructions for AI coding agents working with this codebase.
|
||||
## Code Style
|
||||
|
||||
- Do not use emojis in code, output, or documentation. Unicode symbols (✓, ✗, →, ⚠) are acceptable.
|
||||
- CLI colored output uses `cli/src/color.rs`. This module respects the `NO_COLOR` environment variable. Never use hardcoded ANSI color codes.
|
||||
|
||||
<!-- opensrc:start -->
|
||||
|
||||
|
||||
@@ -74,10 +74,11 @@ agent-browser scroll <dir> [px] # Scroll (up/down/left/right)
|
||||
agent-browser scrollintoview <sel> # Scroll element into view (alias: scrollinto)
|
||||
agent-browser drag <src> <tgt> # Drag and drop
|
||||
agent-browser upload <sel> <files> # Upload files
|
||||
agent-browser screenshot [path] # Take screenshot (--full for full page)
|
||||
agent-browser screenshot [path] # Take screenshot (--full for full page, base64 png to stdout if no path)
|
||||
agent-browser pdf <path> # Save as PDF
|
||||
agent-browser snapshot # Accessibility tree with refs (best for AI)
|
||||
agent-browser eval <js> # Run JavaScript
|
||||
agent-browser connect <port> # Connect to browser via CDP
|
||||
agent-browser close # Close browser (aliases: quit, exit)
|
||||
```
|
||||
|
||||
@@ -465,12 +466,16 @@ export async function handler() {
|
||||
Connect to an existing browser via Chrome DevTools Protocol:
|
||||
|
||||
```bash
|
||||
# Connect to Electron app
|
||||
agent-browser --cdp 9222 snapshot
|
||||
# Start Chrome with: google-chrome --remote-debugging-port=9222
|
||||
|
||||
# Connect to Chrome with remote debugging
|
||||
# (Start Chrome with: google-chrome --remote-debugging-port=9222)
|
||||
agent-browser --cdp 9222 open about:blank
|
||||
# Connect once, then run commands without --cdp
|
||||
agent-browser connect 9222
|
||||
agent-browser snapshot
|
||||
agent-browser tab
|
||||
agent-browser close
|
||||
|
||||
# Or pass --cdp on each command
|
||||
agent-browser --cdp 9222 snapshot
|
||||
```
|
||||
|
||||
This enables control of:
|
||||
@@ -479,6 +484,113 @@ This enables control of:
|
||||
- WebView2 applications
|
||||
- Any browser exposing a CDP endpoint
|
||||
|
||||
## Streaming (Browser Preview)
|
||||
|
||||
Stream the browser viewport via WebSocket for live preview or "pair browsing" where a human can watch and interact alongside an AI agent.
|
||||
|
||||
### Enable Streaming
|
||||
|
||||
Set the `AGENT_BROWSER_STREAM_PORT` environment variable:
|
||||
|
||||
```bash
|
||||
AGENT_BROWSER_STREAM_PORT=9223 agent-browser open example.com
|
||||
```
|
||||
|
||||
This starts a WebSocket server on the specified port that streams the browser viewport and accepts input events.
|
||||
|
||||
### WebSocket Protocol
|
||||
|
||||
Connect to `ws://localhost:9223` to receive frames and send input:
|
||||
|
||||
**Receive frames:**
|
||||
```json
|
||||
{
|
||||
"type": "frame",
|
||||
"data": "<base64-encoded-jpeg>",
|
||||
"metadata": {
|
||||
"deviceWidth": 1280,
|
||||
"deviceHeight": 720,
|
||||
"pageScaleFactor": 1,
|
||||
"offsetTop": 0,
|
||||
"scrollOffsetX": 0,
|
||||
"scrollOffsetY": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Send mouse events:**
|
||||
```json
|
||||
{
|
||||
"type": "input_mouse",
|
||||
"eventType": "mousePressed",
|
||||
"x": 100,
|
||||
"y": 200,
|
||||
"button": "left",
|
||||
"clickCount": 1
|
||||
}
|
||||
```
|
||||
|
||||
**Send keyboard events:**
|
||||
```json
|
||||
{
|
||||
"type": "input_keyboard",
|
||||
"eventType": "keyDown",
|
||||
"key": "Enter",
|
||||
"code": "Enter"
|
||||
}
|
||||
```
|
||||
|
||||
**Send touch events:**
|
||||
```json
|
||||
{
|
||||
"type": "input_touch",
|
||||
"eventType": "touchStart",
|
||||
"touchPoints": [{ "x": 100, "y": 200 }]
|
||||
}
|
||||
```
|
||||
|
||||
### Programmatic API
|
||||
|
||||
For advanced use, control streaming directly via the protocol:
|
||||
|
||||
```typescript
|
||||
import { BrowserManager } from 'agent-browser';
|
||||
|
||||
const browser = new BrowserManager();
|
||||
await browser.launch({ headless: true });
|
||||
await browser.navigate('https://example.com');
|
||||
|
||||
// Start screencast
|
||||
await browser.startScreencast((frame) => {
|
||||
// frame.data is base64-encoded image
|
||||
// frame.metadata contains viewport info
|
||||
console.log('Frame received:', frame.metadata.deviceWidth, 'x', frame.metadata.deviceHeight);
|
||||
}, {
|
||||
format: 'jpeg',
|
||||
quality: 80,
|
||||
maxWidth: 1280,
|
||||
maxHeight: 720,
|
||||
});
|
||||
|
||||
// Inject mouse events
|
||||
await browser.injectMouseEvent({
|
||||
type: 'mousePressed',
|
||||
x: 100,
|
||||
y: 200,
|
||||
button: 'left',
|
||||
});
|
||||
|
||||
// Inject keyboard events
|
||||
await browser.injectKeyboardEvent({
|
||||
type: 'keyDown',
|
||||
key: 'Enter',
|
||||
code: 'Enter',
|
||||
});
|
||||
|
||||
// Stop when done
|
||||
await browser.stopScreencast();
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
agent-browser uses a client-daemon architecture:
|
||||
|
||||
Generated
+1
-1
@@ -4,7 +4,7 @@ version = 4
|
||||
|
||||
[[package]]
|
||||
name = "agent-browser"
|
||||
version = "0.4.4"
|
||||
version = "0.5.0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"serde",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "agent-browser"
|
||||
version = "0.4.4"
|
||||
version = "0.5.0"
|
||||
edition = "2021"
|
||||
description = "Fast browser automation CLI for AI agents"
|
||||
license = "Apache-2.0"
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
//! Color output utilities respecting NO_COLOR environment variable.
|
||||
//!
|
||||
//! When the NO_COLOR environment variable is present (regardless of value),
|
||||
//! all color formatting is disabled per https://no-color.org/
|
||||
|
||||
use std::env;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// Returns true if color output is enabled (NO_COLOR is NOT set)
|
||||
pub fn is_enabled() -> bool {
|
||||
static COLORS_ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*COLORS_ENABLED.get_or_init(|| env::var("NO_COLOR").is_err())
|
||||
}
|
||||
|
||||
/// Format text in red (errors)
|
||||
pub fn red(text: &str) -> String {
|
||||
if is_enabled() {
|
||||
format!("\x1b[31m{}\x1b[0m", text)
|
||||
} else {
|
||||
text.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Format text in green (success)
|
||||
pub fn green(text: &str) -> String {
|
||||
if is_enabled() {
|
||||
format!("\x1b[32m{}\x1b[0m", text)
|
||||
} else {
|
||||
text.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Format text in yellow (warnings)
|
||||
pub fn yellow(text: &str) -> String {
|
||||
if is_enabled() {
|
||||
format!("\x1b[33m{}\x1b[0m", text)
|
||||
} else {
|
||||
text.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Format text in cyan (info/progress)
|
||||
pub fn cyan(text: &str) -> String {
|
||||
if is_enabled() {
|
||||
format!("\x1b[36m{}\x1b[0m", text)
|
||||
} else {
|
||||
text.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Format text in bold
|
||||
pub fn bold(text: &str) -> String {
|
||||
if is_enabled() {
|
||||
format!("\x1b[1m{}\x1b[0m", text)
|
||||
} else {
|
||||
text.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Format text in dim
|
||||
pub fn dim(text: &str) -> String {
|
||||
if is_enabled() {
|
||||
format!("\x1b[2m{}\x1b[0m", text)
|
||||
} else {
|
||||
text.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Red X error indicator
|
||||
pub fn error_indicator() -> &'static str {
|
||||
static INDICATOR: OnceLock<String> = OnceLock::new();
|
||||
INDICATOR.get_or_init(|| {
|
||||
if is_enabled() {
|
||||
"\x1b[31m✗\x1b[0m".to_string()
|
||||
} else {
|
||||
"✗".to_string()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Green checkmark success indicator
|
||||
pub fn success_indicator() -> &'static str {
|
||||
static INDICATOR: OnceLock<String> = OnceLock::new();
|
||||
INDICATOR.get_or_init(|| {
|
||||
if is_enabled() {
|
||||
"\x1b[32m✓\x1b[0m".to_string()
|
||||
} else {
|
||||
"✓".to_string()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Yellow warning indicator
|
||||
pub fn warning_indicator() -> &'static str {
|
||||
static INDICATOR: OnceLock<String> = OnceLock::new();
|
||||
INDICATOR.get_or_init(|| {
|
||||
if is_enabled() {
|
||||
"\x1b[33m⚠\x1b[0m".to_string()
|
||||
} else {
|
||||
"⚠".to_string()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Get console log color prefix by level
|
||||
pub fn console_level_prefix(level: &str) -> String {
|
||||
if !is_enabled() {
|
||||
return format!("[{}]", level);
|
||||
}
|
||||
|
||||
let color = match level {
|
||||
"error" => "\x1b[31m",
|
||||
"warning" => "\x1b[33m",
|
||||
"info" => "\x1b[36m",
|
||||
_ => "",
|
||||
};
|
||||
if color.is_empty() {
|
||||
format!("[{}]", level)
|
||||
} else {
|
||||
format!("{}[{}]\x1b[0m", color, level)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_red_contains_ansi_codes() {
|
||||
// Test the format structure (actual color depends on NO_COLOR env)
|
||||
let formatted = format!("\x1b[31m{}\x1b[0m", "error");
|
||||
assert!(formatted.contains("\x1b[31m"));
|
||||
assert!(formatted.contains("\x1b[0m"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_green_contains_ansi_codes() {
|
||||
let formatted = format!("\x1b[32m{}\x1b[0m", "success");
|
||||
assert!(formatted.contains("\x1b[32m"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_console_level_prefix_contains_level() {
|
||||
// Regardless of color state, the level text should be present
|
||||
assert!(console_level_prefix("error").contains("error"));
|
||||
assert!(console_level_prefix("warning").contains("warning"));
|
||||
assert!(console_level_prefix("info").contains("info"));
|
||||
assert!(console_level_prefix("log").contains("log"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_indicators_contain_symbols() {
|
||||
// Regardless of color state, symbols should be present
|
||||
assert!(error_indicator().contains('✗'));
|
||||
assert!(success_indicator().contains('✓'));
|
||||
assert!(warning_indicator().contains('⚠'));
|
||||
}
|
||||
}
|
||||
+344
-26
@@ -75,7 +75,12 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
context: cmd.to_string(),
|
||||
usage: "open <url>",
|
||||
})?;
|
||||
let url = if url.starts_with("http") {
|
||||
let url_lower = url.to_lowercase();
|
||||
let url = if url_lower.starts_with("http://")
|
||||
|| url_lower.starts_with("https://")
|
||||
|| url_lower.starts_with("about:")
|
||||
|| url_lower.starts_with("data:")
|
||||
|| url_lower.starts_with("file:") {
|
||||
url.to_string()
|
||||
} else {
|
||||
format!("https://{}", url)
|
||||
@@ -153,13 +158,18 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
"select" => {
|
||||
let sel = rest.get(0).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "select".to_string(),
|
||||
usage: "select <selector> <value>",
|
||||
usage: "select <selector> <value...>",
|
||||
})?;
|
||||
let val = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
let _val = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "select".to_string(),
|
||||
usage: "select <selector> <value>",
|
||||
usage: "select <selector> <value...>",
|
||||
})?;
|
||||
Ok(json!({ "id": id, "action": "select", "selector": sel, "value": val }))
|
||||
let values = &rest[1..];
|
||||
if values.len() == 1 {
|
||||
Ok(json!({ "id": id, "action": "select", "selector": sel, "values": values[0] }))
|
||||
} else {
|
||||
Ok(json!({ "id": id, "action": "select", "selector": sel, "values": values }))
|
||||
}
|
||||
}
|
||||
"drag" => {
|
||||
let src = rest.get(0).ok_or_else(|| ParseError::MissingArguments {
|
||||
@@ -273,7 +283,11 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
|
||||
// === Screenshot/PDF ===
|
||||
"screenshot" => {
|
||||
Ok(json!({ "id": id, "action": "screenshot", "path": rest.get(0), "fullPage": flags.full }))
|
||||
let mut cmd = json!({ "id": id, "action": "screenshot", "fullPage": flags.full });
|
||||
if let Some(path) = rest.get(0) {
|
||||
cmd["path"] = json!(path);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
"pdf" => {
|
||||
let path = rest.get(0).ok_or_else(|| ParseError::MissingArguments {
|
||||
@@ -323,6 +337,19 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
// === Close ===
|
||||
"close" | "quit" | "exit" => Ok(json!({ "id": id, "action": "close" })),
|
||||
|
||||
// === Connect (CDP) ===
|
||||
"connect" => {
|
||||
let port_str = rest.get(0).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "connect".to_string(),
|
||||
usage: "connect <port>",
|
||||
})?;
|
||||
let port: u16 = port_str.parse().map_err(|_| ParseError::MissingArguments {
|
||||
context: format!("connect: invalid port '{}'", port_str),
|
||||
usage: "connect <port>",
|
||||
})?;
|
||||
Ok(json!({ "id": id, "action": "launch", "cdpPort": port }))
|
||||
}
|
||||
|
||||
// === Get ===
|
||||
"get" => parse_get(&rest, &id),
|
||||
|
||||
@@ -367,10 +394,20 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
// === Tabs ===
|
||||
"tab" => {
|
||||
match rest.get(0).map(|s| *s) {
|
||||
Some("new") => Ok(json!({ "id": id, "action": "tab_new", "url": rest.get(1) })),
|
||||
Some("new") => {
|
||||
let mut cmd = json!({ "id": id, "action": "tab_new" });
|
||||
if let Some(url) = rest.get(1) {
|
||||
cmd["url"] = json!(url);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
Some("list") => Ok(json!({ "id": id, "action": "tab_list" })),
|
||||
Some("close") => {
|
||||
Ok(json!({ "id": id, "action": "tab_close", "index": rest.get(1).and_then(|s| s.parse::<i32>().ok()) }))
|
||||
let mut cmd = json!({ "id": id, "action": "tab_close" });
|
||||
if let Some(index) = rest.get(1).and_then(|s| s.parse::<i32>().ok()) {
|
||||
cmd["index"] = json!(index);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
Some(n) if n.parse::<i32>().is_ok() => {
|
||||
Ok(json!({ "id": id, "action": "tab_switch", "index": n.parse::<i32>().unwrap() }))
|
||||
@@ -398,7 +435,7 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
// === Frame ===
|
||||
"frame" => {
|
||||
if rest.get(0).map(|s| *s) == Some("main") {
|
||||
Ok(json!({ "id": id, "action": "frame_main" }))
|
||||
Ok(json!({ "id": id, "action": "mainframe" }))
|
||||
} else {
|
||||
let sel = rest.get(0).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "frame".to_string(),
|
||||
@@ -413,7 +450,11 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
const VALID: &[&str] = &["accept", "dismiss"];
|
||||
match rest.get(0).map(|s| *s) {
|
||||
Some("accept") => {
|
||||
Ok(json!({ "id": id, "action": "dialog", "response": "accept", "promptText": rest.get(1) }))
|
||||
let mut cmd = json!({ "id": id, "action": "dialog", "response": "accept" });
|
||||
if let Some(prompt_text) = rest.get(1) {
|
||||
cmd["promptText"] = json!(prompt_text);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
Some("dismiss") => Ok(json!({ "id": id, "action": "dialog", "response": "dismiss" })),
|
||||
Some(sub) => Err(ParseError::UnknownSubcommand {
|
||||
@@ -431,8 +472,14 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
"trace" => {
|
||||
const VALID: &[&str] = &["start", "stop"];
|
||||
match rest.get(0).map(|s| *s) {
|
||||
Some("start") => Ok(json!({ "id": id, "action": "trace_start", "path": rest.get(1) })),
|
||||
Some("stop") => Ok(json!({ "id": id, "action": "trace_stop", "path": rest.get(1) })),
|
||||
Some("start") => Ok(json!({ "id": id, "action": "trace_start" })),
|
||||
Some("stop") => {
|
||||
let path = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "trace stop".to_string(),
|
||||
usage: "trace stop <path>",
|
||||
})?;
|
||||
Ok(json!({ "id": id, "action": "trace_stop", "path": path }))
|
||||
},
|
||||
Some(sub) => Err(ParseError::UnknownSubcommand {
|
||||
subcommand: sub.to_string(),
|
||||
valid_options: VALID,
|
||||
@@ -443,6 +490,60 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// === Recording (Playwright native video recording) ===
|
||||
"record" => {
|
||||
const VALID: &[&str] = &["start", "stop", "restart"];
|
||||
match rest.get(0).map(|s| *s) {
|
||||
Some("start") => {
|
||||
let path = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "record start".to_string(),
|
||||
usage: "record start <output.webm> [url]",
|
||||
})?;
|
||||
// Optional URL parameter
|
||||
let url = rest.get(2);
|
||||
let mut cmd = json!({ "id": id, "action": "recording_start", "path": path });
|
||||
if let Some(u) = url {
|
||||
// Add https:// prefix if needed
|
||||
let url_str = if u.starts_with("http") {
|
||||
u.to_string()
|
||||
} else {
|
||||
format!("https://{}", u)
|
||||
};
|
||||
cmd["url"] = json!(url_str);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
Some("stop") => Ok(json!({ "id": id, "action": "recording_stop" })),
|
||||
Some("restart") => {
|
||||
let path = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "record restart".to_string(),
|
||||
usage: "record restart <output.webm> [url]",
|
||||
})?;
|
||||
// Optional URL parameter
|
||||
let url = rest.get(2);
|
||||
let mut cmd = json!({ "id": id, "action": "recording_restart", "path": path });
|
||||
if let Some(u) = url {
|
||||
// Add https:// prefix if needed
|
||||
let url_str = if u.starts_with("http") {
|
||||
u.to_string()
|
||||
} else {
|
||||
format!("https://{}", u)
|
||||
};
|
||||
cmd["url"] = json!(url_str);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
Some(sub) => Err(ParseError::UnknownSubcommand {
|
||||
subcommand: sub.to_string(),
|
||||
valid_options: VALID,
|
||||
}),
|
||||
None => Err(ParseError::MissingArguments {
|
||||
context: "record".to_string(),
|
||||
usage: "record <start|stop|restart> [path] [url]",
|
||||
}),
|
||||
}
|
||||
}
|
||||
"console" => {
|
||||
let clear = rest.iter().any(|&s| s == "--clear");
|
||||
Ok(json!({ "id": id, "action": "console", "clear": clear }))
|
||||
@@ -495,7 +596,7 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
||||
}
|
||||
|
||||
fn parse_get(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
const VALID: &[&str] = &["text", "html", "value", "attr", "url", "title", "count", "box"];
|
||||
const VALID: &[&str] = &["text", "html", "value", "attr", "url", "title", "count", "box", "styles"];
|
||||
|
||||
match rest.get(0).map(|s| *s) {
|
||||
Some("text") => {
|
||||
@@ -546,13 +647,20 @@ fn parse_get(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
})?;
|
||||
Ok(json!({ "id": id, "action": "boundingbox", "selector": sel }))
|
||||
}
|
||||
Some("styles") => {
|
||||
let sel = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
|
||||
context: "get styles".to_string(),
|
||||
usage: "get styles <selector>",
|
||||
})?;
|
||||
Ok(json!({ "id": id, "action": "styles", "selector": sel }))
|
||||
}
|
||||
Some(sub) => Err(ParseError::UnknownSubcommand {
|
||||
subcommand: sub.to_string(),
|
||||
valid_options: VALID,
|
||||
}),
|
||||
None => Err(ParseError::MissingArguments {
|
||||
context: "get".to_string(),
|
||||
usage: "get <text|html|value|attr|url|title|count|box> [args...]",
|
||||
usage: "get <text|html|value|attr|url|title|count|box|styles> [args...]",
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -630,15 +738,39 @@ fn parse_find(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
};
|
||||
|
||||
match *locator {
|
||||
"role" => Ok(json!({ "id": id, "action": "getbyrole", "role": value, "subaction": subaction, "value": fill_value, "name": name, "exact": exact })),
|
||||
"role" => {
|
||||
let mut cmd = json!({ "id": id, "action": "getbyrole", "role": value, "subaction": subaction, "name": name, "exact": exact });
|
||||
if let Some(v) = fill_value { cmd["value"] = json!(v); }
|
||||
Ok(cmd)
|
||||
}
|
||||
"text" => Ok(json!({ "id": id, "action": "getbytext", "text": value, "subaction": subaction, "exact": exact })),
|
||||
"label" => Ok(json!({ "id": id, "action": "getbylabel", "label": value, "subaction": subaction, "value": fill_value, "exact": exact })),
|
||||
"placeholder" => Ok(json!({ "id": id, "action": "getbyplaceholder", "placeholder": value, "subaction": subaction, "value": fill_value, "exact": exact })),
|
||||
"label" => {
|
||||
let mut cmd = json!({ "id": id, "action": "getbylabel", "label": value, "subaction": subaction, "exact": exact });
|
||||
if let Some(v) = fill_value { cmd["value"] = json!(v); }
|
||||
Ok(cmd)
|
||||
}
|
||||
"placeholder" => {
|
||||
let mut cmd = json!({ "id": id, "action": "getbyplaceholder", "placeholder": value, "subaction": subaction, "exact": exact });
|
||||
if let Some(v) = fill_value { cmd["value"] = json!(v); }
|
||||
Ok(cmd)
|
||||
}
|
||||
"alt" => Ok(json!({ "id": id, "action": "getbyalttext", "text": value, "subaction": subaction, "exact": exact })),
|
||||
"title" => Ok(json!({ "id": id, "action": "getbytitle", "text": value, "subaction": subaction, "exact": exact })),
|
||||
"testid" => Ok(json!({ "id": id, "action": "getbytestid", "testId": value, "subaction": subaction, "value": fill_value })),
|
||||
"first" => Ok(json!({ "id": id, "action": "nth", "selector": value, "index": 0, "subaction": subaction, "value": fill_value })),
|
||||
"last" => Ok(json!({ "id": id, "action": "nth", "selector": value, "index": -1, "subaction": subaction, "value": fill_value })),
|
||||
"testid" => {
|
||||
let mut cmd = json!({ "id": id, "action": "getbytestid", "testId": value, "subaction": subaction });
|
||||
if let Some(v) = fill_value { cmd["value"] = json!(v); }
|
||||
Ok(cmd)
|
||||
}
|
||||
"first" => {
|
||||
let mut cmd = json!({ "id": id, "action": "nth", "selector": value, "index": 0, "subaction": subaction });
|
||||
if let Some(v) = fill_value { cmd["value"] = json!(v); }
|
||||
Ok(cmd)
|
||||
}
|
||||
"last" => {
|
||||
let mut cmd = json!({ "id": id, "action": "nth", "selector": value, "index": -1, "subaction": subaction });
|
||||
if let Some(v) = fill_value { cmd["value"] = json!(v); }
|
||||
Ok(cmd)
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
@@ -661,7 +793,9 @@ fn parse_find(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(json!({ "id": id, "action": "nth", "selector": sel, "index": idx, "subaction": sub, "value": fv }))
|
||||
let mut cmd = json!({ "id": id, "action": "nth", "selector": sel, "index": idx, "subaction": sub });
|
||||
if let Some(v) = fv { cmd["value"] = json!(v); }
|
||||
Ok(cmd)
|
||||
}
|
||||
_ => Err(ParseError::UnknownSubcommand {
|
||||
subcommand: locator.to_string(),
|
||||
@@ -702,7 +836,7 @@ fn parse_mouse(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
Some("wheel") => {
|
||||
let dy = rest.get(1).and_then(|s| s.parse::<i32>().ok()).unwrap_or(100);
|
||||
let dx = rest.get(2).and_then(|s| s.parse::<i32>().ok()).unwrap_or(0);
|
||||
Ok(json!({ "id": id, "action": "mousewheel", "deltaX": dx, "deltaY": dy }))
|
||||
Ok(json!({ "id": id, "action": "wheel", "deltaX": dx, "deltaY": dy }))
|
||||
}
|
||||
Some(sub) => Err(ParseError::UnknownSubcommand {
|
||||
subcommand: sub.to_string(),
|
||||
@@ -800,8 +934,12 @@ fn parse_set(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
} else {
|
||||
"no-preference"
|
||||
};
|
||||
let reduced = rest.iter().any(|&s| s == "reduced-motion");
|
||||
Ok(json!({ "id": id, "action": "media", "colorScheme": color, "reducedMotion": reduced }))
|
||||
let reduced = if rest.iter().any(|&s| s == "reduced-motion") {
|
||||
"reduce"
|
||||
} else {
|
||||
"no-preference"
|
||||
};
|
||||
Ok(json!({ "id": id, "action": "emulatemedia", "colorScheme": color, "reducedMotion": reduced }))
|
||||
}
|
||||
Some(sub) => Err(ParseError::UnknownSubcommand {
|
||||
subcommand: sub.to_string(),
|
||||
@@ -828,12 +966,22 @@ fn parse_network(rest: &[&str], id: &str) -> Result<Value, ParseError> {
|
||||
let body = body_idx.and_then(|i| rest.get(i + 1).map(|s| *s));
|
||||
Ok(json!({ "id": id, "action": "route", "url": url, "abort": abort, "body": body }))
|
||||
}
|
||||
Some("unroute") => Ok(json!({ "id": id, "action": "unroute", "url": rest.get(1) })),
|
||||
Some("unroute") => {
|
||||
let mut cmd = json!({ "id": id, "action": "unroute" });
|
||||
if let Some(url) = rest.get(1) {
|
||||
cmd["url"] = json!(url);
|
||||
}
|
||||
Ok(cmd)
|
||||
},
|
||||
Some("requests") => {
|
||||
let clear = rest.iter().any(|&s| s == "--clear");
|
||||
let filter_idx = rest.iter().position(|&s| s == "--filter");
|
||||
let filter = filter_idx.and_then(|i| rest.get(i + 1).map(|s| *s));
|
||||
Ok(json!({ "id": id, "action": "requests", "clear": clear, "filter": filter }))
|
||||
let mut cmd = json!({ "id": id, "action": "requests", "clear": clear });
|
||||
if let Some(f) = filter {
|
||||
cmd["filter"] = json!(f);
|
||||
}
|
||||
Ok(cmd)
|
||||
}
|
||||
Some(sub) => Err(ParseError::UnknownSubcommand {
|
||||
subcommand: sub.to_string(),
|
||||
@@ -901,7 +1049,9 @@ mod tests {
|
||||
debug: false,
|
||||
headers: None,
|
||||
executable_path: None,
|
||||
extensions: Vec::new(),
|
||||
cdp: None,
|
||||
proxy: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1146,12 +1296,46 @@ mod tests {
|
||||
assert_eq!(cmd["text"], "some text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_select() {
|
||||
let cmd = parse_command(&args("select #menu option1"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "select");
|
||||
assert_eq!(cmd["selector"], "#menu");
|
||||
assert_eq!(cmd["values"], "option1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_select_multiple_values() {
|
||||
let cmd = parse_command(
|
||||
&args("select #menu opt1 opt2 opt3"),
|
||||
&default_flags(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(cmd["action"], "select");
|
||||
assert_eq!(cmd["selector"], "#menu");
|
||||
assert_eq!(cmd["values"], json!(["opt1", "opt2", "opt3"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_frame_main() {
|
||||
let cmd = parse_command(&args("frame main"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "mainframe");
|
||||
}
|
||||
|
||||
// === Tabs ===
|
||||
|
||||
#[test]
|
||||
fn test_tab_new() {
|
||||
let cmd = parse_command(&args("tab new"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "tab_new");
|
||||
assert!(cmd.get("url").is_none(), "url should not be present when not provided");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tab_new_with_url() {
|
||||
let cmd = parse_command(&args("tab new https://example.com"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "tab_new");
|
||||
assert_eq!(cmd["url"], "https://example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1179,6 +1363,14 @@ mod tests {
|
||||
fn test_screenshot() {
|
||||
let cmd = parse_command(&args("screenshot"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "screenshot");
|
||||
assert!(cmd.get("path").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_screenshot_path() {
|
||||
let cmd = parse_command(&args("screenshot out.png"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "screenshot");
|
||||
assert_eq!(cmd["path"], "out.png");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1272,6 +1464,82 @@ mod tests {
|
||||
|
||||
// === Unknown command ===
|
||||
|
||||
// === Record Tests ===
|
||||
|
||||
#[test]
|
||||
fn test_record_start() {
|
||||
let cmd = parse_command(&args("record start output.webm"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "recording_start");
|
||||
assert_eq!(cmd["path"], "output.webm");
|
||||
assert!(cmd.get("url").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_start_with_url() {
|
||||
let cmd = parse_command(&args("record start demo.webm https://example.com"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "recording_start");
|
||||
assert_eq!(cmd["path"], "demo.webm");
|
||||
assert_eq!(cmd["url"], "https://example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_start_with_url_no_protocol() {
|
||||
let cmd = parse_command(&args("record start demo.webm example.com"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "recording_start");
|
||||
assert_eq!(cmd["path"], "demo.webm");
|
||||
assert_eq!(cmd["url"], "https://example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_start_missing_path() {
|
||||
let result = parse_command(&args("record start"), &default_flags());
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), ParseError::MissingArguments { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_stop() {
|
||||
let cmd = parse_command(&args("record stop"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "recording_stop");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_restart() {
|
||||
let cmd = parse_command(&args("record restart output.webm"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "recording_restart");
|
||||
assert_eq!(cmd["path"], "output.webm");
|
||||
assert!(cmd.get("url").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_restart_with_url() {
|
||||
let cmd = parse_command(&args("record restart demo.webm https://example.com"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "recording_restart");
|
||||
assert_eq!(cmd["path"], "demo.webm");
|
||||
assert_eq!(cmd["url"], "https://example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_restart_missing_path() {
|
||||
let result = parse_command(&args("record restart"), &default_flags());
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), ParseError::MissingArguments { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_invalid_subcommand() {
|
||||
let result = parse_command(&args("record foo"), &default_flags());
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), ParseError::UnknownSubcommand { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_missing_subcommand() {
|
||||
let result = parse_command(&args("record"), &default_flags());
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), ParseError::MissingArguments { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_command() {
|
||||
let result = parse_command(&args("unknowncommand"), &default_flags());
|
||||
@@ -1315,4 +1583,54 @@ mod tests {
|
||||
assert!(matches!(err, ParseError::MissingArguments { .. }));
|
||||
assert!(err.format().contains("get text"));
|
||||
}
|
||||
|
||||
// === Protocol alignment tests ===
|
||||
|
||||
#[test]
|
||||
fn test_mouse_wheel() {
|
||||
let cmd = parse_command(&args("mouse wheel 100 50"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "wheel");
|
||||
assert_eq!(cmd["deltaY"], 100);
|
||||
assert_eq!(cmd["deltaX"], 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_media() {
|
||||
let cmd = parse_command(&args("set media dark"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "emulatemedia");
|
||||
assert_eq!(cmd["colorScheme"], "dark");
|
||||
assert_eq!(cmd["reducedMotion"], "no-preference");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_media_reduced_motion() {
|
||||
let cmd = parse_command(&args("set media light reduced-motion"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "emulatemedia");
|
||||
assert_eq!(cmd["colorScheme"], "light");
|
||||
assert_eq!(cmd["reducedMotion"], "reduce");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_first_no_value() {
|
||||
let cmd = parse_command(&args("find first a click"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "nth");
|
||||
assert_eq!(cmd["index"], 0);
|
||||
assert!(cmd.get("value").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_first_with_value() {
|
||||
let cmd = parse_command(&args("find first input fill hello"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "nth");
|
||||
assert_eq!(cmd["index"], 0);
|
||||
assert_eq!(cmd["value"], "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_nth_no_value() {
|
||||
let cmd = parse_command(&args("find nth 2 a click"), &default_flags()).unwrap();
|
||||
assert_eq!(cmd["action"], "nth");
|
||||
assert_eq!(cmd["index"], 2);
|
||||
assert!(cmd.get("value").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
+35
-13
@@ -104,7 +104,9 @@ fn get_port_for_session(session: &str) -> u16 {
|
||||
for c in session.chars() {
|
||||
hash = ((hash << 5).wrapping_sub(hash)).wrapping_add(c as i32);
|
||||
}
|
||||
49152 + ((hash.abs() as u16) % 16383)
|
||||
// Correct logic: first take absolute modulo, then cast to u16
|
||||
// Using unsigned_abs() to safely handle i32::MIN
|
||||
49152 + ((hash.unsigned_abs() as u32 % 16383) as u16)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -140,7 +142,8 @@ fn is_daemon_running(session: &str) -> bool {
|
||||
fn daemon_ready(session: &str) -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
get_socket_path(session).exists()
|
||||
let socket_path = get_socket_path(session);
|
||||
UnixStream::connect(&socket_path).is_ok()
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
@@ -159,24 +162,38 @@ pub struct DaemonResult {
|
||||
pub already_running: bool,
|
||||
}
|
||||
|
||||
pub fn ensure_daemon(session: &str, headed: bool, executable_path: Option<&str>) -> Result<DaemonResult, String> {
|
||||
pub fn ensure_daemon(
|
||||
session: &str,
|
||||
headed: bool,
|
||||
executable_path: Option<&str>,
|
||||
extensions: &[String],
|
||||
) -> Result<DaemonResult, String> {
|
||||
if is_daemon_running(session) && daemon_ready(session) {
|
||||
return Ok(DaemonResult { already_running: true });
|
||||
return Ok(DaemonResult {
|
||||
already_running: true,
|
||||
});
|
||||
}
|
||||
|
||||
let exe_path = env::current_exe().map_err(|e| e.to_string())?;
|
||||
let exe_dir = exe_path.parent().unwrap();
|
||||
|
||||
let daemon_paths = [
|
||||
let mut daemon_paths = vec![
|
||||
exe_dir.join("daemon.js"),
|
||||
exe_dir.join("../dist/daemon.js"),
|
||||
PathBuf::from("dist/daemon.js"),
|
||||
];
|
||||
|
||||
// Check AGENT_BROWSER_HOME environment variable
|
||||
if let Ok(home) = env::var("AGENT_BROWSER_HOME") {
|
||||
let home_path = PathBuf::from(&home);
|
||||
daemon_paths.insert(0, home_path.join("dist/daemon.js"));
|
||||
daemon_paths.insert(1, home_path.join("daemon.js"));
|
||||
}
|
||||
|
||||
let daemon_path = daemon_paths
|
||||
.iter()
|
||||
.find(|p| p.exists())
|
||||
.ok_or("Daemon not found. Run from project directory or ensure daemon.js is alongside binary.")?;
|
||||
.ok_or("Daemon not found. Set AGENT_BROWSER_HOME environment variable or run from project directory.")?;
|
||||
|
||||
// Spawn daemon as a fully detached background process
|
||||
#[cfg(unix)]
|
||||
@@ -196,6 +213,10 @@ pub fn ensure_daemon(session: &str, headed: bool, executable_path: Option<&str>)
|
||||
cmd.env("AGENT_BROWSER_EXECUTABLE_PATH", path);
|
||||
}
|
||||
|
||||
if !extensions.is_empty() {
|
||||
cmd.env("AGENT_BROWSER_EXTENSIONS", extensions.join(","));
|
||||
}
|
||||
|
||||
// Create new process group and session to fully detach
|
||||
unsafe {
|
||||
cmd.pre_exec(|| {
|
||||
@@ -216,13 +237,10 @@ pub fn ensure_daemon(session: &str, headed: bool, executable_path: Option<&str>)
|
||||
{
|
||||
use std::os::windows::process::CommandExt;
|
||||
|
||||
// On Windows, use cmd.exe to run node to ensure proper PATH resolution.
|
||||
// This handles cases where node.exe isn't directly in PATH but node.cmd is.
|
||||
// Pass the entire command as a single string to /c to handle paths with spaces.
|
||||
let cmd_string = format!("node \"{}\"", daemon_path.display());
|
||||
let mut cmd = Command::new("cmd");
|
||||
cmd.arg("/c")
|
||||
.arg(&cmd_string)
|
||||
// On Windows, call node directly. Command::new handles PATH resolution (node.exe or node.cmd)
|
||||
// and automatically quotes arguments containing spaces.
|
||||
let mut cmd = Command::new("node");
|
||||
cmd.arg(daemon_path)
|
||||
.env("AGENT_BROWSER_DAEMON", "1")
|
||||
.env("AGENT_BROWSER_SESSION", session);
|
||||
|
||||
@@ -234,6 +252,10 @@ pub fn ensure_daemon(session: &str, headed: bool, executable_path: Option<&str>)
|
||||
cmd.env("AGENT_BROWSER_EXECUTABLE_PATH", path);
|
||||
}
|
||||
|
||||
if !extensions.is_empty() {
|
||||
cmd.env("AGENT_BROWSER_EXTENSIONS", extensions.join(","));
|
||||
}
|
||||
|
||||
// CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
|
||||
const CREATE_NEW_PROCESS_GROUP: u32 = 0x00000200;
|
||||
const DETACHED_PROCESS: u32 = 0x00000008;
|
||||
|
||||
+23
-2
@@ -9,9 +9,16 @@ pub struct Flags {
|
||||
pub headers: Option<String>,
|
||||
pub executable_path: Option<String>,
|
||||
pub cdp: Option<String>,
|
||||
pub extensions: Vec<String>,
|
||||
pub proxy: Option<String>,
|
||||
}
|
||||
|
||||
pub fn parse_flags(args: &[String]) -> Flags {
|
||||
let extensions_env = env::var("AGENT_BROWSER_EXTENSIONS")
|
||||
.ok()
|
||||
.map(|s| s.split(',').map(|p| p.trim().to_string()).filter(|p| !p.is_empty()).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut flags = Flags {
|
||||
json: false,
|
||||
full: false,
|
||||
@@ -21,6 +28,8 @@ pub fn parse_flags(args: &[String]) -> Flags {
|
||||
headers: None,
|
||||
executable_path: env::var("AGENT_BROWSER_EXECUTABLE_PATH").ok(),
|
||||
cdp: None,
|
||||
extensions: extensions_env,
|
||||
proxy: None,
|
||||
};
|
||||
|
||||
let mut i = 0;
|
||||
@@ -47,13 +56,25 @@ pub fn parse_flags(args: &[String]) -> Flags {
|
||||
flags.executable_path = Some(s.clone());
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
},
|
||||
"--extension" => {
|
||||
if let Some(s) = args.get(i + 1) {
|
||||
flags.extensions.push(s.clone());
|
||||
i += 1;
|
||||
}
|
||||
},
|
||||
"--cdp" => {
|
||||
if let Some(s) = args.get(i + 1) {
|
||||
flags.cdp = Some(s.clone());
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
"--proxy" => {
|
||||
if let Some(p) = args.get(i + 1) {
|
||||
flags.proxy = Some(p.clone());
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
i += 1;
|
||||
@@ -68,7 +89,7 @@ pub fn clean_args(args: &[String]) -> Vec<String> {
|
||||
// Global flags that should be stripped from command args
|
||||
const GLOBAL_FLAGS: &[&str] = &["--json", "--full", "--headed", "--debug"];
|
||||
// Global flags that take a value (need to skip the next arg too)
|
||||
const GLOBAL_FLAGS_WITH_VALUE: &[&str] = &["--session", "--headers", "--executable-path", "--cdp"];
|
||||
const GLOBAL_FLAGS_WITH_VALUE: &[&str] = &["--session", "--headers", "--executable-path", "--cdp", "--extension", "--proxy"];
|
||||
|
||||
for arg in args.iter() {
|
||||
if skip_next {
|
||||
|
||||
+32
-13
@@ -1,3 +1,4 @@
|
||||
use crate::color;
|
||||
use std::process::{exit, Command, Stdio};
|
||||
|
||||
pub fn run_install(with_deps: bool) {
|
||||
@@ -5,9 +6,15 @@ pub fn run_install(with_deps: bool) {
|
||||
|
||||
if is_linux {
|
||||
if with_deps {
|
||||
println!("\x1b[36mInstalling system dependencies...\x1b[0m");
|
||||
println!("{}", color::cyan("Installing system dependencies..."));
|
||||
|
||||
let (pkg_mgr, deps) = if which_exists("apt-get") {
|
||||
let libasound = if package_exists_apt("libasound2t64") {
|
||||
"libasound2t64"
|
||||
} else {
|
||||
"libasound2"
|
||||
};
|
||||
|
||||
(
|
||||
"apt-get",
|
||||
vec![
|
||||
@@ -30,7 +37,7 @@ pub fn run_install(with_deps: bool) {
|
||||
"libcairo2",
|
||||
"libgdk-pixbuf-2.0-0",
|
||||
"libxrender1",
|
||||
"libasound2",
|
||||
libasound,
|
||||
"libfreetype6",
|
||||
"libfontconfig1",
|
||||
"libdbus-1-3",
|
||||
@@ -93,7 +100,7 @@ pub fn run_install(with_deps: bool) {
|
||||
],
|
||||
)
|
||||
} else {
|
||||
eprintln!("\x1b[31m✗\x1b[0m No supported package manager found (apt-get, dnf, or yum)");
|
||||
eprintln!("{} No supported package manager found (apt-get, dnf, or yum)", color::error_indicator());
|
||||
exit(1);
|
||||
};
|
||||
|
||||
@@ -112,22 +119,23 @@ pub fn run_install(with_deps: bool) {
|
||||
|
||||
match status {
|
||||
Ok(s) if s.success() => {
|
||||
println!("\x1b[32m✓\x1b[0m System dependencies installed")
|
||||
println!("{} System dependencies installed", color::success_indicator())
|
||||
}
|
||||
Ok(_) => eprintln!(
|
||||
"\x1b[33m⚠\x1b[0m Failed to install some dependencies. You may need to run manually with sudo."
|
||||
"{} Failed to install some dependencies. You may need to run manually with sudo.",
|
||||
color::warning_indicator()
|
||||
),
|
||||
Err(e) => eprintln!("\x1b[33m⚠\x1b[0m Could not run install command: {}", e),
|
||||
Err(e) => eprintln!("{} Could not run install command: {}", color::warning_indicator(), e),
|
||||
}
|
||||
} else {
|
||||
println!("\x1b[33m⚠\x1b[0m Linux detected. If browser fails to launch, run:");
|
||||
println!("{} Linux detected. If browser fails to launch, run:", color::warning_indicator());
|
||||
println!(" agent-browser install --with-deps");
|
||||
println!(" or: npx playwright install-deps chromium");
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
println!("\x1b[36mInstalling Chromium browser...\x1b[0m");
|
||||
println!("{}", color::cyan("Installing Chromium browser..."));
|
||||
|
||||
// On Windows, we need to use cmd.exe to run npx because npx is actually npx.cmd
|
||||
// and Command::new() doesn't resolve .cmd files the way the shell does.
|
||||
@@ -144,23 +152,23 @@ pub fn run_install(with_deps: bool) {
|
||||
|
||||
match status {
|
||||
Ok(s) if s.success() => {
|
||||
println!("\x1b[32m✓\x1b[0m Chromium installed successfully");
|
||||
println!("{} Chromium installed successfully", color::success_indicator());
|
||||
if is_linux && !with_deps {
|
||||
println!();
|
||||
println!("\x1b[33mNote:\x1b[0m If you see \"shared library\" errors when running, use:");
|
||||
println!("{} If you see \"shared library\" errors when running, use:", color::yellow("Note:"));
|
||||
println!(" agent-browser install --with-deps");
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
eprintln!("\x1b[31m✗\x1b[0m Failed to install browser");
|
||||
eprintln!("{} Failed to install browser", color::error_indicator());
|
||||
if is_linux {
|
||||
println!("\x1b[33mTip:\x1b[0m Try installing system dependencies first:");
|
||||
println!("{} Try installing system dependencies first:", color::yellow("Tip:"));
|
||||
println!(" agent-browser install --with-deps");
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("\x1b[31m✗\x1b[0m Failed to run npx: {}", e);
|
||||
eprintln!("{} Failed to run npx: {}", color::error_indicator(), e);
|
||||
eprintln!("Make sure Node.js is installed and npx is in your PATH");
|
||||
exit(1);
|
||||
}
|
||||
@@ -189,3 +197,14 @@ fn which_exists(cmd: &str) -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
fn package_exists_apt(pkg: &str) -> bool {
|
||||
Command::new("apt-cache")
|
||||
.arg("show")
|
||||
.arg(pkg)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
+136
-22
@@ -1,3 +1,4 @@
|
||||
mod color;
|
||||
mod commands;
|
||||
mod connection;
|
||||
mod flags;
|
||||
@@ -21,7 +22,37 @@ use commands::{gen_id, parse_command, ParseError};
|
||||
use connection::{ensure_daemon, send_command};
|
||||
use flags::{clean_args, parse_flags};
|
||||
use install::run_install;
|
||||
use output::{print_command_help, print_help, print_response};
|
||||
use output::{print_command_help, print_help, print_response, print_version};
|
||||
|
||||
fn parse_proxy(proxy_str: &str) -> serde_json::Value {
|
||||
let Some(protocol_end) = proxy_str.find("://") else {
|
||||
return json!({ "server": proxy_str });
|
||||
};
|
||||
let protocol = &proxy_str[..protocol_end + 3];
|
||||
let rest = &proxy_str[protocol_end + 3..];
|
||||
|
||||
let Some(at_pos) = rest.rfind('@') else {
|
||||
return json!({ "server": proxy_str });
|
||||
};
|
||||
|
||||
let creds = &rest[..at_pos];
|
||||
let server_part = &rest[at_pos + 1..];
|
||||
let server = format!("{}{}", protocol, server_part);
|
||||
|
||||
let Some(colon_pos) = creds.find(':') else {
|
||||
return json!({
|
||||
"server": server,
|
||||
"username": creds,
|
||||
"password": ""
|
||||
});
|
||||
};
|
||||
|
||||
json!({
|
||||
"server": server,
|
||||
"username": &creds[..colon_pos],
|
||||
"password": &creds[colon_pos + 1..]
|
||||
})
|
||||
}
|
||||
|
||||
fn run_session(args: &[String], session: &str, json_mode: bool) {
|
||||
let subcommand = args.get(1).map(|s| s.as_str());
|
||||
@@ -77,7 +108,7 @@ fn run_session(args: &[String], session: &str, json_mode: bool) {
|
||||
} else {
|
||||
println!("Active sessions:");
|
||||
for s in &sessions {
|
||||
let marker = if s == session { "→" } else { " " };
|
||||
let marker = if s == session { color::cyan("→") } else { " ".to_string() };
|
||||
println!("{} {}", marker, s);
|
||||
}
|
||||
}
|
||||
@@ -94,16 +125,18 @@ fn run_session(args: &[String], session: &str, json_mode: bool) {
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// Ignore SIGPIPE to prevent panic when piping to head/tail
|
||||
#[cfg(unix)]
|
||||
unsafe {
|
||||
libc::signal(libc::SIGPIPE, libc::SIG_DFL);
|
||||
}
|
||||
|
||||
let args: Vec<String> = env::args().skip(1).collect();
|
||||
let flags = parse_flags(&args);
|
||||
let clean = clean_args(&args);
|
||||
|
||||
let has_help = args.iter().any(|a| a == "--help" || a == "-h");
|
||||
|
||||
if clean.is_empty() {
|
||||
print_help();
|
||||
return;
|
||||
}
|
||||
let has_version = args.iter().any(|a| a == "--version" || a == "-V");
|
||||
|
||||
if has_help {
|
||||
if let Some(cmd) = clean.get(0) {
|
||||
@@ -115,6 +148,16 @@ fn main() {
|
||||
return;
|
||||
}
|
||||
|
||||
if has_version {
|
||||
print_version();
|
||||
return;
|
||||
}
|
||||
|
||||
if clean.is_empty() {
|
||||
print_help();
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle install separately
|
||||
if clean.get(0).map(|s| s.as_str()) == Some("install") {
|
||||
let with_deps = args.iter().any(|a| a == "--with-deps" || a == "-d");
|
||||
@@ -143,28 +186,33 @@ fn main() {
|
||||
error_type
|
||||
);
|
||||
} else {
|
||||
eprintln!("\x1b[31m{}\x1b[0m", e.format());
|
||||
eprintln!("{}", color::red(&e.format()));
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
let daemon_result = match ensure_daemon(&flags.session, flags.headed, flags.executable_path.as_deref()) {
|
||||
let daemon_result = match ensure_daemon(&flags.session, flags.headed, flags.executable_path.as_deref(), &flags.extensions) {
|
||||
Ok(result) => result,
|
||||
Err(e) => {
|
||||
if flags.json {
|
||||
println!(r#"{{"success":false,"error":"{}"}}"#, e);
|
||||
} else {
|
||||
eprintln!("\x1b[31m✗\x1b[0m {}", e);
|
||||
eprintln!("{} {}", color::error_indicator(), e);
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
// Warn if executable_path was specified but daemon was already running
|
||||
if daemon_result.already_running && flags.executable_path.is_some() {
|
||||
if daemon_result.already_running && (flags.executable_path.is_some() || !flags.extensions.is_empty()) {
|
||||
if !flags.json {
|
||||
eprintln!("\x1b[33m⚠\x1b[0m --executable-path ignored: daemon already running. Use 'agent-browser close' first to restart with new path.");
|
||||
if flags.executable_path.is_some() {
|
||||
eprintln!("{} --executable-path ignored: daemon already running. Use 'agent-browser close' first to restart with new path.", color::warning_indicator());
|
||||
}
|
||||
if !flags.extensions.is_empty() {
|
||||
eprintln!("{} --extension ignored: daemon already running. Use 'agent-browser close' first to restart with extensions.", color::warning_indicator());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,7 +224,7 @@ fn main() {
|
||||
if flags.json {
|
||||
println!(r#"{{"success":false,"error":"{}"}}"#, msg);
|
||||
} else {
|
||||
eprintln!("\x1b[31m✗\x1b[0m {}", msg);
|
||||
eprintln!("{} {}", color::error_indicator(), msg);
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
@@ -185,7 +233,7 @@ fn main() {
|
||||
if flags.json {
|
||||
println!(r#"{{"success":false,"error":"{}"}}"#, msg);
|
||||
} else {
|
||||
eprintln!("\x1b[31m✗\x1b[0m {}", msg);
|
||||
eprintln!("{} {}", color::error_indicator(), msg);
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
@@ -195,7 +243,7 @@ fn main() {
|
||||
if flags.json {
|
||||
println!(r#"{{"success":false,"error":"{}"}}"#, msg);
|
||||
} else {
|
||||
eprintln!("\x1b[31m✗\x1b[0m {}", msg);
|
||||
eprintln!("{} {}", color::error_indicator(), msg);
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
@@ -217,23 +265,30 @@ fn main() {
|
||||
if flags.json {
|
||||
println!(r#"{{"success":false,"error":"{}"}}"#, msg);
|
||||
} else {
|
||||
eprintln!("\x1b[31m✗\x1b[0m {}", msg);
|
||||
eprintln!("{} {}", color::error_indicator(), msg);
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Launch headed browser if --headed flag is set (without CDP)
|
||||
if flags.headed && flags.cdp.is_none() {
|
||||
let launch_cmd = json!({
|
||||
// Launch headed browser or proxy if flags are set (without CDP)
|
||||
if (flags.headed || flags.proxy.is_some()) && flags.cdp.is_none() {
|
||||
let mut launch_cmd = json!({
|
||||
"id": gen_id(),
|
||||
"action": "launch",
|
||||
"headless": false
|
||||
"headless": !flags.headed
|
||||
});
|
||||
|
||||
if let Some(ref proxy_str) = flags.proxy {
|
||||
let proxy_obj = parse_proxy(proxy_str);
|
||||
launch_cmd.as_object_mut()
|
||||
.expect("json! macro guarantees object type")
|
||||
.insert("proxy".to_string(), proxy_obj);
|
||||
}
|
||||
|
||||
if let Err(e) = send_command(launch_cmd, &flags.session) {
|
||||
if !flags.json {
|
||||
eprintln!("\x1b[33m⚠\x1b[0m Could not launch headed browser: {}", e);
|
||||
eprintln!("{} Could not configure browser: {}", color::warning_indicator(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -250,9 +305,68 @@ fn main() {
|
||||
if flags.json {
|
||||
println!(r#"{{"success":false,"error":"{}"}}"#, e);
|
||||
} else {
|
||||
eprintln!("\x1b[31m✗\x1b[0m {}", e);
|
||||
eprintln!("{} {}", color::error_indicator(), e);
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_proxy_simple() {
|
||||
let result = parse_proxy("http://proxy.com:8080");
|
||||
assert_eq!(result["server"], "http://proxy.com:8080");
|
||||
assert!(result.get("username").is_none());
|
||||
assert!(result.get("password").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_proxy_with_auth() {
|
||||
let result = parse_proxy("http://user:pass@proxy.com:8080");
|
||||
assert_eq!(result["server"], "http://proxy.com:8080");
|
||||
assert_eq!(result["username"], "user");
|
||||
assert_eq!(result["password"], "pass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_proxy_username_only() {
|
||||
let result = parse_proxy("http://user@proxy.com:8080");
|
||||
assert_eq!(result["server"], "http://proxy.com:8080");
|
||||
assert_eq!(result["username"], "user");
|
||||
assert_eq!(result["password"], "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_proxy_no_protocol() {
|
||||
let result = parse_proxy("proxy.com:8080");
|
||||
assert_eq!(result["server"], "proxy.com:8080");
|
||||
assert!(result.get("username").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_proxy_socks5() {
|
||||
let result = parse_proxy("socks5://proxy.com:1080");
|
||||
assert_eq!(result["server"], "socks5://proxy.com:1080");
|
||||
assert!(result.get("username").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_proxy_socks5_with_auth() {
|
||||
let result = parse_proxy("socks5://admin:secret@proxy.com:1080");
|
||||
assert_eq!(result["server"], "socks5://proxy.com:1080");
|
||||
assert_eq!(result["username"], "admin");
|
||||
assert_eq!(result["password"], "secret");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_proxy_complex_password() {
|
||||
let result = parse_proxy("http://user:p@ss:w0rd@proxy.com:8080");
|
||||
assert_eq!(result["server"], "http://proxy.com:8080");
|
||||
assert_eq!(result["username"], "user");
|
||||
assert_eq!(result["password"], "p@ss:w0rd");
|
||||
}
|
||||
}
|
||||
|
||||
+168
-23
@@ -1,3 +1,4 @@
|
||||
use crate::color;
|
||||
use crate::connection::Response;
|
||||
|
||||
pub fn print_response(resp: &Response, json_mode: bool) {
|
||||
@@ -8,7 +9,8 @@ pub fn print_response(resp: &Response, json_mode: bool) {
|
||||
|
||||
if !resp.success {
|
||||
eprintln!(
|
||||
"\x1b[31m✗\x1b[0m {}",
|
||||
"{} {}",
|
||||
color::error_indicator(),
|
||||
resp.error.as_deref().unwrap_or("Unknown error")
|
||||
);
|
||||
return;
|
||||
@@ -18,8 +20,8 @@ pub fn print_response(resp: &Response, json_mode: bool) {
|
||||
// Navigation response
|
||||
if let Some(url) = data.get("url").and_then(|v| v.as_str()) {
|
||||
if let Some(title) = data.get("title").and_then(|v| v.as_str()) {
|
||||
println!("\x1b[32m✓\x1b[0m \x1b[1m{}\x1b[0m", title);
|
||||
println!("\x1b[2m {}\x1b[0m", url);
|
||||
println!("{} {}", color::success_indicator(), color::bold(title));
|
||||
println!(" {}", color::dim(url));
|
||||
return;
|
||||
}
|
||||
println!("{}", url);
|
||||
@@ -85,23 +87,17 @@ pub fn print_response(resp: &Response, json_mode: bool) {
|
||||
.unwrap_or("Untitled");
|
||||
let url = tab.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let active = tab.get("active").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
let marker = if active { "→" } else { " " };
|
||||
let marker = if active { color::cyan("→") } else { " ".to_string() };
|
||||
println!("{} [{}] {} - {}", marker, i, title, url);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Console logs
|
||||
if let Some(logs) = data.get("logs").and_then(|v| v.as_array()) {
|
||||
if let Some(logs) = data.get("messages").and_then(|v| v.as_array()) {
|
||||
for log in logs {
|
||||
let level = log.get("type").and_then(|v| v.as_str()).unwrap_or("log");
|
||||
let text = log.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let color = match level {
|
||||
"error" => "\x1b[31m",
|
||||
"warning" => "\x1b[33m",
|
||||
"info" => "\x1b[36m",
|
||||
_ => "\x1b[0m",
|
||||
};
|
||||
println!("{}[{}]\x1b[0m {}", color, level, text);
|
||||
println!("{} {}", color::console_level_prefix(level), text);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -109,7 +105,7 @@ pub fn print_response(resp: &Response, json_mode: bool) {
|
||||
if let Some(errors) = data.get("errors").and_then(|v| v.as_array()) {
|
||||
for err in errors {
|
||||
let msg = err.get("message").and_then(|v| v.as_str()).unwrap_or("");
|
||||
println!("\x1b[31m✗\x1b[0m {}", msg);
|
||||
println!("{} {}", color::error_indicator(), msg);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -122,6 +118,27 @@ pub fn print_response(resp: &Response, json_mode: bool) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Network requests
|
||||
if let Some(requests) = data.get("requests").and_then(|v| v.as_array()) {
|
||||
if requests.is_empty() {
|
||||
println!("No requests captured");
|
||||
} else {
|
||||
for req in requests {
|
||||
let method = req.get("method").and_then(|v| v.as_str()).unwrap_or("GET");
|
||||
let url = req.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let resource_type = req.get("resourceType").and_then(|v| v.as_str()).unwrap_or("");
|
||||
println!("{} {} ({})", method, url, resource_type);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Cleared requests
|
||||
if let Some(cleared) = data.get("cleared").and_then(|v| v.as_bool()) {
|
||||
if cleared {
|
||||
println!("\x1b[32m✓\x1b[0m Request log cleared");
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Bounding box
|
||||
if let Some(box_data) = data.get("box") {
|
||||
println!(
|
||||
@@ -130,18 +147,91 @@ pub fn print_response(resp: &Response, json_mode: bool) {
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Closed
|
||||
if data.get("closed").is_some() {
|
||||
println!("\x1b[32m✓\x1b[0m Browser closed");
|
||||
// Element styles
|
||||
if let Some(elements) = data.get("elements").and_then(|v| v.as_array()) {
|
||||
for (i, el) in elements.iter().enumerate() {
|
||||
let tag = el.get("tag").and_then(|v| v.as_str()).unwrap_or("?");
|
||||
let text = el.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
println!("[{}] {} \"{}\"", i, tag, text);
|
||||
|
||||
if let Some(box_data) = el.get("box") {
|
||||
let w = box_data.get("width").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let h = box_data.get("height").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let x = box_data.get("x").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let y = box_data.get("y").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
println!(" box: {}x{} at ({}, {})", w, h, x, y);
|
||||
}
|
||||
|
||||
if let Some(styles) = el.get("styles") {
|
||||
let font_size = styles.get("fontSize").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let font_weight = styles.get("fontWeight").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let font_family = styles.get("fontFamily").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let color = styles.get("color").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let bg = styles.get("backgroundColor").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let radius = styles.get("borderRadius").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
println!(" font: {} {} {}", font_size, font_weight, font_family);
|
||||
println!(" color: {}", color);
|
||||
println!(" background: {}", bg);
|
||||
if radius != "0px" {
|
||||
println!(" border-radius: {}", radius);
|
||||
}
|
||||
}
|
||||
println!();
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Screenshot path
|
||||
// Closed
|
||||
if data.get("closed").is_some() {
|
||||
println!("{} Browser closed", color::success_indicator());
|
||||
return;
|
||||
}
|
||||
// Recording start (has "started" field)
|
||||
if let Some(started) = data.get("started").and_then(|v| v.as_bool()) {
|
||||
if started {
|
||||
if let Some(path) = data.get("path").and_then(|v| v.as_str()) {
|
||||
println!("{} Recording started: {}", color::success_indicator(), path);
|
||||
} else {
|
||||
println!("{} Recording started", color::success_indicator());
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Recording restart (has "stopped" field - from recording_restart action)
|
||||
if data.get("stopped").is_some() {
|
||||
let path = data.get("path").and_then(|v| v.as_str()).unwrap_or("unknown");
|
||||
if let Some(prev_path) = data.get("previousPath").and_then(|v| v.as_str()) {
|
||||
println!("{} Recording restarted: {} (previous saved to {})", color::success_indicator(), path, prev_path);
|
||||
} else {
|
||||
println!("{} Recording started: {}", color::success_indicator(), path);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Recording stop (has "frames" field - from recording_stop action)
|
||||
if data.get("frames").is_some() {
|
||||
if let Some(path) = data.get("path").and_then(|v| v.as_str()) {
|
||||
if let Some(error) = data.get("error").and_then(|v| v.as_str()) {
|
||||
println!("{} Recording saved to {} - {}", color::warning_indicator(), path, error);
|
||||
} else {
|
||||
println!("{} Recording saved to {}", color::success_indicator(), path);
|
||||
}
|
||||
} else {
|
||||
println!("{} Recording stopped", color::success_indicator());
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Screenshot path (no "started" or "frames" field)
|
||||
if let Some(path) = data.get("path").and_then(|v| v.as_str()) {
|
||||
println!("\x1b[32m✓\x1b[0m Screenshot saved to {}", path);
|
||||
println!("{} Screenshot saved to {}", color::success_indicator(), color::green(path));
|
||||
return;
|
||||
}
|
||||
// Screenshot base64
|
||||
if let Some(base64) = data.get("base64").and_then(|v| v.as_str()) {
|
||||
println!("{}", base64);
|
||||
return;
|
||||
}
|
||||
// Default success
|
||||
println!("\x1b[32m✓\x1b[0m Done");
|
||||
println!("{} Done", color::success_indicator());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,9 +440,9 @@ Examples:
|
||||
"select" => r##"
|
||||
agent-browser select - Select a dropdown option
|
||||
|
||||
Usage: agent-browser select <selector> <value>
|
||||
Usage: agent-browser select <selector> <value...>
|
||||
|
||||
Selects an option in a <select> dropdown by its value attribute.
|
||||
Selects one or more options in a <select> dropdown by value.
|
||||
|
||||
Global Options:
|
||||
--json Output as JSON
|
||||
@@ -361,6 +451,7 @@ Global Options:
|
||||
Examples:
|
||||
agent-browser select "#country" "US"
|
||||
agent-browser select @e5 "option2"
|
||||
agent-browser select "#menu" "opt1" "opt2" "opt3"
|
||||
"##,
|
||||
"drag" => r##"
|
||||
agent-browser drag - Drag and drop
|
||||
@@ -642,6 +733,7 @@ Subcommands:
|
||||
url Get current URL
|
||||
count <selector> Count matching elements
|
||||
box <selector> Get bounding box (x, y, width, height)
|
||||
styles <selector> Get computed styles of elements
|
||||
|
||||
Global Options:
|
||||
--json Output as JSON
|
||||
@@ -656,6 +748,8 @@ Examples:
|
||||
agent-browser get url
|
||||
agent-browser get count "li.item"
|
||||
agent-browser get box "#header"
|
||||
agent-browser get styles "button"
|
||||
agent-browser get styles @e1
|
||||
"##,
|
||||
|
||||
// === Is ===
|
||||
@@ -979,6 +1073,42 @@ Examples:
|
||||
agent-browser trace stop ./debug-trace.zip
|
||||
"##,
|
||||
|
||||
// === Record (video) ===
|
||||
"record" => r##"
|
||||
agent-browser record - Record browser session to video
|
||||
|
||||
Usage: agent-browser record start <path.webm> [url]
|
||||
agent-browser record stop
|
||||
agent-browser record restart <path.webm> [url]
|
||||
|
||||
Record the browser to a WebM video file using Playwright's native recording.
|
||||
Creates a fresh browser context but preserves cookies and localStorage.
|
||||
If no URL is provided, automatically navigates to your current page.
|
||||
|
||||
Operations:
|
||||
start <path> [url] Start recording (defaults to current URL if omitted)
|
||||
stop Stop recording and save video
|
||||
restart <path> [url] Stop current recording (if any) and start a new one
|
||||
|
||||
Global Options:
|
||||
--json Output as JSON
|
||||
--session <name> Use specific session
|
||||
|
||||
Examples:
|
||||
# Record from current page (preserves login state)
|
||||
agent-browser open https://app.example.com/dashboard
|
||||
agent-browser snapshot -i # Explore and plan
|
||||
agent-browser record start ./demo.webm
|
||||
agent-browser click @e3 # Execute planned actions
|
||||
agent-browser record stop
|
||||
|
||||
# Or specify a different URL
|
||||
agent-browser record start ./demo.webm https://example.com
|
||||
|
||||
# Restart recording with a new file (stops previous, starts new)
|
||||
agent-browser record restart ./take2.webm
|
||||
"##,
|
||||
|
||||
// === Console/Errors ===
|
||||
"console" => r##"
|
||||
agent-browser console - View console logs
|
||||
@@ -1121,7 +1251,7 @@ Core Commands:
|
||||
focus <sel> Focus element
|
||||
check <sel> Check checkbox
|
||||
uncheck <sel> Uncheck checkbox
|
||||
select <sel> <val> Select dropdown option
|
||||
select <sel> <val...> Select dropdown option
|
||||
drag <src> <dst> Drag and drop
|
||||
upload <sel> <files...> Upload files
|
||||
scroll <dir> [px] Scroll (up/down/left/right)
|
||||
@@ -1131,6 +1261,7 @@ Core Commands:
|
||||
pdf <path> Save as PDF
|
||||
snapshot Accessibility tree with refs (for AI)
|
||||
eval <js> Run JavaScript
|
||||
connect <port> Connect to browser via CDP (e.g., connect 9222)
|
||||
close Close browser
|
||||
|
||||
Navigation:
|
||||
@@ -1139,7 +1270,7 @@ Navigation:
|
||||
reload Reload page
|
||||
|
||||
Get Info: agent-browser get <what> [selector]
|
||||
text, html, value, attr <name>, title, url, count, box
|
||||
text, html, value, attr <name>, title, url, count, box, styles
|
||||
|
||||
Check State: agent-browser is <what> <selector>
|
||||
visible, enabled, checked
|
||||
@@ -1169,6 +1300,8 @@ Tabs:
|
||||
|
||||
Debug:
|
||||
trace start|stop [path] Record trace
|
||||
record start <path> [url] Start video recording (WebM)
|
||||
record stop Stop and save video
|
||||
console [--clear] View console logs
|
||||
errors [--clear] View page errors
|
||||
highlight <sel> Highlight element
|
||||
@@ -1191,11 +1324,19 @@ Options:
|
||||
--session <name> Isolated session (or AGENT_BROWSER_SESSION env)
|
||||
--headers <json> HTTP headers scoped to URL's origin (for auth)
|
||||
--executable-path <path> Custom browser executable (or AGENT_BROWSER_EXECUTABLE_PATH)
|
||||
--extension <path> Load browser extensions (repeatable).
|
||||
--proxy <url> Proxy server (http://[user:pass@]host:port)
|
||||
--json JSON output
|
||||
--full, -f Full page screenshot
|
||||
--headed Show browser window (not headless)
|
||||
--cdp <port> Connect via CDP (Chrome DevTools Protocol)
|
||||
--debug Debug output
|
||||
--version, -V Show version
|
||||
|
||||
Environment:
|
||||
AGENT_BROWSER_SESSION Session name (default: "default")
|
||||
AGENT_BROWSER_EXECUTABLE_PATH Custom browser executable path
|
||||
AGENT_BROWSER_STREAM_PORT Enable WebSocket streaming on port (e.g., 9223)
|
||||
|
||||
Examples:
|
||||
agent-browser open example.com
|
||||
@@ -1209,3 +1350,7 @@ Examples:
|
||||
"#
|
||||
);
|
||||
}
|
||||
|
||||
pub fn print_version() {
|
||||
println!("agent-browser {}", env!("CARGO_PKG_VERSION"));
|
||||
}
|
||||
|
||||
@@ -6,12 +6,16 @@ export default function CDPMode() {
|
||||
<div className="prose">
|
||||
<h1>CDP Mode</h1>
|
||||
<p>Connect to an existing browser via Chrome DevTools Protocol:</p>
|
||||
<CodeBlock code={`# Connect to Electron app
|
||||
agent-browser --cdp 9222 snapshot
|
||||
<CodeBlock code={`# Start Chrome with: google-chrome --remote-debugging-port=9222
|
||||
|
||||
# Connect to Chrome with remote debugging
|
||||
# (Start Chrome with: google-chrome --remote-debugging-port=9222)
|
||||
agent-browser --cdp 9222 open about:blank`} />
|
||||
# Connect once, then run commands without --cdp
|
||||
agent-browser connect 9222
|
||||
agent-browser snapshot
|
||||
agent-browser tab
|
||||
agent-browser close
|
||||
|
||||
# Or pass --cdp on each command
|
||||
agent-browser --cdp 9222 snapshot`} />
|
||||
|
||||
<h2>Use cases</h2>
|
||||
<p>This enables control of:</p>
|
||||
|
||||
@@ -12,7 +12,8 @@ agent-browser snapshot # Get accessibility tree with refs
|
||||
agent-browser click @e2 # Click by ref from snapshot
|
||||
agent-browser fill @e3 "test@example.com" # Fill by ref
|
||||
agent-browser get text @e1 # Get text by ref
|
||||
agent-browser screenshot page.png
|
||||
agent-browser screenshot # Base64 png to stdout
|
||||
agent-browser screenshot page.png # Save to file
|
||||
agent-browser close`} />
|
||||
|
||||
<h2>Traditional selectors</h2>
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
import { CodeBlock } from "@/components/code-block";
|
||||
|
||||
export default function Streaming() {
|
||||
return (
|
||||
<div className="max-w-2xl mx-auto px-4 sm:px-6 py-8 sm:py-12">
|
||||
<div className="prose">
|
||||
<h1>Streaming</h1>
|
||||
<p>
|
||||
Stream the browser viewport via WebSocket for live preview or "pair browsing"
|
||||
where a human can watch and interact alongside an AI agent.
|
||||
</p>
|
||||
|
||||
<h2>Enable streaming</h2>
|
||||
<p>
|
||||
Set the <code>AGENT_BROWSER_STREAM_PORT</code> environment variable to start
|
||||
a WebSocket server:
|
||||
</p>
|
||||
<CodeBlock code={`AGENT_BROWSER_STREAM_PORT=9223 agent-browser open example.com`} />
|
||||
|
||||
<p>
|
||||
The server streams viewport frames and accepts input events (mouse, keyboard, touch).
|
||||
</p>
|
||||
|
||||
<h2>WebSocket protocol</h2>
|
||||
<p>Connect to <code>ws://localhost:9223</code> to receive frames and send input.</p>
|
||||
|
||||
<h3>Frame messages</h3>
|
||||
<p>The server sends frame messages with base64-encoded images:</p>
|
||||
<CodeBlock code={`{
|
||||
"type": "frame",
|
||||
"data": "<base64-encoded-jpeg>",
|
||||
"metadata": {
|
||||
"deviceWidth": 1280,
|
||||
"deviceHeight": 720,
|
||||
"pageScaleFactor": 1,
|
||||
"offsetTop": 0,
|
||||
"scrollOffsetX": 0,
|
||||
"scrollOffsetY": 0
|
||||
}
|
||||
}`} />
|
||||
|
||||
<h3>Status messages</h3>
|
||||
<p>Connection and screencast status:</p>
|
||||
<CodeBlock code={`{
|
||||
"type": "status",
|
||||
"connected": true,
|
||||
"screencasting": true,
|
||||
"viewportWidth": 1280,
|
||||
"viewportHeight": 720
|
||||
}`} />
|
||||
|
||||
<h2>Input injection</h2>
|
||||
<p>Send input events to control the browser remotely.</p>
|
||||
|
||||
<h3>Mouse events</h3>
|
||||
<CodeBlock code={`// Click
|
||||
{
|
||||
"type": "input_mouse",
|
||||
"eventType": "mousePressed",
|
||||
"x": 100,
|
||||
"y": 200,
|
||||
"button": "left",
|
||||
"clickCount": 1
|
||||
}
|
||||
|
||||
// Release
|
||||
{
|
||||
"type": "input_mouse",
|
||||
"eventType": "mouseReleased",
|
||||
"x": 100,
|
||||
"y": 200,
|
||||
"button": "left"
|
||||
}
|
||||
|
||||
// Move
|
||||
{
|
||||
"type": "input_mouse",
|
||||
"eventType": "mouseMoved",
|
||||
"x": 150,
|
||||
"y": 250
|
||||
}
|
||||
|
||||
// Scroll
|
||||
{
|
||||
"type": "input_mouse",
|
||||
"eventType": "mouseWheel",
|
||||
"x": 100,
|
||||
"y": 200,
|
||||
"deltaX": 0,
|
||||
"deltaY": 100
|
||||
}`} />
|
||||
|
||||
<h3>Keyboard events</h3>
|
||||
<CodeBlock code={`// Key down
|
||||
{
|
||||
"type": "input_keyboard",
|
||||
"eventType": "keyDown",
|
||||
"key": "Enter",
|
||||
"code": "Enter"
|
||||
}
|
||||
|
||||
// Key up
|
||||
{
|
||||
"type": "input_keyboard",
|
||||
"eventType": "keyUp",
|
||||
"key": "Enter",
|
||||
"code": "Enter"
|
||||
}
|
||||
|
||||
// Type character
|
||||
{
|
||||
"type": "input_keyboard",
|
||||
"eventType": "char",
|
||||
"text": "a"
|
||||
}
|
||||
|
||||
// With modifiers (1=Alt, 2=Ctrl, 4=Meta, 8=Shift)
|
||||
{
|
||||
"type": "input_keyboard",
|
||||
"eventType": "keyDown",
|
||||
"key": "c",
|
||||
"code": "KeyC",
|
||||
"modifiers": 2
|
||||
}`} />
|
||||
|
||||
<h3>Touch events</h3>
|
||||
<CodeBlock code={`// Touch start
|
||||
{
|
||||
"type": "input_touch",
|
||||
"eventType": "touchStart",
|
||||
"touchPoints": [{ "x": 100, "y": 200 }]
|
||||
}
|
||||
|
||||
// Touch move
|
||||
{
|
||||
"type": "input_touch",
|
||||
"eventType": "touchMove",
|
||||
"touchPoints": [{ "x": 150, "y": 250 }]
|
||||
}
|
||||
|
||||
// Touch end
|
||||
{
|
||||
"type": "input_touch",
|
||||
"eventType": "touchEnd",
|
||||
"touchPoints": []
|
||||
}
|
||||
|
||||
// Multi-touch (pinch zoom)
|
||||
{
|
||||
"type": "input_touch",
|
||||
"eventType": "touchStart",
|
||||
"touchPoints": [
|
||||
{ "x": 100, "y": 200, "id": 0 },
|
||||
{ "x": 200, "y": 200, "id": 1 }
|
||||
]
|
||||
}`} />
|
||||
|
||||
<h2>Programmatic API</h2>
|
||||
<p>For advanced use, control streaming directly via the TypeScript API:</p>
|
||||
<CodeBlock code={`import { BrowserManager } from 'agent-browser';
|
||||
|
||||
const browser = new BrowserManager();
|
||||
await browser.launch({ headless: true });
|
||||
await browser.navigate('https://example.com');
|
||||
|
||||
// Start screencast with callback
|
||||
await browser.startScreencast((frame) => {
|
||||
console.log('Frame:', frame.metadata.deviceWidth, 'x', frame.metadata.deviceHeight);
|
||||
// frame.data is base64-encoded image
|
||||
}, {
|
||||
format: 'jpeg', // or 'png'
|
||||
quality: 80, // 0-100, jpeg only
|
||||
maxWidth: 1280,
|
||||
maxHeight: 720,
|
||||
everyNthFrame: 1
|
||||
});
|
||||
|
||||
// Inject mouse event
|
||||
await browser.injectMouseEvent({
|
||||
type: 'mousePressed',
|
||||
x: 100,
|
||||
y: 200,
|
||||
button: 'left',
|
||||
clickCount: 1
|
||||
});
|
||||
|
||||
// Inject keyboard event
|
||||
await browser.injectKeyboardEvent({
|
||||
type: 'keyDown',
|
||||
key: 'Enter',
|
||||
code: 'Enter'
|
||||
});
|
||||
|
||||
// Inject touch event
|
||||
await browser.injectTouchEvent({
|
||||
type: 'touchStart',
|
||||
touchPoints: [{ x: 100, y: 200 }]
|
||||
});
|
||||
|
||||
// Check if screencasting
|
||||
console.log('Active:', browser.isScreencasting());
|
||||
|
||||
// Stop screencast
|
||||
await browser.stopScreencast();`} />
|
||||
|
||||
<h2>Use cases</h2>
|
||||
<ul>
|
||||
<li><strong>Pair browsing</strong> - Human watches and assists AI agent in real-time</li>
|
||||
<li><strong>Remote preview</strong> - View browser output in a separate UI</li>
|
||||
<li><strong>Recording</strong> - Capture frames for video generation</li>
|
||||
<li><strong>Mobile testing</strong> - Inject touch events for mobile emulation</li>
|
||||
<li><strong>Accessibility testing</strong> - Manual interaction during automated tests</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -12,6 +12,7 @@ const navigation = [
|
||||
{ name: "Selectors", href: "/selectors" },
|
||||
{ name: "Sessions", href: "/sessions" },
|
||||
{ name: "Snapshots", href: "/snapshots" },
|
||||
{ name: "Streaming", href: "/streaming" },
|
||||
{ name: "Agent Mode", href: "/agent-mode" },
|
||||
{ name: "CDP Mode", href: "/cdp-mode" },
|
||||
];
|
||||
|
||||
+3
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "agent-browser",
|
||||
"version": "0.4.4",
|
||||
"version": "0.5.0",
|
||||
"description": "Headless browser automation CLI for AI agents",
|
||||
"type": "module",
|
||||
"main": "dist/daemon.js",
|
||||
@@ -53,10 +53,12 @@
|
||||
"homepage": "https://github.com/vercel-labs/agent-browser#readme",
|
||||
"dependencies": {
|
||||
"playwright-core": "^1.57.0",
|
||||
"ws": "^8.19.0",
|
||||
"zod": "^3.22.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.10.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"husky": "^9.1.7",
|
||||
"lint-staged": "^15.2.11",
|
||||
"playwright": "^1.57.0",
|
||||
|
||||
Generated
+27
@@ -11,6 +11,9 @@ importers:
|
||||
playwright-core:
|
||||
specifier: ^1.57.0
|
||||
version: 1.57.0
|
||||
ws:
|
||||
specifier: ^8.19.0
|
||||
version: 8.19.0
|
||||
zod:
|
||||
specifier: ^3.22.4
|
||||
version: 3.25.76
|
||||
@@ -18,6 +21,9 @@ importers:
|
||||
'@types/node':
|
||||
specifier: ^20.10.0
|
||||
version: 20.19.28
|
||||
'@types/ws':
|
||||
specifier: ^8.18.1
|
||||
version: 8.18.1
|
||||
husky:
|
||||
specifier: ^9.1.7
|
||||
version: 9.1.7
|
||||
@@ -341,6 +347,9 @@ packages:
|
||||
'@types/node@20.19.28':
|
||||
resolution: {integrity: sha512-VyKBr25BuFDzBFCK5sUM6ZXiWfqgCTwTAOK8qzGV/m9FCirXYDlmczJ+d5dXBAQALGCdRRdbteKYfJ84NGEusw==}
|
||||
|
||||
'@types/ws@8.18.1':
|
||||
resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
|
||||
|
||||
'@vitest/expect@4.0.16':
|
||||
resolution: {integrity: sha512-eshqULT2It7McaJkQGLkPjPjNph+uevROGuIMJdG3V+0BSR2w9u6J9Lwu+E8cK5TETlfou8GRijhafIMhXsimA==}
|
||||
|
||||
@@ -805,6 +814,18 @@ packages:
|
||||
resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
ws@8.19.0:
|
||||
resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
peerDependencies:
|
||||
bufferutil: ^4.0.1
|
||||
utf-8-validate: '>=5.0.2'
|
||||
peerDependenciesMeta:
|
||||
bufferutil:
|
||||
optional: true
|
||||
utf-8-validate:
|
||||
optional: true
|
||||
|
||||
yaml@2.8.2:
|
||||
resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==}
|
||||
engines: {node: '>= 14.6'}
|
||||
@@ -985,6 +1006,10 @@ snapshots:
|
||||
dependencies:
|
||||
undici-types: 6.21.0
|
||||
|
||||
'@types/ws@8.18.1':
|
||||
dependencies:
|
||||
'@types/node': 20.19.28
|
||||
|
||||
'@vitest/expect@4.0.16':
|
||||
dependencies:
|
||||
'@standard-schema/spec': 1.1.0
|
||||
@@ -1427,6 +1452,8 @@ snapshots:
|
||||
string-width: 7.2.0
|
||||
strip-ansi: 7.1.2
|
||||
|
||||
ws@8.19.0: {}
|
||||
|
||||
yaml@2.8.2: {}
|
||||
|
||||
zod@3.25.76: {}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: agent-browser
|
||||
description: Automates browser interactions for web testing, form filling, screenshots, and data extraction. Use when the user needs to navigate websites, interact with web pages, fill forms, take screenshots, test web applications, or extract information from web pages.
|
||||
allowed-tools: Bash(agent-browser:*)
|
||||
---
|
||||
|
||||
# Browser Automation with agent-browser
|
||||
@@ -35,49 +36,86 @@ agent-browser close # Close browser
|
||||
|
||||
### Snapshot (page analysis)
|
||||
```bash
|
||||
agent-browser snapshot # Full accessibility tree
|
||||
agent-browser snapshot -i # Interactive elements only (recommended)
|
||||
agent-browser snapshot -c # Compact output
|
||||
agent-browser snapshot -d 3 # Limit depth to 3
|
||||
agent-browser snapshot # Full accessibility tree
|
||||
agent-browser snapshot -i # Interactive elements only (recommended)
|
||||
agent-browser snapshot -c # Compact output
|
||||
agent-browser snapshot -d 3 # Limit depth to 3
|
||||
agent-browser snapshot -s "#main" # Scope to CSS selector
|
||||
```
|
||||
|
||||
### Interactions (use @refs from snapshot)
|
||||
```bash
|
||||
agent-browser click @e1 # Click
|
||||
agent-browser dblclick @e1 # Double-click
|
||||
agent-browser focus @e1 # Focus element
|
||||
agent-browser fill @e2 "text" # Clear and type
|
||||
agent-browser type @e2 "text" # Type without clearing
|
||||
agent-browser press Enter # Press key
|
||||
agent-browser press Control+a # Key combination
|
||||
agent-browser keydown Shift # Hold key down
|
||||
agent-browser keyup Shift # Release key
|
||||
agent-browser hover @e1 # Hover
|
||||
agent-browser check @e1 # Check checkbox
|
||||
agent-browser uncheck @e1 # Uncheck checkbox
|
||||
agent-browser select @e1 "value" # Select dropdown
|
||||
agent-browser scroll down 500 # Scroll page
|
||||
agent-browser scrollintoview @e1 # Scroll element into view
|
||||
agent-browser drag @e1 @e2 # Drag and drop
|
||||
agent-browser upload @e1 file.pdf # Upload files
|
||||
```
|
||||
|
||||
### Get information
|
||||
```bash
|
||||
agent-browser get text @e1 # Get element text
|
||||
agent-browser get html @e1 # Get innerHTML
|
||||
agent-browser get value @e1 # Get input value
|
||||
agent-browser get attr @e1 href # Get attribute
|
||||
agent-browser get title # Get page title
|
||||
agent-browser get url # Get current URL
|
||||
agent-browser get count ".item" # Count matching elements
|
||||
agent-browser get box @e1 # Get bounding box
|
||||
```
|
||||
|
||||
### Screenshots
|
||||
### Check state
|
||||
```bash
|
||||
agent-browser is visible @e1 # Check if visible
|
||||
agent-browser is enabled @e1 # Check if enabled
|
||||
agent-browser is checked @e1 # Check if checked
|
||||
```
|
||||
|
||||
### Screenshots & PDF
|
||||
```bash
|
||||
agent-browser screenshot # Screenshot to stdout
|
||||
agent-browser screenshot path.png # Save to file
|
||||
agent-browser screenshot --full # Full page
|
||||
agent-browser pdf output.pdf # Save as PDF
|
||||
```
|
||||
|
||||
### Video recording
|
||||
```bash
|
||||
agent-browser record start ./demo.webm # Start recording (uses current URL + state)
|
||||
agent-browser click @e1 # Perform actions
|
||||
agent-browser record stop # Stop and save video
|
||||
agent-browser record restart ./take2.webm # Stop current + start new recording
|
||||
```
|
||||
Recording creates a fresh context but preserves cookies/storage from your session. If no URL is provided, it automatically returns to your current page. For smooth demos, explore first, then start recording.
|
||||
|
||||
### Wait
|
||||
```bash
|
||||
agent-browser wait @e1 # Wait for element
|
||||
agent-browser wait 2000 # Wait milliseconds
|
||||
agent-browser wait --text "Success" # Wait for text
|
||||
agent-browser wait --url "**/dashboard" # Wait for URL pattern
|
||||
agent-browser wait --load networkidle # Wait for network idle
|
||||
agent-browser wait --fn "window.ready" # Wait for JS condition
|
||||
```
|
||||
|
||||
### Mouse control
|
||||
```bash
|
||||
agent-browser mouse move 100 200 # Move mouse
|
||||
agent-browser mouse down left # Press button
|
||||
agent-browser mouse up left # Release button
|
||||
agent-browser mouse wheel 100 # Scroll wheel
|
||||
```
|
||||
|
||||
### Semantic locators (alternative to refs)
|
||||
@@ -85,6 +123,66 @@ agent-browser wait --load networkidle # Wait for network idle
|
||||
agent-browser find role button click --name "Submit"
|
||||
agent-browser find text "Sign In" click
|
||||
agent-browser find label "Email" fill "user@test.com"
|
||||
agent-browser find first ".item" click
|
||||
agent-browser find nth 2 "a" text
|
||||
```
|
||||
|
||||
### Browser settings
|
||||
```bash
|
||||
agent-browser set viewport 1920 1080 # Set viewport size
|
||||
agent-browser set device "iPhone 14" # Emulate device
|
||||
agent-browser set geo 37.7749 -122.4194 # Set geolocation
|
||||
agent-browser set offline on # Toggle offline mode
|
||||
agent-browser set headers '{"X-Key":"v"}' # Extra HTTP headers
|
||||
agent-browser set credentials user pass # HTTP basic auth
|
||||
agent-browser set media dark # Emulate color scheme
|
||||
```
|
||||
|
||||
### Cookies & Storage
|
||||
```bash
|
||||
agent-browser cookies # Get all cookies
|
||||
agent-browser cookies set name value # Set cookie
|
||||
agent-browser cookies clear # Clear cookies
|
||||
agent-browser storage local # Get all localStorage
|
||||
agent-browser storage local key # Get specific key
|
||||
agent-browser storage local set k v # Set value
|
||||
agent-browser storage local clear # Clear all
|
||||
```
|
||||
|
||||
### Network
|
||||
```bash
|
||||
agent-browser network route <url> # Intercept requests
|
||||
agent-browser network route <url> --abort # Block requests
|
||||
agent-browser network route <url> --body '{}' # Mock response
|
||||
agent-browser network unroute [url] # Remove routes
|
||||
agent-browser network requests # View tracked requests
|
||||
agent-browser network requests --filter api # Filter requests
|
||||
```
|
||||
|
||||
### Tabs & Windows
|
||||
```bash
|
||||
agent-browser tab # List tabs
|
||||
agent-browser tab new [url] # New tab
|
||||
agent-browser tab 2 # Switch to tab
|
||||
agent-browser tab close # Close tab
|
||||
agent-browser window new # New window
|
||||
```
|
||||
|
||||
### Frames
|
||||
```bash
|
||||
agent-browser frame "#iframe" # Switch to iframe
|
||||
agent-browser frame main # Back to main frame
|
||||
```
|
||||
|
||||
### Dialogs
|
||||
```bash
|
||||
agent-browser dialog accept [text] # Accept dialog
|
||||
agent-browser dialog dismiss # Dismiss dialog
|
||||
```
|
||||
|
||||
### JavaScript
|
||||
```bash
|
||||
agent-browser eval "document.title" # Run JavaScript
|
||||
```
|
||||
|
||||
## Example: Form submission
|
||||
@@ -137,7 +235,18 @@ agent-browser get text @e1 --json
|
||||
## Debugging
|
||||
|
||||
```bash
|
||||
agent-browser open example.com --headed # Show browser window
|
||||
agent-browser console # View console messages
|
||||
agent-browser errors # View page errors
|
||||
agent-browser record start ./debug.webm # Record from current page
|
||||
agent-browser record stop # Save recording
|
||||
agent-browser open example.com --headed # Show browser window
|
||||
agent-browser --cdp 9222 snapshot # Connect via CDP
|
||||
agent-browser console # View console messages
|
||||
agent-browser console --clear # Clear console
|
||||
agent-browser errors # View page errors
|
||||
agent-browser errors --clear # Clear errors
|
||||
agent-browser highlight @e1 # Highlight element
|
||||
agent-browser trace start # Start recording trace
|
||||
agent-browser trace stop trace.zip # Stop and save trace
|
||||
```
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { toAIFriendlyError } from './actions.js';
|
||||
|
||||
describe('toAIFriendlyError', () => {
|
||||
describe('element blocked by overlay', () => {
|
||||
it('should detect intercepts pointer events even when Timeout is in message', () => {
|
||||
// This is the exact error from Playwright when a cookie banner blocks an element
|
||||
// Bug: Previously this was incorrectly reported as "not found or not visible"
|
||||
const error = new Error(
|
||||
'TimeoutError: locator.click: Timeout 10000ms exceeded.\n' +
|
||||
'Call log:\n' +
|
||||
" - waiting for getByRole('link', { name: 'Anmelden', exact: true }).first()\n" +
|
||||
' - locator resolved to <a href="https://example.com/login">Anmelden</a>\n' +
|
||||
' - attempting click action\n' +
|
||||
' 2 x waiting for element to be visible, enabled and stable\n' +
|
||||
' - element is visible, enabled and stable\n' +
|
||||
' - scrolling into view if needed\n' +
|
||||
' - done scrolling\n' +
|
||||
' - <body class="font-sans antialiased">...</body> intercepts pointer events\n' +
|
||||
' - retrying click action'
|
||||
);
|
||||
|
||||
const result = toAIFriendlyError(error, '@e4');
|
||||
|
||||
// Must NOT say "not found" - the element WAS found
|
||||
expect(result.message).not.toContain('not found');
|
||||
// Must indicate the element is blocked
|
||||
expect(result.message).toContain('blocked by another element');
|
||||
expect(result.message).toContain('modal or overlay');
|
||||
});
|
||||
|
||||
it('should suggest dismissing cookie banners', () => {
|
||||
const error = new Error('<div class="cookie-overlay"> intercepts pointer events');
|
||||
const result = toAIFriendlyError(error, '@e1');
|
||||
|
||||
expect(result.message).toContain('cookie banners');
|
||||
});
|
||||
});
|
||||
});
|
||||
+246
-15
@@ -1,5 +1,5 @@
|
||||
import type { Page, Frame } from 'playwright-core';
|
||||
import type { BrowserManager } from './browser.js';
|
||||
import type { BrowserManager, ScreencastFrame } from './browser.js';
|
||||
import type {
|
||||
Command,
|
||||
Response,
|
||||
@@ -26,6 +26,7 @@ import type {
|
||||
SelectCommand,
|
||||
HoverCommand,
|
||||
ContentCommand,
|
||||
TabNewCommand,
|
||||
TabSwitchCommand,
|
||||
TabCloseCommand,
|
||||
WindowNewCommand,
|
||||
@@ -49,6 +50,7 @@ import type {
|
||||
IsCheckedCommand,
|
||||
CountCommand,
|
||||
BoundingBoxCommand,
|
||||
StylesCommand,
|
||||
TraceStartCommand,
|
||||
TraceStopCommand,
|
||||
HarStopCommand,
|
||||
@@ -94,6 +96,14 @@ import type {
|
||||
MultiSelectCommand,
|
||||
WaitForDownloadCommand,
|
||||
ResponseBodyCommand,
|
||||
ScreencastStartCommand,
|
||||
ScreencastStopCommand,
|
||||
InputMouseCommand,
|
||||
InputKeyboardCommand,
|
||||
InputTouchCommand,
|
||||
RecordingStartCommand,
|
||||
RecordingStopCommand,
|
||||
RecordingRestartCommand,
|
||||
NavigateData,
|
||||
ScreenshotData,
|
||||
EvaluateData,
|
||||
@@ -102,9 +112,29 @@ import type {
|
||||
TabNewData,
|
||||
TabSwitchData,
|
||||
TabCloseData,
|
||||
ScreencastStartData,
|
||||
ScreencastStopData,
|
||||
RecordingStartData,
|
||||
RecordingStopData,
|
||||
RecordingRestartData,
|
||||
InputEventData,
|
||||
StylesData,
|
||||
} from './types.js';
|
||||
import { successResponse, errorResponse } from './protocol.js';
|
||||
|
||||
// Callback for screencast frames - will be set by the daemon when streaming is active
|
||||
let screencastFrameCallback: ((frame: ScreencastFrame) => void) | null = null;
|
||||
|
||||
/**
|
||||
* Set the callback for screencast frames
|
||||
* This is called by the daemon to set up frame streaming
|
||||
*/
|
||||
export function setScreencastFrameCallback(
|
||||
callback: ((frame: ScreencastFrame) => void) | null
|
||||
): void {
|
||||
screencastFrameCallback = callback;
|
||||
}
|
||||
|
||||
// Snapshot response type
|
||||
interface SnapshotData {
|
||||
snapshot: string;
|
||||
@@ -113,8 +143,9 @@ interface SnapshotData {
|
||||
|
||||
/**
|
||||
* Convert Playwright errors to AI-friendly messages
|
||||
* @internal Exported for testing
|
||||
*/
|
||||
function toAIFriendlyError(error: unknown, selector: string): Error {
|
||||
export function toAIFriendlyError(error: unknown, selector: string): Error {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
// Handle strict mode violation (multiple elements match)
|
||||
@@ -129,7 +160,24 @@ function toAIFriendlyError(error: unknown, selector: string): Error {
|
||||
);
|
||||
}
|
||||
|
||||
// Handle element not found
|
||||
// Handle element not interactable (must be checked BEFORE timeout case)
|
||||
// This includes cases where an overlay/modal blocks the element
|
||||
if (message.includes('intercepts pointer events')) {
|
||||
return new Error(
|
||||
`Element "${selector}" is blocked by another element (likely a modal or overlay). ` +
|
||||
`Try dismissing any modals/cookie banners first.`
|
||||
);
|
||||
}
|
||||
|
||||
// Handle element not visible
|
||||
if (message.includes('not visible') && !message.includes('Timeout')) {
|
||||
return new Error(
|
||||
`Element "${selector}" is not visible. ` +
|
||||
`Try scrolling it into view or check if it's hidden.`
|
||||
);
|
||||
}
|
||||
|
||||
// Handle element not found (timeout waiting for element)
|
||||
if (
|
||||
message.includes('waiting for') &&
|
||||
(message.includes('to be visible') || message.includes('Timeout'))
|
||||
@@ -140,14 +188,6 @@ function toAIFriendlyError(error: unknown, selector: string): Error {
|
||||
);
|
||||
}
|
||||
|
||||
// Handle element not interactable
|
||||
if (message.includes('intercepts pointer events') || message.includes('not visible')) {
|
||||
return new Error(
|
||||
`Element "${selector}" is not interactable (may be hidden or covered). ` +
|
||||
`Try scrolling it into view or check if a modal/overlay is blocking it.`
|
||||
);
|
||||
}
|
||||
|
||||
// Return original error for unknown cases
|
||||
return error instanceof Error ? error : new Error(message);
|
||||
}
|
||||
@@ -280,6 +320,8 @@ export async function executeCommand(command: Command, browser: BrowserManager):
|
||||
return await handleCount(command, browser);
|
||||
case 'boundingbox':
|
||||
return await handleBoundingBox(command, browser);
|
||||
case 'styles':
|
||||
return await handleStyles(command, browser);
|
||||
case 'video_start':
|
||||
return await handleVideoStart(command, browser);
|
||||
case 'video_stop':
|
||||
@@ -386,6 +428,22 @@ export async function executeCommand(command: Command, browser: BrowserManager):
|
||||
return await handleWaitForDownload(command, browser);
|
||||
case 'responsebody':
|
||||
return await handleResponseBody(command, browser);
|
||||
case 'screencast_start':
|
||||
return await handleScreencastStart(command, browser);
|
||||
case 'screencast_stop':
|
||||
return await handleScreencastStop(command, browser);
|
||||
case 'input_mouse':
|
||||
return await handleInputMouse(command, browser);
|
||||
case 'input_keyboard':
|
||||
return await handleInputKeyboard(command, browser);
|
||||
case 'input_touch':
|
||||
return await handleInputTouch(command, browser);
|
||||
case 'recording_start':
|
||||
return await handleRecordingStart(command, browser);
|
||||
case 'recording_stop':
|
||||
return await handleRecordingStop(command, browser);
|
||||
case 'recording_restart':
|
||||
return await handleRecordingRestart(command, browser);
|
||||
default: {
|
||||
// TypeScript narrows to never here, but we handle it for safety
|
||||
const unknownCommand = command as { id: string; action: string };
|
||||
@@ -656,10 +714,17 @@ async function handleClose(
|
||||
}
|
||||
|
||||
async function handleTabNew(
|
||||
command: Command & { action: 'tab_new' },
|
||||
command: TabNewCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<TabNewData>> {
|
||||
const result = await browser.newTab();
|
||||
|
||||
// Navigate to URL if provided (same pattern as handleNavigate)
|
||||
if (command.url) {
|
||||
const page = browser.getPage();
|
||||
await page.goto(command.url, { waitUntil: 'domcontentloaded' });
|
||||
}
|
||||
|
||||
return successResponse(command.id, result);
|
||||
}
|
||||
|
||||
@@ -678,7 +743,7 @@ async function handleTabSwitch(
|
||||
command: TabSwitchCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<TabSwitchData>> {
|
||||
const result = browser.switchTo(command.index);
|
||||
const result = await browser.switchTo(command.index);
|
||||
const page = browser.getPage();
|
||||
return successResponse(command.id, {
|
||||
...result,
|
||||
@@ -1185,6 +1250,62 @@ async function handleBoundingBox(
|
||||
return successResponse(command.id, { box });
|
||||
}
|
||||
|
||||
async function handleStyles(
|
||||
command: StylesCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<StylesData>> {
|
||||
const page = browser.getPage();
|
||||
|
||||
// Shared extraction logic as a string to be eval'd in browser context
|
||||
const extractStylesScript = `(function(el) {
|
||||
const s = getComputedStyle(el);
|
||||
const r = el.getBoundingClientRect();
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
text: el.innerText?.trim().slice(0, 80) || null,
|
||||
box: {
|
||||
x: Math.round(r.x),
|
||||
y: Math.round(r.y),
|
||||
width: Math.round(r.width),
|
||||
height: Math.round(r.height),
|
||||
},
|
||||
styles: {
|
||||
fontSize: s.fontSize,
|
||||
fontWeight: s.fontWeight,
|
||||
fontFamily: s.fontFamily.split(',')[0].trim().replace(/"/g, ''),
|
||||
color: s.color,
|
||||
backgroundColor: s.backgroundColor,
|
||||
borderRadius: s.borderRadius,
|
||||
border: s.border !== 'none' && s.borderWidth !== '0px' ? s.border : null,
|
||||
boxShadow: s.boxShadow !== 'none' ? s.boxShadow : null,
|
||||
padding: s.padding,
|
||||
},
|
||||
};
|
||||
})`;
|
||||
|
||||
// Check if it's a ref - single element
|
||||
if (browser.isRef(command.selector)) {
|
||||
const locator = browser.getLocator(command.selector);
|
||||
const element = (await locator.evaluate((el, script) => {
|
||||
const fn = eval(script);
|
||||
return fn(el);
|
||||
}, extractStylesScript)) as StylesData['elements'][0];
|
||||
return successResponse(command.id, { elements: [element] });
|
||||
}
|
||||
|
||||
// CSS selector - can match multiple elements
|
||||
const elements = (await page.$$eval(
|
||||
command.selector,
|
||||
(els, script) => {
|
||||
const fn = eval(script);
|
||||
return els.map((el) => fn(el));
|
||||
},
|
||||
extractStylesScript
|
||||
)) as StylesData['elements'];
|
||||
|
||||
return successResponse(command.id, { elements });
|
||||
}
|
||||
|
||||
// Advanced handlers
|
||||
|
||||
async function handleVideoStart(
|
||||
@@ -1383,8 +1504,8 @@ async function handleInputValue(
|
||||
command: InputValueCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response> {
|
||||
const page = browser.getPage();
|
||||
const value = await page.locator(command.selector).inputValue();
|
||||
const locator = browser.getLocator(command.selector);
|
||||
const value = await locator.inputValue();
|
||||
return successResponse(command.id, { value });
|
||||
}
|
||||
|
||||
@@ -1769,3 +1890,113 @@ async function handleResponseBody(
|
||||
body: parsed,
|
||||
});
|
||||
}
|
||||
|
||||
// Screencast and input injection handlers
|
||||
|
||||
async function handleScreencastStart(
|
||||
command: ScreencastStartCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<ScreencastStartData>> {
|
||||
if (!screencastFrameCallback) {
|
||||
throw new Error('Screencast frame callback not set. Start the streaming server first.');
|
||||
}
|
||||
|
||||
await browser.startScreencast(screencastFrameCallback, {
|
||||
format: command.format,
|
||||
quality: command.quality,
|
||||
maxWidth: command.maxWidth,
|
||||
maxHeight: command.maxHeight,
|
||||
everyNthFrame: command.everyNthFrame,
|
||||
});
|
||||
|
||||
return successResponse(command.id, {
|
||||
started: true,
|
||||
format: command.format ?? 'jpeg',
|
||||
quality: command.quality ?? 80,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleScreencastStop(
|
||||
command: ScreencastStopCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<ScreencastStopData>> {
|
||||
await browser.stopScreencast();
|
||||
return successResponse(command.id, { stopped: true });
|
||||
}
|
||||
|
||||
async function handleInputMouse(
|
||||
command: InputMouseCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<InputEventData>> {
|
||||
await browser.injectMouseEvent({
|
||||
type: command.type,
|
||||
x: command.x,
|
||||
y: command.y,
|
||||
button: command.button,
|
||||
clickCount: command.clickCount,
|
||||
deltaX: command.deltaX,
|
||||
deltaY: command.deltaY,
|
||||
modifiers: command.modifiers,
|
||||
});
|
||||
return successResponse(command.id, { injected: true });
|
||||
}
|
||||
|
||||
async function handleInputKeyboard(
|
||||
command: InputKeyboardCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<InputEventData>> {
|
||||
await browser.injectKeyboardEvent({
|
||||
type: command.type,
|
||||
key: command.key,
|
||||
code: command.code,
|
||||
text: command.text,
|
||||
modifiers: command.modifiers,
|
||||
});
|
||||
return successResponse(command.id, { injected: true });
|
||||
}
|
||||
|
||||
async function handleInputTouch(
|
||||
command: InputTouchCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<InputEventData>> {
|
||||
await browser.injectTouchEvent({
|
||||
type: command.type,
|
||||
touchPoints: command.touchPoints,
|
||||
modifiers: command.modifiers,
|
||||
});
|
||||
return successResponse(command.id, { injected: true });
|
||||
}
|
||||
|
||||
// Recording handlers (Playwright native video recording)
|
||||
|
||||
async function handleRecordingStart(
|
||||
command: RecordingStartCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<RecordingStartData>> {
|
||||
await browser.startRecording(command.path, command.url);
|
||||
return successResponse(command.id, {
|
||||
started: true,
|
||||
path: command.path,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleRecordingStop(
|
||||
command: RecordingStopCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<RecordingStopData>> {
|
||||
const result = await browser.stopRecording();
|
||||
return successResponse(command.id, result);
|
||||
}
|
||||
|
||||
async function handleRecordingRestart(
|
||||
command: RecordingRestartCommand,
|
||||
browser: BrowserManager
|
||||
): Promise<Response<RecordingRestartData>> {
|
||||
const result = await browser.restartRecording(command.path, command.url);
|
||||
return successResponse(command.id, {
|
||||
started: true,
|
||||
path: command.path,
|
||||
previousPath: result.previousPath,
|
||||
stopped: result.stopped,
|
||||
});
|
||||
}
|
||||
|
||||
+283
-1
@@ -1,5 +1,6 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
||||
import { BrowserManager } from './browser.js';
|
||||
import { chromium } from 'playwright-core';
|
||||
|
||||
describe('BrowserManager', () => {
|
||||
let browser: BrowserManager;
|
||||
@@ -378,4 +379,285 @@ describe('BrowserManager', () => {
|
||||
await expect(browser.clearScopedHeaders('https://never-set.com')).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('CDP session', () => {
|
||||
it('should create CDP session on demand', async () => {
|
||||
const cdp = await browser.getCDPSession();
|
||||
expect(cdp).toBeDefined();
|
||||
});
|
||||
|
||||
it('should reuse existing CDP session', async () => {
|
||||
const cdp1 = await browser.getCDPSession();
|
||||
const cdp2 = await browser.getCDPSession();
|
||||
expect(cdp1).toBe(cdp2);
|
||||
});
|
||||
|
||||
it('should filter out pages with empty URLs during CDP connection', async () => {
|
||||
const mockBrowser = {
|
||||
contexts: () => [
|
||||
{
|
||||
pages: () => [
|
||||
{ url: () => 'http://example.com', on: vi.fn() },
|
||||
{ url: () => '', on: vi.fn() }, // This page should be filtered out
|
||||
{ url: () => 'http://anothersite.com', on: vi.fn() },
|
||||
],
|
||||
on: vi.fn(),
|
||||
},
|
||||
],
|
||||
close: vi.fn(),
|
||||
};
|
||||
const spy = vi.spyOn(chromium, 'connectOverCDP').mockResolvedValue(mockBrowser as any);
|
||||
|
||||
const cdpBrowser = new BrowserManager();
|
||||
await cdpBrowser.launch({ cdpPort: 9222 });
|
||||
|
||||
// Should have 2 pages, not 3
|
||||
expect(cdpBrowser.getPages().length).toBe(2);
|
||||
|
||||
// Verify that the empty URL page is not in the list
|
||||
const urls = cdpBrowser.getPages().map((p) => p.url());
|
||||
expect(urls).not.toContain('');
|
||||
expect(urls).toContain('http://example.com');
|
||||
spy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('screencast', () => {
|
||||
it('should report screencasting state correctly', () => {
|
||||
expect(browser.isScreencasting()).toBe(false);
|
||||
});
|
||||
|
||||
it('should start screencast', async () => {
|
||||
const frames: Array<{ data: string }> = [];
|
||||
await browser.startScreencast((frame) => {
|
||||
frames.push(frame);
|
||||
});
|
||||
expect(browser.isScreencasting()).toBe(true);
|
||||
|
||||
// Wait a bit for at least one frame
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
|
||||
await browser.stopScreencast();
|
||||
expect(browser.isScreencasting()).toBe(false);
|
||||
expect(frames.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should start screencast with custom options', async () => {
|
||||
const frames: Array<{ data: string }> = [];
|
||||
await browser.startScreencast(
|
||||
(frame) => {
|
||||
frames.push(frame);
|
||||
},
|
||||
{
|
||||
format: 'png',
|
||||
quality: 100,
|
||||
maxWidth: 800,
|
||||
maxHeight: 600,
|
||||
everyNthFrame: 1,
|
||||
}
|
||||
);
|
||||
expect(browser.isScreencasting()).toBe(true);
|
||||
|
||||
// Wait for a frame
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
|
||||
await browser.stopScreencast();
|
||||
expect(frames.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should throw when starting screencast twice', async () => {
|
||||
await browser.startScreencast(() => {});
|
||||
await expect(browser.startScreencast(() => {})).rejects.toThrow('Screencast already active');
|
||||
await browser.stopScreencast();
|
||||
});
|
||||
|
||||
it('should handle stop when not screencasting', async () => {
|
||||
// Should not throw
|
||||
await expect(browser.stopScreencast()).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('tab switch invalidates CDP session', () => {
|
||||
// Clean up any extra tabs before each test
|
||||
beforeEach(async () => {
|
||||
// Close all tabs except the first one
|
||||
const tabs = await browser.listTabs();
|
||||
for (let i = tabs.length - 1; i > 0; i--) {
|
||||
await browser.closeTab(i);
|
||||
}
|
||||
// Ensure we're on tab 0
|
||||
await browser.switchTo(0);
|
||||
// Stop any active screencast
|
||||
if (browser.isScreencasting()) {
|
||||
await browser.stopScreencast();
|
||||
}
|
||||
});
|
||||
|
||||
it('should not invalidate CDP when switching to same tab', async () => {
|
||||
// Get CDP session for current tab
|
||||
const cdp1 = await browser.getCDPSession();
|
||||
|
||||
// Switch to same tab - should NOT invalidate
|
||||
await browser.switchTo(0);
|
||||
|
||||
// Should be the same session
|
||||
const cdp2 = await browser.getCDPSession();
|
||||
expect(cdp2).toBe(cdp1);
|
||||
});
|
||||
|
||||
it('should invalidate CDP session on tab switch', async () => {
|
||||
// Get CDP session for tab 0
|
||||
const cdp1 = await browser.getCDPSession();
|
||||
expect(cdp1).toBeDefined();
|
||||
|
||||
// Create new tab - this switches to the new tab automatically
|
||||
await browser.newTab();
|
||||
|
||||
// Get CDP session - should be different since we're on a new page
|
||||
const cdp2 = await browser.getCDPSession();
|
||||
expect(cdp2).toBeDefined();
|
||||
|
||||
// Sessions should be different objects (different pages have different CDP sessions)
|
||||
expect(cdp2).not.toBe(cdp1);
|
||||
});
|
||||
|
||||
it('should stop screencast on tab switch', async () => {
|
||||
// Start screencast on tab 0
|
||||
await browser.startScreencast(() => {});
|
||||
expect(browser.isScreencasting()).toBe(true);
|
||||
|
||||
// Create new tab and switch
|
||||
await browser.newTab();
|
||||
await browser.switchTo(1);
|
||||
|
||||
// Screencast should be stopped (it's page-specific)
|
||||
expect(browser.isScreencasting()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('input injection', () => {
|
||||
it('should inject mouse move event', async () => {
|
||||
await expect(
|
||||
browser.injectMouseEvent({
|
||||
type: 'mouseMoved',
|
||||
x: 100,
|
||||
y: 100,
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should inject mouse click events', async () => {
|
||||
await expect(
|
||||
browser.injectMouseEvent({
|
||||
type: 'mousePressed',
|
||||
x: 100,
|
||||
y: 100,
|
||||
button: 'left',
|
||||
clickCount: 1,
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
|
||||
await expect(
|
||||
browser.injectMouseEvent({
|
||||
type: 'mouseReleased',
|
||||
x: 100,
|
||||
y: 100,
|
||||
button: 'left',
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should inject mouse wheel event', async () => {
|
||||
await expect(
|
||||
browser.injectMouseEvent({
|
||||
type: 'mouseWheel',
|
||||
x: 100,
|
||||
y: 100,
|
||||
deltaX: 0,
|
||||
deltaY: 100,
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should inject keyboard events', async () => {
|
||||
await expect(
|
||||
browser.injectKeyboardEvent({
|
||||
type: 'keyDown',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
|
||||
await expect(
|
||||
browser.injectKeyboardEvent({
|
||||
type: 'keyUp',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should inject char event', async () => {
|
||||
// CDP char events only accept single characters
|
||||
await expect(
|
||||
browser.injectKeyboardEvent({
|
||||
type: 'char',
|
||||
text: 'h',
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should inject keyboard with modifiers', async () => {
|
||||
await expect(
|
||||
browser.injectKeyboardEvent({
|
||||
type: 'keyDown',
|
||||
key: 'c',
|
||||
code: 'KeyC',
|
||||
modifiers: 2, // Ctrl
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should inject touch events', async () => {
|
||||
await expect(
|
||||
browser.injectTouchEvent({
|
||||
type: 'touchStart',
|
||||
touchPoints: [{ x: 100, y: 100 }],
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
|
||||
await expect(
|
||||
browser.injectTouchEvent({
|
||||
type: 'touchMove',
|
||||
touchPoints: [{ x: 150, y: 150 }],
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
|
||||
await expect(
|
||||
browser.injectTouchEvent({
|
||||
type: 'touchEnd',
|
||||
touchPoints: [],
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should inject multi-touch events', async () => {
|
||||
await expect(
|
||||
browser.injectTouchEvent({
|
||||
type: 'touchStart',
|
||||
touchPoints: [
|
||||
{ x: 100, y: 100, id: 0 },
|
||||
{ x: 200, y: 200, id: 1 },
|
||||
],
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
|
||||
await expect(
|
||||
browser.injectTouchEvent({
|
||||
type: 'touchEnd',
|
||||
touchPoints: [],
|
||||
})
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+535
-28
@@ -11,10 +11,39 @@ import {
|
||||
type Request,
|
||||
type Route,
|
||||
type Locator,
|
||||
type CDPSession,
|
||||
type Video,
|
||||
} from 'playwright-core';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { existsSync, mkdirSync, rmSync } from 'node:fs';
|
||||
import type { LaunchCommand } from './types.js';
|
||||
import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js';
|
||||
|
||||
// Screencast frame data from CDP
|
||||
export interface ScreencastFrame {
|
||||
data: string; // base64 encoded image
|
||||
metadata: {
|
||||
offsetTop: number;
|
||||
pageScaleFactor: number;
|
||||
deviceWidth: number;
|
||||
deviceHeight: number;
|
||||
scrollOffsetX: number;
|
||||
scrollOffsetY: number;
|
||||
timestamp?: number;
|
||||
};
|
||||
sessionId: number;
|
||||
}
|
||||
|
||||
// Screencast options
|
||||
export interface ScreencastOptions {
|
||||
format?: 'jpeg' | 'png';
|
||||
quality?: number; // 0-100, only for jpeg
|
||||
maxWidth?: number;
|
||||
maxHeight?: number;
|
||||
everyNthFrame?: number;
|
||||
}
|
||||
|
||||
interface TrackedRequest {
|
||||
url: string;
|
||||
method: string;
|
||||
@@ -40,6 +69,7 @@ interface PageError {
|
||||
export class BrowserManager {
|
||||
private browser: Browser | null = null;
|
||||
private cdpPort: number | null = null;
|
||||
private isPersistentContext: boolean = false;
|
||||
private contexts: BrowserContext[] = [];
|
||||
private pages: Page[] = [];
|
||||
private activePageIndex: number = 0;
|
||||
@@ -54,11 +84,24 @@ export class BrowserManager {
|
||||
private lastSnapshot: string = '';
|
||||
private scopedHeaderRoutes: Map<string, (route: Route) => Promise<void>> = new Map();
|
||||
|
||||
// CDP session for screencast and input injection
|
||||
private cdpSession: CDPSession | null = null;
|
||||
private screencastActive: boolean = false;
|
||||
private screencastSessionId: number = 0;
|
||||
private frameCallback: ((frame: ScreencastFrame) => void) | null = null;
|
||||
private screencastFrameHandler: ((params: any) => void) | null = null;
|
||||
|
||||
// Video recording (Playwright native)
|
||||
private recordingContext: BrowserContext | null = null;
|
||||
private recordingPage: Page | null = null;
|
||||
private recordingOutputPath: string = '';
|
||||
private recordingTempDir: string = '';
|
||||
|
||||
/**
|
||||
* Check if browser is launched
|
||||
*/
|
||||
isLaunched(): boolean {
|
||||
return this.browser !== null;
|
||||
return this.browser !== null || this.isPersistentContext;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -605,12 +648,16 @@ export class BrowserManager {
|
||||
*/
|
||||
async launch(options: LaunchCommand): Promise<void> {
|
||||
const cdpPort = options.cdpPort;
|
||||
const hasExtensions = !!options.extensions?.length;
|
||||
|
||||
if (this.browser) {
|
||||
const switchingFromCdpToBrowser = !cdpPort && this.cdpPort !== null;
|
||||
const needsCdpReconnect = !!cdpPort && this.needsCdpReconnect(cdpPort);
|
||||
if (hasExtensions && cdpPort) {
|
||||
throw new Error('Extensions cannot be used with CDP connection');
|
||||
}
|
||||
|
||||
if (switchingFromCdpToBrowser || needsCdpReconnect) {
|
||||
if (this.isLaunched()) {
|
||||
const needsRelaunch =
|
||||
(!cdpPort && this.cdpPort !== null) || (!!cdpPort && this.needsCdpReconnect(cdpPort));
|
||||
if (needsRelaunch) {
|
||||
await this.close();
|
||||
} else {
|
||||
return;
|
||||
@@ -622,35 +669,50 @@ export class BrowserManager {
|
||||
return;
|
||||
}
|
||||
|
||||
// Select browser type
|
||||
const browserType = options.browser ?? 'chromium';
|
||||
if (hasExtensions && browserType !== 'chromium') {
|
||||
throw new Error('Extensions are only supported in Chromium');
|
||||
}
|
||||
|
||||
const launcher =
|
||||
browserType === 'firefox' ? firefox : browserType === 'webkit' ? webkit : chromium;
|
||||
const viewport = options.viewport ?? { width: 1280, height: 720 };
|
||||
|
||||
// Launch browser
|
||||
this.browser = await launcher.launch({
|
||||
headless: options.headless ?? true,
|
||||
executablePath: options.executablePath,
|
||||
});
|
||||
this.cdpPort = null;
|
||||
|
||||
// Create context with viewport and optional headers
|
||||
const context = await this.browser.newContext({
|
||||
viewport: options.viewport ?? { width: 1280, height: 720 },
|
||||
extraHTTPHeaders: options.headers,
|
||||
});
|
||||
|
||||
// Set default timeout to 10 seconds (Playwright default is 30s)
|
||||
context.setDefaultTimeout(10000);
|
||||
let context: BrowserContext;
|
||||
if (hasExtensions) {
|
||||
const extPaths = options.extensions!.join(',');
|
||||
const session = process.env.AGENT_BROWSER_SESSION || 'default';
|
||||
context = await launcher.launchPersistentContext(
|
||||
path.join(os.tmpdir(), `agent-browser-ext-${session}`),
|
||||
{
|
||||
headless: false,
|
||||
executablePath: options.executablePath,
|
||||
args: [`--disable-extensions-except=${extPaths}`, `--load-extension=${extPaths}`],
|
||||
viewport,
|
||||
extraHTTPHeaders: options.headers,
|
||||
...(options.proxy && { proxy: options.proxy }),
|
||||
}
|
||||
);
|
||||
this.isPersistentContext = true;
|
||||
} else {
|
||||
this.browser = await launcher.launch({
|
||||
headless: options.headless ?? true,
|
||||
executablePath: options.executablePath,
|
||||
});
|
||||
this.cdpPort = null;
|
||||
context = await this.browser.newContext({
|
||||
viewport,
|
||||
extraHTTPHeaders: options.headers,
|
||||
...(options.proxy && { proxy: options.proxy }),
|
||||
});
|
||||
}
|
||||
|
||||
context.setDefaultTimeout(60000);
|
||||
this.contexts.push(context);
|
||||
|
||||
// Create initial page
|
||||
const page = await context.newPage();
|
||||
const page = context.pages()[0] ?? (await context.newPage());
|
||||
this.pages.push(page);
|
||||
this.activePageIndex = 0;
|
||||
|
||||
// Automatically start console and error tracking
|
||||
this.setupPageTracking(page);
|
||||
}
|
||||
|
||||
@@ -676,7 +738,9 @@ export class BrowserManager {
|
||||
throw new Error('No browser context found. Make sure the app has an open window.');
|
||||
}
|
||||
|
||||
const allPages = contexts.flatMap((context) => context.pages());
|
||||
// Filter out pages with empty URLs, which can cause Playwright to hang
|
||||
const allPages = contexts.flatMap((context) => context.pages()).filter((page) => page.url());
|
||||
|
||||
if (allPages.length === 0) {
|
||||
throw new Error('No page found. Make sure the app has loaded content.');
|
||||
}
|
||||
@@ -751,6 +815,9 @@ export class BrowserManager {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
// Invalidate CDP session since we're switching to a new page
|
||||
await this.invalidateCDPSession();
|
||||
|
||||
const context = this.contexts[0]; // Use first context for tabs
|
||||
const page = await context.newPage();
|
||||
this.pages.push(page);
|
||||
@@ -776,7 +843,7 @@ export class BrowserManager {
|
||||
const context = await this.browser.newContext({
|
||||
viewport: viewport ?? { width: 1280, height: 720 },
|
||||
});
|
||||
context.setDefaultTimeout(10000);
|
||||
context.setDefaultTimeout(60000);
|
||||
this.contexts.push(context);
|
||||
|
||||
const page = await context.newPage();
|
||||
@@ -789,14 +856,36 @@ export class BrowserManager {
|
||||
return { index: this.activePageIndex, total: this.pages.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate the current CDP session (must be called before switching pages)
|
||||
* This ensures screencast and input injection work correctly after tab switch
|
||||
*/
|
||||
private async invalidateCDPSession(): Promise<void> {
|
||||
// Stop screencast if active (it's tied to the current page's CDP session)
|
||||
if (this.screencastActive) {
|
||||
await this.stopScreencast();
|
||||
}
|
||||
|
||||
// Detach and clear the CDP session
|
||||
if (this.cdpSession) {
|
||||
await this.cdpSession.detach().catch(() => {});
|
||||
this.cdpSession = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Switch to a specific tab/page by index
|
||||
*/
|
||||
switchTo(index: number): { index: number; url: string; title: string } {
|
||||
async switchTo(index: number): Promise<{ index: number; url: string; title: string }> {
|
||||
if (index < 0 || index >= this.pages.length) {
|
||||
throw new Error(`Invalid tab index: ${index}. Available: 0-${this.pages.length - 1}`);
|
||||
}
|
||||
|
||||
// Invalidate CDP session before switching (it's page-specific)
|
||||
if (index !== this.activePageIndex) {
|
||||
await this.invalidateCDPSession();
|
||||
}
|
||||
|
||||
this.activePageIndex = index;
|
||||
const page = this.pages[index];
|
||||
|
||||
@@ -821,6 +910,11 @@ export class BrowserManager {
|
||||
throw new Error('Cannot close the last tab. Use "close" to close the browser.');
|
||||
}
|
||||
|
||||
// If closing the active tab, invalidate CDP session first
|
||||
if (targetIndex === this.activePageIndex) {
|
||||
await this.invalidateCDPSession();
|
||||
}
|
||||
|
||||
const page = this.pages[targetIndex];
|
||||
await page.close();
|
||||
this.pages.splice(targetIndex, 1);
|
||||
@@ -850,10 +944,421 @@ export class BrowserManager {
|
||||
return tabs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a CDP session for the current page
|
||||
* Only works with Chromium-based browsers
|
||||
*/
|
||||
async getCDPSession(): Promise<CDPSession> {
|
||||
if (this.cdpSession) {
|
||||
return this.cdpSession;
|
||||
}
|
||||
|
||||
const page = this.getPage();
|
||||
const context = page.context();
|
||||
|
||||
// Create a new CDP session attached to the page
|
||||
this.cdpSession = await context.newCDPSession(page);
|
||||
return this.cdpSession;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if screencast is currently active
|
||||
*/
|
||||
isScreencasting(): boolean {
|
||||
return this.screencastActive;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start screencast - streams viewport frames via CDP
|
||||
* @param callback Function called for each frame
|
||||
* @param options Screencast options
|
||||
*/
|
||||
async startScreencast(
|
||||
callback: (frame: ScreencastFrame) => void,
|
||||
options?: ScreencastOptions
|
||||
): Promise<void> {
|
||||
if (this.screencastActive) {
|
||||
throw new Error('Screencast already active');
|
||||
}
|
||||
|
||||
const cdp = await this.getCDPSession();
|
||||
this.frameCallback = callback;
|
||||
this.screencastActive = true;
|
||||
|
||||
// Create and store the frame handler so we can remove it later
|
||||
this.screencastFrameHandler = async (params: any) => {
|
||||
const frame: ScreencastFrame = {
|
||||
data: params.data,
|
||||
metadata: params.metadata,
|
||||
sessionId: params.sessionId,
|
||||
};
|
||||
|
||||
// Acknowledge the frame to receive the next one
|
||||
await cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId });
|
||||
|
||||
// Call the callback with the frame
|
||||
if (this.frameCallback) {
|
||||
this.frameCallback(frame);
|
||||
}
|
||||
};
|
||||
|
||||
// Listen for screencast frames
|
||||
cdp.on('Page.screencastFrame', this.screencastFrameHandler);
|
||||
|
||||
// Start the screencast
|
||||
await cdp.send('Page.startScreencast', {
|
||||
format: options?.format ?? 'jpeg',
|
||||
quality: options?.quality ?? 80,
|
||||
maxWidth: options?.maxWidth ?? 1280,
|
||||
maxHeight: options?.maxHeight ?? 720,
|
||||
everyNthFrame: options?.everyNthFrame ?? 1,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop screencast
|
||||
*/
|
||||
async stopScreencast(): Promise<void> {
|
||||
if (!this.screencastActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const cdp = await this.getCDPSession();
|
||||
await cdp.send('Page.stopScreencast');
|
||||
|
||||
// Remove the event listener to prevent accumulation
|
||||
if (this.screencastFrameHandler) {
|
||||
cdp.off('Page.screencastFrame', this.screencastFrameHandler);
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors when stopping
|
||||
}
|
||||
|
||||
this.screencastActive = false;
|
||||
this.frameCallback = null;
|
||||
this.screencastFrameHandler = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject a mouse event via CDP
|
||||
*/
|
||||
async injectMouseEvent(params: {
|
||||
type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel';
|
||||
x: number;
|
||||
y: number;
|
||||
button?: 'left' | 'right' | 'middle' | 'none';
|
||||
clickCount?: number;
|
||||
deltaX?: number;
|
||||
deltaY?: number;
|
||||
modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift
|
||||
}): Promise<void> {
|
||||
const cdp = await this.getCDPSession();
|
||||
|
||||
const cdpButton =
|
||||
params.button === 'left'
|
||||
? 'left'
|
||||
: params.button === 'right'
|
||||
? 'right'
|
||||
: params.button === 'middle'
|
||||
? 'middle'
|
||||
: 'none';
|
||||
|
||||
await cdp.send('Input.dispatchMouseEvent', {
|
||||
type: params.type,
|
||||
x: params.x,
|
||||
y: params.y,
|
||||
button: cdpButton,
|
||||
clickCount: params.clickCount ?? 1,
|
||||
deltaX: params.deltaX ?? 0,
|
||||
deltaY: params.deltaY ?? 0,
|
||||
modifiers: params.modifiers ?? 0,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject a keyboard event via CDP
|
||||
*/
|
||||
async injectKeyboardEvent(params: {
|
||||
type: 'keyDown' | 'keyUp' | 'char';
|
||||
key?: string;
|
||||
code?: string;
|
||||
text?: string;
|
||||
modifiers?: number; // 1=Alt, 2=Ctrl, 4=Meta, 8=Shift
|
||||
}): Promise<void> {
|
||||
const cdp = await this.getCDPSession();
|
||||
|
||||
await cdp.send('Input.dispatchKeyEvent', {
|
||||
type: params.type,
|
||||
key: params.key,
|
||||
code: params.code,
|
||||
text: params.text,
|
||||
modifiers: params.modifiers ?? 0,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject touch event via CDP (for mobile emulation)
|
||||
*/
|
||||
async injectTouchEvent(params: {
|
||||
type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel';
|
||||
touchPoints: Array<{ x: number; y: number; id?: number }>;
|
||||
modifiers?: number;
|
||||
}): Promise<void> {
|
||||
const cdp = await this.getCDPSession();
|
||||
|
||||
await cdp.send('Input.dispatchTouchEvent', {
|
||||
type: params.type,
|
||||
touchPoints: params.touchPoints.map((tp, i) => ({
|
||||
x: tp.x,
|
||||
y: tp.y,
|
||||
id: tp.id ?? i,
|
||||
})),
|
||||
modifiers: params.modifiers ?? 0,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if video recording is currently active
|
||||
*/
|
||||
isRecording(): boolean {
|
||||
return this.recordingContext !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start recording to a video file using Playwright's native video recording.
|
||||
* Creates a fresh browser context with video recording enabled.
|
||||
* Automatically captures current URL and transfers cookies/storage if no URL provided.
|
||||
*
|
||||
* @param outputPath - Path to the output video file (will be .webm)
|
||||
* @param url - Optional URL to navigate to (defaults to current page URL)
|
||||
*/
|
||||
async startRecording(outputPath: string, url?: string): Promise<void> {
|
||||
if (this.recordingContext) {
|
||||
throw new Error(
|
||||
"Recording already in progress. Run 'record stop' first, or use 'record restart' to stop and start a new recording."
|
||||
);
|
||||
}
|
||||
|
||||
if (!this.browser) {
|
||||
throw new Error('Browser not launched. Call launch first.');
|
||||
}
|
||||
|
||||
// Check if output file already exists
|
||||
if (existsSync(outputPath)) {
|
||||
throw new Error(`Output file already exists: ${outputPath}`);
|
||||
}
|
||||
|
||||
// Validate output path is .webm (Playwright native format)
|
||||
if (!outputPath.endsWith('.webm')) {
|
||||
throw new Error(
|
||||
'Playwright native recording only supports WebM format. Please use a .webm extension.'
|
||||
);
|
||||
}
|
||||
|
||||
// Auto-capture current URL if none provided
|
||||
const currentPage = this.pages.length > 0 ? this.pages[this.activePageIndex] : null;
|
||||
const currentContext = this.contexts.length > 0 ? this.contexts[0] : null;
|
||||
if (!url && currentPage) {
|
||||
const currentUrl = currentPage.url();
|
||||
if (currentUrl && currentUrl !== 'about:blank') {
|
||||
url = currentUrl;
|
||||
}
|
||||
}
|
||||
|
||||
// Capture state from current context (cookies + storage)
|
||||
let storageState:
|
||||
| {
|
||||
cookies: Array<{
|
||||
name: string;
|
||||
value: string;
|
||||
domain: string;
|
||||
path: string;
|
||||
expires: number;
|
||||
httpOnly: boolean;
|
||||
secure: boolean;
|
||||
sameSite: 'Strict' | 'Lax' | 'None';
|
||||
}>;
|
||||
origins: Array<{
|
||||
origin: string;
|
||||
localStorage: Array<{ name: string; value: string }>;
|
||||
}>;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
if (currentContext) {
|
||||
try {
|
||||
storageState = await currentContext.storageState();
|
||||
} catch {
|
||||
// Ignore errors - context might be closed or invalid
|
||||
}
|
||||
}
|
||||
|
||||
// Create a temp directory for video recording
|
||||
const session = process.env.AGENT_BROWSER_SESSION || 'default';
|
||||
this.recordingTempDir = path.join(
|
||||
os.tmpdir(),
|
||||
`agent-browser-recording-${session}-${Date.now()}`
|
||||
);
|
||||
mkdirSync(this.recordingTempDir, { recursive: true });
|
||||
|
||||
this.recordingOutputPath = outputPath;
|
||||
|
||||
// Create a new context with video recording enabled and restored state
|
||||
const viewport = { width: 1280, height: 720 };
|
||||
this.recordingContext = await this.browser.newContext({
|
||||
viewport,
|
||||
recordVideo: {
|
||||
dir: this.recordingTempDir,
|
||||
size: viewport,
|
||||
},
|
||||
storageState,
|
||||
});
|
||||
this.recordingContext.setDefaultTimeout(10000);
|
||||
|
||||
// Create a page in the recording context
|
||||
this.recordingPage = await this.recordingContext.newPage();
|
||||
|
||||
// Add the recording context and page to our managed lists
|
||||
this.contexts.push(this.recordingContext);
|
||||
this.pages.push(this.recordingPage);
|
||||
this.activePageIndex = this.pages.length - 1;
|
||||
|
||||
// Set up page tracking
|
||||
this.setupPageTracking(this.recordingPage);
|
||||
|
||||
// Invalidate CDP session since we switched pages
|
||||
await this.invalidateCDPSession();
|
||||
|
||||
// Navigate to URL if provided or captured
|
||||
if (url) {
|
||||
await this.recordingPage.goto(url, { waitUntil: 'load' });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop recording and save the video file
|
||||
* @returns Recording result with path
|
||||
*/
|
||||
async stopRecording(): Promise<{ path: string; frames: number; error?: string }> {
|
||||
if (!this.recordingContext || !this.recordingPage) {
|
||||
return { path: '', frames: 0, error: 'No recording in progress' };
|
||||
}
|
||||
|
||||
const outputPath = this.recordingOutputPath;
|
||||
|
||||
try {
|
||||
// Get the video object before closing the page
|
||||
const video = this.recordingPage.video();
|
||||
|
||||
// Remove recording page/context from our managed lists before closing
|
||||
const pageIndex = this.pages.indexOf(this.recordingPage);
|
||||
if (pageIndex !== -1) {
|
||||
this.pages.splice(pageIndex, 1);
|
||||
}
|
||||
const contextIndex = this.contexts.indexOf(this.recordingContext);
|
||||
if (contextIndex !== -1) {
|
||||
this.contexts.splice(contextIndex, 1);
|
||||
}
|
||||
|
||||
// Close the page to finalize the video
|
||||
await this.recordingPage.close();
|
||||
|
||||
// Save the video to the desired output path
|
||||
if (video) {
|
||||
await video.saveAs(outputPath);
|
||||
}
|
||||
|
||||
// Clean up temp directory
|
||||
if (this.recordingTempDir) {
|
||||
rmSync(this.recordingTempDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// Close the recording context
|
||||
await this.recordingContext.close();
|
||||
|
||||
// Reset recording state
|
||||
this.recordingContext = null;
|
||||
this.recordingPage = null;
|
||||
this.recordingOutputPath = '';
|
||||
this.recordingTempDir = '';
|
||||
|
||||
// Adjust active page index
|
||||
if (this.pages.length > 0) {
|
||||
this.activePageIndex = Math.min(this.activePageIndex, this.pages.length - 1);
|
||||
} else {
|
||||
this.activePageIndex = 0;
|
||||
}
|
||||
|
||||
// Invalidate CDP session since we may have switched pages
|
||||
await this.invalidateCDPSession();
|
||||
|
||||
return { path: outputPath, frames: 0 }; // Playwright doesn't expose frame count
|
||||
} catch (error) {
|
||||
// Clean up temp directory on error
|
||||
if (this.recordingTempDir) {
|
||||
rmSync(this.recordingTempDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// Reset state on error
|
||||
this.recordingContext = null;
|
||||
this.recordingPage = null;
|
||||
this.recordingOutputPath = '';
|
||||
this.recordingTempDir = '';
|
||||
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return { path: outputPath, frames: 0, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Restart recording - stops current recording (if any) and starts a new one.
|
||||
* Convenience method that combines stopRecording and startRecording.
|
||||
*
|
||||
* @param outputPath - Path to the output video file (must be .webm)
|
||||
* @param url - Optional URL to navigate to (defaults to current page URL)
|
||||
* @returns Result from stopping the previous recording (if any)
|
||||
*/
|
||||
async restartRecording(
|
||||
outputPath: string,
|
||||
url?: string
|
||||
): Promise<{ previousPath?: string; stopped: boolean }> {
|
||||
let previousPath: string | undefined;
|
||||
let stopped = false;
|
||||
|
||||
// Stop current recording if active
|
||||
if (this.recordingContext) {
|
||||
const result = await this.stopRecording();
|
||||
previousPath = result.path;
|
||||
stopped = true;
|
||||
}
|
||||
|
||||
// Start new recording
|
||||
await this.startRecording(outputPath, url);
|
||||
|
||||
return { previousPath, stopped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the browser and clean up
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
// Stop recording if active (saves video)
|
||||
if (this.recordingContext) {
|
||||
await this.stopRecording();
|
||||
}
|
||||
|
||||
// Stop screencast if active
|
||||
if (this.screencastActive) {
|
||||
await this.stopScreencast();
|
||||
}
|
||||
|
||||
// Clean up CDP session
|
||||
if (this.cdpSession) {
|
||||
await this.cdpSession.detach().catch(() => {});
|
||||
this.cdpSession = null;
|
||||
}
|
||||
|
||||
// CDP: only disconnect, don't close external app's pages
|
||||
if (this.cdpPort !== null) {
|
||||
if (this.browser) {
|
||||
@@ -877,8 +1382,10 @@ export class BrowserManager {
|
||||
this.pages = [];
|
||||
this.contexts = [];
|
||||
this.cdpPort = null;
|
||||
this.isPersistentContext = false;
|
||||
this.activePageIndex = 0;
|
||||
this.refMap = {};
|
||||
this.lastSnapshot = '';
|
||||
this.frameCallback = null;
|
||||
}
|
||||
}
|
||||
|
||||
+57
-3
@@ -5,6 +5,7 @@ import * as os from 'os';
|
||||
import { BrowserManager } from './browser.js';
|
||||
import { parseCommand, serializeResponse, errorResponse } from './protocol.js';
|
||||
import { executeCommand } from './actions.js';
|
||||
import { StreamServer } from './stream-server.js';
|
||||
|
||||
// Platform detection
|
||||
const isWindows = process.platform === 'win32';
|
||||
@@ -12,6 +13,12 @@ const isWindows = process.platform === 'win32';
|
||||
// Session support - each session gets its own socket/pid
|
||||
let currentSession = process.env.AGENT_BROWSER_SESSION || 'default';
|
||||
|
||||
// Stream server for browser preview
|
||||
let streamServer: StreamServer | null = null;
|
||||
|
||||
// Default stream port (can be overridden with AGENT_BROWSER_STREAM_PORT)
|
||||
const DEFAULT_STREAM_PORT = 9223;
|
||||
|
||||
/**
|
||||
* Set the current session
|
||||
*/
|
||||
@@ -105,8 +112,10 @@ export function getConnectionInfo(
|
||||
*/
|
||||
export function cleanupSocket(session?: string): void {
|
||||
const pidFile = getPidFile(session);
|
||||
const streamPortFile = getStreamPortFile(session);
|
||||
try {
|
||||
if (fs.existsSync(pidFile)) fs.unlinkSync(pidFile);
|
||||
if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile);
|
||||
if (isWindows) {
|
||||
const portFile = getPortFile(session);
|
||||
if (fs.existsSync(portFile)) fs.unlinkSync(portFile);
|
||||
@@ -120,15 +129,40 @@ export function cleanupSocket(session?: string): void {
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the daemon server
|
||||
* Get the stream port file path
|
||||
*/
|
||||
export async function startDaemon(): Promise<void> {
|
||||
export function getStreamPortFile(session?: string): string {
|
||||
const sess = session ?? currentSession;
|
||||
return path.join(os.tmpdir(), `agent-browser-${sess}.stream`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the daemon server
|
||||
* @param options.streamPort Port for WebSocket stream server (0 to disable)
|
||||
*/
|
||||
export async function startDaemon(options?: { streamPort?: number }): Promise<void> {
|
||||
// Clean up any stale socket
|
||||
cleanupSocket();
|
||||
|
||||
const browser = new BrowserManager();
|
||||
let shuttingDown = false;
|
||||
|
||||
// Start stream server if port is specified (or use default if env var is set)
|
||||
const streamPort =
|
||||
options?.streamPort ??
|
||||
(process.env.AGENT_BROWSER_STREAM_PORT
|
||||
? parseInt(process.env.AGENT_BROWSER_STREAM_PORT, 10)
|
||||
: 0);
|
||||
|
||||
if (streamPort > 0) {
|
||||
streamServer = new StreamServer(browser, streamPort);
|
||||
await streamServer.start();
|
||||
|
||||
// Write stream port to file for clients to discover
|
||||
const streamPortFile = getStreamPortFile();
|
||||
fs.writeFileSync(streamPortFile, streamPort.toString());
|
||||
}
|
||||
|
||||
const server = net.createServer((socket) => {
|
||||
let buffer = '';
|
||||
|
||||
@@ -158,11 +192,17 @@ export async function startDaemon(): Promise<void> {
|
||||
parseResult.command.action !== 'launch' &&
|
||||
parseResult.command.action !== 'close'
|
||||
) {
|
||||
const extensions = process.env.AGENT_BROWSER_EXTENSIONS
|
||||
? process.env.AGENT_BROWSER_EXTENSIONS.split(',')
|
||||
.map((p) => p.trim())
|
||||
.filter(Boolean)
|
||||
: undefined;
|
||||
await browser.launch({
|
||||
id: 'auto',
|
||||
action: 'launch',
|
||||
headless: true,
|
||||
headless: process.env.AGENT_BROWSER_HEADED !== '1',
|
||||
executablePath: process.env.AGENT_BROWSER_EXECUTABLE_PATH,
|
||||
extensions: extensions,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -227,6 +267,20 @@ export async function startDaemon(): Promise<void> {
|
||||
const shutdown = async () => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
|
||||
// Stop stream server if running
|
||||
if (streamServer) {
|
||||
await streamServer.stop();
|
||||
streamServer = null;
|
||||
// Clean up stream port file
|
||||
const streamPortFile = getStreamPortFile();
|
||||
try {
|
||||
if (fs.existsSync(streamPortFile)) fs.unlinkSync(streamPortFile);
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
server.close();
|
||||
cleanupSocket();
|
||||
|
||||
@@ -15,6 +15,22 @@ describe('parseCommand', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse navigate with headers', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'navigate',
|
||||
url: 'https://example.com',
|
||||
headers: { Authorization: 'Bearer token' },
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.action).toBe('navigate');
|
||||
expect(result.command.headers).toEqual({ Authorization: 'Bearer token' });
|
||||
}
|
||||
});
|
||||
|
||||
it('should reject navigate without url', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'navigate' }));
|
||||
expect(result.success).toBe(false);
|
||||
@@ -373,6 +389,14 @@ describe('parseCommand', () => {
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse tab_new with url', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'tab_new', url: 'https://example.com' }));
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect((result.command as { url?: string }).url).toBe('https://example.com');
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse tab_list', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'tab_list' }));
|
||||
expect(result.success).toBe(true);
|
||||
@@ -620,6 +644,391 @@ describe('parseCommand', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('screencast', () => {
|
||||
it('should parse screencast_start with defaults', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'screencast_start' }));
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.action).toBe('screencast_start');
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse screencast_start with all options', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'screencast_start',
|
||||
format: 'png',
|
||||
quality: 90,
|
||||
maxWidth: 1920,
|
||||
maxHeight: 1080,
|
||||
everyNthFrame: 2,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.format).toBe('png');
|
||||
expect(result.command.quality).toBe(90);
|
||||
expect(result.command.maxWidth).toBe(1920);
|
||||
expect(result.command.maxHeight).toBe(1080);
|
||||
expect(result.command.everyNthFrame).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('should reject screencast_start with invalid format', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'screencast_start', format: 'gif' }));
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject screencast_start with quality out of range', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'screencast_start', quality: 150 }));
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject screencast_start with negative maxWidth', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'screencast_start', maxWidth: -100 }));
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should parse screencast_stop', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'screencast_stop' }));
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.action).toBe('screencast_stop');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('input injection', () => {
|
||||
describe('input_mouse', () => {
|
||||
it('should parse mousePressed event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mousePressed',
|
||||
x: 100,
|
||||
y: 200,
|
||||
button: 'left',
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.action).toBe('input_mouse');
|
||||
expect(result.command.type).toBe('mousePressed');
|
||||
expect(result.command.x).toBe(100);
|
||||
expect(result.command.y).toBe(200);
|
||||
expect(result.command.button).toBe('left');
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse mouseReleased event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mouseReleased',
|
||||
x: 100,
|
||||
y: 200,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse mouseMoved event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mouseMoved',
|
||||
x: 150,
|
||||
y: 250,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse mouseWheel event with deltas', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mouseWheel',
|
||||
x: 100,
|
||||
y: 200,
|
||||
deltaX: 0,
|
||||
deltaY: 100,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.deltaX).toBe(0);
|
||||
expect(result.command.deltaY).toBe(100);
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse mouse event with modifiers', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mousePressed',
|
||||
x: 100,
|
||||
y: 200,
|
||||
modifiers: 6, // Ctrl + Meta
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.modifiers).toBe(6);
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse mouse event with clickCount', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mousePressed',
|
||||
x: 100,
|
||||
y: 200,
|
||||
clickCount: 2,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.clickCount).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('should reject input_mouse with invalid type', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'invalid',
|
||||
x: 100,
|
||||
y: 200,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject input_mouse without x coordinate', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mousePressed',
|
||||
y: 200,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject input_mouse without y coordinate', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_mouse',
|
||||
type: 'mousePressed',
|
||||
x: 100,
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('input_keyboard', () => {
|
||||
it('should parse keyDown event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_keyboard',
|
||||
type: 'keyDown',
|
||||
key: 'Enter',
|
||||
code: 'Enter',
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.action).toBe('input_keyboard');
|
||||
expect(result.command.type).toBe('keyDown');
|
||||
expect(result.command.key).toBe('Enter');
|
||||
expect(result.command.code).toBe('Enter');
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse keyUp event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_keyboard',
|
||||
type: 'keyUp',
|
||||
key: 'a',
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse char event with text', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_keyboard',
|
||||
type: 'char',
|
||||
text: 'hello',
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.text).toBe('hello');
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse keyboard event with modifiers', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_keyboard',
|
||||
type: 'keyDown',
|
||||
key: 'c',
|
||||
modifiers: 2, // Ctrl
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.modifiers).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('should reject input_keyboard with invalid type', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_keyboard',
|
||||
type: 'invalid',
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('input_touch', () => {
|
||||
it('should parse touchStart event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'touchStart',
|
||||
touchPoints: [{ x: 100, y: 200 }],
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.action).toBe('input_touch');
|
||||
expect(result.command.type).toBe('touchStart');
|
||||
expect(result.command.touchPoints).toHaveLength(1);
|
||||
expect(result.command.touchPoints[0].x).toBe(100);
|
||||
expect(result.command.touchPoints[0].y).toBe(200);
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse touchEnd event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'touchEnd',
|
||||
touchPoints: [],
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse touchMove event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'touchMove',
|
||||
touchPoints: [{ x: 150, y: 250 }],
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse touchCancel event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'touchCancel',
|
||||
touchPoints: [],
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse multi-touch event', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'touchStart',
|
||||
touchPoints: [
|
||||
{ x: 100, y: 200, id: 0 },
|
||||
{ x: 300, y: 400, id: 1 },
|
||||
],
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.touchPoints).toHaveLength(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('should parse touch event with modifiers', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'touchStart',
|
||||
touchPoints: [{ x: 100, y: 200 }],
|
||||
modifiers: 8, // Shift
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.command.modifiers).toBe(8);
|
||||
}
|
||||
});
|
||||
|
||||
it('should reject input_touch with invalid type', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'invalid',
|
||||
touchPoints: [],
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject input_touch without touchPoints', () => {
|
||||
const result = parseCommand(
|
||||
cmd({
|
||||
id: '1',
|
||||
action: 'input_touch',
|
||||
type: 'touchStart',
|
||||
})
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('invalid commands', () => {
|
||||
it('should reject unknown action', () => {
|
||||
const result = parseCommand(cmd({ id: '1', action: 'unknown' }));
|
||||
|
||||
+94
-1
@@ -19,12 +19,24 @@ const launchSchema = baseCommandSchema.extend({
|
||||
.optional(),
|
||||
browser: z.enum(['chromium', 'firefox', 'webkit']).optional(),
|
||||
cdpPort: z.number().positive().optional(),
|
||||
executablePath: z.string().optional(),
|
||||
extensions: z.array(z.string()).optional(),
|
||||
headers: z.record(z.string()).optional(),
|
||||
proxy: z
|
||||
.object({
|
||||
server: z.string().min(1),
|
||||
bypass: z.string().optional(),
|
||||
username: z.string().optional(),
|
||||
password: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
const navigateSchema = baseCommandSchema.extend({
|
||||
action: z.literal('navigate'),
|
||||
url: z.string().min(1),
|
||||
waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle']).optional(),
|
||||
headers: z.record(z.string()).optional(),
|
||||
});
|
||||
|
||||
const clickSchema = baseCommandSchema.extend({
|
||||
@@ -295,6 +307,11 @@ const boundingBoxSchema = baseCommandSchema.extend({
|
||||
selector: z.string().min(1),
|
||||
});
|
||||
|
||||
const stylesSchema = baseCommandSchema.extend({
|
||||
action: z.literal('styles'),
|
||||
selector: z.string().min(1),
|
||||
});
|
||||
|
||||
const videoStartSchema = baseCommandSchema.extend({
|
||||
action: z.literal('video_start'),
|
||||
path: z.string().min(1),
|
||||
@@ -304,6 +321,23 @@ const videoStopSchema = baseCommandSchema.extend({
|
||||
action: z.literal('video_stop'),
|
||||
});
|
||||
|
||||
// Recording schemas (Playwright native video recording)
|
||||
const recordingStartSchema = baseCommandSchema.extend({
|
||||
action: z.literal('recording_start'),
|
||||
path: z.string().min(1),
|
||||
url: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
const recordingStopSchema = baseCommandSchema.extend({
|
||||
action: z.literal('recording_stop'),
|
||||
});
|
||||
|
||||
const recordingRestartSchema = baseCommandSchema.extend({
|
||||
action: z.literal('recording_restart'),
|
||||
path: z.string().min(1),
|
||||
url: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
const traceStartSchema = baseCommandSchema.extend({
|
||||
action: z.literal('trace_start'),
|
||||
screenshots: z.boolean().optional(),
|
||||
@@ -585,6 +619,55 @@ const responseBodySchema = baseCommandSchema.extend({
|
||||
timeout: z.number().positive().optional(),
|
||||
});
|
||||
|
||||
// Screencast schemas for streaming browser viewport
|
||||
const screencastStartSchema = baseCommandSchema.extend({
|
||||
action: z.literal('screencast_start'),
|
||||
format: z.enum(['jpeg', 'png']).optional(),
|
||||
quality: z.number().min(0).max(100).optional(),
|
||||
maxWidth: z.number().positive().optional(),
|
||||
maxHeight: z.number().positive().optional(),
|
||||
everyNthFrame: z.number().positive().optional(),
|
||||
});
|
||||
|
||||
const screencastStopSchema = baseCommandSchema.extend({
|
||||
action: z.literal('screencast_stop'),
|
||||
});
|
||||
|
||||
// Input injection schemas for pair browsing
|
||||
const inputMouseSchema = baseCommandSchema.extend({
|
||||
action: z.literal('input_mouse'),
|
||||
type: z.enum(['mousePressed', 'mouseReleased', 'mouseMoved', 'mouseWheel']),
|
||||
x: z.number(),
|
||||
y: z.number(),
|
||||
button: z.enum(['left', 'right', 'middle', 'none']).optional(),
|
||||
clickCount: z.number().positive().optional(),
|
||||
deltaX: z.number().optional(),
|
||||
deltaY: z.number().optional(),
|
||||
modifiers: z.number().optional(),
|
||||
});
|
||||
|
||||
const inputKeyboardSchema = baseCommandSchema.extend({
|
||||
action: z.literal('input_keyboard'),
|
||||
type: z.enum(['keyDown', 'keyUp', 'char']),
|
||||
key: z.string().optional(),
|
||||
code: z.string().optional(),
|
||||
text: z.string().optional(),
|
||||
modifiers: z.number().optional(),
|
||||
});
|
||||
|
||||
const inputTouchSchema = baseCommandSchema.extend({
|
||||
action: z.literal('input_touch'),
|
||||
type: z.enum(['touchStart', 'touchEnd', 'touchMove', 'touchCancel']),
|
||||
touchPoints: z.array(
|
||||
z.object({
|
||||
x: z.number(),
|
||||
y: z.number(),
|
||||
id: z.number().optional(),
|
||||
})
|
||||
),
|
||||
modifiers: z.number().optional(),
|
||||
});
|
||||
|
||||
const pressSchema = baseCommandSchema.extend({
|
||||
action: z.literal('press'),
|
||||
key: z.string().min(1),
|
||||
@@ -593,7 +676,7 @@ const pressSchema = baseCommandSchema.extend({
|
||||
|
||||
const screenshotSchema = baseCommandSchema.extend({
|
||||
action: z.literal('screenshot'),
|
||||
path: z.string().optional(),
|
||||
path: z.string().nullable().optional(),
|
||||
fullPage: z.boolean().optional(),
|
||||
selector: z.string().min(1).optional(),
|
||||
format: z.enum(['png', 'jpeg']).optional(),
|
||||
@@ -653,6 +736,7 @@ const closeSchema = baseCommandSchema.extend({
|
||||
// Tab/Window schemas
|
||||
const tabNewSchema = baseCommandSchema.extend({
|
||||
action: z.literal('tab_new'),
|
||||
url: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
const tabListSchema = baseCommandSchema.extend({
|
||||
@@ -742,8 +826,12 @@ const commandSchema = z.discriminatedUnion('action', [
|
||||
isCheckedSchema,
|
||||
countSchema,
|
||||
boundingBoxSchema,
|
||||
stylesSchema,
|
||||
videoStartSchema,
|
||||
videoStopSchema,
|
||||
recordingStartSchema,
|
||||
recordingStopSchema,
|
||||
recordingRestartSchema,
|
||||
traceStartSchema,
|
||||
traceStopSchema,
|
||||
harStartSchema,
|
||||
@@ -795,6 +883,11 @@ const commandSchema = z.discriminatedUnion('action', [
|
||||
multiSelectSchema,
|
||||
waitForDownloadSchema,
|
||||
responseBodySchema,
|
||||
screencastStartSchema,
|
||||
screencastStopSchema,
|
||||
inputMouseSchema,
|
||||
inputKeyboardSchema,
|
||||
inputTouchSchema,
|
||||
]);
|
||||
|
||||
// Parse result type
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
import { WebSocketServer, WebSocket } from 'ws';
|
||||
import type { BrowserManager, ScreencastFrame } from './browser.js';
|
||||
import { setScreencastFrameCallback } from './actions.js';
|
||||
|
||||
// Message types for WebSocket communication
|
||||
export interface FrameMessage {
|
||||
type: 'frame';
|
||||
data: string; // base64 encoded image
|
||||
metadata: {
|
||||
offsetTop: number;
|
||||
pageScaleFactor: number;
|
||||
deviceWidth: number;
|
||||
deviceHeight: number;
|
||||
scrollOffsetX: number;
|
||||
scrollOffsetY: number;
|
||||
timestamp?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface InputMouseMessage {
|
||||
type: 'input_mouse';
|
||||
eventType: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel';
|
||||
x: number;
|
||||
y: number;
|
||||
button?: 'left' | 'right' | 'middle' | 'none';
|
||||
clickCount?: number;
|
||||
deltaX?: number;
|
||||
deltaY?: number;
|
||||
modifiers?: number;
|
||||
}
|
||||
|
||||
export interface InputKeyboardMessage {
|
||||
type: 'input_keyboard';
|
||||
eventType: 'keyDown' | 'keyUp' | 'char';
|
||||
key?: string;
|
||||
code?: string;
|
||||
text?: string;
|
||||
modifiers?: number;
|
||||
}
|
||||
|
||||
export interface InputTouchMessage {
|
||||
type: 'input_touch';
|
||||
eventType: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel';
|
||||
touchPoints: Array<{ x: number; y: number; id?: number }>;
|
||||
modifiers?: number;
|
||||
}
|
||||
|
||||
export interface StatusMessage {
|
||||
type: 'status';
|
||||
connected: boolean;
|
||||
screencasting: boolean;
|
||||
viewportWidth?: number;
|
||||
viewportHeight?: number;
|
||||
}
|
||||
|
||||
export interface ErrorMessage {
|
||||
type: 'error';
|
||||
message: string;
|
||||
}
|
||||
|
||||
export type StreamMessage =
|
||||
| FrameMessage
|
||||
| InputMouseMessage
|
||||
| InputKeyboardMessage
|
||||
| InputTouchMessage
|
||||
| StatusMessage
|
||||
| ErrorMessage;
|
||||
|
||||
/**
|
||||
* WebSocket server for streaming browser viewport and receiving input
|
||||
*/
|
||||
export class StreamServer {
|
||||
private wss: WebSocketServer | null = null;
|
||||
private clients: Set<WebSocket> = new Set();
|
||||
private browser: BrowserManager;
|
||||
private port: number;
|
||||
private isScreencasting: boolean = false;
|
||||
|
||||
constructor(browser: BrowserManager, port: number = 9223) {
|
||||
this.browser = browser;
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the WebSocket server
|
||||
*/
|
||||
start(): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
this.wss = new WebSocketServer({ port: this.port });
|
||||
|
||||
this.wss.on('connection', (ws) => {
|
||||
this.handleConnection(ws);
|
||||
});
|
||||
|
||||
this.wss.on('error', (error) => {
|
||||
console.error('[StreamServer] WebSocket error:', error);
|
||||
reject(error);
|
||||
});
|
||||
|
||||
this.wss.on('listening', () => {
|
||||
console.log(`[StreamServer] Listening on port ${this.port}`);
|
||||
|
||||
// Set up the screencast frame callback
|
||||
setScreencastFrameCallback((frame) => {
|
||||
this.broadcastFrame(frame);
|
||||
});
|
||||
|
||||
resolve();
|
||||
});
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the WebSocket server
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
// Stop screencasting
|
||||
if (this.isScreencasting) {
|
||||
await this.stopScreencast();
|
||||
}
|
||||
|
||||
// Clear the callback
|
||||
setScreencastFrameCallback(null);
|
||||
|
||||
// Close all clients
|
||||
for (const client of this.clients) {
|
||||
client.close();
|
||||
}
|
||||
this.clients.clear();
|
||||
|
||||
// Close the server
|
||||
if (this.wss) {
|
||||
return new Promise((resolve) => {
|
||||
this.wss!.close(() => {
|
||||
this.wss = null;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a new WebSocket connection
|
||||
*/
|
||||
private handleConnection(ws: WebSocket): void {
|
||||
console.log('[StreamServer] Client connected');
|
||||
this.clients.add(ws);
|
||||
|
||||
// Send initial status
|
||||
this.sendStatus(ws);
|
||||
|
||||
// Start screencasting if this is the first client
|
||||
if (this.clients.size === 1 && !this.isScreencasting) {
|
||||
this.startScreencast().catch((error) => {
|
||||
console.error('[StreamServer] Failed to start screencast:', error);
|
||||
this.sendError(ws, error.message);
|
||||
});
|
||||
}
|
||||
|
||||
// Handle messages from client
|
||||
ws.on('message', (data) => {
|
||||
try {
|
||||
const message = JSON.parse(data.toString()) as StreamMessage;
|
||||
this.handleMessage(message, ws);
|
||||
} catch (error) {
|
||||
console.error('[StreamServer] Failed to parse message:', error);
|
||||
}
|
||||
});
|
||||
|
||||
// Handle client disconnect
|
||||
ws.on('close', () => {
|
||||
console.log('[StreamServer] Client disconnected');
|
||||
this.clients.delete(ws);
|
||||
|
||||
// Stop screencasting if no more clients
|
||||
if (this.clients.size === 0 && this.isScreencasting) {
|
||||
this.stopScreencast().catch((error) => {
|
||||
console.error('[StreamServer] Failed to stop screencast:', error);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('error', (error) => {
|
||||
console.error('[StreamServer] Client error:', error);
|
||||
this.clients.delete(ws);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle incoming messages from clients
|
||||
*/
|
||||
private async handleMessage(message: StreamMessage, ws: WebSocket): Promise<void> {
|
||||
try {
|
||||
switch (message.type) {
|
||||
case 'input_mouse':
|
||||
await this.browser.injectMouseEvent({
|
||||
type: message.eventType,
|
||||
x: message.x,
|
||||
y: message.y,
|
||||
button: message.button,
|
||||
clickCount: message.clickCount,
|
||||
deltaX: message.deltaX,
|
||||
deltaY: message.deltaY,
|
||||
modifiers: message.modifiers,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'input_keyboard':
|
||||
await this.browser.injectKeyboardEvent({
|
||||
type: message.eventType,
|
||||
key: message.key,
|
||||
code: message.code,
|
||||
text: message.text,
|
||||
modifiers: message.modifiers,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'input_touch':
|
||||
await this.browser.injectTouchEvent({
|
||||
type: message.eventType,
|
||||
touchPoints: message.touchPoints,
|
||||
modifiers: message.modifiers,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'status':
|
||||
// Client is requesting status
|
||||
this.sendStatus(ws);
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
this.sendError(ws, errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Broadcast a frame to all connected clients
|
||||
*/
|
||||
private broadcastFrame(frame: ScreencastFrame): void {
|
||||
const message: FrameMessage = {
|
||||
type: 'frame',
|
||||
data: frame.data,
|
||||
metadata: frame.metadata,
|
||||
};
|
||||
|
||||
const payload = JSON.stringify(message);
|
||||
|
||||
for (const client of this.clients) {
|
||||
if (client.readyState === WebSocket.OPEN) {
|
||||
client.send(payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send status to a client
|
||||
*/
|
||||
private sendStatus(ws: WebSocket): void {
|
||||
let viewportWidth: number | undefined;
|
||||
let viewportHeight: number | undefined;
|
||||
|
||||
try {
|
||||
const page = this.browser.getPage();
|
||||
const viewport = page.viewportSize();
|
||||
viewportWidth = viewport?.width;
|
||||
viewportHeight = viewport?.height;
|
||||
} catch {
|
||||
// Browser not launched yet
|
||||
}
|
||||
|
||||
const message: StatusMessage = {
|
||||
type: 'status',
|
||||
connected: true,
|
||||
screencasting: this.isScreencasting,
|
||||
viewportWidth,
|
||||
viewportHeight,
|
||||
};
|
||||
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify(message));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an error to a client
|
||||
*/
|
||||
private sendError(ws: WebSocket, errorMessage: string): void {
|
||||
const message: ErrorMessage = {
|
||||
type: 'error',
|
||||
message: errorMessage,
|
||||
};
|
||||
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify(message));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start screencasting
|
||||
*/
|
||||
private async startScreencast(): Promise<void> {
|
||||
// Set flag immediately to prevent race conditions with concurrent calls
|
||||
if (this.isScreencasting) return;
|
||||
this.isScreencasting = true;
|
||||
|
||||
try {
|
||||
// Check if browser is launched
|
||||
if (!this.browser.isLaunched()) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.browser.startScreencast((frame) => this.broadcastFrame(frame), {
|
||||
format: 'jpeg',
|
||||
quality: 80,
|
||||
maxWidth: 1280,
|
||||
maxHeight: 720,
|
||||
everyNthFrame: 1,
|
||||
});
|
||||
|
||||
// Notify all clients
|
||||
for (const client of this.clients) {
|
||||
this.sendStatus(client);
|
||||
}
|
||||
} catch (error) {
|
||||
// Reset flag on failure so caller can retry
|
||||
this.isScreencasting = false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop screencasting
|
||||
*/
|
||||
private async stopScreencast(): Promise<void> {
|
||||
if (!this.isScreencasting) return;
|
||||
|
||||
await this.browser.stopScreencast();
|
||||
this.isScreencasting = false;
|
||||
|
||||
// Notify all clients
|
||||
for (const client of this.clients) {
|
||||
this.sendStatus(client);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the port the server is running on
|
||||
*/
|
||||
getPort(): number {
|
||||
return this.port;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of connected clients
|
||||
*/
|
||||
getClientCount(): number {
|
||||
return this.clients.size;
|
||||
}
|
||||
}
|
||||
+139
-2
@@ -15,6 +15,13 @@ export interface LaunchCommand extends BaseCommand {
|
||||
headers?: Record<string, string>;
|
||||
executablePath?: string;
|
||||
cdpPort?: number;
|
||||
extensions?: string[];
|
||||
proxy?: {
|
||||
server: string;
|
||||
bypass?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface NavigateCommand extends BaseCommand {
|
||||
@@ -309,6 +316,12 @@ export interface BoundingBoxCommand extends BaseCommand {
|
||||
selector: string;
|
||||
}
|
||||
|
||||
// Computed styles
|
||||
export interface StylesCommand extends BaseCommand {
|
||||
action: 'styles';
|
||||
selector: string;
|
||||
}
|
||||
|
||||
// More semantic locators
|
||||
export interface GetByAltTextCommand extends BaseCommand {
|
||||
action: 'getbyalttext';
|
||||
@@ -458,7 +471,50 @@ export interface ResponseBodyCommand extends BaseCommand {
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
// Video recording
|
||||
// Screencast commands for streaming browser viewport
|
||||
export interface ScreencastStartCommand extends BaseCommand {
|
||||
action: 'screencast_start';
|
||||
format?: 'jpeg' | 'png';
|
||||
quality?: number; // 0-100, jpeg only
|
||||
maxWidth?: number;
|
||||
maxHeight?: number;
|
||||
everyNthFrame?: number;
|
||||
}
|
||||
|
||||
export interface ScreencastStopCommand extends BaseCommand {
|
||||
action: 'screencast_stop';
|
||||
}
|
||||
|
||||
// Input injection commands for pair browsing
|
||||
export interface InputMouseCommand extends BaseCommand {
|
||||
action: 'input_mouse';
|
||||
type: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel';
|
||||
x: number;
|
||||
y: number;
|
||||
button?: 'left' | 'right' | 'middle' | 'none';
|
||||
clickCount?: number;
|
||||
deltaX?: number;
|
||||
deltaY?: number;
|
||||
modifiers?: number;
|
||||
}
|
||||
|
||||
export interface InputKeyboardCommand extends BaseCommand {
|
||||
action: 'input_keyboard';
|
||||
type: 'keyDown' | 'keyUp' | 'char';
|
||||
key?: string;
|
||||
code?: string;
|
||||
text?: string;
|
||||
modifiers?: number;
|
||||
}
|
||||
|
||||
export interface InputTouchCommand extends BaseCommand {
|
||||
action: 'input_touch';
|
||||
type: 'touchStart' | 'touchEnd' | 'touchMove' | 'touchCancel';
|
||||
touchPoints: Array<{ x: number; y: number; id?: number }>;
|
||||
modifiers?: number;
|
||||
}
|
||||
|
||||
// Video recording (Playwright native - requires launch-time setup)
|
||||
export interface VideoStartCommand extends BaseCommand {
|
||||
action: 'video_start';
|
||||
path: string;
|
||||
@@ -468,6 +524,23 @@ export interface VideoStopCommand extends BaseCommand {
|
||||
action: 'video_stop';
|
||||
}
|
||||
|
||||
// Screen recording (Playwright native - creates fresh recording context)
|
||||
export interface RecordingStartCommand extends BaseCommand {
|
||||
action: 'recording_start';
|
||||
path: string;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface RecordingStopCommand extends BaseCommand {
|
||||
action: 'recording_stop';
|
||||
}
|
||||
|
||||
export interface RecordingRestartCommand extends BaseCommand {
|
||||
action: 'recording_restart';
|
||||
path: string;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
// Tracing
|
||||
export interface TraceStartCommand extends BaseCommand {
|
||||
action: 'trace_start';
|
||||
@@ -705,6 +778,7 @@ export interface CloseCommand extends BaseCommand {
|
||||
// Tab/Window commands
|
||||
export interface TabNewCommand extends BaseCommand {
|
||||
action: 'tab_new';
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface TabListCommand extends BaseCommand {
|
||||
@@ -789,8 +863,12 @@ export type Command =
|
||||
| IsCheckedCommand
|
||||
| CountCommand
|
||||
| BoundingBoxCommand
|
||||
| StylesCommand
|
||||
| VideoStartCommand
|
||||
| VideoStopCommand
|
||||
| RecordingStartCommand
|
||||
| RecordingStopCommand
|
||||
| RecordingRestartCommand
|
||||
| TraceStartCommand
|
||||
| TraceStopCommand
|
||||
| HarStartCommand
|
||||
@@ -841,7 +919,12 @@ export type Command =
|
||||
| InsertTextCommand
|
||||
| MultiSelectCommand
|
||||
| WaitForDownloadCommand
|
||||
| ResponseBodyCommand;
|
||||
| ResponseBodyCommand
|
||||
| ScreencastStartCommand
|
||||
| ScreencastStopCommand
|
||||
| InputMouseCommand
|
||||
| InputKeyboardCommand
|
||||
| InputTouchCommand;
|
||||
|
||||
// Response types
|
||||
export interface SuccessResponse<T = unknown> {
|
||||
@@ -909,6 +992,60 @@ export interface TabCloseData {
|
||||
remaining: number;
|
||||
}
|
||||
|
||||
export interface ScreencastStartData {
|
||||
started: boolean;
|
||||
format: string;
|
||||
quality: number;
|
||||
}
|
||||
|
||||
export interface ScreencastStopData {
|
||||
stopped: boolean;
|
||||
}
|
||||
|
||||
export interface RecordingStartData {
|
||||
started: boolean;
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface RecordingStopData {
|
||||
path: string;
|
||||
frames: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface RecordingRestartData {
|
||||
started: boolean;
|
||||
path: string;
|
||||
previousPath?: string;
|
||||
stopped: boolean;
|
||||
}
|
||||
|
||||
export interface InputEventData {
|
||||
injected: boolean;
|
||||
}
|
||||
|
||||
// Element styles data
|
||||
export interface ElementStyleInfo {
|
||||
tag: string;
|
||||
text: string | null;
|
||||
box: { x: number; y: number; width: number; height: number };
|
||||
styles: {
|
||||
fontSize: string;
|
||||
fontWeight: string;
|
||||
fontFamily: string;
|
||||
color: string;
|
||||
backgroundColor: string;
|
||||
borderRadius: string;
|
||||
border: string | null;
|
||||
boxShadow: string | null;
|
||||
padding: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface StylesData {
|
||||
elements: ElementStyleInfo[];
|
||||
}
|
||||
|
||||
// Browser state
|
||||
export interface BrowserState {
|
||||
browser: Browser | null;
|
||||
|
||||
Reference in New Issue
Block a user