chore: 更新 Cloudflare 及浏览器自动化攻防文章并补发 blog 链接
This commit is contained in:
@@ -229,6 +229,7 @@ flowchart TD
|
|||||||
- Prefer `--headed` for high-friction targets.
|
- Prefer `--headed` for high-friction targets.
|
||||||
- Reuse session state with one stable `--session-name` for continuity (when omitted, it defaults to `--session`).
|
- Reuse session state with one stable `--session-name` for continuity (when omitted, it defaults to `--session`).
|
||||||
- Keep locale/timezone consistent with target market.
|
- Keep locale/timezone consistent with target market.
|
||||||
|
- For challenge-heavy pages, prefer `--wait-until domcontentloaded` on `open`/`navigate` to avoid `load` stalls.
|
||||||
- Use `--risk-mode block` in strict pipelines that require explicit operator intervention on verification pages.
|
- Use `--risk-mode block` in strict pipelines that require explicit operator intervention on verification pages.
|
||||||
- For `cookies set`, use either `--url <url>`, or `--domain <domain> --path <path>` together.
|
- For `cookies set`, use either `--url <url>`, or `--domain <domain> --path <path>` together.
|
||||||
- If `--url`, `--domain`, and `--path` are all omitted, the cookie is scoped from the current page URL.
|
- If `--url`, `--domain`, and `--path` are all omitted, the cookie is scoped from the current page URL.
|
||||||
@@ -240,11 +241,13 @@ Run public detector checks after stealth changes:
|
|||||||
```bash
|
```bash
|
||||||
node scripts/check-sannysoft-webdriver.js --binary ./cli/target/release/agent-browser
|
node scripts/check-sannysoft-webdriver.js --binary ./cli/target/release/agent-browser
|
||||||
node scripts/check-creepjs-headless.js --binary ./cli/target/release/agent-browser
|
node scripts/check-creepjs-headless.js --binary ./cli/target/release/agent-browser
|
||||||
|
node scripts/check-stealth-regression.js --binary ./cli/target/release/agent-browser
|
||||||
|
pnpm run check:turnstile-testkey
|
||||||
```
|
```
|
||||||
|
|
||||||
## Doctor Diagnostics
|
## Doctor Diagnostics
|
||||||
|
|
||||||
Use `doctor` to quickly diagnose local CDP and tab-group plugin readiness:
|
Use `doctor` to quickly diagnose local CDP, sourceURL sanitization, and tab-group plugin readiness:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
agent-browser doctor
|
agent-browser doctor
|
||||||
@@ -255,6 +258,8 @@ agent-browser --json doctor
|
|||||||
|
|
||||||
- CDP probe status (preferred `:9333` plus common ports)
|
- CDP probe status (preferred `:9333` plus common ports)
|
||||||
- DevToolsActivePort discovery from local Chrome profiles
|
- DevToolsActivePort discovery from local Chrome profiles
|
||||||
|
- CDP Runtime.evaluate sourceURL sanitization probe
|
||||||
|
- Plugin handshake page context check (internal page vs normal `http(s)` page)
|
||||||
- Tab-group extension handshake (when currently attached in CDP mode)
|
- Tab-group extension handshake (when currently attached in CDP mode)
|
||||||
|
|
||||||
## Upstream Compatibility
|
## Upstream Compatibility
|
||||||
|
|||||||
@@ -197,6 +197,22 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if let Some(ref wait_until) = flags.wait_until {
|
||||||
|
if matches!(
|
||||||
|
wait_until.as_str(),
|
||||||
|
"load" | "domcontentloaded" | "networkidle"
|
||||||
|
) {
|
||||||
|
nav_cmd["waitUntil"] = json!(wait_until);
|
||||||
|
} else {
|
||||||
|
return Err(ParseError::InvalidValue {
|
||||||
|
message: format!(
|
||||||
|
"Invalid --wait-until value: {} (expected load, domcontentloaded, or networkidle)",
|
||||||
|
wait_until
|
||||||
|
),
|
||||||
|
usage: "open <url>",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(nav_cmd)
|
Ok(nav_cmd)
|
||||||
}
|
}
|
||||||
"back" => Ok(json!({ "id": id, "action": "back" })),
|
"back" => Ok(json!({ "id": id, "action": "back" })),
|
||||||
@@ -2068,6 +2084,7 @@ mod tests {
|
|||||||
tab_group: None,
|
tab_group: None,
|
||||||
tab_group_plugin_id: None,
|
tab_group_plugin_id: None,
|
||||||
risk_mode: None,
|
risk_mode: None,
|
||||||
|
wait_until: None,
|
||||||
cli_tab_group: false,
|
cli_tab_group: false,
|
||||||
cli_tab_group_plugin_id: false,
|
cli_tab_group_plugin_id: false,
|
||||||
}
|
}
|
||||||
@@ -2349,6 +2366,14 @@ mod tests {
|
|||||||
assert_eq!(cmd["riskMode"], "block");
|
assert_eq!(cmd["riskMode"], "block");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_navigate_with_wait_until() {
|
||||||
|
let mut flags = default_flags();
|
||||||
|
flags.wait_until = Some("domcontentloaded".to_string());
|
||||||
|
let cmd = parse_command(&args("open https://example.com"), &flags).unwrap();
|
||||||
|
assert_eq!(cmd["waitUntil"], "domcontentloaded");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_navigate_with_multiple_headers() {
|
fn test_navigate_with_multiple_headers() {
|
||||||
let mut flags = default_flags();
|
let mut flags = default_flags();
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ pub struct Config {
|
|||||||
pub tab_group: Option<String>,
|
pub tab_group: Option<String>,
|
||||||
pub tab_group_plugin_id: Option<String>,
|
pub tab_group_plugin_id: Option<String>,
|
||||||
pub risk_mode: Option<String>,
|
pub risk_mode: Option<String>,
|
||||||
|
pub wait_until: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Config {
|
impl Config {
|
||||||
@@ -76,6 +77,7 @@ impl Config {
|
|||||||
tab_group: other.tab_group.or(self.tab_group),
|
tab_group: other.tab_group.or(self.tab_group),
|
||||||
tab_group_plugin_id: other.tab_group_plugin_id.or(self.tab_group_plugin_id),
|
tab_group_plugin_id: other.tab_group_plugin_id.or(self.tab_group_plugin_id),
|
||||||
risk_mode: other.risk_mode.or(self.risk_mode),
|
risk_mode: other.risk_mode.or(self.risk_mode),
|
||||||
|
wait_until: other.wait_until.or(self.wait_until),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -145,6 +147,7 @@ fn extract_config_path(args: &[String]) -> Option<Option<String>> {
|
|||||||
"--tab-group",
|
"--tab-group",
|
||||||
"--tab-group-plugin-id",
|
"--tab-group-plugin-id",
|
||||||
"--risk-mode",
|
"--risk-mode",
|
||||||
|
"--wait-until",
|
||||||
];
|
];
|
||||||
let mut i = 0;
|
let mut i = 0;
|
||||||
while i < args.len() {
|
while i < args.len() {
|
||||||
@@ -220,6 +223,9 @@ pub struct Flags {
|
|||||||
/// How verification/captcha detections are handled on navigation:
|
/// How verification/captcha detections are handled on navigation:
|
||||||
/// `off` (disable), `warn` (retry and warn), `block` (fail fast).
|
/// `off` (disable), `warn` (retry and warn), `block` (fail fast).
|
||||||
pub risk_mode: Option<String>,
|
pub risk_mode: Option<String>,
|
||||||
|
/// Navigation wait strategy passed to navigate/open commands:
|
||||||
|
/// `load`, `domcontentloaded`, or `networkidle`.
|
||||||
|
pub wait_until: Option<String>,
|
||||||
|
|
||||||
// Track which launch-time options were explicitly passed via CLI
|
// Track which launch-time options were explicitly passed via CLI
|
||||||
// (as opposed to being set only via environment variables)
|
// (as opposed to being set only via environment variables)
|
||||||
@@ -316,6 +322,7 @@ pub fn parse_flags(args: &[String]) -> Flags {
|
|||||||
.ok()
|
.ok()
|
||||||
.or(config.risk_mode)
|
.or(config.risk_mode)
|
||||||
.map(|s| s.to_ascii_lowercase()),
|
.map(|s| s.to_ascii_lowercase()),
|
||||||
|
wait_until: config.wait_until.map(|s| s.to_ascii_lowercase()),
|
||||||
cli_executable_path: false,
|
cli_executable_path: false,
|
||||||
cli_extensions: false,
|
cli_extensions: false,
|
||||||
cli_state: false,
|
cli_state: false,
|
||||||
@@ -509,6 +516,12 @@ pub fn parse_flags(args: &[String]) -> Flags {
|
|||||||
i += 1;
|
i += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
"--wait-until" => {
|
||||||
|
if let Some(s) = args.get(i + 1) {
|
||||||
|
flags.wait_until = Some(s.to_ascii_lowercase());
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
"--config" => {
|
"--config" => {
|
||||||
// Already handled by load_config(); skip the value
|
// Already handled by load_config(); skip the value
|
||||||
i += 1;
|
i += 1;
|
||||||
@@ -568,6 +581,7 @@ pub fn clean_args(args: &[String]) -> Vec<String> {
|
|||||||
"--tab-group",
|
"--tab-group",
|
||||||
"--tab-group-plugin-id",
|
"--tab-group-plugin-id",
|
||||||
"--risk-mode",
|
"--risk-mode",
|
||||||
|
"--wait-until",
|
||||||
"--config",
|
"--config",
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -918,6 +932,18 @@ mod tests {
|
|||||||
assert_eq!(cleaned, vec!["open", "example.com"]);
|
assert_eq!(cleaned, vec!["open", "example.com"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_wait_until_flag() {
|
||||||
|
let flags = parse_flags(&args("--wait-until domcontentloaded open example.com"));
|
||||||
|
assert_eq!(flags.wait_until.as_deref(), Some("domcontentloaded"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_clean_args_removes_wait_until() {
|
||||||
|
let cleaned = clean_args(&args("--wait-until networkidle open example.com"));
|
||||||
|
assert_eq!(cleaned, vec!["open", "example.com"]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_cli_multiple_flags_tracking() {
|
fn test_cli_multiple_flags_tracking() {
|
||||||
let flags = parse_flags(&args(
|
let flags = parse_flags(&args(
|
||||||
|
|||||||
+8
-3
@@ -874,11 +874,13 @@ Global Options:
|
|||||||
--session <name> Use specific session
|
--session <name> Use specific session
|
||||||
--headers <json> Set HTTP headers (scoped to this origin)
|
--headers <json> Set HTTP headers (scoped to this origin)
|
||||||
--risk-mode <mode> Risk handling for verify/captcha pages: off, warn, block
|
--risk-mode <mode> Risk handling for verify/captcha pages: off, warn, block
|
||||||
|
--wait-until <mode> Navigation wait strategy: load, domcontentloaded, networkidle
|
||||||
--headed Show browser window
|
--headed Show browser window
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
agent-browser open example.com
|
agent-browser open example.com
|
||||||
agent-browser --risk-mode block open example.com
|
agent-browser --risk-mode block open example.com
|
||||||
|
agent-browser --wait-until domcontentloaded open example.com
|
||||||
agent-browser open https://github.com
|
agent-browser open https://github.com
|
||||||
agent-browser open localhost:3000
|
agent-browser open localhost:3000
|
||||||
agent-browser open api.example.com --headers '{"Authorization": "Bearer token"}'
|
agent-browser open api.example.com --headers '{"Authorization": "Bearer token"}'
|
||||||
@@ -2232,19 +2234,21 @@ Examples:
|
|||||||
}
|
}
|
||||||
"doctor" => {
|
"doctor" => {
|
||||||
r##"
|
r##"
|
||||||
agent-browser doctor - Diagnose CDP and tab-group plugin health
|
agent-browser doctor - Diagnose CDP, sourceURL sanitization, and tab-group plugin health
|
||||||
|
|
||||||
Usage: agent-browser doctor
|
Usage: agent-browser doctor
|
||||||
|
|
||||||
Runs a non-destructive health check focused on:
|
Runs a non-destructive health check focused on:
|
||||||
- CDP endpoint reachability (preferred :9333 + common ports)
|
- CDP endpoint reachability (preferred :9333 + common ports)
|
||||||
- DevToolsActivePort discovery from local Chrome profiles
|
- DevToolsActivePort discovery from local Chrome profiles
|
||||||
|
- CDP sourceURL sanitization probe (Runtime.evaluate leakage check)
|
||||||
|
- Plugin handshake page context suitability (internal page vs http(s))
|
||||||
- Tab-group plugin handshake status (when connected via CDP)
|
- Tab-group plugin handshake status (when connected via CDP)
|
||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
- doctor does not accept positional arguments
|
- doctor does not accept positional arguments
|
||||||
- If browser is not already connected, doctor will still report CDP probe results
|
- If browser is not already connected, doctor will still report CDP probe results
|
||||||
- Plugin handshake requires a live CDP page and the extension to be installed
|
- Plugin handshake requires CDP mode, a normal http(s) page, and the extension installed
|
||||||
|
|
||||||
Global Options:
|
Global Options:
|
||||||
--json Output as JSON
|
--json Output as JSON
|
||||||
@@ -2483,7 +2487,7 @@ Sessions:
|
|||||||
Setup:
|
Setup:
|
||||||
install Install browser binaries
|
install Install browser binaries
|
||||||
install --with-deps Also install system dependencies (Linux)
|
install --with-deps Also install system dependencies (Linux)
|
||||||
doctor Diagnose CDP + plugin health
|
doctor Diagnose CDP + sourceURL + plugin health
|
||||||
|
|
||||||
Snapshot Options:
|
Snapshot Options:
|
||||||
-i, --interactive Only interactive elements
|
-i, --interactive Only interactive elements
|
||||||
@@ -2520,6 +2524,7 @@ Options:
|
|||||||
--tab-group <name> Base title for agent tab groups (CDP plugin mode; silent no-op if plugin unavailable)
|
--tab-group <name> Base title for agent tab groups (CDP plugin mode; silent no-op if plugin unavailable)
|
||||||
--tab-group-plugin-id <id> Expected Chrome extension ID for tab-group handshake (or AGENT_BROWSER_TAB_GROUP_PLUGIN_ID)
|
--tab-group-plugin-id <id> Expected Chrome extension ID for tab-group handshake (or AGENT_BROWSER_TAB_GROUP_PLUGIN_ID)
|
||||||
--risk-mode <mode> Verify/captcha handling: off, warn, block (or AGENT_BROWSER_RISK_MODE)
|
--risk-mode <mode> Verify/captcha handling: off, warn, block (or AGENT_BROWSER_RISK_MODE)
|
||||||
|
--wait-until <mode> Navigation wait strategy for open/navigate: load, domcontentloaded, networkidle
|
||||||
--session-name <name> Auto-save/restore session state (defaults to --session)
|
--session-name <name> Auto-save/restore session state (defaults to --session)
|
||||||
--content-boundaries Wrap page output in boundary markers (or AGENT_BROWSER_CONTENT_BOUNDARIES)
|
--content-boundaries Wrap page output in boundary markers (or AGENT_BROWSER_CONTENT_BOUNDARIES)
|
||||||
--max-output <chars> Truncate page output to N chars (or AGENT_BROWSER_MAX_OUTPUT)
|
--max-output <chars> Truncate page output to N chars (or AGENT_BROWSER_MAX_OUTPUT)
|
||||||
|
|||||||
@@ -0,0 +1,272 @@
|
|||||||
|
# PRD: CLI Web 数据采集体验优化(以小红书场景为例)
|
||||||
|
|
||||||
|
- 文档版本: v0.1
|
||||||
|
- 状态: Draft
|
||||||
|
- 作者: Codex
|
||||||
|
- 日期: 2026-03-04
|
||||||
|
|
||||||
|
## 1. 背景与问题
|
||||||
|
|
||||||
|
在使用 `agent-browser` CLI 执行「小红书宠物博主采集(100 条)」时,当前流程可完成任务,但存在明显的可用性与稳定性痛点:
|
||||||
|
|
||||||
|
1. 网络层可观测性不足,响应体抓取不稳定,需注入脚本劫持。
|
||||||
|
2. 分页采集依赖手工 `scroll down + wait`,重复劳动且易漏数据。
|
||||||
|
3. 结构化导出缺少一站式命令,需要 `eval` 二次解析。
|
||||||
|
4. 页面交互依赖文本选择,页面文案变动后脆弱。
|
||||||
|
5. 反爬失败时缺少可解释的自动回退策略。
|
||||||
|
6. 用户对“可抓字段”预期不清(例如搜索接口无联系方式)。
|
||||||
|
7. 长会话缺少快照与断点续抓机制。
|
||||||
|
|
||||||
|
## 2. 目标与非目标
|
||||||
|
|
||||||
|
## 2.1 目标
|
||||||
|
|
||||||
|
1. 将常见采集链路从“脚本拼接”降为“CLI 原生命令组合”。
|
||||||
|
2. 让关键动作具备可观测性(日志)和可恢复性(快照/续跑)。
|
||||||
|
3. 降低站点轻微改版、反爬限制带来的失败率。
|
||||||
|
|
||||||
|
## 2.2 非目标
|
||||||
|
|
||||||
|
1. 不承诺绕过平台强风控或登录体系。
|
||||||
|
2. 不在本期实现完整通用爬虫 DSL。
|
||||||
|
3. 不默认抓取平台未公开展示的隐私字段。
|
||||||
|
|
||||||
|
## 3. 目标用户与核心场景
|
||||||
|
|
||||||
|
1. 增长/运营: 按关键词采集账号基础数据并导出 CSV。
|
||||||
|
2. 测试/研发: 复现抓取问题,定位请求失败原因。
|
||||||
|
3. AI Agent 工作流: 在 CLI 内稳定执行“搜索 -> 翻页 -> 提取 -> 导出”。
|
||||||
|
|
||||||
|
## 4. 需求范围与优先级
|
||||||
|
|
||||||
|
## 4.1 P0
|
||||||
|
|
||||||
|
1. `network capture` 增强模式(可过滤、可落盘 response body)。
|
||||||
|
2. `scroll-collect` 自动滚动采集(按页数或直到无新增)。
|
||||||
|
3. `extract` / `extract-to` 结构化导出(JSON/CSV)。
|
||||||
|
|
||||||
|
## 4.2 P1
|
||||||
|
|
||||||
|
1. 语义选择器与 fallback 链(role/aria/data/text)。
|
||||||
|
2. 401/403/406 智能回退(页面触发 + 回包监听)。
|
||||||
|
3. 可抓字段矩阵与二段式采集文档提示。
|
||||||
|
|
||||||
|
## 4.3 P2
|
||||||
|
|
||||||
|
1. `session snapshot` + `crawl resume` 断点续抓。
|
||||||
|
|
||||||
|
## 5. CLI 方案设计
|
||||||
|
|
||||||
|
## 5.1 网络捕获增强
|
||||||
|
|
||||||
|
命令草案:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
agent-browser network capture --match '/api/sns/web/v1/search/usersearch' --save ./out.ndjson
|
||||||
|
agent-browser network capture --domain edith.xiaohongshu.com --method POST --save ./xhs_usersearch.ndjson
|
||||||
|
```
|
||||||
|
|
||||||
|
参数:
|
||||||
|
|
||||||
|
- `--match <regex>`: 按 URL 正则过滤。
|
||||||
|
- `--domain <host>`: 按域名过滤。
|
||||||
|
- `--method <GET|POST|...>`: 按方法过滤。
|
||||||
|
- `--status <code|range>`: 按状态过滤。
|
||||||
|
- `--save <path>`: NDJSON 输出文件。
|
||||||
|
- `--include-body <request|response|both>`: 控制 body 输出范围。
|
||||||
|
- `--max-body-bytes <n>`: 单条 body 截断阈值。
|
||||||
|
|
||||||
|
NDJSON 记录结构:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"ts": "2026-03-04T10:00:00.123Z",
|
||||||
|
"session_id": "sess_abc",
|
||||||
|
"request_id": "req_123",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "https://edith.xiaohongshu.com/api/sns/web/v1/search/usersearch",
|
||||||
|
"status": 200,
|
||||||
|
"duration_ms": 312,
|
||||||
|
"request_headers": {"content-type": "application/json"},
|
||||||
|
"request_body": "{...}",
|
||||||
|
"response_headers": {"content-type": "application/json"},
|
||||||
|
"response_body": "{...}",
|
||||||
|
"truncated": false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5.2 自动滚动采集
|
||||||
|
|
||||||
|
命令草案:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
agent-browser scroll-collect --until no-new-items --max-steps 200 --idle-rounds 3
|
||||||
|
agent-browser scroll-collect --pages 20 --wait-ms 1200
|
||||||
|
```
|
||||||
|
|
||||||
|
行为:
|
||||||
|
|
||||||
|
1. 每轮执行滚动与等待。
|
||||||
|
2. 基于 DOM 项数量或网络新增请求判断“是否有新增”。
|
||||||
|
3. 达到停止条件后输出结束原因。
|
||||||
|
|
||||||
|
输出示例:
|
||||||
|
|
||||||
|
```text
|
||||||
|
step=1 new_items=15 total_items=15
|
||||||
|
step=2 new_items=15 total_items=30
|
||||||
|
...
|
||||||
|
stop_reason=no-new-items idle_rounds=3 total_items=135
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5.3 结构化提取与导出
|
||||||
|
|
||||||
|
命令草案:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
agent-browser extract --from network --match usersearch --fields 'name,fans,note_count,red_id'
|
||||||
|
agent-browser extract-to --from network --match usersearch --fields 'name,fans,note_count,red_id,url' --format csv --out ./users.csv
|
||||||
|
```
|
||||||
|
|
||||||
|
参数:
|
||||||
|
|
||||||
|
- `--from <network|dom|eval>`: 数据源。
|
||||||
|
- `--match <pattern>`: 来源过滤(URL/事件名)。
|
||||||
|
- `--query <JMESPath|JSONPath>`: 自定义提取表达式。
|
||||||
|
- `--fields <a,b,c>`: 字段映射快捷写法。
|
||||||
|
- `--dedupe-by <field>`: 去重键。
|
||||||
|
- `--limit <n>`: 限制条数。
|
||||||
|
- `--format <json|ndjson|csv>`: 输出格式。
|
||||||
|
- `--out <path>`: 文件输出路径。
|
||||||
|
|
||||||
|
## 5.4 语义选择器与回退链
|
||||||
|
|
||||||
|
命令草案:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
agent-browser click --selector 'role=tab[name="用户"]' --fallback 'aria=用户,text=用户'
|
||||||
|
agent-browser find --selector 'data-testid=user-tab' --fallback 'role=tab[name="用户"],text=用户'
|
||||||
|
```
|
||||||
|
|
||||||
|
策略:
|
||||||
|
|
||||||
|
1. 主选择器失败后按 fallback 顺序重试。
|
||||||
|
2. 日志打印每次尝试与失败原因。
|
||||||
|
|
||||||
|
## 5.5 反爬失败自动回退
|
||||||
|
|
||||||
|
命令草案:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
agent-browser request replay --on-status 401,403,406 --fallback page-action
|
||||||
|
```
|
||||||
|
|
||||||
|
策略:
|
||||||
|
|
||||||
|
1. 直接请求失败后自动回退到页面行为触发。
|
||||||
|
2. 自动复用 UA/Referer/Cookie Jar。
|
||||||
|
3. 捕获最终有效响应并给出“回退成功/失败”日志。
|
||||||
|
|
||||||
|
## 5.6 会话快照与断点续抓
|
||||||
|
|
||||||
|
命令草案:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
agent-browser session snapshot save ./snapshots/xhs-20260304.json
|
||||||
|
agent-browser crawl resume --snapshot ./snapshots/xhs-20260304.json --out ./users.csv
|
||||||
|
```
|
||||||
|
|
||||||
|
快照最小字段:
|
||||||
|
|
||||||
|
- 当前 URL
|
||||||
|
- 关键词/筛选参数
|
||||||
|
- 已抓 user_id 集合摘要(可哈希分片)
|
||||||
|
- 分页进度(page/scroll step)
|
||||||
|
- 导出配置(fields/format/out)
|
||||||
|
|
||||||
|
## 6. 错误码设计(草案)
|
||||||
|
|
||||||
|
- `AB_NET_CAPTURE_BODY_UNAVAILABLE` (1001): 响应体不可用(被浏览器策略阻断或已释放)。
|
||||||
|
- `AB_SCROLL_TIMEOUT_NO_PROGRESS` (1101): 滚动超时且无新增。
|
||||||
|
- `AB_EXTRACT_QUERY_INVALID` (1201): 提取表达式语法错误。
|
||||||
|
- `AB_EXTRACT_OUTPUT_FAILED` (1202): 导出失败(权限/路径不可写)。
|
||||||
|
- `AB_SELECTOR_NOT_FOUND` (1301): 主选择器与 fallback 全部失败。
|
||||||
|
- `AB_REQUEST_BLOCKED_406` (1406): 请求被风控拦截,且回退链路失败。
|
||||||
|
- `AB_RESUME_SNAPSHOT_INVALID` (1501): 快照损坏或版本不兼容。
|
||||||
|
|
||||||
|
要求:
|
||||||
|
|
||||||
|
1. CLI 退出码与错误码可映射。
|
||||||
|
2. 错误输出提供 `hint`(下一步建议命令)。
|
||||||
|
|
||||||
|
## 7. 日志与可观测性
|
||||||
|
|
||||||
|
默认人类可读,开启 `--log-format json` 输出结构化日志。
|
||||||
|
|
||||||
|
JSON 日志字段:
|
||||||
|
|
||||||
|
- `ts`
|
||||||
|
- `level`
|
||||||
|
- `session_id`
|
||||||
|
- `command`
|
||||||
|
- `event`
|
||||||
|
- `step`
|
||||||
|
- `url`
|
||||||
|
- `status`
|
||||||
|
- `error_code`
|
||||||
|
- `message`
|
||||||
|
- `hint`
|
||||||
|
|
||||||
|
示例:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"ts":"2026-03-04T10:11:22.123Z","level":"INFO","command":"scroll-collect","event":"step","step":12,"new_items":15,"total_items":180}
|
||||||
|
{"ts":"2026-03-04T10:13:01.001Z","level":"WARN","command":"request replay","event":"fallback","status":406,"message":"direct request blocked, fallback to page-action"}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 8. 文档与帮助信息更新要求
|
||||||
|
|
||||||
|
当功能落地时,需要同步更新以下位置(按仓库规范):
|
||||||
|
|
||||||
|
1. `cli/src/output.rs`(`--help`、示例、环境变量)
|
||||||
|
2. `README.md`(命令选项、样例)
|
||||||
|
3. `skills/agent-browser/SKILL.md`(Agent 工作流)
|
||||||
|
4. `docs/src/app/`(新增/更新 MDX 页面,表格使用 HTML `<table>`)
|
||||||
|
5. 对应源码内联注释
|
||||||
|
|
||||||
|
## 9. 验收用例(首批)
|
||||||
|
|
||||||
|
1. `network capture` 能稳定保存目标接口完整 request/response body。
|
||||||
|
2. 设置 `--max-body-bytes` 后被截断记录带 `truncated=true`。
|
||||||
|
3. `scroll-collect --pages 5` 精确执行 5 轮并退出。
|
||||||
|
4. `scroll-collect --until no-new-items` 在连续空增量 N 轮后退出。
|
||||||
|
5. `extract-to ... --format csv` 产出可打开 CSV 且列名正确。
|
||||||
|
6. `extract --dedupe-by user_id` 去重结果稳定。
|
||||||
|
7. selector 主规则失败时,fallback 生效并成功点击。
|
||||||
|
8. 对 406 场景触发自动回退并成功捕获有效响应。
|
||||||
|
9. 回退失败时返回 `AB_REQUEST_BLOCKED_406` 且提供 hint。
|
||||||
|
10. `session snapshot save/load` 前后任务可恢复。
|
||||||
|
11. `crawl resume` 不重复导出已抓 ID。
|
||||||
|
12. `--log-format json` 日志字段完整,便于机器消费。
|
||||||
|
|
||||||
|
## 10. 里程碑建议
|
||||||
|
|
||||||
|
1. M1(1 周): `network capture` + `scroll-collect`。
|
||||||
|
2. M2(1 周): `extract-to` + selector fallback。
|
||||||
|
3. M3(1 周): 406 回退链路 + 文档补全。
|
||||||
|
4. M4(1 周): snapshot/resume + 稳定性打磨。
|
||||||
|
|
||||||
|
## 11. 风险与缓解
|
||||||
|
|
||||||
|
1. 平台策略变化导致规则失效。
|
||||||
|
缓解: 增加站点适配层与策略开关,保留回退日志。
|
||||||
|
2. 响应体过大带来内存与 IO 压力。
|
||||||
|
缓解: 流式写入 NDJSON + 截断阈值。
|
||||||
|
3. 通用提取表达式学习成本高。
|
||||||
|
缓解: 提供字段模板与场景 presets。
|
||||||
|
|
||||||
|
## 12. 开放问题
|
||||||
|
|
||||||
|
1. `extract` 表达式标准优先 JSONPath 还是 JMESPath?
|
||||||
|
2. `session snapshot` 是否需要加密(含 cookie 元信息)?
|
||||||
|
3. 是否提供站点模板(如 `preset xiaohongshu-user-search`)以降低上手成本?
|
||||||
@@ -1,189 +1,192 @@
|
|||||||
# 从「能跑」到「长期稳定」:agent-browser-stealth 的攻防工程实践
|
# 浏览器自动化攻防方案设计:检测模型与分层控制面
|
||||||
|
|
||||||
高风控站点对自动化会话的判断,通常不是单一规则命中,而是多信号打分。
|
本文聚焦浏览器自动化的攻防方案设计,按两个部分组织:
|
||||||
要点不在“补一个 patch”,而在“让整组信号在同一会话内自洽”。
|
|
||||||
|
|
||||||
项目地址:[leeguooooo/agent-browser](https://github.com/leeguooooo/agent-browser)
|
1. **原理**:风险评分系统如何形成结论
|
||||||
|
2. **控制面**:如何用分层设计降低风险与波动
|
||||||
|
|
||||||
|
本文不包含命令行操作与工程实现步骤。
|
||||||
|
|
||||||
|
Turnstile 专题内容见:
|
||||||
|
[Cloudflare Turnstile 攻防方案设计:系统原理与控制面](https://blog.misonote.com/zh/posts/cloudflare-turnstile-stability-principles/)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 检测系统如何做判断
|
## 一、原理
|
||||||
|
|
||||||
大多数检测系统会同时看三类问题:
|
### 1.1 风险评分不是单点命中
|
||||||
|
|
||||||
1. **一致性**:UA、语言、时区、渲染能力是否互相匹配
|
高风控站点的“是否挑战/是否降权”通常来自多维评分,而不是某一条规则的二元判断。
|
||||||
2. **稀有性**:是否出现低频但高度可疑的组合(如某些 headless 特征并存)
|
|
||||||
3. **时序性**:输入、点击、等待、重试是否呈现机械节奏
|
|
||||||
|
|
||||||
评估通常是累积分值而非二元判断。
|
主要输入维度:
|
||||||
同一个会话里的轻微异常可以被容忍,但跨维度冲突叠加后,容易触发挑战页或高频二次验证。
|
|
||||||
|
1. **一致性**:同一身份在不同表面是否互相矛盾
|
||||||
|
2. **稀有性**:低频异常组合是否出现
|
||||||
|
3. **时序性**:行为时间序列是否呈机械统计特征
|
||||||
|
4. **执行完整性**:关键链路(挑战脚本、跨域资源、worker)是否被破坏
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
flowchart LR
|
flowchart LR
|
||||||
A["采集信号"] --> B["一致性检查"]
|
A["环境与行为"] --> B["一致性评分"]
|
||||||
A --> C["稀有性评估"]
|
A --> C["稀有性评分"]
|
||||||
A --> D["时序行为评估"]
|
A --> D["时序评分"]
|
||||||
B --> E["风险分值"]
|
A --> E["执行完整性评分"]
|
||||||
C --> E
|
B --> F["综合风险"]
|
||||||
D --> E
|
|
||||||
E --> F{"超过阈值?"}
|
|
||||||
F -->|否| G["继续放行"]
|
|
||||||
F -->|是| H["挑战/验证/限流"]
|
|
||||||
```
|
|
||||||
|
|
||||||
这个模型对应的治理原则很直接:
|
|
||||||
|
|
||||||
- 优先消除跨维度冲突
|
|
||||||
- 再处理低频高危特征
|
|
||||||
- 最后处理行为时序的机械性
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 指纹治理:从“补点”改成“信号闭环”
|
|
||||||
|
|
||||||
指纹治理按 `launch -> CDP -> init script` 三层执行。
|
|
||||||
核心目标是把“可见信号”变成一张一致的画像,而不是局部拟真。
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart LR
|
|
||||||
A["Launch 层"] --> B["CDP 层"]
|
|
||||||
B --> C["Init Script 层"]
|
|
||||||
C --> D["统一画像"]
|
|
||||||
D --> E["降低冲突分值"]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Launch 层(本地启动时的基础面)
|
|
||||||
|
|
||||||
Launch 层处理“浏览器刚启动就暴露”的特征面:
|
|
||||||
|
|
||||||
- `--disable-blink-features=AutomationControlled`
|
|
||||||
- `--use-gl=angle`
|
|
||||||
- `--use-angle=default`
|
|
||||||
- 默认 UA 清洗(未自定义 UA 时去掉 `HeadlessChrome`)
|
|
||||||
|
|
||||||
原理:这层不追求“真实用户画像”,而是先消除明显自动化标识,避免会话在首屏前就进入高风险。
|
|
||||||
|
|
||||||
### CDP 层(运行时协议面)
|
|
||||||
|
|
||||||
CDP 层治理的是“同一身份在不同字段里的自我矛盾”:
|
|
||||||
|
|
||||||
- 同步覆盖 `userAgent`、`acceptLanguage`、`userAgentMetadata`
|
|
||||||
- 覆盖应持续作用于新旧 target
|
|
||||||
- 设置不透明背景,降低透明渲染特征
|
|
||||||
|
|
||||||
典型冲突示例:
|
|
||||||
|
|
||||||
- `userAgent` 显示某平台版本,但 `userAgentMetadata` brand/version 不对应
|
|
||||||
- 语言首选项与请求头不一致
|
|
||||||
|
|
||||||
这类冲突往往比“是否 headless”更早触发评分上升。
|
|
||||||
|
|
||||||
### Init Script 层(页面脚本前)
|
|
||||||
|
|
||||||
Init 层治理页面 JS 可直接探测的运行时表面。
|
|
||||||
重点不是数量,而是覆盖高频检查路径。
|
|
||||||
|
|
||||||
高频治理面:
|
|
||||||
|
|
||||||
- Runtime 身份:`navigator.webdriver`、`chrome.runtime`、`cdc_`
|
|
||||||
- Navigator 能力:`languages/plugins/mimeTypes/permissions/userAgentData`
|
|
||||||
- 渲染能力:WebGL vendor/renderer
|
|
||||||
- 窗口屏幕:`outer*` / `screen*` / `avail*`
|
|
||||||
- 能力暴露:`share/contacts/contentIndex/mediaDevices/pdfViewerEnabled`
|
|
||||||
- 边缘特征:`connection/hardwareConcurrency/performance.memory`
|
|
||||||
|
|
||||||
### 指纹排障优先级
|
|
||||||
|
|
||||||
| 优先级 | 信号面 | 常见现象 | 先做什么 |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| P0 | UA + metadata + language | 首屏挑战页 | 先统一三者,再看其它项 |
|
|
||||||
| P1 | webdriver/runtime 痕迹 | 关键动作前即拦截 | 验证 init 注入是否在页面脚本前生效 |
|
|
||||||
| P2 | WebGL/screen | 间歇性二次验证 | 对齐渲染与窗口参数 |
|
|
||||||
| P3 | connection/memory 等边缘面 | 长链路后段异常 | 增量修复并对照回归 |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 行为治理:让时序分布接近真实交互
|
|
||||||
|
|
||||||
行为检测通常不关心单次点击,而关注一段时间序列。
|
|
||||||
真正会被命中的,是“低方差、强周期、强同步”的机器节奏。
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart TD
|
|
||||||
A["动作计划"] --> B["输入节奏分布"]
|
|
||||||
A --> C["鼠标轨迹分布"]
|
|
||||||
A --> D["等待/思考时间分布"]
|
|
||||||
A --> E["重试退避分布"]
|
|
||||||
B --> F["执行时序"]
|
|
||||||
C --> F
|
C --> F
|
||||||
D --> F
|
D --> F
|
||||||
E --> F
|
E --> F
|
||||||
|
F --> G{"放行/挑战/限流"}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 输入节奏
|
### 1.2 一致性:约束集合而非单点修饰
|
||||||
|
|
||||||
固定字符延迟(例如全程 100ms)很容易形成可分辨模式。
|
一致性问题的本质是“同一身份在多个观测面上的约束必须同时成立”。
|
||||||
更稳妥的做法是“基线 + 抖动 + 语义停顿”:
|
|
||||||
|
|
||||||
- 基线延迟围绕输入场景变化
|
#### 1.2.1 约束集合示意
|
||||||
- 每字符有扰动,不保持等间隔
|
|
||||||
- 词边界、字段切换处出现较长停顿
|
|
||||||
|
|
||||||
### 鼠标轨迹
|
可以把身份一致性建模为“约束图”:
|
||||||
|
|
||||||
坐标瞬移与恒速直线是高风险模式。
|
```mermaid
|
||||||
轨迹应包含:
|
flowchart TD
|
||||||
|
UA["UA 字符串"] --> UACH["UA-CH / userAgentMetadata"]
|
||||||
|
UA --> LangH["Accept-Language"]
|
||||||
|
LangH --> LangJS["navigator.language(s)"]
|
||||||
|
LangJS --> Intl["Intl locale/timeZone"]
|
||||||
|
Plat["platform"] --> Rend["渲染能力/WebGL"]
|
||||||
|
Rend --> Win["窗口/屏幕参数"]
|
||||||
|
UACH --> Plat
|
||||||
|
```
|
||||||
|
|
||||||
- 曲线路径
|
图中每条边表示“两个表面必须相互一致”,否则会形成冲突分值。
|
||||||
- 中间采样点
|
|
||||||
- 速度变化(起步、调整、收敛)
|
|
||||||
|
|
||||||
### 等待与思考时间
|
#### 1.2.2 典型冲突类型
|
||||||
|
|
||||||
固定等待常量会形成明显周期。
|
- UA 显示平台/版本与 UA-CH 不一致
|
||||||
建议使用区间采样,让同类操作在时间上有自然波动。
|
- `Accept-Language` 与 `navigator.languages` 不一致
|
||||||
|
- `Intl` 时区与偏移/地区推断不一致
|
||||||
|
- 设备声明与渲染能力组合异常
|
||||||
|
|
||||||
### 重试退避
|
工程含义:
|
||||||
|
|
||||||
命中风险后继续等间隔重试,通常会放大风险分值。
|
- 修一个点可能打破另一个点
|
||||||
退避策略应具备:
|
- 设计顺序应是“先定约束集合,再决定每个表面如何满足约束”
|
||||||
|
|
||||||
- 间隔递增
|
### 1.3 稀有性:组合风险而非单值风险
|
||||||
- 抖动扰动
|
|
||||||
- 次数上限
|
|
||||||
|
|
||||||
### 行为反模式
|
稀有性来自“低频组合”,其危险性来自共现而非单项。
|
||||||
|
|
||||||
- 全链路固定输入延迟
|
可以将稀有性理解为“联合分布”偏离:
|
||||||
- 点击前无移动直接命中目标
|
|
||||||
- 所有等待都是同一个常量
|
- 单项偏离:可被容忍
|
||||||
- 重试间隔完全一致
|
- 多项共现偏离:风险迅速累积
|
||||||
- 所有站点使用同一动作模板
|
|
||||||
|
工程含义:
|
||||||
|
|
||||||
|
- 目标是减少低频组合在同一会话内叠加
|
||||||
|
- 目标不是拟合某个固定画像
|
||||||
|
|
||||||
|
### 1.4 时序性:统计特征而非行为语义
|
||||||
|
|
||||||
|
行为检测通常关注统计分布特征:
|
||||||
|
|
||||||
|
- 低方差:动作间隔过于稳定
|
||||||
|
- 强周期:间隔呈固定节奏
|
||||||
|
- 强同步:不同类型动作间隔一致
|
||||||
|
|
||||||
|
工程含义:
|
||||||
|
|
||||||
|
- 行为治理的目标是“分布塑形”(variance/jitter/backoff)
|
||||||
|
- 行为治理不是“添加更多动作”
|
||||||
|
|
||||||
|
### 1.5 执行完整性:上游条件
|
||||||
|
|
||||||
|
执行完整性属于“系统是否能正确运行”的前置条件。
|
||||||
|
|
||||||
|
- challenge 脚本、跨域 iframe、跨域 worker 的语义被破坏时,失败率会显著上升
|
||||||
|
- 此类失败可能与“是否被识别”为不同类别的问题
|
||||||
|
|
||||||
|
工程原则:
|
||||||
|
|
||||||
|
> 执行链路保护优先于信号修饰。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2026-03 实战更新:Cloudflare 验证页恢复策略
|
## 二、控制面(分层设计)
|
||||||
|
|
||||||
在真实使用中,`dash.cloudflare.com` 一类站点常见 `Just a moment... / Performing security verification` 挑战页。
|
### 2.1 控制面总览
|
||||||
关键问题不只是“被识别”,还包括“客户端过早刷新把挑战流程重置”,导致长期卡在验证中。
|
|
||||||
|
|
||||||
本次修复的关键点:
|
攻防方案可以拆为四层控制面:
|
||||||
|
|
||||||
1. **风险信号增强**:从 `URL + Title` 扩展为 `URL + Title + PageText`
|
1. **启动控制**:治理启动早期显式风险
|
||||||
覆盖 `Performing security verification`、`This website uses a security service...` 等文本证据。
|
2. **协议控制**:治理协议层身份一致性
|
||||||
2. **恢复策略调整**:`risk-mode=warn` 先等待挑战自动放行,再进入重试
|
3. **运行时控制**:治理页面脚本可观测表面
|
||||||
避免“重试即刷新”打断 Cloudflare 的挑战倒计时。
|
4. **行为与会话控制**:治理时序分布与上下文漂移
|
||||||
3. **会话稳定化**:未显式传 `--session-name` 时,默认跟随 `--session`
|
|
||||||
降低会话漂移造成的重复挑战概率。
|
```mermaid
|
||||||
4. **挑战窗口友好化**:减少过早刷新导致的挑战重置
|
flowchart LR
|
||||||
让验证流程有足够时间自动完成,降低反复卡住的概率。
|
A["启动控制"] --> B["协议控制"]
|
||||||
|
B --> C["运行时控制"]
|
||||||
|
C --> D["行为与会话控制"]
|
||||||
|
D --> E["一致性与稳定性"]
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 启动控制
|
||||||
|
|
||||||
|
目标:降低会话早期显式风险。
|
||||||
|
|
||||||
|
设计约束:
|
||||||
|
|
||||||
|
- 只处理高置信度自动化标识
|
||||||
|
- 避免引入与协议层/运行时层不一致的改动
|
||||||
|
|
||||||
|
### 2.3 协议控制
|
||||||
|
|
||||||
|
目标:将身份约束集合落实到协议层输出。
|
||||||
|
|
||||||
|
设计要点:
|
||||||
|
|
||||||
|
- 将 UA 与 UA-CH 视为同一约束集合的不同投影
|
||||||
|
- 覆盖范围需要与目标(页面/worker/子目标)一致
|
||||||
|
|
||||||
|
### 2.4 运行时控制
|
||||||
|
|
||||||
|
目标:覆盖高频探测面,同时保证不破坏执行语义。
|
||||||
|
|
||||||
|
设计要点:
|
||||||
|
|
||||||
|
- 优先治理高频、可解释的探测路径
|
||||||
|
- 对跨域挑战链路对象设置严格注入边界
|
||||||
|
|
||||||
|
### 2.5 行为与会话控制
|
||||||
|
|
||||||
|
目标:塑形时间分布,减少上下文漂移。
|
||||||
|
|
||||||
|
设计要点:
|
||||||
|
|
||||||
|
- 行为治理以统计分布为目标(variance/jitter/backoff)
|
||||||
|
- 会话治理以一致上下文为目标(避免身份漂移)
|
||||||
|
|
||||||
|
### 2.6 挑战场景控制面摘要(Turnstile)
|
||||||
|
|
||||||
|
Turnstile 场景下的关键控制面可抽象为:
|
||||||
|
|
||||||
|
1. 能力令牌语义:服务端验证、有限时效、单次消费
|
||||||
|
2. 作用域收缩:`hostname/action/cdata` 收缩滥用空间
|
||||||
|
3. 执行链路保护:跨域脚本/iframe/worker 语义保护
|
||||||
|
4. 摩擦与安全分离:clearance 属于体验层,不替代安全决策层
|
||||||
|
|
||||||
|
该摘要用于将 Turnstile 纳入统一控制面框架;细节见专题文章。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 结语
|
## 三、方案设计优先级
|
||||||
|
|
||||||
指纹治理的重点是跨维度一致性。
|
控制面设计通常按以下优先级推进:
|
||||||
行为治理的重点是时间分布去机械化。
|
|
||||||
把这两部分统一治理,通常能显著降低攻防波动。
|
|
||||||
|
|
||||||
项目地址:[leeguooooo/agent-browser](https://github.com/leeguooooo/agent-browser)
|
1. 执行完整性(保证链路可运行)
|
||||||
|
2. 一致性约束集合(消除跨表面矛盾)
|
||||||
|
3. 稀有性控制(避免低频组合叠加)
|
||||||
|
4. 时序分布塑形(降低机械统计特征)
|
||||||
|
5. 体验优化(降低重复挑战摩擦)
|
||||||
|
|
||||||
|
该顺序的含义是先保证“系统正确性”,再优化“稳定性与摩擦”。
|
||||||
|
|||||||
@@ -0,0 +1,250 @@
|
|||||||
|
# Cloudflare Turnstile 攻防方案设计:系统原理与控制面
|
||||||
|
|
||||||
|
本文聚焦 Turnstile 的攻防方案设计:
|
||||||
|
|
||||||
|
1. **系统原理**:token 的安全语义、挑战执行链路、风险评分的输入输出
|
||||||
|
2. **控制面设计**:在不同攻击面下,哪些约束是必要的、哪些约束容易引入副作用
|
||||||
|
|
||||||
|
本文不包含命令行操作与工程实现步骤。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、系统原理
|
||||||
|
|
||||||
|
### 1.1 Turnstile 是“能力令牌”系统
|
||||||
|
|
||||||
|
Turnstile 的本质是签发一个短生命周期、单次消费的能力令牌(capability token)。
|
||||||
|
|
||||||
|
- **签发端**:浏览器端完成挑战执行后获得 token
|
||||||
|
- **消费端**:业务服务端通过 Siteverify 验证 token 并决定是否放行
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
A["浏览器端挑战执行"] --> B["token"]
|
||||||
|
B --> C["业务服务端"]
|
||||||
|
C --> D["Siteverify"]
|
||||||
|
D --> E{"放行/拒绝"}
|
||||||
|
```
|
||||||
|
|
||||||
|
关键含义:
|
||||||
|
|
||||||
|
- 前端任何“通过”状态都不是业务放行条件
|
||||||
|
- 业务放行条件是“token 被正确消费”
|
||||||
|
|
||||||
|
### 1.2 Token 的三条安全语义
|
||||||
|
|
||||||
|
token 的安全语义可以抽象为三条约束:
|
||||||
|
|
||||||
|
1. **必须服务端验证**:不允许仅以前端回调作为依据
|
||||||
|
2. **有限时效**:token 超过时效窗口即失效
|
||||||
|
3. **单次消费**:同一 token 重复消费应失败
|
||||||
|
|
||||||
|
这三条语义分别封装了三个常见攻击目标:
|
||||||
|
|
||||||
|
- 伪通过:绕过服务端验证
|
||||||
|
- 延迟提交:绕过时效窗口
|
||||||
|
- 重放/并发:绕过单次消费
|
||||||
|
|
||||||
|
### 1.3 挑战执行链路是“跨域执行系统”
|
||||||
|
|
||||||
|
Turnstile 的 token 产生依赖多组件协作,且跨域链路占主导:
|
||||||
|
|
||||||
|
- `api.js` 脚本
|
||||||
|
- challenge iframe
|
||||||
|
- challenge worker
|
||||||
|
- 跨域资源请求
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
A["加载 api.js"] --> B["创建 iframe"]
|
||||||
|
B --> C["执行 worker"]
|
||||||
|
C --> D["收集信号 + 风险评估"]
|
||||||
|
D --> E["签发 token"]
|
||||||
|
```
|
||||||
|
|
||||||
|
该链路的工程含义:
|
||||||
|
|
||||||
|
- 任何对跨域脚本/iframe/worker 的语义改写,都可能导致 token 生成失败或质量下降
|
||||||
|
- token 失败不一定意味着“被识别”,也可能是“链路被破坏”
|
||||||
|
|
||||||
|
### 1.4 风险评分:输入不是“真假”,而是“自洽程度”
|
||||||
|
|
||||||
|
挑战执行阶段会收集环境与行为信号,形成风险评分。
|
||||||
|
|
||||||
|
- **信号输入**:环境一致性(UA/UA-CH、语言/时区、渲染能力、能力暴露)
|
||||||
|
- **行为输入**:时序分布(方差、周期性、同步性)
|
||||||
|
|
||||||
|
风险评分的关键不是“拟合某种固定画像”,而是“同一身份在多表面是否自洽”。
|
||||||
|
|
||||||
|
### 1.5 作用域绑定:hostname / action / cdata
|
||||||
|
|
||||||
|
服务端校验时提供用于绑定业务语义的字段:
|
||||||
|
|
||||||
|
- `hostname`:token 允许的站点作用域
|
||||||
|
- `action`:token 允许的动作作用域
|
||||||
|
- `cdata`:token 允许的上下文作用域
|
||||||
|
|
||||||
|
这些字段的作用是“收缩 token 可被滥用的范围”,而不是“提高通过率”。
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
A["token"] --> B["hostname 作用域"]
|
||||||
|
A --> C["action 作用域"]
|
||||||
|
A --> D["cdata 作用域"]
|
||||||
|
B --> E["降低站外盗用收益"]
|
||||||
|
C --> F["降低动作错配收益"]
|
||||||
|
D --> G["降低跨流程重放收益"]
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.6 Token 状态机(能力令牌视角)
|
||||||
|
|
||||||
|
从能力令牌视角,token 生命周期可抽象为:
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
stateDiagram-v2
|
||||||
|
[*] --> Issued: challenge ok
|
||||||
|
Issued --> Consumed: siteverify ok
|
||||||
|
Issued --> Expired: time window
|
||||||
|
Issued --> Rejected: binding mismatch
|
||||||
|
Issued --> Replayed: reused
|
||||||
|
Replayed --> Rejected
|
||||||
|
Expired --> Rejected
|
||||||
|
Consumed --> [*]
|
||||||
|
```
|
||||||
|
|
||||||
|
设计目标是让“非法路径”快速失败,并且失败类型可被服务端语义区分。
|
||||||
|
|
||||||
|
### 1.7 攻击树(高层)
|
||||||
|
|
||||||
|
Turnstile 的主要攻击目标可以抽象为:
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
A["绕过业务动作门禁"] --> B["伪造或跳过服务端验证"]
|
||||||
|
A --> C["重放 token"]
|
||||||
|
A --> D["扩大 token 作用域"]
|
||||||
|
A --> E["破坏挑战执行以制造降级路径"]
|
||||||
|
C --> C1["并发提交"]
|
||||||
|
C --> C2["延迟提交"]
|
||||||
|
D --> D1["Any Hostname"]
|
||||||
|
D --> D2["action/cdata 缺失"]
|
||||||
|
```
|
||||||
|
|
||||||
|
该攻击树强调设计重点:
|
||||||
|
|
||||||
|
- 安全决策必须在服务端闭环
|
||||||
|
- token 必须被作用域收缩并按语义消费
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、控制面设计(攻防视角)
|
||||||
|
|
||||||
|
### 2.1 控制面分层
|
||||||
|
|
||||||
|
Turnstile 防线可以分为四层控制面:
|
||||||
|
|
||||||
|
1. **挑战执行控制**:保证脚本/iframe/worker 跨域链路完整
|
||||||
|
2. **服务端消费控制**:保证 token 的语义被正确消费
|
||||||
|
3. **作用域控制**:收缩 `hostname/action/cdata` 的可用范围
|
||||||
|
4. **摩擦控制**:clearance 用于降低挑战摩擦(不作为安全决策依据)
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
A["挑战执行控制"] --> E["token 可生成"]
|
||||||
|
A --> F["token 质量"]
|
||||||
|
B["服务端消费控制"] --> G["安全决策闭环"]
|
||||||
|
C["作用域控制"] --> H["滥用收益收缩"]
|
||||||
|
D["摩擦控制"] --> I["挑战频率下降"]
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 挑战执行控制:跨域语义保护优先
|
||||||
|
|
||||||
|
挑战执行链路对跨域执行语义高度敏感。
|
||||||
|
|
||||||
|
原则:
|
||||||
|
|
||||||
|
- 跨域脚本/iframe/worker 避免语义改写
|
||||||
|
- 所有指纹修饰必须先满足“不破坏挑战执行”这一硬约束
|
||||||
|
|
||||||
|
该原则的工程含义:
|
||||||
|
|
||||||
|
- “执行完整性”是上游条件
|
||||||
|
- “信号修饰”是下游优化
|
||||||
|
|
||||||
|
### 2.3 服务端消费控制:把 token 当作能力消费
|
||||||
|
|
||||||
|
服务端消费控制的设计关键在于“放行条件定义”,而不是“接口调用细节”。
|
||||||
|
|
||||||
|
放行条件应体现三类约束:
|
||||||
|
|
||||||
|
- 真实性:校验 `success`
|
||||||
|
- 作用域:校验 `hostname`
|
||||||
|
- 语义绑定:校验 `action/cdata`
|
||||||
|
|
||||||
|
并且必须贯彻 token 的两个安全语义:
|
||||||
|
|
||||||
|
- 时效性:过期拒绝
|
||||||
|
- 单次性:重放拒绝
|
||||||
|
|
||||||
|
从攻防角度,该层解决的是“绕过与重放”。
|
||||||
|
|
||||||
|
### 2.4 作用域控制:Hostname Management 与 Any Hostname
|
||||||
|
|
||||||
|
Hostname 管理解决“站外盗用”的攻击面。
|
||||||
|
|
||||||
|
- 启用 Hostname Management:收缩 token 可用站点范围
|
||||||
|
- 启用 Any Hostname:扩大 token 可用站点范围
|
||||||
|
|
||||||
|
设计结论:
|
||||||
|
|
||||||
|
- Any Hostname 不是“更灵活”,而是“扩大攻击面”,必须用更强的服务端约束做补偿控制(来源域白名单 + 业务绑定)。
|
||||||
|
|
||||||
|
### 2.5 摩擦控制:Pre-clearance 与 cf_clearance 的边界
|
||||||
|
|
||||||
|
Pre-clearance 通过后可产生 clearance,用于后续 WAF 挑战联动。
|
||||||
|
|
||||||
|
边界定义:
|
||||||
|
|
||||||
|
- clearance 用于体验层(降低重复挑战摩擦)
|
||||||
|
- Siteverify 用于安全决策层(业务放行依据)
|
||||||
|
|
||||||
|
将两者混用会引入“体验信号替代安全信号”的设计缺陷。
|
||||||
|
|
||||||
|
### 2.6 高对抗场景:代理池与设备关联
|
||||||
|
|
||||||
|
在代理池与分布式滥用场景中,单一 IP 维度约束容易失效。
|
||||||
|
|
||||||
|
设计方向是引入更稳定的关联维度(例如设备级 ephemeral id),用于聚类与阈值策略。
|
||||||
|
|
||||||
|
该层属于平台能力与业务风控的交界:
|
||||||
|
|
||||||
|
- 平台提供关联信号
|
||||||
|
- 业务定义动作分层、阈值与处置策略
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 三、方案设计优先级
|
||||||
|
|
||||||
|
Turnstile 攻防设计通常按以下优先级推进:
|
||||||
|
|
||||||
|
1. 服务端消费语义闭环(真实性 + 作用域 + 绑定 + 单次性 + 时效性)
|
||||||
|
2. 挑战执行链路完整性(跨域语义保护)
|
||||||
|
3. 信号一致性(减少跨字段矛盾)
|
||||||
|
4. 行为时序(降低机械分布)
|
||||||
|
5. 体验优化(clearance 等摩擦控制)
|
||||||
|
|
||||||
|
该顺序的含义是先定义“正确的安全决策”,再优化“挑战摩擦与通过率波动”。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 官方参考(概念与配置)
|
||||||
|
|
||||||
|
- Widgets: <https://developers.cloudflare.com/turnstile/concepts/widget/>
|
||||||
|
- Widget configurations: <https://developers.cloudflare.com/turnstile/get-started/client-side-rendering/widget-configurations/>
|
||||||
|
- Server-side validation: <https://developers.cloudflare.com/turnstile/get-started/server-side-validation/>
|
||||||
|
- CSP: <https://developers.cloudflare.com/turnstile/reference/content-security-policy/>
|
||||||
|
- Hostname management: <https://developers.cloudflare.com/turnstile/additional-configuration/hostname-management/>
|
||||||
|
- Any Hostname: <https://developers.cloudflare.com/turnstile/additional-configuration/hostname-management/any-hostname/>
|
||||||
|
- Pre-clearance: <https://developers.cloudflare.com/turnstile/additional-configuration/hostname-management/pre-clearance/>
|
||||||
|
- Cloudflare clearance: <https://developers.cloudflare.com/cloudflare-challenges/concepts/clearance/>
|
||||||
|
- Ephemeral IDs: <https://developers.cloudflare.com/turnstile/additional-configuration/ephemeral-id/>
|
||||||
@@ -1,105 +1,97 @@
|
|||||||
# 使用 agent-browser-stealth 代替 agent-browser
|
# agent-browser 与 agent-browser-stealth:能力差异与选型
|
||||||
|
|
||||||
很多人已经把 AI Agent 接入了浏览器自动化,但上线后会遇到同一个问题:
|
本文给出 `agent-browser` 与 `agent-browser-stealth` 的技术差异、适用场景和升级验证步骤。
|
||||||
|
|
||||||
同样流程在不同网站表现不一致;流程写完了,关键站点还是过不去。
|
|
||||||
|
|
||||||
问题通常不在“会不会自动化”,而在“能不能在真实网站里稳定自动化”。
|
|
||||||
|
|
||||||
这就是为什么要从 `agent-browser` 升级到 `agent-browser-stealth`。
|
|
||||||
|
|
||||||
项目地址:[leeguooooo/agent-browser](https://github.com/leeguooooo/agent-browser)
|
项目地址:[leeguooooo/agent-browser](https://github.com/leeguooooo/agent-browser)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 为什么要替换
|
## 1. 定位差异
|
||||||
|
|
||||||
### 1) 让 AI 真正可以使用浏览器
|
- `agent-browser`:标准浏览器自动化能力
|
||||||
|
- `agent-browser-stealth`:在标准自动化能力基础上,增加反检测与高风控场景稳定性能力
|
||||||
很多网站已经有反爬和自动化检测策略。
|
|
||||||
在这些场景里,传统自动化链路会出现高频验证、中断、重试失败,最终让 AI 任务卡在关键步骤。
|
|
||||||
|
|
||||||
`agent-browser-stealth` 的目标很明确:让 AI 在真实网站环境中保持更高可用性。
|
|
||||||
|
|
||||||
### 2) 应对“限制 AI 浏览器”的站点策略
|
|
||||||
|
|
||||||
部分站点会对自动化浏览器做额外限制,包括:
|
|
||||||
|
|
||||||
- 触发挑战页
|
|
||||||
- 关键页面二次验证
|
|
||||||
- 会话中途降权或限流
|
|
||||||
|
|
||||||
`agent-browser-stealth` 提供更完整的防识别能力,降低这类限制对任务成功率的影响。
|
|
||||||
|
|
||||||
### 3) 让 Agent 和用户共享同一个浏览器
|
|
||||||
|
|
||||||
很多自动化失败发生在“登录前后状态切换”环节。
|
|
||||||
`agent-browser-stealth` 支持 Agent 复用用户正在使用的浏览器会话,直接继承已登录状态,减少重复登录和验证码干扰。
|
|
||||||
|
|
||||||
对业务流程的价值是直接的:
|
|
||||||
|
|
||||||
- 缩短执行路径
|
|
||||||
- 降低登录步骤失败率
|
|
||||||
- 提升整体成功率与执行速度
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 典型站点效果(示例)
|
## 2. 核心能力对比
|
||||||
|
|
||||||
以亚马逊这类高风控电商站点为例,很多 AI 浏览器流程过去常见的问题是:
|
|
||||||
|
|
||||||
- 能打开首页,但关键操作前触发验证
|
|
||||||
- 搜索、跳转、加购这类连续动作中途被打断
|
|
||||||
- 会话偶发失效,任务难以完整执行
|
|
||||||
|
|
||||||
切换到 `agent-browser-stealth` 后,可显著提升这类流程的可执行性,常见可完成动作包括:
|
|
||||||
|
|
||||||
- 商品搜索与详情浏览
|
|
||||||
- 购物车相关操作
|
|
||||||
- 已登录状态下的页面导航与信息读取
|
|
||||||
|
|
||||||
除了电商站点,下面两类场景也常见明显改善:
|
|
||||||
|
|
||||||
- 社媒/内容平台:多步骤跳转流程更稳定
|
|
||||||
- SaaS 后台系统:登录后连续操作中断率降低
|
|
||||||
|
|
||||||
说明:不同账号状态、网络环境、站点实时策略会影响最终效果。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 适合哪些场景
|
|
||||||
|
|
||||||
- AI 客服或运营 Agent 需要在多站点执行后台操作
|
|
||||||
- 自动化流程经常卡在登录、验证、跳转环节
|
|
||||||
- 需要“人机协同”:用户和 Agent 共用一个会话处理复杂任务
|
|
||||||
- 对稳定性要求高的生产任务(不是 Demo)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 迁移成本高吗
|
|
||||||
|
|
||||||
迁移成本通常很低,命令习惯可以保持一致。
|
|
||||||
多数场景可以先做“无侵入替换”,再按业务流程逐步优化。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 一张表看差异
|
|
||||||
|
|
||||||
| 维度 | agent-browser | agent-browser-stealth |
|
| 维度 | agent-browser | agent-browser-stealth |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| 目标 | 标准自动化能力 | 面向真实风控环境的稳定自动化 |
|
| 自动化基础能力 | 支持 | 支持 |
|
||||||
| 站点兼容性 | 普通站点可用 | 高风控站点可用性更高 |
|
| 指纹一致性治理 | 基础 | 多层(launch/CDP/init-script) |
|
||||||
| AI 执行稳定性 | 受验证页影响明显 | 对验证/限制策略更稳 |
|
| 高风控站点稳定性 | 一般 | 更高 |
|
||||||
| 登录链路 | 常需重复处理登录步骤 | 支持复用用户浏览器状态,减少登录干扰 |
|
| 会话连续性(附着现有浏览器) | 支持 | 支持,默认附着策略更明确 |
|
||||||
| 生产可用性 | 适合基础自动化 | 更适合生产级 AI 浏览器任务 |
|
| Cloudflare/Turnstile 回归工具 | 无专用脚本 | `check:turnstile-testkey` |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 结论
|
## 3. Cloudflare/Turnstile 相关能力(v0.15.2-fork.2+)
|
||||||
|
|
||||||
如果目标是“让 AI 在真实网站里稳定完成任务”,`agent-browser-stealth` 是更合适的选择。
|
### 3.1 挑战链路保护
|
||||||
如果目标只是“脚本在理想环境跑通”,`agent-browser` 已经足够。
|
|
||||||
|
|
||||||
在生产环境里,真正的差异通常体现在四个字:**可用与稳定**。
|
- 同源 worker 注入保留
|
||||||
|
- 跨域 challenge worker 不做注入改写
|
||||||
|
- 降低 challenge worker 执行异常概率
|
||||||
|
|
||||||
|
### 3.2 导航等待策略
|
||||||
|
|
||||||
|
`open/navigate` 支持:
|
||||||
|
|
||||||
|
- `--wait-until load`
|
||||||
|
- `--wait-until domcontentloaded`
|
||||||
|
- `--wait-until networkidle`
|
||||||
|
|
||||||
|
挑战页建议优先 `domcontentloaded`,减少 `load` 阶段超时误判。
|
||||||
|
|
||||||
|
### 3.3 确定性回归
|
||||||
|
|
||||||
|
提供官方 test key 回归脚本:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm run check:turnstile-testkey
|
||||||
|
```
|
||||||
|
|
||||||
|
通过特征:输出 `XXXX.DUMMY.TOKEN.XXXX`。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 适用场景
|
||||||
|
|
||||||
|
优先使用 `agent-browser-stealth` 的场景:
|
||||||
|
|
||||||
|
1. 目标站点存在挑战页/验证码/限流
|
||||||
|
2. 自动化链路对稳定性要求高
|
||||||
|
3. 需要长期回归验证与版本门禁
|
||||||
|
|
||||||
|
使用 `agent-browser` 的场景:
|
||||||
|
|
||||||
|
1. 低风控站点
|
||||||
|
2. 以基础自动化能力验证为主
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 升级验证步骤
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1) 检查版本
|
||||||
|
agent-browser -V
|
||||||
|
|
||||||
|
# 2) 关闭旧 daemon,避免版本漂移
|
||||||
|
agent-browser --session default close
|
||||||
|
|
||||||
|
# 3) 运行确定性回归
|
||||||
|
pnpm run check:turnstile-testkey
|
||||||
|
|
||||||
|
# 4) 可选:真实站点回归
|
||||||
|
agent-browser --wait-until domcontentloaded open https://www.anyviewer.com/cloudflare.html
|
||||||
|
```
|
||||||
|
|
||||||
|
如果启用域名白名单(`AGENT_BROWSER_ALLOWED_DOMAINS`),需包含 `challenges.cloudflare.com`。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 结论
|
||||||
|
|
||||||
|
`agent-browser-stealth` 适用于高风控与稳定性敏感场景;`agent-browser` 适用于标准自动化场景。
|
||||||
|
选型建议按目标站点风控强度与回归要求决定。
|
||||||
|
|
||||||
项目地址:[leeguooooo/agent-browser](https://github.com/leeguooooo/agent-browser)
|
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ agent-browser pdf <path> # Save page as PDF
|
|||||||
agent-browser snapshot # Accessibility tree with refs
|
agent-browser snapshot # Accessibility tree with refs
|
||||||
agent-browser eval <js> # Run JavaScript
|
agent-browser eval <js> # Run JavaScript
|
||||||
agent-browser connect <port|url> # Connect to browser via CDP
|
agent-browser connect <port|url> # Connect to browser via CDP
|
||||||
agent-browser doctor # Diagnose CDP + tab-group plugin health
|
agent-browser doctor # Diagnose CDP + sourceURL + tab-group plugin health
|
||||||
agent-browser --version # Show CLI version
|
agent-browser --version # Show CLI version
|
||||||
agent-browser close # Close browser (aliases: quit, exit)
|
agent-browser close # Close browser (aliases: quit, exit)
|
||||||
```
|
```
|
||||||
@@ -253,7 +253,8 @@ agent-browser console --clear # Clear console log
|
|||||||
agent-browser errors # View page errors
|
agent-browser errors # View page errors
|
||||||
agent-browser errors --clear # Clear error log
|
agent-browser errors --clear # Clear error log
|
||||||
agent-browser highlight <sel> # Highlight element
|
agent-browser highlight <sel> # Highlight element
|
||||||
agent-browser doctor # Diagnose CDP + plugin handshake status
|
agent-browser doctor # Diagnose CDP + sourceURL + plugin handshake status
|
||||||
|
pnpm run check:turnstile-testkey # Deterministic Turnstile smoke check (official test key)
|
||||||
```
|
```
|
||||||
|
|
||||||
## State management
|
## State management
|
||||||
@@ -310,6 +311,7 @@ agent-browser reload # Reload page
|
|||||||
--auto-connect # Auto-discover and connect to running Chrome
|
--auto-connect # Auto-discover and connect to running Chrome
|
||||||
--tab-group <name> # Base title for agent tab groups (CDP plugin mode)
|
--tab-group <name> # Base title for agent tab groups (CDP plugin mode)
|
||||||
--tab-group-plugin-id <id> # Expected extension ID for tab-group handshake
|
--tab-group-plugin-id <id> # Expected extension ID for tab-group handshake
|
||||||
|
--wait-until <mode> # Navigation wait strategy for open/navigate (load, domcontentloaded, networkidle)
|
||||||
--debug # Debug output (includes stealth connection type + capabilities)
|
--debug # Debug output (includes stealth connection type + capabilities)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -36,6 +36,8 @@
|
|||||||
"test:watch": "vitest",
|
"test:watch": "vitest",
|
||||||
"test:e2e:dogfood": "vitest run test/e2e/dogfood.eval.ts",
|
"test:e2e:dogfood": "vitest run test/e2e/dogfood.eval.ts",
|
||||||
"check:daemon-pid-recovery": "node scripts/check-daemon-pid-recovery.js",
|
"check:daemon-pid-recovery": "node scripts/check-daemon-pid-recovery.js",
|
||||||
|
"check:stealth-regression": "node scripts/check-stealth-regression.js",
|
||||||
|
"check:turnstile-testkey": "pnpm exec tsx scripts/check-turnstile-testkey.ts",
|
||||||
"postinstall": "node scripts/postinstall.js",
|
"postinstall": "node scripts/postinstall.js",
|
||||||
"verify:native-version": "node scripts/verify-native-version.js",
|
"verify:native-version": "node scripts/verify-native-version.js",
|
||||||
"clawhub:sync": "bash scripts/clawhub-sync.sh",
|
"clawhub:sync": "bash scripts/clawhub-sync.sh",
|
||||||
|
|||||||
Executable
+199
@@ -0,0 +1,199 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
/**
|
||||||
|
* End-to-end stealth regression check across key anti-bot targets.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* node scripts/check-stealth-regression.js
|
||||||
|
* node scripts/check-stealth-regression.js --binary ./cli/target/release/agent-browser
|
||||||
|
* node scripts/check-stealth-regression.js --session-name stealth-regression
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
import { existsSync, mkdirSync } from 'node:fs';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const rootDir = join(__dirname, '..');
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const getArgValue = (name, fallback) => {
|
||||||
|
const index = args.indexOf(name);
|
||||||
|
if (index === -1 || index + 1 >= args.length) return fallback;
|
||||||
|
return args[index + 1];
|
||||||
|
};
|
||||||
|
|
||||||
|
const sessionName = getArgValue('--session-name', 'stealth-regression');
|
||||||
|
const screenshotDir = getArgValue('--screenshot-dir', join('/tmp', 'agent-browser-stealth-regression'));
|
||||||
|
const binaryArg = getArgValue('--binary', '');
|
||||||
|
|
||||||
|
const candidates = [
|
||||||
|
binaryArg,
|
||||||
|
join(rootDir, 'cli', 'target', 'release', 'agent-browser'),
|
||||||
|
join(rootDir, 'bin', 'agent-browser.js'),
|
||||||
|
'agent-browser-stealth',
|
||||||
|
'agent-browser',
|
||||||
|
].filter(Boolean);
|
||||||
|
|
||||||
|
function tryResolveBinary() {
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (candidate.includes('/') && !existsSync(candidate)) continue;
|
||||||
|
const probe = spawnSync(candidate, ['--version'], { encoding: 'utf8' });
|
||||||
|
if (probe.status === 0) return candidate;
|
||||||
|
}
|
||||||
|
throw new Error(
|
||||||
|
`Unable to find a runnable agent-browser binary. Tried: ${candidates.join(', ')}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const binary = tryResolveBinary();
|
||||||
|
const sessionArgs = ['--session', sessionName, '--session-name', sessionName];
|
||||||
|
|
||||||
|
function runBinary(commandArgs, { allowFailure = false } = {}) {
|
||||||
|
const result = spawnSync(binary, commandArgs, {
|
||||||
|
encoding: 'utf8',
|
||||||
|
maxBuffer: 10 * 1024 * 1024,
|
||||||
|
});
|
||||||
|
if (result.status !== 0 && !allowFailure) {
|
||||||
|
const stderr = (result.stderr || '').trim();
|
||||||
|
const stdout = (result.stdout || '').trim();
|
||||||
|
throw new Error(
|
||||||
|
`Command failed: ${binary} ${commandArgs.join(' ')}\n` +
|
||||||
|
`${stderr || stdout || `exit code ${result.status}`}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function runJson(actionArgs, options = {}) {
|
||||||
|
const result = runBinary([...sessionArgs, '--json', ...actionArgs], options);
|
||||||
|
const output = (result.stdout || '').trim();
|
||||||
|
if (!output) return null;
|
||||||
|
try {
|
||||||
|
return JSON.parse(output);
|
||||||
|
} catch {
|
||||||
|
throw new Error(`Expected JSON output, got:\n${output}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const genericRiskScript = `(() => {
|
||||||
|
const lowerTitle = String(document.title || '').toLowerCase();
|
||||||
|
const lowerBody = String(document.body?.innerText || '').toLowerCase();
|
||||||
|
const hasCloudflare =
|
||||||
|
lowerTitle.includes('just a moment') ||
|
||||||
|
lowerTitle.includes('performing security verification') ||
|
||||||
|
lowerBody.includes('performing security verification') ||
|
||||||
|
lowerBody.includes('checking your browser') ||
|
||||||
|
lowerBody.includes('cloudflare');
|
||||||
|
const hasCaptcha =
|
||||||
|
lowerBody.includes('captcha') ||
|
||||||
|
lowerBody.includes('recaptcha') ||
|
||||||
|
lowerBody.includes('hcaptcha') ||
|
||||||
|
lowerBody.includes('turnstile');
|
||||||
|
return {
|
||||||
|
title: document.title || '',
|
||||||
|
url: location.href,
|
||||||
|
hasCloudflare,
|
||||||
|
hasCaptcha,
|
||||||
|
hasTurnstile:
|
||||||
|
!!document.querySelector('.cf-turnstile, iframe[src*="challenges.cloudflare.com"], [name="cf-turnstile-response"]'),
|
||||||
|
bodySample: lowerBody.slice(0, 600),
|
||||||
|
};
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
const sannysoftScript = `(() => {
|
||||||
|
const normalize = (s) => String(s || '').replace(/\\s+/g, ' ').trim();
|
||||||
|
const rows = Array.from(document.querySelectorAll('tr'));
|
||||||
|
const failed = rows.filter((row) => /failed|fail/i.test(normalize(row.innerText)));
|
||||||
|
return {
|
||||||
|
failedCount: failed.length,
|
||||||
|
failedRows: failed.map((row) => normalize(row.innerText)),
|
||||||
|
navigatorWebdriver: navigator.webdriver,
|
||||||
|
navigatorVendor: navigator.vendor,
|
||||||
|
};
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
function sanitizeFileSegment(input) {
|
||||||
|
return input.replace(/[^a-zA-Z0-9._-]+/g, '-');
|
||||||
|
}
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
mkdirSync(screenshotDir, { recursive: true });
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
const targets = [
|
||||||
|
'https://bot.sannysoft.com/',
|
||||||
|
'https://chatgpt.com/',
|
||||||
|
'https://super86.cc/login',
|
||||||
|
];
|
||||||
|
|
||||||
|
runBinary([...sessionArgs, 'close'], { allowFailure: true });
|
||||||
|
|
||||||
|
const report = {
|
||||||
|
binary,
|
||||||
|
sessionName,
|
||||||
|
timestamp,
|
||||||
|
screenshotDir,
|
||||||
|
doctor: null,
|
||||||
|
targets: [],
|
||||||
|
ok: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const doctorResp = runJson(['doctor']);
|
||||||
|
report.doctor = doctorResp?.data ?? null;
|
||||||
|
|
||||||
|
for (const target of targets) {
|
||||||
|
const entry = {
|
||||||
|
target,
|
||||||
|
open: null,
|
||||||
|
risk: null,
|
||||||
|
sannysoft: null,
|
||||||
|
screenshot: null,
|
||||||
|
ok: true,
|
||||||
|
error: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const openResp = runJson(['open', target]);
|
||||||
|
entry.open = openResp?.data ?? null;
|
||||||
|
|
||||||
|
runJson(['wait', '3000'], { allowFailure: true });
|
||||||
|
|
||||||
|
const riskResp = runJson(['eval', genericRiskScript]);
|
||||||
|
entry.risk = riskResp?.data?.result ?? null;
|
||||||
|
|
||||||
|
if (target.includes('bot.sannysoft.com')) {
|
||||||
|
const sannysoftResp = runJson(['eval', sannysoftScript]);
|
||||||
|
entry.sannysoft = sannysoftResp?.data?.result ?? null;
|
||||||
|
if ((entry.sannysoft?.failedCount ?? 1) > 0) {
|
||||||
|
entry.ok = false;
|
||||||
|
}
|
||||||
|
} else if (entry.risk?.hasCloudflare || entry.risk?.hasCaptcha) {
|
||||||
|
entry.ok = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const host = sanitizeFileSegment(new URL(target).host);
|
||||||
|
const shotPath = join(
|
||||||
|
screenshotDir,
|
||||||
|
`${host}-${Date.now().toString(36)}.png`
|
||||||
|
);
|
||||||
|
runJson(['screenshot', '--full', shotPath], { allowFailure: true });
|
||||||
|
entry.screenshot = shotPath;
|
||||||
|
} catch (error) {
|
||||||
|
entry.ok = false;
|
||||||
|
entry.error = error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!entry.ok) report.ok = false;
|
||||||
|
report.targets.push(entry);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
runBinary([...sessionArgs, 'close'], { allowFailure: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(JSON.stringify(report, null, 2));
|
||||||
|
process.exit(report.ok ? 0 : 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
#!/usr/bin/env tsx
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deterministic Turnstile check using Cloudflare official testing sitekey.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* pnpm run check:turnstile-testkey
|
||||||
|
* pnpm run check:turnstile-testkey -- --headed
|
||||||
|
*/
|
||||||
|
|
||||||
|
import http from 'node:http';
|
||||||
|
import { BrowserManager } from '../src/browser.js';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const headed = args.includes('--headed');
|
||||||
|
const waitMsRaw = args.includes('--wait-ms')
|
||||||
|
? args[args.indexOf('--wait-ms') + 1]
|
||||||
|
: undefined;
|
||||||
|
const waitMs = Number.isFinite(Number(waitMsRaw)) ? Number(waitMsRaw) : 9000;
|
||||||
|
|
||||||
|
const html = `<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>agent-browser turnstile testkey</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Turnstile Testkey Probe</h1>
|
||||||
|
<div class="cf-turnstile" data-sitekey="1x00000000000000000000AA" data-callback="onTurnstileToken"></div>
|
||||||
|
<script>
|
||||||
|
window.__turnstileToken = '';
|
||||||
|
function onTurnstileToken(token) {
|
||||||
|
window.__turnstileToken = token || '';
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script>
|
||||||
|
</body>
|
||||||
|
</html>`;
|
||||||
|
|
||||||
|
function createServer(): Promise<http.Server> {
|
||||||
|
const server = http.createServer((_, res) => {
|
||||||
|
res.writeHead(200, {
|
||||||
|
'Content-Type': 'text/html; charset=utf-8',
|
||||||
|
'Cache-Control': 'no-store',
|
||||||
|
});
|
||||||
|
res.end(html);
|
||||||
|
});
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
server.on('error', reject);
|
||||||
|
server.listen(0, '127.0.0.1', () => resolve(server));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeServer(server: http.Server): Promise<void> {
|
||||||
|
return new Promise((resolve) => server.close(() => resolve()));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main(): Promise<void> {
|
||||||
|
const server = await createServer();
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === 'string') {
|
||||||
|
await closeServer(server);
|
||||||
|
throw new Error('Unable to start local HTTP server for Turnstile probe');
|
||||||
|
}
|
||||||
|
|
||||||
|
const localUrl = `http://127.0.0.1:${address.port}/`;
|
||||||
|
const browser = new BrowserManager();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await browser.launch({
|
||||||
|
id: 'turnstile-testkey',
|
||||||
|
action: 'launch',
|
||||||
|
browser: 'chromium',
|
||||||
|
stealth: true,
|
||||||
|
headless: !headed,
|
||||||
|
});
|
||||||
|
|
||||||
|
const page = browser.getPage();
|
||||||
|
await page.goto(localUrl, { waitUntil: 'domcontentloaded', timeout: 45_000 });
|
||||||
|
await page.waitForTimeout(waitMs);
|
||||||
|
|
||||||
|
const result = await page.evaluate(() => {
|
||||||
|
const hidden = document.querySelector('input[name="cf-turnstile-response"]') as
|
||||||
|
| HTMLInputElement
|
||||||
|
| null;
|
||||||
|
const hiddenValue = hidden?.value || '';
|
||||||
|
const callbackValue =
|
||||||
|
typeof (window as any).__turnstileToken === 'string'
|
||||||
|
? (window as any).__turnstileToken
|
||||||
|
: '';
|
||||||
|
const token = hiddenValue || callbackValue || '';
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: location.href,
|
||||||
|
title: document.title || '',
|
||||||
|
tokenLength: token.length,
|
||||||
|
tokenSample: token.slice(0, 40),
|
||||||
|
isDummyToken: token.includes('DUMMY'),
|
||||||
|
hiddenFieldLength: hiddenValue.length,
|
||||||
|
callbackLength: callbackValue.length,
|
||||||
|
widgetCount: document.querySelectorAll('.cf-turnstile').length,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const report = {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
headed,
|
||||||
|
waitMs,
|
||||||
|
ok: result.isDummyToken,
|
||||||
|
result,
|
||||||
|
};
|
||||||
|
|
||||||
|
console.log(JSON.stringify(report, null, 2));
|
||||||
|
process.exit(result.isDummyToken ? 0 : 1);
|
||||||
|
} finally {
|
||||||
|
await browser.close().catch(() => {});
|
||||||
|
await closeServer(server);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error(error instanceof Error ? error.message : String(error));
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -52,7 +52,7 @@ agent-browser open https://example.com && agent-browser wait --load networkidle
|
|||||||
# Navigation
|
# Navigation
|
||||||
agent-browser open <url> # Navigate (aliases: goto, navigate)
|
agent-browser open <url> # Navigate (aliases: goto, navigate)
|
||||||
agent-browser --risk-mode block open <url> # Block if verification/captcha interstitial is detected
|
agent-browser --risk-mode block open <url> # Block if verification/captcha interstitial is detected
|
||||||
agent-browser doctor # Diagnose CDP + tab-group plugin health
|
agent-browser doctor # Diagnose CDP + sourceURL + tab-group plugin health
|
||||||
agent-browser close # Close browser
|
agent-browser close # Close browser
|
||||||
agent-browser --version # Show CLI version (fork builds include upstream/fork)
|
agent-browser --version # Show CLI version (fork builds include upstream/fork)
|
||||||
|
|
||||||
@@ -236,8 +236,14 @@ agent-browser --cdp 9222 snapshot
|
|||||||
# Debug auto-attach behavior
|
# Debug auto-attach behavior
|
||||||
agent-browser --debug snapshot
|
agent-browser --debug snapshot
|
||||||
|
|
||||||
# Diagnose CDP + plugin handshake status
|
# Diagnose CDP + sourceURL + plugin handshake status
|
||||||
agent-browser doctor
|
agent-browser doctor
|
||||||
|
|
||||||
|
# Avoid `open` timeout on challenge-heavy pages
|
||||||
|
agent-browser --wait-until domcontentloaded open https://example.com
|
||||||
|
|
||||||
|
# Deterministic Turnstile smoke check (official test key)
|
||||||
|
pnpm run check:turnstile-testkey
|
||||||
```
|
```
|
||||||
|
|
||||||
### Color Scheme (Dark Mode)
|
### Color Scheme (Dark Mode)
|
||||||
|
|||||||
+86
-2
@@ -2643,6 +2643,54 @@ export class BrowserManager {
|
|||||||
checks.push({ name, status, message, ...(details ? { details } : {}) });
|
checks.push({ name, status, message, ...(details ? { details } : {}) });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Probe whether CDP Runtime.evaluate responses still leak automation-only
|
||||||
|
* sourceURL labels such as `__playwright_evaluation_script__`.
|
||||||
|
*/
|
||||||
|
private async runDoctorSourceUrlProbe(checks: DoctorCheck[], launched: boolean): Promise<void> {
|
||||||
|
if (!launched) {
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'cdp:sourceurl-sanitized',
|
||||||
|
'skip',
|
||||||
|
'Browser is not launched; sourceURL probe skipped'
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cdp = await this.getCDPSession();
|
||||||
|
const response = await cdp.send('Runtime.evaluate', {
|
||||||
|
expression:
|
||||||
|
"(() => { throw new Error('doctor-sourceurl'); })()\\n//# sourceURL=__playwright_evaluation_script__",
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
const raw = JSON.stringify(response);
|
||||||
|
const leakedMarkers = [
|
||||||
|
'__playwright_evaluation_script__',
|
||||||
|
'__puppeteer_evaluation_script__',
|
||||||
|
'sourceURL=',
|
||||||
|
].filter((marker) => raw.includes(marker));
|
||||||
|
const leaked = leakedMarkers.length > 0;
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'cdp:sourceurl-sanitized',
|
||||||
|
leaked ? 'fail' : 'pass',
|
||||||
|
leaked
|
||||||
|
? 'CDP Runtime.evaluate response still exposes automation sourceURL markers'
|
||||||
|
: 'CDP Runtime.evaluate response is sourceURL-sanitized',
|
||||||
|
leaked ? { leakedMarkers } : undefined
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'cdp:sourceurl-sanitized',
|
||||||
|
'warn',
|
||||||
|
`Unable to run Runtime.evaluate sourceURL probe: ${error instanceof Error ? error.message : String(error)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private buildDoctorTabGroupIntent(): TabGroupIntent {
|
private buildDoctorTabGroupIntent(): TabGroupIntent {
|
||||||
const session = this.getAgentSessionName();
|
const session = this.getAgentSessionName();
|
||||||
const pluginId =
|
const pluginId =
|
||||||
@@ -2665,11 +2713,13 @@ export class BrowserManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run connection diagnostics for CDP discovery and tab-group plugin handshake.
|
* Run connection diagnostics for CDP discovery, sourceURL sanitization, and
|
||||||
|
* tab-group plugin readiness/handshake.
|
||||||
* This is intentionally side-effect-light: it does not navigate or force launch.
|
* This is intentionally side-effect-light: it does not navigate or force launch.
|
||||||
*/
|
*/
|
||||||
async runDoctor(): Promise<DoctorData> {
|
async runDoctor(): Promise<DoctorData> {
|
||||||
const checks: DoctorCheck[] = [];
|
const checks: DoctorCheck[] = [];
|
||||||
|
const launched = this.isLaunched();
|
||||||
const preferredPort = 9333;
|
const preferredPort = 9333;
|
||||||
const discovered: DoctorData['cdp']['discovered'] = [];
|
const discovered: DoctorData['cdp']['discovered'] = [];
|
||||||
const devToolsActivePort: DoctorData['cdp']['devToolsActivePort'] = [];
|
const devToolsActivePort: DoctorData['cdp']['devToolsActivePort'] = [];
|
||||||
@@ -2761,6 +2811,8 @@ export class BrowserManager {
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
await this.runDoctorSourceUrlProbe(checks, launched);
|
||||||
|
|
||||||
const pluginIntent = this.buildDoctorTabGroupIntent();
|
const pluginIntent = this.buildDoctorTabGroupIntent();
|
||||||
const pluginResult: DoctorData['plugin'] = {
|
const pluginResult: DoctorData['plugin'] = {
|
||||||
configuredPluginId: pluginIntent.pluginId,
|
configuredPluginId: pluginIntent.pluginId,
|
||||||
@@ -2769,7 +2821,13 @@ export class BrowserManager {
|
|||||||
message: 'Browser is not launched; plugin handshake skipped',
|
message: 'Browser is not launched; plugin handshake skipped',
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!this.isLaunched()) {
|
if (!launched) {
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'plugin:handshake-context',
|
||||||
|
'skip',
|
||||||
|
'Browser is not launched; plugin context check skipped'
|
||||||
|
);
|
||||||
this.addDoctorCheck(
|
this.addDoctorCheck(
|
||||||
checks,
|
checks,
|
||||||
'plugin:tab-group-handshake',
|
'plugin:tab-group-handshake',
|
||||||
@@ -2778,6 +2836,12 @@ export class BrowserManager {
|
|||||||
{ configuredPluginId: pluginIntent.pluginId }
|
{ configuredPluginId: pluginIntent.pluginId }
|
||||||
);
|
);
|
||||||
} else if (this.stealthConnectionKind !== 'cdp') {
|
} else if (this.stealthConnectionKind !== 'cdp') {
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'plugin:handshake-context',
|
||||||
|
'skip',
|
||||||
|
`Current connection mode is ${this.stealthConnectionKind}; plugin context check only applies to CDP`
|
||||||
|
);
|
||||||
pluginResult.mode = 'non-cdp';
|
pluginResult.mode = 'non-cdp';
|
||||||
pluginResult.status = 'skip';
|
pluginResult.status = 'skip';
|
||||||
pluginResult.message = `Current connection mode is ${this.stealthConnectionKind}; plugin handshake only applies to CDP`;
|
pluginResult.message = `Current connection mode is ${this.stealthConnectionKind}; plugin handshake only applies to CDP`;
|
||||||
@@ -2793,13 +2857,33 @@ export class BrowserManager {
|
|||||||
try {
|
try {
|
||||||
const page = this.getPage();
|
const page = this.getPage();
|
||||||
if (page.isClosed()) {
|
if (page.isClosed()) {
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'plugin:handshake-context',
|
||||||
|
'fail',
|
||||||
|
'Active page is closed; cannot test plugin handshake context'
|
||||||
|
);
|
||||||
pluginResult.status = 'fail';
|
pluginResult.status = 'fail';
|
||||||
pluginResult.message = 'Active page is closed; cannot run plugin handshake';
|
pluginResult.message = 'Active page is closed; cannot run plugin handshake';
|
||||||
} else if (!this.canInjectTabGroupScript(page)) {
|
} else if (!this.canInjectTabGroupScript(page)) {
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'plugin:handshake-context',
|
||||||
|
'warn',
|
||||||
|
'Active page is an internal browser page; open a normal http(s) page before testing plugin handshake',
|
||||||
|
{ url: this.getSafePageUrl(page) }
|
||||||
|
);
|
||||||
pluginResult.status = 'warn';
|
pluginResult.status = 'warn';
|
||||||
pluginResult.message =
|
pluginResult.message =
|
||||||
'Active page is an internal browser page; open a normal http(s) page to test plugin handshake';
|
'Active page is an internal browser page; open a normal http(s) page to test plugin handshake';
|
||||||
} else {
|
} else {
|
||||||
|
this.addDoctorCheck(
|
||||||
|
checks,
|
||||||
|
'plugin:handshake-context',
|
||||||
|
'pass',
|
||||||
|
'Active page is a normal page; plugin handshake can be tested',
|
||||||
|
{ url: this.getSafePageUrl(page) }
|
||||||
|
);
|
||||||
const response = await this.requestTabGroupPlugin(page, pluginIntent);
|
const response = await this.requestTabGroupPlugin(page, pluginIntent);
|
||||||
if (!response) {
|
if (!response) {
|
||||||
pluginResult.status = 'fail';
|
pluginResult.status = 'fail';
|
||||||
|
|||||||
@@ -302,6 +302,29 @@ describe('Stealth mode', () => {
|
|||||||
expect(raw).not.toContain('sourceURL=');
|
expect(raw).not.toContain('sourceURL=');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('doctor reports CDP sourceURL probe as pass in launched chromium sessions', async () => {
|
||||||
|
browser = new BrowserManager();
|
||||||
|
await browser.launch({ headless: true, stealth: true });
|
||||||
|
|
||||||
|
const report = await browser.runDoctor();
|
||||||
|
const check = report.checks.find((entry) => entry.name === 'cdp:sourceurl-sanitized');
|
||||||
|
|
||||||
|
expect(check).toBeDefined();
|
||||||
|
expect(check?.status).toBe('pass');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('doctor marks plugin handshake context as skip outside CDP mode', async () => {
|
||||||
|
browser = new BrowserManager();
|
||||||
|
await browser.launch({ headless: true, stealth: true });
|
||||||
|
|
||||||
|
const report = await browser.runDoctor();
|
||||||
|
const check = report.checks.find((entry) => entry.name === 'plugin:handshake-context');
|
||||||
|
|
||||||
|
expect(check).toBeDefined();
|
||||||
|
expect(check?.status).toBe('skip');
|
||||||
|
expect(check?.message).toContain('only applies to CDP');
|
||||||
|
});
|
||||||
|
|
||||||
it('exposes contacts manager and content index APIs', async () => {
|
it('exposes contacts manager and content index APIs', async () => {
|
||||||
browser = new BrowserManager();
|
browser = new BrowserManager();
|
||||||
await browser.launch({ headless: true, stealth: true });
|
await browser.launch({ headless: true, stealth: true });
|
||||||
@@ -354,4 +377,36 @@ describe('Stealth mode', () => {
|
|||||||
expect(workerSignals.hasDownlinkMaxOnProto).toBe(true);
|
expect(workerSignals.hasDownlinkMaxOnProto).toBe(true);
|
||||||
expect(typeof workerSignals.downlinkMax).toBe('number');
|
expect(typeof workerSignals.downlinkMax).toBe('number');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('skips worker wrapping for cross-origin blob URLs', async () => {
|
||||||
|
browser = new BrowserManager();
|
||||||
|
await browser.launch({ headless: true, stealth: true });
|
||||||
|
|
||||||
|
const signals = await browser.getPage().evaluate(() => {
|
||||||
|
const nativeCreateObjectURL = URL.createObjectURL;
|
||||||
|
const nativeRevokeObjectURL = URL.revokeObjectURL;
|
||||||
|
let createCalls = 0;
|
||||||
|
let revokeCalls = 0;
|
||||||
|
|
||||||
|
(URL as any).createObjectURL = (...args: unknown[]) => {
|
||||||
|
createCalls += 1;
|
||||||
|
return nativeCreateObjectURL.apply(URL, args as [Blob | MediaSource]);
|
||||||
|
};
|
||||||
|
(URL as any).revokeObjectURL = (...args: unknown[]) => {
|
||||||
|
revokeCalls += 1;
|
||||||
|
return nativeRevokeObjectURL.apply(URL, args as [string]);
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
new Worker('blob:https://challenges.cloudflare.com/11111111-1111-1111-1111-111111111111');
|
||||||
|
} catch {}
|
||||||
|
|
||||||
|
(URL as any).createObjectURL = nativeCreateObjectURL;
|
||||||
|
(URL as any).revokeObjectURL = nativeRevokeObjectURL;
|
||||||
|
return { createCalls, revokeCalls };
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(signals.createCalls).toBe(0);
|
||||||
|
expect(signals.revokeCalls).toBe(0);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
+27
-2
@@ -1262,7 +1262,8 @@ function patchNavigatorConnection(): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ensure dedicated workers expose navigator.connection.downlinkMax too.
|
* Ensure same-origin dedicated workers expose navigator.connection.downlinkMax too.
|
||||||
|
* Skip cross-origin worker URLs to avoid breaking anti-bot challenge workers.
|
||||||
*/
|
*/
|
||||||
function patchWorkerConnection(): string {
|
function patchWorkerConnection(): string {
|
||||||
return `(function(){
|
return `(function(){
|
||||||
@@ -1305,12 +1306,36 @@ function patchWorkerConnection(): string {
|
|||||||
: \`importScripts(\${JSON.stringify(scriptUrl)});\`;
|
: \`importScripts(\${JSON.stringify(scriptUrl)});\`;
|
||||||
return \`\${workerPrelude}\\n\${loader}\`;
|
return \`\${workerPrelude}\\n\${loader}\`;
|
||||||
};
|
};
|
||||||
|
const resolveWorkerUrl = (value) => {
|
||||||
|
try {
|
||||||
|
return new URL(String(value), location.href);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const shouldPatchWorker = (value) => {
|
||||||
|
const resolved = resolveWorkerUrl(value);
|
||||||
|
if (!resolved) return false;
|
||||||
|
if (resolved.protocol === 'blob:') return resolved.origin === location.origin;
|
||||||
|
if (resolved.protocol === 'http:' || resolved.protocol === 'https:') {
|
||||||
|
return resolved.origin === location.origin;
|
||||||
|
}
|
||||||
|
if (resolved.protocol === 'file:') return location.protocol === 'file:';
|
||||||
|
return false;
|
||||||
|
};
|
||||||
const WrappedWorker = function(scriptURL, options) {
|
const WrappedWorker = function(scriptURL, options) {
|
||||||
|
if (!shouldPatchWorker(scriptURL)) {
|
||||||
|
return new NativeWorker(scriptURL, options);
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
const source = buildPatchedScript(scriptURL, options);
|
const source = buildPatchedScript(scriptURL, options);
|
||||||
const blob = new Blob([source], { type: 'application/javascript' });
|
const blob = new Blob([source], { type: 'application/javascript' });
|
||||||
const patchedUrl = URL.createObjectURL(blob);
|
const patchedUrl = URL.createObjectURL(blob);
|
||||||
return new NativeWorker(patchedUrl, options);
|
const worker = new NativeWorker(patchedUrl, options);
|
||||||
|
try {
|
||||||
|
setTimeout(() => URL.revokeObjectURL(patchedUrl), 0);
|
||||||
|
} catch {}
|
||||||
|
return worker;
|
||||||
} catch {
|
} catch {
|
||||||
return new NativeWorker(scriptURL, options);
|
return new NativeWorker(scriptURL, options);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user