From 77f2caa1bc1a181a7aa998ed4fc30e133ca6d3d2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 24 Feb 2026 07:22:55 -0600 Subject: [PATCH 1/3] feat: add --download-path option (#536) * feat: add --download-path option Adds a `--download-path` flag (and `AGENT_BROWSER_DOWNLOAD_PATH` env / `downloadPath` config key) to set a default download directory for browser downloads. Without this, Playwright stores downloads in a temp directory that is deleted when the browser closes. The new option passes through to Playwright's `downloadsPath` on `launch()` and `launchPersistentContext()`. Fixes #507 * improvements * fixes * fixes --- README.md | 1 + cli/src/commands.rs | 2 ++ cli/src/connection.rs | 9 +++++++ cli/src/flags.rs | 29 +++++++++++++++++++++ cli/src/main.rs | 17 +++++++++++- cli/src/output.rs | 2 ++ docs/src/app/commands/page.mdx | 2 ++ docs/src/app/configuration/page.mdx | 2 ++ skills/agent-browser/SKILL.md | 5 ++++ src/browser.ts | 40 ++++++++++++++++++++++++++++- src/protocol.ts | 1 + src/types.ts | 1 + 12 files changed, 109 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9e1f1f4..fc0eca3 100644 --- a/README.md +++ b/README.md @@ -466,6 +466,7 @@ This is useful for multimodal AI models that can reason about visual layout, unl | `--cdp ` | Connect via Chrome DevTools Protocol (port or WebSocket URL) | | `--auto-connect` | Auto-discover and connect to running Chrome (or `AGENT_BROWSER_AUTO_CONNECT` env) | | `--color-scheme ` | Color scheme: `dark`, `light`, `no-preference` (or `AGENT_BROWSER_COLOR_SCHEME` env) | +| `--download-path ` | Default download directory (or `AGENT_BROWSER_DOWNLOAD_PATH` env) | | `--config ` | Use a custom config file (or `AGENT_BROWSER_CONFIG` env) | | `--debug` | Debug output | diff --git a/cli/src/commands.rs b/cli/src/commands.rs index fb1b257..46abdb2 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -1896,8 +1896,10 @@ mod tests { cli_proxy_bypass: false, cli_allow_file_access: false, cli_annotate: false, + cli_download_path: false, annotate: false, color_scheme: None, + download_path: None, } } diff --git a/cli/src/connection.rs b/cli/src/connection.rs index 220d522..c08bf75 100644 --- a/cli/src/connection.rs +++ b/cli/src/connection.rs @@ -220,6 +220,7 @@ pub fn ensure_daemon( provider: Option<&str>, device: Option<&str>, session_name: Option<&str>, + download_path: Option<&str>, ) -> Result { // Check if daemon is running AND responsive if is_daemon_running(session) && daemon_ready(session) { @@ -364,6 +365,10 @@ pub fn ensure_daemon( cmd.env("AGENT_BROWSER_SESSION_NAME", sn); } + if let Some(dp) = download_path { + cmd.env("AGENT_BROWSER_DOWNLOAD_PATH", dp); + } + // Create new process group and session to fully detach unsafe { cmd.pre_exec(|| { @@ -447,6 +452,10 @@ pub fn ensure_daemon( cmd.env("AGENT_BROWSER_SESSION_NAME", sn); } + if let Some(dp) = download_path { + cmd.env("AGENT_BROWSER_DOWNLOAD_PATH", dp); + } + // CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS const CREATE_NEW_PROCESS_GROUP: u32 = 0x00000200; const DETACHED_PROCESS: u32 = 0x00000008; diff --git a/cli/src/flags.rs b/cli/src/flags.rs index c2a1e6d..1103ebe 100644 --- a/cli/src/flags.rs +++ b/cli/src/flags.rs @@ -34,6 +34,7 @@ pub struct Config { pub headers: Option, pub annotate: Option, pub color_scheme: Option, + pub download_path: Option, } impl Config { @@ -68,6 +69,7 @@ impl Config { headers: other.headers.or(self.headers), annotate: other.annotate.or(self.annotate), color_scheme: other.color_scheme.or(self.color_scheme), + download_path: other.download_path.or(self.download_path), } } } @@ -132,6 +134,7 @@ fn extract_config_path(args: &[String]) -> Option> { "--device", "--session-name", "--color-scheme", + "--download-path", ]; let mut i = 0; while i < args.len() { @@ -203,6 +206,7 @@ pub struct Flags { pub session_name: Option, pub annotate: bool, pub color_scheme: Option, + pub download_path: Option, // Track which launch-time options were explicitly passed via CLI // (as opposed to being set only via environment variables) @@ -216,6 +220,7 @@ pub struct Flags { pub cli_proxy_bypass: bool, pub cli_allow_file_access: bool, pub cli_annotate: bool, + pub cli_download_path: bool, } pub fn parse_flags(args: &[String]) -> Flags { @@ -285,6 +290,8 @@ pub fn parse_flags(args: &[String]) -> Flags { || config.annotate.unwrap_or(false), color_scheme: env::var("AGENT_BROWSER_COLOR_SCHEME").ok() .or(config.color_scheme), + download_path: env::var("AGENT_BROWSER_DOWNLOAD_PATH").ok() + .or(config.download_path), cli_executable_path: false, cli_extensions: false, cli_profile: false, @@ -295,6 +302,7 @@ pub fn parse_flags(args: &[String]) -> Flags { cli_proxy_bypass: false, cli_allow_file_access: false, cli_annotate: false, + cli_download_path: false, }; let mut i = 0; @@ -440,6 +448,13 @@ pub fn parse_flags(args: &[String]) -> Flags { i += 1; } } + "--download-path" => { + if let Some(s) = args.get(i + 1) { + flags.download_path = Some(s.clone()); + flags.cli_download_path = true; + i += 1; + } + } "--config" => { // Already handled by load_config(); skip the value i += 1; @@ -484,6 +499,7 @@ pub fn clean_args(args: &[String]) -> Vec { "--device", "--session-name", "--color-scheme", + "--download-path", "--config", ]; @@ -674,6 +690,19 @@ mod tests { assert!(!flags.cli_annotate); } + #[test] + fn test_cli_download_path_tracking() { + let flags = parse_flags(&args("--download-path /tmp/dl snapshot")); + assert!(flags.cli_download_path); + assert_eq!(flags.download_path, Some("/tmp/dl".to_string())); + } + + #[test] + fn test_cli_download_path_not_set_without_flag() { + let flags = parse_flags(&args("snapshot")); + assert!(!flags.cli_download_path); + } + #[test] fn test_cli_multiple_flags_tracking() { let flags = parse_flags(&args( diff --git a/cli/src/main.rs b/cli/src/main.rs index 73d6c08..39d78de 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -226,6 +226,7 @@ fn main() { flags.provider.as_deref(), flags.device.as_deref(), flags.session_name.as_deref(), + flags.download_path.as_deref(), ) { Ok(result) => result, Err(e) => { @@ -281,6 +282,7 @@ fn main() { }, flags.ignore_https_errors.then_some("--ignore-https-errors"), flags.cli_allow_file_access.then_some("--allow-file-access"), + flags.cli_download_path.then_some("--download-path"), ] .into_iter() .flatten() @@ -362,6 +364,10 @@ fn main() { launch_cmd["colorScheme"] = json!(cs); } + if let Some(ref dp) = flags.download_path { + launch_cmd["downloadPath"] = json!(dp); + } + let err = match send_command(launch_cmd, &flags.session) { Ok(resp) if resp.success => None, Ok(resp) => Some( @@ -448,6 +454,10 @@ fn main() { launch_cmd["colorScheme"] = json!(cs); } + if let Some(ref dp) = flags.download_path { + launch_cmd["downloadPath"] = json!(dp); + } + let err = match send_command(launch_cmd, &flags.session) { Ok(resp) if resp.success => None, Ok(resp) => Some( @@ -507,7 +517,8 @@ fn main() { || flags.args.is_some() || flags.user_agent.is_some() || flags.allow_file_access - || flags.color_scheme.is_some()) + || flags.color_scheme.is_some() + || flags.download_path.is_some()) && flags.cdp.is_none() && flags.provider.is_none() { @@ -573,6 +584,10 @@ fn main() { launch_cmd["colorScheme"] = json!(cs); } + if let Some(ref dp) = flags.download_path { + launch_cmd["downloadPath"] = json!(dp); + } + match send_command(launch_cmd, &flags.session) { Ok(resp) if !resp.success => { // Launch command failed (e.g., invalid state file, profile error) diff --git a/cli/src/output.rs b/cli/src/output.rs index 0438338..18d2921 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -2106,6 +2106,7 @@ Options: --cdp Connect via CDP (Chrome DevTools Protocol) --auto-connect Auto-discover and connect to running Chrome --color-scheme Color scheme: dark, light, no-preference (or AGENT_BROWSER_COLOR_SCHEME) + --download-path Default download directory (or AGENT_BROWSER_DOWNLOAD_PATH) --session-name Auto-save/restore session state (cookies, localStorage) --config Use a custom config file (or AGENT_BROWSER_CONFIG env) --debug Debug output @@ -2148,6 +2149,7 @@ Environment: AGENT_BROWSER_AUTO_CONNECT Auto-discover and connect to running Chrome AGENT_BROWSER_ALLOW_FILE_ACCESS Allow file:// URLs to access local files AGENT_BROWSER_COLOR_SCHEME Color scheme preference (dark, light, no-preference) + AGENT_BROWSER_DOWNLOAD_PATH Default download directory for browser downloads AGENT_BROWSER_DEFAULT_TIMEOUT Default Playwright timeout in ms (default: 25000) AGENT_BROWSER_SESSION_NAME Auto-save/load state persistence name AGENT_BROWSER_STATE_EXPIRE_DAYS Auto-delete saved states older than N days (default: 30) diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx index 9cc7dc8..fd97517 100644 --- a/docs/src/app/commands/page.mdx +++ b/docs/src/app/commands/page.mdx @@ -109,6 +109,8 @@ agent-browser download # Click element to trigger download agent-browser wait --download [path] # Wait for any download to complete ``` +Use `--download-path ` (or `AGENT_BROWSER_DOWNLOAD_PATH` env) to set a default download directory. Without it, downloads go to a temporary directory that is deleted when the browser closes. + ## Mouse ```bash diff --git a/docs/src/app/configuration/page.mdx b/docs/src/app/configuration/page.mdx index b7015a1..d476c1d 100644 --- a/docs/src/app/configuration/page.mdx +++ b/docs/src/app/configuration/page.mdx @@ -73,6 +73,7 @@ Every CLI flag can be set in the config file using its camelCase equivalent: cdp--cdpstring autoConnect--auto-connectboolean colorScheme--color-schemestring (dark, light, no-preference) + downloadPath--download-pathstring headers--headersstring (JSON) @@ -151,6 +152,7 @@ These environment variables configure additional daemon and runtime behavior: AGENT_BROWSER_AUTO_CONNECTAuto-discover and connect to a running Chrome instance.(disabled) AGENT_BROWSER_ALLOW_FILE_ACCESSAllow file:// URLs to access local files.(disabled) AGENT_BROWSER_COLOR_SCHEMEColor scheme preference (dark, light, no-preference).(none) + AGENT_BROWSER_DOWNLOAD_PATHDefault directory for browser downloads.(temp directory) AGENT_BROWSER_DEFAULT_TIMEOUTDefault Playwright timeout in ms. Keep below 30000 to avoid IPC timeouts.25000 AGENT_BROWSER_SESSION_NAMEAuto-save/load state persistence name.(none) AGENT_BROWSER_STATE_EXPIRE_DAYSAuto-delete saved session states older than N days.30 diff --git a/skills/agent-browser/SKILL.md b/skills/agent-browser/SKILL.md index b62f88f..25d237f 100644 --- a/skills/agent-browser/SKILL.md +++ b/skills/agent-browser/SKILL.md @@ -79,6 +79,11 @@ agent-browser wait --load networkidle # Wait for network idle agent-browser wait --url "**/page" # Wait for URL pattern agent-browser wait 2000 # Wait milliseconds +# Downloads +agent-browser download @e1 ./file.pdf # Click element to trigger download +agent-browser wait --download ./output.zip # Wait for any download to complete +agent-browser --download-path ./downloads open # Set default download directory + # Capture agent-browser screenshot # Screenshot to temp dir agent-browser screenshot --full # Full page screenshot diff --git a/src/browser.ts b/src/browser.ts index 3b025df..93538d1 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -16,7 +16,7 @@ import { } from 'playwright-core'; import path from 'node:path'; import os from 'node:os'; -import { existsSync, mkdirSync, rmSync, readFileSync } from 'node:fs'; +import { existsSync, mkdirSync, rmSync, readFileSync, statSync } from 'node:fs'; import { writeFile, mkdir } from 'node:fs/promises'; import type { LaunchCommand, TraceEvent } from './types.js'; import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js'; @@ -116,6 +116,7 @@ export class BrowserManager { private lastSnapshot: string = ''; private scopedHeaderRoutes: Map Promise> = new Map(); private colorScheme: 'light' | 'dark' | 'no-preference' | null = null; + private downloadPath: string | null = null; /** * Set the persistent color scheme preference. @@ -1173,6 +1174,17 @@ export class BrowserManager { this.colorScheme = options.colorScheme; } + if (options.downloadPath) { + this.downloadPath = options.downloadPath; + } + + if (this.downloadPath && (cdpEndpoint || options.autoConnect)) { + const warning = + "--download-path is ignored when connecting via CDP or auto-connect (downloads use the remote browser's configuration)"; + this.launchWarnings.push(warning); + console.error(`[WARN] ${warning}`); + } + if (cdpEndpoint) { await this.connectViaCDP(cdpEndpoint); return; @@ -1186,6 +1198,12 @@ export class BrowserManager { // Cloud browser providers require explicit opt-in via -p flag or AGENT_BROWSER_PROVIDER env var // -p flag takes precedence over env var const provider = options.provider ?? process.env.AGENT_BROWSER_PROVIDER; + if (this.downloadPath && provider) { + const warning = + "--download-path is ignored when using a cloud provider (downloads use the remote browser's configuration)"; + this.launchWarnings.push(warning); + console.error(`[WARN] ${warning}`); + } if (provider === 'browserbase') { await this.connectToBrowserbase(); return; @@ -1201,6 +1219,23 @@ export class BrowserManager { return; } + if (this.downloadPath) { + const resolved = path.resolve(this.downloadPath); + const stat = statSync(resolved, { throwIfNoEntry: false }); + if (stat && !stat.isDirectory()) { + throw new Error(`Download path is not a directory: ${resolved}`); + } + if (!stat) { + try { + mkdirSync(resolved, { recursive: true }); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + throw new Error(`Cannot create download directory '${resolved}': ${msg}`); + } + } + this.downloadPath = resolved; + } + const browserType = options.browser ?? 'chromium'; if (hasExtensions && browserType !== 'chromium') { throw new Error('Extensions are only supported in Chromium'); @@ -1258,6 +1293,7 @@ export class BrowserManager { ...(options.proxy && { proxy: options.proxy }), ignoreHTTPSErrors: options.ignoreHTTPSErrors ?? false, ...(this.colorScheme && { colorScheme: this.colorScheme }), + ...(this.downloadPath && { downloadsPath: this.downloadPath }), } ); this.isPersistentContext = true; @@ -1275,6 +1311,7 @@ export class BrowserManager { ...(options.proxy && { proxy: options.proxy }), ignoreHTTPSErrors: options.ignoreHTTPSErrors ?? false, ...(this.colorScheme && { colorScheme: this.colorScheme }), + ...(this.downloadPath && { downloadsPath: this.downloadPath }), }); this.isPersistentContext = true; } else { @@ -1283,6 +1320,7 @@ export class BrowserManager { headless: options.headless ?? true, executablePath: options.executablePath, args: baseArgs, + ...(this.downloadPath && { downloadsPath: this.downloadPath }), }); this.cdpEndpoint = null; diff --git a/src/protocol.ts b/src/protocol.ts index f9d685f..fbd5643 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -50,6 +50,7 @@ const launchSchema = baseCommandSchema.extend({ ignoreHTTPSErrors: z.boolean().optional(), allowFileAccess: z.boolean().optional(), colorScheme: z.enum(['light', 'dark', 'no-preference']).optional(), + downloadPath: z.string().optional(), profile: z.string().optional(), storageState: z.string().optional(), }); diff --git a/src/types.ts b/src/types.ts index 857f282..ddffd40 100644 --- a/src/types.ts +++ b/src/types.ts @@ -32,6 +32,7 @@ export interface LaunchCommand extends BaseCommand { ignoreHTTPSErrors?: boolean; allowFileAccess?: boolean; // Enable file:// URL access and cross-origin file requests colorScheme?: 'light' | 'dark' | 'no-preference'; // Persistent color scheme override + downloadPath?: string; // Directory for browser downloads (Playwright's downloadsPath) // Auto-load state file for session persistence autoStateFilePath?: string; } From f319195974ff38436e5d0862c949b4e616a2e0ed Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 24 Feb 2026 07:40:46 -0600 Subject: [PATCH 2/3] add --selector flag to scroll command (#537) * add --selector flag to scroll command The `scroll` command uses `window.scrollBy()`, which has no effect on apps that use custom scrollable containers (e.g. a nested div with overflow-y: auto). The backend `handleScroll` already supports a `selector` parameter, but the CLI never exposed it. This adds `-s` / `--selector` to the `scroll` command so users can target a specific scrollable element: agent-browser scroll down 500 --selector "div.scroll-container" Also fixes the backend to apply `direction`/`amount` when a selector is present (previously those fields were only used in the no-selector branch). Closes #501 * fixes --- README.md | 2 +- cli/src/commands.rs | 129 +++++++++++++++++++++++++++++++-- cli/src/output.rs | 8 +- docs/src/app/commands/page.mdx | 2 +- skills/agent-browser/SKILL.md | 1 + src/actions.ts | 50 ++++++------- 6 files changed, 157 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index fc0eca3..acf519b 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ agent-browser hover # Hover element agent-browser select # Select dropdown option agent-browser check # Check checkbox agent-browser uncheck # Uncheck checkbox -agent-browser scroll [px] # Scroll (up/down/left/right) +agent-browser scroll [px] # Scroll (up/down/left/right, --selector ) agent-browser scrollintoview # Scroll element into view (alias: scrollinto) agent-browser drag # Drag and drop agent-browser upload # Upload files diff --git a/cli/src/commands.rs b/cli/src/commands.rs index 46abdb2..21e0e80 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -298,12 +298,48 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result { - let dir = rest.first().unwrap_or(&"down"); - let amount = rest - .get(1) - .and_then(|s| s.parse::().ok()) - .unwrap_or(300); - Ok(json!({ "id": id, "action": "scroll", "direction": dir, "amount": amount })) + let mut cmd = json!({ "id": id, "action": "scroll" }); + let obj = cmd.as_object_mut().unwrap(); + let mut positional_index = 0; + let mut i = 0; + while i < rest.len() { + match rest[i] { + "-s" | "--selector" => { + if let Some(s) = rest.get(i + 1) { + obj.insert("selector".to_string(), json!(s)); + i += 1; + } else { + return Err(ParseError::MissingArguments { + context: "scroll --selector".to_string(), + usage: "scroll [direction] [amount] [--selector ]", + }); + } + } + arg if arg.starts_with('-') => {} + _ => { + match positional_index { + 0 => { + obj.insert("direction".to_string(), json!(rest[i])); + } + 1 => { + if let Ok(n) = rest[i].parse::() { + obj.insert("amount".to_string(), json!(n)); + } + } + _ => {} + } + positional_index += 1; + } + } + i += 1; + } + if !obj.contains_key("direction") { + obj.insert("direction".to_string(), json!("down")); + } + if !obj.contains_key("amount") { + obj.insert("amount".to_string(), json!(300)); + } + Ok(cmd) } "scrollintoview" | "scrollinto" => { let sel = rest.first().ok_or_else(|| ParseError::MissingArguments { @@ -3436,4 +3472,85 @@ mod tests { ParseError::MissingArguments { .. } )); } + + // === Scroll Tests === + + #[test] + fn test_scroll_defaults() { + let cmd = parse_command(&args("scroll"), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "scroll"); + assert_eq!(cmd["direction"], "down"); + assert_eq!(cmd["amount"], 300); + assert!(cmd.get("selector").is_none()); + } + + #[test] + fn test_scroll_direction_and_amount() { + let cmd = parse_command(&args("scroll up 200"), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "scroll"); + assert_eq!(cmd["direction"], "up"); + assert_eq!(cmd["amount"], 200); + } + + #[test] + fn test_scroll_with_selector() { + let cmd = parse_command( + &args("scroll down 500 --selector div.scroll-container"), + &default_flags(), + ) + .unwrap(); + assert_eq!(cmd["action"], "scroll"); + assert_eq!(cmd["direction"], "down"); + assert_eq!(cmd["amount"], 500); + assert_eq!(cmd["selector"], "div.scroll-container"); + } + + #[test] + fn test_scroll_with_selector_short_flag() { + let cmd = parse_command( + &args("scroll left 100 -s .sidebar"), + &default_flags(), + ) + .unwrap(); + assert_eq!(cmd["action"], "scroll"); + assert_eq!(cmd["direction"], "left"); + assert_eq!(cmd["amount"], 100); + assert_eq!(cmd["selector"], ".sidebar"); + } + + #[test] + fn test_scroll_selector_before_positional() { + let cmd = parse_command( + &args("scroll --selector .panel down 400"), + &default_flags(), + ) + .unwrap(); + assert_eq!(cmd["action"], "scroll"); + assert_eq!(cmd["direction"], "down"); + assert_eq!(cmd["amount"], 400); + assert_eq!(cmd["selector"], ".panel"); + } + + #[test] + fn test_scroll_selector_only() { + let cmd = parse_command( + &args("scroll --selector .content"), + &default_flags(), + ) + .unwrap(); + assert_eq!(cmd["action"], "scroll"); + assert_eq!(cmd["direction"], "down"); + assert_eq!(cmd["amount"], 300); + assert_eq!(cmd["selector"], ".content"); + } + + #[test] + fn test_scroll_selector_missing_value() { + let result = parse_command(&args("scroll down 500 --selector"), &default_flags()); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + ParseError::MissingArguments { .. } + )); + } } diff --git a/cli/src/output.rs b/cli/src/output.rs index 18d2921..0d986d3 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -973,14 +973,17 @@ Use Cases: r##" agent-browser scroll - Scroll the page -Usage: agent-browser scroll [direction] [amount] +Usage: agent-browser scroll [direction] [amount] [options] -Scrolls the page in the specified direction. +Scrolls the page or a specific element in the specified direction. Arguments: direction up, down, left, right (default: down) amount Pixels to scroll (default: 300) +Options: + -s, --selector CSS selector for a scrollable container + Global Options: --json Output as JSON --session Use specific session @@ -990,6 +993,7 @@ Examples: agent-browser scroll down 500 agent-browser scroll up 200 agent-browser scroll left 100 + agent-browser scroll down 500 --selector "div.scroll-container" "## } "scrollintoview" | "scrollinto" => { diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx index fd97517..1a56432 100644 --- a/docs/src/app/commands/page.mdx +++ b/docs/src/app/commands/page.mdx @@ -22,7 +22,7 @@ agent-browser focus # Focus element agent-browser select # Select dropdown option agent-browser check # Check checkbox agent-browser uncheck # Uncheck checkbox -agent-browser scroll [px] # Scroll (up/down/left/right) +agent-browser scroll [px] # Scroll (up/down/left/right, --selector ) agent-browser scrollintoview # Scroll element into view agent-browser drag # Drag and drop agent-browser upload # Upload files diff --git a/skills/agent-browser/SKILL.md b/skills/agent-browser/SKILL.md index 25d237f..e691527 100644 --- a/skills/agent-browser/SKILL.md +++ b/skills/agent-browser/SKILL.md @@ -67,6 +67,7 @@ agent-browser press Enter # Press key agent-browser keyboard type "text" # Type at current focus (no selector) agent-browser keyboard inserttext "text" # Insert without key events agent-browser scroll down 500 # Scroll page +agent-browser scroll down 500 --selector "div.content" # Scroll within a specific container # Get information agent-browser get text @e1 # Get element text diff --git a/src/actions.ts b/src/actions.ts index aa92865..fc012e6 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -882,41 +882,41 @@ async function handleWait(command: WaitCommand, browser: BrowserManager): Promis async function handleScroll(command: ScrollCommand, browser: BrowserManager): Promise { const page = browser.getPage(); + let deltaX = command.x ?? 0; + let deltaY = command.y ?? 0; + const hasExplicitDelta = command.x !== undefined || command.y !== undefined; + + if (command.direction) { + const amount = command.amount ?? 100; + switch (command.direction) { + case 'up': + deltaY = -amount; + break; + case 'down': + deltaY = amount; + break; + case 'left': + deltaX = -amount; + break; + case 'right': + deltaX = amount; + break; + } + } + if (command.selector) { const element = browser.getLocator(command.selector); await element.scrollIntoViewIfNeeded(); - if (command.x !== undefined || command.y !== undefined) { + if (hasExplicitDelta || deltaX !== 0 || deltaY !== 0) { await element.evaluate( (el, { x, y }) => { - el.scrollBy(x ?? 0, y ?? 0); + el.scrollBy(x, y); }, - { x: command.x, y: command.y } + { x: deltaX, y: deltaY } ); } } else { - // Scroll the page - let deltaX = command.x ?? 0; - let deltaY = command.y ?? 0; - - if (command.direction) { - const amount = command.amount ?? 100; - switch (command.direction) { - case 'up': - deltaY = -amount; - break; - case 'down': - deltaY = amount; - break; - case 'left': - deltaX = -amount; - break; - case 'right': - deltaX = amount; - break; - } - } - await page.evaluate(`window.scrollBy(${deltaX}, ${deltaY})`); } From c0e2b80f8cc991fbd699e0a0a201f0ec2623720e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 24 Feb 2026 11:35:50 -0600 Subject: [PATCH 3/3] add dogfood skill for agent-driven exploratory qa (#538) * dogfood skill * evals * haiku * fixes * caching * fixes * don't use npx --- .gitignore | 5 + package.json | 2 + pnpm-lock.yaml | 174 ++++++++++++ skills/dogfood/SKILL.md | 216 +++++++++++++++ skills/dogfood/references/issue-taxonomy.md | 109 ++++++++ .../templates/dogfood-report-template.md | 53 ++++ test/e2e/dogfood.eval.ts | 261 ++++++++++++++++++ test/e2e/dogfood.test.ts | 210 ++++++++++++++ test/e2e/fixtures/buggy-app.html | 159 +++++++++++ vitest.config.ts | 2 +- 10 files changed, 1190 insertions(+), 1 deletion(-) create mode 100644 skills/dogfood/SKILL.md create mode 100644 skills/dogfood/references/issue-taxonomy.md create mode 100644 skills/dogfood/templates/dogfood-report-template.md create mode 100644 test/e2e/dogfood.eval.ts create mode 100644 test/e2e/dogfood.test.ts create mode 100644 test/e2e/fixtures/buggy-app.html diff --git a/.gitignore b/.gitignore index 331632b..d1d9e00 100644 --- a/.gitignore +++ b/.gitignore @@ -27,10 +27,15 @@ npm-debug.log* .DS_Store Thumbs.db +# Python +__pycache__/ + # Test artifacts *.png *.jpeg *.jpg +*.webm +test/e2e/.dogfood-output/ # Package manager package-lock.json diff --git a/package.json b/package.json index 6d8c1e0..dff3069 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "format:check": "prettier --check 'src/**/*.ts'", "test": "vitest run", "test:watch": "vitest", + "test:e2e:dogfood": "vitest run test/e2e/dogfood.eval.ts", "postinstall": "node scripts/postinstall.js", "changeset": "changeset", "ci:version": "changeset version && pnpm run version:sync && pnpm install --no-frozen-lockfile", @@ -62,6 +63,7 @@ "zod": "^3.22.4" }, "devDependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.2.52", "@changesets/cli": "^2.29.8", "@types/node": "^20.10.0", "@types/ws": "^8.18.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3488e40..ce8f6b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ importers: specifier: ^3.22.4 version: 3.25.76 devDependencies: + '@anthropic-ai/claude-agent-sdk': + specifier: ^0.2.52 + version: 0.2.52(zod@3.25.76) '@changesets/cli': specifier: ^2.29.8 version: 2.29.8(@types/node@20.19.28) @@ -57,6 +60,12 @@ importers: packages: + '@anthropic-ai/claude-agent-sdk@0.2.52': + resolution: {integrity: sha512-rdTQUu/HjKlDNNxJuhtXY6LJDOLvzVBU7sXFuFIG6CEC/nFfcvYq035EyjVw4nzu7lLZim/m+g2yZ8uNIcbaFw==} + engines: {node: '>=18.0.0'} + peerDependencies: + zod: ^4.0.0 + '@appium/logger@1.7.1': resolution: {integrity: sha512-9C2o9X/lBEDBUnKfAi3mRo9oG7Z03nmISLwsGkWxIWjMAvBdJD0RRSJMekWVKzfXN3byrI1WlCXTITzN4LAoLw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0, npm: '>=8'} @@ -276,6 +285,95 @@ packages: cpu: [x64] os: [win32] + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@inquirer/external-editor@1.0.3': resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} engines: {node: '>=18'} @@ -1950,6 +2048,20 @@ packages: snapshots: + '@anthropic-ai/claude-agent-sdk@0.2.52(zod@3.25.76)': + dependencies: + zod: 3.25.76 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + '@appium/logger@1.7.1': dependencies: console-control-strings: 1.1.0 @@ -2181,6 +2293,68 @@ snapshots: '@esbuild/win32-x64@0.27.2': optional: true + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + '@inquirer/external-editor@1.0.3(@types/node@20.19.28)': dependencies: chardet: 2.1.1 diff --git a/skills/dogfood/SKILL.md b/skills/dogfood/SKILL.md new file mode 100644 index 0000000..be25ce5 --- /dev/null +++ b/skills/dogfood/SKILL.md @@ -0,0 +1,216 @@ +--- +name: dogfood +description: Systematically explore and test a web application to find bugs, UX issues, and other problems. Use when asked to "dogfood", "QA", "exploratory test", "find issues", "bug hunt", "test this app/site/platform", or review the quality of a web application. Produces a structured report with full reproduction evidence -- step-by-step screenshots, repro videos, and detailed repro steps for every issue -- so findings can be handed directly to the responsible teams. +allowed-tools: Bash(agent-browser:*), Bash(npx agent-browser:*) +--- + +# Dogfood + +Systematically explore a web application, find issues, and produce a report with full reproduction evidence for every finding. + +## Setup + +Only the **Target URL** is required. Everything else has sensible defaults -- use them unless the user explicitly provides an override. + +| Parameter | Default | Example override | +|-----------|---------|-----------------| +| **Target URL** | _(required)_ | `vercel.com`, `http://localhost:3000` | +| **Session name** | Slugified domain (e.g., `vercel.com` -> `vercel-com`) | `--session my-session` | +| **Output directory** | `./dogfood-output/` | `Output directory: /tmp/qa` | +| **Scope** | Full app | `Focus on the billing page` | +| **Authentication** | None | `Sign in to user@example.com` | + +If the user says something like "dogfood vercel.com", start immediately with defaults. Do not ask clarifying questions unless authentication is mentioned but credentials are missing. + +Always use `agent-browser` directly -- never `npx agent-browser`. The direct binary uses the fast Rust client. `npx` routes through Node.js and is significantly slower. + +## Workflow + +``` +1. Initialize Set up session, output dirs, report file +2. Authenticate Sign in if needed, save state +3. Orient Navigate to starting point, take initial snapshot +4. Explore Systematically visit pages and test features +5. Document Screenshot + record each issue as found +6. Wrap up Update summary counts, close session +``` + +### 1. Initialize + +```bash +mkdir -p {OUTPUT_DIR}/screenshots {OUTPUT_DIR}/videos +``` + +Copy the report template into the output directory and fill in the header fields: + +```bash +cp {SKILL_DIR}/templates/dogfood-report-template.md {OUTPUT_DIR}/report.md +``` + +Start a named session: + +```bash +agent-browser --session {SESSION} open {TARGET_URL} +agent-browser --session {SESSION} wait --load networkidle +``` + +### 2. Authenticate + +If the app requires login: + +```bash +agent-browser --session {SESSION} snapshot -i +# Identify login form refs, fill credentials +agent-browser --session {SESSION} fill @e1 "{EMAIL}" +agent-browser --session {SESSION} fill @e2 "{PASSWORD}" +agent-browser --session {SESSION} click @e3 +agent-browser --session {SESSION} wait --load networkidle +``` + +For OTP/email codes: ask the user, wait for their response, then enter the code. + +After successful login, save state for potential reuse: + +```bash +agent-browser --session {SESSION} state save {OUTPUT_DIR}/auth-state.json +``` + +### 3. Orient + +Take an initial annotated screenshot and snapshot to understand the app structure: + +```bash +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/initial.png +agent-browser --session {SESSION} snapshot -i +``` + +Identify the main navigation elements and map out the sections to visit. + +### 4. Explore + +Read [references/issue-taxonomy.md](references/issue-taxonomy.md) for the full list of what to look for and the exploration checklist. + +**Strategy -- work through the app systematically:** + +- Start from the main navigation. Visit each top-level section. +- Within each section, test interactive elements: click buttons, fill forms, open dropdowns/modals. +- Check edge cases: empty states, error handling, boundary inputs. +- Try realistic end-to-end workflows (create, edit, delete flows). +- Check the browser console for errors periodically. + +**At each page:** + +```bash +agent-browser --session {SESSION} snapshot -i +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/{page-name}.png +agent-browser --session {SESSION} errors +agent-browser --session {SESSION} console +``` + +Use your judgment on how deep to go. Spend more time on core features and less on peripheral pages. If you find a cluster of issues in one area, investigate deeper. + +### 5. Document Issues (Repro-First) + +Steps 4 and 5 happen together -- explore and document in a single pass. When you find an issue, stop exploring and document it immediately before moving on. Do not explore the whole app first and document later. + +Every issue must be reproducible. When you find something wrong, do not just note it -- prove it with evidence. The goal is that someone reading the report can see exactly what happened and replay it. + +**Choose the right level of evidence for the issue:** + +#### Interactive / behavioral issues (functional, ux, console errors on action) + +These require user interaction to reproduce -- use full repro with video and step-by-step screenshots: + +1. **Start a repro video** _before_ reproducing: + +```bash +agent-browser --session {SESSION} record start {OUTPUT_DIR}/videos/issue-{NNN}-repro.webm +``` + +2. **Walk through the steps at human pace.** Pause 1-2 seconds between actions so the video is watchable. Take a screenshot at each step: + +```bash +agent-browser --session {SESSION} screenshot {OUTPUT_DIR}/screenshots/issue-{NNN}-step-1.png +sleep 1 +# Perform action (click, fill, etc.) +sleep 1 +agent-browser --session {SESSION} screenshot {OUTPUT_DIR}/screenshots/issue-{NNN}-step-2.png +sleep 1 +# ...continue until the issue manifests +``` + +3. **Capture the broken state.** Pause so the viewer can see it, then take an annotated screenshot: + +```bash +sleep 2 +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/issue-{NNN}-result.png +``` + +4. **Stop the video:** + +```bash +agent-browser --session {SESSION} record stop +``` + +5. Write numbered repro steps in the report, each referencing its screenshot. + +#### Static / visible-on-load issues (typos, placeholder text, clipped text, misalignment, console errors on load) + +These are visible without interaction -- a single annotated screenshot is sufficient. No video, no multi-step repro: + +```bash +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/issue-{NNN}.png +``` + +Write a brief description and reference the screenshot in the report. Set **Repro Video** to `N/A`. + +--- + +**For all issues:** + +1. **Append to the report immediately.** Do not batch issues for later. Write each one as you find it so nothing is lost if the session is interrupted. + +2. **Increment the issue counter** (ISSUE-001, ISSUE-002, ...). + +### 6. Wrap Up + +Aim to find **5-10 well-documented issues**, then wrap up. Depth of evidence matters more than total count -- 5 issues with full repro beats 20 with vague descriptions. + +After exploring: + +1. Re-read the report and update the summary severity counts so they match the actual issues. Every `### ISSUE-` block must be reflected in the totals. +2. Close the session: + +```bash +agent-browser --session {SESSION} close +``` + +3. Tell the user the report is ready and summarize findings: total issues, breakdown by severity, and the most critical items. + +## Guidance + +- **Repro is everything.** Every issue needs proof -- but match the evidence to the issue. Interactive bugs need video and step-by-step screenshots. Static bugs (typos, placeholder text, visual glitches visible on load) only need a single annotated screenshot. +- **Don't record video for static issues.** A typo or clipped text doesn't benefit from a video. Save video for issues that involve user interaction, timing, or state changes. +- **For interactive issues, screenshot each step.** Capture the before, the action, and the after -- so someone can see the full sequence. +- **Write repro steps that map to screenshots.** Each numbered step in the report should reference its corresponding screenshot. A reader should be able to follow the steps visually without touching a browser. +- **Be thorough but use judgment.** You are not following a test script -- you are exploring like a real user would. If something feels off, investigate. +- **Write findings incrementally.** Append each issue to the report as you discover it. If the session is interrupted, findings are preserved. Never batch all issues for the end. +- **Never delete output files.** Do not `rm` screenshots, videos, or the report mid-session. Do not close the session and restart. Work forward, not backward. +- **Never read the target app's source code.** You are testing as a user, not auditing code. Do not read HTML, JS, or config files of the app under test. All findings must come from what you observe in the browser. +- **Check the console.** Many issues are invisible in the UI but show up as JS errors or failed requests. +- **Test like a user, not a robot.** Try common workflows end-to-end. Click things a real user would click. Enter realistic data. +- **Type like a human.** When filling form fields during video recording, use `type` instead of `fill` -- it types character-by-character. Use `fill` only outside of video recording when speed matters. +- **Pace repro videos for humans.** Add `sleep 1` between actions and `sleep 2` before the final result screenshot. Videos should be watchable at 1x speed -- a human reviewing the report needs to see what happened, not a blur of instant state changes. +- **Be efficient with commands.** Batch multiple `agent-browser` commands in a single shell call when they are independent (e.g., `agent-browser ... screenshot ... && agent-browser ... console`). Use `agent-browser --session {SESSION} scroll down 300` for scrolling -- do not use `key` or `evaluate` to scroll. + +## References + +| Reference | When to Read | +|-----------|--------------| +| [references/issue-taxonomy.md](references/issue-taxonomy.md) | Start of session -- calibrate what to look for, severity levels, exploration checklist | + +## Templates + +| Template | Purpose | +|----------|---------| +| [templates/dogfood-report-template.md](templates/dogfood-report-template.md) | Copy into output directory as the report file | diff --git a/skills/dogfood/references/issue-taxonomy.md b/skills/dogfood/references/issue-taxonomy.md new file mode 100644 index 0000000..c3edbe5 --- /dev/null +++ b/skills/dogfood/references/issue-taxonomy.md @@ -0,0 +1,109 @@ +# Issue Taxonomy + +Reference for categorizing issues found during dogfooding. Read this at the start of a dogfood session to calibrate what to look for. + +## Contents + +- [Severity Levels](#severity-levels) +- [Categories](#categories) +- [Exploration Checklist](#exploration-checklist) + +## Severity Levels + +| Severity | Definition | +|----------|------------| +| **critical** | Blocks a core workflow, causes data loss, or crashes the app | +| **high** | Major feature broken or unusable, no workaround | +| **medium** | Feature works but with noticeable problems, workaround exists | +| **low** | Minor cosmetic or polish issue | + +## Categories + +### Visual / UI + +- Layout broken or misaligned elements +- Overlapping or clipped text +- Inconsistent spacing, padding, or margins +- Missing or broken icons/images +- Dark mode / light mode rendering issues +- Responsive layout problems (viewport sizes) +- Z-index stacking issues (elements hidden behind others) +- Font rendering issues (wrong font, size, weight) +- Color contrast problems +- Animation glitches or jank + +### Functional + +- Broken links (404, wrong destination) +- Buttons or controls that do nothing on click +- Form validation that rejects valid input or accepts invalid input +- Incorrect redirects +- Features that fail silently +- State not persisted when expected (lost on refresh, navigation) +- Race conditions (double-submit, stale data) +- Broken search or filtering +- Pagination issues +- File upload/download failures + +### UX + +- Confusing or unclear navigation +- Missing loading indicators or feedback after actions +- Slow or unresponsive interactions (>300ms perceived delay) +- Unclear error messages +- Missing confirmation for destructive actions +- Dead ends (no way to go back or proceed) +- Inconsistent patterns across similar features +- Missing keyboard shortcuts or focus management +- Unintuitive defaults +- Missing empty states or unhelpful empty states + +### Content + +- Typos or grammatical errors +- Outdated or incorrect text +- Placeholder or lorem ipsum content left in +- Truncated text without tooltip or expansion +- Missing or wrong labels +- Inconsistent terminology + +### Performance + +- Slow page loads (>3s) +- Janky scrolling or animations +- Large layout shifts (content jumping) +- Excessive network requests (check via console/network) +- Memory leaks (page slows over time) +- Unoptimized images (large file sizes) + +### Console / Errors + +- JavaScript exceptions in console +- Failed network requests (4xx, 5xx) +- Deprecation warnings +- CORS errors +- Mixed content warnings +- Unhandled promise rejections + +### Accessibility + +- Missing alt text on images +- Unlabeled form inputs +- Poor keyboard navigation (can't tab to elements) +- Focus traps +- Insufficient color contrast +- Missing ARIA attributes on dynamic content +- Screen reader incompatible patterns + +## Exploration Checklist + +Use this as a guide for what to test on each page/feature: + +1. **Visual scan** -- Take an annotated screenshot. Look for layout, alignment, and rendering issues. +2. **Interactive elements** -- Click every button, link, and control. Do they work? Is there feedback? +3. **Forms** -- Fill and submit. Test empty submission, invalid input, and edge cases. +4. **Navigation** -- Follow all navigation paths. Check breadcrumbs, back button, deep links. +5. **States** -- Check empty states, loading states, error states, and full/overflow states. +6. **Console** -- Check for JS errors, failed requests, and warnings. +7. **Responsiveness** -- If relevant, test at different viewport sizes. +8. **Auth boundaries** -- Test what happens when not logged in, with different roles if applicable. diff --git a/skills/dogfood/templates/dogfood-report-template.md b/skills/dogfood/templates/dogfood-report-template.md new file mode 100644 index 0000000..a7732a4 --- /dev/null +++ b/skills/dogfood/templates/dogfood-report-template.md @@ -0,0 +1,53 @@ +# Dogfood Report: {APP_NAME} + +| Field | Value | +|-------|-------| +| **Date** | {DATE} | +| **App URL** | {URL} | +| **Session** | {SESSION_NAME} | +| **Scope** | {SCOPE} | + +## Summary + +| Severity | Count | +|----------|-------| +| Critical | 0 | +| High | 0 | +| Medium | 0 | +| Low | 0 | +| **Total** | **0** | + +## Issues + + + +### ISSUE-001: {Short title} + +| Field | Value | +|-------|-------| +| **Severity** | critical / high / medium / low | +| **Category** | visual / functional / ux / content / performance / console / accessibility | +| **URL** | {page URL where issue was found} | +| **Repro Video** | {path to video, or N/A for static issues} | + +**Description** + +{What is wrong, what was expected, and what actually happened.} + +**Repro Steps** + + + +1. Navigate to {URL} + ![Step 1](screenshots/issue-001-step-1.png) + +2. {Action -- e.g., click "Settings" in the sidebar} + ![Step 2](screenshots/issue-001-step-2.png) + +3. {Action -- e.g., type "test" in the search field and press Enter} + ![Step 3](screenshots/issue-001-step-3.png) + +4. **Observe:** {what goes wrong -- e.g., the page shows a blank white screen instead of search results} + ![Result](screenshots/issue-001-result.png) + +--- diff --git a/test/e2e/dogfood.eval.ts b/test/e2e/dogfood.eval.ts new file mode 100644 index 0000000..6e5469e --- /dev/null +++ b/test/e2e/dogfood.eval.ts @@ -0,0 +1,261 @@ +import { describe, it, expect, beforeAll } from 'vitest'; +import { query } from '@anthropic-ai/claude-agent-sdk'; +import type { SDKMessage, SDKResultMessage } from '@anthropic-ai/claude-agent-sdk'; +import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, readdirSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +const AI_GATEWAY_URL = + process.env.ANTHROPIC_BASE_URL || 'https://ai-gateway.vercel.sh'; +const API_KEY = process.env.AI_GATEWAY_API_KEY; +const MODEL = process.env.DOGFOOD_MODEL || 'anthropic/claude-haiku-4.5'; +const CUSTOM_URL = process.env.DOGFOOD_URL; + +const FIXTURE_PATH = path.resolve('test/e2e/fixtures/buggy-app.html'); +const SKILL_PATH = path.resolve('skills/dogfood/SKILL.md'); +const TARGET_URL = CUSTOM_URL || `file://${FIXTURE_PATH}`; +const IS_FIXTURE = !CUSTOM_URL; + +const OUTPUT_DIR = path.resolve('test/e2e/.dogfood-output'); +const EVAL_TIMEOUT = 10 * 60 * 1000; + +async function runDogfood(outputDir: string): Promise<{ + result: SDKResultMessage | null; + messages: SDKMessage[]; + toolsUsed: Set; +}> { + const instruction = [ + `Read the dogfood skill at ${SKILL_PATH} and follow its workflow.`, + `Dogfood ${TARGET_URL}`, + `Output directory: ${outputDir}`, + ].join(' '); + + const messages: SDKMessage[] = []; + const toolsUsed = new Set(); + let result: SDKResultMessage | null = null; + + const conversation = query({ + prompt: instruction, + options: { + model: MODEL, + cwd: process.cwd(), + allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], + permissionMode: 'bypassPermissions', + allowDangerouslySkipPermissions: true, + maxTurns: 80, + maxBudgetUsd: 2, + settingSources: ['project'], + persistSession: false, + env: { + ...process.env, + ANTHROPIC_BASE_URL: AI_GATEWAY_URL, + ANTHROPIC_API_KEY: API_KEY, + }, + }, + }); + + const verbose = process.env.DOGFOOD_VERBOSE !== '0'; + const log = verbose ? (msg: string) => process.stderr.write(` [dogfood] ${msg}\n`) : () => {}; + + const chatLogPath = path.join(outputDir, 'chat-log.jsonl'); + writeFileSync(chatLogPath, ''); + + function appendToLog(entry: Record) { + appendFileSync(chatLogPath, JSON.stringify(entry) + '\n'); + } + + for await (const message of conversation) { + messages.push(message); + + if (message.type === 'system' && message.subtype === 'init') { + log(`session started (model: ${message.model})`); + appendToLog({ type: 'system', subtype: 'init', model: message.model }); + } + + if (message.type === 'assistant' && message.message?.content) { + const logParts: Record[] = []; + for (const block of message.message.content) { + if ('type' in block && block.type === 'tool_use') { + toolsUsed.add(block.name); + const input = block.input as Record; + let preview: string; + if (block.name === 'Bash') { + const cmd = String(input.command ?? ''); + const firstLine = cmd.split('\n').find(l => l.trim() && !l.trim().startsWith('#')) ?? cmd.split('\n')[0]; + preview = firstLine.trim().slice(0, 200); + } else if (block.name === 'Write') { + preview = String(input.file_path ?? input.path ?? ''); + } else if (block.name === 'Read') { + preview = String(input.file_path ?? input.path ?? ''); + } else if (block.name === 'Edit') { + preview = String(input.file_path ?? input.path ?? ''); + } else { + preview = JSON.stringify(input).slice(0, 120); + } + log(`${block.name}: ${preview}`); + logParts.push({ tool: block.name, input: block.input }); + } + if ('type' in block && block.type === 'text' && block.text) { + const line = block.text.split('\n')[0].slice(0, 120); + if (line.trim()) log(line); + logParts.push({ text: block.text }); + } + } + appendToLog({ type: 'assistant', content: logParts }); + } + + if (message.type === 'result') { + result = message; + const cost = `$${message.total_cost_usd.toFixed(4)}`; + const usage = message.usage; + const cacheRead = usage.cache_read_input_tokens ?? 0; + const cacheCreate = usage.cache_creation_input_tokens ?? 0; + const inputTokens = usage.input_tokens ?? 0; + const cacheInfo = cacheRead > 0 + ? ` | cache: ${cacheRead} read, ${cacheCreate} created, ${inputTokens} uncached` + : ''; + if (message.subtype === 'success') { + log(`done (${message.num_turns} turns, ${cost}${cacheInfo})`); + } else { + log(`stopped: ${message.subtype} (${message.num_turns} turns, ${cost}${cacheInfo})`); + } + appendToLog({ type: 'result', subtype: message.subtype, num_turns: message.num_turns, cost: message.total_cost_usd }); + } + } + + log(`chat log: ${chatLogPath}`); + + return { result, messages, toolsUsed }; +} + +function findFiles(dir: string, ext: string): string[] { + if (!existsSync(dir)) return []; + return readdirSync(dir, { recursive: true }) + .map(String) + .filter((f) => f.endsWith(ext)); +} + +describe.skipIf(!API_KEY)('Dogfood e2e eval (Agent SDK)', () => { + const outputDir = OUTPUT_DIR; + let evalResult: Awaited>; + + beforeAll(async () => { + if (existsSync(outputDir)) { + rmSync(outputDir, { recursive: true, force: true }); + } + mkdirSync(outputDir, { recursive: true }); + evalResult = await runDogfood(outputDir); + }, EVAL_TIMEOUT); + + it('completes without hard failure', () => { + expect(evalResult.result, 'No result message received').toBeTruthy(); + const acceptable = ['success', 'error_max_turns', 'error_max_budget_usd']; + expect( + acceptable, + `Agent failed unexpectedly: ${evalResult.result!.subtype}` + ).toContain(evalResult.result!.subtype); + }); + + it('used agent-browser via Bash tool', () => { + expect( + evalResult.toolsUsed.has('Bash'), + 'Agent never used Bash (needed for agent-browser commands)' + ).toBe(true); + }); + + it('produced a report file', () => { + const reportPath = path.join(outputDir, 'report.md'); + expect(existsSync(reportPath), 'report.md not found in output dir').toBe( + true + ); + }); + + it('found a minimum number of issues', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + + const issueBlocks = report.match(/###\s+ISSUE-\d+/g) || []; + if (IS_FIXTURE) { + expect( + issueBlocks.length, + `Expected >=2 issues from fixture, found ${issueBlocks.length}` + ).toBeGreaterThanOrEqual(2); + } else { + expect(issueBlocks.length).toBeGreaterThanOrEqual(1); + } + }); + + it('each issue has required fields and repro evidence', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + + const issueSections = report.split(/(?=###\s+ISSUE-\d+)/).slice(1); + for (const section of issueSections) { + const issueId = section.match(/ISSUE-\d+/)?.[0] ?? 'unknown'; + + expect(section, `${issueId}: missing Severity`).toMatch( + /\*\*Severity\*\*/i + ); + + const sevMatch = section.match( + /\*\*Severity\*\*\s*\|?\s*(critical|high|medium|low)/i + ); + expect(sevMatch, `${issueId}: invalid severity value`).toBeTruthy(); + + expect(section, `${issueId}: missing Category`).toMatch( + /\*\*Category\*\*/i + ); + + expect(section, `${issueId}: missing URL`).toMatch(/\*\*URL\*\*/i); + + expect(section, `${issueId}: missing Repro Video field`).toMatch( + /\*\*Repro Video\*\*/i + ); + + const hasScreenshot = /!\[.*?\]\(.*?\)/.test(section); + const hasReproSteps = /\*\*Repro Steps\*\*/i.test(section); + expect( + hasScreenshot || hasReproSteps, + `${issueId}: needs either screenshot refs or repro steps` + ).toBe(true); + } + }); + + it('has a summary table with non-zero total', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + + expect(report, 'Missing Summary section').toContain('## Summary'); + const totalMatch = report.match(/\*\*Total\*\*\s*\|?\s*\*\*(\d+)\*\*/); + expect(totalMatch, 'Summary Total not found').toBeTruthy(); + if (totalMatch) { + const total = parseInt(totalMatch[1], 10); + expect(total, 'Summary Total should be > 0').toBeGreaterThan(0); + } + }); + + it('produced screenshot files', () => { + const screenshotsDir = path.join(outputDir, 'screenshots'); + const screenshots = findFiles(screenshotsDir, '.png'); + expect( + screenshots.length, + 'No screenshot files found in output' + ).toBeGreaterThan(0); + }); + + it('produced video files for interactive issues', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + const hasVideoRefs = /videos\/issue-\d+/.test(report); + if (!hasVideoRefs) return; + const videosDir = path.join(outputDir, 'videos'); + const videos = findFiles(videosDir, '.webm'); + expect( + videos.length, + 'Report references videos but none were found' + ).toBeGreaterThan(0); + }); +}); diff --git a/test/e2e/dogfood.test.ts b/test/e2e/dogfood.test.ts new file mode 100644 index 0000000..e0d5d31 --- /dev/null +++ b/test/e2e/dogfood.test.ts @@ -0,0 +1,210 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync } from 'node:fs'; +import path from 'node:path'; + +const SKILL_DIR = path.resolve('skills/dogfood'); +const SKILL_MD = path.join(SKILL_DIR, 'SKILL.md'); +const TAXONOMY_MD = path.join(SKILL_DIR, 'references', 'issue-taxonomy.md'); +const TEMPLATE_MD = path.join(SKILL_DIR, 'templates', 'dogfood-report-template.md'); + +function readSkillFile(filePath: string): string { + return readFileSync(filePath, 'utf-8'); +} + +function parseFrontmatter(content: string): Record { + const match = content.match(/^---\n([\s\S]*?)\n---/); + if (!match) return {}; + const fields: Record = {}; + for (const line of match[1].split('\n')) { + const colonIdx = line.indexOf(':'); + if (colonIdx > 0) { + fields[line.slice(0, colonIdx).trim()] = line.slice(colonIdx + 1).trim(); + } + } + return fields; +} + +describe('Dogfood skill: file structure', () => { + it('SKILL.md exists', () => { + expect(existsSync(SKILL_MD)).toBe(true); + }); + + it('references/issue-taxonomy.md exists', () => { + expect(existsSync(TAXONOMY_MD)).toBe(true); + }); + + it('templates/dogfood-report-template.md exists', () => { + expect(existsSync(TEMPLATE_MD)).toBe(true); + }); +}); + +describe('Dogfood skill: SKILL.md frontmatter', () => { + const content = readSkillFile(SKILL_MD); + const frontmatter = parseFrontmatter(content); + + it('has name field', () => { + expect(frontmatter.name).toBe('dogfood'); + }); + + it('has description field', () => { + expect(frontmatter.description).toBeTruthy(); + expect(frontmatter.description!.length).toBeGreaterThan(50); + }); + + it('has allowed-tools field', () => { + expect(frontmatter['allowed-tools']).toBeTruthy(); + expect(frontmatter['allowed-tools']).toContain('agent-browser'); + }); +}); + +describe('Dogfood skill: SKILL.md body references', () => { + const content = readSkillFile(SKILL_MD); + + it('references issue-taxonomy.md', () => { + expect(content).toContain('references/issue-taxonomy.md'); + }); + + it('references dogfood-report-template.md', () => { + expect(content).toContain('templates/dogfood-report-template.md'); + }); + + it('referenced files exist on disk', () => { + const refPattern = /\[.*?\]\((references\/.*?\.md|templates\/.*?\.md)\)/g; + const refs = [...content.matchAll(refPattern)].map((m) => m[1]); + expect(refs.length).toBeGreaterThan(0); + for (const ref of refs) { + const fullPath = path.join(SKILL_DIR, ref); + expect(existsSync(fullPath), `Missing: ${ref}`).toBe(true); + } + }); +}); + +describe('Dogfood skill: report template', () => { + const template = readSkillFile(TEMPLATE_MD); + + it('has ISSUE- prefix in issue blocks', () => { + expect(template).toContain('ISSUE-'); + }); + + it('has Severity field', () => { + expect(template).toContain('**Severity**'); + }); + + it('has Category field', () => { + expect(template).toContain('**Category**'); + }); + + it('has URL field', () => { + expect(template).toContain('**URL**'); + }); + + it('has Repro Video field', () => { + expect(template).toContain('**Repro Video**'); + }); + + it('has Repro Steps section', () => { + expect(template).toContain('**Repro Steps**'); + }); + + it('has screenshot image references in repro steps', () => { + expect(template).toMatch(/!\[.*?\]\(screenshots\//); + }); + + it('lists all valid severity values', () => { + expect(template).toMatch(/critical\s*\/\s*high\s*\/\s*medium\s*\/\s*low/); + }); + + it('lists all valid category values', () => { + const categoryLine = template + .split('\n') + .find((l) => l.includes('**Category**')); + expect(categoryLine).toBeTruthy(); + for (const cat of [ + 'visual', + 'functional', + 'ux', + 'content', + 'performance', + 'console', + 'accessibility', + ]) { + expect(categoryLine!.toLowerCase()).toContain(cat); + } + }); + + it('has Summary table with severity counts', () => { + expect(template).toContain('## Summary'); + for (const sev of ['Critical', 'High', 'Medium', 'Low', 'Total']) { + expect(template).toContain(sev); + } + }); +}); + +describe('Dogfood skill: issue taxonomy', () => { + const taxonomy = readSkillFile(TAXONOMY_MD); + + it('has severity level definitions', () => { + expect(taxonomy).toContain('## Severity Levels'); + for (const sev of ['critical', 'high', 'medium', 'low']) { + expect(taxonomy.toLowerCase()).toContain(`**${sev}**`); + } + }); + + it('has all 7 category sections', () => { + const expectedCategories = [ + 'Visual', + 'Functional', + 'UX', + 'Content', + 'Performance', + 'Console', + 'Accessibility', + ]; + for (const cat of expectedCategories) { + expect(taxonomy).toMatch(new RegExp(`###\\s+.*${cat}`, 'i')); + } + }); + + it('has exploration checklist', () => { + expect(taxonomy).toContain('## Exploration Checklist'); + }); + + it('checklist has numbered items', () => { + const checklistSection = taxonomy.split('## Exploration Checklist')[1]; + expect(checklistSection).toBeTruthy(); + const numberedItems = checklistSection!.match(/^\d+\./gm); + expect(numberedItems!.length).toBeGreaterThanOrEqual(5); + }); +}); + +describe('Dogfood skill: cross-consistency', () => { + const template = readSkillFile(TEMPLATE_MD); + const taxonomy = readSkillFile(TAXONOMY_MD); + + it('every category in template exists in taxonomy', () => { + const categoryLine = template + .split('\n') + .find((l) => l.includes('**Category**')); + expect(categoryLine).toBeTruthy(); + + const categories = categoryLine! + .split('|') + .pop()! + .split('/') + .map((c) => c.trim().toLowerCase()) + .filter(Boolean); + + for (const cat of categories) { + expect( + taxonomy.toLowerCase(), + `Category "${cat}" from template not found in taxonomy` + ).toMatch(new RegExp(`###\\s+.*${cat}`)); + } + }); + + it('every severity in template exists in taxonomy', () => { + for (const sev of ['critical', 'high', 'medium', 'low']) { + expect(taxonomy.toLowerCase()).toContain(`**${sev}**`); + } + }); +}); diff --git a/test/e2e/fixtures/buggy-app.html b/test/e2e/fixtures/buggy-app.html new file mode 100644 index 0000000..6c45d83 --- /dev/null +++ b/test/e2e/fixtures/buggy-app.html @@ -0,0 +1,159 @@ + + + + + + Buggy App - Dogfood Test Fixture + + + + +
+

Buggy App

+ +
+ +
+ +

Welocme to the Dashboard

+ + +
+

Quick Actions

+

Perform common tasks from here.

+
+ + + +
+
+ + +
+

System Status

+ +
+ The system is currently operating normally. All services are online and responding within expected latency thresholds. Last health check completed at 14:32 UTC. +
+ +
+
+ All systems operational +
+
+ + +
+

Recent Activity

+ +

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.

+
+ + +
+

Contact Support

+
+ + + + + + +
+
+ + +
+

Notifications

+ +
+
+
+
+ +
+ © 2025 Buggy App Inc. All rights reserved. +
+ + + + + diff --git a/vitest.config.ts b/vitest.config.ts index 31a6325..f7e8d9c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { globals: true, - include: ['src/**/*.test.ts', 'test/**/*.test.ts'], + include: ['src/**/*.test.ts', 'test/**/*.test.ts', 'test/**/*.eval.ts'], testTimeout: 30000, }, });