diff --git a/agent-browser.schema.json b/agent-browser.schema.json deleted file mode 100644 index 48e38ba..0000000 --- a/agent-browser.schema.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Agent Browser Configuration", - "description": "Configuration file for agent-browser (e.g., agent-browser.json or ~/.agent-browser/config.json)", - "type": "object", - "properties": { - "headed": { - "type": "boolean", - "description": "Show browser window instead of running headless." - }, - "json": { - "type": "boolean", - "description": "Output in JSON format." - }, - "debug": { - "type": "boolean", - "description": "Enable debug output." - }, - "session": { - "type": "string", - "description": "Session identifier." - }, - "sessionName": { - "type": "string", - "description": "Auto-save/load state persistence name." - }, - "executablePath": { - "type": "string", - "description": "Path to a custom browser executable." - }, - "extensions": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Paths to browser extensions. Extensions from user-level and project-level configs are concatenated." - }, - "profile": { - "type": "string", - "description": "Path to the browser profile data directory." - }, - "state": { - "type": "string", - "description": "Path to load/save browser state." - }, - "proxy": { - "type": "string", - "description": "Proxy server URL (e.g., http://localhost:8080)." - }, - "proxyBypass": { - "type": "string", - "description": "Comma-separated domains to bypass the proxy (e.g., localhost,*.internal.com)." - }, - "args": { - "type": "string", - "description": "Additional comma-separated launch arguments for the browser." - }, - "userAgent": { - "type": "string", - "description": "Custom User-Agent string." - }, - "provider": { - "type": "string", - "description": "Provider to use, such as 'ios'." - }, - "device": { - "type": "string", - "description": "Device name or identifier for emulation or providers (e.g., 'iPhone 16 Pro')." - }, - "ignoreHttpsErrors": { - "type": "boolean", - "description": "Ignore HTTPS errors during navigation." - }, - "allowFileAccess": { - "type": "boolean", - "description": "Allow file:// URLs to access local files." - }, - "cdp": { - "type": "string", - "description": "Chrome DevTools Protocol endpoint URL." - }, - "autoConnect": { - "type": "boolean", - "description": "Auto-discover and connect to a running Chrome instance." - }, - "annotate": { - "type": "boolean", - "description": "Annotated screenshot with numbered element labels." - }, - "colorScheme": { - "type": "string", - "enum": ["dark", "light", "no-preference"], - "description": "Color scheme preference." - }, - "downloadPath": { - "type": "string", - "description": "Default directory for browser downloads." - }, - "contentBoundaries": { - "type": "boolean", - "description": "Wrap page output in boundary markers for LLM safety." - }, - "maxOutput": { - "type": "integer", - "minimum": 0, - "description": "Max characters for page output (truncates beyond limit)." - }, - "allowedDomains": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Allowed domain patterns (e.g., ['example.com', '*.example.com'])." - }, - "actionPolicy": { - "type": "string", - "description": "Path to action policy JSON file." - }, - "confirmActions": { - "type": "string", - "description": "Comma-separated action categories requiring confirmation." - }, - "confirmInteractive": { - "type": "boolean", - "description": "Enable interactive confirmation prompts (auto-denies if stdin is not a TTY)." - }, - "engine": { - "type": "string", - "enum": ["chrome", "lightpanda"], - "default": "chrome", - "description": "Browser engine to use." - }, - "screenshotDir": { - "type": "string", - "description": "Default screenshot output directory." - }, - "screenshotQuality": { - "type": "integer", - "minimum": 0, - "maximum": 100, - "description": "JPEG quality for screenshots (0-100)." - }, - "screenshotFormat": { - "type": "string", - "enum": ["png", "jpeg"], - "description": "Screenshot format." - }, - "idleTimeout": { - "type": "string", - "description": "Auto-shutdown the daemon after inactivity (e.g., '30s', '5m', '1h', or raw milliseconds like '60000')." - }, - "model": { - "type": "string", - "description": "AI model for chat command (e.g., 'openai/gpt-4o')." - }, - "noAutoDialog": { - "type": "boolean", - "description": "Disable automatic dismissal of alert/beforeunload dialogs." - }, - "headers": { - "type": "string", - "description": "Custom HTTP headers supplied as a JSON-formatted string." - } - }, - "additionalProperties": true -} diff --git a/docs/public/schema.json b/docs/public/schema.json deleted file mode 100644 index 48e38ba..0000000 --- a/docs/public/schema.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Agent Browser Configuration", - "description": "Configuration file for agent-browser (e.g., agent-browser.json or ~/.agent-browser/config.json)", - "type": "object", - "properties": { - "headed": { - "type": "boolean", - "description": "Show browser window instead of running headless." - }, - "json": { - "type": "boolean", - "description": "Output in JSON format." - }, - "debug": { - "type": "boolean", - "description": "Enable debug output." - }, - "session": { - "type": "string", - "description": "Session identifier." - }, - "sessionName": { - "type": "string", - "description": "Auto-save/load state persistence name." - }, - "executablePath": { - "type": "string", - "description": "Path to a custom browser executable." - }, - "extensions": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Paths to browser extensions. Extensions from user-level and project-level configs are concatenated." - }, - "profile": { - "type": "string", - "description": "Path to the browser profile data directory." - }, - "state": { - "type": "string", - "description": "Path to load/save browser state." - }, - "proxy": { - "type": "string", - "description": "Proxy server URL (e.g., http://localhost:8080)." - }, - "proxyBypass": { - "type": "string", - "description": "Comma-separated domains to bypass the proxy (e.g., localhost,*.internal.com)." - }, - "args": { - "type": "string", - "description": "Additional comma-separated launch arguments for the browser." - }, - "userAgent": { - "type": "string", - "description": "Custom User-Agent string." - }, - "provider": { - "type": "string", - "description": "Provider to use, such as 'ios'." - }, - "device": { - "type": "string", - "description": "Device name or identifier for emulation or providers (e.g., 'iPhone 16 Pro')." - }, - "ignoreHttpsErrors": { - "type": "boolean", - "description": "Ignore HTTPS errors during navigation." - }, - "allowFileAccess": { - "type": "boolean", - "description": "Allow file:// URLs to access local files." - }, - "cdp": { - "type": "string", - "description": "Chrome DevTools Protocol endpoint URL." - }, - "autoConnect": { - "type": "boolean", - "description": "Auto-discover and connect to a running Chrome instance." - }, - "annotate": { - "type": "boolean", - "description": "Annotated screenshot with numbered element labels." - }, - "colorScheme": { - "type": "string", - "enum": ["dark", "light", "no-preference"], - "description": "Color scheme preference." - }, - "downloadPath": { - "type": "string", - "description": "Default directory for browser downloads." - }, - "contentBoundaries": { - "type": "boolean", - "description": "Wrap page output in boundary markers for LLM safety." - }, - "maxOutput": { - "type": "integer", - "minimum": 0, - "description": "Max characters for page output (truncates beyond limit)." - }, - "allowedDomains": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Allowed domain patterns (e.g., ['example.com', '*.example.com'])." - }, - "actionPolicy": { - "type": "string", - "description": "Path to action policy JSON file." - }, - "confirmActions": { - "type": "string", - "description": "Comma-separated action categories requiring confirmation." - }, - "confirmInteractive": { - "type": "boolean", - "description": "Enable interactive confirmation prompts (auto-denies if stdin is not a TTY)." - }, - "engine": { - "type": "string", - "enum": ["chrome", "lightpanda"], - "default": "chrome", - "description": "Browser engine to use." - }, - "screenshotDir": { - "type": "string", - "description": "Default screenshot output directory." - }, - "screenshotQuality": { - "type": "integer", - "minimum": 0, - "maximum": 100, - "description": "JPEG quality for screenshots (0-100)." - }, - "screenshotFormat": { - "type": "string", - "enum": ["png", "jpeg"], - "description": "Screenshot format." - }, - "idleTimeout": { - "type": "string", - "description": "Auto-shutdown the daemon after inactivity (e.g., '30s', '5m', '1h', or raw milliseconds like '60000')." - }, - "model": { - "type": "string", - "description": "AI model for chat command (e.g., 'openai/gpt-4o')." - }, - "noAutoDialog": { - "type": "boolean", - "description": "Disable automatic dismissal of alert/beforeunload dialogs." - }, - "headers": { - "type": "string", - "description": "Custom HTTP headers supplied as a JSON-formatted string." - } - }, - "additionalProperties": true -} diff --git a/docs/src/lib/github.ts b/docs/src/lib/github.ts deleted file mode 100644 index 2c3f78e..0000000 --- a/docs/src/lib/github.ts +++ /dev/null @@ -1,20 +0,0 @@ -const REPO = "vercel-labs/agent-browser"; -const REVALIDATE = 86400; - -export async function getStarCount(): Promise { - try { - const res = await fetch(`https://api.github.com/repos/${REPO}`, { - headers: { Accept: "application/vnd.github.v3+json" }, - next: { revalidate: REVALIDATE }, - }); - if (!res.ok) return ""; - const data = await res.json(); - const count = data.stargazers_count; - if (typeof count !== "number") return ""; - if (count >= 1000) - return `${(count / 1000).toFixed(count >= 10000 ? 0 : 1)}k`; - return String(count); - } catch { - return ""; - } -} diff --git a/evals/.env.example b/evals/.env.example deleted file mode 100644 index 0a16d9f..0000000 --- a/evals/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -# Vercel AI Gateway key (required) -AI_GATEWAY_API_KEY= diff --git a/evals/.gitignore b/evals/.gitignore deleted file mode 100644 index 8c68c1a..0000000 --- a/evals/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -node_modules/ -dist/ -bun.lockb diff --git a/evals/README.md b/evals/README.md deleted file mode 100644 index 72ebdf6..0000000 --- a/evals/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Skills Evals - -Tests whether the thin SKILL.md + CLI-served skills approach works: do agents load the right skill via `agent-browser skills get`, then produce correct agent-browser commands? - -## Prerequisites - -- [Bun](https://bun.sh) installed -- `AI_GATEWAY_API_KEY` set (Vercel AI Gateway key) -- One or both CLIs installed: - - `claude` CLI (`npm i -g @anthropic-ai/claude-code`) for the Claude provider - - `codex` CLI (`npm i -g @openai/codex`) for the Codex provider - -The evals route all calls through the Vercel AI Gateway (`https://ai-gateway.vercel.sh`). Set your key before running: - -```bash -export AI_GATEWAY_API_KEY=gw_your_key_here -``` - -Or copy `.env.example` to `.env` and source it. - -## Usage - -```bash -cd evals - -# Run all evals (default: Claude provider) -bun run run.ts - -# Use Codex provider -bun run run.ts --provider codex - -# Filter by category -bun run run.ts --category skill-loading -bun run run.ts --category skill-selection -bun run run.ts --category command-usage - -# Use a specific model (overrides provider default) -bun run run.ts --model anthropic/claude-opus-4.6 -bun run run.ts --provider codex --model openai/gpt-4.1 - -# Enable LLM judge for quality scoring (1-5) -bun run run.ts --judge - -# JSON output (for CI or further analysis) -bun run run.ts --json - -# Combine options -bun run run.ts --provider codex --category skill-selection --judge -``` - -Or via package scripts: - -```bash -bun run eval # run all (Claude) -bun run eval:claude # run all (Claude, explicit) -bun run eval:codex # run all (Codex) -bun run eval:judge # run all with LLM judge -bun run eval:json # JSON output -``` - -## Providers - - - - - -
ProviderCLIDefault ModelNotes
claudeclaude -panthropic/claude-sonnet-4.6Uses ANTHROPIC_API_KEY + ANTHROPIC_BASE_URL env vars
codexcodex exec --jsonopenai/o3Writes ~/.codex/config.toml with AI Gateway config
- -The LLM judge always uses Claude (anthropic/claude-opus-4.6), regardless of the eval provider. - -## Eval Categories - -### skill-loading - -Tests that the agent runs `agent-browser skills get` before issuing browser commands. The thin SKILL.md instructs agents to load skills first; these evals verify compliance. - -### skill-selection - -Tests that the agent picks the correct specialized skill for the task. For example, a Slack task should load the `slack` skill, not the generic `agent-browser` skill. - -### command-usage - -Tests that the agent produces correct agent-browser commands for common workflows: navigation + screenshot, form filling with snapshot-interact pattern, diffing, authentication, data extraction. - -## How It Works - -1. Each eval case provides a user task prompt -2. The thin `skills/agent-browser/SKILL.md` is injected as context (simulating a skill installation) -3. The chosen provider CLI is called to get a single response -4. Pattern matching checks for expected/forbidden command patterns (pass/fail) -5. Optionally, a second Claude call judges response quality on a 1-5 scale - -## Adding Cases - -Create or edit files in `cases/`. Each file exports a `cases` array of `EvalCase` objects: - -```typescript -import type { EvalCase } from "../lib/types.ts"; - -export const cases: EvalCase[] = [ - { - id: "xx-01", - name: "Description of what this tests", - category: "skill-loading", - prompt: "The user task to send to the model", - expectedPatterns: ["regex.*that.*must.*match"], - forbiddenPatterns: ["regex.*that.*must.*not.*match"], - rubric: "1 - worst ... 5 - best", - }, -]; -``` - -Then import and add the cases to `ALL_CASES` in `run.ts`. - -## Output - -Console mode shows pass/fail per case with failed pattern details: - -``` -skill-loading ----------------------------------------------------------------------- - ✓ Loads skill before opening a page PASS 3200ms - ✗ Loads skill before form interaction FAIL 2800ms - ✗ Expected pattern not found: agent-browser skills get -``` - -JSON mode (`--json`) outputs structured results for programmatic consumption. diff --git a/evals/bun.lock b/evals/bun.lock deleted file mode 100644 index 72107d0..0000000 --- a/evals/bun.lock +++ /dev/null @@ -1,19 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agent-browser-evals", - "dependencies": { - "bun-types": "^1.3.12", - }, - }, - }, - "packages": { - "@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="], - - "bun-types": ["bun-types@1.3.12", "", { "dependencies": { "@types/node": "*" } }, "sha512-HqOLj5PoFajAQciOMRiIZGNoKxDJSr6qigAttOX40vJuSp6DN/CxWp9s3C1Xwm4oH7ybueITwiaOcWXoYVoRkA=="], - - "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], - } -} diff --git a/evals/cases/command-usage.ts b/evals/cases/command-usage.ts deleted file mode 100644 index 4fb47a9..0000000 --- a/evals/cases/command-usage.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type { EvalCase } from "../lib/types.ts"; - -const RUBRIC = ` -1 - Agent does not produce valid agent-browser commands -2 - Agent uses agent-browser but with wrong commands or missing steps -3 - Agent uses correct commands but skips the snapshot-interact workflow -4 - Agent follows the correct workflow with appropriate commands -5 - Agent follows the optimal workflow: navigate, snapshot, interact with refs, re-snapshot as needed -`.trim(); - -const COMMAND_CONTEXT = `You already ran \`agent-browser skills get core\` and loaded these commands: -- agent-browser open (navigate to a page) -- agent-browser snapshot -i (get interactive elements with refs like @e1, @e2) -- agent-browser click @ref (click element) -- agent-browser fill @ref "text" (clear and type) -- agent-browser type @ref "text" (type without clearing) -- agent-browser select @ref "option" (select dropdown) -- agent-browser screenshot (screenshot to temp dir) -- agent-browser screenshot --full (full page screenshot) -- agent-browser diff url (compare two pages) -- agent-browser diff snapshot (compare current vs last snapshot) -- agent-browser state save ./file.json (save auth state) -- agent-browser state load ./file.json (restore auth state) -- agent-browser get text @ref (get element text) -- agent-browser wait (wait for element or time) -- agent-browser --session-name open (named session with auto-save) - -Workflow: open -> snapshot -i -> interact with refs -> re-snapshot after changes.`; - -export const cases: EvalCase[] = [ - { - id: "cu-01", - name: "Navigate and screenshot workflow", - category: "command-usage", - prompt: "Open example.com and take a screenshot", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "agent-browser\\s+screenshot", - ], - rubric: RUBRIC, - }, - { - id: "cu-02", - name: "Form filling workflow", - category: "command-usage", - prompt: - "Go to example.com/signup, fill in name as 'Jane Doe' and email as 'jane@test.com', then submit", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "agent-browser\\s+snapshot", - "agent-browser\\s+(fill|type)", - "agent-browser\\s+(click|press|key)", - ], - rubric: RUBRIC, - }, - { - id: "cu-03", - name: "Snapshot with element refs", - category: "command-usage", - prompt: "Get all interactive elements on example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "agent-browser\\s+snapshot", - ], - rubric: RUBRIC, - }, - { - id: "cu-04", - name: "Diff comparison workflow", - category: "command-usage", - prompt: - "Compare the homepage of staging.example.com and prod.example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+diff|staging\\.example\\.com.*prod\\.example\\.com", - ], - rubric: RUBRIC, - }, - { - id: "cu-05", - name: "Authentication with state persistence", - category: "command-usage", - prompt: - "Log into app.example.com, then save the auth state for future sessions", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "state\\s+save|--session-name|auth\\s+save", - ], - rubric: RUBRIC, - }, - { - id: "cu-06", - name: "Data extraction workflow", - category: "command-usage", - prompt: - "Extract the text content of the main heading on example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "snapshot|get\\s+text", - ], - rubric: RUBRIC, - }, - { - id: "cu-07", - name: "Full-page screenshot", - category: "command-usage", - prompt: "Take a full-page screenshot of example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate|screenshot)", - "screenshot.*--full", - ], - rubric: RUBRIC, - }, -]; diff --git a/evals/cases/skill-loading.ts b/evals/cases/skill-loading.ts deleted file mode 100644 index 8f544df..0000000 --- a/evals/cases/skill-loading.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { EvalCase } from "../lib/types.ts"; - -const RUBRIC = ` -1 - Agent does not mention agent-browser skills or load any skill -2 - Agent mentions skills but does not run the skills get command -3 - Agent runs skills get but for the wrong skill or at the wrong time -4 - Agent runs skills get for the correct skill before using agent-browser -5 - Agent runs skills get first, then follows the loaded skill's workflow correctly -`.trim(); - -export const cases: EvalCase[] = [ - { - id: "sl-01", - name: "Loads skill before opening a page", - category: "skill-loading", - prompt: "Open example.com and take a screenshot", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-02", - name: "Loads skill before form interaction", - category: "skill-loading", - prompt: - "Go to example.com/contact and fill out the contact form with test data", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-03", - name: "Loads skill before data extraction", - category: "skill-loading", - prompt: - "Scrape all product names and prices from shop.example.com", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-04", - name: "Loads skill before authentication task", - category: "skill-loading", - prompt: - "Log into my GitHub account and check my notifications", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-05", - name: "Uses skills list to discover available skills", - category: "skill-loading", - prompt: - "I need to automate some browser tasks. What skills are available for agent-browser?", - expectedPatterns: [ - "agent-browser skills (list|get)", - ], - rubric: RUBRIC, - }, -]; diff --git a/evals/cases/skill-selection.ts b/evals/cases/skill-selection.ts deleted file mode 100644 index 0439f63..0000000 --- a/evals/cases/skill-selection.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { EvalCase } from "../lib/types.ts"; - -const RUBRIC = ` -1 - Agent does not load any skill or loads a completely wrong one -2 - Agent loads the generic agent-browser skill when a specialized one exists -3 - Agent loads a related but suboptimal skill -4 - Agent loads the correct specialized skill -5 - Agent loads the correct skill and explains why it chose it -`.trim(); - -export const cases: EvalCase[] = [ - { - id: "ss-01", - name: "Selects slack skill for Slack tasks", - category: "skill-selection", - prompt: "Check my Slack unreads and summarize any messages mentioning me", - expectedPatterns: [ - "skills get slack", - ], - rubric: RUBRIC, - }, - { - id: "ss-02", - name: "Selects electron skill for VS Code automation", - category: "skill-selection", - prompt: "Automate VS Code to open a project and run a terminal command", - expectedPatterns: [ - "skills get electron", - ], - rubric: RUBRIC, - }, - { - id: "ss-03", - name: "Selects dogfood skill for QA/testing", - category: "skill-selection", - prompt: "QA test http://localhost:3000 and find any bugs or UX issues", - expectedPatterns: [ - "skills get dogfood", - ], - rubric: RUBRIC, - }, - { - id: "ss-04", - name: "Selects agentcore skill for AWS cloud browsers", - category: "skill-selection", - prompt: - "Run browser automation on AWS using AgentCore cloud browsers", - expectedPatterns: [ - "skills get agentcore", - ], - rubric: RUBRIC, - }, - { - id: "ss-05", - name: "Selects vercel-sandbox skill for Vercel environments", - category: "skill-selection", - prompt: - "Run headless Chrome inside a Vercel Sandbox microVM to test my deployed Next.js app", - expectedPatterns: [ - "skills get vercel-sandbox", - ], - rubric: RUBRIC, - }, - { - id: "ss-06", - name: "Selects electron skill for Discord automation", - category: "skill-selection", - prompt: "Automate the Discord desktop app to send a message in a channel", - expectedPatterns: [ - "skills get electron", - ], - rubric: RUBRIC, - }, - { - id: "ss-07", - name: "Selects dogfood skill for exploratory testing", - category: "skill-selection", - prompt: "Dogfood vercel.com and write up a bug report", - expectedPatterns: [ - "skills get dogfood", - ], - rubric: RUBRIC, - }, - { - id: "ss-08", - name: "Selects core skill for general browser tasks", - category: "skill-selection", - prompt: "Navigate to hacker news and screenshot the front page", - expectedPatterns: [ - "skills get core", - ], - rubric: RUBRIC, - }, -]; diff --git a/evals/lib/claude.ts b/evals/lib/claude.ts deleted file mode 100644 index 069b184..0000000 --- a/evals/lib/claude.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { readFileSync } from "fs"; -import { resolve, dirname } from "path"; -import { fileURLToPath } from "url"; -import type { Provider, ProviderOptions, ProviderResponse } from "./types.ts"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const SKILL_PATH = resolve(__dirname, "../../skills/agent-browser/SKILL.md"); - -const AI_GATEWAY_URL = "https://ai-gateway.vercel.sh"; -const DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"; - -let cachedSkillContent: string | null = null; - -function getSkillContent(): string { - if (!cachedSkillContent) { - cachedSkillContent = readFileSync(SKILL_PATH, "utf-8"); - } - return cachedSkillContent; -} - -function buildPrompt(userTask: string, context?: string): string { - const skill = getSkillContent(); - const parts = [ - "You have the following skill installed:\n", - "", - skill, - "\n", - ]; - if (context) { - parts.push(context + "\n"); - } - parts.push( - `Complete this task: ${userTask}\n`, - "Show the exact shell commands you would run. Do not explain, just show the commands.", - ); - return parts.join("\n"); -} - -function getGatewayEnv(): Record { - const apiKey = process.env.AI_GATEWAY_API_KEY; - if (!apiKey) { - throw new Error( - "AI_GATEWAY_API_KEY is not set. Export it before running evals.", - ); - } - return { - ...(process.env as Record), - ANTHROPIC_API_KEY: apiKey, - ANTHROPIC_BASE_URL: AI_GATEWAY_URL, - }; -} - -function spawnClaude( - prompt: string, - model: string, - timeout: number, -): Promise<{ output: string; stderr: string; exitCode: number }> { - const proc = Bun.spawn( - ["claude", "-p", "--output-format", "text", "--model", model, prompt], - { - stdout: "pipe", - stderr: "pipe", - env: getGatewayEnv(), - }, - ); - - return Promise.race([ - (async () => { - const output = await new Response(proc.stdout).text(); - const stderr = await new Response(proc.stderr).text(); - const exitCode = await proc.exited; - return { output, stderr, exitCode }; - })(), - new Promise((_, reject) => - setTimeout(() => { - proc.kill(); - reject(new Error(`Timed out after ${timeout}ms`)); - }, timeout), - ), - ]); -} - -export const claudeProvider: Provider = { - name: "claude", - defaultModel: DEFAULT_MODEL, - - async call( - userPrompt: string, - options: ProviderOptions = {}, - context?: string, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 60_000 } = options; - const prompt = buildPrompt(userPrompt, context); - const start = performance.now(); - - try { - const result = await spawnClaude(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `claude exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - return { output: result.output.trim(), durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, - - async callRaw( - prompt: string, - options: ProviderOptions = {}, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 60_000 } = options; - const start = performance.now(); - - try { - const result = await spawnClaude(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `claude exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - return { output: result.output.trim(), durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, -}; diff --git a/evals/lib/codex.ts b/evals/lib/codex.ts deleted file mode 100644 index 3790219..0000000 --- a/evals/lib/codex.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { readFileSync, writeFileSync, mkdirSync, existsSync } from "fs"; -import { resolve, dirname, join } from "path"; -import { fileURLToPath } from "url"; -import { tmpdir } from "os"; -import type { Provider, ProviderOptions, ProviderResponse } from "./types.ts"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const SKILL_PATH = resolve(__dirname, "../../skills/agent-browser/SKILL.md"); - -const AI_GATEWAY_URL = "https://ai-gateway.vercel.sh/v1"; -const DEFAULT_MODEL = "openai/o3"; - -let cachedSkillContent: string | null = null; - -function getSkillContent(): string { - if (!cachedSkillContent) { - cachedSkillContent = readFileSync(SKILL_PATH, "utf-8"); - } - return cachedSkillContent; -} - -function buildPrompt(userTask: string, context?: string): string { - const skill = getSkillContent(); - const parts = [ - "You have the following skill installed:\n", - "", - skill, - "\n", - ]; - if (context) { - parts.push(context + "\n"); - } - parts.push( - `Complete this task: ${userTask}\n`, - "Show the exact shell commands you would run. Do not explain, just show the commands.", - ); - return parts.join("\n"); -} - -let evalHome: string | null = null; - -function getEvalHome(model: string): string { - if (!evalHome) { - evalHome = join(tmpdir(), `agent-browser-evals-${process.pid}`); - } - const configDir = join(evalHome, ".codex"); - const configPath = join(configDir, "config.toml"); - - const config = `model = "${model}" -model_provider = "vercel-ai-gateway" - -[model_providers.vercel-ai-gateway] -name = "Vercel AI Gateway" -base_url = "${AI_GATEWAY_URL}" -env_key = "AI_GATEWAY_API_KEY" -wire_api = "responses" -`; - - if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); - } - writeFileSync(configPath, config, "utf-8"); - return evalHome; -} - -function getCodexEnv(model: string): Record { - const apiKey = process.env.AI_GATEWAY_API_KEY; - if (!apiKey) { - throw new Error( - "AI_GATEWAY_API_KEY is not set. Export it before running evals.", - ); - } - return { - ...(process.env as Record), - HOME: getEvalHome(model), - AI_GATEWAY_API_KEY: apiKey, - }; -} - -function parseJsonlOutput(raw: string): string { - const lines = raw.split("\n"); - const textParts: string[] = []; - - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed) continue; - - try { - const parsed = JSON.parse(trimmed); - const eventType = parsed.type as string; - - if (eventType === "item.completed") { - const item = parsed.item as Record | undefined; - if (item?.type === "agent_message") { - const text = item.text as string | undefined; - if (text) textParts.push(text); - } - } - } catch { - // Non-JSON line (e.g. stderr leak), skip - } - } - - return textParts.join("\n\n").trim(); -} - -function spawnCodex( - prompt: string, - model: string, - timeout: number, -): Promise<{ output: string; stderr: string; exitCode: number }> { - const escaped = prompt.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); - const proc = Bun.spawn( - [ - "codex", - "exec", - "--dangerously-bypass-approvals-and-sandbox", - "--json", - escaped, - ], - { - stdout: "pipe", - stderr: "pipe", - env: getCodexEnv(model), - }, - ); - - return Promise.race([ - (async () => { - const output = await new Response(proc.stdout).text(); - const stderr = await new Response(proc.stderr).text(); - const exitCode = await proc.exited; - return { output, stderr, exitCode }; - })(), - new Promise((_, reject) => - setTimeout(() => { - proc.kill(); - reject(new Error(`Timed out after ${timeout}ms`)); - }, timeout), - ), - ]); -} - -export const codexProvider: Provider = { - name: "codex", - defaultModel: DEFAULT_MODEL, - - async call( - userPrompt: string, - options: ProviderOptions = {}, - context?: string, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 120_000 } = options; - const prompt = buildPrompt(userPrompt, context); - const start = performance.now(); - - try { - const result = await spawnCodex(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `codex exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - const output = parseJsonlOutput(result.output); - return { output, durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, - - async callRaw( - prompt: string, - options: ProviderOptions = {}, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 120_000 } = options; - const start = performance.now(); - - try { - const result = await spawnCodex(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `codex exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - const output = parseJsonlOutput(result.output); - return { output, durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, -}; diff --git a/evals/lib/judge.ts b/evals/lib/judge.ts deleted file mode 100644 index d5fca6a..0000000 --- a/evals/lib/judge.ts +++ /dev/null @@ -1,140 +0,0 @@ -import type { - EvalCase, - PatternResult, - JudgeResult, - EvalResult, - Provider, - ProviderOptions, -} from "./types.ts"; -import { claudeProvider } from "./claude.ts"; - -function testPatterns( - response: string, - evalCase: EvalCase, -): { pass: boolean; results: PatternResult[] } { - const results: PatternResult[] = []; - let pass = true; - - for (const pattern of evalCase.expectedPatterns) { - const regex = new RegExp(pattern, "is"); - const matched = regex.test(response); - results.push({ pattern, matched, type: "expected" }); - if (!matched) pass = false; - } - - if (evalCase.forbiddenPatterns) { - for (const pattern of evalCase.forbiddenPatterns) { - const regex = new RegExp(pattern, "is"); - const matched = regex.test(response); - results.push({ pattern, matched, type: "forbidden" }); - if (matched) pass = false; - } - } - - return { pass, results }; -} - -const JUDGE_PROMPT_TEMPLATE = `You are an eval judge scoring an AI agent's response to a browser automation task. - -The agent was given a task and a skill file that instructs it to use agent-browser CLI commands. -Score the response on a scale of 1-5 based on the rubric below. - -Rubric: -{rubric} - -Response to judge: - -{response} - - -Reply with ONLY a JSON object (no markdown fences, no other text): -{{"score": <1-5>, "reasoning": ""}}`; - -const JUDGE_MODEL = "anthropic/claude-opus-4.6"; - -async function runLLMJudge( - response: string, - rubric: string, - options: ProviderOptions, -): Promise { - const prompt = JUDGE_PROMPT_TEMPLATE.replace("{rubric}", rubric).replace( - "{response}", - response, - ); - - // Judge always uses Claude regardless of eval provider - const result = await claudeProvider.callRaw(prompt, { - model: JUDGE_MODEL, - timeout: options.timeout ?? 30_000, - }); - - if (result.error) { - return { score: 0, reasoning: `Judge error: ${result.error}` }; - } - - try { - const cleaned = result.output.replace(/```json\n?|```\n?/g, "").trim(); - const parsed = JSON.parse(cleaned); - return { - score: Math.max(0, Math.min(5, Number(parsed.score) || 0)), - reasoning: String(parsed.reasoning || ""), - }; - } catch { - return { - score: 0, - reasoning: `Failed to parse judge response: ${result.output.slice(0, 200)}`, - }; - } -} - -export async function evaluate( - evalCase: EvalCase, - provider: Provider, - options: { model?: string; judge?: boolean; timeout?: number } = {}, -): Promise { - const providerOptions: ProviderOptions = { - model: options.model, - timeout: options.timeout, - }; - - const response = await provider.call( - evalCase.prompt, - providerOptions, - evalCase.context, - ); - - if (response.error) { - return { - caseId: evalCase.id, - caseName: evalCase.name, - category: evalCase.category, - pass: false, - patternResults: [], - response: "", - durationMs: response.durationMs, - error: response.error, - }; - } - - const { pass, results } = testPatterns(response.output, evalCase); - - let judge: JudgeResult | undefined; - if (options.judge && evalCase.rubric) { - judge = await runLLMJudge( - response.output, - evalCase.rubric, - providerOptions, - ); - } - - return { - caseId: evalCase.id, - caseName: evalCase.name, - category: evalCase.category, - pass, - patternResults: results, - judge, - response: response.output, - durationMs: response.durationMs, - }; -} diff --git a/evals/lib/providers.ts b/evals/lib/providers.ts deleted file mode 100644 index 64391f1..0000000 --- a/evals/lib/providers.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Provider, ProviderName } from "./types.ts"; -import { claudeProvider } from "./claude.ts"; -import { codexProvider } from "./codex.ts"; - -const providers: Record = { - claude: claudeProvider, - codex: codexProvider, -}; - -export function getProvider(name: ProviderName): Provider { - const provider = providers[name]; - if (!provider) { - throw new Error(`Unknown provider: ${name}. Use "claude" or "codex".`); - } - return provider; -} diff --git a/evals/lib/reporter.ts b/evals/lib/reporter.ts deleted file mode 100644 index 275960a..0000000 --- a/evals/lib/reporter.ts +++ /dev/null @@ -1,142 +0,0 @@ -import type { EvalResult, EvalSummary, Category } from "./types.ts"; - -const PASS = "\x1b[32m\u2713\x1b[0m"; -const FAIL = "\x1b[31m\u2717\x1b[0m"; -const ERR = "\x1b[33m!\x1b[0m"; -const DIM = "\x1b[2m"; -const RESET = "\x1b[0m"; -const BOLD = "\x1b[1m"; - -function padRight(str: string, len: number): string { - return str + " ".repeat(Math.max(0, len - str.length)); -} - -export function printResult(result: EvalResult): void { - const icon = result.error ? ERR : result.pass ? PASS : FAIL; - const status = result.error ? "ERROR" : result.pass ? "PASS" : "FAIL"; - const duration = `${DIM}${result.durationMs}ms${RESET}`; - - console.log(` ${icon} ${padRight(result.caseName, 50)} ${status} ${duration}`); - - if (result.error) { - console.log(` ${DIM}Error: ${result.error}${RESET}`); - return; - } - - const failedExpected = result.patternResults.filter( - (p) => p.type === "expected" && !p.matched, - ); - const matchedForbidden = result.patternResults.filter( - (p) => p.type === "forbidden" && p.matched, - ); - - for (const p of failedExpected) { - console.log(` ${FAIL} Expected pattern not found: ${DIM}${p.pattern}${RESET}`); - } - for (const p of matchedForbidden) { - console.log(` ${FAIL} Forbidden pattern matched: ${DIM}${p.pattern}${RESET}`); - } - - if (result.judge) { - console.log( - ` ${DIM}Judge: ${result.judge.score}/5 - ${result.judge.reasoning}${RESET}`, - ); - } -} - -export function printCategoryHeader(category: string): void { - console.log(`\n${BOLD}${category}${RESET}`); - console.log(`${"─".repeat(70)}`); -} - -export function computeSummary( - results: EvalResult[], - totalDurationMs: number, -): EvalSummary { - const byCategory: Record = { - "skill-loading": { total: 0, passed: 0 }, - "skill-selection": { total: 0, passed: 0 }, - "command-usage": { total: 0, passed: 0 }, - }; - - let passed = 0; - let failed = 0; - let errors = 0; - - for (const r of results) { - byCategory[r.category].total++; - if (r.error) { - errors++; - } else if (r.pass) { - passed++; - byCategory[r.category].passed++; - } else { - failed++; - } - } - - return { - total: results.length, - passed, - failed, - errors, - byCategory, - durationMs: totalDurationMs, - }; -} - -export function printSummary(summary: EvalSummary): void { - console.log(`\n${BOLD}Summary${RESET}`); - console.log(`${"═".repeat(70)}`); - - for (const [cat, stats] of Object.entries(summary.byCategory)) { - if (stats.total === 0) continue; - const pct = Math.round((stats.passed / stats.total) * 100); - const bar = stats.passed === stats.total ? PASS : FAIL; - console.log(` ${bar} ${padRight(cat, 20)} ${stats.passed}/${stats.total} (${pct}%)`); - } - - console.log(`${"─".repeat(70)}`); - const totalPct = summary.total > 0 - ? Math.round((summary.passed / summary.total) * 100) - : 0; - const icon = summary.failed === 0 && summary.errors === 0 ? PASS : FAIL; - console.log( - ` ${icon} ${BOLD}Total: ${summary.passed}/${summary.total} passed (${totalPct}%)${RESET}`, - ); - if (summary.errors > 0) { - console.log(` ${ERR} ${summary.errors} error(s)`); - } - console.log(` ${DIM}Duration: ${(summary.durationMs / 1000).toFixed(1)}s${RESET}\n`); -} - -export function printResultsJson( - results: EvalResult[], - summary: EvalSummary, -): void { - const output = { - summary: { - total: summary.total, - passed: summary.passed, - failed: summary.failed, - errors: summary.errors, - passRate: summary.total > 0 - ? Math.round((summary.passed / summary.total) * 100) - : 0, - durationMs: summary.durationMs, - byCategory: summary.byCategory, - }, - results: results.map((r) => ({ - id: r.caseId, - name: r.caseName, - category: r.category, - pass: r.pass, - durationMs: r.durationMs, - error: r.error, - patterns: r.patternResults, - judge: r.judge, - response: r.response, - })), - }; - console.log(JSON.stringify(output, null, 2)); -} diff --git a/evals/lib/types.ts b/evals/lib/types.ts deleted file mode 100644 index 265afa7..0000000 --- a/evals/lib/types.ts +++ /dev/null @@ -1,78 +0,0 @@ -export type Category = "skill-loading" | "skill-selection" | "command-usage"; -export type ProviderName = "claude" | "codex"; - -export interface ProviderOptions { - model?: string; - timeout?: number; -} - -export interface ProviderResponse { - output: string; - durationMs: number; - error?: string; -} - -export interface Provider { - name: ProviderName; - defaultModel: string; - call(prompt: string, options?: ProviderOptions, context?: string): Promise; - callRaw(prompt: string, options?: ProviderOptions): Promise; -} - -export interface EvalCase { - id: string; - name: string; - category: Category; - /** The user task prompt sent to the model */ - prompt: string; - /** Additional context injected after the skill content (e.g., simulated skill output) */ - context?: string; - /** Regex patterns that must all match in the response */ - expectedPatterns: string[]; - /** Regex patterns that must NOT match in the response */ - forbiddenPatterns?: string[]; - /** Rubric for LLM judge quality scoring (1-5) */ - rubric?: string; -} - -export interface PatternResult { - pattern: string; - matched: boolean; - type: "expected" | "forbidden"; -} - -export interface JudgeResult { - score: number; - reasoning: string; -} - -export interface EvalResult { - caseId: string; - caseName: string; - category: Category; - pass: boolean; - patternResults: PatternResult[]; - judge?: JudgeResult; - response: string; - durationMs: number; - error?: string; -} - -export interface EvalSummary { - total: number; - passed: number; - failed: number; - errors: number; - byCategory: Record; - durationMs: number; -} - -export interface RunOptions { - provider: ProviderName; - model: string; - category?: Category; - judge: boolean; - json: boolean; - concurrency: number; - timeout: number; -} diff --git a/evals/package.json b/evals/package.json deleted file mode 100644 index fb7f824..0000000 --- a/evals/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "agent-browser-evals", - "version": "1.0.0", - "private": true, - "type": "module", - "scripts": { - "eval": "bun run run.ts", - "eval:claude": "bun run run.ts --provider claude", - "eval:codex": "bun run run.ts --provider codex", - "eval:judge": "bun run run.ts --judge", - "eval:json": "bun run run.ts --json" - }, - "devDependencies": { - "bun-types": "^1.3.12" - } -} diff --git a/evals/run.ts b/evals/run.ts deleted file mode 100644 index a4a6d77..0000000 --- a/evals/run.ts +++ /dev/null @@ -1,149 +0,0 @@ -import type { - EvalCase, - EvalResult, - Category, - ProviderName, - RunOptions, -} from "./lib/types.ts"; -import { getProvider } from "./lib/providers.ts"; -import { evaluate } from "./lib/judge.ts"; -import { - printResult, - printCategoryHeader, - computeSummary, - printSummary, - printResultsJson, -} from "./lib/reporter.ts"; -import { cases as skillLoadingCases } from "./cases/skill-loading.ts"; -import { cases as skillSelectionCases } from "./cases/skill-selection.ts"; -import { cases as commandUsageCases } from "./cases/command-usage.ts"; - -const ALL_CASES: EvalCase[] = [ - ...skillLoadingCases, - ...skillSelectionCases, - ...commandUsageCases, -]; - -function parseArgs(args: string[]): RunOptions { - const options: RunOptions = { - provider: "claude", - model: "", - judge: false, - json: false, - concurrency: 1, - timeout: 60_000, - }; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - switch (arg) { - case "--provider": - options.provider = (args[++i] ?? "claude") as ProviderName; - break; - case "--model": - options.model = args[++i] ?? ""; - break; - case "--category": - options.category = args[++i] as Category; - break; - case "--judge": - options.judge = true; - break; - case "--json": - options.json = true; - break; - case "--timeout": - options.timeout = parseInt(args[++i] ?? "60000", 10); - break; - case "--help": - case "-h": - printUsage(); - process.exit(0); - } - } - - return options; -} - -function printUsage(): void { - console.log( - ` -agent-browser skills evals - -Usage: bun run evals/run.ts [options] - -Options: - --provider Provider to use: claude, codex (default: claude) - --model Model override (default: provider's default model) - --category Filter by category: skill-loading, skill-selection, command-usage - --judge Enable LLM judge for quality scoring (costs extra API calls) - --json Output results as JSON - --timeout Timeout per eval case in milliseconds (default: 60000) - --help, -h Show this help - -Providers: - claude Uses Claude CLI via Vercel AI Gateway (default model: anthropic/claude-sonnet-4.6) - codex Uses Codex CLI via Vercel AI Gateway (default model: openai/o3) -`.trim(), - ); -} - -async function main(): Promise { - const options = parseArgs(process.argv.slice(2)); - const provider = getProvider(options.provider); - const model = options.model || provider.defaultModel; - - let cases = ALL_CASES; - if (options.category) { - cases = cases.filter((c) => c.category === options.category); - } - - if (cases.length === 0) { - console.error("No eval cases match the given filters."); - process.exit(1); - } - - if (!options.json) { - console.log( - `\nRunning ${cases.length} eval(s) with provider=${provider.name} model=${model}` + - (options.judge ? " + LLM judge" : ""), - ); - } - - const results: EvalResult[] = []; - const startTime = performance.now(); - let currentCategory: string | null = null; - - for (const evalCase of cases) { - if (!options.json && evalCase.category !== currentCategory) { - currentCategory = evalCase.category; - printCategoryHeader(currentCategory); - } - - const result = await evaluate(evalCase, provider, { - model, - judge: options.judge, - timeout: options.timeout, - }); - - results.push(result); - - if (!options.json) { - printResult(result); - } - } - - const totalDurationMs = Math.round(performance.now() - startTime); - const summary = computeSummary(results, totalDurationMs); - - if (options.json) { - printResultsJson(results, summary); - } else { - printSummary(summary); - } - - const exitCode = summary.failed > 0 || summary.errors > 0 ? 1 : 0; - process.exit(exitCode); -} - -main(); diff --git a/evals/tsconfig.json b/evals/tsconfig.json deleted file mode 100644 index 21dbb10..0000000 --- a/evals/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "bundler", - "esModuleInterop": true, - "strict": true, - "skipLibCheck": true, - "outDir": "dist", - "declaration": true, - "types": ["bun-types"] - }, - "include": ["*.ts", "lib/**/*.ts", "cases/**/*.ts"] -} diff --git a/package.json b/package.json index 7cecbed..dec22d7 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "bin", "scripts", "skills", + "skill-data", "extensions" ], "bin": { diff --git a/packages/dashboard/src/app/favicon.ico b/packages/dashboard/src/app/favicon.ico deleted file mode 100644 index 718d6fe..0000000 Binary files a/packages/dashboard/src/app/favicon.ico and /dev/null differ diff --git a/packages/dashboard/src/components/chat-panel.tsx b/packages/dashboard/src/components/chat-panel.tsx deleted file mode 100644 index 0a4a98e..0000000 --- a/packages/dashboard/src/components/chat-panel.tsx +++ /dev/null @@ -1,822 +0,0 @@ -"use client"; - -import { useRef, useEffect, useState, useCallback, useMemo } from "react"; -import { useAtomValue } from "jotai/react"; -import { useChat } from "@ai-sdk/react"; -import { DefaultChatTransport } from "ai"; -import { Streamdown } from "streamdown"; -import { getChatApiUrl, chatModelAtom, availableModelsAtom } from "@/store/chat"; -import { activeSessionNameAtom } from "@/store/sessions"; -import { ModelSelector } from "@/components/model-selector"; -import { shikiTheme } from "@/lib/shiki-theme"; -import { ScrollArea } from "@/components/ui/scroll-area"; -import { cn } from "@/lib/utils"; -import { ArrowUp, Square, Trash2, ChevronRight, ImagePlus, X, Loader, Copy, Check, Download } from "lucide-react"; - -type ExtraProps = { node?: unknown }; -type MdImgProps = React.ImgHTMLAttributes & ExtraProps; -type MdHeadingProps = React.HTMLAttributes & ExtraProps; -type MdAnchorProps = React.AnchorHTMLAttributes & ExtraProps; -type MdPreProps = React.HTMLAttributes & ExtraProps; -type MdCodeProps = React.HTMLAttributes & ExtraProps; - -const chatComponents = { - img: ({ node: _node, src, alt, ...props }: MdImgProps) => { - if (typeof src === "string" && src.startsWith("data:image/")) { - return {alt}; - } - return null; - }, - h1: ({ node: _node, ...props }: MdHeadingProps) =>

, - h2: ({ node: _node, ...props }: MdHeadingProps) =>

, - h3: ({ node: _node, ...props }: MdHeadingProps) =>

, - h4: ({ node: _node, ...props }: MdHeadingProps) =>

, - h5: ({ node: _node, ...props }: MdHeadingProps) =>

, - h6: ({ node: _node, ...props }: MdHeadingProps) =>

, - a: ({ node: _node, href, children, ...props }: MdAnchorProps) => ( - - {children} - - ), - pre: ({ node: _node, ...props }: MdPreProps) => ( - 

-  ),
-  code: ({ className, children, node: _node, ...props }: MdCodeProps) => {
-    if (className?.includes("language-")) {
-      return {children};
-    }
-    return (
-      
-        {children}
-      
-    );
-  },
-};
-
-const STORAGE_PREFIX = "dashboard-chat-";
-const IMAGE_DATA_URL_RE = /data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g;
-
-function stripImagesForStorage(messages: unknown[]): unknown[] {
-  const json = JSON.stringify(messages);
-  return JSON.parse(json.replace(IMAGE_DATA_URL_RE, "[image stripped]"));
-}
-
-const SUGGESTIONS = [
-  "Go to vercel.com",
-  "Take a screenshot",
-  "What's on the page?",
-  "Click the first link",
-];
-
-interface ToolInvocationPart {
-  type: string;
-  toolCallId: string;
-  state: string;
-  input?: Record;
-  output?: unknown;
-}
-
-function isToolPart(part: { type: string }): part is ToolInvocationPart {
-  return part.type.startsWith("tool-");
-}
-
-function truncateOutput(text: string, maxLines = 30): string {
-  const lines = text.split("\n");
-  if (lines.length <= maxLines) return text;
-  return lines.slice(0, maxLines).join("\n") + `\n... (${lines.length - maxLines} more lines)`;
-}
-
-function parseOutputObject(raw: unknown): Record | null {
-  if (typeof raw === "string") {
-    try {
-      const parsed = JSON.parse(raw);
-      if (typeof parsed === "object" && parsed !== null) return parsed;
-    } catch { /* not JSON */ }
-    return null;
-  }
-  if (typeof raw === "object" && raw !== null) return raw as Record;
-  return null;
-}
-
-function formatOutput(raw: unknown): string | null {
-  if (typeof raw === "string") {
-    if (!raw.trim()) return null;
-    const obj = parseOutputObject(raw);
-    if (obj) {
-      if (typeof obj.text === "string" && obj.image) return obj.text as string;
-      const { image: _, ...rest } = obj;
-      return JSON.stringify(rest, null, 2);
-    }
-    return raw;
-  }
-  if (typeof raw === "object" && raw !== null) {
-    const r = raw as Record;
-    if (typeof r.text === "string" && r.image) return r.text as string;
-    const { image: _, ...rest } = r;
-    return JSON.stringify(rest, null, 2);
-  }
-  return null;
-}
-
-function extractImageUrl(raw: unknown): string | null {
-  const obj = parseOutputObject(raw);
-  if (!obj) return null;
-  const img = obj.image;
-  if (typeof img === "string" && img.startsWith("data:image/")) return img;
-  return null;
-}
-
-function ToolCallBlock({ part, onImageLoad }: { part: ToolInvocationPart; onImageLoad?: () => void }) {
-  const [expanded, setExpanded] = useState(false);
-  const toolName = part.type.split("-").slice(1).join("-");
-  const command = (part.input as { command?: string })?.command ?? toolName;
-  const isDone = part.state === "output-available";
-  const isRunning = !isDone;
-  const output = isDone ? formatOutput(part.output) : null;
-  const hasOutput = !!output;
-  const imageUrl = isDone ? extractImageUrl(part.output) : null;
-  const canExpand = hasOutput && !isRunning;
-
-  return (
-    

-      
 canExpand && setExpanded(!expanded)}
-      >
-        

-          {isRunning ? (
-            
-          ) : (
-            
-          )}
-          {command}
-        

-        {expanded && hasOutput && (
-          

-            
-              {truncateOutput(output)}
-            

-          

-        )}
-      

-      {imageUrl && (
-        Screenshot
-      )}
-    

-  );
-}
-
-const DEFAULT_CONTEXT_WINDOW = 128000;
-
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
-function formatTokenCount(n: number): string {
-  if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
-  if (n >= 1_000) return `${(n / 1_000).toFixed(0)}K`;
-  return `${n}`;
-}
-
-function ContextMeter({ used, total }: { used: number; total: number }) {
-  const ratio = Math.min(used / total, 1);
-  const size = 16;
-  const strokeWidth = 2;
-  const r = (size - strokeWidth) / 2;
-  const circumference = 2 * Math.PI * r;
-  const offset = circumference * (1 - ratio);
-  const color =
-    ratio > 0.9 ? "text-destructive" : ratio > 0.7 ? "text-yellow-500" : "text-muted-foreground/50";
-
-  return (
-    

-      
-        
-        
-      
-    

-  );
-}
-
-const DEFAULT_MODEL = "anthropic/claude-sonnet-4.6";
-
-function useTimeAgo(ts: number | undefined) {
-  const [, setTick] = useState(0);
-  useEffect(() => {
-    if (!ts) return;
-    const id = setInterval(() => setTick((t) => t + 1), 30_000);
-    return () => clearInterval(id);
-  }, [ts]);
-  if (!ts) return "";
-  const diff = Math.floor((Date.now() - ts) / 1000);
-  if (diff < 5) return "just now";
-  if (diff < 60) return `${diff}s ago`;
-  const mins = Math.floor(diff / 60);
-  if (mins < 60) return `${mins}m ago`;
-  const hrs = Math.floor(mins / 60);
-  return `${hrs}h ago`;
-}
-
-function MessageFooter({ model, timestamp, text }: { model: string; timestamp?: number; text: string }) {
-  const [copied, setCopied] = useState(false);
-  const timeAgo = useTimeAgo(timestamp);
-  const shortModel = model.includes("/") ? model.split("/").pop()! : model;
-
-  const handleCopy = useCallback(() => {
-    navigator.clipboard.writeText(text).then(() => {
-      setCopied(true);
-      setTimeout(() => setCopied(false), 2000);
-    });
-  }, [text]);
-
-  return (
-    

-      {shortModel}
-      {timeAgo && (
-        <>
-          ·
-          {timeAgo}
-        
-      )}
-      
-    

-  );
-}
-
-interface PendingImage {
-  file: File;
-  preview: string;
-}
-
-export function ChatPanel() {
-  const [input, setInput] = useState("");
-  const [errorDismissed, setErrorDismissed] = useState(false);
-  const [pendingImages, setPendingImages] = useState([]);
-  const fileInputRef = useRef(null);
-  const defaultModel = useAtomValue(chatModelAtom);
-  const [selectedModel, setSelectedModel] = useState(defaultModel || DEFAULT_MODEL);
-  const messagesEndRef = useRef(null);
-  const inputRef = useRef(null);
-  const sessionName = useAtomValue(activeSessionNameAtom);
-  const chatId = sessionName || "default";
-  const storageKey = `${STORAGE_PREFIX}${chatId}`;
-  const sessionRef = useRef(chatId);
-  sessionRef.current = chatId;
-  const modelRef = useRef(selectedModel);
-  modelRef.current = selectedModel;
-  const messageTimestamps = useRef>({});
-
-  useEffect(() => {
-    if (defaultModel) setSelectedModel(defaultModel);
-  }, [defaultModel]);
-
-  const transport = useRef(
-    new DefaultChatTransport({
-      api: getChatApiUrl(),
-      body: () => ({
-        session: sessionRef.current,
-        model: modelRef.current,
-      }),
-    }),
-  ).current;
-
-  const { messages, sendMessage, stop, status, setMessages, error } = useChat({
-    id: chatId,
-    transport,
-    onError: () => setErrorDismissed(false),
-  });
-
-  const visibleError = error && !errorDismissed ? error : undefined;
-  const isLoading = status === "streaming" || status === "submitted";
-  const hasMessages = messages.length > 0 || !!visibleError;
-
-  useEffect(() => {
-    for (const msg of messages) {
-      if (msg.role === "assistant" && !messageTimestamps.current[msg.id]) {
-        messageTimestamps.current[msg.id] = Date.now();
-      }
-    }
-  }, [messages]);
-
-  const models = useAtomValue(availableModelsAtom);
-  const estimatedTokens = useMemo(() => {
-    let total = 0;
-    for (const msg of messages) {
-      for (const part of msg.parts) {
-        if (part.type === "text") total += estimateTokens(part.text);
-        else if (isToolPart(part)) {
-          if (part.input) total += estimateTokens(JSON.stringify(part.input));
-          if (part.output) {
-            const raw = typeof part.output === "string" ? part.output : JSON.stringify(part.output);
-            const stripped = raw.replace(/"image"\s*:\s*"data:[^"]*"/g, '"image":"[omitted]"');
-            total += estimateTokens(stripped);
-          }
-        }
-      }
-    }
-    return total;
-  }, [messages]);
-  const contextWindow = useMemo(() => {
-    const match = models.find((m) => m.id === selectedModel);
-    return match?.context_window ?? DEFAULT_CONTEXT_WINDOW;
-  }, [models, selectedModel]);
-
-  useEffect(() => {
-    inputRef.current?.focus();
-  }, []);
-
-  const scrollToBottom = useCallback(() => {
-    messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
-  }, []);
-
-  useEffect(() => {
-    scrollToBottom();
-  }, [messages, visibleError, scrollToBottom]);
-
-  // Restore messages from localStorage when chatId changes
-  useEffect(() => {
-    try {
-      const stored = localStorage.getItem(storageKey);
-      if (stored) {
-        const parsed = JSON.parse(stored);
-        if (Array.isArray(parsed) && parsed.length > 0) {
-          setMessages(parsed);
-          return;
-        }
-      }
-    } catch {
-      // ignore
-    }
-    setMessages([]);
-  }, [chatId, storageKey, setMessages]);
-
-  // Persist messages to localStorage (strip base64 images to save space)
-  useEffect(() => {
-    if (isLoading) return;
-    if (messages.length === 0) {
-      localStorage.removeItem(storageKey);
-      return;
-    }
-    try {
-      localStorage.setItem(storageKey, JSON.stringify(stripImagesForStorage(messages)));
-    } catch {
-      // ignore quota
-    }
-  }, [messages, isLoading, storageKey]);
-
-  const addImages = useCallback((files: FileList | null) => {
-    if (!files) return;
-    const images = Array.from(files).filter((f) => f.type.startsWith("image/"));
-    setPendingImages((prev) => [
-      ...prev,
-      ...images.map((file) => ({ file, preview: URL.createObjectURL(file) })),
-    ]);
-  }, []);
-
-  const removeImage = useCallback((index: number) => {
-    setPendingImages((prev) => {
-      const next = [...prev];
-      URL.revokeObjectURL(next[index].preview);
-      next.splice(index, 1);
-      return next;
-    });
-  }, []);
-
-  const handleSubmit = useCallback(
-    (e: React.FormEvent) => {
-      e.preventDefault();
-      if ((!input.trim() && pendingImages.length === 0) || isLoading) return;
-      const dt = new DataTransfer();
-      for (const img of pendingImages) dt.items.add(img.file);
-      const files = dt.files.length > 0 ? dt.files : undefined;
-      sendMessage({ text: input, files });
-      setInput("");
-      setPendingImages((prev) => {
-        for (const p of prev) URL.revokeObjectURL(p.preview);
-        return [];
-      });
-    },
-    [input, isLoading, sendMessage, pendingImages],
-  );
-
-  const lastCompactedId = useRef(null);
-  useEffect(() => {
-    if (isLoading || messages.length === 0) return;
-    const lastAssistant = [...messages].reverse().find((m) => m.role === "assistant");
-    if (!lastAssistant) return;
-    if (lastAssistant.id === lastCompactedId.current) return;
-    const meta = (lastAssistant as any).metadata as
-      | { compacted?: boolean; summary?: string; keepLastN?: number }
-      | undefined;
-    if (!meta?.compacted || typeof meta.keepLastN !== "number") return;
-
-    lastCompactedId.current = lastAssistant.id;
-    const keep = meta.keepLastN;
-    if (keep >= messages.length) return;
-
-    const summaryMsg = {
-      id: `compaction-${Date.now()}`,
-      role: "assistant" as const,
-      parts: [
-        {
-          type: "text" as const,
-          text: `*Earlier messages were summarized to stay within the context window.*`,
-        },
-      ],
-    };
-
-    const kept = messages.slice(messages.length - keep);
-    setMessages([summaryMsg as any, ...kept]);
-  }, [isLoading, messages, setMessages]);
-
-  const handleClear = useCallback(() => {
-    setMessages([]);
-    setErrorDismissed(true);
-    localStorage.removeItem(storageKey);
-    requestAnimationFrame(() => inputRef.current?.focus());
-  }, [setMessages, storageKey]);
-
-  const handleDownload = useCallback(() => {
-    const data = messages.map((msg) => ({
-      id: msg.id,
-      role: msg.role,
-      parts: msg.parts.map((p) => {
-        if (p.type === "text") return { type: "text", text: p.text };
-        if (p.type === "file") return { type: "file", filename: (p as any).filename };
-        if (isToolPart(p)) {
-          const out = typeof p.output === "string" ? p.output : JSON.stringify(p.output);
-          const stripped = out?.replace(/"image":"data:[^"]*"/g, '"image":"[stripped]"');
-          return {
-            type: p.type,
-            toolName: (p as any).toolName,
-            state: (p as any).state,
-            input: (p as any).input,
-            output: stripped,
-          };
-        }
-        return { type: p.type };
-      }),
-    }));
-    const json = JSON.stringify({ session: chatId, model: selectedModel, messages: data }, null, 2);
-    const blob = new Blob([json], { type: "application/json" });
-    const url = URL.createObjectURL(blob);
-    const a = document.createElement("a");
-    a.href = url;
-    a.download = `chat-${chatId}-${Date.now()}.json`;
-    a.click();
-    URL.revokeObjectURL(url);
-  }, [messages, chatId, selectedModel]);
-
-  const hasVisibleContent = (parts: (typeof messages)[number]["parts"]): boolean => {
-    return parts.some(
-      (p) => (p.type === "text" && p.text.length > 0) || p.type === "file" || isToolPart(p),
-    );
-  };
-
-  return (
-    

-      {hasMessages && (
-        

-          
-          
-        

-      )}
-
-      
-        

-          {!hasMessages && !isLoading && (
-            

-              

-                Control the browser with natural language:
-              

-              

-                {SUGGESTIONS.map((s) => (
-                  
-                ))}
-              

-            

-          )}
-
-          {messages.map((message) => {
-            if (message.id.startsWith("compaction-")) {
-              return (
-                

-                  

-                  Earlier messages summarized
-                  

-                

-              );
-            }
-            if (!hasVisibleContent(message.parts)) return null;
-            return (
-              

-                {message.role === "user" ? (
-                  

-                    {message.parts.some((p) => p.type === "file") && (
-                      

-                        {message.parts
-                          .filter((p): p is Extract => p.type === "file")
-                          .map((p, i) => (
-                            {p.filename
-                          ))}
-                      

-                    )}
-                    

-                      {message.parts
-                        .filter((p): p is Extract => p.type === "text")
-                        .map((p) => p.text)
-                        .join("")}
-                    

-                  

-                ) : (
-                  

-                    {(() => {
-                      type Group = { type: "tools" | "text"; items: (typeof message.parts)[number][] };
-                      const groups: Group[] = [];
-                      for (const part of message.parts) {
-                        const groupType = isToolPart(part) ? "tools" : "text";
-                        const last = groups[groups.length - 1];
-                        if (last && last.type === groupType) {
-                          last.items.push(part);
-                        } else {
-                          groups.push({ type: groupType, items: [part] });
-                        }
-                      }
-
-                      return groups.map((group, gi) => {
-                        if (group.type === "tools") {
-                          return (
-                            

-                              {group.items.map((part) => {
-                                if (!isToolPart(part)) return null;
-                                return ;
-                              })}
-                            

-                          );
-                        }
-                        const combinedText = group.items
-                          .filter((p): p is Extract => p.type === "text" && !!p.text)
-                          .map((p) => p.text)
-                          .join("");
-                        if (!combinedText) return null;
-                        return (
-                          

-                            
-                              {combinedText}
-                            
-                          

-                        );
-                      });
-                    })()}
-                    {(() => {
-                      const isLast = message === messages[messages.length - 1];
-                      const isComplete = !isLast || !isLoading;
-                      if (!isComplete) return null;
-                      const fullText = message.parts
-                        .filter((p): p is Extract => p.type === "text" && !!p.text)
-                        .map((p) => p.text)
-                        .join("");
-                      return (
-                        
-                      );
-                    })()}
-                  

-                )}
-              

-            );
-          })}
-
-          {isLoading && messages.length > 0 && (() => {
-            const lastMsg = messages[messages.length - 1];
-            const lastPart = lastMsg?.parts[lastMsg.parts.length - 1];
-            const noVisibleContent = !lastMsg || !hasVisibleContent(lastMsg.parts);
-            const lastIsCompletedTool = lastPart && isToolPart(lastPart) && lastPart.state === "output-available";
-            if (noVisibleContent || lastIsCompletedTool) {
-              return (
-                
-                  Working...
-                
-              );
-            }
-            return null;
-          })()}
-
-          {visibleError && (
-            

-              {(() => {
-                try {
-                  const parsed = JSON.parse(visibleError.message);
-                  return parsed.message || parsed.error || visibleError.message;
-                } catch {
-                  return visibleError.message || "Something went wrong.";
-                }
-              })()}
-            

-          )}
-
-          

-        

-      
-
-      

-        

-          {pendingImages.length > 0 && (
-            

-              {pendingImages.map((img, i) => (
-                

-                  {img.file.name}
-                  
-                

-              ))}
-            

-          )}
-          

-