From 86c4cff26eef50bbbedec57ed3a18398d143c687 Mon Sep 17 00:00:00 2001 From: leeguooooo Date: Sat, 9 May 2026 00:24:27 +0900 Subject: [PATCH] chore(fork): drop upstream-only docs/, evals/, packages/dashboard, schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These directories are TypeScript-side tooling that the fork dropped at v0.24.0 to keep the repo focused on the stealth CLI binary. Upstream either kept evolving them (docs, packages/dashboard) or added new ones (evals/) — they came back during the v0.27.0 rebase, so prune again. Also include skill-data/ in package.json `files` so the specialized skills (electron, slack, dogfood, etc.) that upstream relocated from skills/ to skill-data/ still ship in the npm tarball. --- agent-browser.schema.json | 166 ---- docs/public/schema.json | 166 ---- docs/src/lib/github.ts | 20 - evals/.env.example | 2 - evals/.gitignore | 3 - evals/README.md | 127 --- evals/bun.lock | 19 - evals/cases/command-usage.ts | 120 --- evals/cases/skill-loading.ts | 66 -- evals/cases/skill-selection.ts | 94 -- evals/lib/claude.ts | 142 --- evals/lib/codex.ts | 205 ----- evals/lib/judge.ts | 140 --- evals/lib/providers.ts | 16 - evals/lib/reporter.ts | 142 --- evals/lib/types.ts | 78 -- evals/package.json | 16 - evals/run.ts | 149 ---- evals/tsconfig.json | 14 - package.json | 1 + packages/dashboard/src/app/favicon.ico | Bin 25931 -> 0 bytes .../dashboard/src/components/chat-panel.tsx | 822 ------------------ .../src/components/model-selector.tsx | 97 --- .../src/components/theme-provider.tsx | 10 - .../dashboard/src/components/theme-toggle.tsx | 23 - .../dashboard/src/components/ui/command.tsx | 98 --- .../dashboard/src/components/ui/popover.tsx | 30 - .../dashboard/src/lib/dashboard-routes.ts | 42 - packages/dashboard/src/lib/shiki-theme.ts | 160 ---- packages/dashboard/src/store/chat.ts | 81 -- 30 files changed, 1 insertion(+), 3048 deletions(-) delete mode 100644 agent-browser.schema.json delete mode 100644 docs/public/schema.json delete mode 100644 docs/src/lib/github.ts delete mode 100644 evals/.env.example delete mode 100644 evals/.gitignore delete mode 100644 evals/README.md delete mode 100644 evals/bun.lock delete mode 100644 evals/cases/command-usage.ts delete mode 100644 evals/cases/skill-loading.ts delete mode 100644 evals/cases/skill-selection.ts delete mode 100644 evals/lib/claude.ts delete mode 100644 evals/lib/codex.ts delete mode 100644 evals/lib/judge.ts delete mode 100644 evals/lib/providers.ts delete mode 100644 evals/lib/reporter.ts delete mode 100644 evals/lib/types.ts delete mode 100644 evals/package.json delete mode 100644 evals/run.ts delete mode 100644 evals/tsconfig.json delete mode 100644 packages/dashboard/src/app/favicon.ico delete mode 100644 packages/dashboard/src/components/chat-panel.tsx delete mode 100644 packages/dashboard/src/components/model-selector.tsx delete mode 100644 packages/dashboard/src/components/theme-provider.tsx delete mode 100644 packages/dashboard/src/components/theme-toggle.tsx delete mode 100644 packages/dashboard/src/components/ui/command.tsx delete mode 100644 packages/dashboard/src/components/ui/popover.tsx delete mode 100644 packages/dashboard/src/lib/dashboard-routes.ts delete mode 100644 packages/dashboard/src/lib/shiki-theme.ts delete mode 100644 packages/dashboard/src/store/chat.ts diff --git a/agent-browser.schema.json b/agent-browser.schema.json deleted file mode 100644 index 48e38ba..0000000 --- a/agent-browser.schema.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Agent Browser Configuration", - "description": "Configuration file for agent-browser (e.g., agent-browser.json or ~/.agent-browser/config.json)", - "type": "object", - "properties": { - "headed": { - "type": "boolean", - "description": "Show browser window instead of running headless." - }, - "json": { - "type": "boolean", - "description": "Output in JSON format." - }, - "debug": { - "type": "boolean", - "description": "Enable debug output." - }, - "session": { - "type": "string", - "description": "Session identifier." - }, - "sessionName": { - "type": "string", - "description": "Auto-save/load state persistence name." - }, - "executablePath": { - "type": "string", - "description": "Path to a custom browser executable." - }, - "extensions": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Paths to browser extensions. Extensions from user-level and project-level configs are concatenated." - }, - "profile": { - "type": "string", - "description": "Path to the browser profile data directory." - }, - "state": { - "type": "string", - "description": "Path to load/save browser state." - }, - "proxy": { - "type": "string", - "description": "Proxy server URL (e.g., http://localhost:8080)." - }, - "proxyBypass": { - "type": "string", - "description": "Comma-separated domains to bypass the proxy (e.g., localhost,*.internal.com)." - }, - "args": { - "type": "string", - "description": "Additional comma-separated launch arguments for the browser." - }, - "userAgent": { - "type": "string", - "description": "Custom User-Agent string." - }, - "provider": { - "type": "string", - "description": "Provider to use, such as 'ios'." - }, - "device": { - "type": "string", - "description": "Device name or identifier for emulation or providers (e.g., 'iPhone 16 Pro')." - }, - "ignoreHttpsErrors": { - "type": "boolean", - "description": "Ignore HTTPS errors during navigation." - }, - "allowFileAccess": { - "type": "boolean", - "description": "Allow file:// URLs to access local files." - }, - "cdp": { - "type": "string", - "description": "Chrome DevTools Protocol endpoint URL." - }, - "autoConnect": { - "type": "boolean", - "description": "Auto-discover and connect to a running Chrome instance." - }, - "annotate": { - "type": "boolean", - "description": "Annotated screenshot with numbered element labels." - }, - "colorScheme": { - "type": "string", - "enum": ["dark", "light", "no-preference"], - "description": "Color scheme preference." - }, - "downloadPath": { - "type": "string", - "description": "Default directory for browser downloads." - }, - "contentBoundaries": { - "type": "boolean", - "description": "Wrap page output in boundary markers for LLM safety." - }, - "maxOutput": { - "type": "integer", - "minimum": 0, - "description": "Max characters for page output (truncates beyond limit)." - }, - "allowedDomains": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Allowed domain patterns (e.g., ['example.com', '*.example.com'])." - }, - "actionPolicy": { - "type": "string", - "description": "Path to action policy JSON file." - }, - "confirmActions": { - "type": "string", - "description": "Comma-separated action categories requiring confirmation." - }, - "confirmInteractive": { - "type": "boolean", - "description": "Enable interactive confirmation prompts (auto-denies if stdin is not a TTY)." - }, - "engine": { - "type": "string", - "enum": ["chrome", "lightpanda"], - "default": "chrome", - "description": "Browser engine to use." - }, - "screenshotDir": { - "type": "string", - "description": "Default screenshot output directory." - }, - "screenshotQuality": { - "type": "integer", - "minimum": 0, - "maximum": 100, - "description": "JPEG quality for screenshots (0-100)." - }, - "screenshotFormat": { - "type": "string", - "enum": ["png", "jpeg"], - "description": "Screenshot format." - }, - "idleTimeout": { - "type": "string", - "description": "Auto-shutdown the daemon after inactivity (e.g., '30s', '5m', '1h', or raw milliseconds like '60000')." - }, - "model": { - "type": "string", - "description": "AI model for chat command (e.g., 'openai/gpt-4o')." - }, - "noAutoDialog": { - "type": "boolean", - "description": "Disable automatic dismissal of alert/beforeunload dialogs." - }, - "headers": { - "type": "string", - "description": "Custom HTTP headers supplied as a JSON-formatted string." - } - }, - "additionalProperties": true -} diff --git a/docs/public/schema.json b/docs/public/schema.json deleted file mode 100644 index 48e38ba..0000000 --- a/docs/public/schema.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Agent Browser Configuration", - "description": "Configuration file for agent-browser (e.g., agent-browser.json or ~/.agent-browser/config.json)", - "type": "object", - "properties": { - "headed": { - "type": "boolean", - "description": "Show browser window instead of running headless." - }, - "json": { - "type": "boolean", - "description": "Output in JSON format." - }, - "debug": { - "type": "boolean", - "description": "Enable debug output." - }, - "session": { - "type": "string", - "description": "Session identifier." - }, - "sessionName": { - "type": "string", - "description": "Auto-save/load state persistence name." - }, - "executablePath": { - "type": "string", - "description": "Path to a custom browser executable." - }, - "extensions": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Paths to browser extensions. Extensions from user-level and project-level configs are concatenated." - }, - "profile": { - "type": "string", - "description": "Path to the browser profile data directory." - }, - "state": { - "type": "string", - "description": "Path to load/save browser state." - }, - "proxy": { - "type": "string", - "description": "Proxy server URL (e.g., http://localhost:8080)." - }, - "proxyBypass": { - "type": "string", - "description": "Comma-separated domains to bypass the proxy (e.g., localhost,*.internal.com)." - }, - "args": { - "type": "string", - "description": "Additional comma-separated launch arguments for the browser." - }, - "userAgent": { - "type": "string", - "description": "Custom User-Agent string." - }, - "provider": { - "type": "string", - "description": "Provider to use, such as 'ios'." - }, - "device": { - "type": "string", - "description": "Device name or identifier for emulation or providers (e.g., 'iPhone 16 Pro')." - }, - "ignoreHttpsErrors": { - "type": "boolean", - "description": "Ignore HTTPS errors during navigation." - }, - "allowFileAccess": { - "type": "boolean", - "description": "Allow file:// URLs to access local files." - }, - "cdp": { - "type": "string", - "description": "Chrome DevTools Protocol endpoint URL." - }, - "autoConnect": { - "type": "boolean", - "description": "Auto-discover and connect to a running Chrome instance." - }, - "annotate": { - "type": "boolean", - "description": "Annotated screenshot with numbered element labels." - }, - "colorScheme": { - "type": "string", - "enum": ["dark", "light", "no-preference"], - "description": "Color scheme preference." - }, - "downloadPath": { - "type": "string", - "description": "Default directory for browser downloads." - }, - "contentBoundaries": { - "type": "boolean", - "description": "Wrap page output in boundary markers for LLM safety." - }, - "maxOutput": { - "type": "integer", - "minimum": 0, - "description": "Max characters for page output (truncates beyond limit)." - }, - "allowedDomains": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Allowed domain patterns (e.g., ['example.com', '*.example.com'])." - }, - "actionPolicy": { - "type": "string", - "description": "Path to action policy JSON file." - }, - "confirmActions": { - "type": "string", - "description": "Comma-separated action categories requiring confirmation." - }, - "confirmInteractive": { - "type": "boolean", - "description": "Enable interactive confirmation prompts (auto-denies if stdin is not a TTY)." - }, - "engine": { - "type": "string", - "enum": ["chrome", "lightpanda"], - "default": "chrome", - "description": "Browser engine to use." - }, - "screenshotDir": { - "type": "string", - "description": "Default screenshot output directory." - }, - "screenshotQuality": { - "type": "integer", - "minimum": 0, - "maximum": 100, - "description": "JPEG quality for screenshots (0-100)." - }, - "screenshotFormat": { - "type": "string", - "enum": ["png", "jpeg"], - "description": "Screenshot format." - }, - "idleTimeout": { - "type": "string", - "description": "Auto-shutdown the daemon after inactivity (e.g., '30s', '5m', '1h', or raw milliseconds like '60000')." - }, - "model": { - "type": "string", - "description": "AI model for chat command (e.g., 'openai/gpt-4o')." - }, - "noAutoDialog": { - "type": "boolean", - "description": "Disable automatic dismissal of alert/beforeunload dialogs." - }, - "headers": { - "type": "string", - "description": "Custom HTTP headers supplied as a JSON-formatted string." - } - }, - "additionalProperties": true -} diff --git a/docs/src/lib/github.ts b/docs/src/lib/github.ts deleted file mode 100644 index 2c3f78e..0000000 --- a/docs/src/lib/github.ts +++ /dev/null @@ -1,20 +0,0 @@ -const REPO = "vercel-labs/agent-browser"; -const REVALIDATE = 86400; - -export async function getStarCount(): Promise { - try { - const res = await fetch(`https://api.github.com/repos/${REPO}`, { - headers: { Accept: "application/vnd.github.v3+json" }, - next: { revalidate: REVALIDATE }, - }); - if (!res.ok) return ""; - const data = await res.json(); - const count = data.stargazers_count; - if (typeof count !== "number") return ""; - if (count >= 1000) - return `${(count / 1000).toFixed(count >= 10000 ? 0 : 1)}k`; - return String(count); - } catch { - return ""; - } -} diff --git a/evals/.env.example b/evals/.env.example deleted file mode 100644 index 0a16d9f..0000000 --- a/evals/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -# Vercel AI Gateway key (required) -AI_GATEWAY_API_KEY= diff --git a/evals/.gitignore b/evals/.gitignore deleted file mode 100644 index 8c68c1a..0000000 --- a/evals/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -node_modules/ -dist/ -bun.lockb diff --git a/evals/README.md b/evals/README.md deleted file mode 100644 index 72ebdf6..0000000 --- a/evals/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Skills Evals - -Tests whether the thin SKILL.md + CLI-served skills approach works: do agents load the right skill via `agent-browser skills get`, then produce correct agent-browser commands? - -## Prerequisites - -- [Bun](https://bun.sh) installed -- `AI_GATEWAY_API_KEY` set (Vercel AI Gateway key) -- One or both CLIs installed: - - `claude` CLI (`npm i -g @anthropic-ai/claude-code`) for the Claude provider - - `codex` CLI (`npm i -g @openai/codex`) for the Codex provider - -The evals route all calls through the Vercel AI Gateway (`https://ai-gateway.vercel.sh`). Set your key before running: - -```bash -export AI_GATEWAY_API_KEY=gw_your_key_here -``` - -Or copy `.env.example` to `.env` and source it. - -## Usage - -```bash -cd evals - -# Run all evals (default: Claude provider) -bun run run.ts - -# Use Codex provider -bun run run.ts --provider codex - -# Filter by category -bun run run.ts --category skill-loading -bun run run.ts --category skill-selection -bun run run.ts --category command-usage - -# Use a specific model (overrides provider default) -bun run run.ts --model anthropic/claude-opus-4.6 -bun run run.ts --provider codex --model openai/gpt-4.1 - -# Enable LLM judge for quality scoring (1-5) -bun run run.ts --judge - -# JSON output (for CI or further analysis) -bun run run.ts --json - -# Combine options -bun run run.ts --provider codex --category skill-selection --judge -``` - -Or via package scripts: - -```bash -bun run eval # run all (Claude) -bun run eval:claude # run all (Claude, explicit) -bun run eval:codex # run all (Codex) -bun run eval:judge # run all with LLM judge -bun run eval:json # JSON output -``` - -## Providers - - - - - -
ProviderCLIDefault ModelNotes
claudeclaude -panthropic/claude-sonnet-4.6Uses ANTHROPIC_API_KEY + ANTHROPIC_BASE_URL env vars
codexcodex exec --jsonopenai/o3Writes ~/.codex/config.toml with AI Gateway config
- -The LLM judge always uses Claude (anthropic/claude-opus-4.6), regardless of the eval provider. - -## Eval Categories - -### skill-loading - -Tests that the agent runs `agent-browser skills get` before issuing browser commands. The thin SKILL.md instructs agents to load skills first; these evals verify compliance. - -### skill-selection - -Tests that the agent picks the correct specialized skill for the task. For example, a Slack task should load the `slack` skill, not the generic `agent-browser` skill. - -### command-usage - -Tests that the agent produces correct agent-browser commands for common workflows: navigation + screenshot, form filling with snapshot-interact pattern, diffing, authentication, data extraction. - -## How It Works - -1. Each eval case provides a user task prompt -2. The thin `skills/agent-browser/SKILL.md` is injected as context (simulating a skill installation) -3. The chosen provider CLI is called to get a single response -4. Pattern matching checks for expected/forbidden command patterns (pass/fail) -5. Optionally, a second Claude call judges response quality on a 1-5 scale - -## Adding Cases - -Create or edit files in `cases/`. Each file exports a `cases` array of `EvalCase` objects: - -```typescript -import type { EvalCase } from "../lib/types.ts"; - -export const cases: EvalCase[] = [ - { - id: "xx-01", - name: "Description of what this tests", - category: "skill-loading", - prompt: "The user task to send to the model", - expectedPatterns: ["regex.*that.*must.*match"], - forbiddenPatterns: ["regex.*that.*must.*not.*match"], - rubric: "1 - worst ... 5 - best", - }, -]; -``` - -Then import and add the cases to `ALL_CASES` in `run.ts`. - -## Output - -Console mode shows pass/fail per case with failed pattern details: - -``` -skill-loading ----------------------------------------------------------------------- - ✓ Loads skill before opening a page PASS 3200ms - ✗ Loads skill before form interaction FAIL 2800ms - ✗ Expected pattern not found: agent-browser skills get -``` - -JSON mode (`--json`) outputs structured results for programmatic consumption. diff --git a/evals/bun.lock b/evals/bun.lock deleted file mode 100644 index 72107d0..0000000 --- a/evals/bun.lock +++ /dev/null @@ -1,19 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agent-browser-evals", - "dependencies": { - "bun-types": "^1.3.12", - }, - }, - }, - "packages": { - "@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="], - - "bun-types": ["bun-types@1.3.12", "", { "dependencies": { "@types/node": "*" } }, "sha512-HqOLj5PoFajAQciOMRiIZGNoKxDJSr6qigAttOX40vJuSp6DN/CxWp9s3C1Xwm4oH7ybueITwiaOcWXoYVoRkA=="], - - "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], - } -} diff --git a/evals/cases/command-usage.ts b/evals/cases/command-usage.ts deleted file mode 100644 index 4fb47a9..0000000 --- a/evals/cases/command-usage.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type { EvalCase } from "../lib/types.ts"; - -const RUBRIC = ` -1 - Agent does not produce valid agent-browser commands -2 - Agent uses agent-browser but with wrong commands or missing steps -3 - Agent uses correct commands but skips the snapshot-interact workflow -4 - Agent follows the correct workflow with appropriate commands -5 - Agent follows the optimal workflow: navigate, snapshot, interact with refs, re-snapshot as needed -`.trim(); - -const COMMAND_CONTEXT = `You already ran \`agent-browser skills get core\` and loaded these commands: -- agent-browser open (navigate to a page) -- agent-browser snapshot -i (get interactive elements with refs like @e1, @e2) -- agent-browser click @ref (click element) -- agent-browser fill @ref "text" (clear and type) -- agent-browser type @ref "text" (type without clearing) -- agent-browser select @ref "option" (select dropdown) -- agent-browser screenshot (screenshot to temp dir) -- agent-browser screenshot --full (full page screenshot) -- agent-browser diff url (compare two pages) -- agent-browser diff snapshot (compare current vs last snapshot) -- agent-browser state save ./file.json (save auth state) -- agent-browser state load ./file.json (restore auth state) -- agent-browser get text @ref (get element text) -- agent-browser wait (wait for element or time) -- agent-browser --session-name open (named session with auto-save) - -Workflow: open -> snapshot -i -> interact with refs -> re-snapshot after changes.`; - -export const cases: EvalCase[] = [ - { - id: "cu-01", - name: "Navigate and screenshot workflow", - category: "command-usage", - prompt: "Open example.com and take a screenshot", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "agent-browser\\s+screenshot", - ], - rubric: RUBRIC, - }, - { - id: "cu-02", - name: "Form filling workflow", - category: "command-usage", - prompt: - "Go to example.com/signup, fill in name as 'Jane Doe' and email as 'jane@test.com', then submit", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "agent-browser\\s+snapshot", - "agent-browser\\s+(fill|type)", - "agent-browser\\s+(click|press|key)", - ], - rubric: RUBRIC, - }, - { - id: "cu-03", - name: "Snapshot with element refs", - category: "command-usage", - prompt: "Get all interactive elements on example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "agent-browser\\s+snapshot", - ], - rubric: RUBRIC, - }, - { - id: "cu-04", - name: "Diff comparison workflow", - category: "command-usage", - prompt: - "Compare the homepage of staging.example.com and prod.example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+diff|staging\\.example\\.com.*prod\\.example\\.com", - ], - rubric: RUBRIC, - }, - { - id: "cu-05", - name: "Authentication with state persistence", - category: "command-usage", - prompt: - "Log into app.example.com, then save the auth state for future sessions", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "state\\s+save|--session-name|auth\\s+save", - ], - rubric: RUBRIC, - }, - { - id: "cu-06", - name: "Data extraction workflow", - category: "command-usage", - prompt: - "Extract the text content of the main heading on example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate)", - "snapshot|get\\s+text", - ], - rubric: RUBRIC, - }, - { - id: "cu-07", - name: "Full-page screenshot", - category: "command-usage", - prompt: "Take a full-page screenshot of example.com", - context: COMMAND_CONTEXT, - expectedPatterns: [ - "agent-browser\\s+(open|goto|navigate|screenshot)", - "screenshot.*--full", - ], - rubric: RUBRIC, - }, -]; diff --git a/evals/cases/skill-loading.ts b/evals/cases/skill-loading.ts deleted file mode 100644 index 8f544df..0000000 --- a/evals/cases/skill-loading.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { EvalCase } from "../lib/types.ts"; - -const RUBRIC = ` -1 - Agent does not mention agent-browser skills or load any skill -2 - Agent mentions skills but does not run the skills get command -3 - Agent runs skills get but for the wrong skill or at the wrong time -4 - Agent runs skills get for the correct skill before using agent-browser -5 - Agent runs skills get first, then follows the loaded skill's workflow correctly -`.trim(); - -export const cases: EvalCase[] = [ - { - id: "sl-01", - name: "Loads skill before opening a page", - category: "skill-loading", - prompt: "Open example.com and take a screenshot", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-02", - name: "Loads skill before form interaction", - category: "skill-loading", - prompt: - "Go to example.com/contact and fill out the contact form with test data", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-03", - name: "Loads skill before data extraction", - category: "skill-loading", - prompt: - "Scrape all product names and prices from shop.example.com", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-04", - name: "Loads skill before authentication task", - category: "skill-loading", - prompt: - "Log into my GitHub account and check my notifications", - expectedPatterns: [ - "agent-browser skills get", - ], - rubric: RUBRIC, - }, - { - id: "sl-05", - name: "Uses skills list to discover available skills", - category: "skill-loading", - prompt: - "I need to automate some browser tasks. What skills are available for agent-browser?", - expectedPatterns: [ - "agent-browser skills (list|get)", - ], - rubric: RUBRIC, - }, -]; diff --git a/evals/cases/skill-selection.ts b/evals/cases/skill-selection.ts deleted file mode 100644 index 0439f63..0000000 --- a/evals/cases/skill-selection.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { EvalCase } from "../lib/types.ts"; - -const RUBRIC = ` -1 - Agent does not load any skill or loads a completely wrong one -2 - Agent loads the generic agent-browser skill when a specialized one exists -3 - Agent loads a related but suboptimal skill -4 - Agent loads the correct specialized skill -5 - Agent loads the correct skill and explains why it chose it -`.trim(); - -export const cases: EvalCase[] = [ - { - id: "ss-01", - name: "Selects slack skill for Slack tasks", - category: "skill-selection", - prompt: "Check my Slack unreads and summarize any messages mentioning me", - expectedPatterns: [ - "skills get slack", - ], - rubric: RUBRIC, - }, - { - id: "ss-02", - name: "Selects electron skill for VS Code automation", - category: "skill-selection", - prompt: "Automate VS Code to open a project and run a terminal command", - expectedPatterns: [ - "skills get electron", - ], - rubric: RUBRIC, - }, - { - id: "ss-03", - name: "Selects dogfood skill for QA/testing", - category: "skill-selection", - prompt: "QA test http://localhost:3000 and find any bugs or UX issues", - expectedPatterns: [ - "skills get dogfood", - ], - rubric: RUBRIC, - }, - { - id: "ss-04", - name: "Selects agentcore skill for AWS cloud browsers", - category: "skill-selection", - prompt: - "Run browser automation on AWS using AgentCore cloud browsers", - expectedPatterns: [ - "skills get agentcore", - ], - rubric: RUBRIC, - }, - { - id: "ss-05", - name: "Selects vercel-sandbox skill for Vercel environments", - category: "skill-selection", - prompt: - "Run headless Chrome inside a Vercel Sandbox microVM to test my deployed Next.js app", - expectedPatterns: [ - "skills get vercel-sandbox", - ], - rubric: RUBRIC, - }, - { - id: "ss-06", - name: "Selects electron skill for Discord automation", - category: "skill-selection", - prompt: "Automate the Discord desktop app to send a message in a channel", - expectedPatterns: [ - "skills get electron", - ], - rubric: RUBRIC, - }, - { - id: "ss-07", - name: "Selects dogfood skill for exploratory testing", - category: "skill-selection", - prompt: "Dogfood vercel.com and write up a bug report", - expectedPatterns: [ - "skills get dogfood", - ], - rubric: RUBRIC, - }, - { - id: "ss-08", - name: "Selects core skill for general browser tasks", - category: "skill-selection", - prompt: "Navigate to hacker news and screenshot the front page", - expectedPatterns: [ - "skills get core", - ], - rubric: RUBRIC, - }, -]; diff --git a/evals/lib/claude.ts b/evals/lib/claude.ts deleted file mode 100644 index 069b184..0000000 --- a/evals/lib/claude.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { readFileSync } from "fs"; -import { resolve, dirname } from "path"; -import { fileURLToPath } from "url"; -import type { Provider, ProviderOptions, ProviderResponse } from "./types.ts"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const SKILL_PATH = resolve(__dirname, "../../skills/agent-browser/SKILL.md"); - -const AI_GATEWAY_URL = "https://ai-gateway.vercel.sh"; -const DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"; - -let cachedSkillContent: string | null = null; - -function getSkillContent(): string { - if (!cachedSkillContent) { - cachedSkillContent = readFileSync(SKILL_PATH, "utf-8"); - } - return cachedSkillContent; -} - -function buildPrompt(userTask: string, context?: string): string { - const skill = getSkillContent(); - const parts = [ - "You have the following skill installed:\n", - "", - skill, - "\n", - ]; - if (context) { - parts.push(context + "\n"); - } - parts.push( - `Complete this task: ${userTask}\n`, - "Show the exact shell commands you would run. Do not explain, just show the commands.", - ); - return parts.join("\n"); -} - -function getGatewayEnv(): Record { - const apiKey = process.env.AI_GATEWAY_API_KEY; - if (!apiKey) { - throw new Error( - "AI_GATEWAY_API_KEY is not set. Export it before running evals.", - ); - } - return { - ...(process.env as Record), - ANTHROPIC_API_KEY: apiKey, - ANTHROPIC_BASE_URL: AI_GATEWAY_URL, - }; -} - -function spawnClaude( - prompt: string, - model: string, - timeout: number, -): Promise<{ output: string; stderr: string; exitCode: number }> { - const proc = Bun.spawn( - ["claude", "-p", "--output-format", "text", "--model", model, prompt], - { - stdout: "pipe", - stderr: "pipe", - env: getGatewayEnv(), - }, - ); - - return Promise.race([ - (async () => { - const output = await new Response(proc.stdout).text(); - const stderr = await new Response(proc.stderr).text(); - const exitCode = await proc.exited; - return { output, stderr, exitCode }; - })(), - new Promise((_, reject) => - setTimeout(() => { - proc.kill(); - reject(new Error(`Timed out after ${timeout}ms`)); - }, timeout), - ), - ]); -} - -export const claudeProvider: Provider = { - name: "claude", - defaultModel: DEFAULT_MODEL, - - async call( - userPrompt: string, - options: ProviderOptions = {}, - context?: string, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 60_000 } = options; - const prompt = buildPrompt(userPrompt, context); - const start = performance.now(); - - try { - const result = await spawnClaude(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `claude exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - return { output: result.output.trim(), durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, - - async callRaw( - prompt: string, - options: ProviderOptions = {}, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 60_000 } = options; - const start = performance.now(); - - try { - const result = await spawnClaude(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `claude exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - return { output: result.output.trim(), durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, -}; diff --git a/evals/lib/codex.ts b/evals/lib/codex.ts deleted file mode 100644 index 3790219..0000000 --- a/evals/lib/codex.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { readFileSync, writeFileSync, mkdirSync, existsSync } from "fs"; -import { resolve, dirname, join } from "path"; -import { fileURLToPath } from "url"; -import { tmpdir } from "os"; -import type { Provider, ProviderOptions, ProviderResponse } from "./types.ts"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const SKILL_PATH = resolve(__dirname, "../../skills/agent-browser/SKILL.md"); - -const AI_GATEWAY_URL = "https://ai-gateway.vercel.sh/v1"; -const DEFAULT_MODEL = "openai/o3"; - -let cachedSkillContent: string | null = null; - -function getSkillContent(): string { - if (!cachedSkillContent) { - cachedSkillContent = readFileSync(SKILL_PATH, "utf-8"); - } - return cachedSkillContent; -} - -function buildPrompt(userTask: string, context?: string): string { - const skill = getSkillContent(); - const parts = [ - "You have the following skill installed:\n", - "", - skill, - "\n", - ]; - if (context) { - parts.push(context + "\n"); - } - parts.push( - `Complete this task: ${userTask}\n`, - "Show the exact shell commands you would run. Do not explain, just show the commands.", - ); - return parts.join("\n"); -} - -let evalHome: string | null = null; - -function getEvalHome(model: string): string { - if (!evalHome) { - evalHome = join(tmpdir(), `agent-browser-evals-${process.pid}`); - } - const configDir = join(evalHome, ".codex"); - const configPath = join(configDir, "config.toml"); - - const config = `model = "${model}" -model_provider = "vercel-ai-gateway" - -[model_providers.vercel-ai-gateway] -name = "Vercel AI Gateway" -base_url = "${AI_GATEWAY_URL}" -env_key = "AI_GATEWAY_API_KEY" -wire_api = "responses" -`; - - if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); - } - writeFileSync(configPath, config, "utf-8"); - return evalHome; -} - -function getCodexEnv(model: string): Record { - const apiKey = process.env.AI_GATEWAY_API_KEY; - if (!apiKey) { - throw new Error( - "AI_GATEWAY_API_KEY is not set. Export it before running evals.", - ); - } - return { - ...(process.env as Record), - HOME: getEvalHome(model), - AI_GATEWAY_API_KEY: apiKey, - }; -} - -function parseJsonlOutput(raw: string): string { - const lines = raw.split("\n"); - const textParts: string[] = []; - - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed) continue; - - try { - const parsed = JSON.parse(trimmed); - const eventType = parsed.type as string; - - if (eventType === "item.completed") { - const item = parsed.item as Record | undefined; - if (item?.type === "agent_message") { - const text = item.text as string | undefined; - if (text) textParts.push(text); - } - } - } catch { - // Non-JSON line (e.g. stderr leak), skip - } - } - - return textParts.join("\n\n").trim(); -} - -function spawnCodex( - prompt: string, - model: string, - timeout: number, -): Promise<{ output: string; stderr: string; exitCode: number }> { - const escaped = prompt.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); - const proc = Bun.spawn( - [ - "codex", - "exec", - "--dangerously-bypass-approvals-and-sandbox", - "--json", - escaped, - ], - { - stdout: "pipe", - stderr: "pipe", - env: getCodexEnv(model), - }, - ); - - return Promise.race([ - (async () => { - const output = await new Response(proc.stdout).text(); - const stderr = await new Response(proc.stderr).text(); - const exitCode = await proc.exited; - return { output, stderr, exitCode }; - })(), - new Promise((_, reject) => - setTimeout(() => { - proc.kill(); - reject(new Error(`Timed out after ${timeout}ms`)); - }, timeout), - ), - ]); -} - -export const codexProvider: Provider = { - name: "codex", - defaultModel: DEFAULT_MODEL, - - async call( - userPrompt: string, - options: ProviderOptions = {}, - context?: string, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 120_000 } = options; - const prompt = buildPrompt(userPrompt, context); - const start = performance.now(); - - try { - const result = await spawnCodex(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `codex exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - const output = parseJsonlOutput(result.output); - return { output, durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, - - async callRaw( - prompt: string, - options: ProviderOptions = {}, - ): Promise { - const { model = DEFAULT_MODEL, timeout = 120_000 } = options; - const start = performance.now(); - - try { - const result = await spawnCodex(prompt, model, timeout); - const durationMs = Math.round(performance.now() - start); - - if (result.exitCode !== 0) { - return { - output: "", - durationMs, - error: `codex exited with code ${result.exitCode}: ${result.stderr}`, - }; - } - - const output = parseJsonlOutput(result.output); - return { output, durationMs }; - } catch (err) { - const durationMs = Math.round(performance.now() - start); - const message = err instanceof Error ? err.message : String(err); - return { output: "", durationMs, error: message }; - } - }, -}; diff --git a/evals/lib/judge.ts b/evals/lib/judge.ts deleted file mode 100644 index d5fca6a..0000000 --- a/evals/lib/judge.ts +++ /dev/null @@ -1,140 +0,0 @@ -import type { - EvalCase, - PatternResult, - JudgeResult, - EvalResult, - Provider, - ProviderOptions, -} from "./types.ts"; -import { claudeProvider } from "./claude.ts"; - -function testPatterns( - response: string, - evalCase: EvalCase, -): { pass: boolean; results: PatternResult[] } { - const results: PatternResult[] = []; - let pass = true; - - for (const pattern of evalCase.expectedPatterns) { - const regex = new RegExp(pattern, "is"); - const matched = regex.test(response); - results.push({ pattern, matched, type: "expected" }); - if (!matched) pass = false; - } - - if (evalCase.forbiddenPatterns) { - for (const pattern of evalCase.forbiddenPatterns) { - const regex = new RegExp(pattern, "is"); - const matched = regex.test(response); - results.push({ pattern, matched, type: "forbidden" }); - if (matched) pass = false; - } - } - - return { pass, results }; -} - -const JUDGE_PROMPT_TEMPLATE = `You are an eval judge scoring an AI agent's response to a browser automation task. - -The agent was given a task and a skill file that instructs it to use agent-browser CLI commands. -Score the response on a scale of 1-5 based on the rubric below. - -Rubric: -{rubric} - -Response to judge: - -{response} - - -Reply with ONLY a JSON object (no markdown fences, no other text): -{{"score": <1-5>, "reasoning": ""}}`; - -const JUDGE_MODEL = "anthropic/claude-opus-4.6"; - -async function runLLMJudge( - response: string, - rubric: string, - options: ProviderOptions, -): Promise { - const prompt = JUDGE_PROMPT_TEMPLATE.replace("{rubric}", rubric).replace( - "{response}", - response, - ); - - // Judge always uses Claude regardless of eval provider - const result = await claudeProvider.callRaw(prompt, { - model: JUDGE_MODEL, - timeout: options.timeout ?? 30_000, - }); - - if (result.error) { - return { score: 0, reasoning: `Judge error: ${result.error}` }; - } - - try { - const cleaned = result.output.replace(/```json\n?|```\n?/g, "").trim(); - const parsed = JSON.parse(cleaned); - return { - score: Math.max(0, Math.min(5, Number(parsed.score) || 0)), - reasoning: String(parsed.reasoning || ""), - }; - } catch { - return { - score: 0, - reasoning: `Failed to parse judge response: ${result.output.slice(0, 200)}`, - }; - } -} - -export async function evaluate( - evalCase: EvalCase, - provider: Provider, - options: { model?: string; judge?: boolean; timeout?: number } = {}, -): Promise { - const providerOptions: ProviderOptions = { - model: options.model, - timeout: options.timeout, - }; - - const response = await provider.call( - evalCase.prompt, - providerOptions, - evalCase.context, - ); - - if (response.error) { - return { - caseId: evalCase.id, - caseName: evalCase.name, - category: evalCase.category, - pass: false, - patternResults: [], - response: "", - durationMs: response.durationMs, - error: response.error, - }; - } - - const { pass, results } = testPatterns(response.output, evalCase); - - let judge: JudgeResult | undefined; - if (options.judge && evalCase.rubric) { - judge = await runLLMJudge( - response.output, - evalCase.rubric, - providerOptions, - ); - } - - return { - caseId: evalCase.id, - caseName: evalCase.name, - category: evalCase.category, - pass, - patternResults: results, - judge, - response: response.output, - durationMs: response.durationMs, - }; -} diff --git a/evals/lib/providers.ts b/evals/lib/providers.ts deleted file mode 100644 index 64391f1..0000000 --- a/evals/lib/providers.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Provider, ProviderName } from "./types.ts"; -import { claudeProvider } from "./claude.ts"; -import { codexProvider } from "./codex.ts"; - -const providers: Record = { - claude: claudeProvider, - codex: codexProvider, -}; - -export function getProvider(name: ProviderName): Provider { - const provider = providers[name]; - if (!provider) { - throw new Error(`Unknown provider: ${name}. Use "claude" or "codex".`); - } - return provider; -} diff --git a/evals/lib/reporter.ts b/evals/lib/reporter.ts deleted file mode 100644 index 275960a..0000000 --- a/evals/lib/reporter.ts +++ /dev/null @@ -1,142 +0,0 @@ -import type { EvalResult, EvalSummary, Category } from "./types.ts"; - -const PASS = "\x1b[32m\u2713\x1b[0m"; -const FAIL = "\x1b[31m\u2717\x1b[0m"; -const ERR = "\x1b[33m!\x1b[0m"; -const DIM = "\x1b[2m"; -const RESET = "\x1b[0m"; -const BOLD = "\x1b[1m"; - -function padRight(str: string, len: number): string { - return str + " ".repeat(Math.max(0, len - str.length)); -} - -export function printResult(result: EvalResult): void { - const icon = result.error ? ERR : result.pass ? PASS : FAIL; - const status = result.error ? "ERROR" : result.pass ? "PASS" : "FAIL"; - const duration = `${DIM}${result.durationMs}ms${RESET}`; - - console.log(` ${icon} ${padRight(result.caseName, 50)} ${status} ${duration}`); - - if (result.error) { - console.log(` ${DIM}Error: ${result.error}${RESET}`); - return; - } - - const failedExpected = result.patternResults.filter( - (p) => p.type === "expected" && !p.matched, - ); - const matchedForbidden = result.patternResults.filter( - (p) => p.type === "forbidden" && p.matched, - ); - - for (const p of failedExpected) { - console.log(` ${FAIL} Expected pattern not found: ${DIM}${p.pattern}${RESET}`); - } - for (const p of matchedForbidden) { - console.log(` ${FAIL} Forbidden pattern matched: ${DIM}${p.pattern}${RESET}`); - } - - if (result.judge) { - console.log( - ` ${DIM}Judge: ${result.judge.score}/5 - ${result.judge.reasoning}${RESET}`, - ); - } -} - -export function printCategoryHeader(category: string): void { - console.log(`\n${BOLD}${category}${RESET}`); - console.log(`${"─".repeat(70)}`); -} - -export function computeSummary( - results: EvalResult[], - totalDurationMs: number, -): EvalSummary { - const byCategory: Record = { - "skill-loading": { total: 0, passed: 0 }, - "skill-selection": { total: 0, passed: 0 }, - "command-usage": { total: 0, passed: 0 }, - }; - - let passed = 0; - let failed = 0; - let errors = 0; - - for (const r of results) { - byCategory[r.category].total++; - if (r.error) { - errors++; - } else if (r.pass) { - passed++; - byCategory[r.category].passed++; - } else { - failed++; - } - } - - return { - total: results.length, - passed, - failed, - errors, - byCategory, - durationMs: totalDurationMs, - }; -} - -export function printSummary(summary: EvalSummary): void { - console.log(`\n${BOLD}Summary${RESET}`); - console.log(`${"═".repeat(70)}`); - - for (const [cat, stats] of Object.entries(summary.byCategory)) { - if (stats.total === 0) continue; - const pct = Math.round((stats.passed / stats.total) * 100); - const bar = stats.passed === stats.total ? PASS : FAIL; - console.log(` ${bar} ${padRight(cat, 20)} ${stats.passed}/${stats.total} (${pct}%)`); - } - - console.log(`${"─".repeat(70)}`); - const totalPct = summary.total > 0 - ? Math.round((summary.passed / summary.total) * 100) - : 0; - const icon = summary.failed === 0 && summary.errors === 0 ? PASS : FAIL; - console.log( - ` ${icon} ${BOLD}Total: ${summary.passed}/${summary.total} passed (${totalPct}%)${RESET}`, - ); - if (summary.errors > 0) { - console.log(` ${ERR} ${summary.errors} error(s)`); - } - console.log(` ${DIM}Duration: ${(summary.durationMs / 1000).toFixed(1)}s${RESET}\n`); -} - -export function printResultsJson( - results: EvalResult[], - summary: EvalSummary, -): void { - const output = { - summary: { - total: summary.total, - passed: summary.passed, - failed: summary.failed, - errors: summary.errors, - passRate: summary.total > 0 - ? Math.round((summary.passed / summary.total) * 100) - : 0, - durationMs: summary.durationMs, - byCategory: summary.byCategory, - }, - results: results.map((r) => ({ - id: r.caseId, - name: r.caseName, - category: r.category, - pass: r.pass, - durationMs: r.durationMs, - error: r.error, - patterns: r.patternResults, - judge: r.judge, - response: r.response, - })), - }; - console.log(JSON.stringify(output, null, 2)); -} diff --git a/evals/lib/types.ts b/evals/lib/types.ts deleted file mode 100644 index 265afa7..0000000 --- a/evals/lib/types.ts +++ /dev/null @@ -1,78 +0,0 @@ -export type Category = "skill-loading" | "skill-selection" | "command-usage"; -export type ProviderName = "claude" | "codex"; - -export interface ProviderOptions { - model?: string; - timeout?: number; -} - -export interface ProviderResponse { - output: string; - durationMs: number; - error?: string; -} - -export interface Provider { - name: ProviderName; - defaultModel: string; - call(prompt: string, options?: ProviderOptions, context?: string): Promise; - callRaw(prompt: string, options?: ProviderOptions): Promise; -} - -export interface EvalCase { - id: string; - name: string; - category: Category; - /** The user task prompt sent to the model */ - prompt: string; - /** Additional context injected after the skill content (e.g., simulated skill output) */ - context?: string; - /** Regex patterns that must all match in the response */ - expectedPatterns: string[]; - /** Regex patterns that must NOT match in the response */ - forbiddenPatterns?: string[]; - /** Rubric for LLM judge quality scoring (1-5) */ - rubric?: string; -} - -export interface PatternResult { - pattern: string; - matched: boolean; - type: "expected" | "forbidden"; -} - -export interface JudgeResult { - score: number; - reasoning: string; -} - -export interface EvalResult { - caseId: string; - caseName: string; - category: Category; - pass: boolean; - patternResults: PatternResult[]; - judge?: JudgeResult; - response: string; - durationMs: number; - error?: string; -} - -export interface EvalSummary { - total: number; - passed: number; - failed: number; - errors: number; - byCategory: Record; - durationMs: number; -} - -export interface RunOptions { - provider: ProviderName; - model: string; - category?: Category; - judge: boolean; - json: boolean; - concurrency: number; - timeout: number; -} diff --git a/evals/package.json b/evals/package.json deleted file mode 100644 index fb7f824..0000000 --- a/evals/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "agent-browser-evals", - "version": "1.0.0", - "private": true, - "type": "module", - "scripts": { - "eval": "bun run run.ts", - "eval:claude": "bun run run.ts --provider claude", - "eval:codex": "bun run run.ts --provider codex", - "eval:judge": "bun run run.ts --judge", - "eval:json": "bun run run.ts --json" - }, - "devDependencies": { - "bun-types": "^1.3.12" - } -} diff --git a/evals/run.ts b/evals/run.ts deleted file mode 100644 index a4a6d77..0000000 --- a/evals/run.ts +++ /dev/null @@ -1,149 +0,0 @@ -import type { - EvalCase, - EvalResult, - Category, - ProviderName, - RunOptions, -} from "./lib/types.ts"; -import { getProvider } from "./lib/providers.ts"; -import { evaluate } from "./lib/judge.ts"; -import { - printResult, - printCategoryHeader, - computeSummary, - printSummary, - printResultsJson, -} from "./lib/reporter.ts"; -import { cases as skillLoadingCases } from "./cases/skill-loading.ts"; -import { cases as skillSelectionCases } from "./cases/skill-selection.ts"; -import { cases as commandUsageCases } from "./cases/command-usage.ts"; - -const ALL_CASES: EvalCase[] = [ - ...skillLoadingCases, - ...skillSelectionCases, - ...commandUsageCases, -]; - -function parseArgs(args: string[]): RunOptions { - const options: RunOptions = { - provider: "claude", - model: "", - judge: false, - json: false, - concurrency: 1, - timeout: 60_000, - }; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - switch (arg) { - case "--provider": - options.provider = (args[++i] ?? "claude") as ProviderName; - break; - case "--model": - options.model = args[++i] ?? ""; - break; - case "--category": - options.category = args[++i] as Category; - break; - case "--judge": - options.judge = true; - break; - case "--json": - options.json = true; - break; - case "--timeout": - options.timeout = parseInt(args[++i] ?? "60000", 10); - break; - case "--help": - case "-h": - printUsage(); - process.exit(0); - } - } - - return options; -} - -function printUsage(): void { - console.log( - ` -agent-browser skills evals - -Usage: bun run evals/run.ts [options] - -Options: - --provider Provider to use: claude, codex (default: claude) - --model Model override (default: provider's default model) - --category Filter by category: skill-loading, skill-selection, command-usage - --judge Enable LLM judge for quality scoring (costs extra API calls) - --json Output results as JSON - --timeout Timeout per eval case in milliseconds (default: 60000) - --help, -h Show this help - -Providers: - claude Uses Claude CLI via Vercel AI Gateway (default model: anthropic/claude-sonnet-4.6) - codex Uses Codex CLI via Vercel AI Gateway (default model: openai/o3) -`.trim(), - ); -} - -async function main(): Promise { - const options = parseArgs(process.argv.slice(2)); - const provider = getProvider(options.provider); - const model = options.model || provider.defaultModel; - - let cases = ALL_CASES; - if (options.category) { - cases = cases.filter((c) => c.category === options.category); - } - - if (cases.length === 0) { - console.error("No eval cases match the given filters."); - process.exit(1); - } - - if (!options.json) { - console.log( - `\nRunning ${cases.length} eval(s) with provider=${provider.name} model=${model}` + - (options.judge ? " + LLM judge" : ""), - ); - } - - const results: EvalResult[] = []; - const startTime = performance.now(); - let currentCategory: string | null = null; - - for (const evalCase of cases) { - if (!options.json && evalCase.category !== currentCategory) { - currentCategory = evalCase.category; - printCategoryHeader(currentCategory); - } - - const result = await evaluate(evalCase, provider, { - model, - judge: options.judge, - timeout: options.timeout, - }); - - results.push(result); - - if (!options.json) { - printResult(result); - } - } - - const totalDurationMs = Math.round(performance.now() - startTime); - const summary = computeSummary(results, totalDurationMs); - - if (options.json) { - printResultsJson(results, summary); - } else { - printSummary(summary); - } - - const exitCode = summary.failed > 0 || summary.errors > 0 ? 1 : 0; - process.exit(exitCode); -} - -main(); diff --git a/evals/tsconfig.json b/evals/tsconfig.json deleted file mode 100644 index 21dbb10..0000000 --- a/evals/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "bundler", - "esModuleInterop": true, - "strict": true, - "skipLibCheck": true, - "outDir": "dist", - "declaration": true, - "types": ["bun-types"] - }, - "include": ["*.ts", "lib/**/*.ts", "cases/**/*.ts"] -} diff --git a/package.json b/package.json index 7cecbed..dec22d7 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "bin", "scripts", "skills", + "skill-data", "extensions" ], "bin": { diff --git a/packages/dashboard/src/app/favicon.ico b/packages/dashboard/src/app/favicon.ico deleted file mode 100644 index 718d6fea4835ec2d246af9800eddb7ffb276240c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25931 zcmeHv30#a{`}aL_*G&7qml|y<+KVaDM2m#dVr!KsA!#An?kSQM(q<_dDNCpjEux83 zLb9Z^XxbDl(w>%i@8hT6>)&Gu{h#Oeyszu?xtw#Zb1mO{pgX9699l+Qppw7jXaYf~-84xW z)w4x8?=youko|}Vr~(D$UXIbiXABHh`p1?nn8Po~fxRJv}|0e(BPs|G`(TT%kKVJAdg5*Z|x0leQq0 zkdUBvb#>9F()jo|T~kx@OM8$9wzs~t2l;K=woNssA3l6|sx2r3+kdfVW@e^8e*E}v zA1y5{bRi+3Z`uD3{F7LgFJDdvm;nJilkzDku>BwXH(8ItVCXk*-lSJnR?-2UN%hJ){&rlvg`CDTj z)Bzo!3v7Ou#83zEDEFcKt(f1E0~=rqeEbTnMvWR#{+9pg%7G8y>u1OVRUSoox-ovF z2Ydma(;=YuBY(eI|04{hXzZD6_f(v~H;C~y5=DhAC{MMS>2fm~1H_t2$56pc$NH8( z5bH|<)71dV-_oCHIrzrT`2s-5w_+2CM0$95I6X8p^r!gHp+j_gd;9O<1~CEQQGS8) zS9Qh3#p&JM-G8rHekNmKVewU;pJRcTAog68KYo^dRo}(M>36U4Us zfgYWSiHZL3;lpWT=zNAW>Dh#mB!_@Lg%$ms8N-;aPqMn+C2HqZgz&9~Eu z4|Kp<`$q)Uw1R?y(~S>ePdonHxpV1#eSP1B;Ogo+-Pk}6#0GsZZ5!||ev2MGdh}_m z{DeR7?0-1^zVs&`AV6Vt;r3`I`OI_wgs*w=eO%_#7Kepl{B@xiyCANc(l zzIyd4y|c6PXWq9-|KM8(zIk8LPk(>a)zyFWjhT!$HJ$qX1vo@d25W<fvZQ2zUz5WRc(UnFMKHwe1| zWmlB1qdbiA(C0jmnV<}GfbKtmcu^2*P^O?MBLZKt|As~ge8&AAO~2K@zbXelK|4T<{|y4`raF{=72kC2Kn(L4YyenWgrPiv z@^mr$t{#X5VuIMeL!7Ab6_kG$&#&5p*Z{+?5U|TZ`B!7llpVmp@skYz&n^8QfPJzL z0G6K_OJM9x+Wu2gfN45phANGt{7=C>i34CV{Xqlx(fWpeAoj^N0Biu`w+MVcCUyU* zDZuzO0>4Z6fbu^T_arWW5n!E45vX8N=bxTVeFoep_G#VmNlQzAI_KTIc{6>c+04vr zx@W}zE5JNSU>!THJ{J=cqjz+4{L4A{Ob9$ZJ*S1?Ggg3klFp!+Y1@K+pK1DqI|_gq z5ZDXVpge8-cs!o|;K73#YXZ3AShj50wBvuq3NTOZ`M&qtjj#GOFfgExjg8Gn8>Vq5 z`85n+9|!iLCZF5$HJ$Iu($dm?8~-ofu}tEc+-pyke=3!im#6pk_Wo8IA|fJwD&~~F zc16osQ)EBo58U7XDuMexaPRjU@h8tXe%S{fA0NH3vGJFhuyyO!Uyl2^&EOpX{9As0 zWj+P>{@}jxH)8|r;2HdupP!vie{sJ28b&bo!8`D^x}TE$%zXNb^X1p@0PJ86`dZyj z%ce7*{^oo+6%&~I!8hQy-vQ7E)0t0ybH4l%KltWOo~8cO`T=157JqL(oq_rC%ea&4 z2NcTJe-HgFjNg-gZ$6!Y`SMHrlj}Etf7?r!zQTPPSv}{so2e>Fjs1{gzk~LGeesX%r(Lh6rbhSo_n)@@G-FTQy93;l#E)hgP@d_SGvyCp0~o(Y;Ee8{ zdVUDbHm5`2taPUOY^MAGOw*>=s7=Gst=D+p+2yON!0%Hk` zz5mAhyT4lS*T3LS^WSxUy86q&GnoHxzQ6vm8)VS}_zuqG?+3td68_x;etQAdu@sc6 zQJ&5|4(I?~3d-QOAODHpZ=hlSg(lBZ!JZWCtHHSj`0Wh93-Uk)_S%zsJ~aD>{`A0~ z9{AG(e|q3g5B%wYKRxiL2Y$8(4w6bzchKuloQW#e&S3n+P- z8!ds-%f;TJ1>)v)##>gd{PdS2Oc3VaR`fr=`O8QIO(6(N!A?pr5C#6fc~Ge@N%Vvu zaoAX2&(a6eWy_q&UwOhU)|P3J0Qc%OdhzW=F4D|pt0E4osw;%<%Dn58hAWD^XnZD= z>9~H(3bmLtxpF?a7su6J7M*x1By7YSUbxGi)Ot0P77`}P3{)&5Un{KD?`-e?r21!4vTTnN(4Y6Lin?UkSM z`MXCTC1@4A4~mvz%Rh2&EwY))LeoT=*`tMoqcEXI>TZU9WTP#l?uFv+@Dn~b(>xh2 z;>B?;Tz2SR&KVb>vGiBSB`@U7VIWFSo=LDSb9F{GF^DbmWAfpms8Sx9OX4CnBJca3 zlj9(x!dIjN?OG1X4l*imJNvRCk}F%!?SOfiOq5y^mZW)jFL@a|r-@d#f7 z2gmU8L3IZq0ynIws=}~m^#@&C%J6QFo~Mo4V`>v7MI-_!EBMMtb%_M&kvAaN)@ZVw z+`toz&WG#HkWDjnZE!6nk{e-oFdL^$YnbOCN}JC&{$#$O27@|Tn-skXr)2ml2~O!5 zX+gYoxhoc7qoU?C^3~&!U?kRFtnSEecWuH0B0OvLodgUAi}8p1 zrO6RSXHH}DMc$&|?D004DiOVMHV8kXCP@7NKB zgaZq^^O<7PoKEp72kby@W0Z!Y*Ay{&vfg#C&gG@YVR9g?FEocMUi1gSN$+V+ayF45{a zuDZDTN}mS|;BO%gEf}pjBfN2-gIrU#G5~cucA;dokXW89%>AyXJJI z9X4UlIWA|ZYHgbI z5?oFk@A=Ik7lrEQPDH!H+b`7_Y~aDb_qa=B2^Y&Ow41cU=4WDd40dp5(QS-WMN-=Y z9g;6_-JdNU;|6cPwf$ak*aJIcwL@1n$#l~zi{c{EW?T;DaW*E8DYq?Umtz{nJ&w-M zEMyTDrC&9K$d|kZe2#ws6)L=7K+{ zQw{XnV6UC$6-rW0emqm8wJoeZK)wJIcV?dST}Z;G0Arq{dVDu0&4kd%N!3F1*;*pW zR&qUiFzK=@44#QGw7k1`3t_d8&*kBV->O##t|tonFc2YWrL7_eqg+=+k;!F-`^b8> z#KWCE8%u4k@EprxqiV$VmmtiWxDLgnGu$Vs<8rppV5EajBXL4nyyZM$SWVm!wnCj-B!Wjqj5-5dNXukI2$$|Bu3Lrw}z65Lc=1G z^-#WuQOj$hwNGG?*CM_TO8Bg-1+qc>J7k5c51U8g?ZU5n?HYor;~JIjoWH-G>AoUP ztrWWLbRNqIjW#RT*WqZgPJXU7C)VaW5}MiijYbABmzoru6EmQ*N8cVK7a3|aOB#O& zBl8JY2WKfmj;h#Q!pN%9o@VNLv{OUL?rixHwOZuvX7{IJ{(EdPpuVFoQqIOa7giLVkBOKL@^smUA!tZ1CKRK}#SSM)iQHk)*R~?M!qkCruaS!#oIL1c z?J;U~&FfH#*98^G?i}pA{ z9Jg36t4=%6mhY(quYq*vSxptes9qy|7xSlH?G=S@>u>Ebe;|LVhs~@+06N<4CViBk zUiY$thvX;>Tby6z9Y1edAMQaiH zm^r3v#$Q#2T=X>bsY#D%s!bhs^M9PMAcHbCc0FMHV{u-dwlL;a1eJ63v5U*?Q_8JO zT#50!RD619#j_Uf))0ooADz~*9&lN!bBDRUgE>Vud-i5ck%vT=r^yD*^?Mp@Q^v+V zG#-?gKlr}Eeqifb{|So?HM&g91P8|av8hQoCmQXkd?7wIJwb z_^v8bbg`SAn{I*4bH$u(RZ6*xUhuA~hc=8czK8SHEKTzSxgbwi~9(OqJB&gwb^l4+m`k*Q;_?>Y-APi1{k zAHQ)P)G)f|AyjSgcCFps)Fh6Bca*Xznq36!pV6Az&m{O8$wGFD? zY&O*3*J0;_EqM#jh6^gMQKpXV?#1?>$ml1xvh8nSN>-?H=V;nJIwB07YX$e6vLxH( zqYwQ>qxwR(i4f)DLd)-$P>T-no_c!LsN@)8`e;W@)-Hj0>nJ-}Kla4-ZdPJzI&Mce zv)V_j;(3ERN3_@I$N<^|4Lf`B;8n+bX@bHbcZTopEmDI*Jfl)-pFDvo6svPRoo@(x z);_{lY<;);XzT`dBFpRmGrr}z5u1=pC^S-{ce6iXQlLGcItwJ^mZx{m$&DA_oEZ)B{_bYPq-HA zcH8WGoBG(aBU_j)vEy+_71T34@4dmSg!|M8Vf92Zj6WH7Q7t#OHQqWgFE3ARt+%!T z?oLovLVlnf?2c7pTc)~cc^($_8nyKwsN`RA-23ed3sdj(ys%pjjM+9JrctL;dy8a( z@en&CQmnV(()bu|Y%G1-4a(6x{aLytn$T-;(&{QIJB9vMox11U-1HpD@d(QkaJdEb zG{)+6Dos_L+O3NpWo^=gR?evp|CqEG?L&Ut#D*KLaRFOgOEK(Kq1@!EGcTfo+%A&I z=dLbB+d$u{sh?u)xP{PF8L%;YPPW53+@{>5W=Jt#wQpN;0_HYdw1{ksf_XhO4#2F= zyPx6Lx2<92L-;L5PD`zn6zwIH`Jk($?Qw({erA$^bC;q33hv!d!>%wRhj# zal^hk+WGNg;rJtb-EB(?czvOM=H7dl=vblBwAv>}%1@{}mnpUznfq1cE^sgsL0*4I zJ##!*B?=vI_OEVis5o+_IwMIRrpQyT_Sq~ZU%oY7c5JMIADzpD!Upz9h@iWg_>>~j zOLS;wp^i$-E?4<_cp?RiS%Rd?i;f*mOz=~(&3lo<=@(nR!_Rqiprh@weZlL!t#NCc zO!QTcInq|%#>OVgobj{~ixEUec`E25zJ~*DofsQdzIa@5^nOXj2T;8O`l--(QyU^$t?TGY^7#&FQ+2SS3B#qK*k3`ye?8jUYSajE5iBbJls75CCc(m3dk{t?- zopcER9{Z?TC)mk~gpi^kbbu>b-+a{m#8-y2^p$ka4n60w;Sc2}HMf<8JUvhCL0B&Btk)T`ctE$*qNW8L$`7!r^9T+>=<=2qaq-;ll2{`{Rg zc5a0ZUI$oG&j-qVOuKa=*v4aY#IsoM+1|c4Z)<}lEDvy;5huB@1RJPquU2U*U-;gu z=En2m+qjBzR#DEJDO`WU)hdd{Vj%^0V*KoyZ|5lzV87&g_j~NCjwv0uQVqXOb*QrQ zy|Qn`hxx(58c70$E;L(X0uZZ72M1!6oeg)(cdKO ze0gDaTz+ohR-#d)NbAH4x{I(21yjwvBQfmpLu$)|m{XolbgF!pmsqJ#D}(ylp6uC> z{bqtcI#hT#HW=wl7>p!38sKsJ`r8}lt-q%Keqy%u(xk=yiIJiUw6|5IvkS+#?JTBl z8H5(Q?l#wzazujH!8o>1xtn8#_w+397*_cy8!pQGP%K(Ga3pAjsaTbbXJlQF_+m+-UpUUent@xM zg%jqLUExj~o^vQ3Gl*>wh=_gOr2*|U64_iXb+-111aH}$TjeajM+I20xw(((>fej-@CIz4S1pi$(#}P7`4({6QS2CaQS4NPENDp>sAqD z$bH4KGzXGffkJ7R>V>)>tC)uax{UsN*dbeNC*v}#8Y#OWYwL4t$ePR?VTyIs!wea+ z5Urmc)X|^`MG~*dS6pGSbU+gPJoq*^a=_>$n4|P^w$sMBBy@f*Z^Jg6?n5?oId6f{ z$LW4M|4m502z0t7g<#Bx%X;9<=)smFolV&(V^(7Cv2-sxbxopQ!)*#ZRhTBpx1)Fc zNm1T%bONzv6@#|dz(w02AH8OXe>kQ#1FMCzO}2J_mST)+ExmBr9cva-@?;wnmWMOk z{3_~EX_xadgJGv&H@zK_8{(x84`}+c?oSBX*Ge3VdfTt&F}yCpFP?CpW+BE^cWY0^ zb&uBN!Ja3UzYHK-CTyA5=L zEMW{l3Usky#ly=7px648W31UNV@K)&Ub&zP1c7%)`{);I4b0Q<)B}3;NMG2JH=X$U zfIW4)4n9ZM`-yRj67I)YSLDK)qfUJ_ij}a#aZN~9EXrh8eZY2&=uY%2N0UFF7<~%M zsB8=erOWZ>Ct_#^tHZ|*q`H;A)5;ycw*IcmVxi8_0Xk}aJA^ath+E;xg!x+As(M#0=)3!NJR6H&9+zd#iP(m0PIW8$ z1Y^VX`>jm`W!=WpF*{ioM?C9`yOR>@0q=u7o>BP-eSHqCgMDj!2anwH?s%i2p+Q7D zzszIf5XJpE)IG4;d_(La-xenmF(tgAxK`Y4sQ}BSJEPs6N_U2vI{8=0C_F?@7<(G; zo$~G=8p+076G;`}>{MQ>t>7cm=zGtfbdDXm6||jUU|?X?CaE?(<6bKDYKeHlz}DA8 zXT={X=yp_R;HfJ9h%?eWvQ!dRgz&Su*JfNt!Wu>|XfU&68iRikRrHRW|ZxzRR^`eIGt zIeiDgVS>IeExKVRWW8-=A=yA`}`)ZkWBrZD`hpWIxBGkh&f#ijr449~m`j6{4jiJ*C!oVA8ZC?$1RM#K(_b zL9TW)kN*Y4%^-qPpMP7d4)o?Nk#>aoYHT(*g)qmRUb?**F@pnNiy6Fv9rEiUqD(^O zzyS?nBrX63BTRYduaG(0VVG2yJRe%o&rVrLjbxTaAFTd8s;<<@Qs>u(<193R8>}2_ zuwp{7;H2a*X7_jryzriZXMg?bTuegABb^87@SsKkr2)0Gyiax8KQWstw^v#ix45EVrcEhr>!NMhprl$InQMzjSFH54x5k9qHc`@9uKQzvL4ihcq{^B zPrVR=o_ic%Y>6&rMN)hTZsI7I<3&`#(nl+3y3ys9A~&^=4?PL&nd8)`OfG#n zwAMN$1&>K++c{^|7<4P=2y(B{jJsQ0a#U;HTo4ZmWZYvI{+s;Td{Yzem%0*k#)vjpB zia;J&>}ICate44SFYY3vEelqStQWFihx%^vQ@Do(sOy7yR2@WNv7Y9I^yL=nZr3mb zXKV5t@=?-Sk|b{XMhA7ZGB@2hqsx}4xwCW!in#C zI@}scZlr3-NFJ@NFaJlhyfcw{k^vvtGl`N9xSo**rDW4S}i zM9{fMPWo%4wYDG~BZ18BD+}h|GQKc-g^{++3MY>}W_uq7jGHx{mwE9fZiPCoxN$+7 zrODGGJrOkcPQUB(FD5aoS4g~7#6NR^ma7-!>mHuJfY5kTe6PpNNKC9GGRiu^L31uG z$7v`*JknQHsYB!Tm_W{a32TM099djW%5e+j0Ve_ct}IM>XLF1Ap+YvcrLV=|CKo6S zb+9Nl3_YdKP6%Cxy@6TxZ>;4&nTneadr z_ES90ydCev)LV!dN=#(*f}|ZORFdvkYBni^aLbUk>BajeWIOcmHP#8S)*2U~QKI%S zyrLmtPqb&TphJ;>yAxri#;{uyk`JJqODDw%(Z=2`1uc}br^V%>j!gS)D*q*f_-qf8&D;W1dJgQMlaH5er zN2U<%Smb7==vE}dDI8K7cKz!vs^73o9f>2sgiTzWcwY|BMYHH5%Vn7#kiw&eItCqa zIkR2~Q}>X=Ar8W|^Ms41Fm8o6IB2_j60eOeBB1Br!boW7JnoeX6Gs)?7rW0^5psc- zjS16yb>dFn>KPOF;imD}e!enuIniFzv}n$m2#gCCv4jM#ArwlzZ$7@9&XkFxZ4n!V zj3dyiwW4Ki2QG{@i>yuZXQizw_OkZI^-3otXC{!(lUpJF33gI60ak;Uqitp74|B6I zgg{b=Iz}WkhCGj1M=hu4#Aw173YxIVbISaoc z-nLZC*6Tgivd5V`K%GxhBsp@SUU60-rfc$=wb>zdJzXS&-5(NRRodFk;Kxk!S(O(a0e7oY=E( zAyS;Ow?6Q&XA+cnkCb{28_1N8H#?J!*$MmIwLq^*T_9-z^&UE@A(z9oGYtFy6EZef LrJugUA?W`A8`#=m diff --git a/packages/dashboard/src/components/chat-panel.tsx b/packages/dashboard/src/components/chat-panel.tsx deleted file mode 100644 index 0a4a98e..0000000 --- a/packages/dashboard/src/components/chat-panel.tsx +++ /dev/null @@ -1,822 +0,0 @@ -"use client"; - -import { useRef, useEffect, useState, useCallback, useMemo } from "react"; -import { useAtomValue } from "jotai/react"; -import { useChat } from "@ai-sdk/react"; -import { DefaultChatTransport } from "ai"; -import { Streamdown } from "streamdown"; -import { getChatApiUrl, chatModelAtom, availableModelsAtom } from "@/store/chat"; -import { activeSessionNameAtom } from "@/store/sessions"; -import { ModelSelector } from "@/components/model-selector"; -import { shikiTheme } from "@/lib/shiki-theme"; -import { ScrollArea } from "@/components/ui/scroll-area"; -import { cn } from "@/lib/utils"; -import { ArrowUp, Square, Trash2, ChevronRight, ImagePlus, X, Loader, Copy, Check, Download } from "lucide-react"; - -type ExtraProps = { node?: unknown }; -type MdImgProps = React.ImgHTMLAttributes & ExtraProps; -type MdHeadingProps = React.HTMLAttributes & ExtraProps; -type MdAnchorProps = React.AnchorHTMLAttributes & ExtraProps; -type MdPreProps = React.HTMLAttributes & ExtraProps; -type MdCodeProps = React.HTMLAttributes & ExtraProps; - -const chatComponents = { - img: ({ node: _node, src, alt, ...props }: MdImgProps) => { - if (typeof src === "string" && src.startsWith("data:image/")) { - return {alt}; - } - return null; - }, - h1: ({ node: _node, ...props }: MdHeadingProps) =>

, - h2: ({ node: _node, ...props }: MdHeadingProps) =>

, - h3: ({ node: _node, ...props }: MdHeadingProps) =>

, - h4: ({ node: _node, ...props }: MdHeadingProps) =>

, - h5: ({ node: _node, ...props }: MdHeadingProps) =>

, - h6: ({ node: _node, ...props }: MdHeadingProps) =>

, - a: ({ node: _node, href, children, ...props }: MdAnchorProps) => ( - - {children} - - ), - pre: ({ node: _node, ...props }: MdPreProps) => ( - 

-  ),
-  code: ({ className, children, node: _node, ...props }: MdCodeProps) => {
-    if (className?.includes("language-")) {
-      return {children};
-    }
-    return (
-      
-        {children}
-      
-    );
-  },
-};
-
-const STORAGE_PREFIX = "dashboard-chat-";
-const IMAGE_DATA_URL_RE = /data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g;
-
-function stripImagesForStorage(messages: unknown[]): unknown[] {
-  const json = JSON.stringify(messages);
-  return JSON.parse(json.replace(IMAGE_DATA_URL_RE, "[image stripped]"));
-}
-
-const SUGGESTIONS = [
-  "Go to vercel.com",
-  "Take a screenshot",
-  "What's on the page?",
-  "Click the first link",
-];
-
-interface ToolInvocationPart {
-  type: string;
-  toolCallId: string;
-  state: string;
-  input?: Record;
-  output?: unknown;
-}
-
-function isToolPart(part: { type: string }): part is ToolInvocationPart {
-  return part.type.startsWith("tool-");
-}
-
-function truncateOutput(text: string, maxLines = 30): string {
-  const lines = text.split("\n");
-  if (lines.length <= maxLines) return text;
-  return lines.slice(0, maxLines).join("\n") + `\n... (${lines.length - maxLines} more lines)`;
-}
-
-function parseOutputObject(raw: unknown): Record | null {
-  if (typeof raw === "string") {
-    try {
-      const parsed = JSON.parse(raw);
-      if (typeof parsed === "object" && parsed !== null) return parsed;
-    } catch { /* not JSON */ }
-    return null;
-  }
-  if (typeof raw === "object" && raw !== null) return raw as Record;
-  return null;
-}
-
-function formatOutput(raw: unknown): string | null {
-  if (typeof raw === "string") {
-    if (!raw.trim()) return null;
-    const obj = parseOutputObject(raw);
-    if (obj) {
-      if (typeof obj.text === "string" && obj.image) return obj.text as string;
-      const { image: _, ...rest } = obj;
-      return JSON.stringify(rest, null, 2);
-    }
-    return raw;
-  }
-  if (typeof raw === "object" && raw !== null) {
-    const r = raw as Record;
-    if (typeof r.text === "string" && r.image) return r.text as string;
-    const { image: _, ...rest } = r;
-    return JSON.stringify(rest, null, 2);
-  }
-  return null;
-}
-
-function extractImageUrl(raw: unknown): string | null {
-  const obj = parseOutputObject(raw);
-  if (!obj) return null;
-  const img = obj.image;
-  if (typeof img === "string" && img.startsWith("data:image/")) return img;
-  return null;
-}
-
-function ToolCallBlock({ part, onImageLoad }: { part: ToolInvocationPart; onImageLoad?: () => void }) {
-  const [expanded, setExpanded] = useState(false);
-  const toolName = part.type.split("-").slice(1).join("-");
-  const command = (part.input as { command?: string })?.command ?? toolName;
-  const isDone = part.state === "output-available";
-  const isRunning = !isDone;
-  const output = isDone ? formatOutput(part.output) : null;
-  const hasOutput = !!output;
-  const imageUrl = isDone ? extractImageUrl(part.output) : null;
-  const canExpand = hasOutput && !isRunning;
-
-  return (
-    

-      
 canExpand && setExpanded(!expanded)}
-      >
-        

-          {isRunning ? (
-            
-          ) : (
-            
-          )}
-          {command}
-        

-        {expanded && hasOutput && (
-          

-            
-              {truncateOutput(output)}
-            

-          

-        )}
-      

-      {imageUrl && (
-        Screenshot
-      )}
-    

-  );
-}
-
-const DEFAULT_CONTEXT_WINDOW = 128000;
-
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
-function formatTokenCount(n: number): string {
-  if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
-  if (n >= 1_000) return `${(n / 1_000).toFixed(0)}K`;
-  return `${n}`;
-}
-
-function ContextMeter({ used, total }: { used: number; total: number }) {
-  const ratio = Math.min(used / total, 1);
-  const size = 16;
-  const strokeWidth = 2;
-  const r = (size - strokeWidth) / 2;
-  const circumference = 2 * Math.PI * r;
-  const offset = circumference * (1 - ratio);
-  const color =
-    ratio > 0.9 ? "text-destructive" : ratio > 0.7 ? "text-yellow-500" : "text-muted-foreground/50";
-
-  return (
-    

-      
-        
-        
-      
-    

-  );
-}
-
-const DEFAULT_MODEL = "anthropic/claude-sonnet-4.6";
-
-function useTimeAgo(ts: number | undefined) {
-  const [, setTick] = useState(0);
-  useEffect(() => {
-    if (!ts) return;
-    const id = setInterval(() => setTick((t) => t + 1), 30_000);
-    return () => clearInterval(id);
-  }, [ts]);
-  if (!ts) return "";
-  const diff = Math.floor((Date.now() - ts) / 1000);
-  if (diff < 5) return "just now";
-  if (diff < 60) return `${diff}s ago`;
-  const mins = Math.floor(diff / 60);
-  if (mins < 60) return `${mins}m ago`;
-  const hrs = Math.floor(mins / 60);
-  return `${hrs}h ago`;
-}
-
-function MessageFooter({ model, timestamp, text }: { model: string; timestamp?: number; text: string }) {
-  const [copied, setCopied] = useState(false);
-  const timeAgo = useTimeAgo(timestamp);
-  const shortModel = model.includes("/") ? model.split("/").pop()! : model;
-
-  const handleCopy = useCallback(() => {
-    navigator.clipboard.writeText(text).then(() => {
-      setCopied(true);
-      setTimeout(() => setCopied(false), 2000);
-    });
-  }, [text]);
-
-  return (
-    

-      {shortModel}
-      {timeAgo && (
-        <>
-          ·
-          {timeAgo}
-        
-      )}
-      
-    

-  );
-}
-
-interface PendingImage {
-  file: File;
-  preview: string;
-}
-
-export function ChatPanel() {
-  const [input, setInput] = useState("");
-  const [errorDismissed, setErrorDismissed] = useState(false);
-  const [pendingImages, setPendingImages] = useState([]);
-  const fileInputRef = useRef(null);
-  const defaultModel = useAtomValue(chatModelAtom);
-  const [selectedModel, setSelectedModel] = useState(defaultModel || DEFAULT_MODEL);
-  const messagesEndRef = useRef(null);
-  const inputRef = useRef(null);
-  const sessionName = useAtomValue(activeSessionNameAtom);
-  const chatId = sessionName || "default";
-  const storageKey = `${STORAGE_PREFIX}${chatId}`;
-  const sessionRef = useRef(chatId);
-  sessionRef.current = chatId;
-  const modelRef = useRef(selectedModel);
-  modelRef.current = selectedModel;
-  const messageTimestamps = useRef>({});
-
-  useEffect(() => {
-    if (defaultModel) setSelectedModel(defaultModel);
-  }, [defaultModel]);
-
-  const transport = useRef(
-    new DefaultChatTransport({
-      api: getChatApiUrl(),
-      body: () => ({
-        session: sessionRef.current,
-        model: modelRef.current,
-      }),
-    }),
-  ).current;
-
-  const { messages, sendMessage, stop, status, setMessages, error } = useChat({
-    id: chatId,
-    transport,
-    onError: () => setErrorDismissed(false),
-  });
-
-  const visibleError = error && !errorDismissed ? error : undefined;
-  const isLoading = status === "streaming" || status === "submitted";
-  const hasMessages = messages.length > 0 || !!visibleError;
-
-  useEffect(() => {
-    for (const msg of messages) {
-      if (msg.role === "assistant" && !messageTimestamps.current[msg.id]) {
-        messageTimestamps.current[msg.id] = Date.now();
-      }
-    }
-  }, [messages]);
-
-  const models = useAtomValue(availableModelsAtom);
-  const estimatedTokens = useMemo(() => {
-    let total = 0;
-    for (const msg of messages) {
-      for (const part of msg.parts) {
-        if (part.type === "text") total += estimateTokens(part.text);
-        else if (isToolPart(part)) {
-          if (part.input) total += estimateTokens(JSON.stringify(part.input));
-          if (part.output) {
-            const raw = typeof part.output === "string" ? part.output : JSON.stringify(part.output);
-            const stripped = raw.replace(/"image"\s*:\s*"data:[^"]*"/g, '"image":"[omitted]"');
-            total += estimateTokens(stripped);
-          }
-        }
-      }
-    }
-    return total;
-  }, [messages]);
-  const contextWindow = useMemo(() => {
-    const match = models.find((m) => m.id === selectedModel);
-    return match?.context_window ?? DEFAULT_CONTEXT_WINDOW;
-  }, [models, selectedModel]);
-
-  useEffect(() => {
-    inputRef.current?.focus();
-  }, []);
-
-  const scrollToBottom = useCallback(() => {
-    messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
-  }, []);
-
-  useEffect(() => {
-    scrollToBottom();
-  }, [messages, visibleError, scrollToBottom]);
-
-  // Restore messages from localStorage when chatId changes
-  useEffect(() => {
-    try {
-      const stored = localStorage.getItem(storageKey);
-      if (stored) {
-        const parsed = JSON.parse(stored);
-        if (Array.isArray(parsed) && parsed.length > 0) {
-          setMessages(parsed);
-          return;
-        }
-      }
-    } catch {
-      // ignore
-    }
-    setMessages([]);
-  }, [chatId, storageKey, setMessages]);
-
-  // Persist messages to localStorage (strip base64 images to save space)
-  useEffect(() => {
-    if (isLoading) return;
-    if (messages.length === 0) {
-      localStorage.removeItem(storageKey);
-      return;
-    }
-    try {
-      localStorage.setItem(storageKey, JSON.stringify(stripImagesForStorage(messages)));
-    } catch {
-      // ignore quota
-    }
-  }, [messages, isLoading, storageKey]);
-
-  const addImages = useCallback((files: FileList | null) => {
-    if (!files) return;
-    const images = Array.from(files).filter((f) => f.type.startsWith("image/"));
-    setPendingImages((prev) => [
-      ...prev,
-      ...images.map((file) => ({ file, preview: URL.createObjectURL(file) })),
-    ]);
-  }, []);
-
-  const removeImage = useCallback((index: number) => {
-    setPendingImages((prev) => {
-      const next = [...prev];
-      URL.revokeObjectURL(next[index].preview);
-      next.splice(index, 1);
-      return next;
-    });
-  }, []);
-
-  const handleSubmit = useCallback(
-    (e: React.FormEvent) => {
-      e.preventDefault();
-      if ((!input.trim() && pendingImages.length === 0) || isLoading) return;
-      const dt = new DataTransfer();
-      for (const img of pendingImages) dt.items.add(img.file);
-      const files = dt.files.length > 0 ? dt.files : undefined;
-      sendMessage({ text: input, files });
-      setInput("");
-      setPendingImages((prev) => {
-        for (const p of prev) URL.revokeObjectURL(p.preview);
-        return [];
-      });
-    },
-    [input, isLoading, sendMessage, pendingImages],
-  );
-
-  const lastCompactedId = useRef(null);
-  useEffect(() => {
-    if (isLoading || messages.length === 0) return;
-    const lastAssistant = [...messages].reverse().find((m) => m.role === "assistant");
-    if (!lastAssistant) return;
-    if (lastAssistant.id === lastCompactedId.current) return;
-    const meta = (lastAssistant as any).metadata as
-      | { compacted?: boolean; summary?: string; keepLastN?: number }
-      | undefined;
-    if (!meta?.compacted || typeof meta.keepLastN !== "number") return;
-
-    lastCompactedId.current = lastAssistant.id;
-    const keep = meta.keepLastN;
-    if (keep >= messages.length) return;
-
-    const summaryMsg = {
-      id: `compaction-${Date.now()}`,
-      role: "assistant" as const,
-      parts: [
-        {
-          type: "text" as const,
-          text: `*Earlier messages were summarized to stay within the context window.*`,
-        },
-      ],
-    };
-
-    const kept = messages.slice(messages.length - keep);
-    setMessages([summaryMsg as any, ...kept]);
-  }, [isLoading, messages, setMessages]);
-
-  const handleClear = useCallback(() => {
-    setMessages([]);
-    setErrorDismissed(true);
-    localStorage.removeItem(storageKey);
-    requestAnimationFrame(() => inputRef.current?.focus());
-  }, [setMessages, storageKey]);
-
-  const handleDownload = useCallback(() => {
-    const data = messages.map((msg) => ({
-      id: msg.id,
-      role: msg.role,
-      parts: msg.parts.map((p) => {
-        if (p.type === "text") return { type: "text", text: p.text };
-        if (p.type === "file") return { type: "file", filename: (p as any).filename };
-        if (isToolPart(p)) {
-          const out = typeof p.output === "string" ? p.output : JSON.stringify(p.output);
-          const stripped = out?.replace(/"image":"data:[^"]*"/g, '"image":"[stripped]"');
-          return {
-            type: p.type,
-            toolName: (p as any).toolName,
-            state: (p as any).state,
-            input: (p as any).input,
-            output: stripped,
-          };
-        }
-        return { type: p.type };
-      }),
-    }));
-    const json = JSON.stringify({ session: chatId, model: selectedModel, messages: data }, null, 2);
-    const blob = new Blob([json], { type: "application/json" });
-    const url = URL.createObjectURL(blob);
-    const a = document.createElement("a");
-    a.href = url;
-    a.download = `chat-${chatId}-${Date.now()}.json`;
-    a.click();
-    URL.revokeObjectURL(url);
-  }, [messages, chatId, selectedModel]);
-
-  const hasVisibleContent = (parts: (typeof messages)[number]["parts"]): boolean => {
-    return parts.some(
-      (p) => (p.type === "text" && p.text.length > 0) || p.type === "file" || isToolPart(p),
-    );
-  };
-
-  return (
-    

-      {hasMessages && (
-        

-          
-          
-        

-      )}
-
-      
-        

-          {!hasMessages && !isLoading && (
-            

-              

-                Control the browser with natural language:
-              

-              

-                {SUGGESTIONS.map((s) => (
-                  
-                ))}
-              

-            

-          )}
-
-          {messages.map((message) => {
-            if (message.id.startsWith("compaction-")) {
-              return (
-                

-                  

-                  Earlier messages summarized
-                  

-                

-              );
-            }
-            if (!hasVisibleContent(message.parts)) return null;
-            return (
-              

-                {message.role === "user" ? (
-                  

-                    {message.parts.some((p) => p.type === "file") && (
-                      

-                        {message.parts
-                          .filter((p): p is Extract => p.type === "file")
-                          .map((p, i) => (
-                            {p.filename
-                          ))}
-                      

-                    )}
-                    

-                      {message.parts
-                        .filter((p): p is Extract => p.type === "text")
-                        .map((p) => p.text)
-                        .join("")}
-                    

-                  

-                ) : (
-                  

-                    {(() => {
-                      type Group = { type: "tools" | "text"; items: (typeof message.parts)[number][] };
-                      const groups: Group[] = [];
-                      for (const part of message.parts) {
-                        const groupType = isToolPart(part) ? "tools" : "text";
-                        const last = groups[groups.length - 1];
-                        if (last && last.type === groupType) {
-                          last.items.push(part);
-                        } else {
-                          groups.push({ type: groupType, items: [part] });
-                        }
-                      }
-
-                      return groups.map((group, gi) => {
-                        if (group.type === "tools") {
-                          return (
-                            

-                              {group.items.map((part) => {
-                                if (!isToolPart(part)) return null;
-                                return ;
-                              })}
-                            

-                          );
-                        }
-                        const combinedText = group.items
-                          .filter((p): p is Extract => p.type === "text" && !!p.text)
-                          .map((p) => p.text)
-                          .join("");
-                        if (!combinedText) return null;
-                        return (
-                          

-                            
-                              {combinedText}
-                            
-                          

-                        );
-                      });
-                    })()}
-                    {(() => {
-                      const isLast = message === messages[messages.length - 1];
-                      const isComplete = !isLast || !isLoading;
-                      if (!isComplete) return null;
-                      const fullText = message.parts
-                        .filter((p): p is Extract => p.type === "text" && !!p.text)
-                        .map((p) => p.text)
-                        .join("");
-                      return (
-                        
-                      );
-                    })()}
-                  

-                )}
-              

-            );
-          })}
-
-          {isLoading && messages.length > 0 && (() => {
-            const lastMsg = messages[messages.length - 1];
-            const lastPart = lastMsg?.parts[lastMsg.parts.length - 1];
-            const noVisibleContent = !lastMsg || !hasVisibleContent(lastMsg.parts);
-            const lastIsCompletedTool = lastPart && isToolPart(lastPart) && lastPart.state === "output-available";
-            if (noVisibleContent || lastIsCompletedTool) {
-              return (
-                
-                  Working...
-                
-              );
-            }
-            return null;
-          })()}
-
-          {visibleError && (
-            

-              {(() => {
-                try {
-                  const parsed = JSON.parse(visibleError.message);
-                  return parsed.message || parsed.error || visibleError.message;
-                } catch {
-                  return visibleError.message || "Something went wrong.";
-                }
-              })()}
-            

-          )}
-
-          

-        

-      
-
-      

-        

-          {pendingImages.length > 0 && (
-            

-              {pendingImages.map((img, i) => (
-                

-                  {img.file.name}
-                  
-                

-              ))}
-            

-          )}
-          

-