From c0e2b80f8cc991fbd699e0a0a201f0ec2623720e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 24 Feb 2026 11:35:50 -0600 Subject: [PATCH] add dogfood skill for agent-driven exploratory qa (#538) * dogfood skill * evals * haiku * fixes * caching * fixes * don't use npx --- .gitignore | 5 + package.json | 2 + pnpm-lock.yaml | 174 ++++++++++++ skills/dogfood/SKILL.md | 216 +++++++++++++++ skills/dogfood/references/issue-taxonomy.md | 109 ++++++++ .../templates/dogfood-report-template.md | 53 ++++ test/e2e/dogfood.eval.ts | 261 ++++++++++++++++++ test/e2e/dogfood.test.ts | 210 ++++++++++++++ test/e2e/fixtures/buggy-app.html | 159 +++++++++++ vitest.config.ts | 2 +- 10 files changed, 1190 insertions(+), 1 deletion(-) create mode 100644 skills/dogfood/SKILL.md create mode 100644 skills/dogfood/references/issue-taxonomy.md create mode 100644 skills/dogfood/templates/dogfood-report-template.md create mode 100644 test/e2e/dogfood.eval.ts create mode 100644 test/e2e/dogfood.test.ts create mode 100644 test/e2e/fixtures/buggy-app.html diff --git a/.gitignore b/.gitignore index 331632b..d1d9e00 100644 --- a/.gitignore +++ b/.gitignore @@ -27,10 +27,15 @@ npm-debug.log* .DS_Store Thumbs.db +# Python +__pycache__/ + # Test artifacts *.png *.jpeg *.jpg +*.webm +test/e2e/.dogfood-output/ # Package manager package-lock.json diff --git a/package.json b/package.json index 6d8c1e0..dff3069 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "format:check": "prettier --check 'src/**/*.ts'", "test": "vitest run", "test:watch": "vitest", + "test:e2e:dogfood": "vitest run test/e2e/dogfood.eval.ts", "postinstall": "node scripts/postinstall.js", "changeset": "changeset", "ci:version": "changeset version && pnpm run version:sync && pnpm install --no-frozen-lockfile", @@ -62,6 +63,7 @@ "zod": "^3.22.4" }, "devDependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.2.52", "@changesets/cli": "^2.29.8", "@types/node": "^20.10.0", "@types/ws": "^8.18.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3488e40..ce8f6b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ importers: specifier: ^3.22.4 version: 3.25.76 devDependencies: + '@anthropic-ai/claude-agent-sdk': + specifier: ^0.2.52 + version: 0.2.52(zod@3.25.76) '@changesets/cli': specifier: ^2.29.8 version: 2.29.8(@types/node@20.19.28) @@ -57,6 +60,12 @@ importers: packages: + '@anthropic-ai/claude-agent-sdk@0.2.52': + resolution: {integrity: sha512-rdTQUu/HjKlDNNxJuhtXY6LJDOLvzVBU7sXFuFIG6CEC/nFfcvYq035EyjVw4nzu7lLZim/m+g2yZ8uNIcbaFw==} + engines: {node: '>=18.0.0'} + peerDependencies: + zod: ^4.0.0 + '@appium/logger@1.7.1': resolution: {integrity: sha512-9C2o9X/lBEDBUnKfAi3mRo9oG7Z03nmISLwsGkWxIWjMAvBdJD0RRSJMekWVKzfXN3byrI1WlCXTITzN4LAoLw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0, npm: '>=8'} @@ -276,6 +285,95 @@ packages: cpu: [x64] os: [win32] + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@inquirer/external-editor@1.0.3': resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} engines: {node: '>=18'} @@ -1950,6 +2048,20 @@ packages: snapshots: + '@anthropic-ai/claude-agent-sdk@0.2.52(zod@3.25.76)': + dependencies: + zod: 3.25.76 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + '@appium/logger@1.7.1': dependencies: console-control-strings: 1.1.0 @@ -2181,6 +2293,68 @@ snapshots: '@esbuild/win32-x64@0.27.2': optional: true + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + '@inquirer/external-editor@1.0.3(@types/node@20.19.28)': dependencies: chardet: 2.1.1 diff --git a/skills/dogfood/SKILL.md b/skills/dogfood/SKILL.md new file mode 100644 index 0000000..be25ce5 --- /dev/null +++ b/skills/dogfood/SKILL.md @@ -0,0 +1,216 @@ +--- +name: dogfood +description: Systematically explore and test a web application to find bugs, UX issues, and other problems. Use when asked to "dogfood", "QA", "exploratory test", "find issues", "bug hunt", "test this app/site/platform", or review the quality of a web application. Produces a structured report with full reproduction evidence -- step-by-step screenshots, repro videos, and detailed repro steps for every issue -- so findings can be handed directly to the responsible teams. +allowed-tools: Bash(agent-browser:*), Bash(npx agent-browser:*) +--- + +# Dogfood + +Systematically explore a web application, find issues, and produce a report with full reproduction evidence for every finding. + +## Setup + +Only the **Target URL** is required. Everything else has sensible defaults -- use them unless the user explicitly provides an override. + +| Parameter | Default | Example override | +|-----------|---------|-----------------| +| **Target URL** | _(required)_ | `vercel.com`, `http://localhost:3000` | +| **Session name** | Slugified domain (e.g., `vercel.com` -> `vercel-com`) | `--session my-session` | +| **Output directory** | `./dogfood-output/` | `Output directory: /tmp/qa` | +| **Scope** | Full app | `Focus on the billing page` | +| **Authentication** | None | `Sign in to user@example.com` | + +If the user says something like "dogfood vercel.com", start immediately with defaults. Do not ask clarifying questions unless authentication is mentioned but credentials are missing. + +Always use `agent-browser` directly -- never `npx agent-browser`. The direct binary uses the fast Rust client. `npx` routes through Node.js and is significantly slower. + +## Workflow + +``` +1. Initialize Set up session, output dirs, report file +2. Authenticate Sign in if needed, save state +3. Orient Navigate to starting point, take initial snapshot +4. Explore Systematically visit pages and test features +5. Document Screenshot + record each issue as found +6. Wrap up Update summary counts, close session +``` + +### 1. Initialize + +```bash +mkdir -p {OUTPUT_DIR}/screenshots {OUTPUT_DIR}/videos +``` + +Copy the report template into the output directory and fill in the header fields: + +```bash +cp {SKILL_DIR}/templates/dogfood-report-template.md {OUTPUT_DIR}/report.md +``` + +Start a named session: + +```bash +agent-browser --session {SESSION} open {TARGET_URL} +agent-browser --session {SESSION} wait --load networkidle +``` + +### 2. Authenticate + +If the app requires login: + +```bash +agent-browser --session {SESSION} snapshot -i +# Identify login form refs, fill credentials +agent-browser --session {SESSION} fill @e1 "{EMAIL}" +agent-browser --session {SESSION} fill @e2 "{PASSWORD}" +agent-browser --session {SESSION} click @e3 +agent-browser --session {SESSION} wait --load networkidle +``` + +For OTP/email codes: ask the user, wait for their response, then enter the code. + +After successful login, save state for potential reuse: + +```bash +agent-browser --session {SESSION} state save {OUTPUT_DIR}/auth-state.json +``` + +### 3. Orient + +Take an initial annotated screenshot and snapshot to understand the app structure: + +```bash +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/initial.png +agent-browser --session {SESSION} snapshot -i +``` + +Identify the main navigation elements and map out the sections to visit. + +### 4. Explore + +Read [references/issue-taxonomy.md](references/issue-taxonomy.md) for the full list of what to look for and the exploration checklist. + +**Strategy -- work through the app systematically:** + +- Start from the main navigation. Visit each top-level section. +- Within each section, test interactive elements: click buttons, fill forms, open dropdowns/modals. +- Check edge cases: empty states, error handling, boundary inputs. +- Try realistic end-to-end workflows (create, edit, delete flows). +- Check the browser console for errors periodically. + +**At each page:** + +```bash +agent-browser --session {SESSION} snapshot -i +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/{page-name}.png +agent-browser --session {SESSION} errors +agent-browser --session {SESSION} console +``` + +Use your judgment on how deep to go. Spend more time on core features and less on peripheral pages. If you find a cluster of issues in one area, investigate deeper. + +### 5. Document Issues (Repro-First) + +Steps 4 and 5 happen together -- explore and document in a single pass. When you find an issue, stop exploring and document it immediately before moving on. Do not explore the whole app first and document later. + +Every issue must be reproducible. When you find something wrong, do not just note it -- prove it with evidence. The goal is that someone reading the report can see exactly what happened and replay it. + +**Choose the right level of evidence for the issue:** + +#### Interactive / behavioral issues (functional, ux, console errors on action) + +These require user interaction to reproduce -- use full repro with video and step-by-step screenshots: + +1. **Start a repro video** _before_ reproducing: + +```bash +agent-browser --session {SESSION} record start {OUTPUT_DIR}/videos/issue-{NNN}-repro.webm +``` + +2. **Walk through the steps at human pace.** Pause 1-2 seconds between actions so the video is watchable. Take a screenshot at each step: + +```bash +agent-browser --session {SESSION} screenshot {OUTPUT_DIR}/screenshots/issue-{NNN}-step-1.png +sleep 1 +# Perform action (click, fill, etc.) +sleep 1 +agent-browser --session {SESSION} screenshot {OUTPUT_DIR}/screenshots/issue-{NNN}-step-2.png +sleep 1 +# ...continue until the issue manifests +``` + +3. **Capture the broken state.** Pause so the viewer can see it, then take an annotated screenshot: + +```bash +sleep 2 +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/issue-{NNN}-result.png +``` + +4. **Stop the video:** + +```bash +agent-browser --session {SESSION} record stop +``` + +5. Write numbered repro steps in the report, each referencing its screenshot. + +#### Static / visible-on-load issues (typos, placeholder text, clipped text, misalignment, console errors on load) + +These are visible without interaction -- a single annotated screenshot is sufficient. No video, no multi-step repro: + +```bash +agent-browser --session {SESSION} screenshot --annotate {OUTPUT_DIR}/screenshots/issue-{NNN}.png +``` + +Write a brief description and reference the screenshot in the report. Set **Repro Video** to `N/A`. + +--- + +**For all issues:** + +1. **Append to the report immediately.** Do not batch issues for later. Write each one as you find it so nothing is lost if the session is interrupted. + +2. **Increment the issue counter** (ISSUE-001, ISSUE-002, ...). + +### 6. Wrap Up + +Aim to find **5-10 well-documented issues**, then wrap up. Depth of evidence matters more than total count -- 5 issues with full repro beats 20 with vague descriptions. + +After exploring: + +1. Re-read the report and update the summary severity counts so they match the actual issues. Every `### ISSUE-` block must be reflected in the totals. +2. Close the session: + +```bash +agent-browser --session {SESSION} close +``` + +3. Tell the user the report is ready and summarize findings: total issues, breakdown by severity, and the most critical items. + +## Guidance + +- **Repro is everything.** Every issue needs proof -- but match the evidence to the issue. Interactive bugs need video and step-by-step screenshots. Static bugs (typos, placeholder text, visual glitches visible on load) only need a single annotated screenshot. +- **Don't record video for static issues.** A typo or clipped text doesn't benefit from a video. Save video for issues that involve user interaction, timing, or state changes. +- **For interactive issues, screenshot each step.** Capture the before, the action, and the after -- so someone can see the full sequence. +- **Write repro steps that map to screenshots.** Each numbered step in the report should reference its corresponding screenshot. A reader should be able to follow the steps visually without touching a browser. +- **Be thorough but use judgment.** You are not following a test script -- you are exploring like a real user would. If something feels off, investigate. +- **Write findings incrementally.** Append each issue to the report as you discover it. If the session is interrupted, findings are preserved. Never batch all issues for the end. +- **Never delete output files.** Do not `rm` screenshots, videos, or the report mid-session. Do not close the session and restart. Work forward, not backward. +- **Never read the target app's source code.** You are testing as a user, not auditing code. Do not read HTML, JS, or config files of the app under test. All findings must come from what you observe in the browser. +- **Check the console.** Many issues are invisible in the UI but show up as JS errors or failed requests. +- **Test like a user, not a robot.** Try common workflows end-to-end. Click things a real user would click. Enter realistic data. +- **Type like a human.** When filling form fields during video recording, use `type` instead of `fill` -- it types character-by-character. Use `fill` only outside of video recording when speed matters. +- **Pace repro videos for humans.** Add `sleep 1` between actions and `sleep 2` before the final result screenshot. Videos should be watchable at 1x speed -- a human reviewing the report needs to see what happened, not a blur of instant state changes. +- **Be efficient with commands.** Batch multiple `agent-browser` commands in a single shell call when they are independent (e.g., `agent-browser ... screenshot ... && agent-browser ... console`). Use `agent-browser --session {SESSION} scroll down 300` for scrolling -- do not use `key` or `evaluate` to scroll. + +## References + +| Reference | When to Read | +|-----------|--------------| +| [references/issue-taxonomy.md](references/issue-taxonomy.md) | Start of session -- calibrate what to look for, severity levels, exploration checklist | + +## Templates + +| Template | Purpose | +|----------|---------| +| [templates/dogfood-report-template.md](templates/dogfood-report-template.md) | Copy into output directory as the report file | diff --git a/skills/dogfood/references/issue-taxonomy.md b/skills/dogfood/references/issue-taxonomy.md new file mode 100644 index 0000000..c3edbe5 --- /dev/null +++ b/skills/dogfood/references/issue-taxonomy.md @@ -0,0 +1,109 @@ +# Issue Taxonomy + +Reference for categorizing issues found during dogfooding. Read this at the start of a dogfood session to calibrate what to look for. + +## Contents + +- [Severity Levels](#severity-levels) +- [Categories](#categories) +- [Exploration Checklist](#exploration-checklist) + +## Severity Levels + +| Severity | Definition | +|----------|------------| +| **critical** | Blocks a core workflow, causes data loss, or crashes the app | +| **high** | Major feature broken or unusable, no workaround | +| **medium** | Feature works but with noticeable problems, workaround exists | +| **low** | Minor cosmetic or polish issue | + +## Categories + +### Visual / UI + +- Layout broken or misaligned elements +- Overlapping or clipped text +- Inconsistent spacing, padding, or margins +- Missing or broken icons/images +- Dark mode / light mode rendering issues +- Responsive layout problems (viewport sizes) +- Z-index stacking issues (elements hidden behind others) +- Font rendering issues (wrong font, size, weight) +- Color contrast problems +- Animation glitches or jank + +### Functional + +- Broken links (404, wrong destination) +- Buttons or controls that do nothing on click +- Form validation that rejects valid input or accepts invalid input +- Incorrect redirects +- Features that fail silently +- State not persisted when expected (lost on refresh, navigation) +- Race conditions (double-submit, stale data) +- Broken search or filtering +- Pagination issues +- File upload/download failures + +### UX + +- Confusing or unclear navigation +- Missing loading indicators or feedback after actions +- Slow or unresponsive interactions (>300ms perceived delay) +- Unclear error messages +- Missing confirmation for destructive actions +- Dead ends (no way to go back or proceed) +- Inconsistent patterns across similar features +- Missing keyboard shortcuts or focus management +- Unintuitive defaults +- Missing empty states or unhelpful empty states + +### Content + +- Typos or grammatical errors +- Outdated or incorrect text +- Placeholder or lorem ipsum content left in +- Truncated text without tooltip or expansion +- Missing or wrong labels +- Inconsistent terminology + +### Performance + +- Slow page loads (>3s) +- Janky scrolling or animations +- Large layout shifts (content jumping) +- Excessive network requests (check via console/network) +- Memory leaks (page slows over time) +- Unoptimized images (large file sizes) + +### Console / Errors + +- JavaScript exceptions in console +- Failed network requests (4xx, 5xx) +- Deprecation warnings +- CORS errors +- Mixed content warnings +- Unhandled promise rejections + +### Accessibility + +- Missing alt text on images +- Unlabeled form inputs +- Poor keyboard navigation (can't tab to elements) +- Focus traps +- Insufficient color contrast +- Missing ARIA attributes on dynamic content +- Screen reader incompatible patterns + +## Exploration Checklist + +Use this as a guide for what to test on each page/feature: + +1. **Visual scan** -- Take an annotated screenshot. Look for layout, alignment, and rendering issues. +2. **Interactive elements** -- Click every button, link, and control. Do they work? Is there feedback? +3. **Forms** -- Fill and submit. Test empty submission, invalid input, and edge cases. +4. **Navigation** -- Follow all navigation paths. Check breadcrumbs, back button, deep links. +5. **States** -- Check empty states, loading states, error states, and full/overflow states. +6. **Console** -- Check for JS errors, failed requests, and warnings. +7. **Responsiveness** -- If relevant, test at different viewport sizes. +8. **Auth boundaries** -- Test what happens when not logged in, with different roles if applicable. diff --git a/skills/dogfood/templates/dogfood-report-template.md b/skills/dogfood/templates/dogfood-report-template.md new file mode 100644 index 0000000..a7732a4 --- /dev/null +++ b/skills/dogfood/templates/dogfood-report-template.md @@ -0,0 +1,53 @@ +# Dogfood Report: {APP_NAME} + +| Field | Value | +|-------|-------| +| **Date** | {DATE} | +| **App URL** | {URL} | +| **Session** | {SESSION_NAME} | +| **Scope** | {SCOPE} | + +## Summary + +| Severity | Count | +|----------|-------| +| Critical | 0 | +| High | 0 | +| Medium | 0 | +| Low | 0 | +| **Total** | **0** | + +## Issues + + + +### ISSUE-001: {Short title} + +| Field | Value | +|-------|-------| +| **Severity** | critical / high / medium / low | +| **Category** | visual / functional / ux / content / performance / console / accessibility | +| **URL** | {page URL where issue was found} | +| **Repro Video** | {path to video, or N/A for static issues} | + +**Description** + +{What is wrong, what was expected, and what actually happened.} + +**Repro Steps** + + + +1. Navigate to {URL} + ![Step 1](screenshots/issue-001-step-1.png) + +2. {Action -- e.g., click "Settings" in the sidebar} + ![Step 2](screenshots/issue-001-step-2.png) + +3. {Action -- e.g., type "test" in the search field and press Enter} + ![Step 3](screenshots/issue-001-step-3.png) + +4. **Observe:** {what goes wrong -- e.g., the page shows a blank white screen instead of search results} + ![Result](screenshots/issue-001-result.png) + +--- diff --git a/test/e2e/dogfood.eval.ts b/test/e2e/dogfood.eval.ts new file mode 100644 index 0000000..6e5469e --- /dev/null +++ b/test/e2e/dogfood.eval.ts @@ -0,0 +1,261 @@ +import { describe, it, expect, beforeAll } from 'vitest'; +import { query } from '@anthropic-ai/claude-agent-sdk'; +import type { SDKMessage, SDKResultMessage } from '@anthropic-ai/claude-agent-sdk'; +import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, readdirSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +const AI_GATEWAY_URL = + process.env.ANTHROPIC_BASE_URL || 'https://ai-gateway.vercel.sh'; +const API_KEY = process.env.AI_GATEWAY_API_KEY; +const MODEL = process.env.DOGFOOD_MODEL || 'anthropic/claude-haiku-4.5'; +const CUSTOM_URL = process.env.DOGFOOD_URL; + +const FIXTURE_PATH = path.resolve('test/e2e/fixtures/buggy-app.html'); +const SKILL_PATH = path.resolve('skills/dogfood/SKILL.md'); +const TARGET_URL = CUSTOM_URL || `file://${FIXTURE_PATH}`; +const IS_FIXTURE = !CUSTOM_URL; + +const OUTPUT_DIR = path.resolve('test/e2e/.dogfood-output'); +const EVAL_TIMEOUT = 10 * 60 * 1000; + +async function runDogfood(outputDir: string): Promise<{ + result: SDKResultMessage | null; + messages: SDKMessage[]; + toolsUsed: Set; +}> { + const instruction = [ + `Read the dogfood skill at ${SKILL_PATH} and follow its workflow.`, + `Dogfood ${TARGET_URL}`, + `Output directory: ${outputDir}`, + ].join(' '); + + const messages: SDKMessage[] = []; + const toolsUsed = new Set(); + let result: SDKResultMessage | null = null; + + const conversation = query({ + prompt: instruction, + options: { + model: MODEL, + cwd: process.cwd(), + allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], + permissionMode: 'bypassPermissions', + allowDangerouslySkipPermissions: true, + maxTurns: 80, + maxBudgetUsd: 2, + settingSources: ['project'], + persistSession: false, + env: { + ...process.env, + ANTHROPIC_BASE_URL: AI_GATEWAY_URL, + ANTHROPIC_API_KEY: API_KEY, + }, + }, + }); + + const verbose = process.env.DOGFOOD_VERBOSE !== '0'; + const log = verbose ? (msg: string) => process.stderr.write(` [dogfood] ${msg}\n`) : () => {}; + + const chatLogPath = path.join(outputDir, 'chat-log.jsonl'); + writeFileSync(chatLogPath, ''); + + function appendToLog(entry: Record) { + appendFileSync(chatLogPath, JSON.stringify(entry) + '\n'); + } + + for await (const message of conversation) { + messages.push(message); + + if (message.type === 'system' && message.subtype === 'init') { + log(`session started (model: ${message.model})`); + appendToLog({ type: 'system', subtype: 'init', model: message.model }); + } + + if (message.type === 'assistant' && message.message?.content) { + const logParts: Record[] = []; + for (const block of message.message.content) { + if ('type' in block && block.type === 'tool_use') { + toolsUsed.add(block.name); + const input = block.input as Record; + let preview: string; + if (block.name === 'Bash') { + const cmd = String(input.command ?? ''); + const firstLine = cmd.split('\n').find(l => l.trim() && !l.trim().startsWith('#')) ?? cmd.split('\n')[0]; + preview = firstLine.trim().slice(0, 200); + } else if (block.name === 'Write') { + preview = String(input.file_path ?? input.path ?? ''); + } else if (block.name === 'Read') { + preview = String(input.file_path ?? input.path ?? ''); + } else if (block.name === 'Edit') { + preview = String(input.file_path ?? input.path ?? ''); + } else { + preview = JSON.stringify(input).slice(0, 120); + } + log(`${block.name}: ${preview}`); + logParts.push({ tool: block.name, input: block.input }); + } + if ('type' in block && block.type === 'text' && block.text) { + const line = block.text.split('\n')[0].slice(0, 120); + if (line.trim()) log(line); + logParts.push({ text: block.text }); + } + } + appendToLog({ type: 'assistant', content: logParts }); + } + + if (message.type === 'result') { + result = message; + const cost = `$${message.total_cost_usd.toFixed(4)}`; + const usage = message.usage; + const cacheRead = usage.cache_read_input_tokens ?? 0; + const cacheCreate = usage.cache_creation_input_tokens ?? 0; + const inputTokens = usage.input_tokens ?? 0; + const cacheInfo = cacheRead > 0 + ? ` | cache: ${cacheRead} read, ${cacheCreate} created, ${inputTokens} uncached` + : ''; + if (message.subtype === 'success') { + log(`done (${message.num_turns} turns, ${cost}${cacheInfo})`); + } else { + log(`stopped: ${message.subtype} (${message.num_turns} turns, ${cost}${cacheInfo})`); + } + appendToLog({ type: 'result', subtype: message.subtype, num_turns: message.num_turns, cost: message.total_cost_usd }); + } + } + + log(`chat log: ${chatLogPath}`); + + return { result, messages, toolsUsed }; +} + +function findFiles(dir: string, ext: string): string[] { + if (!existsSync(dir)) return []; + return readdirSync(dir, { recursive: true }) + .map(String) + .filter((f) => f.endsWith(ext)); +} + +describe.skipIf(!API_KEY)('Dogfood e2e eval (Agent SDK)', () => { + const outputDir = OUTPUT_DIR; + let evalResult: Awaited>; + + beforeAll(async () => { + if (existsSync(outputDir)) { + rmSync(outputDir, { recursive: true, force: true }); + } + mkdirSync(outputDir, { recursive: true }); + evalResult = await runDogfood(outputDir); + }, EVAL_TIMEOUT); + + it('completes without hard failure', () => { + expect(evalResult.result, 'No result message received').toBeTruthy(); + const acceptable = ['success', 'error_max_turns', 'error_max_budget_usd']; + expect( + acceptable, + `Agent failed unexpectedly: ${evalResult.result!.subtype}` + ).toContain(evalResult.result!.subtype); + }); + + it('used agent-browser via Bash tool', () => { + expect( + evalResult.toolsUsed.has('Bash'), + 'Agent never used Bash (needed for agent-browser commands)' + ).toBe(true); + }); + + it('produced a report file', () => { + const reportPath = path.join(outputDir, 'report.md'); + expect(existsSync(reportPath), 'report.md not found in output dir').toBe( + true + ); + }); + + it('found a minimum number of issues', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + + const issueBlocks = report.match(/###\s+ISSUE-\d+/g) || []; + if (IS_FIXTURE) { + expect( + issueBlocks.length, + `Expected >=2 issues from fixture, found ${issueBlocks.length}` + ).toBeGreaterThanOrEqual(2); + } else { + expect(issueBlocks.length).toBeGreaterThanOrEqual(1); + } + }); + + it('each issue has required fields and repro evidence', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + + const issueSections = report.split(/(?=###\s+ISSUE-\d+)/).slice(1); + for (const section of issueSections) { + const issueId = section.match(/ISSUE-\d+/)?.[0] ?? 'unknown'; + + expect(section, `${issueId}: missing Severity`).toMatch( + /\*\*Severity\*\*/i + ); + + const sevMatch = section.match( + /\*\*Severity\*\*\s*\|?\s*(critical|high|medium|low)/i + ); + expect(sevMatch, `${issueId}: invalid severity value`).toBeTruthy(); + + expect(section, `${issueId}: missing Category`).toMatch( + /\*\*Category\*\*/i + ); + + expect(section, `${issueId}: missing URL`).toMatch(/\*\*URL\*\*/i); + + expect(section, `${issueId}: missing Repro Video field`).toMatch( + /\*\*Repro Video\*\*/i + ); + + const hasScreenshot = /!\[.*?\]\(.*?\)/.test(section); + const hasReproSteps = /\*\*Repro Steps\*\*/i.test(section); + expect( + hasScreenshot || hasReproSteps, + `${issueId}: needs either screenshot refs or repro steps` + ).toBe(true); + } + }); + + it('has a summary table with non-zero total', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + + expect(report, 'Missing Summary section').toContain('## Summary'); + const totalMatch = report.match(/\*\*Total\*\*\s*\|?\s*\*\*(\d+)\*\*/); + expect(totalMatch, 'Summary Total not found').toBeTruthy(); + if (totalMatch) { + const total = parseInt(totalMatch[1], 10); + expect(total, 'Summary Total should be > 0').toBeGreaterThan(0); + } + }); + + it('produced screenshot files', () => { + const screenshotsDir = path.join(outputDir, 'screenshots'); + const screenshots = findFiles(screenshotsDir, '.png'); + expect( + screenshots.length, + 'No screenshot files found in output' + ).toBeGreaterThan(0); + }); + + it('produced video files for interactive issues', () => { + const reportPath = path.join(outputDir, 'report.md'); + if (!existsSync(reportPath)) return; + const report = readFileSync(reportPath, 'utf-8'); + const hasVideoRefs = /videos\/issue-\d+/.test(report); + if (!hasVideoRefs) return; + const videosDir = path.join(outputDir, 'videos'); + const videos = findFiles(videosDir, '.webm'); + expect( + videos.length, + 'Report references videos but none were found' + ).toBeGreaterThan(0); + }); +}); diff --git a/test/e2e/dogfood.test.ts b/test/e2e/dogfood.test.ts new file mode 100644 index 0000000..e0d5d31 --- /dev/null +++ b/test/e2e/dogfood.test.ts @@ -0,0 +1,210 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync } from 'node:fs'; +import path from 'node:path'; + +const SKILL_DIR = path.resolve('skills/dogfood'); +const SKILL_MD = path.join(SKILL_DIR, 'SKILL.md'); +const TAXONOMY_MD = path.join(SKILL_DIR, 'references', 'issue-taxonomy.md'); +const TEMPLATE_MD = path.join(SKILL_DIR, 'templates', 'dogfood-report-template.md'); + +function readSkillFile(filePath: string): string { + return readFileSync(filePath, 'utf-8'); +} + +function parseFrontmatter(content: string): Record { + const match = content.match(/^---\n([\s\S]*?)\n---/); + if (!match) return {}; + const fields: Record = {}; + for (const line of match[1].split('\n')) { + const colonIdx = line.indexOf(':'); + if (colonIdx > 0) { + fields[line.slice(0, colonIdx).trim()] = line.slice(colonIdx + 1).trim(); + } + } + return fields; +} + +describe('Dogfood skill: file structure', () => { + it('SKILL.md exists', () => { + expect(existsSync(SKILL_MD)).toBe(true); + }); + + it('references/issue-taxonomy.md exists', () => { + expect(existsSync(TAXONOMY_MD)).toBe(true); + }); + + it('templates/dogfood-report-template.md exists', () => { + expect(existsSync(TEMPLATE_MD)).toBe(true); + }); +}); + +describe('Dogfood skill: SKILL.md frontmatter', () => { + const content = readSkillFile(SKILL_MD); + const frontmatter = parseFrontmatter(content); + + it('has name field', () => { + expect(frontmatter.name).toBe('dogfood'); + }); + + it('has description field', () => { + expect(frontmatter.description).toBeTruthy(); + expect(frontmatter.description!.length).toBeGreaterThan(50); + }); + + it('has allowed-tools field', () => { + expect(frontmatter['allowed-tools']).toBeTruthy(); + expect(frontmatter['allowed-tools']).toContain('agent-browser'); + }); +}); + +describe('Dogfood skill: SKILL.md body references', () => { + const content = readSkillFile(SKILL_MD); + + it('references issue-taxonomy.md', () => { + expect(content).toContain('references/issue-taxonomy.md'); + }); + + it('references dogfood-report-template.md', () => { + expect(content).toContain('templates/dogfood-report-template.md'); + }); + + it('referenced files exist on disk', () => { + const refPattern = /\[.*?\]\((references\/.*?\.md|templates\/.*?\.md)\)/g; + const refs = [...content.matchAll(refPattern)].map((m) => m[1]); + expect(refs.length).toBeGreaterThan(0); + for (const ref of refs) { + const fullPath = path.join(SKILL_DIR, ref); + expect(existsSync(fullPath), `Missing: ${ref}`).toBe(true); + } + }); +}); + +describe('Dogfood skill: report template', () => { + const template = readSkillFile(TEMPLATE_MD); + + it('has ISSUE- prefix in issue blocks', () => { + expect(template).toContain('ISSUE-'); + }); + + it('has Severity field', () => { + expect(template).toContain('**Severity**'); + }); + + it('has Category field', () => { + expect(template).toContain('**Category**'); + }); + + it('has URL field', () => { + expect(template).toContain('**URL**'); + }); + + it('has Repro Video field', () => { + expect(template).toContain('**Repro Video**'); + }); + + it('has Repro Steps section', () => { + expect(template).toContain('**Repro Steps**'); + }); + + it('has screenshot image references in repro steps', () => { + expect(template).toMatch(/!\[.*?\]\(screenshots\//); + }); + + it('lists all valid severity values', () => { + expect(template).toMatch(/critical\s*\/\s*high\s*\/\s*medium\s*\/\s*low/); + }); + + it('lists all valid category values', () => { + const categoryLine = template + .split('\n') + .find((l) => l.includes('**Category**')); + expect(categoryLine).toBeTruthy(); + for (const cat of [ + 'visual', + 'functional', + 'ux', + 'content', + 'performance', + 'console', + 'accessibility', + ]) { + expect(categoryLine!.toLowerCase()).toContain(cat); + } + }); + + it('has Summary table with severity counts', () => { + expect(template).toContain('## Summary'); + for (const sev of ['Critical', 'High', 'Medium', 'Low', 'Total']) { + expect(template).toContain(sev); + } + }); +}); + +describe('Dogfood skill: issue taxonomy', () => { + const taxonomy = readSkillFile(TAXONOMY_MD); + + it('has severity level definitions', () => { + expect(taxonomy).toContain('## Severity Levels'); + for (const sev of ['critical', 'high', 'medium', 'low']) { + expect(taxonomy.toLowerCase()).toContain(`**${sev}**`); + } + }); + + it('has all 7 category sections', () => { + const expectedCategories = [ + 'Visual', + 'Functional', + 'UX', + 'Content', + 'Performance', + 'Console', + 'Accessibility', + ]; + for (const cat of expectedCategories) { + expect(taxonomy).toMatch(new RegExp(`###\\s+.*${cat}`, 'i')); + } + }); + + it('has exploration checklist', () => { + expect(taxonomy).toContain('## Exploration Checklist'); + }); + + it('checklist has numbered items', () => { + const checklistSection = taxonomy.split('## Exploration Checklist')[1]; + expect(checklistSection).toBeTruthy(); + const numberedItems = checklistSection!.match(/^\d+\./gm); + expect(numberedItems!.length).toBeGreaterThanOrEqual(5); + }); +}); + +describe('Dogfood skill: cross-consistency', () => { + const template = readSkillFile(TEMPLATE_MD); + const taxonomy = readSkillFile(TAXONOMY_MD); + + it('every category in template exists in taxonomy', () => { + const categoryLine = template + .split('\n') + .find((l) => l.includes('**Category**')); + expect(categoryLine).toBeTruthy(); + + const categories = categoryLine! + .split('|') + .pop()! + .split('/') + .map((c) => c.trim().toLowerCase()) + .filter(Boolean); + + for (const cat of categories) { + expect( + taxonomy.toLowerCase(), + `Category "${cat}" from template not found in taxonomy` + ).toMatch(new RegExp(`###\\s+.*${cat}`)); + } + }); + + it('every severity in template exists in taxonomy', () => { + for (const sev of ['critical', 'high', 'medium', 'low']) { + expect(taxonomy.toLowerCase()).toContain(`**${sev}**`); + } + }); +}); diff --git a/test/e2e/fixtures/buggy-app.html b/test/e2e/fixtures/buggy-app.html new file mode 100644 index 0000000..6c45d83 --- /dev/null +++ b/test/e2e/fixtures/buggy-app.html @@ -0,0 +1,159 @@ + + + + + + Buggy App - Dogfood Test Fixture + + + + +
+

Buggy App

+ +
+ +
+ +

Welocme to the Dashboard

+ + +
+

Quick Actions

+

Perform common tasks from here.

+
+ + + +
+
+ + +
+

System Status

+ +
+ The system is currently operating normally. All services are online and responding within expected latency thresholds. Last health check completed at 14:32 UTC. +
+ +
+
+ All systems operational +
+
+ + +
+

Recent Activity

+ +

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.

+
+ + +
+

Contact Support

+
+ + + + + + +
+
+ + +
+

Notifications

+ +
+
+
+
+ +
+ © 2025 Buggy App Inc. All rights reserved. +
+ + + + + diff --git a/vitest.config.ts b/vitest.config.ts index 31a6325..f7e8d9c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { globals: true, - include: ['src/**/*.test.ts', 'test/**/*.test.ts'], + include: ['src/**/*.test.ts', 'test/**/*.test.ts', 'test/**/*.eval.ts'], testTimeout: 30000, }, });