Merge remote-tracking branch 'upstream/main'

# Conflicts:
#	README.md
#	cli/src/connection.rs
#	cli/src/flags.rs
#	cli/src/main.rs
#	src/browser.ts
#	src/protocol.ts
This commit is contained in:
leeguooooo
2026-02-25 10:06:20 +09:00
22 changed files with 1467 additions and 59 deletions
+261
View File
@@ -0,0 +1,261 @@
import { describe, it, expect, beforeAll } from 'vitest';
import { query } from '@anthropic-ai/claude-agent-sdk';
import type { SDKMessage, SDKResultMessage } from '@anthropic-ai/claude-agent-sdk';
import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, readdirSync, rmSync } from 'node:fs';
import path from 'node:path';
const AI_GATEWAY_URL =
process.env.ANTHROPIC_BASE_URL || 'https://ai-gateway.vercel.sh';
const API_KEY = process.env.AI_GATEWAY_API_KEY;
const MODEL = process.env.DOGFOOD_MODEL || 'anthropic/claude-haiku-4.5';
const CUSTOM_URL = process.env.DOGFOOD_URL;
const FIXTURE_PATH = path.resolve('test/e2e/fixtures/buggy-app.html');
const SKILL_PATH = path.resolve('skills/dogfood/SKILL.md');
const TARGET_URL = CUSTOM_URL || `file://${FIXTURE_PATH}`;
const IS_FIXTURE = !CUSTOM_URL;
const OUTPUT_DIR = path.resolve('test/e2e/.dogfood-output');
const EVAL_TIMEOUT = 10 * 60 * 1000;
async function runDogfood(outputDir: string): Promise<{
result: SDKResultMessage | null;
messages: SDKMessage[];
toolsUsed: Set<string>;
}> {
const instruction = [
`Read the dogfood skill at ${SKILL_PATH} and follow its workflow.`,
`Dogfood ${TARGET_URL}`,
`Output directory: ${outputDir}`,
].join(' ');
const messages: SDKMessage[] = [];
const toolsUsed = new Set<string>();
let result: SDKResultMessage | null = null;
const conversation = query({
prompt: instruction,
options: {
model: MODEL,
cwd: process.cwd(),
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
permissionMode: 'bypassPermissions',
allowDangerouslySkipPermissions: true,
maxTurns: 80,
maxBudgetUsd: 2,
settingSources: ['project'],
persistSession: false,
env: {
...process.env,
ANTHROPIC_BASE_URL: AI_GATEWAY_URL,
ANTHROPIC_API_KEY: API_KEY,
},
},
});
const verbose = process.env.DOGFOOD_VERBOSE !== '0';
const log = verbose ? (msg: string) => process.stderr.write(` [dogfood] ${msg}\n`) : () => {};
const chatLogPath = path.join(outputDir, 'chat-log.jsonl');
writeFileSync(chatLogPath, '');
function appendToLog(entry: Record<string, unknown>) {
appendFileSync(chatLogPath, JSON.stringify(entry) + '\n');
}
for await (const message of conversation) {
messages.push(message);
if (message.type === 'system' && message.subtype === 'init') {
log(`session started (model: ${message.model})`);
appendToLog({ type: 'system', subtype: 'init', model: message.model });
}
if (message.type === 'assistant' && message.message?.content) {
const logParts: Record<string, unknown>[] = [];
for (const block of message.message.content) {
if ('type' in block && block.type === 'tool_use') {
toolsUsed.add(block.name);
const input = block.input as Record<string, unknown>;
let preview: string;
if (block.name === 'Bash') {
const cmd = String(input.command ?? '');
const firstLine = cmd.split('\n').find(l => l.trim() && !l.trim().startsWith('#')) ?? cmd.split('\n')[0];
preview = firstLine.trim().slice(0, 200);
} else if (block.name === 'Write') {
preview = String(input.file_path ?? input.path ?? '');
} else if (block.name === 'Read') {
preview = String(input.file_path ?? input.path ?? '');
} else if (block.name === 'Edit') {
preview = String(input.file_path ?? input.path ?? '');
} else {
preview = JSON.stringify(input).slice(0, 120);
}
log(`${block.name}: ${preview}`);
logParts.push({ tool: block.name, input: block.input });
}
if ('type' in block && block.type === 'text' && block.text) {
const line = block.text.split('\n')[0].slice(0, 120);
if (line.trim()) log(line);
logParts.push({ text: block.text });
}
}
appendToLog({ type: 'assistant', content: logParts });
}
if (message.type === 'result') {
result = message;
const cost = `$${message.total_cost_usd.toFixed(4)}`;
const usage = message.usage;
const cacheRead = usage.cache_read_input_tokens ?? 0;
const cacheCreate = usage.cache_creation_input_tokens ?? 0;
const inputTokens = usage.input_tokens ?? 0;
const cacheInfo = cacheRead > 0
? ` | cache: ${cacheRead} read, ${cacheCreate} created, ${inputTokens} uncached`
: '';
if (message.subtype === 'success') {
log(`done (${message.num_turns} turns, ${cost}${cacheInfo})`);
} else {
log(`stopped: ${message.subtype} (${message.num_turns} turns, ${cost}${cacheInfo})`);
}
appendToLog({ type: 'result', subtype: message.subtype, num_turns: message.num_turns, cost: message.total_cost_usd });
}
}
log(`chat log: ${chatLogPath}`);
return { result, messages, toolsUsed };
}
function findFiles(dir: string, ext: string): string[] {
if (!existsSync(dir)) return [];
return readdirSync(dir, { recursive: true })
.map(String)
.filter((f) => f.endsWith(ext));
}
describe.skipIf(!API_KEY)('Dogfood e2e eval (Agent SDK)', () => {
const outputDir = OUTPUT_DIR;
let evalResult: Awaited<ReturnType<typeof runDogfood>>;
beforeAll(async () => {
if (existsSync(outputDir)) {
rmSync(outputDir, { recursive: true, force: true });
}
mkdirSync(outputDir, { recursive: true });
evalResult = await runDogfood(outputDir);
}, EVAL_TIMEOUT);
it('completes without hard failure', () => {
expect(evalResult.result, 'No result message received').toBeTruthy();
const acceptable = ['success', 'error_max_turns', 'error_max_budget_usd'];
expect(
acceptable,
`Agent failed unexpectedly: ${evalResult.result!.subtype}`
).toContain(evalResult.result!.subtype);
});
it('used agent-browser via Bash tool', () => {
expect(
evalResult.toolsUsed.has('Bash'),
'Agent never used Bash (needed for agent-browser commands)'
).toBe(true);
});
it('produced a report file', () => {
const reportPath = path.join(outputDir, 'report.md');
expect(existsSync(reportPath), 'report.md not found in output dir').toBe(
true
);
});
it('found a minimum number of issues', () => {
const reportPath = path.join(outputDir, 'report.md');
if (!existsSync(reportPath)) return;
const report = readFileSync(reportPath, 'utf-8');
const issueBlocks = report.match(/###\s+ISSUE-\d+/g) || [];
if (IS_FIXTURE) {
expect(
issueBlocks.length,
`Expected >=2 issues from fixture, found ${issueBlocks.length}`
).toBeGreaterThanOrEqual(2);
} else {
expect(issueBlocks.length).toBeGreaterThanOrEqual(1);
}
});
it('each issue has required fields and repro evidence', () => {
const reportPath = path.join(outputDir, 'report.md');
if (!existsSync(reportPath)) return;
const report = readFileSync(reportPath, 'utf-8');
const issueSections = report.split(/(?=###\s+ISSUE-\d+)/).slice(1);
for (const section of issueSections) {
const issueId = section.match(/ISSUE-\d+/)?.[0] ?? 'unknown';
expect(section, `${issueId}: missing Severity`).toMatch(
/\*\*Severity\*\*/i
);
const sevMatch = section.match(
/\*\*Severity\*\*\s*\|?\s*(critical|high|medium|low)/i
);
expect(sevMatch, `${issueId}: invalid severity value`).toBeTruthy();
expect(section, `${issueId}: missing Category`).toMatch(
/\*\*Category\*\*/i
);
expect(section, `${issueId}: missing URL`).toMatch(/\*\*URL\*\*/i);
expect(section, `${issueId}: missing Repro Video field`).toMatch(
/\*\*Repro Video\*\*/i
);
const hasScreenshot = /!\[.*?\]\(.*?\)/.test(section);
const hasReproSteps = /\*\*Repro Steps\*\*/i.test(section);
expect(
hasScreenshot || hasReproSteps,
`${issueId}: needs either screenshot refs or repro steps`
).toBe(true);
}
});
it('has a summary table with non-zero total', () => {
const reportPath = path.join(outputDir, 'report.md');
if (!existsSync(reportPath)) return;
const report = readFileSync(reportPath, 'utf-8');
expect(report, 'Missing Summary section').toContain('## Summary');
const totalMatch = report.match(/\*\*Total\*\*\s*\|?\s*\*\*(\d+)\*\*/);
expect(totalMatch, 'Summary Total not found').toBeTruthy();
if (totalMatch) {
const total = parseInt(totalMatch[1], 10);
expect(total, 'Summary Total should be > 0').toBeGreaterThan(0);
}
});
it('produced screenshot files', () => {
const screenshotsDir = path.join(outputDir, 'screenshots');
const screenshots = findFiles(screenshotsDir, '.png');
expect(
screenshots.length,
'No screenshot files found in output'
).toBeGreaterThan(0);
});
it('produced video files for interactive issues', () => {
const reportPath = path.join(outputDir, 'report.md');
if (!existsSync(reportPath)) return;
const report = readFileSync(reportPath, 'utf-8');
const hasVideoRefs = /videos\/issue-\d+/.test(report);
if (!hasVideoRefs) return;
const videosDir = path.join(outputDir, 'videos');
const videos = findFiles(videosDir, '.webm');
expect(
videos.length,
'Report references videos but none were found'
).toBeGreaterThan(0);
});
});
+210
View File
@@ -0,0 +1,210 @@
import { describe, it, expect } from 'vitest';
import { readFileSync, existsSync } from 'node:fs';
import path from 'node:path';
const SKILL_DIR = path.resolve('skills/dogfood');
const SKILL_MD = path.join(SKILL_DIR, 'SKILL.md');
const TAXONOMY_MD = path.join(SKILL_DIR, 'references', 'issue-taxonomy.md');
const TEMPLATE_MD = path.join(SKILL_DIR, 'templates', 'dogfood-report-template.md');
function readSkillFile(filePath: string): string {
return readFileSync(filePath, 'utf-8');
}
function parseFrontmatter(content: string): Record<string, string> {
const match = content.match(/^---\n([\s\S]*?)\n---/);
if (!match) return {};
const fields: Record<string, string> = {};
for (const line of match[1].split('\n')) {
const colonIdx = line.indexOf(':');
if (colonIdx > 0) {
fields[line.slice(0, colonIdx).trim()] = line.slice(colonIdx + 1).trim();
}
}
return fields;
}
describe('Dogfood skill: file structure', () => {
it('SKILL.md exists', () => {
expect(existsSync(SKILL_MD)).toBe(true);
});
it('references/issue-taxonomy.md exists', () => {
expect(existsSync(TAXONOMY_MD)).toBe(true);
});
it('templates/dogfood-report-template.md exists', () => {
expect(existsSync(TEMPLATE_MD)).toBe(true);
});
});
describe('Dogfood skill: SKILL.md frontmatter', () => {
const content = readSkillFile(SKILL_MD);
const frontmatter = parseFrontmatter(content);
it('has name field', () => {
expect(frontmatter.name).toBe('dogfood');
});
it('has description field', () => {
expect(frontmatter.description).toBeTruthy();
expect(frontmatter.description!.length).toBeGreaterThan(50);
});
it('has allowed-tools field', () => {
expect(frontmatter['allowed-tools']).toBeTruthy();
expect(frontmatter['allowed-tools']).toContain('agent-browser');
});
});
describe('Dogfood skill: SKILL.md body references', () => {
const content = readSkillFile(SKILL_MD);
it('references issue-taxonomy.md', () => {
expect(content).toContain('references/issue-taxonomy.md');
});
it('references dogfood-report-template.md', () => {
expect(content).toContain('templates/dogfood-report-template.md');
});
it('referenced files exist on disk', () => {
const refPattern = /\[.*?\]\((references\/.*?\.md|templates\/.*?\.md)\)/g;
const refs = [...content.matchAll(refPattern)].map((m) => m[1]);
expect(refs.length).toBeGreaterThan(0);
for (const ref of refs) {
const fullPath = path.join(SKILL_DIR, ref);
expect(existsSync(fullPath), `Missing: ${ref}`).toBe(true);
}
});
});
describe('Dogfood skill: report template', () => {
const template = readSkillFile(TEMPLATE_MD);
it('has ISSUE- prefix in issue blocks', () => {
expect(template).toContain('ISSUE-');
});
it('has Severity field', () => {
expect(template).toContain('**Severity**');
});
it('has Category field', () => {
expect(template).toContain('**Category**');
});
it('has URL field', () => {
expect(template).toContain('**URL**');
});
it('has Repro Video field', () => {
expect(template).toContain('**Repro Video**');
});
it('has Repro Steps section', () => {
expect(template).toContain('**Repro Steps**');
});
it('has screenshot image references in repro steps', () => {
expect(template).toMatch(/!\[.*?\]\(screenshots\//);
});
it('lists all valid severity values', () => {
expect(template).toMatch(/critical\s*\/\s*high\s*\/\s*medium\s*\/\s*low/);
});
it('lists all valid category values', () => {
const categoryLine = template
.split('\n')
.find((l) => l.includes('**Category**'));
expect(categoryLine).toBeTruthy();
for (const cat of [
'visual',
'functional',
'ux',
'content',
'performance',
'console',
'accessibility',
]) {
expect(categoryLine!.toLowerCase()).toContain(cat);
}
});
it('has Summary table with severity counts', () => {
expect(template).toContain('## Summary');
for (const sev of ['Critical', 'High', 'Medium', 'Low', 'Total']) {
expect(template).toContain(sev);
}
});
});
describe('Dogfood skill: issue taxonomy', () => {
const taxonomy = readSkillFile(TAXONOMY_MD);
it('has severity level definitions', () => {
expect(taxonomy).toContain('## Severity Levels');
for (const sev of ['critical', 'high', 'medium', 'low']) {
expect(taxonomy.toLowerCase()).toContain(`**${sev}**`);
}
});
it('has all 7 category sections', () => {
const expectedCategories = [
'Visual',
'Functional',
'UX',
'Content',
'Performance',
'Console',
'Accessibility',
];
for (const cat of expectedCategories) {
expect(taxonomy).toMatch(new RegExp(`###\\s+.*${cat}`, 'i'));
}
});
it('has exploration checklist', () => {
expect(taxonomy).toContain('## Exploration Checklist');
});
it('checklist has numbered items', () => {
const checklistSection = taxonomy.split('## Exploration Checklist')[1];
expect(checklistSection).toBeTruthy();
const numberedItems = checklistSection!.match(/^\d+\./gm);
expect(numberedItems!.length).toBeGreaterThanOrEqual(5);
});
});
describe('Dogfood skill: cross-consistency', () => {
const template = readSkillFile(TEMPLATE_MD);
const taxonomy = readSkillFile(TAXONOMY_MD);
it('every category in template exists in taxonomy', () => {
const categoryLine = template
.split('\n')
.find((l) => l.includes('**Category**'));
expect(categoryLine).toBeTruthy();
const categories = categoryLine!
.split('|')
.pop()!
.split('/')
.map((c) => c.trim().toLowerCase())
.filter(Boolean);
for (const cat of categories) {
expect(
taxonomy.toLowerCase(),
`Category "${cat}" from template not found in taxonomy`
).toMatch(new RegExp(`###\\s+.*${cat}`));
}
});
it('every severity in template exists in taxonomy', () => {
for (const sev of ['critical', 'high', 'medium', 'low']) {
expect(taxonomy.toLowerCase()).toContain(`**${sev}**`);
}
});
});
+159
View File
@@ -0,0 +1,159 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Buggy App - Dogfood Test Fixture</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: system-ui, sans-serif; color: #333; background: #f9f9f9; }
header { background: #1a1a2e; color: #fff; padding: 16px 24px; display: flex; justify-content: space-between; align-items: center; }
header h1 { font-size: 20px; }
nav { display: flex; gap: 16px; }
nav a { color: #ccc; text-decoration: none; }
nav a:hover { color: #fff; }
main { max-width: 960px; margin: 0 auto; padding: 32px 24px; }
.card { background: #fff; border: 1px solid #e0e0e0; border-radius: 8px; padding: 24px; margin-bottom: 24px; }
.card h2 { margin-bottom: 12px; }
.btn { padding: 8px 16px; border: none; border-radius: 4px; cursor: pointer; font-size: 14px; }
.btn-primary { background: #3b82f6; color: #fff; }
.btn-danger { background: #ef4444; color: #fff; }
input, textarea { padding: 8px 12px; border: 1px solid #d0d0d0; border-radius: 4px; font-size: 14px; width: 100%; margin-bottom: 12px; }
label { display: block; margin-bottom: 4px; font-weight: 500; }
footer { text-align: center; padding: 24px; color: #999; font-size: 12px; }
/* BUG: Visual - clipped text via overflow: hidden on a short container */
.clipped-container {
overflow: hidden;
height: 20px;
border: 1px solid #e0e0e0;
padding: 4px 8px;
margin-top: 8px;
}
/* BUG: Visual - misaligned element */
.misaligned {
display: flex;
align-items: flex-start; /* should be center */
gap: 12px;
padding: 12px;
background: #f0f4ff;
border-radius: 8px;
margin-top: 12px;
}
.misaligned .icon {
width: 40px;
height: 40px;
background: #3b82f6;
border-radius: 50%;
flex-shrink: 0;
margin-top: 14px; /* intentionally off */
}
.misaligned .label {
font-size: 16px;
line-height: 40px;
}
</style>
</head>
<body>
<header>
<h1>Buggy App</h1>
<nav>
<a href="#dashboard">Dashboard</a>
<a href="#settings">Settings</a>
<!-- BUG: Functional - broken link to nonexistent page -->
<a href="#/this-page-does-not-exist">Reports</a>
<a href="#help">Help</a>
</nav>
</header>
<main>
<!-- BUG: Content - typo "Welocme" -->
<h2 style="margin-bottom: 24px;">Welocme to the Dashboard</h2>
<!-- Card 1: Functional bug - button throws JS error -->
<div class="card">
<h2>Quick Actions</h2>
<p>Perform common tasks from here.</p>
<div style="margin-top: 12px; display: flex; gap: 8px;">
<!-- BUG: Functional - button throws JS error on click -->
<button class="btn btn-primary" onclick="processAction()">Run Analysis</button>
<button class="btn btn-danger" onclick="deleteAllData()">Delete All Data</button>
</div>
</div>
<!-- Card 2: Visual bugs - clipped text and misaligned element -->
<div class="card">
<h2>System Status</h2>
<!-- BUG: Visual - text is clipped because container is too short -->
<div class="clipped-container">
The system is currently operating normally. All services are online and responding within expected latency thresholds. Last health check completed at 14:32 UTC.
</div>
<!-- BUG: Visual - icon and label are misaligned -->
<div class="misaligned">
<div class="icon"></div>
<span class="label">All systems operational</span>
</div>
</div>
<!-- Card 3: Content bug - placeholder text -->
<div class="card">
<h2>Recent Activity</h2>
<!-- BUG: Content - lorem ipsum placeholder left in -->
<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.</p>
</div>
<!-- Card 4: UX bug - form with no feedback on submit -->
<div class="card">
<h2>Contact Support</h2>
<form id="support-form">
<label for="subject">Subject</label>
<input type="text" id="subject" placeholder="Enter subject">
<label for="message">Message</label>
<textarea id="message" rows="3" placeholder="Describe your issue"></textarea>
<!-- BUG: UX - submit does nothing, no feedback -->
<button type="submit" class="btn btn-primary">Send Message</button>
</form>
</div>
<!-- Card 5: UX bug - empty state with no message -->
<div class="card">
<h2>Notifications</h2>
<!-- BUG: UX - empty container with no empty state message -->
<div id="notifications-list" style="min-height: 60px;">
</div>
</div>
</main>
<footer>
&copy; 2025 Buggy App Inc. All rights reserved.
</footer>
<script>
// BUG: Console - error on page load
console.error("Failed to initialize analytics: endpoint not configured");
// BUG: Console - failed fetch on page load
fetch("https://api.nonexistent-endpoint.invalid/v1/health")
.catch(function() {});
// BUG: Functional - function referenced by button is broken
function processAction() {
// Throws because undefinedService is not defined
undefinedService.runAnalysis();
}
// No confirmation for destructive action
function deleteAllData() {
alert("All data deleted!");
}
// Form submit does nothing
document.getElementById("support-form").addEventListener("submit", function(e) {
e.preventDefault();
});
</script>
</body>
</html>