diff --git a/package.json b/package.json index 14cfa6f..7ac747e 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,9 @@ "test": "vitest run", "test:watch": "vitest", "test:e2e:dogfood": "vitest run test/e2e/dogfood.eval.ts", + "bench": "pnpm build && tsx test/benchmarks/run.ts", + "bench:node": "pnpm build && tsx test/benchmarks/run.ts --node-only", + "bench:native": "pnpm build && tsx test/benchmarks/run.ts --native-only", "postinstall": "node scripts/postinstall.js", "changeset": "changeset", "ci:version": "changeset version && pnpm run version:sync && pnpm install --no-frozen-lockfile", diff --git a/test/benchmarks/run.ts b/test/benchmarks/run.ts new file mode 100644 index 0000000..27481cd --- /dev/null +++ b/test/benchmarks/run.ts @@ -0,0 +1,607 @@ +import { spawn, ChildProcess } from "child_process"; +import * as net from "net"; +import * as os from "os"; +import * as path from "path"; +import * as fs from "fs"; +import { fileURLToPath } from "url"; +import { scenarios, type BenchmarkCommand, type Scenario } from "./scenarios.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// --------------------------------------------------------------------------- +// Socket / daemon helpers +// --------------------------------------------------------------------------- + +function getSocketDir(): string { + if (process.env.AGENT_BROWSER_SOCKET_DIR) { + return process.env.AGENT_BROWSER_SOCKET_DIR; + } + if (process.env.XDG_RUNTIME_DIR) { + return path.join(process.env.XDG_RUNTIME_DIR, "agent-browser"); + } + const home = os.homedir(); + if (home) { + return path.join(home, ".agent-browser"); + } + return path.join(os.tmpdir(), "agent-browser"); +} + +function getSocketPath(session: string): string { + return path.join(getSocketDir(), `${session}.sock`); +} + +function getProjectRoot(): string { + return path.resolve(__dirname, "../.."); +} + +function getNativeBinaryPath(): string { + const root = getProjectRoot(); + const p = os.platform(); + const a = os.arch(); + + const osKey = + p === "darwin" ? "darwin" : p === "linux" ? "linux" : p === "win32" ? "win32" : null; + const archKey = + a === "x64" || a === "x86_64" ? "x64" : a === "arm64" || a === "aarch64" ? "arm64" : null; + + if (!osKey || !archKey) { + throw new Error(`Unsupported platform: ${p}-${a}`); + } + + const ext = p === "win32" ? ".exe" : ""; + const binName = `agent-browser-${osKey}-${archKey}${ext}`; + + const candidates = [ + path.join(root, "cli/target/release/agent-browser"), + path.join(root, "cli/target/debug/agent-browser"), + path.join(root, "bin", binName), + ]; + + for (const candidate of candidates) { + if (fs.existsSync(candidate)) return candidate; + } + + throw new Error( + `Native binary not found. Tried:\n${candidates.map((c) => " " + c).join("\n")}\n` + + 'Run "pnpm build:native" to build the native binary.', + ); +} + +function sendCommand(session: string, cmd: BenchmarkCommand): Promise> { + return new Promise((resolve, reject) => { + const socketPath = getSocketPath(session); + const client = net.createConnection({ path: socketPath }, () => { + client.write(JSON.stringify(cmd) + "\n"); + }); + + let data = ""; + client.on("data", (chunk) => { + data += chunk.toString(); + const newlineIdx = data.indexOf("\n"); + if (newlineIdx !== -1) { + const line = data.slice(0, newlineIdx); + client.destroy(); + try { + resolve(JSON.parse(line)); + } catch { + reject(new Error(`Invalid JSON response: ${line}`)); + } + } + }); + + client.on("error", (err) => reject(err)); + client.on("timeout", () => { + client.destroy(); + reject(new Error("Socket timeout")); + }); + client.setTimeout(30_000); + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitForSocket(session: string, timeoutMs = 15_000): Promise { + const start = Date.now(); + const socketPath = getSocketPath(session); + while (Date.now() - start < timeoutMs) { + if (fs.existsSync(socketPath)) { + try { + await new Promise((resolve, reject) => { + const c = net.createConnection({ path: socketPath }, () => { + c.destroy(); + resolve(); + }); + c.on("error", reject); + c.setTimeout(1000); + c.on("timeout", () => { + c.destroy(); + reject(new Error("timeout")); + }); + }); + return; + } catch { + // not ready yet + } + } + await sleep(100); + } + throw new Error(`Daemon '${session}' did not become ready within ${timeoutMs}ms`); +} + +interface DaemonHandle { + session: string; + process: ChildProcess; +} + +function spawnNodeDaemon(session: string): DaemonHandle { + const daemonPath = path.join(getProjectRoot(), "dist/daemon.js"); + if (!fs.existsSync(daemonPath)) { + throw new Error(`Node daemon not found at ${daemonPath}. Run "pnpm build" first.`); + } + + const child = spawn("node", [daemonPath], { + env: { + ...process.env, + AGENT_BROWSER_DAEMON: "1", + AGENT_BROWSER_SESSION: session, + }, + stdio: ["ignore", "ignore", "pipe"], + detached: true, + }); + + child.stderr?.on("data", (chunk) => { + const msg = chunk.toString().trim(); + if (msg && process.env.BENCH_DEBUG) { + process.stderr.write(`[node-daemon] ${msg}\n`); + } + }); + + return { session, process: child }; +} + +function spawnNativeDaemon(session: string): DaemonHandle { + const binaryPath = getNativeBinaryPath(); + + const child = spawn(binaryPath, [], { + env: { + ...process.env, + AGENT_BROWSER_DAEMON: "1", + AGENT_BROWSER_SESSION: session, + }, + stdio: ["ignore", "ignore", "pipe"], + detached: true, + }); + + child.stderr?.on("data", (chunk) => { + const msg = chunk.toString().trim(); + if (msg && process.env.BENCH_DEBUG) { + process.stderr.write(`[native-daemon] ${msg}\n`); + } + }); + + return { session, process: child }; +} + +async function closeDaemon(handle: DaemonHandle): Promise { + try { + await sendCommand(handle.session, { id: "close", action: "close" }); + } catch { + // daemon may already be gone + } + await sleep(200); + try { + handle.process.kill("SIGTERM"); + } catch { + // already exited + } +} + +function cleanupSockets(): void { + for (const session of ["bench-node", "bench-native"]) { + const sockPath = getSocketPath(session); + const pidPath = sockPath.replace(/\.sock$/, ".pid"); + try { + fs.unlinkSync(sockPath); + } catch { + /* */ + } + try { + fs.unlinkSync(pidPath); + } catch { + /* */ + } + } +} + +// --------------------------------------------------------------------------- +// Statistics (microsecond precision) +// --------------------------------------------------------------------------- + +interface Stats { + avgUs: number; + minUs: number; + maxUs: number; + p50Us: number; + p95Us: number; +} + +function computeStats(timingsUs: number[]): Stats { + const sorted = [...timingsUs].sort((a, b) => a - b); + const sum = sorted.reduce((a, b) => a + b, 0); + return { + avgUs: Math.round(sum / sorted.length), + minUs: sorted[0], + maxUs: sorted[sorted.length - 1], + p50Us: sorted[Math.floor(sorted.length * 0.5)], + p95Us: sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))], + }; +} + +function formatDuration(us: number): string { + if (us >= 1_000_000) return `${(us / 1_000_000).toFixed(2)}s`; + if (us >= 1_000) return `${(us / 1_000).toFixed(1)}ms`; + return `${us}us`; +} + +// --------------------------------------------------------------------------- +// Benchmark runner +// --------------------------------------------------------------------------- + +async function runCommands(session: string, commands: BenchmarkCommand[]): Promise { + for (const cmd of commands) { + const resp = await sendCommand(session, cmd); + if (!(resp as { success?: boolean }).success) { + throw new Error( + `Command '${cmd.action}' failed on session '${session}': ${JSON.stringify(resp)}`, + ); + } + } +} + +async function timeCommands(session: string, commands: BenchmarkCommand[]): Promise { + const start = process.hrtime.bigint(); + await runCommands(session, commands); + const elapsedNs = process.hrtime.bigint() - start; + return Number(elapsedNs / 1000n); // microseconds +} + +interface ScenarioResult { + name: string; + nodeStats: Stats | null; + nativeStats: Stats | null; +} + +async function runScenario( + scenario: Scenario, + sessions: { node?: string; native?: string }, + iterations: number, + warmup: number, +): Promise { + const result: ScenarioResult = { + name: scenario.name, + nodeStats: null, + nativeStats: null, + }; + + for (const [label, session] of Object.entries(sessions)) { + if (!session) continue; + + if (scenario.setup) { + await runCommands(session, scenario.setup); + } + + for (let i = 0; i < warmup; i++) { + await timeCommands(session, scenario.commands); + } + + const timings: number[] = []; + for (let i = 0; i < iterations; i++) { + timings.push(await timeCommands(session, scenario.commands)); + } + + if (scenario.teardown) { + await runCommands(session, scenario.teardown); + } + + const stats = computeStats(timings); + if (label === "node") result.nodeStats = stats; + else result.nativeStats = stats; + } + + return result; +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +function pad(s: string, len: number): string { + return s.padEnd(len); +} + +function rpad(s: string, len: number): string { + return s.padStart(len); +} + +function formatSpeedup(nodeUs: number, nativeUs: number): string { + if (nativeUs === 0 && nodeUs === 0) return " --"; + if (nativeUs === 0) return " >>>"; + const ratio = nodeUs / nativeUs; + return `${ratio.toFixed(1)}x`; +} + +function printResults(results: ScenarioResult[], iterations: number, warmup: number): void { + const bothPaths = results[0].nodeStats !== null && results[0].nativeStats !== null; + + console.log(""); + const header = bothPaths + ? `agent-browser benchmark: node vs native (${iterations} iterations, ${warmup} warmup)` + : `agent-browser benchmark (${iterations} iterations, ${warmup} warmup)`; + console.log(header); + console.log("=".repeat(header.length)); + console.log(""); + + if (bothPaths) { + const nameW = 20; + const colW = 14; + + console.log( + pad("Scenario", nameW) + + rpad("Node (avg)", colW) + + rpad("Native (avg)", colW) + + rpad("Speedup", 10), + ); + console.log("-".repeat(nameW + colW * 2 + 10)); + + let totalNodeUs = 0; + let totalNativeUs = 0; + let count = 0; + + for (const r of results) { + if (!r.nodeStats || !r.nativeStats) continue; + totalNodeUs += r.nodeStats.avgUs; + totalNativeUs += r.nativeStats.avgUs; + count++; + + console.log( + pad(r.name, nameW) + + rpad(formatDuration(r.nodeStats.avgUs), colW) + + rpad(formatDuration(r.nativeStats.avgUs), colW) + + rpad(formatSpeedup(r.nodeStats.avgUs, r.nativeStats.avgUs), 10), + ); + } + + console.log("-".repeat(nameW + colW * 2 + 10)); + + if (count > 0 && totalNativeUs > 0) { + const overallSpeedup = totalNodeUs / totalNativeUs; + const winner = overallSpeedup >= 1.0 ? "native is faster" : "node is faster"; + console.log(`Overall average speedup: ${overallSpeedup.toFixed(1)}x (${winner})`); + console.log(""); + + const allNativeFaster = results.every( + (r) => !r.nodeStats || !r.nativeStats || r.nodeStats.avgUs >= r.nativeStats.avgUs, + ); + if (allNativeFaster) { + console.log("Result: PASS -- native is faster across all scenarios"); + } else { + const slower = results + .filter((r) => r.nodeStats && r.nativeStats && r.nodeStats.avgUs < r.nativeStats.avgUs) + .map((r) => r.name); + console.log(`Result: WARN -- native is slower in: ${slower.join(", ")}`); + } + } + } else { + const nameW = 20; + const label = results[0].nodeStats ? "Node" : "Native"; + console.log( + pad("Scenario", nameW) + + rpad(`${label} avg`, 10) + + rpad("min", 10) + + rpad("max", 10) + + rpad("p50", 10) + + rpad("p95", 10), + ); + console.log("-".repeat(nameW + 50)); + for (const r of results) { + const s = r.nodeStats ?? r.nativeStats; + if (!s) continue; + console.log( + pad(r.name, nameW) + + rpad(formatDuration(s.avgUs), 10) + + rpad(formatDuration(s.minUs), 10) + + rpad(formatDuration(s.maxUs), 10) + + rpad(formatDuration(s.p50Us), 10) + + rpad(formatDuration(s.p95Us), 10), + ); + } + } + + console.log(""); +} + +function writeJsonResults(results: ScenarioResult[], outputPath: string): void { + const toMs = (us: number) => +(us / 1000).toFixed(2); + const json = results.map((r) => ({ + scenario: r.name, + node: r.nodeStats + ? { + avg_ms: toMs(r.nodeStats.avgUs), + min_ms: toMs(r.nodeStats.minUs), + max_ms: toMs(r.nodeStats.maxUs), + p50_ms: toMs(r.nodeStats.p50Us), + p95_ms: toMs(r.nodeStats.p95Us), + } + : null, + native: r.nativeStats + ? { + avg_ms: toMs(r.nativeStats.avgUs), + min_ms: toMs(r.nativeStats.minUs), + max_ms: toMs(r.nativeStats.maxUs), + p50_ms: toMs(r.nativeStats.p50Us), + p95_ms: toMs(r.nativeStats.p95Us), + } + : null, + speedup: + r.nodeStats && r.nativeStats && r.nativeStats.avgUs > 0 + ? +(r.nodeStats.avgUs / r.nativeStats.avgUs).toFixed(2) + : null, + })); + fs.writeFileSync(outputPath, JSON.stringify(json, null, 2) + "\n"); + console.log(`JSON results written to ${outputPath}`); +} + +// --------------------------------------------------------------------------- +// CLI argument parsing +// --------------------------------------------------------------------------- + +interface CliArgs { + iterations: number; + warmup: number; + nodeOnly: boolean; + nativeOnly: boolean; + json: boolean; +} + +function parseArgs(): CliArgs { + const args = process.argv.slice(2); + const result: CliArgs = { + iterations: 10, + warmup: 3, + nodeOnly: false, + nativeOnly: false, + json: false, + }; + + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case "--iterations": + result.iterations = parseInt(args[++i], 10); + break; + case "--warmup": + result.warmup = parseInt(args[++i], 10); + break; + case "--node-only": + result.nodeOnly = true; + break; + case "--native-only": + result.nativeOnly = true; + break; + case "--json": + result.json = true; + break; + default: + console.error(`Unknown flag: ${args[i]}`); + process.exit(1); + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const args = parseArgs(); + const runNode = !args.nativeOnly; + const runNative = !args.nodeOnly; + + cleanupSockets(); + + console.log("Starting benchmark daemons..."); + + let nodeHandle: DaemonHandle | undefined; + let nativeHandle: DaemonHandle | undefined; + + try { + if (runNode) { + nodeHandle = spawnNodeDaemon("bench-node"); + await waitForSocket("bench-node"); + console.log(" Node daemon ready"); + } + + if (runNative) { + nativeHandle = spawnNativeDaemon("bench-native"); + await waitForSocket("bench-native"); + console.log(" Native daemon ready"); + } + + const sessions: { node?: string; native?: string } = {}; + if (runNode) sessions.node = "bench-node"; + if (runNative) sessions.native = "bench-native"; + + // Launch browsers on both daemons + for (const session of Object.values(sessions)) { + const resp = await sendCommand(session, { + id: "launch", + action: "launch", + headless: true, + }); + if (!(resp as { success?: boolean }).success) { + throw new Error(`Failed to launch browser on ${session}: ${JSON.stringify(resp)}`); + } + } + console.log(" Browsers launched"); + console.log(""); + + // Run all scenarios + const results: ScenarioResult[] = []; + for (const scenario of scenarios) { + process.stdout.write(` Running: ${scenario.name}...`); + const result = await runScenario(scenario, sessions, args.iterations, args.warmup); + results.push(result); + + if (result.nodeStats && result.nativeStats) { + const speedup = formatSpeedup(result.nodeStats.avgUs, result.nativeStats.avgUs); + process.stdout.write( + ` node=${formatDuration(result.nodeStats.avgUs)} native=${formatDuration(result.nativeStats.avgUs)} (${speedup})\n`, + ); + } else { + const s = result.nodeStats ?? result.nativeStats; + process.stdout.write(` avg=${s ? formatDuration(s.avgUs) : "??"}\n`); + } + } + + printResults(results, args.iterations, args.warmup); + + if (args.json) { + writeJsonResults(results, path.join(getProjectRoot(), "test/benchmarks/results.json")); + } + + // Close browsers + for (const session of Object.values(sessions)) { + await sendCommand(session, { id: "close", action: "close" }).catch(() => {}); + } + + await sleep(300); + + // CI gate: exit 1 if native is slower overall (total avg across all scenarios) + if (runNode && runNative) { + let totalNodeUs = 0; + let totalNativeUs = 0; + for (const r of results) { + if (r.nodeStats && r.nativeStats) { + totalNodeUs += r.nodeStats.avgUs; + totalNativeUs += r.nativeStats.avgUs; + } + } + if (totalNativeUs > 0 && totalNodeUs / totalNativeUs < 1.0) { + process.exit(1); + } + } + } finally { + if (nodeHandle) await closeDaemon(nodeHandle); + if (nativeHandle) await closeDaemon(nativeHandle); + cleanupSockets(); + } +} + +main().catch((err) => { + console.error("Benchmark failed:", err.message || err); + process.exit(2); +}); diff --git a/test/benchmarks/scenarios.ts b/test/benchmarks/scenarios.ts new file mode 100644 index 0000000..410dfa8 --- /dev/null +++ b/test/benchmarks/scenarios.ts @@ -0,0 +1,119 @@ +export interface BenchmarkCommand { + id: string; + action: string; + [key: string]: unknown; +} + +export interface Scenario { + name: string; + description: string; + /** Commands to run once before measured iterations (e.g. navigate to a page). */ + setup?: BenchmarkCommand[]; + /** The commands whose total execution time is measured per iteration. */ + commands: BenchmarkCommand[]; + /** Commands to run once after measured iterations (e.g. cleanup). */ + teardown?: BenchmarkCommand[]; +} + +const FORM_HTML = [ + "Bench", + "

Benchmark Page

", + "", + "", + "", + "", + "", + "", + "

Ready

", + "Click me", + "
    ", + ...Array.from({ length: 20 }, (_, i) => `
  • Item ${i + 1}
  • `), + "
", + "", +].join(""); + +const INJECT_FORM: BenchmarkCommand = { + id: "inject", + action: "evaluate", + script: `document.open(); document.write(${JSON.stringify(FORM_HTML)}); document.close(); 'ok'`, +}; + +const SETUP_PAGE: BenchmarkCommand[] = [ + { id: "setup-nav", action: "navigate", url: "about:blank", waitUntil: "domcontentloaded" }, + INJECT_FORM, +]; + +export const scenarios: Scenario[] = [ + { + name: "navigate", + description: "Page navigation (about:blank round-trip)", + commands: [ + { id: "nav", action: "navigate", url: "about:blank", waitUntil: "domcontentloaded" }, + ], + }, + { + name: "snapshot", + description: "DOM snapshot (accessibility tree)", + setup: SETUP_PAGE, + commands: [ + { id: "snap", action: "snapshot" }, + ], + }, + { + name: "screenshot", + description: "Screenshot capture", + setup: SETUP_PAGE, + commands: [ + { id: "ss", action: "screenshot" }, + ], + }, + { + name: "evaluate", + description: "JavaScript evaluation", + setup: SETUP_PAGE, + commands: [ + { id: "eval", action: "evaluate", script: "document.title + ' ' + document.querySelectorAll('li').length" }, + ], + }, + { + name: "click", + description: "Element click interaction", + setup: SETUP_PAGE, + commands: [ + { id: "clk", action: "click", selector: "#link" }, + ], + }, + { + name: "fill", + description: "Form field fill", + setup: SETUP_PAGE, + commands: [ + { id: "fill", action: "fill", selector: "#name", value: "Benchmark User" }, + ], + }, + { + name: "tabs", + description: "Tab new + list + switch", + commands: [ + { id: "tnew", action: "tab_new", url: "about:blank" }, + { id: "tlist", action: "tab_list" }, + { id: "tswitch", action: "tab_switch", index: 0 }, + ], + teardown: [ + { id: "tclose", action: "tab_close", index: 1 }, + ], + }, + { + name: "full-workflow", + description: "Realistic agent workflow: navigate, snapshot, click, fill, evaluate, screenshot", + commands: [ + { id: "w-nav", action: "navigate", url: "about:blank", waitUntil: "domcontentloaded" }, + INJECT_FORM, + { id: "w-snap", action: "snapshot" }, + { id: "w-click", action: "click", selector: "#link" }, + { id: "w-fill", action: "fill", selector: "#name", value: "Agent User" }, + { id: "w-eval", action: "evaluate", script: "document.getElementById('name').value" }, + { id: "w-ss", action: "screenshot" }, + ], + }, +];