From ad6e206a90f10f0e9aafaae54a7b549263ce0ac2 Mon Sep 17 00:00:00 2001 From: Provi Date: Mon, 23 Feb 2026 07:44:08 -0800 Subject: [PATCH] feat: add `keyboard` command for raw keyboard input (#521) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `keyboard type` and `keyboard insertText` subcommands that operate on the currently focused element without requiring a selector. Essential for contenteditable editors (Lexical, ProseMirror, CodeMirror, Monaco) where `type ` doesn't trigger the editor's internal event pipeline (beforeinput/DOM mutation). - `keyboard type ` — page.keyboard.type() with real keystrokes - `keyboard insertText ` — page.keyboard.insertText() Note: `keyboard press` intentionally omitted — the existing top-level `press` command already operates on current focus. Co-authored-by: Claude Opus 4.6 --- README.md | 2 + cli/src/commands.rs | 32 +++++++++++++++ cli/src/output.rs | 43 ++++++++++++++++++++ docs/src/app/commands/page.mdx | 2 + skills/agent-browser/SKILL.md | 2 + src/actions.ts | 17 +++++++- src/protocol.ts | 15 ++++++- src/types.ts | 7 +++- test/keyboard.test.ts | 71 ++++++++++++++++++++++++++++++++++ 9 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 test/keyboard.test.ts diff --git a/README.md b/README.md index 9d4618a..28ba46e 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,8 @@ agent-browser focus # Focus element agent-browser type # Type into element agent-browser fill # Clear and fill agent-browser press # Press key (Enter, Tab, Control+a) (alias: key) +agent-browser keyboard type # Type with real keystrokes (no selector, current focus) +agent-browser keyboard inserttext # Insert text without key events (no selector) agent-browser keydown # Hold key down agent-browser keyup # Release key agent-browser hover # Hover element diff --git a/cli/src/commands.rs b/cli/src/commands.rs index 232c989..fb1b257 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -263,6 +263,38 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result { + let sub = rest.first().ok_or_else(|| ParseError::MissingArguments { + context: "keyboard".to_string(), + usage: "keyboard ", + })?; + match *sub { + "type" => { + let text: String = rest[1..].join(" "); + if text.is_empty() { + return Err(ParseError::MissingArguments { + context: "keyboard type".to_string(), + usage: "keyboard type ", + }); + } + Ok(json!({ "id": id, "action": "keyboard", "subaction": "type", "text": text })) + } + "inserttext" | "insertText" => { + let text: String = rest[1..].join(" "); + if text.is_empty() { + return Err(ParseError::MissingArguments { + context: "keyboard inserttext".to_string(), + usage: "keyboard inserttext ", + }); + } + Ok(json!({ "id": id, "action": "keyboard", "subaction": "insertText", "text": text })) + } + _ => Err(ParseError::UnknownSubcommand { + subcommand: sub.to_string(), + valid_options: &["type", "inserttext"], + }), + } + } // === Scroll === "scroll" => { diff --git a/cli/src/output.rs b/cli/src/output.rs index 9e62539..f5bb7c4 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -713,6 +713,11 @@ Global Options: Examples: agent-browser type "#search" "hello" agent-browser type @e2 "additional text" + +See Also: + For typing into contenteditable editors (Lexical, ProseMirror, etc.) + without a selector, use 'keyboard type' instead: + agent-browser keyboard type "# My Heading" "## } "hover" => { @@ -926,6 +931,42 @@ Examples: agent-browser keyup Control "## } + "keyboard" => { + r##" +agent-browser keyboard - Raw keyboard input (no selector needed) + +Usage: agent-browser keyboard + +Sends keyboard input to whatever element currently has focus. +Unlike 'type' which requires a selector, 'keyboard' operates on +the current focus — essential for contenteditable editors like +Lexical, ProseMirror, CodeMirror, and Monaco. + +Subcommands: + type Type text character-by-character with real + key events (keydown, keypress, keyup per char) + inserttext Insert text without key events (like paste) + +Note: For key combos (Enter, Control+a), use the 'press' command +directly — it already operates on the current focus. + +Global Options: + --json Output as JSON + --session Use specific session + +Examples: + agent-browser keyboard type "Hello, World!" + agent-browser keyboard type "# My Heading" + agent-browser keyboard inserttext "pasted content" + +Use Cases: + # Type into a Lexical/ProseMirror contenteditable editor: + agent-browser click "[contenteditable]" + agent-browser keyboard type "# My Heading" + agent-browser press Enter + agent-browser keyboard type "Some paragraph text" +"## + } // === Scroll === "scroll" => { @@ -1958,6 +1999,8 @@ Core Commands: type Type into element fill Clear and fill press Press key (Enter, Tab, Control+a) + keyboard type Type text with real keystrokes (no selector) + keyboard inserttext Insert text without key events hover Hover element focus Focus element check Check checkbox diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx index 6227821..9cc7dc8 100644 --- a/docs/src/app/commands/page.mdx +++ b/docs/src/app/commands/page.mdx @@ -13,6 +13,8 @@ agent-browser dblclick # Double-click agent-browser fill # Clear and fill agent-browser type # Type into element agent-browser press # Press key (Enter, Tab, Control+a) (alias: key) +agent-browser keyboard type # Type at current focus (no selector needed) +agent-browser keyboard inserttext # Insert text without key events agent-browser keydown # Hold key down agent-browser keyup # Release key agent-browser hover # Hover element diff --git a/skills/agent-browser/SKILL.md b/skills/agent-browser/SKILL.md index 834f3e3..e967594 100644 --- a/skills/agent-browser/SKILL.md +++ b/skills/agent-browser/SKILL.md @@ -64,6 +64,8 @@ agent-browser type @e2 "text" # Type without clearing agent-browser select @e1 "option" # Select dropdown option agent-browser check @e1 # Check checkbox agent-browser press Enter # Press key +agent-browser keyboard type "text" # Type at current focus (no selector) +agent-browser keyboard inserttext "text" # Insert without key events agent-browser scroll down 500 # Scroll page # Get information diff --git a/src/actions.ts b/src/actions.ts index 4017940..aa92865 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -1894,8 +1894,21 @@ async function handleKeyboard( browser: BrowserManager ): Promise { const page = browser.getPage(); - await page.keyboard.press(command.keys); - return successResponse(command.id, { pressed: command.keys }); + const sub = command.subaction ?? 'press'; + + switch (sub) { + case 'type': + await page.keyboard.type(command.text ?? '', { delay: command.delay }); + return successResponse(command.id, { typed: true, text: command.text }); + case 'press': + await page.keyboard.press(command.keys ?? ''); + return successResponse(command.id, { pressed: command.keys }); + case 'insertText': + await page.keyboard.insertText(command.text ?? ''); + return successResponse(command.id, { inserted: true, text: command.text }); + default: + return errorResponse(command.id, `Unknown keyboard subaction: ${sub}`); + } } async function handleWheel(command: WheelCommand, browser: BrowserManager): Promise { diff --git a/src/protocol.ts b/src/protocol.ts index d5a7ee0..f9d685f 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -442,7 +442,10 @@ const errorsSchema = baseCommandSchema.extend({ const keyboardSchema = baseCommandSchema.extend({ action: z.literal('keyboard'), - keys: z.string().min(1), + subaction: z.enum(['type', 'press', 'insertText']).optional(), + keys: z.string().min(1).optional(), + text: z.string().min(1).optional(), + delay: z.number().optional(), }); const wheelSchema = baseCommandSchema.extend({ @@ -1058,6 +1061,16 @@ export function parseCommand(input: string): ParseResult { }; } + if (command.action === 'keyboard') { + const sub = command.subaction ?? 'press'; + if ((sub === 'type' || sub === 'insertText') && !command.text) { + return { success: false, error: `keyboard ${sub} requires text`, id }; + } + if (sub === 'press' && !command.keys) { + return { success: false, error: 'keyboard press requires keys', id }; + } + } + return { success: true, command }; } diff --git a/src/types.ts b/src/types.ts index f539705..857f282 100644 --- a/src/types.ts +++ b/src/types.ts @@ -667,10 +667,13 @@ export interface ErrorsCommand extends BaseCommand { clear?: boolean; } -// Keyboard shortcuts +// Raw keyboard input (no selector needed) export interface KeyboardCommand extends BaseCommand { action: 'keyboard'; - keys: string; // e.g., "Control+a", "Shift+Tab" + subaction?: 'type' | 'press' | 'insertText'; // press kept for backward compat + keys?: string; // for legacy press path + text?: string; // for type/insertText + delay?: number; // for type (ms between keystrokes) } // Mouse wheel diff --git a/test/keyboard.test.ts b/test/keyboard.test.ts new file mode 100644 index 0000000..23bab5d --- /dev/null +++ b/test/keyboard.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from 'vitest'; +import { parseCommand } from '../src/protocol.js'; + +describe('keyboard command validation', () => { + it('accepts keyboard type with text', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', subaction: 'type', text: 'hello' }) + ); + expect(result.success).toBe(true); + }); + + it('accepts keyboard insertText with text', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', subaction: 'insertText', text: 'hello' }) + ); + expect(result.success).toBe(true); + }); + + it('accepts keyboard press with keys', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', subaction: 'press', keys: 'Enter' }) + ); + expect(result.success).toBe(true); + }); + + it('accepts legacy keyboard (no subaction) with keys', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', keys: 'Enter' }) + ); + expect(result.success).toBe(true); + }); + + it('rejects keyboard type without text', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', subaction: 'type' }) + ); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain('requires text'); + }); + + it('rejects keyboard insertText without text', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', subaction: 'insertText' }) + ); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain('requires text'); + }); + + it('rejects keyboard press without keys', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', subaction: 'press' }) + ); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain('requires keys'); + }); + + it('rejects legacy keyboard (no subaction) without keys', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard' }) + ); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain('requires keys'); + }); + + it('accepts keyboard type with delay option', () => { + const result = parseCommand( + JSON.stringify({ id: '1', action: 'keyboard', subaction: 'type', text: 'hello', delay: 50 }) + ); + expect(result.success).toBe(true); + }); +});