feat(input): press --hold <ms> for precise timed key-holds + document timed-driving pattern
Dogfooding by driving a canvas game surfaced that per-action shell round-trips (keydown; sleep; keyup) are the slowest, lowest-fidelity way to drive anything timed — each is a process spawn + relay round-trip with ~250ms jitter, so a '0.8s hold' is anything but. - 'press <key> --hold <ms>': keyDown, wait, keyUp all inside the daemon, so the hold duration is precise and it's one round-trip. For games (hold-to-move/ charge) and any press-and-hold. - Documented the real driving pattern in the core skill + --help: script a timed sequence in ONE round-trip with 'batch "press d --hold 900" "press j" "wait 200"' (batch sends each step to the running daemon; --hold/wait block in-daemon), and prefer reading engine state via main-world 'eval' over guessing from pixels. Parser test covers plain/held/missing-duration. Builds on the keydown/keyup full descriptor fix.
This commit is contained in:
+35
-3
@@ -583,11 +583,27 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
|
|||||||
|
|
||||||
// === Keyboard ===
|
// === Keyboard ===
|
||||||
"press" | "key" => {
|
"press" | "key" => {
|
||||||
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let key = rest.iter().find(|a| !a.starts_with("--")).ok_or_else(|| {
|
||||||
|
ParseError::MissingArguments {
|
||||||
context: "press".to_string(),
|
context: "press".to_string(),
|
||||||
usage: "press <key>",
|
usage: "press <key> [--hold <ms>]",
|
||||||
|
}
|
||||||
})?;
|
})?;
|
||||||
Ok(json!({ "id": id, "action": "press", "key": key }))
|
let mut c = json!({ "id": id, "action": "press", "key": key });
|
||||||
|
// `--hold <ms>`: hold the key down for <ms> then release, timed inside
|
||||||
|
// the daemon (one round-trip, no shell-sleep jitter) — for games and
|
||||||
|
// hold-to-charge where keydown+sleep+keyup over 3 round-trips is too
|
||||||
|
// imprecise.
|
||||||
|
if let Some(i) = rest.iter().position(|a| *a == "--hold") {
|
||||||
|
let ms = rest.get(i + 1).and_then(|s| s.parse::<u64>().ok()).ok_or(
|
||||||
|
ParseError::MissingArguments {
|
||||||
|
context: "press --hold".to_string(),
|
||||||
|
usage: "press <key> --hold <ms>",
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
c["hold"] = json!(ms);
|
||||||
|
}
|
||||||
|
Ok(c)
|
||||||
}
|
}
|
||||||
"keydown" => {
|
"keydown" => {
|
||||||
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
let key = rest.first().ok_or_else(|| ParseError::MissingArguments {
|
||||||
@@ -3581,6 +3597,22 @@ mod tests {
|
|||||||
assert_eq!(cmd["url"], "https://example.com");
|
assert_eq!(cmd["url"], "https://example.com");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_press_plain_and_hold() {
|
||||||
|
let cmd = parse_command(&args("press d"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(cmd["action"], "press");
|
||||||
|
assert_eq!(cmd["key"], "d");
|
||||||
|
assert!(cmd.get("hold").is_none());
|
||||||
|
|
||||||
|
let held = parse_command(&args("press d --hold 800"), &default_flags()).unwrap();
|
||||||
|
assert_eq!(held["key"], "d");
|
||||||
|
assert_eq!(held["hold"], 800);
|
||||||
|
|
||||||
|
// Missing/invalid duration is an error, not a silent no-hold.
|
||||||
|
assert!(parse_command(&args("press d --hold"), &default_flags()).is_err());
|
||||||
|
assert!(parse_command(&args("press d --hold abc"), &default_flags()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_navigate_reuse_tab_flag() {
|
fn test_navigate_reuse_tab_flag() {
|
||||||
let cmd = parse_command(
|
let cmd = parse_command(
|
||||||
|
|||||||
@@ -3334,6 +3334,16 @@ async fn handle_press(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
|
|||||||
// Parse modifier+key chords like "Control+a", "Shift+Enter", "Control+Shift+a"
|
// Parse modifier+key chords like "Control+a", "Shift+Enter", "Control+Shift+a"
|
||||||
let (actual_key, modifiers) = parse_key_chord(key);
|
let (actual_key, modifiers) = parse_key_chord(key);
|
||||||
|
|
||||||
|
// `--hold <ms>`: keyDown, wait, keyUp — all inside the daemon so the hold
|
||||||
|
// duration is precise (no shell-sleep / round-trip jitter). For games
|
||||||
|
// (hold-to-move/charge) and any press-and-hold interaction.
|
||||||
|
if let Some(ms) = cmd.get("hold").and_then(|v| v.as_u64()) {
|
||||||
|
interaction::dispatch_single_key(&mgr.client, &session_id, &actual_key, "keyDown").await?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(ms)).await;
|
||||||
|
interaction::dispatch_single_key(&mgr.client, &session_id, &actual_key, "keyUp").await?;
|
||||||
|
return Ok(json!({ "pressed": key, "heldMs": ms }));
|
||||||
|
}
|
||||||
|
|
||||||
interaction::press_key_with_modifiers(&mgr.client, &session_id, &actual_key, modifiers).await?;
|
interaction::press_key_with_modifiers(&mgr.client, &session_id, &actual_key, modifiers).await?;
|
||||||
Ok(json!({ "pressed": key }))
|
Ok(json!({ "pressed": key }))
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-1
@@ -3070,7 +3070,9 @@ Core Commands:
|
|||||||
dblclick <sel> Double-click element
|
dblclick <sel> Double-click element
|
||||||
type <sel> <text> Type into element
|
type <sel> <text> Type into element
|
||||||
fill <sel> <text> Clear and fill
|
fill <sel> <text> Clear and fill
|
||||||
press <key> Press key (Enter, Tab, Control+a)
|
press <key> [--hold <ms>] Press key (Enter, Tab, Control+a). --hold keeps it
|
||||||
|
down <ms> then releases — precise (in-daemon), for
|
||||||
|
games/charge: `press d --hold 800`
|
||||||
keydown <key> Hold a key down (no auto-release) — for games/shortcuts
|
keydown <key> Hold a key down (no auto-release) — for games/shortcuts
|
||||||
keyup <key> Release a held key. Pair with keydown to hold-to-move:
|
keyup <key> Release a held key. Pair with keydown to hold-to-move:
|
||||||
`keydown d` … `keyup d`
|
`keydown d` … `keyup d`
|
||||||
|
|||||||
@@ -309,12 +309,26 @@ detects this and prints a one-line hint. Drive them the screenshot way:
|
|||||||
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
chrome-use screenshot /tmp/s.png # SEE the state (your only read path —
|
||||||
# eval/get text return nothing useful)
|
# eval/get text return nothing useful)
|
||||||
chrome-use click 640 360 # interact by viewport coordinate
|
chrome-use click 640 360 # interact by viewport coordinate
|
||||||
chrome-use keydown d; sleep 0.6; chrome-use keyup d # hold-to-move
|
chrome-use press d --hold 800 # hold-to-move, precise (timed in-daemon —
|
||||||
|
# NOT keydown+shell-sleep+keyup, which
|
||||||
|
# adds ~250ms jitter per round-trip)
|
||||||
chrome-use press Space # discrete actions (jump/attack/confirm)
|
chrome-use press Space # discrete actions (jump/attack/confirm)
|
||||||
```
|
```
|
||||||
|
|
||||||
Each command is a ~250ms round-trip, so this is fine for turn-based / canvas
|
**Don't drive frame-by-frame with one CLI call per action** — that's the slowest,
|
||||||
*apps* but too slow to play a real-time 60fps action game frame-by-frame.
|
lowest-fidelity way (each call is a process spawn + round-trip). Script a *timed
|
||||||
|
sequence in a single round-trip* with `batch` (it sends each step to the running
|
||||||
|
daemon; `press --hold` and `wait` block in-daemon, so timing is precise):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chrome-use batch "press d --hold 900" "press j" "press j" "wait 200" "press d --hold 500"
|
||||||
|
```
|
||||||
|
|
||||||
|
Also try reading real state instead of pixels: `eval` runs in the page's main
|
||||||
|
world, so for a framework/engine game you can often reach its globals (e.g. a
|
||||||
|
Phaser/PIXI/Three instance, a store, `window.__GAME__`) and read positions/score
|
||||||
|
directly — far better than guessing from a screenshot. Still, a real-time 60fps
|
||||||
|
action game is not playable frame-perfect over a CLI; expect to script bursts.
|
||||||
|
|
||||||
## Waiting (read this)
|
## Waiting (read this)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user