native (#594)

* Native Rust rewrite of agent-browser daemon Single-binary Rust implementation replacing the Node.js/Playwright daemon with direct CDP (Chrome DevTools Protocol) communication. Includes full command parity, WebDriver/Safari/iOS backend routing, request tracking, frame context management, CDP protocol codegen, and comprehensive tests. * improvements * fix ci * fixes * faster builds
2026-03-03 15:15:57 -06:00
parent 857c0b25df
commit 51f5fa484c
52 changed files with 55160 additions and 278 deletions
@@ -0,0 +1,707 @@
+use serde_json::Value;
+
+use super::cdp::client::CdpClient;
+use super::cdp::types::*;
+use super::element::{resolve_element_center, resolve_element_object_id, RefMap};
+
+pub async fn click(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+    button: &str,
+    click_count: i32,
+) -> Result<(), String> {
+    let (x, y) = resolve_element_center(client, session_id, ref_map, selector_or_ref).await?;
+    dispatch_click(client, session_id, x, y, button, click_count).await
+}
+
+pub async fn dblclick(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    click(client, session_id, ref_map, selector_or_ref, "left", 2).await
+}
+
+pub async fn hover(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let (x, y) = resolve_element_center(client, session_id, ref_map, selector_or_ref).await?;
+    client
+        .send_command_typed::<_, Value>(
+            "Input.dispatchMouseEvent",
+            &DispatchMouseEventParams {
+                event_type: "mouseMoved".to_string(),
+                x,
+                y,
+                button: None,
+                buttons: None,
+                click_count: None,
+                delta_x: None,
+                delta_y: None,
+                modifiers: None,
+            },
+            Some(session_id),
+        )
+        .await?;
+    Ok(())
+}
+
+pub async fn fill(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+    value: &str,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    // Focus the element
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: "function() { this.focus(); }".to_string(),
+                object_id: Some(object_id.clone()),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    // Select all + delete to clear
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: r#"function() {
+                    this.select && this.select();
+                    this.value = '';
+                    this.dispatchEvent(new Event('input', { bubbles: true }));
+                }"#
+                .to_string(),
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    // Insert text
+    client
+        .send_command_typed::<_, Value>(
+            "Input.insertText",
+            &InsertTextParams {
+                text: value.to_string(),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn type_text(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+    text: &str,
+    clear: bool,
+    delay_ms: Option<u64>,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    // Focus
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: "function() { this.focus(); }".to_string(),
+                object_id: Some(object_id.clone()),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    if clear {
+        client
+            .send_command_typed::<_, Value>(
+                "Runtime.callFunctionOn",
+                &CallFunctionOnParams {
+                    function_declaration: r#"function() {
+                        this.select && this.select();
+                        this.value = '';
+                        this.dispatchEvent(new Event('input', { bubbles: true }));
+                    }"#
+                    .to_string(),
+                    object_id: Some(object_id),
+                    arguments: None,
+                    return_by_value: Some(true),
+                    await_promise: Some(false),
+                },
+                Some(session_id),
+            )
+            .await?;
+    }
+
+    let delay = delay_ms.unwrap_or(0);
+
+    for ch in text.chars() {
+        let text_str = ch.to_string();
+        let (key, code, key_code) = char_to_key_info(ch);
+
+        client
+            .send_command_typed::<_, Value>(
+                "Input.dispatchKeyEvent",
+                &DispatchKeyEventParams {
+                    event_type: "keyDown".to_string(),
+                    key: Some(key.clone()),
+                    code: Some(code.clone()),
+                    text: Some(text_str.clone()),
+                    unmodified_text: Some(text_str.clone()),
+                    windows_virtual_key_code: Some(key_code),
+                    native_virtual_key_code: Some(key_code),
+                    modifiers: None,
+                },
+                Some(session_id),
+            )
+            .await?;
+
+        client
+            .send_command_typed::<_, Value>(
+                "Input.dispatchKeyEvent",
+                &DispatchKeyEventParams {
+                    event_type: "keyUp".to_string(),
+                    key: Some(key),
+                    code: Some(code),
+                    text: None,
+                    unmodified_text: None,
+                    windows_virtual_key_code: Some(key_code),
+                    native_virtual_key_code: Some(key_code),
+                    modifiers: None,
+                },
+                Some(session_id),
+            )
+            .await?;
+
+        if delay > 0 {
+            tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;
+        }
+    }
+
+    Ok(())
+}
+
+pub async fn press_key(client: &CdpClient, session_id: &str, key: &str) -> Result<(), String> {
+    let (key_name, code, key_code) = named_key_info(key);
+
+    client
+        .send_command_typed::<_, Value>(
+            "Input.dispatchKeyEvent",
+            &DispatchKeyEventParams {
+                event_type: "keyDown".to_string(),
+                key: Some(key_name.clone()),
+                code: Some(code.clone()),
+                text: None,
+                unmodified_text: None,
+                windows_virtual_key_code: Some(key_code),
+                native_virtual_key_code: Some(key_code),
+                modifiers: None,
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    client
+        .send_command_typed::<_, Value>(
+            "Input.dispatchKeyEvent",
+            &DispatchKeyEventParams {
+                event_type: "keyUp".to_string(),
+                key: Some(key_name),
+                code: Some(code),
+                text: None,
+                unmodified_text: None,
+                windows_virtual_key_code: Some(key_code),
+                native_virtual_key_code: Some(key_code),
+                modifiers: None,
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn scroll(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: Option<&str>,
+    delta_x: f64,
+    delta_y: f64,
+) -> Result<(), String> {
+    if let Some(sel) = selector_or_ref {
+        let object_id = resolve_element_object_id(client, session_id, ref_map, sel).await?;
+        let js = "function(dx, dy) { this.scrollBy(dx, dy); }".to_string();
+        client
+            .send_command_typed::<_, Value>(
+                "Runtime.callFunctionOn",
+                &CallFunctionOnParams {
+                    function_declaration: js,
+                    object_id: Some(object_id),
+                    arguments: Some(vec![
+                        CallArgument {
+                            value: Some(serde_json::json!(delta_x)),
+                            object_id: None,
+                        },
+                        CallArgument {
+                            value: Some(serde_json::json!(delta_y)),
+                            object_id: None,
+                        },
+                    ]),
+                    return_by_value: Some(true),
+                    await_promise: Some(false),
+                },
+                Some(session_id),
+            )
+            .await?;
+    } else {
+        let js = format!("window.scrollBy({}, {})", delta_x, delta_y);
+        client
+            .send_command_typed::<_, Value>(
+                "Runtime.evaluate",
+                &EvaluateParams {
+                    expression: js,
+                    return_by_value: Some(true),
+                    await_promise: Some(false),
+                },
+                Some(session_id),
+            )
+            .await?;
+    }
+    Ok(())
+}
+
+pub async fn select_option(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+    values: &[String],
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    let js = r#"function(vals) {
+            const options = Array.from(this.options);
+            for (const opt of options) {
+                opt.selected = vals.includes(opt.value) || vals.includes(opt.textContent.trim());
+            }
+            this.dispatchEvent(new Event('change', { bubbles: true }));
+        }"#
+    .to_string();
+
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: js,
+                object_id: Some(object_id),
+                arguments: Some(vec![CallArgument {
+                    value: Some(serde_json::json!(values)),
+                    object_id: None,
+                }]),
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn check(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let is_checked =
+        super::element::is_element_checked(client, session_id, ref_map, selector_or_ref).await?;
+    if !is_checked {
+        click(client, session_id, ref_map, selector_or_ref, "left", 1).await?;
+    }
+    Ok(())
+}
+
+pub async fn uncheck(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let is_checked =
+        super::element::is_element_checked(client, session_id, ref_map, selector_or_ref).await?;
+    if is_checked {
+        click(client, session_id, ref_map, selector_or_ref, "left", 1).await?;
+    }
+    Ok(())
+}
+
+pub async fn focus(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: "function() { this.focus(); }".to_string(),
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn clear(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: r#"function() {
+                    this.focus();
+                    this.value = '';
+                    this.dispatchEvent(new Event('input', { bubbles: true }));
+                    this.dispatchEvent(new Event('change', { bubbles: true }));
+                }"#
+                .to_string(),
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn select_all(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: r#"function() {
+                    this.focus();
+                    if (typeof this.select === 'function') {
+                        this.select();
+                    } else {
+                        const range = document.createRange();
+                        range.selectNodeContents(this);
+                        const sel = window.getSelection();
+                        sel.removeAllRanges();
+                        sel.addRange(range);
+                    }
+                }"#
+                .to_string(),
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn scroll_into_view(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration:
+                    "function() { this.scrollIntoView({ block: 'center', inline: 'center' }); }"
+                        .to_string(),
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn dispatch_event(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+    event_type: &str,
+    event_init: Option<&Value>,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    let init_json = event_init
+        .map(|v| serde_json::to_string(v).unwrap_or("{}".to_string()))
+        .unwrap_or_else(|| "{ bubbles: true }".to_string());
+
+    let js = format!(
+        "function() {{ this.dispatchEvent(new Event({}, {})); }}",
+        serde_json::to_string(event_type).unwrap_or_default(),
+        init_json
+    );
+
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: js,
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn highlight(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let object_id = resolve_element_object_id(client, session_id, ref_map, selector_or_ref).await?;
+
+    client
+        .send_command_typed::<_, Value>(
+            "Runtime.callFunctionOn",
+            &CallFunctionOnParams {
+                function_declaration: r#"function() {
+                    this.style.outline = '2px solid red';
+                    this.style.outlineOffset = '2px';
+                    const el = this;
+                    setTimeout(() => {
+                        el.style.outline = '';
+                        el.style.outlineOffset = '';
+                    }, 3000);
+                }"#
+                .to_string(),
+                object_id: Some(object_id),
+                arguments: None,
+                return_by_value: Some(true),
+                await_promise: Some(false),
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub async fn tap_touch(
+    client: &CdpClient,
+    session_id: &str,
+    ref_map: &RefMap,
+    selector_or_ref: &str,
+) -> Result<(), String> {
+    let (x, y) = resolve_element_center(client, session_id, ref_map, selector_or_ref).await?;
+
+    client
+        .send_command(
+            "Input.dispatchTouchEvent",
+            Some(serde_json::json!({
+                "type": "touchStart",
+                "touchPoints": [{ "x": x, "y": y }],
+            })),
+            Some(session_id),
+        )
+        .await?;
+
+    client
+        .send_command(
+            "Input.dispatchTouchEvent",
+            Some(serde_json::json!({
+                "type": "touchEnd",
+                "touchPoints": [],
+            })),
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+async fn dispatch_click(
+    client: &CdpClient,
+    session_id: &str,
+    x: f64,
+    y: f64,
+    button: &str,
+    click_count: i32,
+) -> Result<(), String> {
+    // Move
+    client
+        .send_command_typed::<_, Value>(
+            "Input.dispatchMouseEvent",
+            &DispatchMouseEventParams {
+                event_type: "mouseMoved".to_string(),
+                x,
+                y,
+                button: None,
+                buttons: None,
+                click_count: None,
+                delta_x: None,
+                delta_y: None,
+                modifiers: None,
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    let button_value = match button {
+        "right" => 2,
+        "middle" => 4,
+        _ => 1,
+    };
+
+    // Press
+    client
+        .send_command_typed::<_, Value>(
+            "Input.dispatchMouseEvent",
+            &DispatchMouseEventParams {
+                event_type: "mousePressed".to_string(),
+                x,
+                y,
+                button: Some(button.to_string()),
+                buttons: Some(button_value),
+                click_count: Some(click_count),
+                delta_x: None,
+                delta_y: None,
+                modifiers: None,
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    // Release
+    client
+        .send_command_typed::<_, Value>(
+            "Input.dispatchMouseEvent",
+            &DispatchMouseEventParams {
+                event_type: "mouseReleased".to_string(),
+                x,
+                y,
+                button: Some(button.to_string()),
+                buttons: Some(0),
+                click_count: Some(click_count),
+                delta_x: None,
+                delta_y: None,
+                modifiers: None,
+            },
+            Some(session_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
+fn char_to_key_info(ch: char) -> (String, String, i32) {
+    match ch {
+        '\n' | '\r' => ("Enter".to_string(), "Enter".to_string(), 13),
+        '\t' => ("Tab".to_string(), "Tab".to_string(), 9),
+        ' ' => (" ".to_string(), "Space".to_string(), 32),
+        _ => {
+            let key = ch.to_string();
+            let code = if ch.is_ascii_alphabetic() {
+                format!("Key{}", ch.to_uppercase())
+            } else if ch.is_ascii_digit() {
+                format!("Digit{}", ch)
+            } else {
+                String::new()
+            };
+            let key_code = ch as i32;
+            (key, code, key_code)
+        }
+    }
+}
+
+fn named_key_info(key: &str) -> (String, String, i32) {
+    match key.to_lowercase().as_str() {
+        "enter" | "return" => ("Enter".to_string(), "Enter".to_string(), 13),
+        "tab" => ("Tab".to_string(), "Tab".to_string(), 9),
+        "escape" | "esc" => ("Escape".to_string(), "Escape".to_string(), 27),
+        "backspace" => ("Backspace".to_string(), "Backspace".to_string(), 8),
+        "delete" => ("Delete".to_string(), "Delete".to_string(), 46),
+        "arrowup" | "up" => ("ArrowUp".to_string(), "ArrowUp".to_string(), 38),
+        "arrowdown" | "down" => ("ArrowDown".to_string(), "ArrowDown".to_string(), 40),
+        "arrowleft" | "left" => ("ArrowLeft".to_string(), "ArrowLeft".to_string(), 37),
+        "arrowright" | "right" => ("ArrowRight".to_string(), "ArrowRight".to_string(), 39),
+        "home" => ("Home".to_string(), "Home".to_string(), 36),
+        "end" => ("End".to_string(), "End".to_string(), 35),
+        "pageup" => ("PageUp".to_string(), "PageUp".to_string(), 33),
+        "pagedown" => ("PageDown".to_string(), "PageDown".to_string(), 34),
+        "space" | " " => (" ".to_string(), "Space".to_string(), 32),
+        _ => {
+            if key.len() == 1 {
+                let ch = key.chars().next().unwrap();
+                char_to_key_info(ch)
+            } else {
+                (key.to_string(), key.to_string(), 0)
+            }
+        }
+    }
+}