oimwiodev/chrome-use
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: support accessibility tree refs in upload command (#1156)

Browse Source 
* fix: support accessibility tree refs in upload command (#1107)

The upload command only accepted CSS selectors while click/fill supported
accessibility tree refs (e.g. e1, @e1, ref=e1). This resolves the API
inconsistency by reusing resolve_element_object_id for all selector types.

* style: apply cargo fmt

---------

Co-authored-by: hyunjinee <leehj0110@kakao.com>
This commit is contained in:
jin.2
2026-04-05 15:38:49 -05:00
committed by GitHub
co-authored by hyunjinee
parent c47756be9b
commit fcb6615f5a
4 changed files with 128 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/native/actions.rs
+2 -1
View File
@@ -4472,7 +4472,8 @@ async fn handle_upload(cmd: &Value, state: &DaemonState) -> Result<Value, String
})
.unwrap_or_default();
mgr.upload_files(selector, &files).await?;
mgr.upload_files(selector, &files, &state.ref_map, &state.iframe_sessions)
.await?;
Ok(json!({ "uploaded": files.len(), "selector": selector }))
}
cli/src/native/browser.rs
+14 -41
View File
@@ -1,5 +1,5 @@
use serde_json::{json, Value};
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -10,6 +10,7 @@ use super::cdp::client::CdpClient;
use super::cdp::discovery::discover_cdp_url;
use super::cdp::lightpanda::{launch_lightpanda, LightpandaLaunchOptions, LightpandaProcess};
use super::cdp::types::*;
use super::element::{resolve_element_object_id, RefMap};
// ---------------------------------------------------------------------------
// Launch validation
@@ -1091,50 +1092,25 @@ impl BrowserManager {
Ok(())
}
pub async fn upload_files(&self, selector: &str, files: &[String]) -> Result<(), String> {
pub async fn upload_files(
&self,
selector: &str,
files: &[String],
ref_map: &RefMap,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let session_id = self.active_session_id()?;
let node_result = self
.client
.send_command(
"DOM.querySelector",
Some(json!({
"nodeId": 1,
"selector": selector,
})),
Some(session_id),
)
.await;
let (object_id, effective_session_id) =
resolve_element_object_id(&self.client, session_id, ref_map, selector, iframe_sessions)
.await?;
// Alternative: resolve via JS
let result: EvaluateResult = self
.client
.send_command_typed(
"Runtime.evaluate",
&EvaluateParams {
expression: format!(
"document.querySelector({})",
serde_json::to_string(selector).unwrap_or_default()
),
return_by_value: Some(false),
await_promise: Some(false),
},
Some(session_id),
)
.await?;
let object_id = result
.result
.object_id
.ok_or("File input element not found")?;
// Get the DOM node from the remote object
let describe: Value = self
.client
.send_command(
"DOM.describeNode",
Some(json!({ "objectId": object_id })),
Some(session_id),
Some(&effective_session_id),
)
.await?;
@@ -1144,9 +1120,6 @@ impl BrowserManager {
.and_then(|v| v.as_i64())
.ok_or("Could not get backendNodeId for file input")?;
// Suppress unused variable warning
let _ = node_result;
self.client
.send_command(
"DOM.setFileInputFiles",
@@ -1154,7 +1127,7 @@ impl BrowserManager {
"files": files,
"backendNodeId": backend_node_id,
})),
Some(session_id),
Some(&effective_session_id),
)
.await?;
cli/src/native/e2e_tests.rs
+94
View File
@@ -34,6 +34,7 @@ fn native_test_fixture_html(name: &str) -> &'static str {
"drag_probe" => include_str!("test_fixtures/drag_probe.html"),
"html5_drag_probe" => include_str!("test_fixtures/html5_drag_probe.html"),
"pointer_capture_probe" => include_str!("test_fixtures/pointer_capture_probe.html"),
"upload_probe" => include_str!("test_fixtures/upload_probe.html"),
_ => panic!("Unknown native test fixture: {}", name),
}
}
@@ -3884,3 +3885,96 @@ async fn e2e_relaunch_on_options_change() {
let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
assert_success(&resp);
}
// ---------------------------------------------------------------------------
// Upload: ref-based selector support (issue #1107)
// ---------------------------------------------------------------------------
#[tokio::test]
#[ignore]
async fn e2e_upload_with_ref_selector() {
let mut state = DaemonState::new();
let resp = execute_command(
&json!({ "id": "1", "action": "launch", "headless": true }),
&mut state,
)
.await;
assert_success(&resp);
let resp = execute_command(
&json!({ "id": "2", "action": "navigate", "url": native_test_fixture_url("upload_probe") }),
&mut state,
)
.await;
assert_success(&resp);
let resp = execute_command(&json!({ "id": "3", "action": "snapshot" }), &mut state).await;
assert_success(&resp);
let snapshot = get_data(&resp)["snapshot"].as_str().unwrap();
// Match by label text, not by role which may vary across Chrome versions
let file_input_ref = snapshot
.lines()
.filter_map(|line| {
if line.contains("Choose file") && line.contains("ref=") {
let start = line.find("ref=")? + 4;
let end = line[start..].find(']')? + start;
Some(line[start..end].to_string())
} else {
None
}
})
.next()
.expect("Snapshot should contain the file input with a ref");
let tmp = std::env::temp_dir().join(format!("ab-upload-ref-{}.txt", std::process::id()));
std::fs::write(&tmp, "test").unwrap();
let resp = execute_command(
&json!({ "id": "4", "action": "upload", "selector": file_input_ref, "files": [tmp.to_string_lossy()] }),
&mut state,
)
.await;
assert_success(&resp);
assert_eq!(get_data(&resp)["uploaded"], 1);
let _ = std::fs::remove_file(&tmp);
let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
assert_success(&resp);
}
#[tokio::test]
#[ignore]
async fn e2e_upload_with_css_selector() {
let mut state = DaemonState::new();
let resp = execute_command(
&json!({ "id": "1", "action": "launch", "headless": true }),
&mut state,
)
.await;
assert_success(&resp);
let resp = execute_command(
&json!({ "id": "2", "action": "navigate", "url": native_test_fixture_url("upload_probe") }),
&mut state,
)
.await;
assert_success(&resp);
let tmp = std::env::temp_dir().join(format!("ab-upload-css-{}.txt", std::process::id()));
std::fs::write(&tmp, "test").unwrap();
let resp = execute_command(
&json!({ "id": "3", "action": "upload", "selector": "#fileInput", "files": [tmp.to_string_lossy()] }),
&mut state,
)
.await;
assert_success(&resp);
assert_eq!(get_data(&resp)["uploaded"], 1);
let _ = std::fs::remove_file(&tmp);
let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
assert_success(&resp);
}
cli/src/native/test_fixtures/upload_probe.html
+18
View File
@@ -0,0 +1,18 @@
<!DOCTYPE html>
<html>
<head><title>Upload Test</title></head>
<body>
<h1>Upload Test</h1>
<label for="fileInput">Choose file:</label>
<input type="file" id="fileInput" name="fileInput">
<div id="result"></div>
<script>
document.getElementById('fileInput').addEventListener('change', function(e) {
var file = e.target.files[0];
if (file) {
document.getElementById('result').textContent = 'uploaded:' + file.name;
}
});
</script>
</body>
</html>