fix: resolve snapshot -C and screenshot --annotate hang over WSS (#842)
* fix: resolve snapshot -C and screenshot --annotate hang over WSS (#841) Root cause: sequential CDP round-trips per element in find_cursor_interactive_elements() and collect_annotations() caused timeouts over high-latency WSS connections (~200ms × 200+ elements exceeds the 30s CDP timeout). Fix: - snapshot -C: Replace per-element CDP calls with a single JS eval that detects cursor:pointer/onclick/tabindex elements in-browser, then batch-resolve via DOM.querySelectorAll + concurrent DOM.describeNode calls using join_all - screenshot --annotate: Replace sequential DOM.resolveNode + getRect calls with concurrent join_all, matching v0.19.0's Promise.all() pattern Behavioral parity with v0.19.0 (Node.js/Playwright): - cursor:pointer detection via getComputedStyle - Inherited cursor:pointer dedup (skip children of pointer parents) - interactiveTags and interactive ARIA roles exclusion - Role differentiation: clickable vs focusable - Text dedup against ARIA tree ref names and quoted strings - Edge case: -i -C shows cursor elements even when ARIA tree is empty Tests: - 5 unit tests for build_dedup_set() helper - 3 e2e regression tests: cursor-interactive detection, annotation scaling to 50 elements, cursor scaling to 100 elements * fix: add hidden/aria-hidden filtering, contentEditable support, and cleanup robustness - Restore hidden/aria-hidden element filtering in cursor-interactive JS (was present in old code, dropped during rewrite) - Add contentEditable detection with 'editable' role and hint - Replace fire-and-forget cleanup with warning on failure - Simplify build_dedup_set to use ref_map only (eliminates fragile tree-text quote parsing; ref_map already has all ref-bearing names) --------- Co-authored-by: ctate <366502+ctate@users.noreply.github.com>
This commit is contained in:
@@ -223,35 +223,87 @@ async fn collect_annotations(
|
||||
session_id: &str,
|
||||
ref_map: &RefMap,
|
||||
) -> Result<Vec<RawAnnotation>, String> {
|
||||
let mut annotations = Vec::new();
|
||||
let entries = ref_map.entries_sorted();
|
||||
if entries.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
for (ref_id, entry) in ref_map.entries_sorted() {
|
||||
let object_id =
|
||||
match super::element::resolve_element_object_id(client, session_id, ref_map, &ref_id)
|
||||
.await
|
||||
// Collect entries that have backend_node_ids for batch resolution.
|
||||
let with_backend_ids: Vec<(String, super::element::RefEntry, i64)> = entries
|
||||
.iter()
|
||||
.filter_map(|(ref_id, entry)| {
|
||||
entry
|
||||
.backend_node_id
|
||||
.map(|bid| (ref_id.clone(), entry.clone(), bid))
|
||||
})
|
||||
.collect();
|
||||
|
||||
if with_backend_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Batch-resolve all backend_node_ids to object IDs using concurrent CDP calls.
|
||||
let resolve_futures: Vec<_> = with_backend_ids
|
||||
.iter()
|
||||
.map(|(_, _, backend_node_id)| {
|
||||
client.send_command(
|
||||
"DOM.resolveNode",
|
||||
Some(serde_json::json!({
|
||||
"backendNodeId": backend_node_id,
|
||||
"objectGroup": "agent-browser-annotate"
|
||||
})),
|
||||
Some(session_id),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let resolve_results = futures_util::future::join_all(resolve_futures).await;
|
||||
|
||||
// Collect resolved object IDs paired with their ref info.
|
||||
let mut resolved: Vec<(String, super::element::RefEntry, String)> = Vec::new();
|
||||
for (i, result) in resolve_results.into_iter().enumerate() {
|
||||
if let Ok(val) = result {
|
||||
if let Some(oid) = val
|
||||
.get("object")
|
||||
.and_then(|o| o.get("objectId"))
|
||||
.and_then(|v| v.as_str())
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let (ref_id, entry, _) = &with_backend_ids[i];
|
||||
resolved.push((ref_id.clone(), entry.clone(), oid.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(rect) = get_rect_for_object(client, session_id, &object_id).await? else {
|
||||
continue;
|
||||
if resolved.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Batch-get bounding rects for all resolved elements using concurrent CDP calls.
|
||||
let rect_futures: Vec<_> = resolved
|
||||
.iter()
|
||||
.map(|(_, _, object_id)| get_rect_for_object(client, session_id, object_id))
|
||||
.collect();
|
||||
|
||||
let rect_results = futures_util::future::join_all(rect_futures).await;
|
||||
|
||||
let mut annotations = Vec::new();
|
||||
for (i, rect_result) in rect_results.into_iter().enumerate() {
|
||||
let rect = match rect_result {
|
||||
Ok(Some(r)) if r.width > 0.0 && r.height > 0.0 => r,
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
if rect.width <= 0.0 || rect.height <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let (ref_id, entry, _) = &resolved[i];
|
||||
let number = ref_id
|
||||
.strip_prefix('e')
|
||||
.and_then(|n| n.parse::<u64>().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
annotations.push(RawAnnotation {
|
||||
ref_id,
|
||||
ref_id: ref_id.clone(),
|
||||
number,
|
||||
role: entry.role,
|
||||
name: (!entry.name.is_empty()).then_some(entry.name),
|
||||
role: entry.role.clone(),
|
||||
name: (!entry.name.is_empty()).then_some(entry.name.clone()),
|
||||
rect,
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user