feat(text): frame-aware text extraction — get text --all-frames / --main + frames (#27)

On listing/marketplace pages (Yahoo Auctions, Rakuten, Mercari shops) the
seller's description lives in a child frame or under a related-items sidebar,
so 'get text body' returned only header/nav boilerplate.

- get text --all-frames: aggregate visible text across every reachable frame.
  Same-process child frames are read via Page.createIsolatedWorld; OOPIFs via
  their auto-attached debugger session (iframe_sessions). Each non-top frame is
  labelled with a '----- frame [kind] url -----' separator.
- get text --main: readability-lite — prefer the densest <main>/<article>
  region over the whole body, dropping global header/nav/footer chrome.
- frames: enumerate frames (kind + url + per-frame text length) so an agent can
  see where a page's text actually lives and pick the right read.

Verified live: inline srcdoc frame text aggregated through --all-frames; Yahoo
Auctions <main> (2881 chars) extracted via --main, stripping the Yahoo header.
This commit is contained in:
leeguooooo
2026-06-15 12:51:59 +09:00
parent f7a657ac46
commit 2707ceb1c4
5 changed files with 370 additions and 1 deletions
+63 -1
View File
@@ -1296,6 +1296,11 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result<Value, ParseErr
// === Get === // === Get ===
"get" => parse_get(&rest, &id), "get" => parse_get(&rest, &id),
// List every frame the session can reach (top + same-process child
// frames + out-of-process iframes), with a text-length per frame so you
// can see where a listing's description actually lives (issue #27).
"frames" => Ok(json!({ "id": id, "action": "frames" })),
// Top-level shortcuts for `get <x>` status reads — users naturally type // Top-level shortcuts for `get <x>` status reads — users naturally type
// `chrome-use url` / `cdp-url` / `title` without the `get` prefix // `chrome-use url` / `cdp-url` / `title` without the `get` prefix
// (and expect `cdp-url`/`cdp_url` to work interchangeably). // (and expect `cdp-url`/`cdp_url` to work interchangeably).
@@ -2388,10 +2393,32 @@ fn parse_get(rest: &[&str], id: &str) -> Result<Value, ParseError> {
match rest.first().copied() { match rest.first().copied() {
Some("text") => { Some("text") => {
// `get text --all-frames` aggregates visible text across every
// frame, including out-of-process iframes invisible to the top
// document (issue #27). The selector is ignored in this mode.
let all_frames = rest[1..]
.iter()
.any(|a| matches!(*a, "--all-frames" | "--frames" | "-a"));
if all_frames {
return Ok(json!({ "id": id, "action": "gettext", "allFrames": true }));
}
// `get text --main` returns the main-content region (readability),
// skipping header/nav/footer/sidebar boilerplate (issue #27).
let main = rest[1..]
.iter()
.any(|a| matches!(*a, "--main" | "--readable" | "-m"));
if main {
return Ok(json!({ "id": id, "action": "gettext", "main": true }));
}
// `get text` with no selector returns the whole page's text (body) — // `get text` with no selector returns the whole page's text (body) —
// a common convenience; previously it errored without a selector // a common convenience; previously it errored without a selector
// (issue #24-D). // (issue #24-D).
let sel = rest.get(1).copied().unwrap_or("body"); let sel = rest
.iter()
.skip(1)
.find(|a| !a.starts_with("--"))
.copied()
.unwrap_or("body");
Ok(json!({ "id": id, "action": "gettext", "selector": sel })) Ok(json!({ "id": id, "action": "gettext", "selector": sel }))
} }
Some("html") => { Some("html") => {
@@ -4761,6 +4788,41 @@ mod tests {
assert_eq!(cmd2["selector"], "h1"); assert_eq!(cmd2["selector"], "h1");
} }
#[test]
fn test_get_text_all_frames() {
// `--all-frames` switches to whole-page, cross-frame aggregation and
// drops the selector (issue #27).
for variant in ["get text --all-frames", "get text --frames", "text -a"] {
let cmd = parse_command(&args(variant), &default_flags()).unwrap();
assert_eq!(cmd["action"], "gettext", "{variant}");
assert_eq!(cmd["allFrames"], true, "{variant}");
assert!(cmd.get("selector").is_none(), "{variant}");
}
// A flag mixed with a selector still triggers all-frames.
let cmd = parse_command(&args("get text body --all-frames"), &default_flags()).unwrap();
assert_eq!(cmd["allFrames"], true);
// Without the flag, a leading flag-like token is skipped for the selector.
let cmd = parse_command(&args("get text main"), &default_flags()).unwrap();
assert_eq!(cmd["selector"], "main");
assert!(cmd.get("allFrames").is_none());
}
#[test]
fn test_frames_command() {
let cmd = parse_command(&args("frames"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "frames");
}
#[test]
fn test_get_text_main() {
for variant in ["get text --main", "get text --readable", "text -m"] {
let cmd = parse_command(&args(variant), &default_flags()).unwrap();
assert_eq!(cmd["action"], "gettext", "{variant}");
assert_eq!(cmd["main"], true, "{variant}");
assert!(cmd.get("selector").is_none(), "{variant}");
}
}
#[test] #[test]
fn test_tab_activate_flag() { fn test_tab_activate_flag() {
let plain = parse_command(&args("tab t3"), &default_flags()).unwrap(); let plain = parse_command(&args("tab t3"), &default_flags()).unwrap();
+68
View File
@@ -1332,6 +1332,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value {
"uncheck" => handle_uncheck(cmd, state).await, "uncheck" => handle_uncheck(cmd, state).await,
"wait" => handle_wait(cmd, state).await, "wait" => handle_wait(cmd, state).await,
"gettext" => handle_gettext(cmd, state).await, "gettext" => handle_gettext(cmd, state).await,
"frames" => handle_frames(cmd, state).await,
"getattribute" => handle_getattribute(cmd, state).await, "getattribute" => handle_getattribute(cmd, state).await,
"isvisible" => handle_isvisible(cmd, state).await, "isvisible" => handle_isvisible(cmd, state).await,
"isenabled" => handle_isenabled(cmd, state).await, "isenabled" => handle_isenabled(cmd, state).await,
@@ -3594,6 +3595,47 @@ async fn handle_wait(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> { async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
let session_id = mgr.active_session_id()?.to_string(); let session_id = mgr.active_session_id()?.to_string();
// `get text --all-frames` aggregates visible text across every frame the
// session can reach — including out-of-process iframes that never show up
// in the top document (#27: Yahoo/Rakuten/Mercari listing descriptions).
if cmd.get("allFrames").and_then(|v| v.as_bool()) == Some(true) {
let frames = super::element::collect_all_frames_text(
&mgr.client,
&session_id,
&state.iframe_sessions,
)
.await?;
let mut combined = String::new();
let mut frame_count = 0usize;
for f in &frames {
let t = f.text.trim();
if t.is_empty() {
continue;
}
frame_count += 1;
if f.kind != "top" {
combined.push_str(&format!("\n\n----- frame [{}] {} -----\n", f.kind, f.url));
}
combined.push_str(t);
}
let url = mgr.get_url().await.unwrap_or_default();
return Ok(json!({
"text": combined,
"origin": url,
"frames": frame_count,
"allFrames": true,
}));
}
// `get text --main` returns the page's main-content region (readability-lite),
// skipping global header/nav/footer/sidebar boilerplate (#27).
if cmd.get("main").and_then(|v| v.as_bool()) == Some(true) {
let text = super::element::get_main_content_text(&mgr.client, &session_id).await?;
let url = mgr.get_url().await.unwrap_or_default();
return Ok(json!({ "text": text, "origin": url, "main": true }));
}
let selector = cmd let selector = cmd
.get("selector") .get("selector")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
@@ -3611,6 +3653,32 @@ async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result<Value, S
Ok(json!({ "text": text, "origin": url })) Ok(json!({ "text": text, "origin": url }))
} }
async fn handle_frames(_cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
let session_id = mgr.active_session_id()?.to_string();
let frames = super::element::collect_all_frames_text(
&mgr.client,
&session_id,
&state.iframe_sessions,
)
.await?;
let list: Vec<Value> = frames
.iter()
.enumerate()
.map(|(i, f)| {
json!({
"index": i,
"kind": f.kind,
"url": f.url,
"frameId": f.frame_id,
"textLen": f.text.trim().chars().count(),
})
})
.collect();
let url = mgr.get_url().await.unwrap_or_default();
Ok(json!({ "frames": list, "count": list.len(), "origin": url }))
}
async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> { async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result<Value, String> {
let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let mgr = state.browser.as_ref().ok_or("Browser not launched")?;
let session_id = mgr.active_session_id()?.to_string(); let session_id = mgr.active_session_id()?.to_string();
+194
View File
@@ -975,6 +975,200 @@ pub async fn get_element_text(
.unwrap_or_default()) .unwrap_or_default())
} }
/// Text content collected from a single frame of the page.
#[derive(Debug, Clone)]
pub struct FrameText {
pub frame_id: String,
pub url: String,
/// "top" | "inline" (same-process child frame) | "oopif" (out-of-process).
pub kind: &'static str,
pub text: String,
}
// The expression we run in every frame to read its visible text. innerText
// honors CSS visibility (skips display:none), textContent is the fallback.
const FRAME_INNERTEXT_JS: &str = "(function(){try{var b=document.body||document.documentElement;return b?(b.innerText||b.textContent||''):'';}catch(e){return '';}})()";
async fn eval_text_default(client: &CdpClient, session_id: &str) -> String {
let res = client
.send_command(
"Runtime.evaluate",
Some(serde_json::json!({
"expression": FRAME_INNERTEXT_JS,
"returnByValue": true,
})),
Some(session_id),
)
.await;
res.ok()
.and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned())
.and_then(|v| v.as_str().map(|s| s.to_string()))
.unwrap_or_default()
}
// Same-process child frames share the top renderer but live in their own
// execution context. Page.createIsolatedWorld hands us a context id bound to
// that frame so Runtime.evaluate reads the child document, not the parent.
async fn eval_text_in_frame(client: &CdpClient, session_id: &str, frame_id: &str) -> String {
let ctx = client
.send_command(
"Page.createIsolatedWorld",
Some(serde_json::json!({ "frameId": frame_id, "worldName": "chrome_use_text" })),
Some(session_id),
)
.await
.ok()
.and_then(|v| v.get("executionContextId").and_then(|c| c.as_i64()));
let Some(ctx_id) = ctx else { return String::new() };
let res = client
.send_command(
"Runtime.evaluate",
Some(serde_json::json!({
"expression": FRAME_INNERTEXT_JS,
"returnByValue": true,
"contextId": ctx_id,
})),
Some(session_id),
)
.await;
res.ok()
.and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned())
.and_then(|v| v.as_str().map(|s| s.to_string()))
.unwrap_or_default()
}
fn flatten_frame_tree(node: &Value, is_top: bool, out: &mut Vec<(String, String, bool)>) {
if let Some(frame) = node.get("frame") {
if let Some(id) = frame.get("id").and_then(|v| v.as_str()) {
let url = frame
.get("url")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
out.push((id.to_string(), url, is_top));
}
}
if let Some(children) = node.get("childFrames").and_then(|v| v.as_array()) {
for child in children {
flatten_frame_tree(child, false, out);
}
}
}
/// Collect visible text from every frame reachable in the active session,
/// including out-of-process iframes (which never appear in the top frame's
/// `Page.getFrameTree` and so are invisible to `document.body.innerText`).
///
/// Same-process child frames are read through `Page.createIsolatedWorld`;
/// OOPIFs are read through their own auto-attached debugger session
/// (`iframe_sessions`, keyed by frameId == targetId). This is the engine
/// behind `get text --all-frames` and `chrome-use frames` — the fix for
/// listing/marketplace pages whose description lives in a child frame (#27).
pub async fn collect_all_frames_text(
client: &CdpClient,
top_session: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<Vec<FrameText>, String> {
let mut out: Vec<FrameText> = Vec::new();
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
// 1. Top session: the top frame plus its same-process descendants. OOPIF
// frames that happen to surface here are skipped — they're read via
// their dedicated session in step 2 (cross-process isolated worlds fail).
let tree = client
.send_command_no_params("Page.getFrameTree", Some(top_session))
.await?;
let mut frames: Vec<(String, String, bool)> = Vec::new();
flatten_frame_tree(&tree["frameTree"], true, &mut frames);
for (fid, url, is_top) in frames {
if iframe_sessions.contains_key(&fid) {
continue;
}
if !seen.insert(fid.clone()) {
continue;
}
let (kind, text) = if is_top {
("top", eval_text_default(client, top_session).await)
} else {
("inline", eval_text_in_frame(client, top_session, &fid).await)
};
out.push(FrameText {
frame_id: fid,
url,
kind,
text,
});
}
// 2. Each out-of-process iframe, read through its own session.
for (fid, sid) in iframe_sessions {
if !seen.insert(fid.clone()) {
continue;
}
let url = client
.send_command_no_params("Page.getFrameTree", Some(sid))
.await
.ok()
.and_then(|t| {
t.get("frameTree")
.and_then(|ft| ft.get("frame"))
.and_then(|f| f.get("url"))
.and_then(|v| v.as_str())
.map(|s| s.to_string())
})
.unwrap_or_default();
let text = eval_text_default(client, sid).await;
out.push(FrameText {
frame_id: fid.clone(),
url,
kind: "oopif",
text,
});
}
Ok(out)
}
// Readability-lite: prefer the page's semantic main-content region over the
// whole body so global header/nav/footer chrome (and, on many listing pages,
// the "related items" sidebar) doesn't drown out the actual content. Runs on
// the live, rendered tree (innerText needs layout — a detached clone returns
// empty), so we pick the densest <main>/<article> region rather than cloning
// and stripping. Falls back to <body> when no substantial main region exists.
const MAIN_CONTENT_JS: &str = r#"(function(){
function txt(el){try{return (el.innerText||'').trim();}catch(e){return '';}}
var sels=['main','[role=main]','article','#main','#contents','#l-content'];
var best=null,bestLen=0;
for(var i=0;i<sels.length;i++){
var els=document.querySelectorAll(sels[i]);
for(var j=0;j<els.length;j++){var l=txt(els[j]).length;if(l>bestLen){bestLen=l;best=els[j];}}
}
if(best&&bestLen>200)return txt(best);
return txt(document.body);
})()"#;
/// Extract the page's main-content text (readability-lite), preferring a
/// semantic `<main>`/`<article>` region over the full body. Used by
/// `get text --main` to avoid header/nav/sidebar boilerplate (#27).
pub async fn get_main_content_text(client: &CdpClient, session_id: &str) -> Result<String, String> {
let res = client
.send_command(
"Runtime.evaluate",
Some(serde_json::json!({
"expression": MAIN_CONTENT_JS,
"returnByValue": true,
})),
Some(session_id),
)
.await?;
Ok(res
.get("result")
.and_then(|r| r.get("value"))
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string())
}
pub async fn get_element_attribute( pub async fn get_element_attribute(
client: &CdpClient, client: &CdpClient,
session_id: &str, session_id: &str,
+34
View File
@@ -342,6 +342,34 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou
} }
return; return;
} }
// Frame list (`chrome-use frames`)
if action == Some("frames") {
if let Some(list) = data.get("frames").and_then(|v| v.as_array()) {
let count = list.len();
println!(
"{}",
color::bold(&format!("{} frame{}", count, if count == 1 { "" } else { "s" }))
);
for f in list {
let idx = f.get("index").and_then(|v| v.as_i64()).unwrap_or(0);
let kind = f.get("kind").and_then(|v| v.as_str()).unwrap_or("?");
let url = f.get("url").and_then(|v| v.as_str()).unwrap_or("");
let len = f.get("textLen").and_then(|v| v.as_i64()).unwrap_or(0);
println!(
" [{}] {:<6} {} chars {}",
idx,
kind,
len,
color::dim(if url.is_empty() { "(about:blank)" } else { url })
);
}
eprintln!(
"{}",
color::dim("read everything with: chrome-use get text --all-frames")
);
}
return;
}
// Title // Title
if let Some(title) = data.get("title").and_then(|v| v.as_str()) { if let Some(title) = data.get("title").and_then(|v| v.as_str()) {
println!("{}", title); println!("{}", title);
@@ -1930,6 +1958,8 @@ Retrieves various types of information from elements or the page.
Subcommands: Subcommands:
text <selector> Get text content of element text <selector> Get text content of element
text --all-frames Aggregate text across ALL frames (incl. iframes)
text --main Main-content text only (skip nav/header/sidebar)
html <selector> Get inner HTML of element html <selector> Get inner HTML of element
value <selector> Get value of input element value <selector> Get value of input element
attr <selector> <name> Get attribute value attr <selector> <name> Get attribute value
@@ -1946,6 +1976,9 @@ Global Options:
Examples: Examples:
chrome-use get text @e1 chrome-use get text @e1
chrome-use get text --all-frames # read iframed content (listing pages)
chrome-use get text --main # main content, no nav/sidebar boilerplate
chrome-use frames # list frames + where the text lives
chrome-use get html "#content" chrome-use get html "#content"
chrome-use get value "#email-input" chrome-use get value "#email-input"
chrome-use get attr "#link" href chrome-use get attr "#link" href
@@ -3155,6 +3188,7 @@ Navigation:
Get Info: chrome-use get <what> [selector] Get Info: chrome-use get <what> [selector]
text, html, value, attr <name>, title, url, count, box, styles, cdp-url text, html, value, attr <name>, title, url, count, box, styles, cdp-url
text --all-frames (cross-frame), text --main (no boilerplate), frames (list)
Check State: chrome-use is <what> <selector> Check State: chrome-use is <what> <selector>
visible, enabled, checked visible, enabled, checked
+11
View File
@@ -208,6 +208,9 @@ For unstructured reading (no refs needed):
```bash ```bash
chrome-use get text @e1 # visible text of an element chrome-use get text @e1 # visible text of an element
chrome-use get text --all-frames # whole page, aggregated across ALL frames
chrome-use get text --main # main content only — skip nav/header/sidebar
chrome-use frames # list every frame + where the text lives
chrome-use get html @e1 # innerHTML chrome-use get html @e1 # innerHTML
chrome-use get attr @e1 href # any attribute chrome-use get attr @e1 href # any attribute
chrome-use get value @e1 # input value chrome-use get value @e1 # input value
@@ -216,6 +219,14 @@ chrome-use get url # current URL
chrome-use get count ".item" # count matching elements chrome-use get count ".item" # count matching elements
``` ```
On listing/marketplace pages (Yahoo Auctions, Rakuten, Mercari shops) the seller's
description often lives in a **child frame** or is buried under a "related items"
sidebar, so a plain `get text body` returns only header/nav boilerplate. When the
text you expect is missing: run `chrome-use frames` to see where it is, then
`get text --all-frames` (reads every reachable frame incl. cross-origin iframes)
or `get text --main` (drops the global chrome). If the content is lazy-loaded,
`scroll` it into view first.
## Interacting ## Interacting
```bash ```bash