diff --git a/cli/src/commands.rs b/cli/src/commands.rs index f1a44ac..d62aff0 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -1296,6 +1296,11 @@ fn parse_command_inner(args: &[String], flags: &Flags) -> Result parse_get(&rest, &id), + // List every frame the session can reach (top + same-process child + // frames + out-of-process iframes), with a text-length per frame so you + // can see where a listing's description actually lives (issue #27). + "frames" => Ok(json!({ "id": id, "action": "frames" })), + // Top-level shortcuts for `get ` status reads — users naturally type // `chrome-use url` / `cdp-url` / `title` without the `get` prefix // (and expect `cdp-url`/`cdp_url` to work interchangeably). @@ -2388,10 +2393,32 @@ fn parse_get(rest: &[&str], id: &str) -> Result { match rest.first().copied() { Some("text") => { + // `get text --all-frames` aggregates visible text across every + // frame, including out-of-process iframes invisible to the top + // document (issue #27). The selector is ignored in this mode. + let all_frames = rest[1..] + .iter() + .any(|a| matches!(*a, "--all-frames" | "--frames" | "-a")); + if all_frames { + return Ok(json!({ "id": id, "action": "gettext", "allFrames": true })); + } + // `get text --main` returns the main-content region (readability), + // skipping header/nav/footer/sidebar boilerplate (issue #27). + let main = rest[1..] + .iter() + .any(|a| matches!(*a, "--main" | "--readable" | "-m")); + if main { + return Ok(json!({ "id": id, "action": "gettext", "main": true })); + } // `get text` with no selector returns the whole page's text (body) — // a common convenience; previously it errored without a selector // (issue #24-D). - let sel = rest.get(1).copied().unwrap_or("body"); + let sel = rest + .iter() + .skip(1) + .find(|a| !a.starts_with("--")) + .copied() + .unwrap_or("body"); Ok(json!({ "id": id, "action": "gettext", "selector": sel })) } Some("html") => { @@ -4761,6 +4788,41 @@ mod tests { assert_eq!(cmd2["selector"], "h1"); } + #[test] + fn test_get_text_all_frames() { + // `--all-frames` switches to whole-page, cross-frame aggregation and + // drops the selector (issue #27). + for variant in ["get text --all-frames", "get text --frames", "text -a"] { + let cmd = parse_command(&args(variant), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "gettext", "{variant}"); + assert_eq!(cmd["allFrames"], true, "{variant}"); + assert!(cmd.get("selector").is_none(), "{variant}"); + } + // A flag mixed with a selector still triggers all-frames. + let cmd = parse_command(&args("get text body --all-frames"), &default_flags()).unwrap(); + assert_eq!(cmd["allFrames"], true); + // Without the flag, a leading flag-like token is skipped for the selector. + let cmd = parse_command(&args("get text main"), &default_flags()).unwrap(); + assert_eq!(cmd["selector"], "main"); + assert!(cmd.get("allFrames").is_none()); + } + + #[test] + fn test_frames_command() { + let cmd = parse_command(&args("frames"), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "frames"); + } + + #[test] + fn test_get_text_main() { + for variant in ["get text --main", "get text --readable", "text -m"] { + let cmd = parse_command(&args(variant), &default_flags()).unwrap(); + assert_eq!(cmd["action"], "gettext", "{variant}"); + assert_eq!(cmd["main"], true, "{variant}"); + assert!(cmd.get("selector").is_none(), "{variant}"); + } + } + #[test] fn test_tab_activate_flag() { let plain = parse_command(&args("tab t3"), &default_flags()).unwrap(); diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 3424cfb..b095410 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -1332,6 +1332,7 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value { "uncheck" => handle_uncheck(cmd, state).await, "wait" => handle_wait(cmd, state).await, "gettext" => handle_gettext(cmd, state).await, + "frames" => handle_frames(cmd, state).await, "getattribute" => handle_getattribute(cmd, state).await, "isvisible" => handle_isvisible(cmd, state).await, "isenabled" => handle_isenabled(cmd, state).await, @@ -3594,6 +3595,47 @@ async fn handle_wait(cmd: &Value, state: &mut DaemonState) -> Result Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); + + // `get text --all-frames` aggregates visible text across every frame the + // session can reach — including out-of-process iframes that never show up + // in the top document (#27: Yahoo/Rakuten/Mercari listing descriptions). + if cmd.get("allFrames").and_then(|v| v.as_bool()) == Some(true) { + let frames = super::element::collect_all_frames_text( + &mgr.client, + &session_id, + &state.iframe_sessions, + ) + .await?; + let mut combined = String::new(); + let mut frame_count = 0usize; + for f in &frames { + let t = f.text.trim(); + if t.is_empty() { + continue; + } + frame_count += 1; + if f.kind != "top" { + combined.push_str(&format!("\n\n----- frame [{}] {} -----\n", f.kind, f.url)); + } + combined.push_str(t); + } + let url = mgr.get_url().await.unwrap_or_default(); + return Ok(json!({ + "text": combined, + "origin": url, + "frames": frame_count, + "allFrames": true, + })); + } + + // `get text --main` returns the page's main-content region (readability-lite), + // skipping global header/nav/footer/sidebar boilerplate (#27). + if cmd.get("main").and_then(|v| v.as_bool()) == Some(true) { + let text = super::element::get_main_content_text(&mgr.client, &session_id).await?; + let url = mgr.get_url().await.unwrap_or_default(); + return Ok(json!({ "text": text, "origin": url, "main": true })); + } + let selector = cmd .get("selector") .and_then(|v| v.as_str()) @@ -3611,6 +3653,32 @@ async fn handle_gettext(cmd: &Value, state: &mut DaemonState) -> Result Result { + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + let session_id = mgr.active_session_id()?.to_string(); + let frames = super::element::collect_all_frames_text( + &mgr.client, + &session_id, + &state.iframe_sessions, + ) + .await?; + let list: Vec = frames + .iter() + .enumerate() + .map(|(i, f)| { + json!({ + "index": i, + "kind": f.kind, + "url": f.url, + "frameId": f.frame_id, + "textLen": f.text.trim().chars().count(), + }) + }) + .collect(); + let url = mgr.get_url().await.unwrap_or_default(); + Ok(json!({ "frames": list, "count": list.len(), "origin": url })) +} + async fn handle_getattribute(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); diff --git a/cli/src/native/element.rs b/cli/src/native/element.rs index a78f7ef..08db3b3 100644 --- a/cli/src/native/element.rs +++ b/cli/src/native/element.rs @@ -975,6 +975,200 @@ pub async fn get_element_text( .unwrap_or_default()) } +/// Text content collected from a single frame of the page. +#[derive(Debug, Clone)] +pub struct FrameText { + pub frame_id: String, + pub url: String, + /// "top" | "inline" (same-process child frame) | "oopif" (out-of-process). + pub kind: &'static str, + pub text: String, +} + +// The expression we run in every frame to read its visible text. innerText +// honors CSS visibility (skips display:none), textContent is the fallback. +const FRAME_INNERTEXT_JS: &str = "(function(){try{var b=document.body||document.documentElement;return b?(b.innerText||b.textContent||''):'';}catch(e){return '';}})()"; + +async fn eval_text_default(client: &CdpClient, session_id: &str) -> String { + let res = client + .send_command( + "Runtime.evaluate", + Some(serde_json::json!({ + "expression": FRAME_INNERTEXT_JS, + "returnByValue": true, + })), + Some(session_id), + ) + .await; + res.ok() + .and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned()) + .and_then(|v| v.as_str().map(|s| s.to_string())) + .unwrap_or_default() +} + +// Same-process child frames share the top renderer but live in their own +// execution context. Page.createIsolatedWorld hands us a context id bound to +// that frame so Runtime.evaluate reads the child document, not the parent. +async fn eval_text_in_frame(client: &CdpClient, session_id: &str, frame_id: &str) -> String { + let ctx = client + .send_command( + "Page.createIsolatedWorld", + Some(serde_json::json!({ "frameId": frame_id, "worldName": "chrome_use_text" })), + Some(session_id), + ) + .await + .ok() + .and_then(|v| v.get("executionContextId").and_then(|c| c.as_i64())); + let Some(ctx_id) = ctx else { return String::new() }; + let res = client + .send_command( + "Runtime.evaluate", + Some(serde_json::json!({ + "expression": FRAME_INNERTEXT_JS, + "returnByValue": true, + "contextId": ctx_id, + })), + Some(session_id), + ) + .await; + res.ok() + .and_then(|v| v.get("result").and_then(|r| r.get("value")).cloned()) + .and_then(|v| v.as_str().map(|s| s.to_string())) + .unwrap_or_default() +} + +fn flatten_frame_tree(node: &Value, is_top: bool, out: &mut Vec<(String, String, bool)>) { + if let Some(frame) = node.get("frame") { + if let Some(id) = frame.get("id").and_then(|v| v.as_str()) { + let url = frame + .get("url") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + out.push((id.to_string(), url, is_top)); + } + } + if let Some(children) = node.get("childFrames").and_then(|v| v.as_array()) { + for child in children { + flatten_frame_tree(child, false, out); + } + } +} + +/// Collect visible text from every frame reachable in the active session, +/// including out-of-process iframes (which never appear in the top frame's +/// `Page.getFrameTree` and so are invisible to `document.body.innerText`). +/// +/// Same-process child frames are read through `Page.createIsolatedWorld`; +/// OOPIFs are read through their own auto-attached debugger session +/// (`iframe_sessions`, keyed by frameId == targetId). This is the engine +/// behind `get text --all-frames` and `chrome-use frames` — the fix for +/// listing/marketplace pages whose description lives in a child frame (#27). +pub async fn collect_all_frames_text( + client: &CdpClient, + top_session: &str, + iframe_sessions: &HashMap, +) -> Result, String> { + let mut out: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + + // 1. Top session: the top frame plus its same-process descendants. OOPIF + // frames that happen to surface here are skipped — they're read via + // their dedicated session in step 2 (cross-process isolated worlds fail). + let tree = client + .send_command_no_params("Page.getFrameTree", Some(top_session)) + .await?; + let mut frames: Vec<(String, String, bool)> = Vec::new(); + flatten_frame_tree(&tree["frameTree"], true, &mut frames); + for (fid, url, is_top) in frames { + if iframe_sessions.contains_key(&fid) { + continue; + } + if !seen.insert(fid.clone()) { + continue; + } + let (kind, text) = if is_top { + ("top", eval_text_default(client, top_session).await) + } else { + ("inline", eval_text_in_frame(client, top_session, &fid).await) + }; + out.push(FrameText { + frame_id: fid, + url, + kind, + text, + }); + } + + // 2. Each out-of-process iframe, read through its own session. + for (fid, sid) in iframe_sessions { + if !seen.insert(fid.clone()) { + continue; + } + let url = client + .send_command_no_params("Page.getFrameTree", Some(sid)) + .await + .ok() + .and_then(|t| { + t.get("frameTree") + .and_then(|ft| ft.get("frame")) + .and_then(|f| f.get("url")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }) + .unwrap_or_default(); + let text = eval_text_default(client, sid).await; + out.push(FrameText { + frame_id: fid.clone(), + url, + kind: "oopif", + text, + }); + } + + Ok(out) +} + +// Readability-lite: prefer the page's semantic main-content region over the +// whole body so global header/nav/footer chrome (and, on many listing pages, +// the "related items" sidebar) doesn't drown out the actual content. Runs on +// the live, rendered tree (innerText needs layout — a detached clone returns +// empty), so we pick the densest
/
region rather than cloning +// and stripping. Falls back to when no substantial main region exists. +const MAIN_CONTENT_JS: &str = r#"(function(){ + function txt(el){try{return (el.innerText||'').trim();}catch(e){return '';}} + var sels=['main','[role=main]','article','#main','#contents','#l-content']; + var best=null,bestLen=0; + for(var i=0;ibestLen){bestLen=l;best=els[j];}} + } + if(best&&bestLen>200)return txt(best); + return txt(document.body); +})()"#; + +/// Extract the page's main-content text (readability-lite), preferring a +/// semantic `
`/`
` region over the full body. Used by +/// `get text --main` to avoid header/nav/sidebar boilerplate (#27). +pub async fn get_main_content_text(client: &CdpClient, session_id: &str) -> Result { + let res = client + .send_command( + "Runtime.evaluate", + Some(serde_json::json!({ + "expression": MAIN_CONTENT_JS, + "returnByValue": true, + })), + Some(session_id), + ) + .await?; + Ok(res + .get("result") + .and_then(|r| r.get("value")) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string()) +} + pub async fn get_element_attribute( client: &CdpClient, session_id: &str, diff --git a/cli/src/output.rs b/cli/src/output.rs index f942cc9..9d90545 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -342,6 +342,34 @@ pub fn print_response_with_opts(resp: &Response, action: Option<&str>, opts: &Ou } return; } + // Frame list (`chrome-use frames`) + if action == Some("frames") { + if let Some(list) = data.get("frames").and_then(|v| v.as_array()) { + let count = list.len(); + println!( + "{}", + color::bold(&format!("{} frame{}", count, if count == 1 { "" } else { "s" })) + ); + for f in list { + let idx = f.get("index").and_then(|v| v.as_i64()).unwrap_or(0); + let kind = f.get("kind").and_then(|v| v.as_str()).unwrap_or("?"); + let url = f.get("url").and_then(|v| v.as_str()).unwrap_or(""); + let len = f.get("textLen").and_then(|v| v.as_i64()).unwrap_or(0); + println!( + " [{}] {:<6} {} chars {}", + idx, + kind, + len, + color::dim(if url.is_empty() { "(about:blank)" } else { url }) + ); + } + eprintln!( + "{}", + color::dim("read everything with: chrome-use get text --all-frames") + ); + } + return; + } // Title if let Some(title) = data.get("title").and_then(|v| v.as_str()) { println!("{}", title); @@ -1930,6 +1958,8 @@ Retrieves various types of information from elements or the page. Subcommands: text Get text content of element + text --all-frames Aggregate text across ALL frames (incl. iframes) + text --main Main-content text only (skip nav/header/sidebar) html Get inner HTML of element value Get value of input element attr Get attribute value @@ -1946,6 +1976,9 @@ Global Options: Examples: chrome-use get text @e1 + chrome-use get text --all-frames # read iframed content (listing pages) + chrome-use get text --main # main content, no nav/sidebar boilerplate + chrome-use frames # list frames + where the text lives chrome-use get html "#content" chrome-use get value "#email-input" chrome-use get attr "#link" href @@ -3155,6 +3188,7 @@ Navigation: Get Info: chrome-use get [selector] text, html, value, attr , title, url, count, box, styles, cdp-url + text --all-frames (cross-frame), text --main (no boilerplate), frames (list) Check State: chrome-use is visible, enabled, checked diff --git a/skill-data/core/SKILL.md b/skill-data/core/SKILL.md index 4c3afcf..a3ad8e8 100644 --- a/skill-data/core/SKILL.md +++ b/skill-data/core/SKILL.md @@ -208,6 +208,9 @@ For unstructured reading (no refs needed): ```bash chrome-use get text @e1 # visible text of an element +chrome-use get text --all-frames # whole page, aggregated across ALL frames +chrome-use get text --main # main content only — skip nav/header/sidebar +chrome-use frames # list every frame + where the text lives chrome-use get html @e1 # innerHTML chrome-use get attr @e1 href # any attribute chrome-use get value @e1 # input value @@ -216,6 +219,14 @@ chrome-use get url # current URL chrome-use get count ".item" # count matching elements ``` +On listing/marketplace pages (Yahoo Auctions, Rakuten, Mercari shops) the seller's +description often lives in a **child frame** or is buried under a "related items" +sidebar, so a plain `get text body` returns only header/nav boilerplate. When the +text you expect is missing: run `chrome-use frames` to see where it is, then +`get text --all-frames` (reads every reachable frame incl. cross-origin iframes) +or `get text --main` (drops the global chrome). If the content is lazy-loaded, +`scroll` it into view first. + ## Interacting ```bash