oimwiodev/chrome-use
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: recording produces correct video duration with real-time ffmpeg encoding (#812)

Browse Source 
* fix: replace screenshot polling with screencast-based piped ffmpeg recording

  Recording previously used Page.captureScreenshot polling at 10fps,
  which was CPU-heavy and produced inconsistent results. Now uses
  Page.startScreencast with throttled acks (35ms interval) to receive
  frames event-driven from Chrome, and pipes JPEG data directly to
  ffmpeg stdin in real-time instead of saving temp files.

  - Spawn ffmpeg at recording start with piped stdin (image2pipe)
  - Background task receives screencast frames, interpolates gaps by
    repeating the last frame based on timestamps, targets 25fps
  - Ack throttling controls Chrome's frame push rate
  - Fix: current frame was never written after the first one
  - Fix: frame count was read before task finished padding
  - Remove tokio-util dependency (replaced CancellationToken with oneshot)
  - Add tokio "process" feature for async child process stdin pipe
  - Extract start/stop_recording_task helpers on DaemonState
  - Add tests for restart, ffmpeg codec selection, and stop without task

* fmt

* chore

* fix: switch WebM codec from VP9 to VP8 for correct framerate and browser
  compatibility

  VP9 realtime encoder ignored input framerate, producing 10fps output
  instead of 25fps. This caused inconsistent playback in browsers.
  VP8 respects -framerate 25 and has wider browser playback support.

* fmt

* fix: add kill_on_drop to ffmpeg process to prevent zombie on task panic

* fix: switch from screencast to screenshot polling for reliable recording duration

  Screencast only pushes frames on visual changes, producing short videos
  on static pages. Screenshot polling captures at a fixed 10fps interval
  regardless of page activity, guaranteeing duration matches wall-clock time.
  ffmpeg piped stdin architecture is preserved — no temp files.

---------

Co-authored-by: hyunjinee <leehj0110@kakao.com>
This commit is contained in:
jin.2
2026-03-15 08:04:27 -05:00
committed by GitHub
co-authored by hyunjinee
parent e78cc05cec
commit 19ba9048a0
3 changed files with 327 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/native/actions.rs
+122 -159
View File
@@ -1,8 +1,9 @@
use serde_json::{json, Value};
use std::env;
use std::io::Write;
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
use tokio::sync::{broadcast, RwLock};
use tokio::sync::{broadcast, oneshot, RwLock};
use super::auth;
use super::browser::{BrowserManager, WaitUntil};
@@ -166,6 +167,31 @@ impl DaemonState {
}
}
/// Spawn a background task that polls screenshots and pipes them to ffmpeg.
async fn start_recording_task(
&mut self,
client: Arc<CdpClient>,
session_id: String,
) -> Result<(), String> {
let shared_count = Arc::new(AtomicU64::new(0));
let (cancel_tx, cancel_rx) = oneshot::channel();
let handle = recording::spawn_recording_task(
client,
session_id,
self.recording_state.output_path.clone(),
shared_count.clone(),
cancel_rx,
);
self.recording_state.capture_task = Some(handle);
self.recording_state.shared_frame_count = Some(shared_count);
self.recording_state.cancel_tx = Some(cancel_tx);
Ok(())
}
async fn stop_recording_task(&mut self) -> Result<(), String> {
recording::stop_recording_task(&mut self.recording_state).await
}
fn drain_cdp_events(
&mut self,
) -> (
@@ -346,21 +372,6 @@ impl DaemonState {
}
}
"Page.screencastFrame" => {
if self.recording_state.active {
if let Some(data) =
event.params.get("data").and_then(|v| v.as_str())
{
if let Ok(bytes) = base64::Engine::decode(
&base64::engine::general_purpose::STANDARD,
data,
) {
recording::recording_add_frame(
&mut self.recording_state,
&bytes,
);
}
}
}
if let Some(sid) =
event.params.get("sessionId").and_then(|v| v.as_i64())
{
@@ -2574,132 +2585,111 @@ async fn handle_recording_start(cmd: &Value, state: &mut DaemonState) -> Result<
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty());
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
let old_session_id = mgr.active_session_id()?.to_string();
let (client, new_session_id) = {
let mgr = state.browser.as_mut().ok_or("Browser not launched")?;
let old_session_id = mgr.active_session_id()?.to_string();
// Capture current URL if no URL specified
let nav_url = if let Some(u) = recording_url {
u.to_string()
} else {
mgr.get_url()
// Capture current URL if no URL specified
let nav_url = if let Some(u) = recording_url {
u.to_string()
} else {
mgr.get_url()
.await
.unwrap_or_else(|_| "about:blank".to_string())
};
// Capture current cookies
let cookies_result = mgr
.client
.send_command_no_params("Network.getAllCookies", Some(&old_session_id))
.await
.unwrap_or_else(|_| "about:blank".to_string())
};
.ok();
// Capture current cookies
let cookies_result = mgr
.client
.send_command_no_params("Network.getAllCookies", Some(&old_session_id))
.await
.ok();
// Create new browser context
let ctx_result = mgr
.client
.send_command_no_params("Target.createBrowserContext", None)
.await?;
let context_id = ctx_result
.get("browserContextId")
.and_then(|v| v.as_str())
.ok_or("Failed to get browserContextId")?
.to_string();
// Create new browser context
let ctx_result = mgr
.client
.send_command_no_params("Target.createBrowserContext", None)
.await?;
let context_id = ctx_result
.get("browserContextId")
.and_then(|v| v.as_str())
.ok_or("Failed to get browserContextId")?
.to_string();
// Create page in new context
let create_result: CreateTargetResult = mgr
.client
.send_command_typed(
"Target.createTarget",
&json!({ "url": "about:blank", "browserContextId": context_id }),
None,
)
.await?;
// Create page in new context
let create_result: CreateTargetResult = mgr
.client
.send_command_typed(
"Target.createTarget",
&json!({ "url": "about:blank", "browserContextId": context_id }),
None,
)
.await?;
let attach_result: AttachToTargetResult = mgr
.client
.send_command_typed(
"Target.attachToTarget",
&AttachToTargetParams {
target_id: create_result.target_id.clone(),
flatten: true,
},
None,
)
.await?;
let attach_result: AttachToTargetResult = mgr
.client
.send_command_typed(
"Target.attachToTarget",
&AttachToTargetParams {
target_id: create_result.target_id.clone(),
flatten: true,
},
None,
)
.await?;
let new_session_id = attach_result.session_id.clone();
mgr.enable_domains_pub(&new_session_id).await?;
let new_session_id = attach_result.session_id.clone();
mgr.enable_domains_pub(&new_session_id).await?;
// Transfer cookies to new context
if let Some(ref cr) = cookies_result {
if let Some(cookie_arr) = cr.get("cookies").and_then(|v| v.as_array()) {
if !cookie_arr.is_empty() {
let _ = mgr
.client
.send_command(
"Network.setCookies",
Some(json!({ "cookies": cookie_arr })),
Some(&new_session_id),
)
.await;
// Transfer cookies to new context
if let Some(ref cr) = cookies_result {
if let Some(cookie_arr) = cr.get("cookies").and_then(|v| v.as_array()) {
if !cookie_arr.is_empty() {
let _ = mgr
.client
.send_command(
"Network.setCookies",
Some(json!({ "cookies": cookie_arr })),
Some(&new_session_id),
)
.await;
}
}
}
}
// Add page and switch to it
mgr.add_page(super::browser::PageInfo {
target_id: create_result.target_id,
session_id: new_session_id.clone(),
url: nav_url.clone(),
title: String::new(),
target_type: "page".to_string(),
});
// Add page and switch to it
mgr.add_page(super::browser::PageInfo {
target_id: create_result.target_id,
session_id: new_session_id.clone(),
url: nav_url.clone(),
title: String::new(),
target_type: "page".to_string(),
});
// Navigate to URL
if nav_url != "about:blank" {
let _ = mgr
.client
.send_command(
"Page.navigate",
Some(json!({ "url": nav_url })),
Some(&new_session_id),
)
.await;
tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
}
// Navigate to URL
if nav_url != "about:blank" {
let _ = mgr
.client
.send_command(
"Page.navigate",
Some(json!({ "url": nav_url })),
Some(&new_session_id),
)
.await;
tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
}
(mgr.client.clone(), new_session_id)
};
let result = recording::recording_start(&mut state.recording_state, path)?;
// Start screencast on new page
stream::start_screencast(&mgr.client, &new_session_id, "jpeg", 80, 1280, 720).await?;
state.screencasting = true;
state.start_recording_task(client, new_session_id).await?;
Ok(result)
}
async fn handle_recording_stop(state: &mut DaemonState) -> Result<Value, String> {
// Stop screencast
if state.screencasting {
if let Some(ref browser) = state.browser {
if let Ok(session_id) = browser.active_session_id() {
let _ = stream::stop_screencast(&browser.client, session_id).await;
}
}
state.screencasting = false;
}
// Drain remaining frames before stopping
let (ack_ids, _, _, _) = state.drain_cdp_events();
if !ack_ids.is_empty() {
if let Some(ref browser) = state.browser {
if let Ok(session_id) = browser.active_session_id() {
for ack_sid in ack_ids {
let _ =
stream::ack_screencast_frame(&browser.client, session_id, ack_sid).await;
}
}
}
}
state.stop_recording_task().await?;
recording::recording_stop(&mut state.recording_state)
}
@@ -2709,22 +2699,14 @@ async fn handle_recording_restart(cmd: &Value, state: &mut DaemonState) -> Resul
.and_then(|v| v.as_str())
.ok_or("Missing 'path' parameter")?;
// Stop screencast, restart recording, start screencast again
if state.screencasting {
if let Some(ref browser) = state.browser {
if let Ok(session_id) = browser.active_session_id() {
let _ = stream::stop_screencast(&browser.client, session_id).await;
}
}
state.screencasting = false;
}
let _ = state.stop_recording_task().await;
let result = recording::recording_restart(&mut state.recording_state, path)?;
if let Some(ref browser) = state.browser {
let session_id = browser.active_session_id()?.to_string();
stream::start_screencast(&browser.client, &session_id, "jpeg", 80, 1280, 720).await?;
state.screencasting = true;
state
.start_recording_task(browser.client.clone(), session_id)
.await?;
}
Ok(result)
@@ -4351,8 +4333,9 @@ async fn handle_video_start(cmd: &Value, state: &mut DaemonState) -> Result<Valu
let session_id = mgr.active_session_id()?.to_string();
recording::recording_start(&mut state.recording_state, path)?;
stream::start_screencast(&mgr.client, &session_id, "jpeg", 80, 1280, 720).await?;
state.screencasting = true;
state
.start_recording_task(mgr.client.clone(), session_id)
.await?;
Ok(json!({
"started": true,
@@ -4368,27 +4351,7 @@ async fn handle_video_stop(state: &mut DaemonState) -> Result<Value, String> {
}));
}
if state.screencasting {
if let Some(ref browser) = state.browser {
if let Ok(session_id) = browser.active_session_id() {
let _ = stream::stop_screencast(&browser.client, session_id).await;
}
}
state.screencasting = false;
}
let (ack_ids, _, _, _) = state.drain_cdp_events();
if !ack_ids.is_empty() {
if let Some(ref browser) = state.browser {
if let Ok(session_id) = browser.active_session_id() {
for ack_sid in ack_ids {
let _ =
stream::ack_screencast_frame(&browser.client, session_id, ack_sid).await;
}
}
}
}
state.stop_recording_task().await?;
recording::recording_stop(&mut state.recording_state)
}