Fix daemon hang on Linux caused by waitpid(-1) race condition (#1098)
* Fix daemon hang on Linux caused by waitpid(-1) race condition Fixes #1035 The SIGCHLD handler added in v0.22.3 called `waitpid(-1, WNOHANG)` to reap zombie Chrome processes. This races with Rust's `Child::try_wait()` / `Child::wait()` because `waitpid(-1)` reaps *any* child in the process, stealing the exit status before the `Child` handle can collect it. The result is `ECHILD` errors in `BrowserManager::has_process_exited()` and `ChromeProcess::kill()`, leaving the daemon in a broken state that manifests as indefinite hangs on Linux servers. The fix removes the global SIGCHLD handler and `reap_children()` function entirely. Instead, the existing 500ms drain interval now checks `mgr.has_process_exited()` (which delegates to `Child::try_wait()`) for targeted, race-free crash detection. When Chrome is detected as crashed, the `BrowserManager` is closed and daemon state is reset. ## Changes - Removed `SIGCHLD` signal handler and `reap_children()` from the Unix daemon event loop - Enhanced the drain interval to detect Chrome crashes via `has_process_exited()` and clean up state - Added 3 regression tests: - Static source scan that fails if `waitpid(-1)` is re-introduced in production code - `try_wait()` correctness test for exit detection without a SIGCHLD handler - Kill detection test simulating a Chrome crash * fmt
This commit is contained in:
+106
-26
@@ -156,12 +156,6 @@ async fn run_socket_server(
|
|||||||
let (reset_tx, mut reset_rx) = mpsc::channel::<()>(64);
|
let (reset_tx, mut reset_rx) = mpsc::channel::<()>(64);
|
||||||
let reset_tx = idle_timeout_ms.map(|_| Arc::new(reset_tx));
|
let reset_tx = idle_timeout_ms.map(|_| Arc::new(reset_tx));
|
||||||
|
|
||||||
// Listen for SIGCHLD to reap zombie child processes (e.g. crashed Chrome).
|
|
||||||
// Without this, a crashed Chrome becomes <defunct> and is never reaped until
|
|
||||||
// the daemon exits.
|
|
||||||
let mut sigchld = signal::unix::signal(signal::unix::SignalKind::child())
|
|
||||||
.map_err(|e| format!("Failed to install SIGCHLD handler: {}", e))?;
|
|
||||||
|
|
||||||
let mut drain_interval = tokio::time::interval(Duration::from_millis(500));
|
let mut drain_interval = tokio::time::interval(Duration::from_millis(500));
|
||||||
drain_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
drain_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||||
|
|
||||||
@@ -185,13 +179,17 @@ async fn run_socket_server(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = sigchld.recv() => {
|
|
||||||
reap_children();
|
|
||||||
}
|
|
||||||
_ = drain_interval.tick() => {
|
_ = drain_interval.tick() => {
|
||||||
let mut s = state.lock().await;
|
let mut s = state.lock().await;
|
||||||
if s.browser.is_some() {
|
if let Some(ref mut mgr) = s.browser {
|
||||||
s.drain_cdp_events_background().await;
|
if mgr.has_process_exited() {
|
||||||
|
let _ = mgr.close().await;
|
||||||
|
s.browser = None;
|
||||||
|
s.screencasting = false;
|
||||||
|
s.update_stream_client().await;
|
||||||
|
} else {
|
||||||
|
s.drain_cdp_events_background().await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = async {
|
_ = async {
|
||||||
@@ -223,17 +221,6 @@ async fn run_socket_server(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reap all zombie child processes by calling waitpid(-1, WNOHANG) in a loop.
|
|
||||||
#[cfg(unix)]
|
|
||||||
fn reap_children() {
|
|
||||||
loop {
|
|
||||||
let result = unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) };
|
|
||||||
if result <= 0 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(windows)]
|
#[cfg(windows)]
|
||||||
async fn run_socket_server(
|
async fn run_socket_server(
|
||||||
socket_path: &PathBuf,
|
socket_path: &PathBuf,
|
||||||
@@ -478,18 +465,111 @@ fn get_port_for_session(session: &str) -> u16 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[cfg(windows)]
|
|
||||||
mod tests {
|
mod tests {
|
||||||
|
#[allow(unused_imports)]
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
#[test]
|
#[test]
|
||||||
fn test_port_matches_client_algorithm() {
|
fn test_port_matches_client_algorithm() {
|
||||||
// These values are computed by the identical djb2 implementation in
|
|
||||||
// connection.rs. Both sides must agree on the port for the daemon to
|
|
||||||
// start successfully.
|
|
||||||
assert_eq!(get_port_for_session("default"), 50838);
|
assert_eq!(get_port_for_session("default"), 50838);
|
||||||
assert_eq!(get_port_for_session("my-session"), 63105);
|
assert_eq!(get_port_for_session("my-session"), 63105);
|
||||||
assert_eq!(get_port_for_session("work"), 51184);
|
assert_eq!(get_port_for_session("work"), 51184);
|
||||||
assert_eq!(get_port_for_session(""), 49152);
|
assert_eq!(get_port_for_session(""), 49152);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Guard against re-introducing `waitpid(-1)` in daemon code.
|
||||||
|
///
|
||||||
|
/// Issue #1035: a SIGCHLD handler that called `waitpid(-1, WNOHANG)` was
|
||||||
|
/// added in v0.22.3 to reap zombie Chrome processes. This races with
|
||||||
|
/// Rust's `Child::try_wait()` / `Child::wait()` because `waitpid(-1)`
|
||||||
|
/// reaps *any* child, stealing the exit status before Rust can collect
|
||||||
|
/// it. The result is ECHILD errors in `BrowserManager::has_process_exited()`
|
||||||
|
/// and `ChromeProcess::kill()`, which can leave the daemon in a broken
|
||||||
|
/// state or cause hangs on certain Linux configurations.
|
||||||
|
///
|
||||||
|
/// The fix uses the existing 500ms drain interval to call
|
||||||
|
/// `has_process_exited()` (which delegates to `Child::try_wait()`)
|
||||||
|
/// for targeted, race-free zombie detection.
|
||||||
|
#[test]
|
||||||
|
fn test_no_waitpid_minus_one_in_daemon() {
|
||||||
|
let source = include_str!("daemon.rs");
|
||||||
|
// Only check production code (everything before `#[cfg(test)]`)
|
||||||
|
let production_code = source.split("#[cfg(test)]").next().unwrap_or(source);
|
||||||
|
assert!(
|
||||||
|
!production_code.contains("waitpid(-1"),
|
||||||
|
"daemon.rs production code must not call waitpid(-1, ...). \
|
||||||
|
Use Child::try_wait() via has_process_exited() instead. \
|
||||||
|
See issue #1035."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify that `Child::try_wait()` correctly detects a crashed child
|
||||||
|
/// without needing a global SIGCHLD handler or `waitpid(-1)`.
|
||||||
|
/// This is what `has_process_exited()` uses in the fixed code.
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn test_child_try_wait_detects_exit_without_sigchld_handler() {
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
|
||||||
|
let mut child = Command::new("/bin/sh")
|
||||||
|
.args(["-c", "exit 42"])
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::null())
|
||||||
|
.spawn()
|
||||||
|
.expect("failed to spawn child");
|
||||||
|
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||||
|
|
||||||
|
match child.try_wait() {
|
||||||
|
Ok(Some(status)) => {
|
||||||
|
assert!(
|
||||||
|
!status.success(),
|
||||||
|
"child exited with code 42, should not be success"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(None) => panic!("try_wait() returned None but child should have exited"),
|
||||||
|
Err(e) => panic!("try_wait() should succeed without waitpid(-1): {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify that `ChromeProcess::has_exited()` (which uses `Child::try_wait()`)
|
||||||
|
/// correctly detects a killed child, the same way the drain interval does
|
||||||
|
/// in the fixed daemon code. This ensures crash detection works without
|
||||||
|
/// a SIGCHLD handler.
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn test_has_exited_detects_killed_process() {
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
|
||||||
|
let mut child = Command::new("/bin/sh")
|
||||||
|
.args(["-c", "sleep 60"])
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::null())
|
||||||
|
.spawn()
|
||||||
|
.expect("failed to spawn child");
|
||||||
|
|
||||||
|
// Process should be running
|
||||||
|
match child.try_wait() {
|
||||||
|
Ok(None) => {} // expected
|
||||||
|
other => panic!("expected Ok(None) for running process, got {:?}", other),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kill it (simulates Chrome crash)
|
||||||
|
child.kill().expect("failed to kill child");
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||||
|
|
||||||
|
// try_wait should detect the exit
|
||||||
|
match child.try_wait() {
|
||||||
|
Ok(Some(_)) => {} // expected: detected the crash
|
||||||
|
other => panic!(
|
||||||
|
"expected Ok(Some(_)) after kill, got {:?}. \
|
||||||
|
Crash detection via try_wait() must work for the drain \
|
||||||
|
interval fix (issue #1035) to function correctly.",
|
||||||
|
other
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user