Stop the source reader when the peer is gone instead of hanging

After a send failed, run_send joined the reader thread, but the reader
only learned the receiver was gone on its next send. An idle growing
capture, or a filter matching nothing, never produced one, so the
process hung -- locally, and as an orphaned remote --send server. The
reader now checks a stop flag every poll, raised once the send loop
ends. The client also kills ssh when a remote-to-local receive fails
locally rather than waiting on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ericandClaude Opus 5.5 committed 2026-11-13 16:12:47 -05:00
1 parent b46d6a936c
commit 3eb779a6aa
3 files changed
+95 -8

No files matched your search

+7
View File
@@ -171,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> {
}; };
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout)); let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
let recv_result = run_recv(source, recv_opts); let recv_result = run_recv(source, recv_opts);
if recv_result.is_err() {
// The remote sender only notices we're gone when its next
// write fails, which on an idle capture may be never. Don't
// wait for that: drop the connection, so its next write
// fails and it stops.
let _ = child.kill();
}
let status = child.wait()?; let status = child.wait()?;
recv_result?; recv_result?;
if !status.success() { if !status.success() {
+60 -1
View File
@@ -9,6 +9,7 @@ use std::fs::File;
use std::io::Write; use std::io::Write;
use std::num::NonZeroUsize; use std::num::NonZeroUsize;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc}; use std::sync::{mpsc, Arc};
use std::time::Duration; use std::time::Duration;
use std::thread; use std::thread;
@@ -80,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
}; };
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH); let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
// Raised once the send loop below is done, for whatever reason. After a
// clean EOF or a source error the reader has already returned and never
// looks; after a failed send it is how the reader learns to quit instead
// of polling the source until it next has something to send, which on an
// idle capture or a filter matching nothing may be never.
let stop = Arc::new(AtomicBool::new(false));
let reader = thread::spawn({ let reader = thread::spawn({
let source_path = opts.source_path.clone(); let source_path = opts.source_path.clone();
let is_partial = opts.is_partial; let is_partial = opts.is_partial;
let final_source_path = opts.final_source_path.clone(); let final_source_path = opts.final_source_path.clone();
let log = opts.log.clone(); let log = opts.log.clone();
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log) let stop = stop.clone();
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
}); });
if opts.compress { if opts.compress {
@@ -140,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
} }
} }
} }
stop.store(true, Ordering::Relaxed);
let reader_result = reader.join().unwrap(); let reader_result = reader.join().unwrap();
let compressor_result = compressor.join().unwrap(); let compressor_result = compressor.join().unwrap();
if let Some(e) = send_err { if let Some(e) = send_err {
@@ -175,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
} }
} }
} }
stop.store(true, Ordering::Relaxed);
let reader_result = reader.join().unwrap(); let reader_result = reader.join().unwrap();
if let Some(e) = send_err { if let Some(e) = send_err {
return Err(e); return Err(e);
@@ -619,4 +629,53 @@ mod tests {
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected); assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
assert!(expected.len() < source.len()); assert!(expected.len() < source.len());
} }
/// A peer that has gone away: every send fails.
struct GoneSink;
impl MessageSink for GoneSink {
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
bail!("peer gone")
}
fn send_finalize(&mut self) -> Result<()> {
bail!("peer gone")
}
fn send_error(&mut self, _msg: &str) -> Result<()> {
bail!("peer gone")
}
}
#[test]
fn a_failed_send_returns_even_though_the_source_never_finishes() {
// The reader used to learn the peer was gone only on its next send.
// A growing capture that stops growing (and is never renamed) never
// gives it one, so run_send waited on the reader forever.
for compress in [false, true] {
let dir = tempdir().unwrap();
let partial = dir.path().join("cap.pcap.partial");
let mut stream = eth_header_bytes();
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
std::fs::write(&partial, &stream).unwrap();
let opts = SendOptions {
source_path: partial.clone(),
is_partial: true,
final_source_path: dir.path().join("cap.pcap"),
compress,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: PacketFilter::match_all(),
log: Arc::new(Logger::none()),
};
let (done_tx, done_rx) = mpsc::channel();
thread::spawn(move || {
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
});
let result = done_rx
.recv_timeout(Duration::from_secs(5))
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
assert!(result.unwrap_err().to_string().contains("peer gone"));
}
}
} }
+28 -7
View File
@@ -4,6 +4,7 @@ use anyhow::{bail, Result};
use std::fs::File; use std::fs::File;
use std::os::unix::fs::FileExt; use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
@@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
/// ///
/// Per the confirmed design: if `source_path` doesn't exist at all when this /// Per the confirmed design: if `source_path` doesn't exist at all when this
/// is called, that's a hard error (no polling for creation). /// is called, that's a hard error (no polling for creation).
///
/// `stop` is how the consumer says it has given up (its peer is gone). The
/// loop checks it every poll, because otherwise it would only find out on
/// its next send -- and a growing source that stops growing, or a filter
/// that matches nothing, may never produce one, leaving whoever joins this
/// thread waiting forever.
pub fn run_tail( pub fn run_tail(
source_path: PathBuf, source_path: PathBuf,
is_partial: bool, is_partial: bool,
@@ -65,6 +72,7 @@ pub fn run_tail(
mut chunker: Box<dyn Chunker>, mut chunker: Box<dyn Chunker>,
events: impl EventSink, events: impl EventSink,
log: Arc<Logger>, log: Arc<Logger>,
stop: Arc<AtomicBool>,
) -> Result<()> { ) -> Result<()> {
if !source_path.exists() { if !source_path.exists() {
log.log(format_args!("tail: source not found: {}", source_path.display())); log.log(format_args!("tail: source not found: {}", source_path.display()));
@@ -75,7 +83,7 @@ pub fn run_tail(
source_path.display(), source_path.display(),
final_path.display() final_path.display()
)); ));
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log); let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
match &result { match &result {
Ok(()) => { Ok(()) => {
log.log(format_args!("tail: done, sending EOF")); log.log(format_args!("tail: done, sending EOF"));
@@ -96,6 +104,7 @@ fn drain(
chunker: &mut dyn Chunker, chunker: &mut dyn Chunker,
events: &impl EventSink, events: &impl EventSink,
log: &Logger, log: &Logger,
stop: &AtomicBool,
) -> Result<()> { ) -> Result<()> {
let file = File::open(source_path)?; let file = File::open(source_path)?;
let mut complete_signal = !is_partial; let mut complete_signal = !is_partial;
@@ -123,6 +132,9 @@ fn drain(
let mut last_stuck_peek: Option<Vec<u8>> = None; let mut last_stuck_peek: Option<Vec<u8>> = None;
loop { loop {
if stop.load(Ordering::Relaxed) {
bail!("stopped: the receiving side is gone");
}
let file_len = file.metadata()?.len(); let file_len = file.metadata()?.len();
// Whether this iteration did real work -- if so, loop again // Whether this iteration did real work -- if so, loop again
// immediately rather than sleeping, so a large already-available // immediately rather than sleeping, so a large already-available
@@ -316,6 +328,10 @@ mod tests {
(data, got_eof) (data, got_eof)
} }
fn no_stop() -> Arc<AtomicBool> {
Arc::new(AtomicBool::new(false))
}
fn header_bytes(magic_le: u32) -> Vec<u8> { fn header_bytes(magic_le: u32) -> Vec<u8> {
let mut h = vec![0u8; GLOBAL_HEADER_LEN]; let mut h = vec![0u8; GLOBAL_HEADER_LEN];
h[0..4].copy_from_slice(&magic_le.to_le_bytes()); h[0..4].copy_from_slice(&magic_le.to_le_bytes());
@@ -340,7 +356,7 @@ mod tests {
std::fs::write(&path, b"hello world").unwrap(); std::fs::write(&path, b"hello world").unwrap();
let (tx, rx) = mpsc::channel(); let (tx, rx) = mpsc::channel();
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap(); run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
let (data, eof) = collect(rx); let (data, eof) = collect(rx);
assert_eq!(data, b"hello world"); assert_eq!(data, b"hello world");
assert!(eof); assert!(eof);
@@ -352,7 +368,7 @@ mod tests {
let path = dir.path().join("nope.pcap.partial"); let path = dir.path().join("nope.pcap.partial");
let final_path = dir.path().join("nope.pcap"); let final_path = dir.path().join("nope.pcap");
let (tx, _rx) = mpsc::channel(); let (tx, _rx) = mpsc::channel();
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err(); let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
assert!(err.to_string().contains("not found")); assert!(err.to_string().contains("not found"));
} }
@@ -369,7 +385,7 @@ mod tests {
let handle = std::thread::spawn({ let handle = std::thread::spawn({
let partial_path = partial_path.clone(); let partial_path = partial_path.clone();
let final_path = final_path.clone(); let final_path = final_path.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())) move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
}); });
std::thread::sleep(Duration::from_millis(60)); std::thread::sleep(Duration::from_millis(60));
@@ -399,7 +415,7 @@ mod tests {
let partial_path = partial_path.clone(); let partial_path = partial_path.clone();
let final_path = final_path.clone(); let final_path = final_path.clone();
let log = log.clone(); let log = log.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log) move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
}); });
std::thread::sleep(Duration::from_millis(60)); std::thread::sleep(Duration::from_millis(60));
@@ -426,7 +442,7 @@ mod tests {
let handle = std::thread::spawn({ let handle = std::thread::spawn({
let partial_path = partial_path.clone(); let partial_path = partial_path.clone();
let final_path = final_path.clone(); let final_path = final_path.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())) move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
}); });
std::thread::sleep(Duration::from_millis(60)); std::thread::sleep(Duration::from_millis(60));
@@ -457,7 +473,7 @@ mod tests {
let handle = std::thread::spawn({ let handle = std::thread::spawn({
let partial_path = partial_path.clone(); let partial_path = partial_path.clone();
let final_path = final_path.clone(); let final_path = final_path.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())) move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
}); });
// Let the reader fully drain the initial 10 bytes. // Let the reader fully drain the initial 10 bytes.
@@ -522,6 +538,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx, tx,
Arc::new(Logger::none()), Arc::new(Logger::none()),
no_stop(),
) )
} }
}); });
@@ -569,6 +586,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx, tx,
Arc::new(Logger::none()), Arc::new(Logger::none()),
no_stop(),
) )
} }
}); });
@@ -624,6 +642,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx, tx,
Arc::new(Logger::none()), Arc::new(Logger::none()),
no_stop(),
) )
} }
}); });
@@ -668,6 +687,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx, tx,
Arc::new(Logger::none()), Arc::new(Logger::none()),
no_stop(),
) )
} }
}); });
@@ -706,6 +726,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)), Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
tx, tx,
Arc::new(Logger::none()), Arc::new(Logger::none()),
no_stop(),
); );
try_collect(rx); try_collect(rx);
assert!(result.is_err()); assert!(result.is_err());