diff --git a/src/client.rs b/src/client.rs index 7081c87..54b8422 100644 --- a/src/client.rs +++ b/src/client.rs @@ -171,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> { }; let source: Box = Box::new(WireSource(stdout)); let recv_result = run_recv(source, recv_opts); + if recv_result.is_err() { + // The remote sender only notices we're gone when its next + // write fails, which on an idle capture may be never. Don't + // wait for that: drop the connection, so its next write + // fails and it stops. + let _ = child.kill(); + } let status = child.wait()?; recv_result?; if !status.success() { diff --git a/src/pipeline.rs b/src/pipeline.rs index 97560eb..a505145 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -9,6 +9,7 @@ use std::fs::File; use std::io::Write; use std::num::NonZeroUsize; use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{mpsc, Arc}; use std::time::Duration; use std::thread; @@ -80,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box) -> Result<()> }; let (tail_tx, tail_rx) = mpsc::sync_channel::(PIPELINE_DEPTH); + // Raised once the send loop below is done, for whatever reason. After a + // clean EOF or a source error the reader has already returned and never + // looks; after a failed send it is how the reader learns to quit instead + // of polling the source until it next has something to send, which on an + // idle capture or a filter matching nothing may be never. + let stop = Arc::new(AtomicBool::new(false)); let reader = thread::spawn({ let source_path = opts.source_path.clone(); let is_partial = opts.is_partial; let final_source_path = opts.final_source_path.clone(); let log = opts.log.clone(); - move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log) + let stop = stop.clone(); + move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop) }); if opts.compress { @@ -140,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box) -> Result<()> } } } + stop.store(true, Ordering::Relaxed); let reader_result = reader.join().unwrap(); let compressor_result = compressor.join().unwrap(); if let Some(e) = send_err { @@ -175,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box) -> Result<()> } } } + stop.store(true, Ordering::Relaxed); let reader_result = reader.join().unwrap(); if let Some(e) = send_err { return Err(e); @@ -619,4 +629,53 @@ mod tests { assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected); assert!(expected.len() < source.len()); } + + /// A peer that has gone away: every send fails. + struct GoneSink; + + impl MessageSink for GoneSink { + fn send_data(&mut self, _payload: &[u8]) -> Result<()> { + bail!("peer gone") + } + fn send_finalize(&mut self) -> Result<()> { + bail!("peer gone") + } + fn send_error(&mut self, _msg: &str) -> Result<()> { + bail!("peer gone") + } + } + + #[test] + fn a_failed_send_returns_even_though_the_source_never_finishes() { + // The reader used to learn the peer was gone only on its next send. + // A growing capture that stops growing (and is never renamed) never + // gives it one, so run_send waited on the reader forever. + for compress in [false, true] { + let dir = tempdir().unwrap(); + let partial = dir.path().join("cap.pcap.partial"); + let mut stream = eth_header_bytes(); + stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60)); + std::fs::write(&partial, &stream).unwrap(); + + let opts = SendOptions { + source_path: partial.clone(), + is_partial: true, + final_source_path: dir.path().join("cap.pcap"), + compress, + chunk_target: crate::chunker::DEFAULT_TARGET_BYTES, + flush_after: crate::chunker::DEFAULT_FLUSH, + level: crate::zstd_frame::DEFAULT_LEVEL, + filter: PacketFilter::match_all(), + log: Arc::new(Logger::none()), + }; + let (done_tx, done_rx) = mpsc::channel(); + thread::spawn(move || { + let _ = done_tx.send(run_send(opts, Box::new(GoneSink))); + }); + let result = done_rx + .recv_timeout(Duration::from_secs(5)) + .unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})")); + assert!(result.unwrap_err().to_string().contains("peer gone")); + } + } } diff --git a/src/tail.rs b/src/tail.rs index 234fbba..d324511 100644 --- a/src/tail.rs +++ b/src/tail.rs @@ -4,6 +4,7 @@ use anyhow::{bail, Result}; use std::fs::File; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender { /// /// Per the confirmed design: if `source_path` doesn't exist at all when this /// is called, that's a hard error (no polling for creation). +/// +/// `stop` is how the consumer says it has given up (its peer is gone). The +/// loop checks it every poll, because otherwise it would only find out on +/// its next send -- and a growing source that stops growing, or a filter +/// that matches nothing, may never produce one, leaving whoever joins this +/// thread waiting forever. pub fn run_tail( source_path: PathBuf, is_partial: bool, @@ -65,6 +72,7 @@ pub fn run_tail( mut chunker: Box, events: impl EventSink, log: Arc, + stop: Arc, ) -> Result<()> { if !source_path.exists() { log.log(format_args!("tail: source not found: {}", source_path.display())); @@ -75,7 +83,7 @@ pub fn run_tail( source_path.display(), final_path.display() )); - let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log); + let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop); match &result { Ok(()) => { log.log(format_args!("tail: done, sending EOF")); @@ -96,6 +104,7 @@ fn drain( chunker: &mut dyn Chunker, events: &impl EventSink, log: &Logger, + stop: &AtomicBool, ) -> Result<()> { let file = File::open(source_path)?; let mut complete_signal = !is_partial; @@ -123,6 +132,9 @@ fn drain( let mut last_stuck_peek: Option> = None; loop { + if stop.load(Ordering::Relaxed) { + bail!("stopped: the receiving side is gone"); + } let file_len = file.metadata()?.len(); // Whether this iteration did real work -- if so, loop again // immediately rather than sleeping, so a large already-available @@ -316,6 +328,10 @@ mod tests { (data, got_eof) } + fn no_stop() -> Arc { + Arc::new(AtomicBool::new(false)) + } + fn header_bytes(magic_le: u32) -> Vec { let mut h = vec![0u8; GLOBAL_HEADER_LEN]; h[0..4].copy_from_slice(&magic_le.to_le_bytes()); @@ -340,7 +356,7 @@ mod tests { std::fs::write(&path, b"hello world").unwrap(); let (tx, rx) = mpsc::channel(); - run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap(); + run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap(); let (data, eof) = collect(rx); assert_eq!(data, b"hello world"); assert!(eof); @@ -352,7 +368,7 @@ mod tests { let path = dir.path().join("nope.pcap.partial"); let final_path = dir.path().join("nope.pcap"); let (tx, _rx) = mpsc::channel(); - let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err(); + let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err(); assert!(err.to_string().contains("not found")); } @@ -369,7 +385,7 @@ mod tests { let handle = std::thread::spawn({ let partial_path = partial_path.clone(); let final_path = final_path.clone(); - move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())) + move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()) }); std::thread::sleep(Duration::from_millis(60)); @@ -399,7 +415,7 @@ mod tests { let partial_path = partial_path.clone(); let final_path = final_path.clone(); let log = log.clone(); - move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log) + move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop()) }); std::thread::sleep(Duration::from_millis(60)); @@ -426,7 +442,7 @@ mod tests { let handle = std::thread::spawn({ let partial_path = partial_path.clone(); let final_path = final_path.clone(); - move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())) + move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()) }); std::thread::sleep(Duration::from_millis(60)); @@ -457,7 +473,7 @@ mod tests { let handle = std::thread::spawn({ let partial_path = partial_path.clone(); let final_path = final_path.clone(); - move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())) + move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()) }); // Let the reader fully drain the initial 10 bytes. @@ -522,6 +538,7 @@ mod tests { Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), tx, Arc::new(Logger::none()), + no_stop(), ) } }); @@ -569,6 +586,7 @@ mod tests { Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), tx, Arc::new(Logger::none()), + no_stop(), ) } }); @@ -624,6 +642,7 @@ mod tests { Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), tx, Arc::new(Logger::none()), + no_stop(), ) } }); @@ -668,6 +687,7 @@ mod tests { Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)), tx, Arc::new(Logger::none()), + no_stop(), ) } }); @@ -706,6 +726,7 @@ mod tests { Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)), tx, Arc::new(Logger::none()), + no_stop(), ); try_collect(rx); assert!(result.is_err());