Stop the source reader when the peer is gone instead of hanging
After a send failed, run_send joined the reader thread, but the reader only learned the receiver was gone on its next send. An idle growing capture, or a filter matching nothing, never produced one, so the process hung -- locally, and as an orphaned remote --send server. The reader now checks a stop flag every poll, raised once the send loop ends. The client also kills ssh when a remote-to-local receive fails locally rather than waiting on it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b46d6a936c
commit
3eb779a6aa
3 files changed
+95
-8
No files matched your search
+60
-1
@@ -9,6 +9,7 @@ use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
@@ -80,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
};
|
||||
|
||||
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
||||
// Raised once the send loop below is done, for whatever reason. After a
|
||||
// clean EOF or a source error the reader has already returned and never
|
||||
// looks; after a failed send it is how the reader learns to quit instead
|
||||
// of polling the source until it next has something to send, which on an
|
||||
// idle capture or a filter matching nothing may be never.
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let reader = thread::spawn({
|
||||
let source_path = opts.source_path.clone();
|
||||
let is_partial = opts.is_partial;
|
||||
let final_source_path = opts.final_source_path.clone();
|
||||
let log = opts.log.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
||||
let stop = stop.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
||||
});
|
||||
|
||||
if opts.compress {
|
||||
@@ -140,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
let compressor_result = compressor.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
@@ -175,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
return Err(e);
|
||||
@@ -619,4 +629,53 @@ mod tests {
|
||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||
assert!(expected.len() < source.len());
|
||||
}
|
||||
|
||||
/// A peer that has gone away: every send fails.
|
||||
struct GoneSink;
|
||||
|
||||
impl MessageSink for GoneSink {
|
||||
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_finalize(&mut self) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
||||
// The reader used to learn the peer was gone only on its next send.
|
||||
// A growing capture that stops growing (and is never renamed) never
|
||||
// gives it one, so run_send waited on the reader forever.
|
||||
for compress in [false, true] {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
||||
std::fs::write(&partial, &stream).unwrap();
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: dir.path().join("cap.pcap"),
|
||||
compress,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
||||
});
|
||||
let result = done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
||||
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user