Stop the source reader when the peer is gone instead of hanging
After a send failed, run_send joined the reader thread, but the reader only learned the receiver was gone on its next send. An idle growing capture, or a filter matching nothing, never produced one, so the process hung -- locally, and as an orphaned remote --send server. The reader now checks a stop flag every poll, raised once the send loop ends. The client also kills ssh when a remote-to-local receive fails locally rather than waiting on it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b46d6a936c
commit
3eb779a6aa
3 files changed
+95
-8
No files matched your search
@@ -171,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
};
|
||||
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
||||
let recv_result = run_recv(source, recv_opts);
|
||||
if recv_result.is_err() {
|
||||
// The remote sender only notices we're gone when its next
|
||||
// write fails, which on an idle capture may be never. Don't
|
||||
// wait for that: drop the connection, so its next write
|
||||
// fails and it stops.
|
||||
let _ = child.kill();
|
||||
}
|
||||
let status = child.wait()?;
|
||||
recv_result?;
|
||||
if !status.success() {
|
||||
|
||||
+60
-1
@@ -9,6 +9,7 @@ use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
@@ -80,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
};
|
||||
|
||||
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
||||
// Raised once the send loop below is done, for whatever reason. After a
|
||||
// clean EOF or a source error the reader has already returned and never
|
||||
// looks; after a failed send it is how the reader learns to quit instead
|
||||
// of polling the source until it next has something to send, which on an
|
||||
// idle capture or a filter matching nothing may be never.
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let reader = thread::spawn({
|
||||
let source_path = opts.source_path.clone();
|
||||
let is_partial = opts.is_partial;
|
||||
let final_source_path = opts.final_source_path.clone();
|
||||
let log = opts.log.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
||||
let stop = stop.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
||||
});
|
||||
|
||||
if opts.compress {
|
||||
@@ -140,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
let compressor_result = compressor.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
@@ -175,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
return Err(e);
|
||||
@@ -619,4 +629,53 @@ mod tests {
|
||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||
assert!(expected.len() < source.len());
|
||||
}
|
||||
|
||||
/// A peer that has gone away: every send fails.
|
||||
struct GoneSink;
|
||||
|
||||
impl MessageSink for GoneSink {
|
||||
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_finalize(&mut self) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
||||
// The reader used to learn the peer was gone only on its next send.
|
||||
// A growing capture that stops growing (and is never renamed) never
|
||||
// gives it one, so run_send waited on the reader forever.
|
||||
for compress in [false, true] {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
||||
std::fs::write(&partial, &stream).unwrap();
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: dir.path().join("cap.pcap"),
|
||||
compress,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
||||
});
|
||||
let result = done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
||||
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+28
-7
@@ -4,6 +4,7 @@ use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
|
||||
///
|
||||
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
||||
/// is called, that's a hard error (no polling for creation).
|
||||
///
|
||||
/// `stop` is how the consumer says it has given up (its peer is gone). The
|
||||
/// loop checks it every poll, because otherwise it would only find out on
|
||||
/// its next send -- and a growing source that stops growing, or a filter
|
||||
/// that matches nothing, may never produce one, leaving whoever joins this
|
||||
/// thread waiting forever.
|
||||
pub fn run_tail(
|
||||
source_path: PathBuf,
|
||||
is_partial: bool,
|
||||
@@ -65,6 +72,7 @@ pub fn run_tail(
|
||||
mut chunker: Box<dyn Chunker>,
|
||||
events: impl EventSink,
|
||||
log: Arc<Logger>,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> Result<()> {
|
||||
if !source_path.exists() {
|
||||
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
||||
@@ -75,7 +83,7 @@ pub fn run_tail(
|
||||
source_path.display(),
|
||||
final_path.display()
|
||||
));
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log);
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
log.log(format_args!("tail: done, sending EOF"));
|
||||
@@ -96,6 +104,7 @@ fn drain(
|
||||
chunker: &mut dyn Chunker,
|
||||
events: &impl EventSink,
|
||||
log: &Logger,
|
||||
stop: &AtomicBool,
|
||||
) -> Result<()> {
|
||||
let file = File::open(source_path)?;
|
||||
let mut complete_signal = !is_partial;
|
||||
@@ -123,6 +132,9 @@ fn drain(
|
||||
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
||||
|
||||
loop {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
bail!("stopped: the receiving side is gone");
|
||||
}
|
||||
let file_len = file.metadata()?.len();
|
||||
// Whether this iteration did real work -- if so, loop again
|
||||
// immediately rather than sleeping, so a large already-available
|
||||
@@ -316,6 +328,10 @@ mod tests {
|
||||
(data, got_eof)
|
||||
}
|
||||
|
||||
fn no_stop() -> Arc<AtomicBool> {
|
||||
Arc::new(AtomicBool::new(false))
|
||||
}
|
||||
|
||||
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
||||
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
||||
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
||||
@@ -340,7 +356,7 @@ mod tests {
|
||||
std::fs::write(&path, b"hello world").unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
|
||||
let (data, eof) = collect(rx);
|
||||
assert_eq!(data, b"hello world");
|
||||
assert!(eof);
|
||||
@@ -352,7 +368,7 @@ mod tests {
|
||||
let path = dir.path().join("nope.pcap.partial");
|
||||
let final_path = dir.path().join("nope.pcap");
|
||||
let (tx, _rx) = mpsc::channel();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
|
||||
assert!(err.to_string().contains("not found"));
|
||||
}
|
||||
|
||||
@@ -369,7 +385,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -399,7 +415,7 @@ mod tests {
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
let log = log.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log)
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -426,7 +442,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -457,7 +473,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
// Let the reader fully drain the initial 10 bytes.
|
||||
@@ -522,6 +538,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -569,6 +586,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -624,6 +642,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -668,6 +687,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -706,6 +726,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
);
|
||||
try_collect(rx);
|
||||
assert!(result.is_err());
|
||||
|
||||
Reference in new issue
Block a user