Stop the source reader when the peer is gone instead of hanging
After a send failed, run_send joined the reader thread, but the reader only learned the receiver was gone on its next send. An idle growing capture, or a filter matching nothing, never produced one, so the process hung -- locally, and as an orphaned remote --send server. The reader now checks a stop flag every poll, raised once the send loop ends. The client also kills ssh when a remote-to-local receive fails locally rather than waiting on it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b46d6a936c
commit
3eb779a6aa
3 files changed
+95
-8
No files matched your search
@@ -171,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
|||||||
};
|
};
|
||||||
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
||||||
let recv_result = run_recv(source, recv_opts);
|
let recv_result = run_recv(source, recv_opts);
|
||||||
|
if recv_result.is_err() {
|
||||||
|
// The remote sender only notices we're gone when its next
|
||||||
|
// write fails, which on an idle capture may be never. Don't
|
||||||
|
// wait for that: drop the connection, so its next write
|
||||||
|
// fails and it stops.
|
||||||
|
let _ = child.kill();
|
||||||
|
}
|
||||||
let status = child.wait()?;
|
let status = child.wait()?;
|
||||||
recv_result?;
|
recv_result?;
|
||||||
if !status.success() {
|
if !status.success() {
|
||||||
|
|||||||
+60
-1
@@ -9,6 +9,7 @@ use std::fs::File;
|
|||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::num::NonZeroUsize;
|
use std::num::NonZeroUsize;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::{mpsc, Arc};
|
use std::sync::{mpsc, Arc};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::thread;
|
use std::thread;
|
||||||
@@ -80,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
||||||
|
// Raised once the send loop below is done, for whatever reason. After a
|
||||||
|
// clean EOF or a source error the reader has already returned and never
|
||||||
|
// looks; after a failed send it is how the reader learns to quit instead
|
||||||
|
// of polling the source until it next has something to send, which on an
|
||||||
|
// idle capture or a filter matching nothing may be never.
|
||||||
|
let stop = Arc::new(AtomicBool::new(false));
|
||||||
let reader = thread::spawn({
|
let reader = thread::spawn({
|
||||||
let source_path = opts.source_path.clone();
|
let source_path = opts.source_path.clone();
|
||||||
let is_partial = opts.is_partial;
|
let is_partial = opts.is_partial;
|
||||||
let final_source_path = opts.final_source_path.clone();
|
let final_source_path = opts.final_source_path.clone();
|
||||||
let log = opts.log.clone();
|
let log = opts.log.clone();
|
||||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
let stop = stop.clone();
|
||||||
|
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
||||||
});
|
});
|
||||||
|
|
||||||
if opts.compress {
|
if opts.compress {
|
||||||
@@ -140,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
stop.store(true, Ordering::Relaxed);
|
||||||
let reader_result = reader.join().unwrap();
|
let reader_result = reader.join().unwrap();
|
||||||
let compressor_result = compressor.join().unwrap();
|
let compressor_result = compressor.join().unwrap();
|
||||||
if let Some(e) = send_err {
|
if let Some(e) = send_err {
|
||||||
@@ -175,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
stop.store(true, Ordering::Relaxed);
|
||||||
let reader_result = reader.join().unwrap();
|
let reader_result = reader.join().unwrap();
|
||||||
if let Some(e) = send_err {
|
if let Some(e) = send_err {
|
||||||
return Err(e);
|
return Err(e);
|
||||||
@@ -619,4 +629,53 @@ mod tests {
|
|||||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||||
assert!(expected.len() < source.len());
|
assert!(expected.len() < source.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A peer that has gone away: every send fails.
|
||||||
|
struct GoneSink;
|
||||||
|
|
||||||
|
impl MessageSink for GoneSink {
|
||||||
|
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
||||||
|
bail!("peer gone")
|
||||||
|
}
|
||||||
|
fn send_finalize(&mut self) -> Result<()> {
|
||||||
|
bail!("peer gone")
|
||||||
|
}
|
||||||
|
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
||||||
|
bail!("peer gone")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
||||||
|
// The reader used to learn the peer was gone only on its next send.
|
||||||
|
// A growing capture that stops growing (and is never renamed) never
|
||||||
|
// gives it one, so run_send waited on the reader forever.
|
||||||
|
for compress in [false, true] {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let partial = dir.path().join("cap.pcap.partial");
|
||||||
|
let mut stream = eth_header_bytes();
|
||||||
|
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
||||||
|
std::fs::write(&partial, &stream).unwrap();
|
||||||
|
|
||||||
|
let opts = SendOptions {
|
||||||
|
source_path: partial.clone(),
|
||||||
|
is_partial: true,
|
||||||
|
final_source_path: dir.path().join("cap.pcap"),
|
||||||
|
compress,
|
||||||
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||||
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||||
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||||
|
filter: PacketFilter::match_all(),
|
||||||
|
log: Arc::new(Logger::none()),
|
||||||
|
};
|
||||||
|
let (done_tx, done_rx) = mpsc::channel();
|
||||||
|
thread::spawn(move || {
|
||||||
|
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
||||||
|
});
|
||||||
|
let result = done_rx
|
||||||
|
.recv_timeout(Duration::from_secs(5))
|
||||||
|
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
||||||
|
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
+28
-7
@@ -4,6 +4,7 @@ use anyhow::{bail, Result};
|
|||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::os::unix::fs::FileExt;
|
use std::os::unix::fs::FileExt;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
@@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
|
|||||||
///
|
///
|
||||||
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
||||||
/// is called, that's a hard error (no polling for creation).
|
/// is called, that's a hard error (no polling for creation).
|
||||||
|
///
|
||||||
|
/// `stop` is how the consumer says it has given up (its peer is gone). The
|
||||||
|
/// loop checks it every poll, because otherwise it would only find out on
|
||||||
|
/// its next send -- and a growing source that stops growing, or a filter
|
||||||
|
/// that matches nothing, may never produce one, leaving whoever joins this
|
||||||
|
/// thread waiting forever.
|
||||||
pub fn run_tail(
|
pub fn run_tail(
|
||||||
source_path: PathBuf,
|
source_path: PathBuf,
|
||||||
is_partial: bool,
|
is_partial: bool,
|
||||||
@@ -65,6 +72,7 @@ pub fn run_tail(
|
|||||||
mut chunker: Box<dyn Chunker>,
|
mut chunker: Box<dyn Chunker>,
|
||||||
events: impl EventSink,
|
events: impl EventSink,
|
||||||
log: Arc<Logger>,
|
log: Arc<Logger>,
|
||||||
|
stop: Arc<AtomicBool>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
if !source_path.exists() {
|
if !source_path.exists() {
|
||||||
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
||||||
@@ -75,7 +83,7 @@ pub fn run_tail(
|
|||||||
source_path.display(),
|
source_path.display(),
|
||||||
final_path.display()
|
final_path.display()
|
||||||
));
|
));
|
||||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log);
|
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
|
||||||
match &result {
|
match &result {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
log.log(format_args!("tail: done, sending EOF"));
|
log.log(format_args!("tail: done, sending EOF"));
|
||||||
@@ -96,6 +104,7 @@ fn drain(
|
|||||||
chunker: &mut dyn Chunker,
|
chunker: &mut dyn Chunker,
|
||||||
events: &impl EventSink,
|
events: &impl EventSink,
|
||||||
log: &Logger,
|
log: &Logger,
|
||||||
|
stop: &AtomicBool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let file = File::open(source_path)?;
|
let file = File::open(source_path)?;
|
||||||
let mut complete_signal = !is_partial;
|
let mut complete_signal = !is_partial;
|
||||||
@@ -123,6 +132,9 @@ fn drain(
|
|||||||
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
|
if stop.load(Ordering::Relaxed) {
|
||||||
|
bail!("stopped: the receiving side is gone");
|
||||||
|
}
|
||||||
let file_len = file.metadata()?.len();
|
let file_len = file.metadata()?.len();
|
||||||
// Whether this iteration did real work -- if so, loop again
|
// Whether this iteration did real work -- if so, loop again
|
||||||
// immediately rather than sleeping, so a large already-available
|
// immediately rather than sleeping, so a large already-available
|
||||||
@@ -316,6 +328,10 @@ mod tests {
|
|||||||
(data, got_eof)
|
(data, got_eof)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn no_stop() -> Arc<AtomicBool> {
|
||||||
|
Arc::new(AtomicBool::new(false))
|
||||||
|
}
|
||||||
|
|
||||||
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
||||||
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
||||||
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
||||||
@@ -340,7 +356,7 @@ mod tests {
|
|||||||
std::fs::write(&path, b"hello world").unwrap();
|
std::fs::write(&path, b"hello world").unwrap();
|
||||||
|
|
||||||
let (tx, rx) = mpsc::channel();
|
let (tx, rx) = mpsc::channel();
|
||||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap();
|
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
|
||||||
let (data, eof) = collect(rx);
|
let (data, eof) = collect(rx);
|
||||||
assert_eq!(data, b"hello world");
|
assert_eq!(data, b"hello world");
|
||||||
assert!(eof);
|
assert!(eof);
|
||||||
@@ -352,7 +368,7 @@ mod tests {
|
|||||||
let path = dir.path().join("nope.pcap.partial");
|
let path = dir.path().join("nope.pcap.partial");
|
||||||
let final_path = dir.path().join("nope.pcap");
|
let final_path = dir.path().join("nope.pcap");
|
||||||
let (tx, _rx) = mpsc::channel();
|
let (tx, _rx) = mpsc::channel();
|
||||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err();
|
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
|
||||||
assert!(err.to_string().contains("not found"));
|
assert!(err.to_string().contains("not found"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -369,7 +385,7 @@ mod tests {
|
|||||||
let handle = std::thread::spawn({
|
let handle = std::thread::spawn({
|
||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||||
});
|
});
|
||||||
|
|
||||||
std::thread::sleep(Duration::from_millis(60));
|
std::thread::sleep(Duration::from_millis(60));
|
||||||
@@ -399,7 +415,7 @@ mod tests {
|
|||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
let log = log.clone();
|
let log = log.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log)
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
|
||||||
});
|
});
|
||||||
|
|
||||||
std::thread::sleep(Duration::from_millis(60));
|
std::thread::sleep(Duration::from_millis(60));
|
||||||
@@ -426,7 +442,7 @@ mod tests {
|
|||||||
let handle = std::thread::spawn({
|
let handle = std::thread::spawn({
|
||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||||
});
|
});
|
||||||
|
|
||||||
std::thread::sleep(Duration::from_millis(60));
|
std::thread::sleep(Duration::from_millis(60));
|
||||||
@@ -457,7 +473,7 @@ mod tests {
|
|||||||
let handle = std::thread::spawn({
|
let handle = std::thread::spawn({
|
||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||||
});
|
});
|
||||||
|
|
||||||
// Let the reader fully drain the initial 10 bytes.
|
// Let the reader fully drain the initial 10 bytes.
|
||||||
@@ -522,6 +538,7 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
|
no_stop(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -569,6 +586,7 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
|
no_stop(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -624,6 +642,7 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
|
no_stop(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -668,6 +687,7 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
|
no_stop(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -706,6 +726,7 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
|
no_stop(),
|
||||||
);
|
);
|
||||||
try_collect(rx);
|
try_collect(rx);
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
|
|||||||
Reference in new issue
Block a user