After a send failed, run_send joined the reader thread, but the reader only learned the receiver was gone on its next send. An idle growing capture, or a filter matching nothing, never produced one, so the process hung -- locally, and as an orphaned remote --send server. The reader now checks a stop flag every poll, raised once the send loop ends. The client also kills ssh when a remote-to-local receive fails locally rather than waiting on it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
682 lines
28 KiB
Rust
682 lines
28 KiB
Rust
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker};
|
|
use crate::filter::PacketFilter;
|
|
use crate::protocol::{Message, MessageSink, MessageSource};
|
|
use crate::tail::{run_tail, SourceEvent};
|
|
use crate::tracelog::Logger;
|
|
use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
|
use anyhow::{bail, Result};
|
|
use std::fs::File;
|
|
use std::io::Write;
|
|
use std::num::NonZeroUsize;
|
|
use std::path::PathBuf;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::sync::{mpsc, Arc};
|
|
use std::time::Duration;
|
|
use std::thread;
|
|
|
|
/// Bound on the reader->compressor and compressor->sender channels: enough to
|
|
/// let stages overlap (disk read, CPU compress, network send happening on
|
|
/// different chunks at once) without unbounded memory growth if one stage is
|
|
/// slower than another.
|
|
const PIPELINE_DEPTH: usize = 8;
|
|
|
|
pub struct SendOptions {
|
|
pub source_path: PathBuf,
|
|
pub is_partial: bool,
|
|
pub final_source_path: PathBuf,
|
|
pub compress: bool,
|
|
pub chunk_target: NonZeroUsize,
|
|
/// How long kept-but-undersized output may wait before being sent
|
|
/// anyway. See `Chunker::should_flush`.
|
|
pub flush_after: Duration,
|
|
pub level: i32,
|
|
/// Which packets to forward. `PacketFilter::match_all()` when no `--ip`/`--mac` was
|
|
/// given -- not a disabled filter, just one that says yes, so there is
|
|
/// only ever one code path through the chunker.
|
|
pub filter: PacketFilter,
|
|
pub log: Arc<Logger>,
|
|
}
|
|
|
|
enum CompressedItem {
|
|
Frame(Vec<u8>),
|
|
Eof,
|
|
Error(String),
|
|
}
|
|
|
|
/// Reads (and, in compress mode, pcap-record-aligned-chunks and zstd-compresses)
|
|
/// the source, sending the result through `sink`. Compress mode runs a 3-stage
|
|
/// pipeline (tail thread -> compress thread -> this thread sends) so that
|
|
/// disk I/O, CPU compression, and network writes overlap instead of
|
|
/// serializing -- see the plan's Performance section. Non-compress mode
|
|
/// collapses to 2 stages since there's no compression work to overlap.
|
|
pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()> {
|
|
// Two independent reasons to parse records, either of which forces the
|
|
// record-aligned chunker:
|
|
// * `compress`: chunk boundaries become zstd frame boundaries, and the
|
|
// format requires every frame to hold a whole number of records
|
|
// (STREAMING_ZSTD_FORMAT.md invariant 3). The cut point *is* the format.
|
|
// * `is_partial`: a growing source needs `confirmed_len()` -- record
|
|
// structure is the only way to tell real content from a preallocating
|
|
// writer's not-yet-written tail. Here the chunker is a validator, not
|
|
// a framer.
|
|
// * a real `--ip`/`--mac` filter: deciding whether to forward a packet means
|
|
// finding the packet, and record boundaries are the only way to know
|
|
// where one starts. Here the chunker is a demultiplexer.
|
|
// A final, uncompressed copy has neither: `run_recv` writes every Data
|
|
// payload verbatim and never inspects a boundary, so alignment would buy
|
|
// nothing, while requiring pcap structure would break copying pcapng,
|
|
// truncated, or otherwise non-pcap files that transfer fine today. So it
|
|
// gets the passthrough chunker, framed on the wire only by tail.rs's
|
|
// READ_WINDOW.
|
|
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial || !opts.filter.is_match_all()
|
|
{
|
|
Box::new(RecordAlignedChunker::with_filter(
|
|
opts.chunk_target,
|
|
opts.flush_after,
|
|
opts.is_partial,
|
|
opts.filter.clone(),
|
|
))
|
|
} else {
|
|
Box::new(RawChunker::new())
|
|
};
|
|
|
|
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
|
// Raised once the send loop below is done, for whatever reason. After a
|
|
// clean EOF or a source error the reader has already returned and never
|
|
// looks; after a failed send it is how the reader learns to quit instead
|
|
// of polling the source until it next has something to send, which on an
|
|
// idle capture or a filter matching nothing may be never.
|
|
let stop = Arc::new(AtomicBool::new(false));
|
|
let reader = thread::spawn({
|
|
let source_path = opts.source_path.clone();
|
|
let is_partial = opts.is_partial;
|
|
let final_source_path = opts.final_source_path.clone();
|
|
let log = opts.log.clone();
|
|
let stop = stop.clone();
|
|
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
|
});
|
|
|
|
if opts.compress {
|
|
let (comp_tx, comp_rx) = mpsc::sync_channel::<CompressedItem>(PIPELINE_DEPTH);
|
|
let level = opts.level;
|
|
let compressor = thread::spawn(move || -> Result<()> {
|
|
for event in tail_rx {
|
|
match event {
|
|
SourceEvent::Chunk(_kind, data) => {
|
|
let frame = compress_frame(&data, level)?;
|
|
if comp_tx.send(CompressedItem::Frame(frame)).is_err() {
|
|
return Ok(()); // receiver gone, sender loop already errored
|
|
}
|
|
}
|
|
SourceEvent::Eof => {
|
|
let _ = comp_tx.send(CompressedItem::Eof);
|
|
return Ok(());
|
|
}
|
|
SourceEvent::Error(e) => {
|
|
let _ = comp_tx.send(CompressedItem::Error(e.clone()));
|
|
bail!(e);
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
});
|
|
|
|
sink.send_data(&MARKER_FRAME)?;
|
|
let mut send_err: Option<anyhow::Error> = None;
|
|
for item in comp_rx {
|
|
match item {
|
|
CompressedItem::Frame(f) => {
|
|
if let Err(e) = sink.send_data(&f) {
|
|
send_err = Some(e);
|
|
break;
|
|
}
|
|
}
|
|
CompressedItem::Eof => {
|
|
if let Err(e) = sink.send_finalize() {
|
|
send_err = Some(e);
|
|
}
|
|
break;
|
|
}
|
|
CompressedItem::Error(e) => {
|
|
// Best-effort: tell the peer why, so it gets a real
|
|
// diagnostic instead of just "stream ended without
|
|
// FINALIZE". If this send also fails (peer already gone),
|
|
// that's fine -- we still report the original error below.
|
|
let _ = sink.send_error(&e);
|
|
send_err = Some(anyhow::anyhow!(e));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
stop.store(true, Ordering::Relaxed);
|
|
let reader_result = reader.join().unwrap();
|
|
let compressor_result = compressor.join().unwrap();
|
|
if let Some(e) = send_err {
|
|
return Err(e);
|
|
}
|
|
reader_result?;
|
|
compressor_result?;
|
|
} else {
|
|
let mut send_err: Option<anyhow::Error> = None;
|
|
let mut sent: u64 = 0;
|
|
for event in tail_rx {
|
|
match event {
|
|
SourceEvent::Chunk(_kind, data) => {
|
|
sent += data.len() as u64;
|
|
if let Err(e) = sink.send_data(&data) {
|
|
opts.log.log(format_args!("send: send_data failed after {sent} bytes: {e}"));
|
|
send_err = Some(e);
|
|
break;
|
|
}
|
|
}
|
|
SourceEvent::Eof => {
|
|
opts.log.log(format_args!("send: tail EOF, {sent} bytes sent, sending FINALIZE"));
|
|
if let Err(e) = sink.send_finalize() {
|
|
send_err = Some(e);
|
|
}
|
|
break;
|
|
}
|
|
SourceEvent::Error(e) => {
|
|
opts.log.log(format_args!("send: tail error after {sent} bytes: {e}"));
|
|
let _ = sink.send_error(&e);
|
|
send_err = Some(anyhow::anyhow!(e));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
stop.store(true, Ordering::Relaxed);
|
|
let reader_result = reader.join().unwrap();
|
|
if let Some(e) = send_err {
|
|
return Err(e);
|
|
}
|
|
reader_result?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub struct RecvOptions {
|
|
pub dest_final: PathBuf,
|
|
pub dest_temp: PathBuf,
|
|
pub log: Arc<Logger>,
|
|
}
|
|
|
|
/// Writes whatever bytes arrive verbatim to a temp path, then atomically
|
|
/// renames to the final path on `Finalize`. A stream that ends without
|
|
/// `Finalize` is an abort: the temp file is left in place and this returns an
|
|
/// error rather than renaming.
|
|
pub fn run_recv(mut source: Box<dyn MessageSource>, opts: RecvOptions) -> Result<()> {
|
|
let mut f = File::create(&opts.dest_temp)?;
|
|
let mut received: u64 = 0;
|
|
opts.log.log(format_args!(
|
|
"recv: writing to {} (final: {})",
|
|
opts.dest_temp.display(),
|
|
opts.dest_final.display()
|
|
));
|
|
loop {
|
|
match source.recv()? {
|
|
Some(Message::Data(payload)) => {
|
|
received += payload.len() as u64;
|
|
f.write_all(&payload)?;
|
|
}
|
|
Some(Message::Finalize) => {
|
|
f.sync_all()?;
|
|
std::fs::rename(&opts.dest_temp, &opts.dest_final)?;
|
|
opts.log.log(format_args!(
|
|
"recv: FINALIZE after {received} bytes, renamed to {}",
|
|
opts.dest_final.display()
|
|
));
|
|
return Ok(());
|
|
}
|
|
None => {
|
|
opts.log.log(format_args!(
|
|
"recv: stream ended without FINALIZE after {received} bytes"
|
|
));
|
|
bail!("transfer aborted: stream ended without FINALIZE")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::channel::{ChannelSink, ChannelSource};
|
|
use tempfile::tempdir;
|
|
|
|
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
|
let mut h = vec![0u8; crate::chunker::GLOBAL_HEADER_LEN];
|
|
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
|
h
|
|
}
|
|
|
|
fn record_bytes(payload: &[u8], ts_sec: u32) -> Vec<u8> {
|
|
let mut r = Vec::new();
|
|
r.extend_from_slice(&ts_sec.to_le_bytes());
|
|
r.extend_from_slice(&[0u8; 4]); // ts_usec
|
|
let incl_len = payload.len() as u32;
|
|
r.extend_from_slice(&incl_len.to_le_bytes());
|
|
r.extend_from_slice(&incl_len.to_le_bytes());
|
|
r.extend_from_slice(payload);
|
|
r
|
|
}
|
|
|
|
fn run_pipe(opts: SendOptions, dest_final: PathBuf, dest_temp: PathBuf) -> Result<()> {
|
|
let (tx, rx) = mpsc::channel::<Message>();
|
|
let recv_opts = RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) };
|
|
let recv_handle = thread::spawn(move || run_recv(Box::new(ChannelSource(rx)), recv_opts));
|
|
run_send(opts, Box::new(ChannelSink(tx)))?;
|
|
recv_handle.join().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn non_compress_already_final_source() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
std::fs::write(&src, b"just some bytes, not necessarily pcap").unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: PacketFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
|
|
|
assert!(!dest_temp.exists());
|
|
assert_eq!(
|
|
std::fs::read(&dest_final).unwrap(),
|
|
b"just some bytes, not necessarily pcap"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn non_compress_partial_source_waits_for_rename() {
|
|
// Under the new chunker-selection (compress || is_partial), a raw
|
|
// partial transfer now routes through RecordAlignedChunker, which
|
|
// requires genuine pcap structure -- see the plan's "behavior
|
|
// change requiring sign-off". A real global header plus a real
|
|
// record is required; the second record is appended before rename
|
|
// so the first one gets its next-record proof quickly rather than
|
|
// waiting out the trailing-record grace period.
|
|
let dir = tempdir().unwrap();
|
|
let partial = dir.path().join("cap.pcap.partial");
|
|
let final_src = dir.path().join("cap.pcap");
|
|
let mut stream = header_bytes(0xA1B2C3D4);
|
|
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
|
|
std::fs::write(&partial, &stream).unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: partial.clone(),
|
|
is_partial: true,
|
|
final_source_path: final_src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: PacketFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
|
|
let (tx, rx) = mpsc::channel::<Message>();
|
|
let recv_handle = thread::spawn({
|
|
let dest_final = dest_final.clone();
|
|
let dest_temp = dest_temp.clone();
|
|
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
|
});
|
|
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
|
|
|
thread::sleep(std::time::Duration::from_millis(60));
|
|
let rec2 = record_bytes(&[2u8; 10], 1_700_000_001);
|
|
{
|
|
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
|
f.write_all(&rec2).unwrap();
|
|
f.sync_all().unwrap();
|
|
}
|
|
stream.extend_from_slice(&rec2);
|
|
thread::sleep(std::time::Duration::from_millis(60));
|
|
std::fs::rename(&partial, &final_src).unwrap();
|
|
|
|
send_handle.join().unwrap().unwrap();
|
|
recv_handle.join().unwrap().unwrap();
|
|
|
|
assert_eq!(std::fs::read(&dest_final).unwrap(), stream);
|
|
}
|
|
|
|
#[test]
|
|
fn compress_mode_produces_conformant_stream_and_round_trips() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let mut stream = header_bytes(0xA1B2C3D4);
|
|
for i in 0..50u8 {
|
|
stream.extend_from_slice(&record_bytes(&[i; 100], 0));
|
|
}
|
|
std::fs::write(&src, &stream).unwrap();
|
|
|
|
let dest_final = dir.path().join("out.pcap.zst");
|
|
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: true,
|
|
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: PacketFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
|
|
|
let out_bytes = std::fs::read(&dest_final).unwrap();
|
|
|
|
// Marker frame present at offset 0.
|
|
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
|
|
|
|
// Whole-file zstd -d output equals the original source, byte for byte.
|
|
let decoded = zstd::stream::decode_all(out_bytes.as_slice()).unwrap();
|
|
assert_eq!(decoded, stream);
|
|
|
|
// Header frame (first frame after the marker) declares exactly 24 bytes.
|
|
let after_marker = &out_bytes[14..];
|
|
let header_content_size = zstd::zstd_safe::get_frame_content_size(after_marker)
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(header_content_size, 24);
|
|
}
|
|
|
|
#[test]
|
|
fn compress_mode_bad_magic_errors_and_leaves_no_final_file() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
std::fs::write(&src, b"not a pcap file at all, 24+ bytes long").unwrap();
|
|
let dest_final = dir.path().join("out.pcap.zst");
|
|
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: true,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: PacketFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
|
|
assert!(!dest_final.exists());
|
|
}
|
|
|
|
fn eth_header_bytes() -> Vec<u8> {
|
|
let mut h = header_bytes(0xA1B2C3D4);
|
|
h[20..24].copy_from_slice(&1u32.to_le_bytes()); // LINKTYPE_ETHERNET
|
|
h
|
|
}
|
|
|
|
/// A record holding an Ethernet/IPv4 packet, padded to `len` captured bytes.
|
|
fn ip_record(src: [u8; 4], dst: [u8; 4], len: usize) -> Vec<u8> {
|
|
let mut pkt = vec![0u8; len.max(34)];
|
|
pkt[12..14].copy_from_slice(&0x0800u16.to_be_bytes());
|
|
pkt[14] = 0x45;
|
|
pkt[26..30].copy_from_slice(&src);
|
|
pkt[30..34].copy_from_slice(&dst);
|
|
record_bytes(&pkt, 1_700_000_000)
|
|
}
|
|
|
|
fn ip_filter(specs: &[&str]) -> PacketFilter {
|
|
PacketFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>(), &[]).unwrap()
|
|
}
|
|
|
|
/// Source stream plus the exact pcap a `--ip 1.1.1.1` transfer should
|
|
/// produce from it: same global header, only the matching records.
|
|
fn filtered_fixture(n: usize) -> (Vec<u8>, Vec<u8>) {
|
|
let mut source = eth_header_bytes();
|
|
let mut expected = eth_header_bytes();
|
|
for i in 0..n {
|
|
if i % 5 == 0 {
|
|
let hit = ip_record([1, 1, 1, 1], [9, 9, 9, 9], 100 + i);
|
|
source.extend_from_slice(&hit);
|
|
expected.extend_from_slice(&hit);
|
|
} else {
|
|
source.extend_from_slice(&ip_record([9, 9, 9, 9], [8, 8, 8, 8], 100 + i));
|
|
}
|
|
}
|
|
(source, expected)
|
|
}
|
|
|
|
#[test]
|
|
fn filtered_transfer_carries_only_matching_packets() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let (source, expected) = filtered_fixture(50);
|
|
std::fs::write(&src, &source).unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: ip_filter(&["1.1.1.1"]),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
|
|
|
// A filter forces the record-aligned chunker even though this is a
|
|
// final, uncompressed copy -- so the output is a real pcap, not a
|
|
// byte-for-byte copy of the source.
|
|
assert_eq!(std::fs::read(&dest_final).unwrap(), expected);
|
|
assert!(expected.len() < source.len());
|
|
}
|
|
|
|
#[test]
|
|
fn filtered_compressed_transfer_round_trips_to_the_filtered_pcap() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let (source, expected) = filtered_fixture(200);
|
|
std::fs::write(&src, &source).unwrap();
|
|
let dest_final = dir.path().join("out.pcap.zst");
|
|
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: true,
|
|
chunk_target: NonZeroUsize::new(512).unwrap(),
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: ip_filter(&["1.1.1.1"]),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
|
|
|
let out_bytes = std::fs::read(&dest_final).unwrap();
|
|
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
|
|
// The whole point of the container: decompressing yields a valid
|
|
// pcap, here the filtered one.
|
|
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
|
// Header frame still declares exactly the 24-byte global header.
|
|
let header_content_size = zstd::zstd_safe::get_frame_content_size(&out_bytes[14..])
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(header_content_size, 24);
|
|
}
|
|
|
|
#[test]
|
|
fn a_filter_matching_nothing_produces_a_header_only_pcap() {
|
|
// Not an error: the transfer completes and the destination is a
|
|
// valid, empty capture.
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let (source, _) = filtered_fixture(20);
|
|
std::fs::write(&src, &source).unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: ip_filter(&["203.0.113.7"]),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
|
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
|
}
|
|
|
|
/// A record holding an Ethernet frame from `src` to `dst` with
|
|
/// `ethertype`, padded to `len` captured bytes.
|
|
fn mac_record(dst: [u8; 6], src: [u8; 6], ethertype: u16, len: usize, ts_sec: u32) -> Vec<u8> {
|
|
let mut pkt = vec![0u8; len.max(14)];
|
|
pkt[0..6].copy_from_slice(&dst);
|
|
pkt[6..12].copy_from_slice(&src);
|
|
pkt[12..14].copy_from_slice(ðertype.to_be_bytes());
|
|
record_bytes(&pkt, ts_sec)
|
|
}
|
|
|
|
#[test]
|
|
fn mac_filtered_growing_capture_carries_only_the_upstream_link() {
|
|
// The scenario --mac exists for: a capture still being written, from
|
|
// which only the traffic on one switch connection is wanted. The
|
|
// source grows in uneven appends that tear records mid-header and
|
|
// mid-payload, so the filter must only ever see confirmed records.
|
|
const UPSTREAM: [u8; 6] = [0x02, 0, 0, 0, 0, 0xaa];
|
|
const OTHER: [u8; 6] = [0x02, 0, 0, 0, 0, 0xbb];
|
|
const BROADCAST: [u8; 6] = [0xff; 6];
|
|
|
|
let mut source = eth_header_bytes();
|
|
let mut expected = eth_header_bytes();
|
|
for i in 0..60u32 {
|
|
let ts = 1_700_000_000 + i;
|
|
let len = 64 + i as usize;
|
|
let (record, kept) = match i % 4 {
|
|
0 => (mac_record(OTHER, UPSTREAM, 0x0800, len, ts), true), // from the switch
|
|
1 => (mac_record(UPSTREAM, OTHER, 0x86DD, len, ts), true), // to the switch
|
|
2 => (mac_record(BROADCAST, UPSTREAM, 0x0806, len, ts), true), // ARP from the switch
|
|
_ => (mac_record(BROADCAST, OTHER, 0x0806, len, ts), false), // someone else's ARP
|
|
};
|
|
source.extend_from_slice(&record);
|
|
if kept {
|
|
expected.extend_from_slice(&record);
|
|
}
|
|
}
|
|
|
|
let dir = tempdir().unwrap();
|
|
let partial = dir.path().join("cap.pcap.partial");
|
|
let final_src = dir.path().join("cap.pcap");
|
|
std::fs::write(&partial, &source[..10]).unwrap();
|
|
let dest_final = dir.path().join("out.pcap.zst");
|
|
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: partial.clone(),
|
|
is_partial: true,
|
|
final_source_path: final_src.clone(),
|
|
compress: true,
|
|
chunk_target: NonZeroUsize::new(512).unwrap(),
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
|
|
let (tx, rx) = mpsc::channel::<Message>();
|
|
let recv_handle = thread::spawn({
|
|
let dest_final = dest_final.clone();
|
|
let dest_temp = dest_temp.clone();
|
|
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
|
});
|
|
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
|
|
|
// Odd-sized appends (an awkward stride relative to the ~80-140 byte
|
|
// records) so the visible end lands inside headers and payloads.
|
|
let mut written = 10;
|
|
for step in [37usize, 501, 13, 977, 250].iter().cycle() {
|
|
if written == source.len() {
|
|
break;
|
|
}
|
|
let end = (written + step).min(source.len());
|
|
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
|
f.write_all(&source[written..end]).unwrap();
|
|
f.sync_all().unwrap();
|
|
written = end;
|
|
thread::sleep(std::time::Duration::from_millis(5));
|
|
}
|
|
std::fs::rename(&partial, &final_src).unwrap();
|
|
|
|
send_handle.join().unwrap().unwrap();
|
|
recv_handle.join().unwrap().unwrap();
|
|
|
|
let out_bytes = std::fs::read(&dest_final).unwrap();
|
|
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
|
assert!(expected.len() < source.len());
|
|
}
|
|
|
|
/// A peer that has gone away: every send fails.
|
|
struct GoneSink;
|
|
|
|
impl MessageSink for GoneSink {
|
|
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
|
bail!("peer gone")
|
|
}
|
|
fn send_finalize(&mut self) -> Result<()> {
|
|
bail!("peer gone")
|
|
}
|
|
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
|
bail!("peer gone")
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
|
// The reader used to learn the peer was gone only on its next send.
|
|
// A growing capture that stops growing (and is never renamed) never
|
|
// gives it one, so run_send waited on the reader forever.
|
|
for compress in [false, true] {
|
|
let dir = tempdir().unwrap();
|
|
let partial = dir.path().join("cap.pcap.partial");
|
|
let mut stream = eth_header_bytes();
|
|
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
|
std::fs::write(&partial, &stream).unwrap();
|
|
|
|
let opts = SendOptions {
|
|
source_path: partial.clone(),
|
|
is_partial: true,
|
|
final_source_path: dir.path().join("cap.pcap"),
|
|
compress,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: PacketFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
let (done_tx, done_rx) = mpsc::channel();
|
|
thread::spawn(move || {
|
|
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
|
});
|
|
let result = done_rx
|
|
.recv_timeout(Duration::from_secs(5))
|
|
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
|
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
|
}
|
|
}
|
|
}
|