534 lines
21 KiB
Rust
534 lines
21 KiB
Rust
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker};
|
|
use crate::filter::IpFilter;
|
|
use crate::protocol::{Message, MessageSink, MessageSource};
|
|
use crate::tail::{run_tail, SourceEvent};
|
|
use crate::tracelog::Logger;
|
|
use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
|
use anyhow::{bail, Result};
|
|
use std::fs::File;
|
|
use std::io::Write;
|
|
use std::path::PathBuf;
|
|
use std::sync::{mpsc, Arc};
|
|
use std::time::Duration;
|
|
use std::thread;
|
|
|
|
/// Bound on the reader->compressor and compressor->sender channels: enough to
|
|
/// let stages overlap (disk read, CPU compress, network send happening on
|
|
/// different chunks at once) without unbounded memory growth if one stage is
|
|
/// slower than another.
|
|
const PIPELINE_DEPTH: usize = 8;
|
|
|
|
pub struct SendOptions {
|
|
pub source_path: PathBuf,
|
|
pub is_partial: bool,
|
|
pub final_source_path: PathBuf,
|
|
pub compress: bool,
|
|
pub chunk_target: usize,
|
|
/// How long kept-but-undersized output may wait before being sent
|
|
/// anyway. See `Chunker::should_flush`.
|
|
pub flush_after: Duration,
|
|
pub level: i32,
|
|
/// Which packets to forward. `IpFilter::match_all()` when no `--ip` was
|
|
/// given -- not a disabled filter, just one that says yes, so there is
|
|
/// only ever one code path through the chunker.
|
|
pub filter: IpFilter,
|
|
pub log: Arc<Logger>,
|
|
}
|
|
|
|
enum CompressedItem {
|
|
Frame(Vec<u8>),
|
|
Eof,
|
|
Error(String),
|
|
}
|
|
|
|
/// Reads (and, in compress mode, pcap-record-aligned-chunks and zstd-compresses)
|
|
/// the source, sending the result through `sink`. Compress mode runs a 3-stage
|
|
/// pipeline (tail thread -> compress thread -> this thread sends) so that
|
|
/// disk I/O, CPU compression, and network writes overlap instead of
|
|
/// serializing -- see the plan's Performance section. Non-compress mode
|
|
/// collapses to 2 stages since there's no compression work to overlap.
|
|
pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()> {
|
|
// Two independent reasons to parse records, either of which forces the
|
|
// record-aligned chunker:
|
|
// * `compress`: chunk boundaries become zstd frame boundaries, and the
|
|
// format requires every frame to hold a whole number of records
|
|
// (STREAMING_ZSTD_FORMAT.md invariant 3). The cut point *is* the format.
|
|
// * `is_partial`: a growing source needs `confirmed_len()` -- record
|
|
// structure is the only way to tell real content from a preallocating
|
|
// writer's not-yet-written tail. Here the chunker is a validator, not
|
|
// a framer.
|
|
// * a real `--ip` filter: deciding whether to forward a packet means
|
|
// finding the packet, and record boundaries are the only way to know
|
|
// where one starts. Here the chunker is a demultiplexer.
|
|
// A final, uncompressed copy has neither: `run_recv` writes every Data
|
|
// payload verbatim and never inspects a boundary, so alignment would buy
|
|
// nothing, while requiring pcap structure would break copying pcapng,
|
|
// truncated, or otherwise non-pcap files that transfer fine today. So it
|
|
// gets the passthrough chunker, framed on the wire only by tail.rs's
|
|
// READ_WINDOW.
|
|
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial || !opts.filter.is_match_all()
|
|
{
|
|
Box::new(RecordAlignedChunker::with_filter(
|
|
opts.chunk_target,
|
|
opts.flush_after,
|
|
opts.is_partial,
|
|
opts.filter.clone(),
|
|
))
|
|
} else {
|
|
Box::new(RawChunker::new())
|
|
};
|
|
|
|
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
|
let reader = thread::spawn({
|
|
let source_path = opts.source_path.clone();
|
|
let is_partial = opts.is_partial;
|
|
let final_source_path = opts.final_source_path.clone();
|
|
let log = opts.log.clone();
|
|
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
|
});
|
|
|
|
if opts.compress {
|
|
let (comp_tx, comp_rx) = mpsc::sync_channel::<CompressedItem>(PIPELINE_DEPTH);
|
|
let level = opts.level;
|
|
let compressor = thread::spawn(move || -> Result<()> {
|
|
for event in tail_rx {
|
|
match event {
|
|
SourceEvent::Chunk(_kind, data) => {
|
|
let frame = compress_frame(&data, level)?;
|
|
if comp_tx.send(CompressedItem::Frame(frame)).is_err() {
|
|
return Ok(()); // receiver gone, sender loop already errored
|
|
}
|
|
}
|
|
SourceEvent::Eof => {
|
|
let _ = comp_tx.send(CompressedItem::Eof);
|
|
return Ok(());
|
|
}
|
|
SourceEvent::Error(e) => {
|
|
let _ = comp_tx.send(CompressedItem::Error(e.clone()));
|
|
bail!(e);
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
});
|
|
|
|
sink.send_data(&MARKER_FRAME)?;
|
|
let mut send_err: Option<anyhow::Error> = None;
|
|
for item in comp_rx {
|
|
match item {
|
|
CompressedItem::Frame(f) => {
|
|
if let Err(e) = sink.send_data(&f) {
|
|
send_err = Some(e);
|
|
break;
|
|
}
|
|
}
|
|
CompressedItem::Eof => {
|
|
if let Err(e) = sink.send_finalize() {
|
|
send_err = Some(e);
|
|
}
|
|
break;
|
|
}
|
|
CompressedItem::Error(e) => {
|
|
// Best-effort: tell the peer why, so it gets a real
|
|
// diagnostic instead of just "stream ended without
|
|
// FINALIZE". If this send also fails (peer already gone),
|
|
// that's fine -- we still report the original error below.
|
|
let _ = sink.send_error(&e);
|
|
send_err = Some(anyhow::anyhow!(e));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
let reader_result = reader.join().unwrap();
|
|
let compressor_result = compressor.join().unwrap();
|
|
if let Some(e) = send_err {
|
|
return Err(e);
|
|
}
|
|
reader_result?;
|
|
compressor_result?;
|
|
} else {
|
|
let mut send_err: Option<anyhow::Error> = None;
|
|
let mut sent: u64 = 0;
|
|
for event in tail_rx {
|
|
match event {
|
|
SourceEvent::Chunk(_kind, data) => {
|
|
sent += data.len() as u64;
|
|
if let Err(e) = sink.send_data(&data) {
|
|
opts.log.log(format_args!("send: send_data failed after {sent} bytes: {e}"));
|
|
send_err = Some(e);
|
|
break;
|
|
}
|
|
}
|
|
SourceEvent::Eof => {
|
|
opts.log.log(format_args!("send: tail EOF, {sent} bytes sent, sending FINALIZE"));
|
|
if let Err(e) = sink.send_finalize() {
|
|
send_err = Some(e);
|
|
}
|
|
break;
|
|
}
|
|
SourceEvent::Error(e) => {
|
|
opts.log.log(format_args!("send: tail error after {sent} bytes: {e}"));
|
|
let _ = sink.send_error(&e);
|
|
send_err = Some(anyhow::anyhow!(e));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
let reader_result = reader.join().unwrap();
|
|
if let Some(e) = send_err {
|
|
return Err(e);
|
|
}
|
|
reader_result?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub struct RecvOptions {
|
|
pub dest_final: PathBuf,
|
|
pub dest_temp: PathBuf,
|
|
pub log: Arc<Logger>,
|
|
}
|
|
|
|
/// Writes whatever bytes arrive verbatim to a temp path, then atomically
|
|
/// renames to the final path on `Finalize`. A stream that ends without
|
|
/// `Finalize` is an abort: the temp file is left in place and this returns an
|
|
/// error rather than renaming.
|
|
pub fn run_recv(mut source: Box<dyn MessageSource>, opts: RecvOptions) -> Result<()> {
|
|
let mut f = File::create(&opts.dest_temp)?;
|
|
let mut received: u64 = 0;
|
|
opts.log.log(format_args!(
|
|
"recv: writing to {} (final: {})",
|
|
opts.dest_temp.display(),
|
|
opts.dest_final.display()
|
|
));
|
|
loop {
|
|
match source.recv()? {
|
|
Some(Message::Data(payload)) => {
|
|
received += payload.len() as u64;
|
|
f.write_all(&payload)?;
|
|
}
|
|
Some(Message::Finalize) => {
|
|
f.sync_all()?;
|
|
std::fs::rename(&opts.dest_temp, &opts.dest_final)?;
|
|
opts.log.log(format_args!(
|
|
"recv: FINALIZE after {received} bytes, renamed to {}",
|
|
opts.dest_final.display()
|
|
));
|
|
return Ok(());
|
|
}
|
|
None => {
|
|
opts.log.log(format_args!(
|
|
"recv: stream ended without FINALIZE after {received} bytes"
|
|
));
|
|
bail!("transfer aborted: stream ended without FINALIZE")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::channel::{ChannelSink, ChannelSource};
|
|
use tempfile::tempdir;
|
|
|
|
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
|
let mut h = vec![0u8; crate::chunker::GLOBAL_HEADER_LEN];
|
|
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
|
h
|
|
}
|
|
|
|
fn record_bytes(payload: &[u8], ts_sec: u32) -> Vec<u8> {
|
|
let mut r = Vec::new();
|
|
r.extend_from_slice(&ts_sec.to_le_bytes());
|
|
r.extend_from_slice(&[0u8; 4]); // ts_usec
|
|
let incl_len = payload.len() as u32;
|
|
r.extend_from_slice(&incl_len.to_le_bytes());
|
|
r.extend_from_slice(&incl_len.to_le_bytes());
|
|
r.extend_from_slice(payload);
|
|
r
|
|
}
|
|
|
|
fn run_pipe(opts: SendOptions, dest_final: PathBuf, dest_temp: PathBuf) -> Result<()> {
|
|
let (tx, rx) = mpsc::channel::<Message>();
|
|
let recv_opts = RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) };
|
|
let recv_handle = thread::spawn(move || run_recv(Box::new(ChannelSource(rx)), recv_opts));
|
|
run_send(opts, Box::new(ChannelSink(tx)))?;
|
|
recv_handle.join().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn non_compress_already_final_source() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
std::fs::write(&src, b"just some bytes, not necessarily pcap").unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: IpFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
|
|
|
assert!(!dest_temp.exists());
|
|
assert_eq!(
|
|
std::fs::read(&dest_final).unwrap(),
|
|
b"just some bytes, not necessarily pcap"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn non_compress_partial_source_waits_for_rename() {
|
|
// Under the new chunker-selection (compress || is_partial), a raw
|
|
// partial transfer now routes through RecordAlignedChunker, which
|
|
// requires genuine pcap structure -- see the plan's "behavior
|
|
// change requiring sign-off". A real global header plus a real
|
|
// record is required; the second record is appended before rename
|
|
// so the first one gets its next-record proof quickly rather than
|
|
// waiting out the trailing-record grace period.
|
|
let dir = tempdir().unwrap();
|
|
let partial = dir.path().join("cap.pcap.partial");
|
|
let final_src = dir.path().join("cap.pcap");
|
|
let mut stream = header_bytes(0xA1B2C3D4);
|
|
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
|
|
std::fs::write(&partial, &stream).unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: partial.clone(),
|
|
is_partial: true,
|
|
final_source_path: final_src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: IpFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
|
|
let (tx, rx) = mpsc::channel::<Message>();
|
|
let recv_handle = thread::spawn({
|
|
let dest_final = dest_final.clone();
|
|
let dest_temp = dest_temp.clone();
|
|
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
|
});
|
|
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
|
|
|
thread::sleep(std::time::Duration::from_millis(60));
|
|
let rec2 = record_bytes(&[2u8; 10], 1_700_000_001);
|
|
{
|
|
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
|
f.write_all(&rec2).unwrap();
|
|
f.sync_all().unwrap();
|
|
}
|
|
stream.extend_from_slice(&rec2);
|
|
thread::sleep(std::time::Duration::from_millis(60));
|
|
std::fs::rename(&partial, &final_src).unwrap();
|
|
|
|
send_handle.join().unwrap().unwrap();
|
|
recv_handle.join().unwrap().unwrap();
|
|
|
|
assert_eq!(std::fs::read(&dest_final).unwrap(), stream);
|
|
}
|
|
|
|
#[test]
|
|
fn compress_mode_produces_conformant_stream_and_round_trips() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let mut stream = header_bytes(0xA1B2C3D4);
|
|
for i in 0..50u8 {
|
|
stream.extend_from_slice(&record_bytes(&[i; 100], 0));
|
|
}
|
|
std::fs::write(&src, &stream).unwrap();
|
|
|
|
let dest_final = dir.path().join("out.pcap.zst");
|
|
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: true,
|
|
chunk_target: 512, // small target to force multiple frames
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: IpFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
|
|
|
let out_bytes = std::fs::read(&dest_final).unwrap();
|
|
|
|
// Marker frame present at offset 0.
|
|
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
|
|
|
|
// Whole-file zstd -d output equals the original source, byte for byte.
|
|
let decoded = zstd::stream::decode_all(out_bytes.as_slice()).unwrap();
|
|
assert_eq!(decoded, stream);
|
|
|
|
// Header frame (first frame after the marker) declares exactly 24 bytes.
|
|
let after_marker = &out_bytes[14..];
|
|
let header_content_size = zstd::zstd_safe::get_frame_content_size(after_marker)
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(header_content_size, 24);
|
|
}
|
|
|
|
#[test]
|
|
fn compress_mode_bad_magic_errors_and_leaves_no_final_file() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
std::fs::write(&src, b"not a pcap file at all, 24+ bytes long").unwrap();
|
|
let dest_final = dir.path().join("out.pcap.zst");
|
|
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: true,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: IpFilter::match_all(),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
|
|
assert!(!dest_final.exists());
|
|
}
|
|
|
|
fn eth_header_bytes() -> Vec<u8> {
|
|
let mut h = header_bytes(0xA1B2C3D4);
|
|
h[20..24].copy_from_slice(&1u32.to_le_bytes()); // LINKTYPE_ETHERNET
|
|
h
|
|
}
|
|
|
|
/// A record holding an Ethernet/IPv4 packet, padded to `len` captured bytes.
|
|
fn ip_record(src: [u8; 4], dst: [u8; 4], len: usize) -> Vec<u8> {
|
|
let mut pkt = vec![0u8; len.max(34)];
|
|
pkt[12..14].copy_from_slice(&0x0800u16.to_be_bytes());
|
|
pkt[14] = 0x45;
|
|
pkt[26..30].copy_from_slice(&src);
|
|
pkt[30..34].copy_from_slice(&dst);
|
|
record_bytes(&pkt, 1_700_000_000)
|
|
}
|
|
|
|
fn ip_filter(specs: &[&str]) -> IpFilter {
|
|
IpFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
|
|
}
|
|
|
|
/// Source stream plus the exact pcap a `--ip 1.1.1.1` transfer should
|
|
/// produce from it: same global header, only the matching records.
|
|
fn filtered_fixture(n: usize) -> (Vec<u8>, Vec<u8>) {
|
|
let mut source = eth_header_bytes();
|
|
let mut expected = eth_header_bytes();
|
|
for i in 0..n {
|
|
if i % 5 == 0 {
|
|
let hit = ip_record([1, 1, 1, 1], [9, 9, 9, 9], 100 + i);
|
|
source.extend_from_slice(&hit);
|
|
expected.extend_from_slice(&hit);
|
|
} else {
|
|
source.extend_from_slice(&ip_record([9, 9, 9, 9], [8, 8, 8, 8], 100 + i));
|
|
}
|
|
}
|
|
(source, expected)
|
|
}
|
|
|
|
#[test]
|
|
fn filtered_transfer_carries_only_matching_packets() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let (source, expected) = filtered_fixture(50);
|
|
std::fs::write(&src, &source).unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: ip_filter(&["1.1.1.1"]),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
|
|
|
// A filter forces the record-aligned chunker even though this is a
|
|
// final, uncompressed copy -- so the output is a real pcap, not a
|
|
// byte-for-byte copy of the source.
|
|
assert_eq!(std::fs::read(&dest_final).unwrap(), expected);
|
|
assert!(expected.len() < source.len());
|
|
}
|
|
|
|
#[test]
|
|
fn filtered_compressed_transfer_round_trips_to_the_filtered_pcap() {
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let (source, expected) = filtered_fixture(200);
|
|
std::fs::write(&src, &source).unwrap();
|
|
let dest_final = dir.path().join("out.pcap.zst");
|
|
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: true,
|
|
chunk_target: 512,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: ip_filter(&["1.1.1.1"]),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
|
|
|
let out_bytes = std::fs::read(&dest_final).unwrap();
|
|
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
|
|
// The whole point of the container: decompressing yields a valid
|
|
// pcap, here the filtered one.
|
|
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
|
// Header frame still declares exactly the 24-byte global header.
|
|
let header_content_size = zstd::zstd_safe::get_frame_content_size(&out_bytes[14..])
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(header_content_size, 24);
|
|
}
|
|
|
|
#[test]
|
|
fn a_filter_matching_nothing_produces_a_header_only_pcap() {
|
|
// Not an error: the transfer completes and the destination is a
|
|
// valid, empty capture.
|
|
let dir = tempdir().unwrap();
|
|
let src = dir.path().join("cap.pcap");
|
|
let (source, _) = filtered_fixture(20);
|
|
std::fs::write(&src, &source).unwrap();
|
|
let dest_final = dir.path().join("out.pcap");
|
|
let dest_temp = dir.path().join("out.pcap.partial");
|
|
|
|
let opts = SendOptions {
|
|
source_path: src.clone(),
|
|
is_partial: false,
|
|
final_source_path: src.clone(),
|
|
compress: false,
|
|
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
filter: ip_filter(&["203.0.113.7"]),
|
|
log: Arc::new(Logger::none()),
|
|
};
|
|
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
|
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
|
}
|
|
}
|