scpcap v0

This commit is contained in:
eric committed 2026-08-25 09:03:56 -04:00
1 parent 64775b8025
commit 9fbef46c9a
14 files changed
+2213 -2

No files matched your search

+317
View File
@@ -0,0 +1,317 @@
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker, DEFAULT_FLUSH};
use crate::protocol::{Message, MessageSink, MessageSource};
use crate::tail::{run_tail, SourceEvent};
use crate::zstd_frame::{compress_frame, MARKER_FRAME};
use anyhow::{bail, Result};
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::sync::mpsc;
use std::thread;
/// Bound on the reader->compressor and compressor->sender channels: enough to
/// let stages overlap (disk read, CPU compress, network send happening on
/// different chunks at once) without unbounded memory growth if one stage is
/// slower than another.
const PIPELINE_DEPTH: usize = 8;
pub struct SendOptions {
pub source_path: PathBuf,
pub is_partial: bool,
pub final_source_path: PathBuf,
pub compress: bool,
pub chunk_target: usize,
pub level: i32,
}
enum CompressedItem {
Frame(Vec<u8>),
Eof,
Error(String),
}
/// Reads (and, in compress mode, pcap-record-aligned-chunks and zstd-compresses)
/// the source, sending the result through `sink`. Compress mode runs a 3-stage
/// pipeline (tail thread -> compress thread -> this thread sends) so that
/// disk I/O, CPU compression, and network writes overlap instead of
/// serializing -- see the plan's Performance section. Non-compress mode
/// collapses to 2 stages since there's no compression work to overlap.
pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()> {
let chunker: Box<dyn Chunker> = if opts.compress {
Box::new(RecordAlignedChunker::new(opts.chunk_target, DEFAULT_FLUSH))
} else {
Box::new(RawChunker::new())
};
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
let reader = thread::spawn({
let source_path = opts.source_path.clone();
let is_partial = opts.is_partial;
let final_source_path = opts.final_source_path.clone();
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx)
});
if opts.compress {
let (comp_tx, comp_rx) = mpsc::sync_channel::<CompressedItem>(PIPELINE_DEPTH);
let level = opts.level;
let compressor = thread::spawn(move || -> Result<()> {
for event in tail_rx {
match event {
SourceEvent::Chunk(_kind, data) => {
let frame = compress_frame(&data, level)?;
if comp_tx.send(CompressedItem::Frame(frame)).is_err() {
return Ok(()); // receiver gone, sender loop already errored
}
}
SourceEvent::Eof => {
let _ = comp_tx.send(CompressedItem::Eof);
return Ok(());
}
SourceEvent::Error(e) => {
let _ = comp_tx.send(CompressedItem::Error(e.clone()));
bail!(e);
}
}
}
Ok(())
});
sink.send_data(&MARKER_FRAME)?;
let mut send_err: Option<anyhow::Error> = None;
for item in comp_rx {
match item {
CompressedItem::Frame(f) => {
if let Err(e) = sink.send_data(&f) {
send_err = Some(e);
break;
}
}
CompressedItem::Eof => {
if let Err(e) = sink.send_finalize() {
send_err = Some(e);
}
break;
}
CompressedItem::Error(e) => {
// Best-effort: tell the peer why, so it gets a real
// diagnostic instead of just "stream ended without
// FINALIZE". If this send also fails (peer already gone),
// that's fine -- we still report the original error below.
let _ = sink.send_error(&e);
send_err = Some(anyhow::anyhow!(e));
break;
}
}
}
let reader_result = reader.join().unwrap();
let compressor_result = compressor.join().unwrap();
if let Some(e) = send_err {
return Err(e);
}
reader_result?;
compressor_result?;
} else {
let mut send_err: Option<anyhow::Error> = None;
for event in tail_rx {
match event {
SourceEvent::Chunk(_kind, data) => {
if let Err(e) = sink.send_data(&data) {
send_err = Some(e);
break;
}
}
SourceEvent::Eof => {
if let Err(e) = sink.send_finalize() {
send_err = Some(e);
}
break;
}
SourceEvent::Error(e) => {
let _ = sink.send_error(&e);
send_err = Some(anyhow::anyhow!(e));
break;
}
}
}
let reader_result = reader.join().unwrap();
if let Some(e) = send_err {
return Err(e);
}
reader_result?;
}
Ok(())
}
pub struct RecvOptions {
pub dest_final: PathBuf,
pub dest_temp: PathBuf,
}
/// Writes whatever bytes arrive verbatim to a temp path, then atomically
/// renames to the final path on `Finalize`. A stream that ends without
/// `Finalize` is an abort: the temp file is left in place and this returns an
/// error rather than renaming.
pub fn run_recv(mut source: Box<dyn MessageSource>, opts: RecvOptions) -> Result<()> {
let mut f = File::create(&opts.dest_temp)?;
loop {
match source.recv()? {
Some(Message::Data(payload)) => f.write_all(&payload)?,
Some(Message::Finalize) => {
f.sync_all()?;
std::fs::rename(&opts.dest_temp, &opts.dest_final)?;
return Ok(());
}
None => bail!("transfer aborted: stream ended without FINALIZE"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::channel::{ChannelSink, ChannelSource};
use tempfile::tempdir;
fn header_bytes(magic_le: u32) -> Vec<u8> {
let mut h = vec![0u8; crate::chunker::GLOBAL_HEADER_LEN];
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
h
}
fn record_bytes(payload: &[u8]) -> Vec<u8> {
let mut r = Vec::new();
r.extend_from_slice(&[0u8; 8]);
let incl_len = payload.len() as u32;
r.extend_from_slice(&incl_len.to_le_bytes());
r.extend_from_slice(&incl_len.to_le_bytes());
r.extend_from_slice(payload);
r
}
fn run_pipe(opts: SendOptions, dest_final: PathBuf, dest_temp: PathBuf) -> Result<()> {
let (tx, rx) = mpsc::channel::<Message>();
let recv_opts = RecvOptions { dest_final, dest_temp };
let recv_handle = thread::spawn(move || run_recv(Box::new(ChannelSource(rx)), recv_opts));
run_send(opts, Box::new(ChannelSink(tx)))?;
recv_handle.join().unwrap()
}
#[test]
fn non_compress_already_final_source() {
let dir = tempdir().unwrap();
let src = dir.path().join("cap.pcap");
std::fs::write(&src, b"just some bytes, not necessarily pcap").unwrap();
let dest_final = dir.path().join("out.pcap");
let dest_temp = dir.path().join("out.pcap.partial");
let opts = SendOptions {
source_path: src.clone(),
is_partial: false,
final_source_path: src.clone(),
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
level: crate::zstd_frame::DEFAULT_LEVEL,
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
assert!(!dest_temp.exists());
assert_eq!(
std::fs::read(&dest_final).unwrap(),
b"just some bytes, not necessarily pcap"
);
}
#[test]
fn non_compress_partial_source_waits_for_rename() {
let dir = tempdir().unwrap();
let partial = dir.path().join("cap.pcap.partial");
let final_src = dir.path().join("cap.pcap");
std::fs::write(&partial, b"growing-data").unwrap();
let dest_final = dir.path().join("out.pcap");
let dest_temp = dir.path().join("out.pcap.partial");
let opts = SendOptions {
source_path: partial.clone(),
is_partial: true,
final_source_path: final_src.clone(),
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
level: crate::zstd_frame::DEFAULT_LEVEL,
};
let (tx, rx) = mpsc::channel::<Message>();
let recv_handle = thread::spawn({
let dest_final = dest_final.clone();
let dest_temp = dest_temp.clone();
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp })
});
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
thread::sleep(std::time::Duration::from_millis(60));
std::fs::rename(&partial, &final_src).unwrap();
send_handle.join().unwrap().unwrap();
recv_handle.join().unwrap().unwrap();
assert_eq!(std::fs::read(&dest_final).unwrap(), b"growing-data");
}
#[test]
fn compress_mode_produces_conformant_stream_and_round_trips() {
let dir = tempdir().unwrap();
let src = dir.path().join("cap.pcap");
let mut stream = header_bytes(0xA1B2C3D4);
for i in 0..50u8 {
stream.extend_from_slice(&record_bytes(&[i; 100]));
}
std::fs::write(&src, &stream).unwrap();
let dest_final = dir.path().join("out.pcap.zst");
let dest_temp = dir.path().join("out.pcap.zst.partial");
let opts = SendOptions {
source_path: src.clone(),
is_partial: false,
final_source_path: src.clone(),
compress: true,
chunk_target: 512, // small target to force multiple frames
level: crate::zstd_frame::DEFAULT_LEVEL,
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
let out_bytes = std::fs::read(&dest_final).unwrap();
// Marker frame present at offset 0.
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
// Whole-file zstd -d output equals the original source, byte for byte.
let decoded = zstd::stream::decode_all(out_bytes.as_slice()).unwrap();
assert_eq!(decoded, stream);
// Header frame (first frame after the marker) declares exactly 24 bytes.
let after_marker = &out_bytes[14..];
let header_content_size = zstd::zstd_safe::get_frame_content_size(after_marker)
.unwrap()
.unwrap();
assert_eq!(header_content_size, 24);
}
#[test]
fn compress_mode_bad_magic_errors_and_leaves_no_final_file() {
let dir = tempdir().unwrap();
let src = dir.path().join("cap.pcap");
std::fs::write(&src, b"not a pcap file at all, 24+ bytes long").unwrap();
let dest_final = dir.path().join("out.pcap.zst");
let dest_temp = dir.path().join("out.pcap.zst.partial");
let opts = SendOptions {
source_path: src.clone(),
is_partial: false,
final_source_path: src.clone(),
compress: true,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
level: crate::zstd_frame::DEFAULT_LEVEL,
};
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
assert!(!dest_final.exists());
}
}