allow inflight filter

This commit is contained in:
eric committed 2026-09-01 08:38:44 -04:00
1 parent aeb827b361
commit 8b111ddfdc
9 files changed
+956 -55

No files matched your search

+170 -3
View File
@@ -1,4 +1,5 @@
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker, DEFAULT_FLUSH};
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker};
use crate::filter::IpFilter;
use crate::protocol::{Message, MessageSink, MessageSource};
use crate::tail::{run_tail, SourceEvent};
use crate::tracelog::Logger;
@@ -8,6 +9,7 @@ use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::sync::{mpsc, Arc};
use std::time::Duration;
use std::thread;
/// Bound on the reader->compressor and compressor->sender channels: enough to
@@ -22,7 +24,14 @@ pub struct SendOptions {
pub final_source_path: PathBuf,
pub compress: bool,
pub chunk_target: usize,
/// How long kept-but-undersized output may wait before being sent
/// anyway. See `Chunker::should_flush`.
pub flush_after: Duration,
pub level: i32,
/// Which packets to forward. `IpFilter::match_all()` when no `--ip` was
/// given -- not a disabled filter, just one that says yes, so there is
/// only ever one code path through the chunker.
pub filter: IpFilter,
pub log: Arc<Logger>,
}
@@ -39,8 +48,32 @@ enum CompressedItem {
/// serializing -- see the plan's Performance section. Non-compress mode
/// collapses to 2 stages since there's no compression work to overlap.
pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()> {
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial {
Box::new(RecordAlignedChunker::new(opts.chunk_target, DEFAULT_FLUSH, opts.is_partial))
// Two independent reasons to parse records, either of which forces the
// record-aligned chunker:
// * `compress`: chunk boundaries become zstd frame boundaries, and the
// format requires every frame to hold a whole number of records
// (STREAMING_ZSTD_FORMAT.md invariant 3). The cut point *is* the format.
// * `is_partial`: a growing source needs `confirmed_len()` -- record
// structure is the only way to tell real content from a preallocating
// writer's not-yet-written tail. Here the chunker is a validator, not
// a framer.
// * a real `--ip` filter: deciding whether to forward a packet means
// finding the packet, and record boundaries are the only way to know
// where one starts. Here the chunker is a demultiplexer.
// A final, uncompressed copy has neither: `run_recv` writes every Data
// payload verbatim and never inspects a boundary, so alignment would buy
// nothing, while requiring pcap structure would break copying pcapng,
// truncated, or otherwise non-pcap files that transfer fine today. So it
// gets the passthrough chunker, framed on the wire only by tail.rs's
// READ_WINDOW.
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial || !opts.filter.is_match_all()
{
Box::new(RecordAlignedChunker::with_filter(
opts.chunk_target,
opts.flush_after,
opts.is_partial,
opts.filter.clone(),
))
} else {
Box::new(RawChunker::new())
};
@@ -238,7 +271,9 @@ mod tests {
final_source_path: src.clone(),
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
@@ -274,7 +309,9 @@ mod tests {
final_source_path: final_src.clone(),
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
log: Arc::new(Logger::none()),
};
@@ -321,7 +358,9 @@ mod tests {
final_source_path: src.clone(),
compress: true,
chunk_target: 512, // small target to force multiple frames
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
@@ -357,10 +396,138 @@ mod tests {
final_source_path: src.clone(),
compress: true,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
log: Arc::new(Logger::none()),
};
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
assert!(!dest_final.exists());
}
fn eth_header_bytes() -> Vec<u8> {
let mut h = header_bytes(0xA1B2C3D4);
h[20..24].copy_from_slice(&1u32.to_le_bytes()); // LINKTYPE_ETHERNET
h
}
/// A record holding an Ethernet/IPv4 packet, padded to `len` captured bytes.
fn ip_record(src: [u8; 4], dst: [u8; 4], len: usize) -> Vec<u8> {
let mut pkt = vec![0u8; len.max(34)];
pkt[12..14].copy_from_slice(&0x0800u16.to_be_bytes());
pkt[14] = 0x45;
pkt[26..30].copy_from_slice(&src);
pkt[30..34].copy_from_slice(&dst);
record_bytes(&pkt, 1_700_000_000)
}
fn ip_filter(specs: &[&str]) -> IpFilter {
IpFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
}
/// Source stream plus the exact pcap a `--ip 1.1.1.1` transfer should
/// produce from it: same global header, only the matching records.
fn filtered_fixture(n: usize) -> (Vec<u8>, Vec<u8>) {
let mut source = eth_header_bytes();
let mut expected = eth_header_bytes();
for i in 0..n {
if i % 5 == 0 {
let hit = ip_record([1, 1, 1, 1], [9, 9, 9, 9], 100 + i);
source.extend_from_slice(&hit);
expected.extend_from_slice(&hit);
} else {
source.extend_from_slice(&ip_record([9, 9, 9, 9], [8, 8, 8, 8], 100 + i));
}
}
(source, expected)
}
#[test]
fn filtered_transfer_carries_only_matching_packets() {
let dir = tempdir().unwrap();
let src = dir.path().join("cap.pcap");
let (source, expected) = filtered_fixture(50);
std::fs::write(&src, &source).unwrap();
let dest_final = dir.path().join("out.pcap");
let dest_temp = dir.path().join("out.pcap.partial");
let opts = SendOptions {
source_path: src.clone(),
is_partial: false,
final_source_path: src.clone(),
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: ip_filter(&["1.1.1.1"]),
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
// A filter forces the record-aligned chunker even though this is a
// final, uncompressed copy -- so the output is a real pcap, not a
// byte-for-byte copy of the source.
assert_eq!(std::fs::read(&dest_final).unwrap(), expected);
assert!(expected.len() < source.len());
}
#[test]
fn filtered_compressed_transfer_round_trips_to_the_filtered_pcap() {
let dir = tempdir().unwrap();
let src = dir.path().join("cap.pcap");
let (source, expected) = filtered_fixture(200);
std::fs::write(&src, &source).unwrap();
let dest_final = dir.path().join("out.pcap.zst");
let dest_temp = dir.path().join("out.pcap.zst.partial");
let opts = SendOptions {
source_path: src.clone(),
is_partial: false,
final_source_path: src.clone(),
compress: true,
chunk_target: 512,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: ip_filter(&["1.1.1.1"]),
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
let out_bytes = std::fs::read(&dest_final).unwrap();
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
// The whole point of the container: decompressing yields a valid
// pcap, here the filtered one.
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
// Header frame still declares exactly the 24-byte global header.
let header_content_size = zstd::zstd_safe::get_frame_content_size(&out_bytes[14..])
.unwrap()
.unwrap();
assert_eq!(header_content_size, 24);
}
#[test]
fn a_filter_matching_nothing_produces_a_header_only_pcap() {
// Not an error: the transfer completes and the destination is a
// valid, empty capture.
let dir = tempdir().unwrap();
let src = dir.path().join("cap.pcap");
let (source, _) = filtered_fixture(20);
std::fs::write(&src, &source).unwrap();
let dest_final = dir.path().join("out.pcap");
let dest_temp = dir.path().join("out.pcap.partial");
let opts = SendOptions {
source_path: src.clone(),
is_partial: false,
final_source_path: src.clone(),
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: ip_filter(&["203.0.113.7"]),
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
}
}