allow inflight filter
This commit is contained in:
1 parent
aeb827b361
commit
8b111ddfdc
9 files changed
+956
-55
No files matched your search
+170
-3
@@ -1,4 +1,5 @@
|
||||
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker, DEFAULT_FLUSH};
|
||||
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker};
|
||||
use crate::filter::IpFilter;
|
||||
use crate::protocol::{Message, MessageSink, MessageSource};
|
||||
use crate::tail::{run_tail, SourceEvent};
|
||||
use crate::tracelog::Logger;
|
||||
@@ -8,6 +9,7 @@ use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
|
||||
/// Bound on the reader->compressor and compressor->sender channels: enough to
|
||||
@@ -22,7 +24,14 @@ pub struct SendOptions {
|
||||
pub final_source_path: PathBuf,
|
||||
pub compress: bool,
|
||||
pub chunk_target: usize,
|
||||
/// How long kept-but-undersized output may wait before being sent
|
||||
/// anyway. See `Chunker::should_flush`.
|
||||
pub flush_after: Duration,
|
||||
pub level: i32,
|
||||
/// Which packets to forward. `IpFilter::match_all()` when no `--ip` was
|
||||
/// given -- not a disabled filter, just one that says yes, so there is
|
||||
/// only ever one code path through the chunker.
|
||||
pub filter: IpFilter,
|
||||
pub log: Arc<Logger>,
|
||||
}
|
||||
|
||||
@@ -39,8 +48,32 @@ enum CompressedItem {
|
||||
/// serializing -- see the plan's Performance section. Non-compress mode
|
||||
/// collapses to 2 stages since there's no compression work to overlap.
|
||||
pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()> {
|
||||
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial {
|
||||
Box::new(RecordAlignedChunker::new(opts.chunk_target, DEFAULT_FLUSH, opts.is_partial))
|
||||
// Two independent reasons to parse records, either of which forces the
|
||||
// record-aligned chunker:
|
||||
// * `compress`: chunk boundaries become zstd frame boundaries, and the
|
||||
// format requires every frame to hold a whole number of records
|
||||
// (STREAMING_ZSTD_FORMAT.md invariant 3). The cut point *is* the format.
|
||||
// * `is_partial`: a growing source needs `confirmed_len()` -- record
|
||||
// structure is the only way to tell real content from a preallocating
|
||||
// writer's not-yet-written tail. Here the chunker is a validator, not
|
||||
// a framer.
|
||||
// * a real `--ip` filter: deciding whether to forward a packet means
|
||||
// finding the packet, and record boundaries are the only way to know
|
||||
// where one starts. Here the chunker is a demultiplexer.
|
||||
// A final, uncompressed copy has neither: `run_recv` writes every Data
|
||||
// payload verbatim and never inspects a boundary, so alignment would buy
|
||||
// nothing, while requiring pcap structure would break copying pcapng,
|
||||
// truncated, or otherwise non-pcap files that transfer fine today. So it
|
||||
// gets the passthrough chunker, framed on the wire only by tail.rs's
|
||||
// READ_WINDOW.
|
||||
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial || !opts.filter.is_match_all()
|
||||
{
|
||||
Box::new(RecordAlignedChunker::with_filter(
|
||||
opts.chunk_target,
|
||||
opts.flush_after,
|
||||
opts.is_partial,
|
||||
opts.filter.clone(),
|
||||
))
|
||||
} else {
|
||||
Box::new(RawChunker::new())
|
||||
};
|
||||
@@ -238,7 +271,9 @@ mod tests {
|
||||
final_source_path: src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
@@ -274,7 +309,9 @@ mod tests {
|
||||
final_source_path: final_src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
|
||||
@@ -321,7 +358,9 @@ mod tests {
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512, // small target to force multiple frames
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
@@ -357,10 +396,138 @@ mod tests {
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
|
||||
assert!(!dest_final.exists());
|
||||
}
|
||||
|
||||
fn eth_header_bytes() -> Vec<u8> {
|
||||
let mut h = header_bytes(0xA1B2C3D4);
|
||||
h[20..24].copy_from_slice(&1u32.to_le_bytes()); // LINKTYPE_ETHERNET
|
||||
h
|
||||
}
|
||||
|
||||
/// A record holding an Ethernet/IPv4 packet, padded to `len` captured bytes.
|
||||
fn ip_record(src: [u8; 4], dst: [u8; 4], len: usize) -> Vec<u8> {
|
||||
let mut pkt = vec![0u8; len.max(34)];
|
||||
pkt[12..14].copy_from_slice(&0x0800u16.to_be_bytes());
|
||||
pkt[14] = 0x45;
|
||||
pkt[26..30].copy_from_slice(&src);
|
||||
pkt[30..34].copy_from_slice(&dst);
|
||||
record_bytes(&pkt, 1_700_000_000)
|
||||
}
|
||||
|
||||
fn ip_filter(specs: &[&str]) -> IpFilter {
|
||||
IpFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
|
||||
}
|
||||
|
||||
/// Source stream plus the exact pcap a `--ip 1.1.1.1` transfer should
|
||||
/// produce from it: same global header, only the matching records.
|
||||
fn filtered_fixture(n: usize) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut source = eth_header_bytes();
|
||||
let mut expected = eth_header_bytes();
|
||||
for i in 0..n {
|
||||
if i % 5 == 0 {
|
||||
let hit = ip_record([1, 1, 1, 1], [9, 9, 9, 9], 100 + i);
|
||||
source.extend_from_slice(&hit);
|
||||
expected.extend_from_slice(&hit);
|
||||
} else {
|
||||
source.extend_from_slice(&ip_record([9, 9, 9, 9], [8, 8, 8, 8], 100 + i));
|
||||
}
|
||||
}
|
||||
(source, expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filtered_transfer_carries_only_matching_packets() {
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap");
|
||||
let (source, expected) = filtered_fixture(50);
|
||||
std::fs::write(&src, &source).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap");
|
||||
let dest_temp = dir.path().join("out.pcap.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: src.clone(),
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["1.1.1.1"]),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
|
||||
// A filter forces the record-aligned chunker even though this is a
|
||||
// final, uncompressed copy -- so the output is a real pcap, not a
|
||||
// byte-for-byte copy of the source.
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), expected);
|
||||
assert!(expected.len() < source.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filtered_compressed_transfer_round_trips_to_the_filtered_pcap() {
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap");
|
||||
let (source, expected) = filtered_fixture(200);
|
||||
std::fs::write(&src, &source).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap.zst");
|
||||
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: src.clone(),
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["1.1.1.1"]),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
|
||||
let out_bytes = std::fs::read(&dest_final).unwrap();
|
||||
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
|
||||
// The whole point of the container: decompressing yields a valid
|
||||
// pcap, here the filtered one.
|
||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||
// Header frame still declares exactly the 24-byte global header.
|
||||
let header_content_size = zstd::zstd_safe::get_frame_content_size(&out_bytes[14..])
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(header_content_size, 24);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_filter_matching_nothing_produces_a_header_only_pcap() {
|
||||
// Not an error: the transfer completes and the destination is a
|
||||
// valid, empty capture.
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap");
|
||||
let (source, _) = filtered_fixture(20);
|
||||
std::fs::write(&src, &source).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap");
|
||||
let dest_temp = dir.path().join("out.pcap.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: src.clone(),
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["203.0.113.7"]),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user