Compare commits
10
Commits
aeb827b361
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb67bbafa1 | ||
|
|
6b68deb047 | ||
|
|
cc698b7826 | ||
|
|
3eb779a6aa | ||
|
|
b46d6a936c | ||
|
|
6a80aaf461 | ||
|
|
f21b6c53e8 | ||
|
|
a9e283a87c | ||
|
|
a7417dcdd6 | ||
|
|
8b111ddfdc |
No files matched your search
+361
-49
@@ -1,11 +1,13 @@
|
||||
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
||||
use anyhow::{Result, bail};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub const GLOBAL_HEADER_LEN: usize = 24;
|
||||
pub const RECORD_HEADER_LEN: usize = 16;
|
||||
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
||||
#[allow(dead_code)]
|
||||
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
|
||||
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
|
||||
/// Sanity bound on a record's declared payload length, per
|
||||
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
||||
/// sources -- see `RecordAlignedChunker::partial`.
|
||||
@@ -146,11 +148,38 @@ pub struct RecordAlignedChunker {
|
||||
header_ready: bool,
|
||||
endianness: Option<Endianness>,
|
||||
snaplen: u32,
|
||||
/// Link-layer type from global-header bytes [20..24], needed to know
|
||||
/// where the IP header starts. Only meaningful once `header_done`.
|
||||
linktype: u32,
|
||||
/// Source bytes read but not yet confirmed as complete records. Every
|
||||
/// record that `rescan` confirms is consumed out of here immediately,
|
||||
/// so this holds only the unconfirmed tail -- bounded by one read
|
||||
/// window, no matter how little the filter keeps.
|
||||
pending: Vec<u8>,
|
||||
complete_len: usize,
|
||||
emitted_len: u64,
|
||||
last_growth: Instant,
|
||||
target: usize,
|
||||
/// Cumulative source bytes consumed (confirmed and removed from
|
||||
/// `pending`), NOT bytes emitted downstream -- the two differ whenever
|
||||
/// the filter drops a record. Source-offset accounting depends only on
|
||||
/// this.
|
||||
consumed_len: u64,
|
||||
/// Confirmed records the filter kept, accumulating across many
|
||||
/// `pending` recycles until they reach `target`. This is what gets
|
||||
/// emitted, so chunk (and therefore zstd frame) size is measured in
|
||||
/// KEPT bytes -- a filter keeping 1% of the traffic still produces
|
||||
/// full-size frames rather than 1%-size ones.
|
||||
kept: Vec<u8>,
|
||||
filter: PacketFilter,
|
||||
/// When the oldest byte currently in `kept` was confirmed, i.e. how
|
||||
/// long the pending output has been waiting. `None` when `kept` is
|
||||
/// empty.
|
||||
///
|
||||
/// This deliberately tracks the age of the OUTPUT, not of the last
|
||||
/// source growth: with a selective filter the source can grow
|
||||
/// continuously (resetting any source-based timer forever) while the
|
||||
/// handful of matching records sit here unsent. Timing from the kept
|
||||
/// bytes is what keeps `flush_after` an actual latency bound rather
|
||||
/// than an idle-only one.
|
||||
kept_since: Option<Instant>,
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
/// Whether this source is a still-growing (partial) file. Gates every
|
||||
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
||||
@@ -161,17 +190,34 @@ pub struct RecordAlignedChunker {
|
||||
}
|
||||
|
||||
impl RecordAlignedChunker {
|
||||
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
|
||||
/// Unfiltered: keeps every record. Equivalent to `with_filter` with
|
||||
/// `PacketFilter::match_all()` -- which is not a separate code path, just
|
||||
/// a filter that says yes, so this and the filtered case exercise the
|
||||
/// same record-confirmation logic. Production goes through
|
||||
/// `with_filter`; this is the convenience the tests use.
|
||||
#[allow(dead_code)]
|
||||
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
|
||||
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
||||
}
|
||||
|
||||
pub fn with_filter(
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
partial: bool,
|
||||
filter: PacketFilter,
|
||||
) -> Self {
|
||||
Self {
|
||||
header_buf: Vec::with_capacity(GLOBAL_HEADER_LEN),
|
||||
header_done: false,
|
||||
header_ready: false,
|
||||
endianness: None,
|
||||
snaplen: 0,
|
||||
linktype: 0,
|
||||
pending: Vec::new(),
|
||||
complete_len: 0,
|
||||
emitted_len: 0,
|
||||
last_growth: Instant::now(),
|
||||
consumed_len: 0,
|
||||
kept: Vec::new(),
|
||||
filter,
|
||||
kept_since: None,
|
||||
target,
|
||||
flush_after,
|
||||
partial,
|
||||
@@ -191,19 +237,49 @@ impl RecordAlignedChunker {
|
||||
}
|
||||
let e = detect_endianness(&self.header_buf)?;
|
||||
self.snaplen = read_u32(&self.header_buf[16..20], e);
|
||||
self.linktype = read_u32(&self.header_buf[20..24], e);
|
||||
self.endianness = Some(e);
|
||||
self.header_done = true;
|
||||
self.header_ready = true;
|
||||
// A filter can only find addresses in an Ethernet frame, so on any
|
||||
// other link type it keeps nothing and the transfer silently
|
||||
// produces a header-only pcap. `tcpdump -i any` (LINUX_SLL, 113) is
|
||||
// the easy way to hit this, so say so out loud. stderr, never
|
||||
// stdout: in --server mode stdout is the wire.
|
||||
if !self.filter.is_match_all() && self.linktype != LINKTYPE_ETHERNET {
|
||||
eprintln!(
|
||||
"scpcap: warning: --ip/--mac filtering needs Ethernet (link type {LINKTYPE_ETHERNET}), \
|
||||
but this capture is link type {}; no packets will match",
|
||||
self.linktype
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Walks `pending` confirming whole records, hands each confirmed
|
||||
/// record to the filter (keeping the matches in `kept`), and consumes
|
||||
/// the confirmed bytes out of `pending`.
|
||||
///
|
||||
/// Confirming/consuming and *emitting* are deliberately separate. If
|
||||
/// consumption were tied to emission, a filter keeping 1% of the
|
||||
/// traffic would hold the other 99% in `pending` for as long as `kept`
|
||||
/// stayed under `target` -- the whole un-emitted source accumulating in
|
||||
/// memory, gigabytes on a large capture. Consuming here caps `pending`
|
||||
/// at the unconfirmed tail while `kept` fills up across many passes.
|
||||
///
|
||||
/// `cursor` is deliberately a local, not a field: every confirmed byte
|
||||
/// leaves `pending` before this returns, so there is no persistent
|
||||
/// "confirmed but still buffered" region for the source-offset
|
||||
/// accounting (or `refresh_tail`) to have to reason about.
|
||||
fn rescan(&mut self, trust_trailing_record: bool) {
|
||||
let e = match self.endianness {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
};
|
||||
let max_incl_len = self.max_incl_len();
|
||||
let mut cursor = 0usize;
|
||||
loop {
|
||||
let remaining = &self.pending[self.complete_len..];
|
||||
let remaining = &self.pending[cursor..];
|
||||
if remaining.len() < RECORD_HEADER_LEN {
|
||||
break;
|
||||
}
|
||||
@@ -224,7 +300,7 @@ impl RecordAlignedChunker {
|
||||
if ts_sec == 0 {
|
||||
break;
|
||||
}
|
||||
if incl_len > self.max_incl_len() {
|
||||
if incl_len > max_incl_len {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -235,13 +311,13 @@ impl RecordAlignedChunker {
|
||||
// Require positive proof the writer has moved past this
|
||||
// record -- the start of the NEXT record's header, itself
|
||||
// looking real -- before confirming it. This makes
|
||||
// complete_len always sit (at least) one record behind
|
||||
// confirmed_len() always sit (at least) one record behind
|
||||
// whatever's currently visible, by construction, for every
|
||||
// record, not just "the last one" as a special case.
|
||||
let after = &remaining[record_total..];
|
||||
let next_is_proven = after.len() >= RECORD_HEADER_LEN
|
||||
&& read_u32(&after[0..4], e) != 0
|
||||
&& (read_u32(&after[8..12], e) as usize) <= self.max_incl_len();
|
||||
&& (read_u32(&after[8..12], e) as usize) <= max_incl_len;
|
||||
|
||||
// Escape hatch (only when the caller is certain nothing
|
||||
// more will EVER arrive): a record that is itself plausible
|
||||
@@ -259,15 +335,30 @@ impl RecordAlignedChunker {
|
||||
break;
|
||||
}
|
||||
}
|
||||
self.complete_len += record_total;
|
||||
|
||||
// Confirmed. Show the captured bytes (everything after the
|
||||
// record header) to the filter, and keep the whole record --
|
||||
// header included -- if it matches, so `kept` stays a valid
|
||||
// sequence of pcap records.
|
||||
let start = cursor;
|
||||
let end = cursor + record_total;
|
||||
if self.filter.matches(&self.pending[start + RECORD_HEADER_LEN..end], self.linktype) {
|
||||
self.kept.extend_from_slice(&self.pending[start..end]);
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
if cursor > 0 {
|
||||
self.pending.drain(..cursor);
|
||||
self.consumed_len += cursor as u64;
|
||||
}
|
||||
if !self.kept.is_empty() && self.kept_since.is_none() {
|
||||
self.kept_since = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
fn drain_complete(&mut self) -> (ChunkKind, Vec<u8>) {
|
||||
let chunk: Vec<u8> = self.pending.drain(..self.complete_len).collect();
|
||||
self.emitted_len += self.complete_len as u64;
|
||||
self.complete_len = 0;
|
||||
(ChunkKind::Data, chunk)
|
||||
fn take_kept(&mut self) -> (ChunkKind, Vec<u8>) {
|
||||
self.kept_since = None;
|
||||
(ChunkKind::Data, std::mem::take(&mut self.kept))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -296,7 +387,6 @@ impl Chunker for RecordAlignedChunker {
|
||||
if !rest.is_empty() {
|
||||
self.pending.extend_from_slice(rest);
|
||||
self.rescan(false);
|
||||
self.last_growth = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -306,19 +396,27 @@ impl Chunker for RecordAlignedChunker {
|
||||
self.header_ready = false;
|
||||
return Some((ChunkKind::Header, std::mem::take(&mut self.header_buf)));
|
||||
}
|
||||
if self.complete_len >= self.target {
|
||||
return Some(self.drain_complete());
|
||||
// All three cut points gate on `kept`, never on how much source was
|
||||
// consumed: a stretch of source where the filter matched nothing
|
||||
// must produce no chunk at all, rather than an empty one. An empty
|
||||
// chunk would become a zero-content zstd frame, which
|
||||
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
||||
if self.kept.len() >= self.target.get() {
|
||||
return Some(self.take_kept());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn should_flush(&self, now: Instant) -> bool {
|
||||
self.complete_len > 0 && now.duration_since(self.last_growth) >= self.flush_after
|
||||
// `kept_since` is Some exactly when `kept` is non-empty, so this
|
||||
// can never ask for a flush that would produce an empty chunk.
|
||||
self.kept_since
|
||||
.is_some_and(|since| now.duration_since(since) >= self.flush_after)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Option<(ChunkKind, Vec<u8>)> {
|
||||
if self.complete_len > 0 {
|
||||
Some(self.drain_complete())
|
||||
if !self.kept.is_empty() {
|
||||
Some(self.take_kept())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -328,8 +426,8 @@ impl Chunker for RecordAlignedChunker {
|
||||
if !self.header_done {
|
||||
bail!("source ended before a complete pcap global header was read");
|
||||
}
|
||||
if self.complete_len > 0 {
|
||||
Ok(Some(self.drain_complete()))
|
||||
if !self.kept.is_empty() {
|
||||
Ok(Some(self.take_kept()))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
@@ -339,7 +437,7 @@ impl Chunker for RecordAlignedChunker {
|
||||
if !self.header_done {
|
||||
self.header_buf.len() as u64
|
||||
} else {
|
||||
GLOBAL_HEADER_LEN as u64 + self.emitted_len + self.pending.len() as u64
|
||||
GLOBAL_HEADER_LEN as u64 + self.consumed_len + self.pending.len() as u64
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,7 +450,14 @@ impl Chunker for RecordAlignedChunker {
|
||||
// treating the stuck zero bytes as already consumed.
|
||||
0
|
||||
} else {
|
||||
GLOBAL_HEADER_LEN as u64 + self.emitted_len + self.complete_len as u64
|
||||
// Source-offset accounting, entirely independent of what the
|
||||
// filter kept: every confirmed record is consumed the moment
|
||||
// `rescan` proves it, so `consumed_len` alone is the confirmed
|
||||
// prefix. (This is the same number the old
|
||||
// `emitted_len + complete_len` produced; consuming on
|
||||
// confirmation just moves the quantity from the second term to
|
||||
// the first.)
|
||||
GLOBAL_HEADER_LEN as u64 + self.consumed_len
|
||||
}
|
||||
}
|
||||
|
||||
@@ -373,30 +478,24 @@ impl Chunker for RecordAlignedChunker {
|
||||
if !self.header_done {
|
||||
return Ok(());
|
||||
}
|
||||
// header_done just flipped true, so complete_len is still 0 here
|
||||
// (nothing beyond the header has been considered yet) --
|
||||
// pending is simply the fresh post-header bytes.
|
||||
// header_done just flipped true, so nothing beyond the header
|
||||
// has been considered yet -- pending is simply the fresh
|
||||
// post-header bytes.
|
||||
self.pending = bytes[GLOBAL_HEADER_LEN..].to_vec();
|
||||
self.rescan(trust_trailing_record);
|
||||
self.last_growth = Instant::now();
|
||||
return Ok(());
|
||||
}
|
||||
// `pending[..complete_len]` is CONFIRMED content that just hasn't
|
||||
// been drained (emitted) yet -- e.g. because it's still under
|
||||
// `target`. `bytes` starts exactly at `confirmed_len()` (checked
|
||||
// above), i.e. exactly where that confirmed prefix ends, so it must
|
||||
// be appended after it, never used to replace the whole buffer --
|
||||
// doing that would silently discard already-confirmed, already
|
||||
// real, not-yet-emitted content on every single re-peek, which is
|
||||
// catastrophic for a long-running growing transfer (each poll would
|
||||
// re-derive from scratch instead of building on prior confirmed
|
||||
// progress, and once the fresh read window is too small to
|
||||
// re-establish the same progress alone -- e.g. capped by
|
||||
// READ_WINDOW -- confirmed_len can never recover).
|
||||
self.pending.truncate(self.complete_len);
|
||||
// `pending` holds only the UNCONFIRMED tail -- `rescan` consumes
|
||||
// every record it confirms, so nothing confirmed is still sitting
|
||||
// here waiting to be emitted. `bytes` is a fresh read starting at
|
||||
// `confirmed_len()` (checked above), i.e. exactly where that
|
||||
// unconfirmed tail begins, so replacing the buffer wholesale is
|
||||
// correct and discards nothing: anything already confirmed lives in
|
||||
// `consumed_len` (for source accounting) and, if the filter kept
|
||||
// it, in `kept` (for emission). Neither is touched here.
|
||||
self.pending.clear();
|
||||
self.pending.extend_from_slice(bytes);
|
||||
self.rescan(trust_trailing_record);
|
||||
self.last_growth = Instant::now();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -411,6 +510,10 @@ mod tests {
|
||||
h
|
||||
}
|
||||
|
||||
fn nz(n: usize) -> NonZeroUsize {
|
||||
NonZeroUsize::new(n).unwrap()
|
||||
}
|
||||
|
||||
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
||||
record_bytes_with_ts(payload, big_endian, 0)
|
||||
}
|
||||
@@ -473,7 +576,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
||||
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
||||
for i in 0..20u8 {
|
||||
@@ -508,7 +611,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
||||
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
|
||||
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
|
||||
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
||||
c.ready_chunk(); // drain header
|
||||
|
||||
@@ -771,4 +874,213 @@ mod tests {
|
||||
"confirmed_len must never regress below what was already established"
|
||||
);
|
||||
}
|
||||
|
||||
// --- IP filtering -------------------------------------------------
|
||||
//
|
||||
// Note that every test above this point already exercises the
|
||||
// filtering code path, with a match-all filter: there is no separate
|
||||
// unfiltered implementation to regress against.
|
||||
|
||||
/// A global header declaring Ethernet (link type 1) at bytes [20..24],
|
||||
/// which the filter needs in order to find an IP header at all.
|
||||
fn eth_header_bytes() -> Vec<u8> {
|
||||
let mut h = header_bytes(0xA1B2C3D4);
|
||||
h[20..24].copy_from_slice(&1u32.to_le_bytes());
|
||||
h
|
||||
}
|
||||
|
||||
/// A pcap record holding an Ethernet/IPv4 packet between `src` and `dst`,
|
||||
/// padded out to `payload_len` total captured bytes.
|
||||
fn ip_record(src: [u8; 4], dst: [u8; 4], payload_len: usize) -> Vec<u8> {
|
||||
let mut pkt = vec![0u8; payload_len.max(34)];
|
||||
pkt[12..14].copy_from_slice(&0x0800u16.to_be_bytes()); // ethertype IPv4
|
||||
pkt[14] = 0x45; // version 4, IHL 5
|
||||
pkt[26..30].copy_from_slice(&src);
|
||||
pkt[30..34].copy_from_slice(&dst);
|
||||
record_bytes_with_ts(&pkt, false, 1_700_000_000)
|
||||
}
|
||||
|
||||
fn filter_for(specs: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>(), &[]).unwrap()
|
||||
}
|
||||
|
||||
fn drain_all(c: &mut RecordAlignedChunker) -> Vec<(ChunkKind, Vec<u8>)> {
|
||||
let mut out = Vec::new();
|
||||
while let Some(chunk) = c.ready_chunk() {
|
||||
out.push(chunk);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
const WANTED: [u8; 4] = [1, 1, 1, 1];
|
||||
const OTHER: [u8; 4] = [9, 9, 9, 9];
|
||||
const THIRD: [u8; 4] = [8, 8, 8, 8];
|
||||
|
||||
#[test]
|
||||
fn keeps_only_matching_records_and_keeps_them_whole() {
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let hit_src = ip_record(WANTED, OTHER, 60);
|
||||
let hit_dst = ip_record(OTHER, WANTED, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&miss);
|
||||
stream.extend_from_slice(&hit_src);
|
||||
stream.extend_from_slice(&miss);
|
||||
stream.extend_from_slice(&hit_dst);
|
||||
stream.extend_from_slice(&miss);
|
||||
c.feed(&stream).unwrap();
|
||||
|
||||
let chunks = drain_all(&mut c);
|
||||
assert_eq!(chunks[0].0, ChunkKind::Header);
|
||||
assert_eq!(chunks[0].1, eth_header_bytes(), "header is never filtered");
|
||||
|
||||
// Records come out whole (16-byte header included) and in order.
|
||||
let data: Vec<u8> = chunks[1..].iter().flat_map(|(_, d)| d.clone()).collect();
|
||||
let mut expected = hit_src.clone();
|
||||
expected.extend_from_slice(&hit_dst);
|
||||
assert_eq!(data, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_counts_kept_bytes_not_source_bytes() {
|
||||
// The reason filtering lives at record confirmation rather than in a
|
||||
// post-pass over emitted chunks: with 1% of traffic kept, chunks
|
||||
// (and therefore zstd frames) must still reach `target`, not shrink
|
||||
// to 1% of it.
|
||||
let hit = ip_record(WANTED, OTHER, 100);
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let target = 10 * hit.len();
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
// 1 kept in every 100 records, 1000 records total => 10 kept.
|
||||
for i in 0..1000 {
|
||||
c.feed(if i % 100 == 0 { &hit } else { &miss }).unwrap();
|
||||
}
|
||||
|
||||
let chunks = drain_all(&mut c);
|
||||
assert_eq!(chunks.len(), 1, "expected one full-size chunk, got {}", chunks.len());
|
||||
assert!(
|
||||
chunks[0].1.len() >= target,
|
||||
"chunk is {} bytes, below the {target}-byte target",
|
||||
chunks[0].1.len()
|
||||
);
|
||||
assert_eq!(chunks[0].1.len(), 10 * hit.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_buffer_does_not_grow_when_nothing_matches() {
|
||||
// Confirmed records are consumed from `pending` immediately rather
|
||||
// than when a chunk is emitted. Without that, a filter that keeps
|
||||
// nothing would accumulate the entire source in memory while
|
||||
// waiting for `kept` to reach `target`.
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let mut c = RecordAlignedChunker::with_filter(
|
||||
DEFAULT_TARGET_BYTES,
|
||||
DEFAULT_FLUSH,
|
||||
false,
|
||||
filter_for(&["1.1.1.1"]),
|
||||
);
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
let mut source_len = GLOBAL_HEADER_LEN as u64;
|
||||
for _ in 0..5000 {
|
||||
c.feed(&miss).unwrap();
|
||||
source_len += miss.len() as u64;
|
||||
}
|
||||
|
||||
assert!(
|
||||
c.pending.len() < miss.len() * 2,
|
||||
"unconfirmed tail grew to {} bytes; confirmed records are not being consumed",
|
||||
c.pending.len()
|
||||
);
|
||||
assert!(c.kept.is_empty());
|
||||
// ...and the source is nonetheless fully accounted for.
|
||||
assert_eq!(c.confirmed_len(), source_len);
|
||||
assert_eq!(c.read_from(), source_len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fully_dropped_region_yields_no_chunk_not_an_empty_one() {
|
||||
// An empty chunk would be compressed into a zero-content zstd frame,
|
||||
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
for _ in 0..20 {
|
||||
c.feed(&miss).unwrap();
|
||||
}
|
||||
// target is 1 byte, so anything kept at all would be emitted.
|
||||
assert!(c.ready_chunk().is_none());
|
||||
assert!(c.flush().is_none());
|
||||
assert!(!c.should_flush(Instant::now() + Duration::from_secs(60)));
|
||||
assert!(c.finish().unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_accounting_is_unaffected_by_what_the_filter_keeps() {
|
||||
// The direct statement of the invariant the rest of the system
|
||||
// depends on: confirmed_len()/read_from() are SOURCE offsets, which
|
||||
// tail.rs compares against the real file length to decide finalize.
|
||||
// Dropping packets must not move them by a single byte.
|
||||
let mut stream = eth_header_bytes();
|
||||
for i in 0..40u8 {
|
||||
let src = if i % 3 == 0 { WANTED } else { OTHER };
|
||||
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
||||
}
|
||||
|
||||
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||
for c in [&mut all, &mut some, &mut none] {
|
||||
c.feed(&stream).unwrap();
|
||||
}
|
||||
|
||||
for c in [&some, &none] {
|
||||
assert_eq!(c.confirmed_len(), all.confirmed_len());
|
||||
assert_eq!(c.read_from(), all.read_from());
|
||||
}
|
||||
assert_eq!(all.confirmed_len(), stream.len() as u64);
|
||||
|
||||
// Meanwhile the kept content really does differ.
|
||||
assert_eq!(all.kept.len(), stream.len() - GLOBAL_HEADER_LEN);
|
||||
assert!(!some.kept.is_empty() && some.kept.len() < all.kept.len());
|
||||
assert!(none.kept.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filtering_a_growing_source_confirms_and_drops_across_refreshes() {
|
||||
// The partial path reaches records through refresh_tail, which
|
||||
// replaces the unconfirmed tail wholesale. A record must be filtered
|
||||
// exactly once even though the bytes around it are re-read.
|
||||
let hit = ip_record(WANTED, OTHER, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&miss);
|
||||
stream.extend_from_slice(&hit);
|
||||
stream.extend_from_slice(&miss);
|
||||
stream.extend_from_slice(&hit);
|
||||
|
||||
// Feed it in growing prefixes, each time re-reading from the current
|
||||
// confirmed offset, exactly as tail.rs does.
|
||||
for end in [40, 100, stream.len() - 10, stream.len()] {
|
||||
let offset = c.confirmed_len() as usize;
|
||||
c.refresh_tail(offset as u64, &stream[offset..end], false).unwrap();
|
||||
}
|
||||
// The trailing record has no next-record proof yet, so only the
|
||||
// first `hit` is confirmed.
|
||||
assert_eq!(c.kept, hit);
|
||||
|
||||
// Source stops growing and the caller vouches for the tail.
|
||||
let offset = c.confirmed_len() as usize;
|
||||
c.refresh_tail(offset as u64, &stream[offset..], true).unwrap();
|
||||
assert_eq!(c.confirmed_len(), stream.len() as u64);
|
||||
let mut both = hit.clone();
|
||||
both.extend_from_slice(&hit);
|
||||
assert_eq!(c.kept, both, "each kept record appears exactly once");
|
||||
}
|
||||
}
|
||||
+38
@@ -33,6 +33,29 @@ pub struct ClientCli {
|
||||
#[arg(long, default_value = "scpcap")]
|
||||
pub remote_exe: String,
|
||||
|
||||
/// How long a partial chunk may wait before being sent anyway, bounding
|
||||
/// latency on a still-growing source: "100ms", "2s", or a bare number of
|
||||
/// milliseconds. Chunks that reach --chunk sooner are sent immediately;
|
||||
/// this only governs the leftover. Matters most with --ip, where a
|
||||
/// selective filter would otherwise take a long time to fill a chunk.
|
||||
#[arg(long, value_name = "DURATION", default_value = "100ms")]
|
||||
pub flush_interval: String,
|
||||
|
||||
/// Only forward packets whose source or destination IP matches. May be
|
||||
/// repeated; a packet is kept if it matches any entry. Accepts a bare
|
||||
/// address ("1.1.1.1", "2606:4700::1111") or a CIDR prefix
|
||||
/// ("10.0.0.0/8"). Requires an Ethernet-linktype pcap source.
|
||||
#[arg(long = "ip", value_name = "ADDR")]
|
||||
pub ip: Vec<String>,
|
||||
|
||||
/// Only forward packets whose Ethernet source or destination MAC
|
||||
/// matches, e.g. to pull one switch connection out of a capture by its
|
||||
/// upstream MAC ("aa:bb:cc:dd:ee:ff"). May be repeated; a packet is kept
|
||||
/// if it matches any entry. Combined with --ip, a packet must match
|
||||
/// both. Requires an Ethernet-linktype pcap source.
|
||||
#[arg(long = "mac", value_name = "ADDR")]
|
||||
pub mac: Vec<String>,
|
||||
|
||||
/// Append a diagnostic trace of this (local) process's transfer events
|
||||
/// to this file -- positions, rename detection, finalize -- for
|
||||
/// figuring out where a stalled transfer is stuck.
|
||||
@@ -80,6 +103,21 @@ pub struct ServerCli {
|
||||
#[arg(long, default_value_t = crate::zstd_frame::DEFAULT_LEVEL)]
|
||||
pub level: i32,
|
||||
|
||||
/// Flush interval for a partial chunk (--send mode). See the client's
|
||||
/// --flush-interval.
|
||||
#[arg(long, value_name = "DURATION", default_value = "100ms")]
|
||||
pub flush_interval: String,
|
||||
|
||||
/// IP filter for --send mode; repeated, one per address/prefix. See the
|
||||
/// client's --ip.
|
||||
#[arg(long = "ip", value_name = "ADDR")]
|
||||
pub ip: Vec<String>,
|
||||
|
||||
/// MAC filter for --send mode; repeated, one per address. See the
|
||||
/// client's --mac.
|
||||
#[arg(long = "mac", value_name = "ADDR")]
|
||||
pub mac: Vec<String>,
|
||||
|
||||
/// Destination final path (--recv mode).
|
||||
#[arg(long)]
|
||||
pub dest_final: Option<String>,
|
||||
|
||||
+150
-1
@@ -1,12 +1,14 @@
|
||||
use crate::channel::{ChannelSink, ChannelSource};
|
||||
use crate::cli::ClientCli;
|
||||
use crate::filter::PacketFilter;
|
||||
use crate::naming::{self, resolve_dest_paths, strip_name, Target};
|
||||
use crate::pipeline::{run_recv, run_send, RecvOptions, SendOptions};
|
||||
use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
|
||||
use crate::ssh::{build_server_command, spawn_remote_server};
|
||||
use crate::tracelog::Logger;
|
||||
use anyhow::{bail, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::thread;
|
||||
|
||||
@@ -35,18 +37,46 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
let is_partial = source_stripped.was_partial;
|
||||
let effective_compress = cli.compress.is_some() && !source_stripped.was_zst;
|
||||
let chunk_target = naming::parse_size(&cli.chunk)?;
|
||||
let flush_after = naming::parse_duration(&cli.flush_interval)?;
|
||||
// Parsed up front so a bad --ip fails before anything is spawned or
|
||||
// opened. Only the SENDING side filters, so this is used directly when
|
||||
// we are the sender (local source) and forwarded as args when the
|
||||
// remote is (see the Remote->Local branch). It is never passed to a
|
||||
// --recv server: the receiver writes bytes verbatim and never parses
|
||||
// them.
|
||||
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
|
||||
// Filtering means reading pcap records, which an already-compressed
|
||||
// source doesn't expose. Without this the transfer would still pick the
|
||||
// record-aligned chunker (a filter forces it) and abort deep inside with
|
||||
// "unrecognized pcap global header magic: 0xfd2fb528" -- the zstd magic.
|
||||
// Such a copy is verbatim and works fine unfiltered, so refuse the
|
||||
// combination rather than the source.
|
||||
if !filter.is_match_all() && source_stripped.was_zst {
|
||||
bail!(
|
||||
"cannot apply --ip/--mac to an already-compressed source ({}): \
|
||||
filtering needs to read pcap records. Copy it without --ip/--mac, \
|
||||
or filter the uncompressed capture.",
|
||||
cli.source
|
||||
);
|
||||
}
|
||||
|
||||
let dest = resolve_dest_paths(dest_target.path(), &source_stripped, effective_compress, &cli.extension);
|
||||
|
||||
match (&source_target, &dest_target) {
|
||||
(Target::Local(src), Target::Local(_)) => {
|
||||
refuse_overlap(
|
||||
[Path::new(src), Path::new(&final_source_path)],
|
||||
[Path::new(&dest.temp_path), Path::new(&dest.final_path)],
|
||||
)?;
|
||||
let send_opts = SendOptions {
|
||||
source_path: PathBuf::from(src),
|
||||
is_partial,
|
||||
final_source_path: PathBuf::from(&final_source_path),
|
||||
compress: effective_compress,
|
||||
chunk_target,
|
||||
flush_after,
|
||||
level: cli.level,
|
||||
filter: filter.clone(),
|
||||
log: log.clone(),
|
||||
};
|
||||
let recv_opts = RecvOptions {
|
||||
@@ -83,7 +113,9 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
final_source_path: PathBuf::from(&final_source_path),
|
||||
compress: effective_compress,
|
||||
chunk_target,
|
||||
flush_after,
|
||||
level: cli.level,
|
||||
filter: filter.clone(),
|
||||
log: log.clone(),
|
||||
};
|
||||
let sink: Box<dyn MessageSink> = Box::new(WireSink(stdin));
|
||||
@@ -104,12 +136,22 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
final_source_path.clone(),
|
||||
"--chunk".to_string(),
|
||||
cli.chunk.clone(),
|
||||
"--flush-interval".to_string(),
|
||||
cli.flush_interval.clone(),
|
||||
"--level".to_string(),
|
||||
cli.level.to_string(),
|
||||
];
|
||||
if is_partial {
|
||||
server_args.push("--partial".to_string());
|
||||
}
|
||||
for spec in &cli.ip {
|
||||
server_args.push("--ip".to_string());
|
||||
server_args.push(spec.clone());
|
||||
}
|
||||
for spec in &cli.mac {
|
||||
server_args.push("--mac".to_string());
|
||||
server_args.push(spec.clone());
|
||||
}
|
||||
if effective_compress {
|
||||
server_args.push("--compress".to_string());
|
||||
server_args.push("zstd".to_string());
|
||||
@@ -129,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
};
|
||||
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
||||
let recv_result = run_recv(source, recv_opts);
|
||||
if recv_result.is_err() {
|
||||
// The remote sender only notices we're gone when its next
|
||||
// write fails, which on an idle capture may be never. Don't
|
||||
// wait for that: drop the connection, so its next write
|
||||
// fails and it stops.
|
||||
let _ = child.kill();
|
||||
}
|
||||
let status = child.wait()?;
|
||||
recv_result?;
|
||||
if !status.success() {
|
||||
@@ -141,3 +190,103 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses a local copy whose destination lands on the source. The receiver
|
||||
/// truncates its in-flight file on open and renames over its final one, so
|
||||
/// either collision destroys the capture being read. Copying a capture into
|
||||
/// its own directory is enough to hit it: `dir/cap.pcap.partial` to `dir/`
|
||||
/// resolves to `dir/cap.pcap`, written via `dir/cap.pcap.partial`.
|
||||
///
|
||||
/// `sources` is the source as given plus its post-rename final path, since a
|
||||
/// growing source moves from one to the other mid-transfer.
|
||||
fn refuse_overlap(sources: [&Path; 2], dests: [&Path; 2]) -> Result<()> {
|
||||
for source in sources {
|
||||
for dest in dests {
|
||||
if same_location(source, dest) {
|
||||
bail!(
|
||||
"destination {} is the source {}: copying would overwrite the capture \
|
||||
being read. Copy it to a different directory or name.",
|
||||
dest.display(),
|
||||
source.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether two paths name the same file, whether or not it exists yet: the
|
||||
/// same inode when both exist (catching hard links and symlinks), otherwise
|
||||
/// the same canonical parent directory and file name.
|
||||
fn same_location(a: &Path, b: &Path) -> bool {
|
||||
if let (Ok(ma), Ok(mb)) = (std::fs::metadata(a), std::fs::metadata(b)) {
|
||||
return ma.dev() == mb.dev() && ma.ino() == mb.ino();
|
||||
}
|
||||
match (resolve(a), resolve(b)) {
|
||||
(Some(ra), Some(rb)) => ra == rb,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The canonical form of `path`, or for a file that doesn't exist yet, its
|
||||
/// canonical parent joined with its name. `None` when even the parent
|
||||
/// doesn't exist, in which case nothing can collide with it.
|
||||
fn resolve(path: &Path) -> Option<PathBuf> {
|
||||
if let Ok(canonical) = std::fs::canonicalize(path) {
|
||||
return Some(canonical);
|
||||
}
|
||||
let parent = match path.parent() {
|
||||
Some(p) if !p.as_os_str().is_empty() => p,
|
||||
_ => Path::new("."),
|
||||
};
|
||||
Some(std::fs::canonicalize(parent).ok()?.join(path.file_name()?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn copying_a_growing_capture_into_its_own_directory_is_refused() {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
std::fs::write(&partial, b"live").unwrap();
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
// What `resolve_dest_paths` makes of DEST = the source's directory.
|
||||
let err = refuse_overlap([&partial, &final_src], [&partial, &final_src]).unwrap_err();
|
||||
assert!(err.to_string().contains("overwrite the capture"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_symlinked_or_relative_spelling_of_the_source_is_still_caught() {
|
||||
let dir = tempdir().unwrap();
|
||||
let real = dir.path().join("real");
|
||||
std::fs::create_dir(&real).unwrap();
|
||||
let src = real.join("cap.pcap");
|
||||
std::fs::write(&src, b"done").unwrap();
|
||||
let link = dir.path().join("link");
|
||||
std::os::unix::fs::symlink(&real, &link).unwrap();
|
||||
let via_link = link.join("cap.pcap");
|
||||
let via_dotdot = real.join("..").join("real").join("cap.pcap.partial");
|
||||
assert!(refuse_overlap([&src, &src], [&via_dotdot, &via_link]).is_err());
|
||||
// Neither exists yet: compared by canonical parent + name.
|
||||
let not_yet = real.join("other.pcap");
|
||||
assert!(refuse_overlap([¬_yet, ¬_yet], [&link.join("other.pcap"), &via_dotdot]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_different_destination_is_allowed() {
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap.partial");
|
||||
std::fs::write(&src, b"live").unwrap();
|
||||
let out = dir.path().join("out");
|
||||
std::fs::create_dir(&out).unwrap();
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
let dests = [out.join("cap.pcap.partial"), out.join("cap.pcap")];
|
||||
assert!(refuse_overlap([&src, &final_src], [&dests[0], &dests[1]]).is_ok());
|
||||
// A destination under a directory that doesn't exist can't collide.
|
||||
let missing = dir.path().join("nope").join("cap.pcap");
|
||||
assert!(refuse_overlap([&src, &final_src], [&missing, &missing]).is_ok());
|
||||
}
|
||||
}
|
||||
+437
@@ -0,0 +1,437 @@
|
||||
use anyhow::{Result, bail};
|
||||
use std::net::IpAddr;
|
||||
|
||||
/// The only link-layer type the IP peek understands. Anything else matches
|
||||
/// nothing -- see `matches`.
|
||||
pub const LINKTYPE_ETHERNET: u32 = 1;
|
||||
|
||||
const ETH_HEADER_LEN: usize = 14;
|
||||
const ETHERTYPE_IPV4: u16 = 0x0800;
|
||||
const ETHERTYPE_IPV6: u16 = 0x86DD;
|
||||
const ETHERTYPE_VLAN: u16 = 0x8100;
|
||||
const ETHERTYPE_QINQ: u16 = 0x88A8;
|
||||
const ETHERTYPE_QINQ_ALT: u16 = 0x9100;
|
||||
/// 802.1Q and one layer of QinQ. Deeper stacking is real but vanishingly
|
||||
/// rare in capture files, and the bound keeps a corrupt/truncated packet
|
||||
/// from walking off into the payload looking for an ethertype.
|
||||
const MAX_VLAN_TAGS: usize = 2;
|
||||
|
||||
const IPV4_HEADER_MIN: usize = 20;
|
||||
const IPV6_HEADER_LEN: usize = 40;
|
||||
|
||||
const MAC_LEN: usize = 6;
|
||||
|
||||
/// An Ethernet MAC address.
|
||||
type Mac = [u8; MAC_LEN];
|
||||
|
||||
/// What to keep from a capture: packets whose IP addresses and Ethernet MAC
|
||||
/// addresses match.
|
||||
///
|
||||
/// Each dimension is a set of entries; a packet satisfies a dimension if
|
||||
/// *either* its source or destination falls in *any* entry. An empty
|
||||
/// dimension is satisfied by everything (without looking at the packet), and
|
||||
/// a packet must satisfy every dimension to be kept -- so `--ip` and `--mac`
|
||||
/// together mean "this IP *on* this MAC".
|
||||
///
|
||||
/// Sized for the expected 1-4 entries: plain vectors scanned linearly. IP
|
||||
/// entries are split by address family so a v4 packet never compares against
|
||||
/// a v6 entry. No sorting (a prefix match isn't an equality test, so
|
||||
/// ordering buys nothing) and no hashing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PacketFilter {
|
||||
/// Set when neither `--ip` nor `--mac` was given: every packet is kept,
|
||||
/// without looking at it at all. This is the default so that the
|
||||
/// filtering code path is the *only* code path -- see the chunker.
|
||||
match_all: bool,
|
||||
/// (network, mask) pairs, network already masked at parse time.
|
||||
v4: Vec<(u32, u32)>,
|
||||
v6: Vec<(u128, u128)>,
|
||||
macs: Vec<Mac>,
|
||||
}
|
||||
|
||||
impl PacketFilter {
|
||||
/// The no-`--ip`/`--mac` default: keeps everything, matches without
|
||||
/// inspecting.
|
||||
pub fn match_all() -> Self {
|
||||
Self { match_all: true, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() }
|
||||
}
|
||||
|
||||
/// Parses `--ip` specs -- a bare address (`1.1.1.1`, `2606:4700::1111`)
|
||||
/// or a CIDR prefix (`10.0.0.0/8`) -- and `--mac` specs
|
||||
/// (`aa:bb:cc:dd:ee:ff`). Two empty lists yield `match_all`.
|
||||
pub fn parse(ip_specs: &[String], mac_specs: &[String]) -> Result<Self> {
|
||||
if ip_specs.is_empty() && mac_specs.is_empty() {
|
||||
return Ok(Self::match_all());
|
||||
}
|
||||
let mut f = Self { match_all: false, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() };
|
||||
for spec in mac_specs {
|
||||
f.macs.push(parse_mac(spec)?);
|
||||
}
|
||||
for spec in ip_specs {
|
||||
let (addr_str, prefix_str) = match spec.split_once('/') {
|
||||
Some((a, p)) => (a, Some(p)),
|
||||
None => (spec.as_str(), None),
|
||||
};
|
||||
let addr: IpAddr = addr_str
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("--ip {spec:?}: not an IP address"))?;
|
||||
let width = if addr.is_ipv4() { 32u32 } else { 128u32 };
|
||||
let prefix = match prefix_str {
|
||||
Some(p) => p
|
||||
.parse::<u32>()
|
||||
.map_err(|_| anyhow::anyhow!("--ip {spec:?}: prefix length is not a number"))?,
|
||||
None => width,
|
||||
};
|
||||
if prefix > width {
|
||||
bail!("--ip {spec:?}: prefix length {prefix} exceeds {width} bits");
|
||||
}
|
||||
match addr {
|
||||
IpAddr::V4(v4) => {
|
||||
let mask = if prefix == 0 { 0 } else { u32::MAX << (32 - prefix) };
|
||||
f.v4.push((u32::from(v4) & mask, mask));
|
||||
}
|
||||
IpAddr::V6(v6) => {
|
||||
let mask = if prefix == 0 { 0 } else { u128::MAX << (128 - prefix) };
|
||||
f.v6.push((u128::from(v6) & mask, mask));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
/// Whether this filter keeps everything, i.e. neither `--ip` nor `--mac`
|
||||
/// was given.
|
||||
/// Callers use it to skip work (and warnings) that only a real filter
|
||||
/// needs.
|
||||
pub fn is_match_all(&self) -> bool {
|
||||
self.match_all
|
||||
}
|
||||
|
||||
/// Whether to keep `packet` -- the captured bytes of one pcap record,
|
||||
/// starting at the link-layer header.
|
||||
///
|
||||
/// Everything unmatched is dropped: packets truncated by snaplen before
|
||||
/// the fields being matched, any link type other than Ethernet, and --
|
||||
/// when `--ip` is set -- non-IP frames (ARP, LLDP). With only `--mac`,
|
||||
/// non-IP frames from a matching MAC are kept.
|
||||
pub fn matches(&self, packet: &[u8], linktype: u32) -> bool {
|
||||
if self.match_all {
|
||||
return true;
|
||||
}
|
||||
if linktype != LINKTYPE_ETHERNET || packet.len() < ETH_HEADER_LEN {
|
||||
return false;
|
||||
}
|
||||
self.mac_matches(packet) && self.ip_matches(packet)
|
||||
}
|
||||
|
||||
/// Whether the frame's source or destination MAC is in the set; an empty
|
||||
/// set matches every frame. `packet` is at least an Ethernet header long.
|
||||
fn mac_matches(&self, packet: &[u8]) -> bool {
|
||||
if self.macs.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let dst: Mac = packet[0..MAC_LEN].try_into().unwrap();
|
||||
let src: Mac = packet[MAC_LEN..2 * MAC_LEN].try_into().unwrap();
|
||||
self.macs.iter().any(|&m| m == src || m == dst)
|
||||
}
|
||||
|
||||
/// Whether the frame's source or destination IP is in the set; an empty
|
||||
/// set matches every frame, IP or not. `packet` is at least an Ethernet
|
||||
/// header long.
|
||||
fn ip_matches(&self, packet: &[u8]) -> bool {
|
||||
if self.v4.is_empty() && self.v6.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let mut ethertype = be16(&packet[12..14]);
|
||||
let mut off = ETH_HEADER_LEN;
|
||||
let mut tags = 0;
|
||||
// A VLAN tag is TPID(2) + TCI(2); the TPID is the ethertype we just
|
||||
// read, so the *next* type field sits 2 bytes into the tag.
|
||||
while matches!(ethertype, ETHERTYPE_VLAN | ETHERTYPE_QINQ | ETHERTYPE_QINQ_ALT) {
|
||||
tags += 1;
|
||||
if tags > MAX_VLAN_TAGS || packet.len() < off + 4 {
|
||||
return false;
|
||||
}
|
||||
ethertype = be16(&packet[off + 2..off + 4]);
|
||||
off += 4;
|
||||
}
|
||||
match ethertype {
|
||||
ETHERTYPE_IPV4 => {
|
||||
if packet.len() < off + IPV4_HEADER_MIN || packet[off] >> 4 != 4 {
|
||||
return false;
|
||||
}
|
||||
let src = be32(&packet[off + 12..off + 16]);
|
||||
let dst = be32(&packet[off + 16..off + 20]);
|
||||
self.v4.iter().any(|&(net, mask)| src & mask == net || dst & mask == net)
|
||||
}
|
||||
ETHERTYPE_IPV6 => {
|
||||
if packet.len() < off + IPV6_HEADER_LEN || packet[off] >> 4 != 6 {
|
||||
return false;
|
||||
}
|
||||
let src = be128(&packet[off + 8..off + 24]);
|
||||
let dst = be128(&packet[off + 24..off + 40]);
|
||||
self.v6.iter().any(|&(net, mask)| src & mask == net || dst & mask == net)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses `aa:bb:cc:dd:ee:ff` (or `-` separated) into a MAC address.
|
||||
fn parse_mac(spec: &str) -> Result<Mac> {
|
||||
let bad = || anyhow::anyhow!("--mac {spec:?}: not a MAC address (expected aa:bb:cc:dd:ee:ff)");
|
||||
let mut mac = [0u8; MAC_LEN];
|
||||
let mut parts = spec.split([':', '-']);
|
||||
for byte in &mut mac {
|
||||
let part = parts.next().ok_or_else(bad)?;
|
||||
// from_str_radix alone would accept a leading '+'.
|
||||
if part.len() != 2 || !part.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return Err(bad());
|
||||
}
|
||||
*byte = u8::from_str_radix(part, 16).map_err(|_| bad())?;
|
||||
}
|
||||
if parts.next().is_some() {
|
||||
return Err(bad());
|
||||
}
|
||||
Ok(mac)
|
||||
}
|
||||
|
||||
fn be16(b: &[u8]) -> u16 {
|
||||
u16::from_be_bytes(b.try_into().unwrap())
|
||||
}
|
||||
|
||||
fn be32(b: &[u8]) -> u32 {
|
||||
u32::from_be_bytes(b.try_into().unwrap())
|
||||
}
|
||||
|
||||
fn be128(b: &[u8]) -> u128 {
|
||||
u128::from_be_bytes(b.try_into().unwrap())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn specs(list: &[&str]) -> Vec<String> {
|
||||
list.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
fn filter(list: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs(list), &[]).unwrap()
|
||||
}
|
||||
|
||||
fn mac_filter(ips: &[&str], macs: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs(ips), &specs(macs)).unwrap()
|
||||
}
|
||||
|
||||
/// An Ethernet frame with `vlan_tags` 802.1Q tags in front of `ethertype`.
|
||||
fn eth(vlan_tags: usize, ethertype: u16, payload: &[u8]) -> Vec<u8> {
|
||||
let mut f = vec![0u8; 12]; // dst + src MAC
|
||||
for _ in 0..vlan_tags {
|
||||
f.extend_from_slice(ÐERTYPE_VLAN.to_be_bytes());
|
||||
f.extend_from_slice(&[0x00, 0x64]); // TCI: VLAN 100
|
||||
}
|
||||
f.extend_from_slice(ðertype.to_be_bytes());
|
||||
f.extend_from_slice(payload);
|
||||
f
|
||||
}
|
||||
|
||||
fn ipv4(src: &str, dst: &str) -> Vec<u8> {
|
||||
let mut p = vec![0u8; IPV4_HEADER_MIN];
|
||||
p[0] = 0x45; // version 4, IHL 5
|
||||
let s: std::net::Ipv4Addr = src.parse().unwrap();
|
||||
let d: std::net::Ipv4Addr = dst.parse().unwrap();
|
||||
p[12..16].copy_from_slice(&s.octets());
|
||||
p[16..20].copy_from_slice(&d.octets());
|
||||
p
|
||||
}
|
||||
|
||||
fn ipv6(src: &str, dst: &str) -> Vec<u8> {
|
||||
let mut p = vec![0u8; IPV6_HEADER_LEN];
|
||||
p[0] = 0x60; // version 6
|
||||
let s: std::net::Ipv6Addr = src.parse().unwrap();
|
||||
let d: std::net::Ipv6Addr = dst.parse().unwrap();
|
||||
p[8..24].copy_from_slice(&s.octets());
|
||||
p[24..40].copy_from_slice(&d.octets());
|
||||
p
|
||||
}
|
||||
|
||||
fn v4_packet(src: &str, dst: &str) -> Vec<u8> {
|
||||
eth(0, ETHERTYPE_IPV4, &ipv4(src, dst))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn match_all_keeps_everything_including_non_ip_and_odd_linktypes() {
|
||||
let f = PacketFilter::match_all();
|
||||
assert!(f.is_match_all());
|
||||
assert!(f.matches(&[], 113));
|
||||
assert!(f.matches(ð(0, 0x0806, &[1, 2, 3]), LINKTYPE_ETHERNET));
|
||||
// An empty --ip list is the same thing.
|
||||
assert!(PacketFilter::parse(&[], &[]).unwrap().is_match_all());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_v4_matches_either_direction_only() {
|
||||
let f = filter(&["1.1.1.1"]);
|
||||
assert!(!f.is_match_all());
|
||||
assert!(f.matches(&v4_packet("1.1.1.1", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(&v4_packet("9.9.9.9", "1.1.1.1"), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(&v4_packet("9.9.9.9", "8.8.8.8"), LINKTYPE_ETHERNET));
|
||||
// Neighbouring address must not match a bare (/32) entry.
|
||||
assert!(!f.matches(&v4_packet("1.1.1.2", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_v6_matches_either_direction() {
|
||||
let f = filter(&["2606:4700::1111"]);
|
||||
let pkt = eth(0, ETHERTYPE_IPV6, &ipv6("2001:db8::1", "2606:4700::1111"));
|
||||
assert!(f.matches(&pkt, LINKTYPE_ETHERNET));
|
||||
let miss = eth(0, ETHERTYPE_IPV6, &ipv6("2001:db8::1", "2606:4700::1112"));
|
||||
assert!(!f.matches(&miss, LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cidr_boundaries_are_exact() {
|
||||
let f = filter(&["10.0.0.0/8"]);
|
||||
assert!(f.matches(&v4_packet("10.0.0.0", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(&v4_packet("10.255.255.255", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(&v4_packet("11.0.0.0", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(&v4_packet("9.255.255.255", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
|
||||
// /0 keeps every packet of that family, but still not non-IP.
|
||||
let all = filter(&["0.0.0.0/0"]);
|
||||
assert!(all.matches(&v4_packet("203.0.113.9", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
assert!(!all.matches(ð(0, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
|
||||
|
||||
let v6 = filter(&["2606:4700::/32"]);
|
||||
assert!(v6.matches(ð(0, ETHERTYPE_IPV6, &ipv6("2606:4700:dead::1", "::1")), LINKTYPE_ETHERNET));
|
||||
assert!(!v6.matches(ð(0, ETHERTYPE_IPV6, &ipv6("2606:4701::1", "::1")), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn families_do_not_cross_match() {
|
||||
// A v4-only filter must never keep a v6 packet, and vice versa --
|
||||
// the split vectors mean the comparison is never even attempted.
|
||||
let v4only = filter(&["1.1.1.1"]);
|
||||
assert!(!v4only.matches(ð(0, ETHERTYPE_IPV6, &ipv6("::1", "::2")), LINKTYPE_ETHERNET));
|
||||
let v6only = filter(&["::1"]);
|
||||
assert!(!v6only.matches(&v4_packet("1.1.1.1", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vlan_tags_are_skipped_single_and_double() {
|
||||
let f = filter(&["1.1.1.1"]);
|
||||
let one = eth(1, ETHERTYPE_IPV4, &ipv4("1.1.1.1", "9.9.9.9"));
|
||||
let two = eth(2, ETHERTYPE_IPV4, &ipv4("9.9.9.9", "1.1.1.1"));
|
||||
assert!(f.matches(&one, LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(&two, LINKTYPE_ETHERNET));
|
||||
// Beyond the tag bound we stop rather than hunt through the payload.
|
||||
let three = eth(3, ETHERTYPE_IPV4, &ipv4("1.1.1.1", "9.9.9.9"));
|
||||
assert!(!f.matches(&three, LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unparseable_packets_are_dropped() {
|
||||
let f = filter(&["1.1.1.1"]);
|
||||
// ARP.
|
||||
assert!(!f.matches(ð(0, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
|
||||
// Truncated before the addresses (snaplen too small).
|
||||
let full = v4_packet("1.1.1.1", "9.9.9.9");
|
||||
assert!(!f.matches(&full[..ETH_HEADER_LEN + 12], LINKTYPE_ETHERNET));
|
||||
// Shorter than an Ethernet header, and empty.
|
||||
assert!(!f.matches(&full[..8], LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(&[], LINKTYPE_ETHERNET));
|
||||
// Right ethertype, wrong version nibble.
|
||||
let mut bad_version = full.clone();
|
||||
bad_version[ETH_HEADER_LEN] = 0x65;
|
||||
assert!(!f.matches(&bad_version, LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_ethernet_linktypes_match_nothing() {
|
||||
let f = filter(&["1.1.1.1"]);
|
||||
let pkt = v4_packet("1.1.1.1", "9.9.9.9");
|
||||
// 113 = LINUX_SLL, what `tcpdump -i any` produces; 101 = RAW; 0 = NULL.
|
||||
for linktype in [0u32, 101, 113, 276] {
|
||||
assert!(!f.matches(&pkt, linktype), "linktype {linktype} should match nothing");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_specs_are_rejected_at_parse() {
|
||||
for spec in ["1.1.1.1/33", "::1/129", "not-an-ip", "1.1.1.1/x", "1.1.1.256"] {
|
||||
assert!(
|
||||
PacketFilter::parse(&specs(&[spec]), &[]).is_err(),
|
||||
"{spec:?} should be rejected"
|
||||
);
|
||||
}
|
||||
// /0 and full-width prefixes are fine.
|
||||
assert!(PacketFilter::parse(&specs(&["1.1.1.1/32", "0.0.0.0/0", "::/0", "::1/128"]), &[]).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_entries_are_all_considered() {
|
||||
let f = filter(&["1.1.1.1", "8.8.8.8", "10.0.0.0/8", "2606:4700::/32"]);
|
||||
assert!(f.matches(&v4_packet("8.8.8.8", "9.9.9.9"), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(&v4_packet("9.9.9.9", "10.1.2.3"), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(ð(0, ETHERTYPE_IPV6, &ipv6("::1", "2606:4700::1")), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(&v4_packet("9.9.9.9", "203.0.113.1"), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
const UPSTREAM: &str = "02:00:00:00:00:aa";
|
||||
const OTHER: &str = "02:00:00:00:00:bb";
|
||||
|
||||
/// An Ethernet frame between two MACs carrying `ethertype`/`payload`.
|
||||
fn eth_between(dst: &str, src: &str, ethertype: u16, payload: &[u8]) -> Vec<u8> {
|
||||
let mut f = parse_mac(dst).unwrap().to_vec();
|
||||
f.extend_from_slice(&parse_mac(src).unwrap());
|
||||
f.extend_from_slice(ðertype.to_be_bytes());
|
||||
f.extend_from_slice(payload);
|
||||
f
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_matches_either_direction_only() {
|
||||
let f = mac_filter(&[], &[UPSTREAM]);
|
||||
assert!(!f.is_match_all());
|
||||
let ip = ipv4("1.1.1.1", "9.9.9.9");
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(ð_between(UPSTREAM, OTHER, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(ð_between(OTHER, OTHER, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_only_keeps_non_ip_frames_and_ignores_vlan_tags() {
|
||||
let f = mac_filter(&[], &[UPSTREAM]);
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_VLAN, &[0u8; 4]), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_and_ip_must_both_match() {
|
||||
let f = mac_filter(&["1.1.1.1"], &[UPSTREAM]);
|
||||
let wanted = ipv4("1.1.1.1", "9.9.9.9");
|
||||
let unwanted = ipv4("8.8.8.8", "9.9.9.9");
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &wanted), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &unwanted), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(ð_between(OTHER, OTHER, ETHERTYPE_IPV4, &wanted), LINKTYPE_ETHERNET));
|
||||
// ARP on the right MAC still fails the IP dimension.
|
||||
assert!(!f.matches(ð_between(OTHER, UPSTREAM, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_specs_parse_strictly() {
|
||||
assert_eq!(parse_mac("AA:bb:0c:0D:ee:ff").unwrap(), [0xaa, 0xbb, 0x0c, 0x0d, 0xee, 0xff]);
|
||||
assert_eq!(parse_mac("aa-bb-cc-dd-ee-ff").unwrap(), [0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff]);
|
||||
for bad in ["", "aa:bb:cc:dd:ee", "aa:bb:cc:dd:ee:ff:00", "aa:bb:cc:dd:ee:fg", "a:bb:cc:dd:ee:ff", "aabbccddeeff", "+a:bb:cc:dd:ee:ff"] {
|
||||
assert!(parse_mac(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_filter_on_non_ethernet_matches_nothing() {
|
||||
let f = mac_filter(&[], &[UPSTREAM]);
|
||||
let pkt = eth_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &ipv4("1.1.1.1", "9.9.9.9"));
|
||||
assert!(!f.matches(&pkt, 113));
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ mod channel;
|
||||
mod chunker;
|
||||
mod cli;
|
||||
mod client;
|
||||
mod filter;
|
||||
mod naming;
|
||||
mod pipeline;
|
||||
mod protocol;
|
||||
|
||||
+51
-8
@@ -1,4 +1,6 @@
|
||||
use anyhow::{bail, Result};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::Duration;
|
||||
use std::path::Path;
|
||||
|
||||
/// Either a local filesystem path or a `[user@]host:path` remote target.
|
||||
@@ -110,7 +112,9 @@ pub fn resolve_dest_paths(
|
||||
}
|
||||
|
||||
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
||||
pub fn parse_size(s: &str) -> Result<usize> {
|
||||
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
|
||||
/// a zero target would cut empty chunks without end.
|
||||
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
bail!("empty size");
|
||||
@@ -125,7 +129,31 @@ pub fn parse_size(s: &str) -> Result<usize> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
||||
Ok(n * mult)
|
||||
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
|
||||
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
|
||||
}
|
||||
|
||||
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
||||
/// explicit "ms"/"s" suffix. "100", "100ms" and "0.1s" are not all spellings
|
||||
/// of the same thing -- only integers are accepted, so use "100ms" or "1s".
|
||||
pub fn parse_duration(s: &str) -> Result<Duration> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
bail!("empty interval");
|
||||
}
|
||||
let (digits, mult_ms) = if let Some(d) = s.strip_suffix("ms") {
|
||||
(d, 1)
|
||||
} else if let Some(d) = s.strip_suffix('s') {
|
||||
(d, 1000)
|
||||
} else {
|
||||
(s, 1)
|
||||
};
|
||||
let n: u64 = digits
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
||||
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
|
||||
Ok(Duration::from_millis(ms))
|
||||
}
|
||||
|
||||
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
||||
@@ -251,14 +279,29 @@ mod tests {
|
||||
assert_eq!(d.temp_path, "cap.pcap.partial");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duration_parsing() {
|
||||
assert_eq!(parse_duration("100").unwrap(), Duration::from_millis(100));
|
||||
assert_eq!(parse_duration("100ms").unwrap(), Duration::from_millis(100));
|
||||
assert_eq!(parse_duration("2s").unwrap(), Duration::from_secs(2));
|
||||
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
||||
// 0 is legal: cut a chunk as soon as anything is kept.
|
||||
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
||||
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
|
||||
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_parsing() {
|
||||
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap(), 512);
|
||||
assert!(parse_size("").is_err());
|
||||
assert!(parse_size("abc").is_err());
|
||||
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap().get(), 512);
|
||||
// 0 would cut empty chunks forever; overflow must not wrap.
|
||||
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
|
||||
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+321
-6
@@ -1,4 +1,5 @@
|
||||
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker, DEFAULT_FLUSH};
|
||||
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker};
|
||||
use crate::filter::PacketFilter;
|
||||
use crate::protocol::{Message, MessageSink, MessageSource};
|
||||
use crate::tail::{run_tail, SourceEvent};
|
||||
use crate::tracelog::Logger;
|
||||
@@ -6,8 +7,11 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
|
||||
/// Bound on the reader->compressor and compressor->sender channels: enough to
|
||||
@@ -21,8 +25,15 @@ pub struct SendOptions {
|
||||
pub is_partial: bool,
|
||||
pub final_source_path: PathBuf,
|
||||
pub compress: bool,
|
||||
pub chunk_target: usize,
|
||||
pub chunk_target: NonZeroUsize,
|
||||
/// How long kept-but-undersized output may wait before being sent
|
||||
/// anyway. See `Chunker::should_flush`.
|
||||
pub flush_after: Duration,
|
||||
pub level: i32,
|
||||
/// Which packets to forward. `PacketFilter::match_all()` when no `--ip`/`--mac` was
|
||||
/// given -- not a disabled filter, just one that says yes, so there is
|
||||
/// only ever one code path through the chunker.
|
||||
pub filter: PacketFilter,
|
||||
pub log: Arc<Logger>,
|
||||
}
|
||||
|
||||
@@ -39,19 +50,50 @@ enum CompressedItem {
|
||||
/// serializing -- see the plan's Performance section. Non-compress mode
|
||||
/// collapses to 2 stages since there's no compression work to overlap.
|
||||
pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()> {
|
||||
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial {
|
||||
Box::new(RecordAlignedChunker::new(opts.chunk_target, DEFAULT_FLUSH, opts.is_partial))
|
||||
// Two independent reasons to parse records, either of which forces the
|
||||
// record-aligned chunker:
|
||||
// * `compress`: chunk boundaries become zstd frame boundaries, and the
|
||||
// format requires every frame to hold a whole number of records
|
||||
// (STREAMING_ZSTD_FORMAT.md invariant 3). The cut point *is* the format.
|
||||
// * `is_partial`: a growing source needs `confirmed_len()` -- record
|
||||
// structure is the only way to tell real content from a preallocating
|
||||
// writer's not-yet-written tail. Here the chunker is a validator, not
|
||||
// a framer.
|
||||
// * a real `--ip`/`--mac` filter: deciding whether to forward a packet means
|
||||
// finding the packet, and record boundaries are the only way to know
|
||||
// where one starts. Here the chunker is a demultiplexer.
|
||||
// A final, uncompressed copy has neither: `run_recv` writes every Data
|
||||
// payload verbatim and never inspects a boundary, so alignment would buy
|
||||
// nothing, while requiring pcap structure would break copying pcapng,
|
||||
// truncated, or otherwise non-pcap files that transfer fine today. So it
|
||||
// gets the passthrough chunker, framed on the wire only by tail.rs's
|
||||
// READ_WINDOW.
|
||||
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial || !opts.filter.is_match_all()
|
||||
{
|
||||
Box::new(RecordAlignedChunker::with_filter(
|
||||
opts.chunk_target,
|
||||
opts.flush_after,
|
||||
opts.is_partial,
|
||||
opts.filter.clone(),
|
||||
))
|
||||
} else {
|
||||
Box::new(RawChunker::new())
|
||||
};
|
||||
|
||||
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
||||
// Raised once the send loop below is done, for whatever reason. After a
|
||||
// clean EOF or a source error the reader has already returned and never
|
||||
// looks; after a failed send it is how the reader learns to quit instead
|
||||
// of polling the source until it next has something to send, which on an
|
||||
// idle capture or a filter matching nothing may be never.
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let reader = thread::spawn({
|
||||
let source_path = opts.source_path.clone();
|
||||
let is_partial = opts.is_partial;
|
||||
let final_source_path = opts.final_source_path.clone();
|
||||
let log = opts.log.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
||||
let stop = stop.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
||||
});
|
||||
|
||||
if opts.compress {
|
||||
@@ -106,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
let compressor_result = compressor.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
@@ -141,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
return Err(e);
|
||||
@@ -238,7 +282,9 @@ mod tests {
|
||||
final_source_path: src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
@@ -274,7 +320,9 @@ mod tests {
|
||||
final_source_path: final_src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
|
||||
@@ -320,8 +368,10 @@ mod tests {
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512, // small target to force multiple frames
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
@@ -357,10 +407,275 @@ mod tests {
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
|
||||
assert!(!dest_final.exists());
|
||||
}
|
||||
|
||||
fn eth_header_bytes() -> Vec<u8> {
|
||||
let mut h = header_bytes(0xA1B2C3D4);
|
||||
h[20..24].copy_from_slice(&1u32.to_le_bytes()); // LINKTYPE_ETHERNET
|
||||
h
|
||||
}
|
||||
|
||||
/// A record holding an Ethernet/IPv4 packet, padded to `len` captured bytes.
|
||||
fn ip_record(src: [u8; 4], dst: [u8; 4], len: usize) -> Vec<u8> {
|
||||
let mut pkt = vec![0u8; len.max(34)];
|
||||
pkt[12..14].copy_from_slice(&0x0800u16.to_be_bytes());
|
||||
pkt[14] = 0x45;
|
||||
pkt[26..30].copy_from_slice(&src);
|
||||
pkt[30..34].copy_from_slice(&dst);
|
||||
record_bytes(&pkt, 1_700_000_000)
|
||||
}
|
||||
|
||||
fn ip_filter(specs: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>(), &[]).unwrap()
|
||||
}
|
||||
|
||||
/// Source stream plus the exact pcap a `--ip 1.1.1.1` transfer should
|
||||
/// produce from it: same global header, only the matching records.
|
||||
fn filtered_fixture(n: usize) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut source = eth_header_bytes();
|
||||
let mut expected = eth_header_bytes();
|
||||
for i in 0..n {
|
||||
if i % 5 == 0 {
|
||||
let hit = ip_record([1, 1, 1, 1], [9, 9, 9, 9], 100 + i);
|
||||
source.extend_from_slice(&hit);
|
||||
expected.extend_from_slice(&hit);
|
||||
} else {
|
||||
source.extend_from_slice(&ip_record([9, 9, 9, 9], [8, 8, 8, 8], 100 + i));
|
||||
}
|
||||
}
|
||||
(source, expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filtered_transfer_carries_only_matching_packets() {
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap");
|
||||
let (source, expected) = filtered_fixture(50);
|
||||
std::fs::write(&src, &source).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap");
|
||||
let dest_temp = dir.path().join("out.pcap.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: src.clone(),
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["1.1.1.1"]),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
|
||||
// A filter forces the record-aligned chunker even though this is a
|
||||
// final, uncompressed copy -- so the output is a real pcap, not a
|
||||
// byte-for-byte copy of the source.
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), expected);
|
||||
assert!(expected.len() < source.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filtered_compressed_transfer_round_trips_to_the_filtered_pcap() {
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap");
|
||||
let (source, expected) = filtered_fixture(200);
|
||||
std::fs::write(&src, &source).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap.zst");
|
||||
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: src.clone(),
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["1.1.1.1"]),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
|
||||
let out_bytes = std::fs::read(&dest_final).unwrap();
|
||||
assert_eq!(&out_bytes[0..14], &MARKER_FRAME[..]);
|
||||
// The whole point of the container: decompressing yields a valid
|
||||
// pcap, here the filtered one.
|
||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||
// Header frame still declares exactly the 24-byte global header.
|
||||
let header_content_size = zstd::zstd_safe::get_frame_content_size(&out_bytes[14..])
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(header_content_size, 24);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_filter_matching_nothing_produces_a_header_only_pcap() {
|
||||
// Not an error: the transfer completes and the destination is a
|
||||
// valid, empty capture.
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap");
|
||||
let (source, _) = filtered_fixture(20);
|
||||
std::fs::write(&src, &source).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap");
|
||||
let dest_temp = dir.path().join("out.pcap.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: src.clone(),
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["203.0.113.7"]),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
||||
}
|
||||
|
||||
/// A record holding an Ethernet frame from `src` to `dst` with
|
||||
/// `ethertype`, padded to `len` captured bytes.
|
||||
fn mac_record(dst: [u8; 6], src: [u8; 6], ethertype: u16, len: usize, ts_sec: u32) -> Vec<u8> {
|
||||
let mut pkt = vec![0u8; len.max(14)];
|
||||
pkt[0..6].copy_from_slice(&dst);
|
||||
pkt[6..12].copy_from_slice(&src);
|
||||
pkt[12..14].copy_from_slice(ðertype.to_be_bytes());
|
||||
record_bytes(&pkt, ts_sec)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_filtered_growing_capture_carries_only_the_upstream_link() {
|
||||
// The scenario --mac exists for: a capture still being written, from
|
||||
// which only the traffic on one switch connection is wanted. The
|
||||
// source grows in uneven appends that tear records mid-header and
|
||||
// mid-payload, so the filter must only ever see confirmed records.
|
||||
const UPSTREAM: [u8; 6] = [0x02, 0, 0, 0, 0, 0xaa];
|
||||
const OTHER: [u8; 6] = [0x02, 0, 0, 0, 0, 0xbb];
|
||||
const BROADCAST: [u8; 6] = [0xff; 6];
|
||||
|
||||
let mut source = eth_header_bytes();
|
||||
let mut expected = eth_header_bytes();
|
||||
for i in 0..60u32 {
|
||||
let ts = 1_700_000_000 + i;
|
||||
let len = 64 + i as usize;
|
||||
let (record, kept) = match i % 4 {
|
||||
0 => (mac_record(OTHER, UPSTREAM, 0x0800, len, ts), true), // from the switch
|
||||
1 => (mac_record(UPSTREAM, OTHER, 0x86DD, len, ts), true), // to the switch
|
||||
2 => (mac_record(BROADCAST, UPSTREAM, 0x0806, len, ts), true), // ARP from the switch
|
||||
_ => (mac_record(BROADCAST, OTHER, 0x0806, len, ts), false), // someone else's ARP
|
||||
};
|
||||
source.extend_from_slice(&record);
|
||||
if kept {
|
||||
expected.extend_from_slice(&record);
|
||||
}
|
||||
}
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
std::fs::write(&partial, &source[..10]).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap.zst");
|
||||
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: final_src.clone(),
|
||||
compress: true,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel::<Message>();
|
||||
let recv_handle = thread::spawn({
|
||||
let dest_final = dest_final.clone();
|
||||
let dest_temp = dest_temp.clone();
|
||||
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
||||
});
|
||||
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
||||
|
||||
// Odd-sized appends (an awkward stride relative to the ~80-140 byte
|
||||
// records) so the visible end lands inside headers and payloads.
|
||||
let mut written = 10;
|
||||
for step in [37usize, 501, 13, 977, 250].iter().cycle() {
|
||||
if written == source.len() {
|
||||
break;
|
||||
}
|
||||
let end = (written + step).min(source.len());
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
||||
f.write_all(&source[written..end]).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
written = end;
|
||||
thread::sleep(std::time::Duration::from_millis(5));
|
||||
}
|
||||
std::fs::rename(&partial, &final_src).unwrap();
|
||||
|
||||
send_handle.join().unwrap().unwrap();
|
||||
recv_handle.join().unwrap().unwrap();
|
||||
|
||||
let out_bytes = std::fs::read(&dest_final).unwrap();
|
||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||
assert!(expected.len() < source.len());
|
||||
}
|
||||
|
||||
/// A peer that has gone away: every send fails.
|
||||
struct GoneSink;
|
||||
|
||||
impl MessageSink for GoneSink {
|
||||
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_finalize(&mut self) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
||||
// The reader used to learn the peer was gone only on its next send.
|
||||
// A growing capture that stops growing (and is never renamed) never
|
||||
// gives it one, so run_send waited on the reader forever.
|
||||
for compress in [false, true] {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
||||
std::fs::write(&partial, &stream).unwrap();
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: dir.path().join("cap.pcap"),
|
||||
compress,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
||||
});
|
||||
let result = done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
||||
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::cli::{ServerCli, ServerRole};
|
||||
use crate::filter::PacketFilter;
|
||||
use crate::naming;
|
||||
use crate::pipeline::{run_recv, run_send, RecvOptions, SendOptions};
|
||||
use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
|
||||
@@ -35,6 +36,11 @@ fn run_send_server(cli: ServerCli) -> Result<()> {
|
||||
None => false,
|
||||
};
|
||||
let chunk_target = naming::parse_size(&cli.chunk)?;
|
||||
let flush_after = naming::parse_duration(&cli.flush_interval)?;
|
||||
// The sending side is the only side that filters -- dropping packets
|
||||
// before they hit the wire is the whole point. `run_recv` never parses
|
||||
// what it writes, so --recv has no --ip/--mac.
|
||||
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: PathBuf::from(source),
|
||||
@@ -42,7 +48,9 @@ fn run_send_server(cli: ServerCli) -> Result<()> {
|
||||
final_source_path: PathBuf::from(final_source),
|
||||
compress,
|
||||
chunk_target,
|
||||
flush_after,
|
||||
level: cli.level,
|
||||
filter,
|
||||
log,
|
||||
};
|
||||
// `Stdout`/`Stdin` (not the `.lock()` guards) are used here: the guards
|
||||
|
||||
+111
-17
@@ -2,8 +2,9 @@ use crate::chunker::{Chunker, ChunkKind};
|
||||
use crate::tracelog::Logger;
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::os::unix::fs::{FileExt, MetadataExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
|
||||
///
|
||||
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
||||
/// is called, that's a hard error (no polling for creation).
|
||||
///
|
||||
/// `stop` is how the consumer says it has given up (its peer is gone). The
|
||||
/// loop checks it every poll, because otherwise it would only find out on
|
||||
/// its next send -- and a growing source that stops growing, or a filter
|
||||
/// that matches nothing, may never produce one, leaving whoever joins this
|
||||
/// thread waiting forever.
|
||||
pub fn run_tail(
|
||||
source_path: PathBuf,
|
||||
is_partial: bool,
|
||||
@@ -65,6 +72,7 @@ pub fn run_tail(
|
||||
mut chunker: Box<dyn Chunker>,
|
||||
events: impl EventSink,
|
||||
log: Arc<Logger>,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> Result<()> {
|
||||
if !source_path.exists() {
|
||||
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
||||
@@ -75,7 +83,7 @@ pub fn run_tail(
|
||||
source_path.display(),
|
||||
final_path.display()
|
||||
));
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log);
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
log.log(format_args!("tail: done, sending EOF"));
|
||||
@@ -96,6 +104,7 @@ fn drain(
|
||||
chunker: &mut dyn Chunker,
|
||||
events: &impl EventSink,
|
||||
log: &Logger,
|
||||
stop: &AtomicBool,
|
||||
) -> Result<()> {
|
||||
let file = File::open(source_path)?;
|
||||
let mut complete_signal = !is_partial;
|
||||
@@ -123,6 +132,9 @@ fn drain(
|
||||
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
||||
|
||||
loop {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
bail!("stopped: the receiving side is gone");
|
||||
}
|
||||
let file_len = file.metadata()?.len();
|
||||
// Whether this iteration did real work -- if so, loop again
|
||||
// immediately rather than sleeping, so a large already-available
|
||||
@@ -170,10 +182,6 @@ fn drain(
|
||||
last_stuck_peek = Some(preview.to_vec());
|
||||
}
|
||||
}
|
||||
} else if chunker.should_flush(Instant::now())
|
||||
&& let Some((kind, data)) = chunker.flush()
|
||||
{
|
||||
send_chunk(events, kind, data)?;
|
||||
}
|
||||
if file_len < confirmed && !shrunk_below_anchor {
|
||||
shrunk_below_anchor = true;
|
||||
@@ -193,14 +201,25 @@ fn drain(
|
||||
emit_ready(chunker, events)?;
|
||||
made_progress = true;
|
||||
}
|
||||
} else if chunker.should_flush(Instant::now())
|
||||
&& let Some((kind, data)) = chunker.flush()
|
||||
{
|
||||
send_chunk(events, kind, data)?;
|
||||
}
|
||||
}
|
||||
|
||||
if !complete_signal && final_path.try_exists()? {
|
||||
// Checked on EVERY iteration, not just ones that read nothing. The
|
||||
// flush timer measures how long output has been waiting, so it has
|
||||
// to be able to fire while the source is still growing: with a
|
||||
// selective filter, a busy source produces a steady trickle of
|
||||
// matching records that would otherwise sit here until they reached
|
||||
// the size target -- which at a 1% match rate means ~100x the target
|
||||
// in source bytes, seconds of added latency on a live capture.
|
||||
// `should_flush` is false whenever there is nothing kept, so this
|
||||
// never cuts an empty chunk.
|
||||
if chunker.should_flush(Instant::now())
|
||||
&& let Some((kind, data)) = chunker.flush()
|
||||
{
|
||||
send_chunk(events, kind, data)?;
|
||||
}
|
||||
|
||||
if !complete_signal && renamed_to(final_path, &file)? {
|
||||
// Rename observed -- signal now, well before finalization, per
|
||||
// section 7.4's repoint-then-drain-then-finalize ordering.
|
||||
complete_signal = true;
|
||||
@@ -262,6 +281,22 @@ fn drain(
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `final_path` now names the very file being read, i.e. the writer's
|
||||
/// rename has happened. This is stricter than section 7.5's "the final path
|
||||
/// exists": names get reused (section 6), so a file left at the final name by
|
||||
/// an earlier capture or copy would otherwise pass for the rename the moment
|
||||
/// tailing starts, and the transfer would finish early, short, at the first
|
||||
/// lull in the writer. Comparing identity also still works when the writer
|
||||
/// renames over such a leftover.
|
||||
fn renamed_to(final_path: &Path, file: &File) -> Result<bool> {
|
||||
let ours = file.metadata()?;
|
||||
match std::fs::metadata(final_path) {
|
||||
Ok(theirs) => Ok(theirs.dev() == ours.dev() && theirs.ino() == ours.ino()),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_ready(chunker: &mut dyn Chunker, events: &impl EventSink) -> Result<()> {
|
||||
while let Some((kind, data)) = chunker.ready_chunk() {
|
||||
send_chunk(events, kind, data)?;
|
||||
@@ -309,6 +344,10 @@ mod tests {
|
||||
(data, got_eof)
|
||||
}
|
||||
|
||||
fn no_stop() -> Arc<AtomicBool> {
|
||||
Arc::new(AtomicBool::new(false))
|
||||
}
|
||||
|
||||
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
||||
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
||||
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
||||
@@ -333,7 +372,7 @@ mod tests {
|
||||
std::fs::write(&path, b"hello world").unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
|
||||
let (data, eof) = collect(rx);
|
||||
assert_eq!(data, b"hello world");
|
||||
assert!(eof);
|
||||
@@ -345,7 +384,7 @@ mod tests {
|
||||
let path = dir.path().join("nope.pcap.partial");
|
||||
let final_path = dir.path().join("nope.pcap");
|
||||
let (tx, _rx) = mpsc::channel();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
|
||||
assert!(err.to_string().contains("not found"));
|
||||
}
|
||||
|
||||
@@ -362,7 +401,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -392,7 +431,7 @@ mod tests {
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
let log = log.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log)
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -419,7 +458,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -450,7 +489,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
// Let the reader fully drain the initial 10 bytes.
|
||||
@@ -515,6 +554,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -562,6 +602,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -617,6 +658,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -661,6 +703,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -699,8 +742,59 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
);
|
||||
try_collect(rx);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_file_at_the_final_name_is_not_the_rename() {
|
||||
// A leftover cap.pcap (an earlier capture reusing the name) used to
|
||||
// pass for the rename the moment tailing started, so the first lull
|
||||
// longer than the grace period finalized the copy short.
|
||||
let dir = tempdir().unwrap();
|
||||
let partial_path = dir.path().join("cap.pcap.partial");
|
||||
let final_path = dir.path().join("cap.pcap");
|
||||
std::fs::write(&final_path, b"yesterday's capture").unwrap();
|
||||
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
|
||||
stream.extend_from_slice(&record_bytes(&[2u8; 10], 1_700_000_001));
|
||||
let mut f = File::create(&partial_path).unwrap();
|
||||
f.write_all(&stream).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || {
|
||||
run_tail(
|
||||
partial_path,
|
||||
true,
|
||||
final_path,
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
|
||||
// A writer lull longer than the grace period, then more packets and
|
||||
// the real rename -- over the stale file.
|
||||
std::thread::sleep(FINALIZE_GRACE_PERIOD + Duration::from_millis(500));
|
||||
let rec3 = record_bytes(&[3u8; 10], 1_700_000_002);
|
||||
f.write_all(&rec3).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
stream.extend_from_slice(&rec3);
|
||||
drop(f);
|
||||
std::fs::rename(&partial_path, &final_path).unwrap();
|
||||
|
||||
handle.join().unwrap().unwrap();
|
||||
let (data, eof) = collect(rx);
|
||||
assert_eq!(data, stream);
|
||||
assert!(eof);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user