Compare commits
8
Commits
a7417dcdd6
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb67bbafa1 | ||
|
|
6b68deb047 | ||
|
|
cc698b7826 | ||
|
|
3eb779a6aa | ||
|
|
b46d6a936c | ||
|
|
6a80aaf461 | ||
|
|
f21b6c53e8 | ||
|
|
a9e283a87c |
No files matched your search
+27
-22
@@ -1,12 +1,13 @@
|
||||
use crate::filter::{IpFilter, LINKTYPE_ETHERNET};
|
||||
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
||||
use anyhow::{Result, bail};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub const GLOBAL_HEADER_LEN: usize = 24;
|
||||
pub const RECORD_HEADER_LEN: usize = 16;
|
||||
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
||||
#[allow(dead_code)]
|
||||
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
|
||||
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
|
||||
/// Sanity bound on a record's declared payload length, per
|
||||
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
||||
/// sources -- see `RecordAlignedChunker::partial`.
|
||||
@@ -166,7 +167,7 @@ pub struct RecordAlignedChunker {
|
||||
/// KEPT bytes -- a filter keeping 1% of the traffic still produces
|
||||
/// full-size frames rather than 1%-size ones.
|
||||
kept: Vec<u8>,
|
||||
filter: IpFilter,
|
||||
filter: PacketFilter,
|
||||
/// When the oldest byte currently in `kept` was confirmed, i.e. how
|
||||
/// long the pending output has been waiting. `None` when `kept` is
|
||||
/// empty.
|
||||
@@ -178,7 +179,7 @@ pub struct RecordAlignedChunker {
|
||||
/// bytes is what keeps `flush_after` an actual latency bound rather
|
||||
/// than an idle-only one.
|
||||
kept_since: Option<Instant>,
|
||||
target: usize,
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
/// Whether this source is a still-growing (partial) file. Gates every
|
||||
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
||||
@@ -190,20 +191,20 @@ pub struct RecordAlignedChunker {
|
||||
|
||||
impl RecordAlignedChunker {
|
||||
/// Unfiltered: keeps every record. Equivalent to `with_filter` with
|
||||
/// `IpFilter::match_all()` -- which is not a separate code path, just
|
||||
/// `PacketFilter::match_all()` -- which is not a separate code path, just
|
||||
/// a filter that says yes, so this and the filtered case exercise the
|
||||
/// same record-confirmation logic. Production goes through
|
||||
/// `with_filter`; this is the convenience the tests use.
|
||||
#[allow(dead_code)]
|
||||
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
|
||||
Self::with_filter(target, flush_after, partial, IpFilter::match_all())
|
||||
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
|
||||
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
||||
}
|
||||
|
||||
pub fn with_filter(
|
||||
target: usize,
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
partial: bool,
|
||||
filter: IpFilter,
|
||||
filter: PacketFilter,
|
||||
) -> Self {
|
||||
Self {
|
||||
header_buf: Vec::with_capacity(GLOBAL_HEADER_LEN),
|
||||
@@ -247,7 +248,7 @@ impl RecordAlignedChunker {
|
||||
// stdout: in --server mode stdout is the wire.
|
||||
if !self.filter.is_match_all() && self.linktype != LINKTYPE_ETHERNET {
|
||||
eprintln!(
|
||||
"scpcap: warning: --ip filtering needs Ethernet (link type {LINKTYPE_ETHERNET}), \
|
||||
"scpcap: warning: --ip/--mac filtering needs Ethernet (link type {LINKTYPE_ETHERNET}), \
|
||||
but this capture is link type {}; no packets will match",
|
||||
self.linktype
|
||||
);
|
||||
@@ -400,7 +401,7 @@ impl Chunker for RecordAlignedChunker {
|
||||
// must produce no chunk at all, rather than an empty one. An empty
|
||||
// chunk would become a zero-content zstd frame, which
|
||||
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
||||
if self.kept.len() >= self.target {
|
||||
if self.kept.len() >= self.target.get() {
|
||||
return Some(self.take_kept());
|
||||
}
|
||||
None
|
||||
@@ -509,6 +510,10 @@ mod tests {
|
||||
h
|
||||
}
|
||||
|
||||
fn nz(n: usize) -> NonZeroUsize {
|
||||
NonZeroUsize::new(n).unwrap()
|
||||
}
|
||||
|
||||
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
||||
record_bytes_with_ts(payload, big_endian, 0)
|
||||
}
|
||||
@@ -571,7 +576,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
||||
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
||||
for i in 0..20u8 {
|
||||
@@ -606,7 +611,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
||||
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
|
||||
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
|
||||
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
||||
c.ready_chunk(); // drain header
|
||||
|
||||
@@ -895,8 +900,8 @@ mod tests {
|
||||
record_bytes_with_ts(&pkt, false, 1_700_000_000)
|
||||
}
|
||||
|
||||
fn filter_for(specs: &[&str]) -> IpFilter {
|
||||
IpFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
|
||||
fn filter_for(specs: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>(), &[]).unwrap()
|
||||
}
|
||||
|
||||
fn drain_all(c: &mut RecordAlignedChunker) -> Vec<(ChunkKind, Vec<u8>)> {
|
||||
@@ -913,7 +918,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn keeps_only_matching_records_and_keeps_them_whole() {
|
||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let hit_src = ip_record(WANTED, OTHER, 60);
|
||||
let hit_dst = ip_record(OTHER, WANTED, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
@@ -946,7 +951,7 @@ mod tests {
|
||||
let hit = ip_record(WANTED, OTHER, 100);
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let target = 10 * hit.len();
|
||||
let mut c = RecordAlignedChunker::with_filter(target, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
@@ -1001,7 +1006,7 @@ mod tests {
|
||||
// An empty chunk would be compressed into a zero-content zstd frame,
|
||||
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
for _ in 0..20 {
|
||||
@@ -1026,9 +1031,9 @@ mod tests {
|
||||
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
||||
}
|
||||
|
||||
let mut all = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, IpFilter::match_all());
|
||||
let mut some = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut none = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||
for c in [&mut all, &mut some, &mut none] {
|
||||
c.feed(&stream).unwrap();
|
||||
}
|
||||
@@ -1052,7 +1057,7 @@ mod tests {
|
||||
// exactly once even though the bytes around it are re-read.
|
||||
let hit = ip_record(WANTED, OTHER, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
let mut c = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&miss);
|
||||
|
||||
+13
@@ -48,6 +48,14 @@ pub struct ClientCli {
|
||||
#[arg(long = "ip", value_name = "ADDR")]
|
||||
pub ip: Vec<String>,
|
||||
|
||||
/// Only forward packets whose Ethernet source or destination MAC
|
||||
/// matches, e.g. to pull one switch connection out of a capture by its
|
||||
/// upstream MAC ("aa:bb:cc:dd:ee:ff"). May be repeated; a packet is kept
|
||||
/// if it matches any entry. Combined with --ip, a packet must match
|
||||
/// both. Requires an Ethernet-linktype pcap source.
|
||||
#[arg(long = "mac", value_name = "ADDR")]
|
||||
pub mac: Vec<String>,
|
||||
|
||||
/// Append a diagnostic trace of this (local) process's transfer events
|
||||
/// to this file -- positions, rename detection, finalize -- for
|
||||
/// figuring out where a stalled transfer is stuck.
|
||||
@@ -105,6 +113,11 @@ pub struct ServerCli {
|
||||
#[arg(long = "ip", value_name = "ADDR")]
|
||||
pub ip: Vec<String>,
|
||||
|
||||
/// MAC filter for --send mode; repeated, one per address. See the
|
||||
/// client's --mac.
|
||||
#[arg(long = "mac", value_name = "ADDR")]
|
||||
pub mac: Vec<String>,
|
||||
|
||||
/// Destination final path (--recv mode).
|
||||
#[arg(long)]
|
||||
pub dest_final: Option<String>,
|
||||
|
||||
+121
-5
@@ -1,13 +1,14 @@
|
||||
use crate::channel::{ChannelSink, ChannelSource};
|
||||
use crate::cli::ClientCli;
|
||||
use crate::filter::IpFilter;
|
||||
use crate::filter::PacketFilter;
|
||||
use crate::naming::{self, resolve_dest_paths, strip_name, Target};
|
||||
use crate::pipeline::{run_recv, run_send, RecvOptions, SendOptions};
|
||||
use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
|
||||
use crate::ssh::{build_server_command, spawn_remote_server};
|
||||
use crate::tracelog::Logger;
|
||||
use anyhow::{bail, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::thread;
|
||||
|
||||
@@ -43,7 +44,7 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
// remote is (see the Remote->Local branch). It is never passed to a
|
||||
// --recv server: the receiver writes bytes verbatim and never parses
|
||||
// them.
|
||||
let filter = IpFilter::parse(&cli.ip)?;
|
||||
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
|
||||
// Filtering means reading pcap records, which an already-compressed
|
||||
// source doesn't expose. Without this the transfer would still pick the
|
||||
// record-aligned chunker (a filter forces it) and abort deep inside with
|
||||
@@ -52,8 +53,8 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
// combination rather than the source.
|
||||
if !filter.is_match_all() && source_stripped.was_zst {
|
||||
bail!(
|
||||
"cannot apply --ip to an already-compressed source ({}): \
|
||||
filtering needs to read pcap records. Copy it without --ip, \
|
||||
"cannot apply --ip/--mac to an already-compressed source ({}): \
|
||||
filtering needs to read pcap records. Copy it without --ip/--mac, \
|
||||
or filter the uncompressed capture.",
|
||||
cli.source
|
||||
);
|
||||
@@ -63,6 +64,10 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
|
||||
match (&source_target, &dest_target) {
|
||||
(Target::Local(src), Target::Local(_)) => {
|
||||
refuse_overlap(
|
||||
[Path::new(src), Path::new(&final_source_path)],
|
||||
[Path::new(&dest.temp_path), Path::new(&dest.final_path)],
|
||||
)?;
|
||||
let send_opts = SendOptions {
|
||||
source_path: PathBuf::from(src),
|
||||
is_partial,
|
||||
@@ -143,6 +148,10 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
server_args.push("--ip".to_string());
|
||||
server_args.push(spec.clone());
|
||||
}
|
||||
for spec in &cli.mac {
|
||||
server_args.push("--mac".to_string());
|
||||
server_args.push(spec.clone());
|
||||
}
|
||||
if effective_compress {
|
||||
server_args.push("--compress".to_string());
|
||||
server_args.push("zstd".to_string());
|
||||
@@ -162,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
};
|
||||
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
||||
let recv_result = run_recv(source, recv_opts);
|
||||
if recv_result.is_err() {
|
||||
// The remote sender only notices we're gone when its next
|
||||
// write fails, which on an idle capture may be never. Don't
|
||||
// wait for that: drop the connection, so its next write
|
||||
// fails and it stops.
|
||||
let _ = child.kill();
|
||||
}
|
||||
let status = child.wait()?;
|
||||
recv_result?;
|
||||
if !status.success() {
|
||||
@@ -174,3 +190,103 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses a local copy whose destination lands on the source. The receiver
|
||||
/// truncates its in-flight file on open and renames over its final one, so
|
||||
/// either collision destroys the capture being read. Copying a capture into
|
||||
/// its own directory is enough to hit it: `dir/cap.pcap.partial` to `dir/`
|
||||
/// resolves to `dir/cap.pcap`, written via `dir/cap.pcap.partial`.
|
||||
///
|
||||
/// `sources` is the source as given plus its post-rename final path, since a
|
||||
/// growing source moves from one to the other mid-transfer.
|
||||
fn refuse_overlap(sources: [&Path; 2], dests: [&Path; 2]) -> Result<()> {
|
||||
for source in sources {
|
||||
for dest in dests {
|
||||
if same_location(source, dest) {
|
||||
bail!(
|
||||
"destination {} is the source {}: copying would overwrite the capture \
|
||||
being read. Copy it to a different directory or name.",
|
||||
dest.display(),
|
||||
source.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether two paths name the same file, whether or not it exists yet: the
|
||||
/// same inode when both exist (catching hard links and symlinks), otherwise
|
||||
/// the same canonical parent directory and file name.
|
||||
fn same_location(a: &Path, b: &Path) -> bool {
|
||||
if let (Ok(ma), Ok(mb)) = (std::fs::metadata(a), std::fs::metadata(b)) {
|
||||
return ma.dev() == mb.dev() && ma.ino() == mb.ino();
|
||||
}
|
||||
match (resolve(a), resolve(b)) {
|
||||
(Some(ra), Some(rb)) => ra == rb,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The canonical form of `path`, or for a file that doesn't exist yet, its
|
||||
/// canonical parent joined with its name. `None` when even the parent
|
||||
/// doesn't exist, in which case nothing can collide with it.
|
||||
fn resolve(path: &Path) -> Option<PathBuf> {
|
||||
if let Ok(canonical) = std::fs::canonicalize(path) {
|
||||
return Some(canonical);
|
||||
}
|
||||
let parent = match path.parent() {
|
||||
Some(p) if !p.as_os_str().is_empty() => p,
|
||||
_ => Path::new("."),
|
||||
};
|
||||
Some(std::fs::canonicalize(parent).ok()?.join(path.file_name()?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn copying_a_growing_capture_into_its_own_directory_is_refused() {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
std::fs::write(&partial, b"live").unwrap();
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
// What `resolve_dest_paths` makes of DEST = the source's directory.
|
||||
let err = refuse_overlap([&partial, &final_src], [&partial, &final_src]).unwrap_err();
|
||||
assert!(err.to_string().contains("overwrite the capture"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_symlinked_or_relative_spelling_of_the_source_is_still_caught() {
|
||||
let dir = tempdir().unwrap();
|
||||
let real = dir.path().join("real");
|
||||
std::fs::create_dir(&real).unwrap();
|
||||
let src = real.join("cap.pcap");
|
||||
std::fs::write(&src, b"done").unwrap();
|
||||
let link = dir.path().join("link");
|
||||
std::os::unix::fs::symlink(&real, &link).unwrap();
|
||||
let via_link = link.join("cap.pcap");
|
||||
let via_dotdot = real.join("..").join("real").join("cap.pcap.partial");
|
||||
assert!(refuse_overlap([&src, &src], [&via_dotdot, &via_link]).is_err());
|
||||
// Neither exists yet: compared by canonical parent + name.
|
||||
let not_yet = real.join("other.pcap");
|
||||
assert!(refuse_overlap([¬_yet, ¬_yet], [&link.join("other.pcap"), &via_dotdot]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_different_destination_is_allowed() {
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap.partial");
|
||||
std::fs::write(&src, b"live").unwrap();
|
||||
let out = dir.path().join("out");
|
||||
std::fs::create_dir(&out).unwrap();
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
let dests = [out.join("cap.pcap.partial"), out.join("cap.pcap")];
|
||||
assert!(refuse_overlap([&src, &final_src], [&dests[0], &dests[1]]).is_ok());
|
||||
// A destination under a directory that doesn't exist can't collide.
|
||||
let missing = dir.path().join("nope").join("cap.pcap");
|
||||
assert!(refuse_overlap([&src, &final_src], [&missing, &missing]).is_ok());
|
||||
}
|
||||
}
|
||||
+149
-29
@@ -19,38 +19,55 @@ const MAX_VLAN_TAGS: usize = 2;
|
||||
const IPV4_HEADER_MIN: usize = 20;
|
||||
const IPV6_HEADER_LEN: usize = 40;
|
||||
|
||||
/// A set of IP addresses/prefixes to keep. A packet matches if *either* its
|
||||
/// source or destination address falls in *any* entry.
|
||||
const MAC_LEN: usize = 6;
|
||||
|
||||
/// An Ethernet MAC address.
|
||||
type Mac = [u8; MAC_LEN];
|
||||
|
||||
/// What to keep from a capture: packets whose IP addresses and Ethernet MAC
|
||||
/// addresses match.
|
||||
///
|
||||
/// Sized for the expected 1-4 entries: two plain vectors, split by address
|
||||
/// family so a v4 packet never compares against a v6 entry, scanned
|
||||
/// linearly. No sorting (a prefix match isn't an equality test, so ordering
|
||||
/// buys nothing) and no hashing.
|
||||
/// Each dimension is a set of entries; a packet satisfies a dimension if
|
||||
/// *either* its source or destination falls in *any* entry. An empty
|
||||
/// dimension is satisfied by everything (without looking at the packet), and
|
||||
/// a packet must satisfy every dimension to be kept -- so `--ip` and `--mac`
|
||||
/// together mean "this IP *on* this MAC".
|
||||
///
|
||||
/// Sized for the expected 1-4 entries: plain vectors scanned linearly. IP
|
||||
/// entries are split by address family so a v4 packet never compares against
|
||||
/// a v6 entry. No sorting (a prefix match isn't an equality test, so
|
||||
/// ordering buys nothing) and no hashing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IpFilter {
|
||||
/// Set when no `--ip` was given: every packet is kept, without looking
|
||||
/// at it at all. This is the default so that the filtering code path is
|
||||
/// the *only* code path -- see the chunker.
|
||||
pub struct PacketFilter {
|
||||
/// Set when neither `--ip` nor `--mac` was given: every packet is kept,
|
||||
/// without looking at it at all. This is the default so that the
|
||||
/// filtering code path is the *only* code path -- see the chunker.
|
||||
match_all: bool,
|
||||
/// (network, mask) pairs, network already masked at parse time.
|
||||
v4: Vec<(u32, u32)>,
|
||||
v6: Vec<(u128, u128)>,
|
||||
macs: Vec<Mac>,
|
||||
}
|
||||
|
||||
impl IpFilter {
|
||||
/// The no-`--ip` default: keeps everything, matches without inspecting.
|
||||
impl PacketFilter {
|
||||
/// The no-`--ip`/`--mac` default: keeps everything, matches without
|
||||
/// inspecting.
|
||||
pub fn match_all() -> Self {
|
||||
Self { match_all: true, v4: Vec::new(), v6: Vec::new() }
|
||||
Self { match_all: true, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() }
|
||||
}
|
||||
|
||||
/// Parses `--ip` specs: a bare address (`1.1.1.1`, `2606:4700::1111`)
|
||||
/// or a CIDR prefix (`10.0.0.0/8`). An empty list yields `match_all`.
|
||||
pub fn parse(specs: &[String]) -> Result<Self> {
|
||||
if specs.is_empty() {
|
||||
/// Parses `--ip` specs -- a bare address (`1.1.1.1`, `2606:4700::1111`)
|
||||
/// or a CIDR prefix (`10.0.0.0/8`) -- and `--mac` specs
|
||||
/// (`aa:bb:cc:dd:ee:ff`). Two empty lists yield `match_all`.
|
||||
pub fn parse(ip_specs: &[String], mac_specs: &[String]) -> Result<Self> {
|
||||
if ip_specs.is_empty() && mac_specs.is_empty() {
|
||||
return Ok(Self::match_all());
|
||||
}
|
||||
let mut f = Self { match_all: false, v4: Vec::new(), v6: Vec::new() };
|
||||
for spec in specs {
|
||||
let mut f = Self { match_all: false, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() };
|
||||
for spec in mac_specs {
|
||||
f.macs.push(parse_mac(spec)?);
|
||||
}
|
||||
for spec in ip_specs {
|
||||
let (addr_str, prefix_str) = match spec.split_once('/') {
|
||||
Some((a, p)) => (a, Some(p)),
|
||||
None => (spec.as_str(), None),
|
||||
@@ -82,7 +99,8 @@ impl IpFilter {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
/// Whether this filter keeps everything, i.e. no `--ip` was given.
|
||||
/// Whether this filter keeps everything, i.e. neither `--ip` nor `--mac`
|
||||
/// was given.
|
||||
/// Callers use it to skip work (and warnings) that only a real filter
|
||||
/// needs.
|
||||
pub fn is_match_all(&self) -> bool {
|
||||
@@ -92,9 +110,10 @@ impl IpFilter {
|
||||
/// Whether to keep `packet` -- the captured bytes of one pcap record,
|
||||
/// starting at the link-layer header.
|
||||
///
|
||||
/// Everything unmatched is dropped: non-IP frames (ARP, LLDP), packets
|
||||
/// truncated by snaplen before the addresses, and any link type other
|
||||
/// than Ethernet.
|
||||
/// Everything unmatched is dropped: packets truncated by snaplen before
|
||||
/// the fields being matched, any link type other than Ethernet, and --
|
||||
/// when `--ip` is set -- non-IP frames (ARP, LLDP). With only `--mac`,
|
||||
/// non-IP frames from a matching MAC are kept.
|
||||
pub fn matches(&self, packet: &[u8], linktype: u32) -> bool {
|
||||
if self.match_all {
|
||||
return true;
|
||||
@@ -102,6 +121,27 @@ impl IpFilter {
|
||||
if linktype != LINKTYPE_ETHERNET || packet.len() < ETH_HEADER_LEN {
|
||||
return false;
|
||||
}
|
||||
self.mac_matches(packet) && self.ip_matches(packet)
|
||||
}
|
||||
|
||||
/// Whether the frame's source or destination MAC is in the set; an empty
|
||||
/// set matches every frame. `packet` is at least an Ethernet header long.
|
||||
fn mac_matches(&self, packet: &[u8]) -> bool {
|
||||
if self.macs.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let dst: Mac = packet[0..MAC_LEN].try_into().unwrap();
|
||||
let src: Mac = packet[MAC_LEN..2 * MAC_LEN].try_into().unwrap();
|
||||
self.macs.iter().any(|&m| m == src || m == dst)
|
||||
}
|
||||
|
||||
/// Whether the frame's source or destination IP is in the set; an empty
|
||||
/// set matches every frame, IP or not. `packet` is at least an Ethernet
|
||||
/// header long.
|
||||
fn ip_matches(&self, packet: &[u8]) -> bool {
|
||||
if self.v4.is_empty() && self.v6.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let mut ethertype = be16(&packet[12..14]);
|
||||
let mut off = ETH_HEADER_LEN;
|
||||
let mut tags = 0;
|
||||
@@ -137,6 +177,25 @@ impl IpFilter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses `aa:bb:cc:dd:ee:ff` (or `-` separated) into a MAC address.
|
||||
fn parse_mac(spec: &str) -> Result<Mac> {
|
||||
let bad = || anyhow::anyhow!("--mac {spec:?}: not a MAC address (expected aa:bb:cc:dd:ee:ff)");
|
||||
let mut mac = [0u8; MAC_LEN];
|
||||
let mut parts = spec.split([':', '-']);
|
||||
for byte in &mut mac {
|
||||
let part = parts.next().ok_or_else(bad)?;
|
||||
// from_str_radix alone would accept a leading '+'.
|
||||
if part.len() != 2 || !part.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return Err(bad());
|
||||
}
|
||||
*byte = u8::from_str_radix(part, 16).map_err(|_| bad())?;
|
||||
}
|
||||
if parts.next().is_some() {
|
||||
return Err(bad());
|
||||
}
|
||||
Ok(mac)
|
||||
}
|
||||
|
||||
fn be16(b: &[u8]) -> u16 {
|
||||
u16::from_be_bytes(b.try_into().unwrap())
|
||||
}
|
||||
@@ -157,8 +216,12 @@ mod tests {
|
||||
list.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
fn filter(list: &[&str]) -> IpFilter {
|
||||
IpFilter::parse(&specs(list)).unwrap()
|
||||
fn filter(list: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs(list), &[]).unwrap()
|
||||
}
|
||||
|
||||
fn mac_filter(ips: &[&str], macs: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs(ips), &specs(macs)).unwrap()
|
||||
}
|
||||
|
||||
/// An Ethernet frame with `vlan_tags` 802.1Q tags in front of `ethertype`.
|
||||
@@ -199,12 +262,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn match_all_keeps_everything_including_non_ip_and_odd_linktypes() {
|
||||
let f = IpFilter::match_all();
|
||||
let f = PacketFilter::match_all();
|
||||
assert!(f.is_match_all());
|
||||
assert!(f.matches(&[], 113));
|
||||
assert!(f.matches(ð(0, 0x0806, &[1, 2, 3]), LINKTYPE_ETHERNET));
|
||||
// An empty --ip list is the same thing.
|
||||
assert!(IpFilter::parse(&[]).unwrap().is_match_all());
|
||||
assert!(PacketFilter::parse(&[], &[]).unwrap().is_match_all());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -298,12 +361,12 @@ mod tests {
|
||||
fn bad_specs_are_rejected_at_parse() {
|
||||
for spec in ["1.1.1.1/33", "::1/129", "not-an-ip", "1.1.1.1/x", "1.1.1.256"] {
|
||||
assert!(
|
||||
IpFilter::parse(&specs(&[spec])).is_err(),
|
||||
PacketFilter::parse(&specs(&[spec]), &[]).is_err(),
|
||||
"{spec:?} should be rejected"
|
||||
);
|
||||
}
|
||||
// /0 and full-width prefixes are fine.
|
||||
assert!(IpFilter::parse(&specs(&["1.1.1.1/32", "0.0.0.0/0", "::/0", "::1/128"])).is_ok());
|
||||
assert!(PacketFilter::parse(&specs(&["1.1.1.1/32", "0.0.0.0/0", "::/0", "::1/128"]), &[]).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -314,4 +377,61 @@ mod tests {
|
||||
assert!(f.matches(ð(0, ETHERTYPE_IPV6, &ipv6("::1", "2606:4700::1")), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(&v4_packet("9.9.9.9", "203.0.113.1"), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
const UPSTREAM: &str = "02:00:00:00:00:aa";
|
||||
const OTHER: &str = "02:00:00:00:00:bb";
|
||||
|
||||
/// An Ethernet frame between two MACs carrying `ethertype`/`payload`.
|
||||
fn eth_between(dst: &str, src: &str, ethertype: u16, payload: &[u8]) -> Vec<u8> {
|
||||
let mut f = parse_mac(dst).unwrap().to_vec();
|
||||
f.extend_from_slice(&parse_mac(src).unwrap());
|
||||
f.extend_from_slice(ðertype.to_be_bytes());
|
||||
f.extend_from_slice(payload);
|
||||
f
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_matches_either_direction_only() {
|
||||
let f = mac_filter(&[], &[UPSTREAM]);
|
||||
assert!(!f.is_match_all());
|
||||
let ip = ipv4("1.1.1.1", "9.9.9.9");
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(ð_between(UPSTREAM, OTHER, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(ð_between(OTHER, OTHER, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_only_keeps_non_ip_frames_and_ignores_vlan_tags() {
|
||||
let f = mac_filter(&[], &[UPSTREAM]);
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_VLAN, &[0u8; 4]), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_and_ip_must_both_match() {
|
||||
let f = mac_filter(&["1.1.1.1"], &[UPSTREAM]);
|
||||
let wanted = ipv4("1.1.1.1", "9.9.9.9");
|
||||
let unwanted = ipv4("8.8.8.8", "9.9.9.9");
|
||||
assert!(f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &wanted), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(ð_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &unwanted), LINKTYPE_ETHERNET));
|
||||
assert!(!f.matches(ð_between(OTHER, OTHER, ETHERTYPE_IPV4, &wanted), LINKTYPE_ETHERNET));
|
||||
// ARP on the right MAC still fails the IP dimension.
|
||||
assert!(!f.matches(ð_between(OTHER, UPSTREAM, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_specs_parse_strictly() {
|
||||
assert_eq!(parse_mac("AA:bb:0c:0D:ee:ff").unwrap(), [0xaa, 0xbb, 0x0c, 0x0d, 0xee, 0xff]);
|
||||
assert_eq!(parse_mac("aa-bb-cc-dd-ee-ff").unwrap(), [0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff]);
|
||||
for bad in ["", "aa:bb:cc:dd:ee", "aa:bb:cc:dd:ee:ff:00", "aa:bb:cc:dd:ee:fg", "a:bb:cc:dd:ee:ff", "aabbccddeeff", "+a:bb:cc:dd:ee:ff"] {
|
||||
assert!(parse_mac(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_filter_on_non_ethernet_matches_nothing() {
|
||||
let f = mac_filter(&[], &[UPSTREAM]);
|
||||
let pkt = eth_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &ipv4("1.1.1.1", "9.9.9.9"));
|
||||
assert!(!f.matches(&pkt, 113));
|
||||
}
|
||||
}
|
||||
+17
-10
@@ -1,4 +1,5 @@
|
||||
use anyhow::{bail, Result};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::Duration;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -111,7 +112,9 @@ pub fn resolve_dest_paths(
|
||||
}
|
||||
|
||||
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
||||
pub fn parse_size(s: &str) -> Result<usize> {
|
||||
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
|
||||
/// a zero target would cut empty chunks without end.
|
||||
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
bail!("empty size");
|
||||
@@ -126,7 +129,8 @@ pub fn parse_size(s: &str) -> Result<usize> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
||||
Ok(n * mult)
|
||||
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
|
||||
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
|
||||
}
|
||||
|
||||
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
||||
@@ -148,7 +152,8 @@ pub fn parse_duration(s: &str) -> Result<Duration> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
||||
Ok(Duration::from_millis(n * mult_ms))
|
||||
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
|
||||
Ok(Duration::from_millis(ms))
|
||||
}
|
||||
|
||||
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
||||
@@ -282,19 +287,21 @@ mod tests {
|
||||
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
||||
// 0 is legal: cut a chunk as soon as anything is kept.
|
||||
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
||||
for bad in ["", "abc", "0.1s", "100m", "-5"] {
|
||||
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
|
||||
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_parsing() {
|
||||
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap(), 512);
|
||||
assert!(parse_size("").is_err());
|
||||
assert!(parse_size("abc").is_err());
|
||||
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap().get(), 512);
|
||||
// 0 would cut empty chunks forever; overflow must not wrap.
|
||||
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
|
||||
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+162
-14
@@ -1,5 +1,5 @@
|
||||
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker};
|
||||
use crate::filter::IpFilter;
|
||||
use crate::filter::PacketFilter;
|
||||
use crate::protocol::{Message, MessageSink, MessageSource};
|
||||
use crate::tail::{run_tail, SourceEvent};
|
||||
use crate::tracelog::Logger;
|
||||
@@ -7,7 +7,9 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
@@ -23,15 +25,15 @@ pub struct SendOptions {
|
||||
pub is_partial: bool,
|
||||
pub final_source_path: PathBuf,
|
||||
pub compress: bool,
|
||||
pub chunk_target: usize,
|
||||
pub chunk_target: NonZeroUsize,
|
||||
/// How long kept-but-undersized output may wait before being sent
|
||||
/// anyway. See `Chunker::should_flush`.
|
||||
pub flush_after: Duration,
|
||||
pub level: i32,
|
||||
/// Which packets to forward. `IpFilter::match_all()` when no `--ip` was
|
||||
/// Which packets to forward. `PacketFilter::match_all()` when no `--ip`/`--mac` was
|
||||
/// given -- not a disabled filter, just one that says yes, so there is
|
||||
/// only ever one code path through the chunker.
|
||||
pub filter: IpFilter,
|
||||
pub filter: PacketFilter,
|
||||
pub log: Arc<Logger>,
|
||||
}
|
||||
|
||||
@@ -57,7 +59,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
// structure is the only way to tell real content from a preallocating
|
||||
// writer's not-yet-written tail. Here the chunker is a validator, not
|
||||
// a framer.
|
||||
// * a real `--ip` filter: deciding whether to forward a packet means
|
||||
// * a real `--ip`/`--mac` filter: deciding whether to forward a packet means
|
||||
// finding the packet, and record boundaries are the only way to know
|
||||
// where one starts. Here the chunker is a demultiplexer.
|
||||
// A final, uncompressed copy has neither: `run_recv` writes every Data
|
||||
@@ -79,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
};
|
||||
|
||||
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
||||
// Raised once the send loop below is done, for whatever reason. After a
|
||||
// clean EOF or a source error the reader has already returned and never
|
||||
// looks; after a failed send it is how the reader learns to quit instead
|
||||
// of polling the source until it next has something to send, which on an
|
||||
// idle capture or a filter matching nothing may be never.
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let reader = thread::spawn({
|
||||
let source_path = opts.source_path.clone();
|
||||
let is_partial = opts.is_partial;
|
||||
let final_source_path = opts.final_source_path.clone();
|
||||
let log = opts.log.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
||||
let stop = stop.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
||||
});
|
||||
|
||||
if opts.compress {
|
||||
@@ -139,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
let compressor_result = compressor.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
@@ -174,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
return Err(e);
|
||||
@@ -273,7 +284,7 @@ mod tests {
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
@@ -311,7 +322,7 @@ mod tests {
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
|
||||
@@ -357,10 +368,10 @@ mod tests {
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512, // small target to force multiple frames
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
@@ -398,7 +409,7 @@ mod tests {
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: IpFilter::match_all(),
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
|
||||
@@ -421,8 +432,8 @@ mod tests {
|
||||
record_bytes(&pkt, 1_700_000_000)
|
||||
}
|
||||
|
||||
fn ip_filter(specs: &[&str]) -> IpFilter {
|
||||
IpFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
|
||||
fn ip_filter(specs: &[&str]) -> PacketFilter {
|
||||
PacketFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>(), &[]).unwrap()
|
||||
}
|
||||
|
||||
/// Source stream plus the exact pcap a `--ip 1.1.1.1` transfer should
|
||||
@@ -485,7 +496,7 @@ mod tests {
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["1.1.1.1"]),
|
||||
@@ -530,4 +541,141 @@ mod tests {
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
||||
}
|
||||
|
||||
/// A record holding an Ethernet frame from `src` to `dst` with
|
||||
/// `ethertype`, padded to `len` captured bytes.
|
||||
fn mac_record(dst: [u8; 6], src: [u8; 6], ethertype: u16, len: usize, ts_sec: u32) -> Vec<u8> {
|
||||
let mut pkt = vec![0u8; len.max(14)];
|
||||
pkt[0..6].copy_from_slice(&dst);
|
||||
pkt[6..12].copy_from_slice(&src);
|
||||
pkt[12..14].copy_from_slice(ðertype.to_be_bytes());
|
||||
record_bytes(&pkt, ts_sec)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_filtered_growing_capture_carries_only_the_upstream_link() {
|
||||
// The scenario --mac exists for: a capture still being written, from
|
||||
// which only the traffic on one switch connection is wanted. The
|
||||
// source grows in uneven appends that tear records mid-header and
|
||||
// mid-payload, so the filter must only ever see confirmed records.
|
||||
const UPSTREAM: [u8; 6] = [0x02, 0, 0, 0, 0, 0xaa];
|
||||
const OTHER: [u8; 6] = [0x02, 0, 0, 0, 0, 0xbb];
|
||||
const BROADCAST: [u8; 6] = [0xff; 6];
|
||||
|
||||
let mut source = eth_header_bytes();
|
||||
let mut expected = eth_header_bytes();
|
||||
for i in 0..60u32 {
|
||||
let ts = 1_700_000_000 + i;
|
||||
let len = 64 + i as usize;
|
||||
let (record, kept) = match i % 4 {
|
||||
0 => (mac_record(OTHER, UPSTREAM, 0x0800, len, ts), true), // from the switch
|
||||
1 => (mac_record(UPSTREAM, OTHER, 0x86DD, len, ts), true), // to the switch
|
||||
2 => (mac_record(BROADCAST, UPSTREAM, 0x0806, len, ts), true), // ARP from the switch
|
||||
_ => (mac_record(BROADCAST, OTHER, 0x0806, len, ts), false), // someone else's ARP
|
||||
};
|
||||
source.extend_from_slice(&record);
|
||||
if kept {
|
||||
expected.extend_from_slice(&record);
|
||||
}
|
||||
}
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
std::fs::write(&partial, &source[..10]).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap.zst");
|
||||
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: final_src.clone(),
|
||||
compress: true,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel::<Message>();
|
||||
let recv_handle = thread::spawn({
|
||||
let dest_final = dest_final.clone();
|
||||
let dest_temp = dest_temp.clone();
|
||||
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
||||
});
|
||||
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
||||
|
||||
// Odd-sized appends (an awkward stride relative to the ~80-140 byte
|
||||
// records) so the visible end lands inside headers and payloads.
|
||||
let mut written = 10;
|
||||
for step in [37usize, 501, 13, 977, 250].iter().cycle() {
|
||||
if written == source.len() {
|
||||
break;
|
||||
}
|
||||
let end = (written + step).min(source.len());
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
||||
f.write_all(&source[written..end]).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
written = end;
|
||||
thread::sleep(std::time::Duration::from_millis(5));
|
||||
}
|
||||
std::fs::rename(&partial, &final_src).unwrap();
|
||||
|
||||
send_handle.join().unwrap().unwrap();
|
||||
recv_handle.join().unwrap().unwrap();
|
||||
|
||||
let out_bytes = std::fs::read(&dest_final).unwrap();
|
||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||
assert!(expected.len() < source.len());
|
||||
}
|
||||
|
||||
/// A peer that has gone away: every send fails.
|
||||
struct GoneSink;
|
||||
|
||||
impl MessageSink for GoneSink {
|
||||
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_finalize(&mut self) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
||||
// The reader used to learn the peer was gone only on its next send.
|
||||
// A growing capture that stops growing (and is never renamed) never
|
||||
// gives it one, so run_send waited on the reader forever.
|
||||
for compress in [false, true] {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
||||
std::fs::write(&partial, &stream).unwrap();
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: dir.path().join("cap.pcap"),
|
||||
compress,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
||||
});
|
||||
let result = done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
||||
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
use crate::cli::{ServerCli, ServerRole};
|
||||
use crate::filter::IpFilter;
|
||||
use crate::filter::PacketFilter;
|
||||
use crate::naming;
|
||||
use crate::pipeline::{run_recv, run_send, RecvOptions, SendOptions};
|
||||
use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
|
||||
@@ -39,8 +39,8 @@ fn run_send_server(cli: ServerCli) -> Result<()> {
|
||||
let flush_after = naming::parse_duration(&cli.flush_interval)?;
|
||||
// The sending side is the only side that filters -- dropping packets
|
||||
// before they hit the wire is the whole point. `run_recv` never parses
|
||||
// what it writes, so --recv has no --ip.
|
||||
let filter = IpFilter::parse(&cli.ip)?;
|
||||
// what it writes, so --recv has no --ip/--mac.
|
||||
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: PathBuf::from(source),
|
||||
|
||||
+96
-9
@@ -2,8 +2,9 @@ use crate::chunker::{Chunker, ChunkKind};
|
||||
use crate::tracelog::Logger;
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::os::unix::fs::{FileExt, MetadataExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
|
||||
///
|
||||
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
||||
/// is called, that's a hard error (no polling for creation).
|
||||
///
|
||||
/// `stop` is how the consumer says it has given up (its peer is gone). The
|
||||
/// loop checks it every poll, because otherwise it would only find out on
|
||||
/// its next send -- and a growing source that stops growing, or a filter
|
||||
/// that matches nothing, may never produce one, leaving whoever joins this
|
||||
/// thread waiting forever.
|
||||
pub fn run_tail(
|
||||
source_path: PathBuf,
|
||||
is_partial: bool,
|
||||
@@ -65,6 +72,7 @@ pub fn run_tail(
|
||||
mut chunker: Box<dyn Chunker>,
|
||||
events: impl EventSink,
|
||||
log: Arc<Logger>,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> Result<()> {
|
||||
if !source_path.exists() {
|
||||
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
||||
@@ -75,7 +83,7 @@ pub fn run_tail(
|
||||
source_path.display(),
|
||||
final_path.display()
|
||||
));
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log);
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
log.log(format_args!("tail: done, sending EOF"));
|
||||
@@ -96,6 +104,7 @@ fn drain(
|
||||
chunker: &mut dyn Chunker,
|
||||
events: &impl EventSink,
|
||||
log: &Logger,
|
||||
stop: &AtomicBool,
|
||||
) -> Result<()> {
|
||||
let file = File::open(source_path)?;
|
||||
let mut complete_signal = !is_partial;
|
||||
@@ -123,6 +132,9 @@ fn drain(
|
||||
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
||||
|
||||
loop {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
bail!("stopped: the receiving side is gone");
|
||||
}
|
||||
let file_len = file.metadata()?.len();
|
||||
// Whether this iteration did real work -- if so, loop again
|
||||
// immediately rather than sleeping, so a large already-available
|
||||
@@ -207,7 +219,7 @@ fn drain(
|
||||
send_chunk(events, kind, data)?;
|
||||
}
|
||||
|
||||
if !complete_signal && final_path.try_exists()? {
|
||||
if !complete_signal && renamed_to(final_path, &file)? {
|
||||
// Rename observed -- signal now, well before finalization, per
|
||||
// section 7.4's repoint-then-drain-then-finalize ordering.
|
||||
complete_signal = true;
|
||||
@@ -269,6 +281,22 @@ fn drain(
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `final_path` now names the very file being read, i.e. the writer's
|
||||
/// rename has happened. This is stricter than section 7.5's "the final path
|
||||
/// exists": names get reused (section 6), so a file left at the final name by
|
||||
/// an earlier capture or copy would otherwise pass for the rename the moment
|
||||
/// tailing starts, and the transfer would finish early, short, at the first
|
||||
/// lull in the writer. Comparing identity also still works when the writer
|
||||
/// renames over such a leftover.
|
||||
fn renamed_to(final_path: &Path, file: &File) -> Result<bool> {
|
||||
let ours = file.metadata()?;
|
||||
match std::fs::metadata(final_path) {
|
||||
Ok(theirs) => Ok(theirs.dev() == ours.dev() && theirs.ino() == ours.ino()),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_ready(chunker: &mut dyn Chunker, events: &impl EventSink) -> Result<()> {
|
||||
while let Some((kind, data)) = chunker.ready_chunk() {
|
||||
send_chunk(events, kind, data)?;
|
||||
@@ -316,6 +344,10 @@ mod tests {
|
||||
(data, got_eof)
|
||||
}
|
||||
|
||||
fn no_stop() -> Arc<AtomicBool> {
|
||||
Arc::new(AtomicBool::new(false))
|
||||
}
|
||||
|
||||
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
||||
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
||||
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
||||
@@ -340,7 +372,7 @@ mod tests {
|
||||
std::fs::write(&path, b"hello world").unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
|
||||
let (data, eof) = collect(rx);
|
||||
assert_eq!(data, b"hello world");
|
||||
assert!(eof);
|
||||
@@ -352,7 +384,7 @@ mod tests {
|
||||
let path = dir.path().join("nope.pcap.partial");
|
||||
let final_path = dir.path().join("nope.pcap");
|
||||
let (tx, _rx) = mpsc::channel();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
|
||||
assert!(err.to_string().contains("not found"));
|
||||
}
|
||||
|
||||
@@ -369,7 +401,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -399,7 +431,7 @@ mod tests {
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
let log = log.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log)
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -426,7 +458,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -457,7 +489,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
// Let the reader fully drain the initial 10 bytes.
|
||||
@@ -522,6 +554,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -569,6 +602,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -624,6 +658,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -668,6 +703,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -706,8 +742,59 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
);
|
||||
try_collect(rx);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_file_at_the_final_name_is_not_the_rename() {
|
||||
// A leftover cap.pcap (an earlier capture reusing the name) used to
|
||||
// pass for the rename the moment tailing started, so the first lull
|
||||
// longer than the grace period finalized the copy short.
|
||||
let dir = tempdir().unwrap();
|
||||
let partial_path = dir.path().join("cap.pcap.partial");
|
||||
let final_path = dir.path().join("cap.pcap");
|
||||
std::fs::write(&final_path, b"yesterday's capture").unwrap();
|
||||
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
|
||||
stream.extend_from_slice(&record_bytes(&[2u8; 10], 1_700_000_001));
|
||||
let mut f = File::create(&partial_path).unwrap();
|
||||
f.write_all(&stream).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || {
|
||||
run_tail(
|
||||
partial_path,
|
||||
true,
|
||||
final_path,
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
|
||||
// A writer lull longer than the grace period, then more packets and
|
||||
// the real rename -- over the stale file.
|
||||
std::thread::sleep(FINALIZE_GRACE_PERIOD + Duration::from_millis(500));
|
||||
let rec3 = record_bytes(&[3u8; 10], 1_700_000_002);
|
||||
f.write_all(&rec3).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
stream.extend_from_slice(&rec3);
|
||||
drop(f);
|
||||
std::fs::rename(&partial_path, &final_path).unwrap();
|
||||
|
||||
handle.join().unwrap().unwrap();
|
||||
let (data, eof) = collect(rx);
|
||||
assert_eq!(data, stream);
|
||||
assert!(eof);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user