Compare commits

...
8 Commits
Author SHA1 Message Date
eric bb67bbafa1 Merge pull request 'New agent who dis' (#3) from new-agent-who-dis into master
Reviewed-on: #3
2026-11-16 16:05:24 +00:00
eric 6b68deb047 Merge pull request 'Add --mac filter and rename IpFilter to PacketFilter' (#2) from mac-filter into master
Reviewed-on: #2
2026-11-13 21:22:16 +00:00
ericandClaude Opus 5.5 cc698b7826 Detect the rename by file identity, not by the final name existing
A file already sitting at the final name -- an earlier capture reusing
the name, or a previous copy -- counted as the rename the moment tailing
started, so the first writer lull longer than the grace period finalized
the transfer short. Treat the rename as observed only once the final path
is the same inode as the fd being tailed, which also still works when the
writer renames over such a leftover.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 16:13:53 -05:00
ericandClaude Opus 5.5 3eb779a6aa Stop the source reader when the peer is gone instead of hanging
After a send failed, run_send joined the reader thread, but the reader
only learned the receiver was gone on its next send. An idle growing
capture, or a filter matching nothing, never produced one, so the
process hung -- locally, and as an orphaned remote --send server. The
reader now checks a stop flag every poll, raised once the send loop
ends. The client also kills ssh when a remote-to-local receive fails
locally rather than waiting on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 16:12:47 -05:00
ericandClaude Opus 5.5 b46d6a936c Make the chunk size non-zero by type; reject overflowing sizes
--chunk 0 made every chunk 'full' while empty, so the chunker emitted
empty chunks (and empty zstd frames) forever -- 11 MB in 3 s from a
424-byte source. parse_size now returns NonZeroUsize, which flows through
SendOptions into RecordAlignedChunker, so a zero target can't reach the
cut logic. parse_size and parse_duration also use checked_mul instead of
wrapping on absurd inputs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 16:11:17 -05:00
ericandClaude Opus 5.5 6a80aaf461 Refuse local copies whose destination is the source
Copying a capture into its own directory resolved the destination to the
source's own path: a growing cap.pcap.partial was truncated to zero when
the receiver opened its in-flight file, and a finished capture was
silently replaced by its (possibly filtered) copy. Compare the source and
its final name against both destination paths by inode, or by canonical
parent + name for files that don't exist yet, and bail before anything
is opened.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 16:10:35 -05:00
ericandClaude Opus 5.5 f21b6c53e8 Add growing-capture --mac pipeline test; mention --mac in filter/server comments
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 16:07:49 -05:00
ericandClaude Sonnet 5.5 a9e283a87c Add --mac filter and rename IpFilter to PacketFilter
A packet is kept if its Ethernet source or destination MAC matches any
--mac entry; combined with --ip, both must match. Empty dimensions match
everything, so the unfiltered case is still the same code path.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-13 14:48:51 -05:00
8 changed files with 588 additions and 92 deletions

No files matched your search

+27 -22
View File
@@ -1,12 +1,13 @@
use crate::filter::{IpFilter, LINKTYPE_ETHERNET};
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
use anyhow::{Result, bail};
use std::num::NonZeroUsize;
use std::time::{Duration, Instant};
pub const GLOBAL_HEADER_LEN: usize = 24;
pub const RECORD_HEADER_LEN: usize = 16;
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
#[allow(dead_code)]
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
/// Sanity bound on a record's declared payload length, per
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
/// sources -- see `RecordAlignedChunker::partial`.
@@ -166,7 +167,7 @@ pub struct RecordAlignedChunker {
/// KEPT bytes -- a filter keeping 1% of the traffic still produces
/// full-size frames rather than 1%-size ones.
kept: Vec<u8>,
filter: IpFilter,
filter: PacketFilter,
/// When the oldest byte currently in `kept` was confirmed, i.e. how
/// long the pending output has been waiting. `None` when `kept` is
/// empty.
@@ -178,7 +179,7 @@ pub struct RecordAlignedChunker {
/// bytes is what keeps `flush_after` an actual latency bound rather
/// than an idle-only one.
kept_since: Option<Instant>,
target: usize,
target: NonZeroUsize,
flush_after: Duration,
/// Whether this source is a still-growing (partial) file. Gates every
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
@@ -190,20 +191,20 @@ pub struct RecordAlignedChunker {
impl RecordAlignedChunker {
/// Unfiltered: keeps every record. Equivalent to `with_filter` with
/// `IpFilter::match_all()` -- which is not a separate code path, just
/// `PacketFilter::match_all()` -- which is not a separate code path, just
/// a filter that says yes, so this and the filtered case exercise the
/// same record-confirmation logic. Production goes through
/// `with_filter`; this is the convenience the tests use.
#[allow(dead_code)]
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
Self::with_filter(target, flush_after, partial, IpFilter::match_all())
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
}
pub fn with_filter(
target: usize,
target: NonZeroUsize,
flush_after: Duration,
partial: bool,
filter: IpFilter,
filter: PacketFilter,
) -> Self {
Self {
header_buf: Vec::with_capacity(GLOBAL_HEADER_LEN),
@@ -247,7 +248,7 @@ impl RecordAlignedChunker {
// stdout: in --server mode stdout is the wire.
if !self.filter.is_match_all() && self.linktype != LINKTYPE_ETHERNET {
eprintln!(
"scpcap: warning: --ip filtering needs Ethernet (link type {LINKTYPE_ETHERNET}), \
"scpcap: warning: --ip/--mac filtering needs Ethernet (link type {LINKTYPE_ETHERNET}), \
but this capture is link type {}; no packets will match",
self.linktype
);
@@ -400,7 +401,7 @@ impl Chunker for RecordAlignedChunker {
// must produce no chunk at all, rather than an empty one. An empty
// chunk would become a zero-content zstd frame, which
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
if self.kept.len() >= self.target {
if self.kept.len() >= self.target.get() {
return Some(self.take_kept());
}
None
@@ -509,6 +510,10 @@ mod tests {
h
}
fn nz(n: usize) -> NonZeroUsize {
NonZeroUsize::new(n).unwrap()
}
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
record_bytes_with_ts(payload, big_endian, 0)
}
@@ -571,7 +576,7 @@ mod tests {
#[test]
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
let mut stream = header_bytes(0xA1B2C3D4);
let mut expected_records: Vec<Vec<u8>> = Vec::new();
for i in 0..20u8 {
@@ -606,7 +611,7 @@ mod tests {
#[test]
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
c.ready_chunk(); // drain header
@@ -895,8 +900,8 @@ mod tests {
record_bytes_with_ts(&pkt, false, 1_700_000_000)
}
fn filter_for(specs: &[&str]) -> IpFilter {
IpFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
fn filter_for(specs: &[&str]) -> PacketFilter {
PacketFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>(), &[]).unwrap()
}
fn drain_all(c: &mut RecordAlignedChunker) -> Vec<(ChunkKind, Vec<u8>)> {
@@ -913,7 +918,7 @@ mod tests {
#[test]
fn keeps_only_matching_records_and_keeps_them_whole() {
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let hit_src = ip_record(WANTED, OTHER, 60);
let hit_dst = ip_record(OTHER, WANTED, 60);
let miss = ip_record(OTHER, THIRD, 60);
@@ -946,7 +951,7 @@ mod tests {
let hit = ip_record(WANTED, OTHER, 100);
let miss = ip_record(OTHER, THIRD, 100);
let target = 10 * hit.len();
let mut c = RecordAlignedChunker::with_filter(target, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
c.feed(&eth_header_bytes()).unwrap();
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
@@ -1001,7 +1006,7 @@ mod tests {
// An empty chunk would be compressed into a zero-content zstd frame,
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
let miss = ip_record(OTHER, THIRD, 100);
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
c.feed(&eth_header_bytes()).unwrap();
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
for _ in 0..20 {
@@ -1026,9 +1031,9 @@ mod tests {
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
}
let mut all = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, IpFilter::match_all());
let mut some = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut none = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
for c in [&mut all, &mut some, &mut none] {
c.feed(&stream).unwrap();
}
@@ -1052,7 +1057,7 @@ mod tests {
// exactly once even though the bytes around it are re-read.
let hit = ip_record(WANTED, OTHER, 60);
let miss = ip_record(OTHER, THIRD, 60);
let mut c = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
let mut stream = eth_header_bytes();
stream.extend_from_slice(&miss);
+13
View File
@@ -48,6 +48,14 @@ pub struct ClientCli {
#[arg(long = "ip", value_name = "ADDR")]
pub ip: Vec<String>,
/// Only forward packets whose Ethernet source or destination MAC
/// matches, e.g. to pull one switch connection out of a capture by its
/// upstream MAC ("aa:bb:cc:dd:ee:ff"). May be repeated; a packet is kept
/// if it matches any entry. Combined with --ip, a packet must match
/// both. Requires an Ethernet-linktype pcap source.
#[arg(long = "mac", value_name = "ADDR")]
pub mac: Vec<String>,
/// Append a diagnostic trace of this (local) process's transfer events
/// to this file -- positions, rename detection, finalize -- for
/// figuring out where a stalled transfer is stuck.
@@ -105,6 +113,11 @@ pub struct ServerCli {
#[arg(long = "ip", value_name = "ADDR")]
pub ip: Vec<String>,
/// MAC filter for --send mode; repeated, one per address. See the
/// client's --mac.
#[arg(long = "mac", value_name = "ADDR")]
pub mac: Vec<String>,
/// Destination final path (--recv mode).
#[arg(long)]
pub dest_final: Option<String>,
+121 -5
View File
@@ -1,13 +1,14 @@
use crate::channel::{ChannelSink, ChannelSource};
use crate::cli::ClientCli;
use crate::filter::IpFilter;
use crate::filter::PacketFilter;
use crate::naming::{self, resolve_dest_paths, strip_name, Target};
use crate::pipeline::{run_recv, run_send, RecvOptions, SendOptions};
use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
use crate::ssh::{build_server_command, spawn_remote_server};
use crate::tracelog::Logger;
use anyhow::{bail, Result};
use std::path::PathBuf;
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::sync::{mpsc, Arc};
use std::thread;
@@ -43,7 +44,7 @@ pub fn run(cli: ClientCli) -> Result<()> {
// remote is (see the Remote->Local branch). It is never passed to a
// --recv server: the receiver writes bytes verbatim and never parses
// them.
let filter = IpFilter::parse(&cli.ip)?;
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
// Filtering means reading pcap records, which an already-compressed
// source doesn't expose. Without this the transfer would still pick the
// record-aligned chunker (a filter forces it) and abort deep inside with
@@ -52,8 +53,8 @@ pub fn run(cli: ClientCli) -> Result<()> {
// combination rather than the source.
if !filter.is_match_all() && source_stripped.was_zst {
bail!(
"cannot apply --ip to an already-compressed source ({}): \
filtering needs to read pcap records. Copy it without --ip, \
"cannot apply --ip/--mac to an already-compressed source ({}): \
filtering needs to read pcap records. Copy it without --ip/--mac, \
or filter the uncompressed capture.",
cli.source
);
@@ -63,6 +64,10 @@ pub fn run(cli: ClientCli) -> Result<()> {
match (&source_target, &dest_target) {
(Target::Local(src), Target::Local(_)) => {
refuse_overlap(
[Path::new(src), Path::new(&final_source_path)],
[Path::new(&dest.temp_path), Path::new(&dest.final_path)],
)?;
let send_opts = SendOptions {
source_path: PathBuf::from(src),
is_partial,
@@ -143,6 +148,10 @@ pub fn run(cli: ClientCli) -> Result<()> {
server_args.push("--ip".to_string());
server_args.push(spec.clone());
}
for spec in &cli.mac {
server_args.push("--mac".to_string());
server_args.push(spec.clone());
}
if effective_compress {
server_args.push("--compress".to_string());
server_args.push("zstd".to_string());
@@ -162,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> {
};
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
let recv_result = run_recv(source, recv_opts);
if recv_result.is_err() {
// The remote sender only notices we're gone when its next
// write fails, which on an idle capture may be never. Don't
// wait for that: drop the connection, so its next write
// fails and it stops.
let _ = child.kill();
}
let status = child.wait()?;
recv_result?;
if !status.success() {
@@ -174,3 +190,103 @@ pub fn run(cli: ClientCli) -> Result<()> {
}
}
}
/// Refuses a local copy whose destination lands on the source. The receiver
/// truncates its in-flight file on open and renames over its final one, so
/// either collision destroys the capture being read. Copying a capture into
/// its own directory is enough to hit it: `dir/cap.pcap.partial` to `dir/`
/// resolves to `dir/cap.pcap`, written via `dir/cap.pcap.partial`.
///
/// `sources` is the source as given plus its post-rename final path, since a
/// growing source moves from one to the other mid-transfer.
fn refuse_overlap(sources: [&Path; 2], dests: [&Path; 2]) -> Result<()> {
for source in sources {
for dest in dests {
if same_location(source, dest) {
bail!(
"destination {} is the source {}: copying would overwrite the capture \
being read. Copy it to a different directory or name.",
dest.display(),
source.display()
);
}
}
}
Ok(())
}
/// Whether two paths name the same file, whether or not it exists yet: the
/// same inode when both exist (catching hard links and symlinks), otherwise
/// the same canonical parent directory and file name.
fn same_location(a: &Path, b: &Path) -> bool {
if let (Ok(ma), Ok(mb)) = (std::fs::metadata(a), std::fs::metadata(b)) {
return ma.dev() == mb.dev() && ma.ino() == mb.ino();
}
match (resolve(a), resolve(b)) {
(Some(ra), Some(rb)) => ra == rb,
_ => false,
}
}
/// The canonical form of `path`, or for a file that doesn't exist yet, its
/// canonical parent joined with its name. `None` when even the parent
/// doesn't exist, in which case nothing can collide with it.
fn resolve(path: &Path) -> Option<PathBuf> {
if let Ok(canonical) = std::fs::canonicalize(path) {
return Some(canonical);
}
let parent = match path.parent() {
Some(p) if !p.as_os_str().is_empty() => p,
_ => Path::new("."),
};
Some(std::fs::canonicalize(parent).ok()?.join(path.file_name()?))
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn copying_a_growing_capture_into_its_own_directory_is_refused() {
let dir = tempdir().unwrap();
let partial = dir.path().join("cap.pcap.partial");
std::fs::write(&partial, b"live").unwrap();
let final_src = dir.path().join("cap.pcap");
// What `resolve_dest_paths` makes of DEST = the source's directory.
let err = refuse_overlap([&partial, &final_src], [&partial, &final_src]).unwrap_err();
assert!(err.to_string().contains("overwrite the capture"), "{err}");
}
#[test]
fn a_symlinked_or_relative_spelling_of_the_source_is_still_caught() {
let dir = tempdir().unwrap();
let real = dir.path().join("real");
std::fs::create_dir(&real).unwrap();
let src = real.join("cap.pcap");
std::fs::write(&src, b"done").unwrap();
let link = dir.path().join("link");
std::os::unix::fs::symlink(&real, &link).unwrap();
let via_link = link.join("cap.pcap");
let via_dotdot = real.join("..").join("real").join("cap.pcap.partial");
assert!(refuse_overlap([&src, &src], [&via_dotdot, &via_link]).is_err());
// Neither exists yet: compared by canonical parent + name.
let not_yet = real.join("other.pcap");
assert!(refuse_overlap([&not_yet, &not_yet], [&link.join("other.pcap"), &via_dotdot]).is_err());
}
#[test]
fn a_different_destination_is_allowed() {
let dir = tempdir().unwrap();
let src = dir.path().join("cap.pcap.partial");
std::fs::write(&src, b"live").unwrap();
let out = dir.path().join("out");
std::fs::create_dir(&out).unwrap();
let final_src = dir.path().join("cap.pcap");
let dests = [out.join("cap.pcap.partial"), out.join("cap.pcap")];
assert!(refuse_overlap([&src, &final_src], [&dests[0], &dests[1]]).is_ok());
// A destination under a directory that doesn't exist can't collide.
let missing = dir.path().join("nope").join("cap.pcap");
assert!(refuse_overlap([&src, &final_src], [&missing, &missing]).is_ok());
}
}
+149 -29
View File
@@ -19,38 +19,55 @@ const MAX_VLAN_TAGS: usize = 2;
const IPV4_HEADER_MIN: usize = 20;
const IPV6_HEADER_LEN: usize = 40;
/// A set of IP addresses/prefixes to keep. A packet matches if *either* its
/// source or destination address falls in *any* entry.
const MAC_LEN: usize = 6;
/// An Ethernet MAC address.
type Mac = [u8; MAC_LEN];
/// What to keep from a capture: packets whose IP addresses and Ethernet MAC
/// addresses match.
///
/// Sized for the expected 1-4 entries: two plain vectors, split by address
/// family so a v4 packet never compares against a v6 entry, scanned
/// linearly. No sorting (a prefix match isn't an equality test, so ordering
/// buys nothing) and no hashing.
/// Each dimension is a set of entries; a packet satisfies a dimension if
/// *either* its source or destination falls in *any* entry. An empty
/// dimension is satisfied by everything (without looking at the packet), and
/// a packet must satisfy every dimension to be kept -- so `--ip` and `--mac`
/// together mean "this IP *on* this MAC".
///
/// Sized for the expected 1-4 entries: plain vectors scanned linearly. IP
/// entries are split by address family so a v4 packet never compares against
/// a v6 entry. No sorting (a prefix match isn't an equality test, so
/// ordering buys nothing) and no hashing.
#[derive(Debug, Clone)]
pub struct IpFilter {
/// Set when no `--ip` was given: every packet is kept, without looking
/// at it at all. This is the default so that the filtering code path is
/// the *only* code path -- see the chunker.
pub struct PacketFilter {
/// Set when neither `--ip` nor `--mac` was given: every packet is kept,
/// without looking at it at all. This is the default so that the
/// filtering code path is the *only* code path -- see the chunker.
match_all: bool,
/// (network, mask) pairs, network already masked at parse time.
v4: Vec<(u32, u32)>,
v6: Vec<(u128, u128)>,
macs: Vec<Mac>,
}
impl IpFilter {
/// The no-`--ip` default: keeps everything, matches without inspecting.
impl PacketFilter {
/// The no-`--ip`/`--mac` default: keeps everything, matches without
/// inspecting.
pub fn match_all() -> Self {
Self { match_all: true, v4: Vec::new(), v6: Vec::new() }
Self { match_all: true, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() }
}
/// Parses `--ip` specs: a bare address (`1.1.1.1`, `2606:4700::1111`)
/// or a CIDR prefix (`10.0.0.0/8`). An empty list yields `match_all`.
pub fn parse(specs: &[String]) -> Result<Self> {
if specs.is_empty() {
/// Parses `--ip` specs -- a bare address (`1.1.1.1`, `2606:4700::1111`)
/// or a CIDR prefix (`10.0.0.0/8`) -- and `--mac` specs
/// (`aa:bb:cc:dd:ee:ff`). Two empty lists yield `match_all`.
pub fn parse(ip_specs: &[String], mac_specs: &[String]) -> Result<Self> {
if ip_specs.is_empty() && mac_specs.is_empty() {
return Ok(Self::match_all());
}
let mut f = Self { match_all: false, v4: Vec::new(), v6: Vec::new() };
for spec in specs {
let mut f = Self { match_all: false, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() };
for spec in mac_specs {
f.macs.push(parse_mac(spec)?);
}
for spec in ip_specs {
let (addr_str, prefix_str) = match spec.split_once('/') {
Some((a, p)) => (a, Some(p)),
None => (spec.as_str(), None),
@@ -82,7 +99,8 @@ impl IpFilter {
Ok(f)
}
/// Whether this filter keeps everything, i.e. no `--ip` was given.
/// Whether this filter keeps everything, i.e. neither `--ip` nor `--mac`
/// was given.
/// Callers use it to skip work (and warnings) that only a real filter
/// needs.
pub fn is_match_all(&self) -> bool {
@@ -92,9 +110,10 @@ impl IpFilter {
/// Whether to keep `packet` -- the captured bytes of one pcap record,
/// starting at the link-layer header.
///
/// Everything unmatched is dropped: non-IP frames (ARP, LLDP), packets
/// truncated by snaplen before the addresses, and any link type other
/// than Ethernet.
/// Everything unmatched is dropped: packets truncated by snaplen before
/// the fields being matched, any link type other than Ethernet, and --
/// when `--ip` is set -- non-IP frames (ARP, LLDP). With only `--mac`,
/// non-IP frames from a matching MAC are kept.
pub fn matches(&self, packet: &[u8], linktype: u32) -> bool {
if self.match_all {
return true;
@@ -102,6 +121,27 @@ impl IpFilter {
if linktype != LINKTYPE_ETHERNET || packet.len() < ETH_HEADER_LEN {
return false;
}
self.mac_matches(packet) && self.ip_matches(packet)
}
/// Whether the frame's source or destination MAC is in the set; an empty
/// set matches every frame. `packet` is at least an Ethernet header long.
fn mac_matches(&self, packet: &[u8]) -> bool {
if self.macs.is_empty() {
return true;
}
let dst: Mac = packet[0..MAC_LEN].try_into().unwrap();
let src: Mac = packet[MAC_LEN..2 * MAC_LEN].try_into().unwrap();
self.macs.iter().any(|&m| m == src || m == dst)
}
/// Whether the frame's source or destination IP is in the set; an empty
/// set matches every frame, IP or not. `packet` is at least an Ethernet
/// header long.
fn ip_matches(&self, packet: &[u8]) -> bool {
if self.v4.is_empty() && self.v6.is_empty() {
return true;
}
let mut ethertype = be16(&packet[12..14]);
let mut off = ETH_HEADER_LEN;
let mut tags = 0;
@@ -137,6 +177,25 @@ impl IpFilter {
}
}
/// Parses `aa:bb:cc:dd:ee:ff` (or `-` separated) into a MAC address.
fn parse_mac(spec: &str) -> Result<Mac> {
let bad = || anyhow::anyhow!("--mac {spec:?}: not a MAC address (expected aa:bb:cc:dd:ee:ff)");
let mut mac = [0u8; MAC_LEN];
let mut parts = spec.split([':', '-']);
for byte in &mut mac {
let part = parts.next().ok_or_else(bad)?;
// from_str_radix alone would accept a leading '+'.
if part.len() != 2 || !part.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(bad());
}
*byte = u8::from_str_radix(part, 16).map_err(|_| bad())?;
}
if parts.next().is_some() {
return Err(bad());
}
Ok(mac)
}
fn be16(b: &[u8]) -> u16 {
u16::from_be_bytes(b.try_into().unwrap())
}
@@ -157,8 +216,12 @@ mod tests {
list.iter().map(|s| s.to_string()).collect()
}
fn filter(list: &[&str]) -> IpFilter {
IpFilter::parse(&specs(list)).unwrap()
fn filter(list: &[&str]) -> PacketFilter {
PacketFilter::parse(&specs(list), &[]).unwrap()
}
fn mac_filter(ips: &[&str], macs: &[&str]) -> PacketFilter {
PacketFilter::parse(&specs(ips), &specs(macs)).unwrap()
}
/// An Ethernet frame with `vlan_tags` 802.1Q tags in front of `ethertype`.
@@ -199,12 +262,12 @@ mod tests {
#[test]
fn match_all_keeps_everything_including_non_ip_and_odd_linktypes() {
let f = IpFilter::match_all();
let f = PacketFilter::match_all();
assert!(f.is_match_all());
assert!(f.matches(&[], 113));
assert!(f.matches(&eth(0, 0x0806, &[1, 2, 3]), LINKTYPE_ETHERNET));
// An empty --ip list is the same thing.
assert!(IpFilter::parse(&[]).unwrap().is_match_all());
assert!(PacketFilter::parse(&[], &[]).unwrap().is_match_all());
}
#[test]
@@ -298,12 +361,12 @@ mod tests {
fn bad_specs_are_rejected_at_parse() {
for spec in ["1.1.1.1/33", "::1/129", "not-an-ip", "1.1.1.1/x", "1.1.1.256"] {
assert!(
IpFilter::parse(&specs(&[spec])).is_err(),
PacketFilter::parse(&specs(&[spec]), &[]).is_err(),
"{spec:?} should be rejected"
);
}
// /0 and full-width prefixes are fine.
assert!(IpFilter::parse(&specs(&["1.1.1.1/32", "0.0.0.0/0", "::/0", "::1/128"])).is_ok());
assert!(PacketFilter::parse(&specs(&["1.1.1.1/32", "0.0.0.0/0", "::/0", "::1/128"]), &[]).is_ok());
}
#[test]
@@ -314,4 +377,61 @@ mod tests {
assert!(f.matches(&eth(0, ETHERTYPE_IPV6, &ipv6("::1", "2606:4700::1")), LINKTYPE_ETHERNET));
assert!(!f.matches(&v4_packet("9.9.9.9", "203.0.113.1"), LINKTYPE_ETHERNET));
}
const UPSTREAM: &str = "02:00:00:00:00:aa";
const OTHER: &str = "02:00:00:00:00:bb";
/// An Ethernet frame between two MACs carrying `ethertype`/`payload`.
fn eth_between(dst: &str, src: &str, ethertype: u16, payload: &[u8]) -> Vec<u8> {
let mut f = parse_mac(dst).unwrap().to_vec();
f.extend_from_slice(&parse_mac(src).unwrap());
f.extend_from_slice(&ethertype.to_be_bytes());
f.extend_from_slice(payload);
f
}
#[test]
fn mac_matches_either_direction_only() {
let f = mac_filter(&[], &[UPSTREAM]);
assert!(!f.is_match_all());
let ip = ipv4("1.1.1.1", "9.9.9.9");
assert!(f.matches(&eth_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
assert!(f.matches(&eth_between(UPSTREAM, OTHER, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
assert!(!f.matches(&eth_between(OTHER, OTHER, ETHERTYPE_IPV4, &ip), LINKTYPE_ETHERNET));
}
#[test]
fn mac_only_keeps_non_ip_frames_and_ignores_vlan_tags() {
let f = mac_filter(&[], &[UPSTREAM]);
assert!(f.matches(&eth_between(OTHER, UPSTREAM, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
assert!(f.matches(&eth_between(OTHER, UPSTREAM, ETHERTYPE_VLAN, &[0u8; 4]), LINKTYPE_ETHERNET));
}
#[test]
fn mac_and_ip_must_both_match() {
let f = mac_filter(&["1.1.1.1"], &[UPSTREAM]);
let wanted = ipv4("1.1.1.1", "9.9.9.9");
let unwanted = ipv4("8.8.8.8", "9.9.9.9");
assert!(f.matches(&eth_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &wanted), LINKTYPE_ETHERNET));
assert!(!f.matches(&eth_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &unwanted), LINKTYPE_ETHERNET));
assert!(!f.matches(&eth_between(OTHER, OTHER, ETHERTYPE_IPV4, &wanted), LINKTYPE_ETHERNET));
// ARP on the right MAC still fails the IP dimension.
assert!(!f.matches(&eth_between(OTHER, UPSTREAM, 0x0806, &[0u8; 28]), LINKTYPE_ETHERNET));
}
#[test]
fn mac_specs_parse_strictly() {
assert_eq!(parse_mac("AA:bb:0c:0D:ee:ff").unwrap(), [0xaa, 0xbb, 0x0c, 0x0d, 0xee, 0xff]);
assert_eq!(parse_mac("aa-bb-cc-dd-ee-ff").unwrap(), [0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff]);
for bad in ["", "aa:bb:cc:dd:ee", "aa:bb:cc:dd:ee:ff:00", "aa:bb:cc:dd:ee:fg", "a:bb:cc:dd:ee:ff", "aabbccddeeff", "+a:bb:cc:dd:ee:ff"] {
assert!(parse_mac(bad).is_err(), "{bad:?} should be rejected");
}
}
#[test]
fn mac_filter_on_non_ethernet_matches_nothing() {
let f = mac_filter(&[], &[UPSTREAM]);
let pkt = eth_between(OTHER, UPSTREAM, ETHERTYPE_IPV4, &ipv4("1.1.1.1", "9.9.9.9"));
assert!(!f.matches(&pkt, 113));
}
}
+17 -10
View File
@@ -1,4 +1,5 @@
use anyhow::{bail, Result};
use std::num::NonZeroUsize;
use std::time::Duration;
use std::path::Path;
@@ -111,7 +112,9 @@ pub fn resolve_dest_paths(
}
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
pub fn parse_size(s: &str) -> Result<usize> {
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
/// a zero target would cut empty chunks without end.
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
let s = s.trim();
if s.is_empty() {
bail!("empty size");
@@ -126,7 +129,8 @@ pub fn parse_size(s: &str) -> Result<usize> {
.trim()
.parse()
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
Ok(n * mult)
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
}
/// Parses a flush/idle interval: a bare number is milliseconds, or an
@@ -148,7 +152,8 @@ pub fn parse_duration(s: &str) -> Result<Duration> {
.trim()
.parse()
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
Ok(Duration::from_millis(n * mult_ms))
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
Ok(Duration::from_millis(ms))
}
/// Single-quote a string for safe inclusion as one argument in a remote shell
@@ -282,19 +287,21 @@ mod tests {
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
// 0 is legal: cut a chunk as soon as anything is kept.
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
for bad in ["", "abc", "0.1s", "100m", "-5"] {
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
}
}
#[test]
fn size_parsing() {
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
assert_eq!(parse_size("512").unwrap(), 512);
assert!(parse_size("").is_err());
assert!(parse_size("abc").is_err());
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
assert_eq!(parse_size("512").unwrap().get(), 512);
// 0 would cut empty chunks forever; overflow must not wrap.
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
}
}
#[test]
+162 -14
View File
@@ -1,5 +1,5 @@
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker};
use crate::filter::IpFilter;
use crate::filter::PacketFilter;
use crate::protocol::{Message, MessageSink, MessageSource};
use crate::tail::{run_tail, SourceEvent};
use crate::tracelog::Logger;
@@ -7,7 +7,9 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
use anyhow::{bail, Result};
use std::fs::File;
use std::io::Write;
use std::num::NonZeroUsize;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc};
use std::time::Duration;
use std::thread;
@@ -23,15 +25,15 @@ pub struct SendOptions {
pub is_partial: bool,
pub final_source_path: PathBuf,
pub compress: bool,
pub chunk_target: usize,
pub chunk_target: NonZeroUsize,
/// How long kept-but-undersized output may wait before being sent
/// anyway. See `Chunker::should_flush`.
pub flush_after: Duration,
pub level: i32,
/// Which packets to forward. `IpFilter::match_all()` when no `--ip` was
/// Which packets to forward. `PacketFilter::match_all()` when no `--ip`/`--mac` was
/// given -- not a disabled filter, just one that says yes, so there is
/// only ever one code path through the chunker.
pub filter: IpFilter,
pub filter: PacketFilter,
pub log: Arc<Logger>,
}
@@ -57,7 +59,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
// structure is the only way to tell real content from a preallocating
// writer's not-yet-written tail. Here the chunker is a validator, not
// a framer.
// * a real `--ip` filter: deciding whether to forward a packet means
// * a real `--ip`/`--mac` filter: deciding whether to forward a packet means
// finding the packet, and record boundaries are the only way to know
// where one starts. Here the chunker is a demultiplexer.
// A final, uncompressed copy has neither: `run_recv` writes every Data
@@ -79,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
};
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
// Raised once the send loop below is done, for whatever reason. After a
// clean EOF or a source error the reader has already returned and never
// looks; after a failed send it is how the reader learns to quit instead
// of polling the source until it next has something to send, which on an
// idle capture or a filter matching nothing may be never.
let stop = Arc::new(AtomicBool::new(false));
let reader = thread::spawn({
let source_path = opts.source_path.clone();
let is_partial = opts.is_partial;
let final_source_path = opts.final_source_path.clone();
let log = opts.log.clone();
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
let stop = stop.clone();
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
});
if opts.compress {
@@ -139,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
}
}
}
stop.store(true, Ordering::Relaxed);
let reader_result = reader.join().unwrap();
let compressor_result = compressor.join().unwrap();
if let Some(e) = send_err {
@@ -174,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
}
}
}
stop.store(true, Ordering::Relaxed);
let reader_result = reader.join().unwrap();
if let Some(e) = send_err {
return Err(e);
@@ -273,7 +284,7 @@ mod tests {
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
filter: PacketFilter::match_all(),
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
@@ -311,7 +322,7 @@ mod tests {
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
filter: PacketFilter::match_all(),
log: Arc::new(Logger::none()),
};
@@ -357,10 +368,10 @@ mod tests {
is_partial: false,
final_source_path: src.clone(),
compress: true,
chunk_target: 512, // small target to force multiple frames
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
filter: PacketFilter::match_all(),
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
@@ -398,7 +409,7 @@ mod tests {
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: IpFilter::match_all(),
filter: PacketFilter::match_all(),
log: Arc::new(Logger::none()),
};
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
@@ -421,8 +432,8 @@ mod tests {
record_bytes(&pkt, 1_700_000_000)
}
fn ip_filter(specs: &[&str]) -> IpFilter {
IpFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
fn ip_filter(specs: &[&str]) -> PacketFilter {
PacketFilter::parse(&specs.iter().map(|s| s.to_string()).collect::<Vec<_>>(), &[]).unwrap()
}
/// Source stream plus the exact pcap a `--ip 1.1.1.1` transfer should
@@ -485,7 +496,7 @@ mod tests {
is_partial: false,
final_source_path: src.clone(),
compress: true,
chunk_target: 512,
chunk_target: NonZeroUsize::new(512).unwrap(),
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: ip_filter(&["1.1.1.1"]),
@@ -530,4 +541,141 @@ mod tests {
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
}
/// A record holding an Ethernet frame from `src` to `dst` with
/// `ethertype`, padded to `len` captured bytes.
fn mac_record(dst: [u8; 6], src: [u8; 6], ethertype: u16, len: usize, ts_sec: u32) -> Vec<u8> {
let mut pkt = vec![0u8; len.max(14)];
pkt[0..6].copy_from_slice(&dst);
pkt[6..12].copy_from_slice(&src);
pkt[12..14].copy_from_slice(&ethertype.to_be_bytes());
record_bytes(&pkt, ts_sec)
}
#[test]
fn mac_filtered_growing_capture_carries_only_the_upstream_link() {
// The scenario --mac exists for: a capture still being written, from
// which only the traffic on one switch connection is wanted. The
// source grows in uneven appends that tear records mid-header and
// mid-payload, so the filter must only ever see confirmed records.
const UPSTREAM: [u8; 6] = [0x02, 0, 0, 0, 0, 0xaa];
const OTHER: [u8; 6] = [0x02, 0, 0, 0, 0, 0xbb];
const BROADCAST: [u8; 6] = [0xff; 6];
let mut source = eth_header_bytes();
let mut expected = eth_header_bytes();
for i in 0..60u32 {
let ts = 1_700_000_000 + i;
let len = 64 + i as usize;
let (record, kept) = match i % 4 {
0 => (mac_record(OTHER, UPSTREAM, 0x0800, len, ts), true), // from the switch
1 => (mac_record(UPSTREAM, OTHER, 0x86DD, len, ts), true), // to the switch
2 => (mac_record(BROADCAST, UPSTREAM, 0x0806, len, ts), true), // ARP from the switch
_ => (mac_record(BROADCAST, OTHER, 0x0806, len, ts), false), // someone else's ARP
};
source.extend_from_slice(&record);
if kept {
expected.extend_from_slice(&record);
}
}
let dir = tempdir().unwrap();
let partial = dir.path().join("cap.pcap.partial");
let final_src = dir.path().join("cap.pcap");
std::fs::write(&partial, &source[..10]).unwrap();
let dest_final = dir.path().join("out.pcap.zst");
let dest_temp = dir.path().join("out.pcap.zst.partial");
let opts = SendOptions {
source_path: partial.clone(),
is_partial: true,
final_source_path: final_src.clone(),
compress: true,
chunk_target: NonZeroUsize::new(512).unwrap(),
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
log: Arc::new(Logger::none()),
};
let (tx, rx) = mpsc::channel::<Message>();
let recv_handle = thread::spawn({
let dest_final = dest_final.clone();
let dest_temp = dest_temp.clone();
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
});
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
// Odd-sized appends (an awkward stride relative to the ~80-140 byte
// records) so the visible end lands inside headers and payloads.
let mut written = 10;
for step in [37usize, 501, 13, 977, 250].iter().cycle() {
if written == source.len() {
break;
}
let end = (written + step).min(source.len());
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
f.write_all(&source[written..end]).unwrap();
f.sync_all().unwrap();
written = end;
thread::sleep(std::time::Duration::from_millis(5));
}
std::fs::rename(&partial, &final_src).unwrap();
send_handle.join().unwrap().unwrap();
recv_handle.join().unwrap().unwrap();
let out_bytes = std::fs::read(&dest_final).unwrap();
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
assert!(expected.len() < source.len());
}
/// A peer that has gone away: every send fails.
struct GoneSink;
impl MessageSink for GoneSink {
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
bail!("peer gone")
}
fn send_finalize(&mut self) -> Result<()> {
bail!("peer gone")
}
fn send_error(&mut self, _msg: &str) -> Result<()> {
bail!("peer gone")
}
}
#[test]
fn a_failed_send_returns_even_though_the_source_never_finishes() {
// The reader used to learn the peer was gone only on its next send.
// A growing capture that stops growing (and is never renamed) never
// gives it one, so run_send waited on the reader forever.
for compress in [false, true] {
let dir = tempdir().unwrap();
let partial = dir.path().join("cap.pcap.partial");
let mut stream = eth_header_bytes();
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
std::fs::write(&partial, &stream).unwrap();
let opts = SendOptions {
source_path: partial.clone(),
is_partial: true,
final_source_path: dir.path().join("cap.pcap"),
compress,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: PacketFilter::match_all(),
log: Arc::new(Logger::none()),
};
let (done_tx, done_rx) = mpsc::channel();
thread::spawn(move || {
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
});
let result = done_rx
.recv_timeout(Duration::from_secs(5))
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
assert!(result.unwrap_err().to_string().contains("peer gone"));
}
}
}
+3 -3
View File
@@ -1,5 +1,5 @@
use crate::cli::{ServerCli, ServerRole};
use crate::filter::IpFilter;
use crate::filter::PacketFilter;
use crate::naming;
use crate::pipeline::{run_recv, run_send, RecvOptions, SendOptions};
use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
@@ -39,8 +39,8 @@ fn run_send_server(cli: ServerCli) -> Result<()> {
let flush_after = naming::parse_duration(&cli.flush_interval)?;
// The sending side is the only side that filters -- dropping packets
// before they hit the wire is the whole point. `run_recv` never parses
// what it writes, so --recv has no --ip.
let filter = IpFilter::parse(&cli.ip)?;
// what it writes, so --recv has no --ip/--mac.
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
let opts = SendOptions {
source_path: PathBuf::from(source),
+96 -9
View File
@@ -2,8 +2,9 @@ use crate::chunker::{Chunker, ChunkKind};
use crate::tracelog::Logger;
use anyhow::{bail, Result};
use std::fs::File;
use std::os::unix::fs::FileExt;
use std::os::unix::fs::{FileExt, MetadataExt};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
///
/// Per the confirmed design: if `source_path` doesn't exist at all when this
/// is called, that's a hard error (no polling for creation).
///
/// `stop` is how the consumer says it has given up (its peer is gone). The
/// loop checks it every poll, because otherwise it would only find out on
/// its next send -- and a growing source that stops growing, or a filter
/// that matches nothing, may never produce one, leaving whoever joins this
/// thread waiting forever.
pub fn run_tail(
source_path: PathBuf,
is_partial: bool,
@@ -65,6 +72,7 @@ pub fn run_tail(
mut chunker: Box<dyn Chunker>,
events: impl EventSink,
log: Arc<Logger>,
stop: Arc<AtomicBool>,
) -> Result<()> {
if !source_path.exists() {
log.log(format_args!("tail: source not found: {}", source_path.display()));
@@ -75,7 +83,7 @@ pub fn run_tail(
source_path.display(),
final_path.display()
));
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log);
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
match &result {
Ok(()) => {
log.log(format_args!("tail: done, sending EOF"));
@@ -96,6 +104,7 @@ fn drain(
chunker: &mut dyn Chunker,
events: &impl EventSink,
log: &Logger,
stop: &AtomicBool,
) -> Result<()> {
let file = File::open(source_path)?;
let mut complete_signal = !is_partial;
@@ -123,6 +132,9 @@ fn drain(
let mut last_stuck_peek: Option<Vec<u8>> = None;
loop {
if stop.load(Ordering::Relaxed) {
bail!("stopped: the receiving side is gone");
}
let file_len = file.metadata()?.len();
// Whether this iteration did real work -- if so, loop again
// immediately rather than sleeping, so a large already-available
@@ -207,7 +219,7 @@ fn drain(
send_chunk(events, kind, data)?;
}
if !complete_signal && final_path.try_exists()? {
if !complete_signal && renamed_to(final_path, &file)? {
// Rename observed -- signal now, well before finalization, per
// section 7.4's repoint-then-drain-then-finalize ordering.
complete_signal = true;
@@ -269,6 +281,22 @@ fn drain(
}
}
/// Whether `final_path` now names the very file being read, i.e. the writer's
/// rename has happened. This is stricter than section 7.5's "the final path
/// exists": names get reused (section 6), so a file left at the final name by
/// an earlier capture or copy would otherwise pass for the rename the moment
/// tailing starts, and the transfer would finish early, short, at the first
/// lull in the writer. Comparing identity also still works when the writer
/// renames over such a leftover.
fn renamed_to(final_path: &Path, file: &File) -> Result<bool> {
let ours = file.metadata()?;
match std::fs::metadata(final_path) {
Ok(theirs) => Ok(theirs.dev() == ours.dev() && theirs.ino() == ours.ino()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(e.into()),
}
}
fn emit_ready(chunker: &mut dyn Chunker, events: &impl EventSink) -> Result<()> {
while let Some((kind, data)) = chunker.ready_chunk() {
send_chunk(events, kind, data)?;
@@ -316,6 +344,10 @@ mod tests {
(data, got_eof)
}
fn no_stop() -> Arc<AtomicBool> {
Arc::new(AtomicBool::new(false))
}
fn header_bytes(magic_le: u32) -> Vec<u8> {
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
@@ -340,7 +372,7 @@ mod tests {
std::fs::write(&path, b"hello world").unwrap();
let (tx, rx) = mpsc::channel();
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap();
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
let (data, eof) = collect(rx);
assert_eq!(data, b"hello world");
assert!(eof);
@@ -352,7 +384,7 @@ mod tests {
let path = dir.path().join("nope.pcap.partial");
let final_path = dir.path().join("nope.pcap");
let (tx, _rx) = mpsc::channel();
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err();
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
assert!(err.to_string().contains("not found"));
}
@@ -369,7 +401,7 @@ mod tests {
let handle = std::thread::spawn({
let partial_path = partial_path.clone();
let final_path = final_path.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
});
std::thread::sleep(Duration::from_millis(60));
@@ -399,7 +431,7 @@ mod tests {
let partial_path = partial_path.clone();
let final_path = final_path.clone();
let log = log.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log)
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
});
std::thread::sleep(Duration::from_millis(60));
@@ -426,7 +458,7 @@ mod tests {
let handle = std::thread::spawn({
let partial_path = partial_path.clone();
let final_path = final_path.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
});
std::thread::sleep(Duration::from_millis(60));
@@ -457,7 +489,7 @@ mod tests {
let handle = std::thread::spawn({
let partial_path = partial_path.clone();
let final_path = final_path.clone();
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
});
// Let the reader fully drain the initial 10 bytes.
@@ -522,6 +554,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx,
Arc::new(Logger::none()),
no_stop(),
)
}
});
@@ -569,6 +602,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx,
Arc::new(Logger::none()),
no_stop(),
)
}
});
@@ -624,6 +658,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx,
Arc::new(Logger::none()),
no_stop(),
)
}
});
@@ -668,6 +703,7 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx,
Arc::new(Logger::none()),
no_stop(),
)
}
});
@@ -706,8 +742,59 @@ mod tests {
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
tx,
Arc::new(Logger::none()),
no_stop(),
);
try_collect(rx);
assert!(result.is_err());
}
#[test]
fn a_stale_file_at_the_final_name_is_not_the_rename() {
// A leftover cap.pcap (an earlier capture reusing the name) used to
// pass for the rename the moment tailing started, so the first lull
// longer than the grace period finalized the copy short.
let dir = tempdir().unwrap();
let partial_path = dir.path().join("cap.pcap.partial");
let final_path = dir.path().join("cap.pcap");
std::fs::write(&final_path, b"yesterday's capture").unwrap();
let mut stream = header_bytes(0xA1B2C3D4);
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
stream.extend_from_slice(&record_bytes(&[2u8; 10], 1_700_000_001));
let mut f = File::create(&partial_path).unwrap();
f.write_all(&stream).unwrap();
f.sync_all().unwrap();
let (tx, rx) = mpsc::channel();
let handle = std::thread::spawn({
let partial_path = partial_path.clone();
let final_path = final_path.clone();
move || {
run_tail(
partial_path,
true,
final_path,
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
tx,
Arc::new(Logger::none()),
no_stop(),
)
}
});
// A writer lull longer than the grace period, then more packets and
// the real rename -- over the stale file.
std::thread::sleep(FINALIZE_GRACE_PERIOD + Duration::from_millis(500));
let rec3 = record_bytes(&[3u8; 10], 1_700_000_002);
f.write_all(&rec3).unwrap();
f.sync_all().unwrap();
stream.extend_from_slice(&rec3);
drop(f);
std::fs::rename(&partial_path, &final_path).unwrap();
handle.join().unwrap().unwrap();
let (data, eof) = collect(rx);
assert_eq!(data, stream);
assert!(eof);
}
}