Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cc698b7826 | ||
|
|
3eb779a6aa | ||
|
|
b46d6a936c | ||
|
|
6a80aaf461 | ||
|
|
f21b6c53e8 |
No files matched your search
+19
-14
@@ -1,12 +1,13 @@
|
||||
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
||||
use anyhow::{Result, bail};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub const GLOBAL_HEADER_LEN: usize = 24;
|
||||
pub const RECORD_HEADER_LEN: usize = 16;
|
||||
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
||||
#[allow(dead_code)]
|
||||
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
|
||||
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
|
||||
/// Sanity bound on a record's declared payload length, per
|
||||
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
||||
/// sources -- see `RecordAlignedChunker::partial`.
|
||||
@@ -178,7 +179,7 @@ pub struct RecordAlignedChunker {
|
||||
/// bytes is what keeps `flush_after` an actual latency bound rather
|
||||
/// than an idle-only one.
|
||||
kept_since: Option<Instant>,
|
||||
target: usize,
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
/// Whether this source is a still-growing (partial) file. Gates every
|
||||
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
||||
@@ -195,12 +196,12 @@ impl RecordAlignedChunker {
|
||||
/// same record-confirmation logic. Production goes through
|
||||
/// `with_filter`; this is the convenience the tests use.
|
||||
#[allow(dead_code)]
|
||||
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
|
||||
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
|
||||
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
||||
}
|
||||
|
||||
pub fn with_filter(
|
||||
target: usize,
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
partial: bool,
|
||||
filter: PacketFilter,
|
||||
@@ -400,7 +401,7 @@ impl Chunker for RecordAlignedChunker {
|
||||
// must produce no chunk at all, rather than an empty one. An empty
|
||||
// chunk would become a zero-content zstd frame, which
|
||||
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
||||
if self.kept.len() >= self.target {
|
||||
if self.kept.len() >= self.target.get() {
|
||||
return Some(self.take_kept());
|
||||
}
|
||||
None
|
||||
@@ -509,6 +510,10 @@ mod tests {
|
||||
h
|
||||
}
|
||||
|
||||
fn nz(n: usize) -> NonZeroUsize {
|
||||
NonZeroUsize::new(n).unwrap()
|
||||
}
|
||||
|
||||
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
||||
record_bytes_with_ts(payload, big_endian, 0)
|
||||
}
|
||||
@@ -571,7 +576,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
||||
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
||||
for i in 0..20u8 {
|
||||
@@ -606,7 +611,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
||||
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
|
||||
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
|
||||
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
||||
c.ready_chunk(); // drain header
|
||||
|
||||
@@ -913,7 +918,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn keeps_only_matching_records_and_keeps_them_whole() {
|
||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let hit_src = ip_record(WANTED, OTHER, 60);
|
||||
let hit_dst = ip_record(OTHER, WANTED, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
@@ -946,7 +951,7 @@ mod tests {
|
||||
let hit = ip_record(WANTED, OTHER, 100);
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let target = 10 * hit.len();
|
||||
let mut c = RecordAlignedChunker::with_filter(target, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
@@ -1001,7 +1006,7 @@ mod tests {
|
||||
// An empty chunk would be compressed into a zero-content zstd frame,
|
||||
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
for _ in 0..20 {
|
||||
@@ -1026,9 +1031,9 @@ mod tests {
|
||||
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
||||
}
|
||||
|
||||
let mut all = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||
let mut some = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut none = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||
for c in [&mut all, &mut some, &mut none] {
|
||||
c.feed(&stream).unwrap();
|
||||
}
|
||||
@@ -1052,7 +1057,7 @@ mod tests {
|
||||
// exactly once even though the bytes around it are re-read.
|
||||
let hit = ip_record(WANTED, OTHER, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
let mut c = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&miss);
|
||||
|
||||
+113
-1
@@ -7,7 +7,8 @@ use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
|
||||
use crate::ssh::{build_server_command, spawn_remote_server};
|
||||
use crate::tracelog::Logger;
|
||||
use anyhow::{bail, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::thread;
|
||||
|
||||
@@ -63,6 +64,10 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
|
||||
match (&source_target, &dest_target) {
|
||||
(Target::Local(src), Target::Local(_)) => {
|
||||
refuse_overlap(
|
||||
[Path::new(src), Path::new(&final_source_path)],
|
||||
[Path::new(&dest.temp_path), Path::new(&dest.final_path)],
|
||||
)?;
|
||||
let send_opts = SendOptions {
|
||||
source_path: PathBuf::from(src),
|
||||
is_partial,
|
||||
@@ -166,6 +171,13 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
};
|
||||
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
||||
let recv_result = run_recv(source, recv_opts);
|
||||
if recv_result.is_err() {
|
||||
// The remote sender only notices we're gone when its next
|
||||
// write fails, which on an idle capture may be never. Don't
|
||||
// wait for that: drop the connection, so its next write
|
||||
// fails and it stops.
|
||||
let _ = child.kill();
|
||||
}
|
||||
let status = child.wait()?;
|
||||
recv_result?;
|
||||
if !status.success() {
|
||||
@@ -178,3 +190,103 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses a local copy whose destination lands on the source. The receiver
|
||||
/// truncates its in-flight file on open and renames over its final one, so
|
||||
/// either collision destroys the capture being read. Copying a capture into
|
||||
/// its own directory is enough to hit it: `dir/cap.pcap.partial` to `dir/`
|
||||
/// resolves to `dir/cap.pcap`, written via `dir/cap.pcap.partial`.
|
||||
///
|
||||
/// `sources` is the source as given plus its post-rename final path, since a
|
||||
/// growing source moves from one to the other mid-transfer.
|
||||
fn refuse_overlap(sources: [&Path; 2], dests: [&Path; 2]) -> Result<()> {
|
||||
for source in sources {
|
||||
for dest in dests {
|
||||
if same_location(source, dest) {
|
||||
bail!(
|
||||
"destination {} is the source {}: copying would overwrite the capture \
|
||||
being read. Copy it to a different directory or name.",
|
||||
dest.display(),
|
||||
source.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether two paths name the same file, whether or not it exists yet: the
|
||||
/// same inode when both exist (catching hard links and symlinks), otherwise
|
||||
/// the same canonical parent directory and file name.
|
||||
fn same_location(a: &Path, b: &Path) -> bool {
|
||||
if let (Ok(ma), Ok(mb)) = (std::fs::metadata(a), std::fs::metadata(b)) {
|
||||
return ma.dev() == mb.dev() && ma.ino() == mb.ino();
|
||||
}
|
||||
match (resolve(a), resolve(b)) {
|
||||
(Some(ra), Some(rb)) => ra == rb,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The canonical form of `path`, or for a file that doesn't exist yet, its
|
||||
/// canonical parent joined with its name. `None` when even the parent
|
||||
/// doesn't exist, in which case nothing can collide with it.
|
||||
fn resolve(path: &Path) -> Option<PathBuf> {
|
||||
if let Ok(canonical) = std::fs::canonicalize(path) {
|
||||
return Some(canonical);
|
||||
}
|
||||
let parent = match path.parent() {
|
||||
Some(p) if !p.as_os_str().is_empty() => p,
|
||||
_ => Path::new("."),
|
||||
};
|
||||
Some(std::fs::canonicalize(parent).ok()?.join(path.file_name()?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn copying_a_growing_capture_into_its_own_directory_is_refused() {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
std::fs::write(&partial, b"live").unwrap();
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
// What `resolve_dest_paths` makes of DEST = the source's directory.
|
||||
let err = refuse_overlap([&partial, &final_src], [&partial, &final_src]).unwrap_err();
|
||||
assert!(err.to_string().contains("overwrite the capture"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_symlinked_or_relative_spelling_of_the_source_is_still_caught() {
|
||||
let dir = tempdir().unwrap();
|
||||
let real = dir.path().join("real");
|
||||
std::fs::create_dir(&real).unwrap();
|
||||
let src = real.join("cap.pcap");
|
||||
std::fs::write(&src, b"done").unwrap();
|
||||
let link = dir.path().join("link");
|
||||
std::os::unix::fs::symlink(&real, &link).unwrap();
|
||||
let via_link = link.join("cap.pcap");
|
||||
let via_dotdot = real.join("..").join("real").join("cap.pcap.partial");
|
||||
assert!(refuse_overlap([&src, &src], [&via_dotdot, &via_link]).is_err());
|
||||
// Neither exists yet: compared by canonical parent + name.
|
||||
let not_yet = real.join("other.pcap");
|
||||
assert!(refuse_overlap([¬_yet, ¬_yet], [&link.join("other.pcap"), &via_dotdot]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_different_destination_is_allowed() {
|
||||
let dir = tempdir().unwrap();
|
||||
let src = dir.path().join("cap.pcap.partial");
|
||||
std::fs::write(&src, b"live").unwrap();
|
||||
let out = dir.path().join("out");
|
||||
std::fs::create_dir(&out).unwrap();
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
let dests = [out.join("cap.pcap.partial"), out.join("cap.pcap")];
|
||||
assert!(refuse_overlap([&src, &final_src], [&dests[0], &dests[1]]).is_ok());
|
||||
// A destination under a directory that doesn't exist can't collide.
|
||||
let missing = dir.path().join("nope").join("cap.pcap");
|
||||
assert!(refuse_overlap([&src, &final_src], [&missing, &missing]).is_ok());
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -50,7 +50,8 @@ pub struct PacketFilter {
|
||||
}
|
||||
|
||||
impl PacketFilter {
|
||||
/// The no-`--ip` default: keeps everything, matches without inspecting.
|
||||
/// The no-`--ip`/`--mac` default: keeps everything, matches without
|
||||
/// inspecting.
|
||||
pub fn match_all() -> Self {
|
||||
Self { match_all: true, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() }
|
||||
}
|
||||
|
||||
+17
-10
@@ -1,4 +1,5 @@
|
||||
use anyhow::{bail, Result};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::Duration;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -111,7 +112,9 @@ pub fn resolve_dest_paths(
|
||||
}
|
||||
|
||||
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
||||
pub fn parse_size(s: &str) -> Result<usize> {
|
||||
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
|
||||
/// a zero target would cut empty chunks without end.
|
||||
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
bail!("empty size");
|
||||
@@ -126,7 +129,8 @@ pub fn parse_size(s: &str) -> Result<usize> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
||||
Ok(n * mult)
|
||||
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
|
||||
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
|
||||
}
|
||||
|
||||
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
||||
@@ -148,7 +152,8 @@ pub fn parse_duration(s: &str) -> Result<Duration> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
||||
Ok(Duration::from_millis(n * mult_ms))
|
||||
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
|
||||
Ok(Duration::from_millis(ms))
|
||||
}
|
||||
|
||||
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
||||
@@ -282,19 +287,21 @@ mod tests {
|
||||
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
||||
// 0 is legal: cut a chunk as soon as anything is kept.
|
||||
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
||||
for bad in ["", "abc", "0.1s", "100m", "-5"] {
|
||||
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
|
||||
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_parsing() {
|
||||
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap(), 512);
|
||||
assert!(parse_size("").is_err());
|
||||
assert!(parse_size("abc").is_err());
|
||||
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap().get(), 512);
|
||||
// 0 would cut empty chunks forever; overflow must not wrap.
|
||||
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
|
||||
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+152
-4
@@ -7,7 +7,9 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
@@ -23,7 +25,7 @@ pub struct SendOptions {
|
||||
pub is_partial: bool,
|
||||
pub final_source_path: PathBuf,
|
||||
pub compress: bool,
|
||||
pub chunk_target: usize,
|
||||
pub chunk_target: NonZeroUsize,
|
||||
/// How long kept-but-undersized output may wait before being sent
|
||||
/// anyway. See `Chunker::should_flush`.
|
||||
pub flush_after: Duration,
|
||||
@@ -79,12 +81,19 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
};
|
||||
|
||||
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
||||
// Raised once the send loop below is done, for whatever reason. After a
|
||||
// clean EOF or a source error the reader has already returned and never
|
||||
// looks; after a failed send it is how the reader learns to quit instead
|
||||
// of polling the source until it next has something to send, which on an
|
||||
// idle capture or a filter matching nothing may be never.
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let reader = thread::spawn({
|
||||
let source_path = opts.source_path.clone();
|
||||
let is_partial = opts.is_partial;
|
||||
let final_source_path = opts.final_source_path.clone();
|
||||
let log = opts.log.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
||||
let stop = stop.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
||||
});
|
||||
|
||||
if opts.compress {
|
||||
@@ -139,6 +148,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
let compressor_result = compressor.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
@@ -174,6 +184,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let reader_result = reader.join().unwrap();
|
||||
if let Some(e) = send_err {
|
||||
return Err(e);
|
||||
@@ -357,7 +368,7 @@ mod tests {
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512, // small target to force multiple frames
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
@@ -485,7 +496,7 @@ mod tests {
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["1.1.1.1"]),
|
||||
@@ -530,4 +541,141 @@ mod tests {
|
||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
||||
}
|
||||
|
||||
/// A record holding an Ethernet frame from `src` to `dst` with
|
||||
/// `ethertype`, padded to `len` captured bytes.
|
||||
fn mac_record(dst: [u8; 6], src: [u8; 6], ethertype: u16, len: usize, ts_sec: u32) -> Vec<u8> {
|
||||
let mut pkt = vec![0u8; len.max(14)];
|
||||
pkt[0..6].copy_from_slice(&dst);
|
||||
pkt[6..12].copy_from_slice(&src);
|
||||
pkt[12..14].copy_from_slice(ðertype.to_be_bytes());
|
||||
record_bytes(&pkt, ts_sec)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mac_filtered_growing_capture_carries_only_the_upstream_link() {
|
||||
// The scenario --mac exists for: a capture still being written, from
|
||||
// which only the traffic on one switch connection is wanted. The
|
||||
// source grows in uneven appends that tear records mid-header and
|
||||
// mid-payload, so the filter must only ever see confirmed records.
|
||||
const UPSTREAM: [u8; 6] = [0x02, 0, 0, 0, 0, 0xaa];
|
||||
const OTHER: [u8; 6] = [0x02, 0, 0, 0, 0, 0xbb];
|
||||
const BROADCAST: [u8; 6] = [0xff; 6];
|
||||
|
||||
let mut source = eth_header_bytes();
|
||||
let mut expected = eth_header_bytes();
|
||||
for i in 0..60u32 {
|
||||
let ts = 1_700_000_000 + i;
|
||||
let len = 64 + i as usize;
|
||||
let (record, kept) = match i % 4 {
|
||||
0 => (mac_record(OTHER, UPSTREAM, 0x0800, len, ts), true), // from the switch
|
||||
1 => (mac_record(UPSTREAM, OTHER, 0x86DD, len, ts), true), // to the switch
|
||||
2 => (mac_record(BROADCAST, UPSTREAM, 0x0806, len, ts), true), // ARP from the switch
|
||||
_ => (mac_record(BROADCAST, OTHER, 0x0806, len, ts), false), // someone else's ARP
|
||||
};
|
||||
source.extend_from_slice(&record);
|
||||
if kept {
|
||||
expected.extend_from_slice(&record);
|
||||
}
|
||||
}
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
std::fs::write(&partial, &source[..10]).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap.zst");
|
||||
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: final_src.clone(),
|
||||
compress: true,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel::<Message>();
|
||||
let recv_handle = thread::spawn({
|
||||
let dest_final = dest_final.clone();
|
||||
let dest_temp = dest_temp.clone();
|
||||
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
||||
});
|
||||
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
||||
|
||||
// Odd-sized appends (an awkward stride relative to the ~80-140 byte
|
||||
// records) so the visible end lands inside headers and payloads.
|
||||
let mut written = 10;
|
||||
for step in [37usize, 501, 13, 977, 250].iter().cycle() {
|
||||
if written == source.len() {
|
||||
break;
|
||||
}
|
||||
let end = (written + step).min(source.len());
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
||||
f.write_all(&source[written..end]).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
written = end;
|
||||
thread::sleep(std::time::Duration::from_millis(5));
|
||||
}
|
||||
std::fs::rename(&partial, &final_src).unwrap();
|
||||
|
||||
send_handle.join().unwrap().unwrap();
|
||||
recv_handle.join().unwrap().unwrap();
|
||||
|
||||
let out_bytes = std::fs::read(&dest_final).unwrap();
|
||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
||||
assert!(expected.len() < source.len());
|
||||
}
|
||||
|
||||
/// A peer that has gone away: every send fails.
|
||||
struct GoneSink;
|
||||
|
||||
impl MessageSink for GoneSink {
|
||||
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_finalize(&mut self) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
||||
bail!("peer gone")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
||||
// The reader used to learn the peer was gone only on its next send.
|
||||
// A growing capture that stops growing (and is never renamed) never
|
||||
// gives it one, so run_send waited on the reader forever.
|
||||
for compress in [false, true] {
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
||||
std::fs::write(&partial, &stream).unwrap();
|
||||
|
||||
let opts = SendOptions {
|
||||
source_path: partial.clone(),
|
||||
is_partial: true,
|
||||
final_source_path: dir.path().join("cap.pcap"),
|
||||
compress,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
||||
});
|
||||
let result = done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
||||
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -39,7 +39,7 @@ fn run_send_server(cli: ServerCli) -> Result<()> {
|
||||
let flush_after = naming::parse_duration(&cli.flush_interval)?;
|
||||
// The sending side is the only side that filters -- dropping packets
|
||||
// before they hit the wire is the whole point. `run_recv` never parses
|
||||
// what it writes, so --recv has no --ip.
|
||||
// what it writes, so --recv has no --ip/--mac.
|
||||
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
|
||||
|
||||
let opts = SendOptions {
|
||||
|
||||
+96
-9
@@ -2,8 +2,9 @@ use crate::chunker::{Chunker, ChunkKind};
|
||||
use crate::tracelog::Logger;
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::os::unix::fs::{FileExt, MetadataExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -58,6 +59,12 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
|
||||
///
|
||||
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
||||
/// is called, that's a hard error (no polling for creation).
|
||||
///
|
||||
/// `stop` is how the consumer says it has given up (its peer is gone). The
|
||||
/// loop checks it every poll, because otherwise it would only find out on
|
||||
/// its next send -- and a growing source that stops growing, or a filter
|
||||
/// that matches nothing, may never produce one, leaving whoever joins this
|
||||
/// thread waiting forever.
|
||||
pub fn run_tail(
|
||||
source_path: PathBuf,
|
||||
is_partial: bool,
|
||||
@@ -65,6 +72,7 @@ pub fn run_tail(
|
||||
mut chunker: Box<dyn Chunker>,
|
||||
events: impl EventSink,
|
||||
log: Arc<Logger>,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> Result<()> {
|
||||
if !source_path.exists() {
|
||||
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
||||
@@ -75,7 +83,7 @@ pub fn run_tail(
|
||||
source_path.display(),
|
||||
final_path.display()
|
||||
));
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log);
|
||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
log.log(format_args!("tail: done, sending EOF"));
|
||||
@@ -96,6 +104,7 @@ fn drain(
|
||||
chunker: &mut dyn Chunker,
|
||||
events: &impl EventSink,
|
||||
log: &Logger,
|
||||
stop: &AtomicBool,
|
||||
) -> Result<()> {
|
||||
let file = File::open(source_path)?;
|
||||
let mut complete_signal = !is_partial;
|
||||
@@ -123,6 +132,9 @@ fn drain(
|
||||
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
||||
|
||||
loop {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
bail!("stopped: the receiving side is gone");
|
||||
}
|
||||
let file_len = file.metadata()?.len();
|
||||
// Whether this iteration did real work -- if so, loop again
|
||||
// immediately rather than sleeping, so a large already-available
|
||||
@@ -207,7 +219,7 @@ fn drain(
|
||||
send_chunk(events, kind, data)?;
|
||||
}
|
||||
|
||||
if !complete_signal && final_path.try_exists()? {
|
||||
if !complete_signal && renamed_to(final_path, &file)? {
|
||||
// Rename observed -- signal now, well before finalization, per
|
||||
// section 7.4's repoint-then-drain-then-finalize ordering.
|
||||
complete_signal = true;
|
||||
@@ -269,6 +281,22 @@ fn drain(
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `final_path` now names the very file being read, i.e. the writer's
|
||||
/// rename has happened. This is stricter than section 7.5's "the final path
|
||||
/// exists": names get reused (section 6), so a file left at the final name by
|
||||
/// an earlier capture or copy would otherwise pass for the rename the moment
|
||||
/// tailing starts, and the transfer would finish early, short, at the first
|
||||
/// lull in the writer. Comparing identity also still works when the writer
|
||||
/// renames over such a leftover.
|
||||
fn renamed_to(final_path: &Path, file: &File) -> Result<bool> {
|
||||
let ours = file.metadata()?;
|
||||
match std::fs::metadata(final_path) {
|
||||
Ok(theirs) => Ok(theirs.dev() == ours.dev() && theirs.ino() == ours.ino()),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_ready(chunker: &mut dyn Chunker, events: &impl EventSink) -> Result<()> {
|
||||
while let Some((kind, data)) = chunker.ready_chunk() {
|
||||
send_chunk(events, kind, data)?;
|
||||
@@ -316,6 +344,10 @@ mod tests {
|
||||
(data, got_eof)
|
||||
}
|
||||
|
||||
fn no_stop() -> Arc<AtomicBool> {
|
||||
Arc::new(AtomicBool::new(false))
|
||||
}
|
||||
|
||||
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
||||
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
||||
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
||||
@@ -340,7 +372,7 @@ mod tests {
|
||||
std::fs::write(&path, b"hello world").unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap();
|
||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
|
||||
let (data, eof) = collect(rx);
|
||||
assert_eq!(data, b"hello world");
|
||||
assert!(eof);
|
||||
@@ -352,7 +384,7 @@ mod tests {
|
||||
let path = dir.path().join("nope.pcap.partial");
|
||||
let final_path = dir.path().join("nope.pcap");
|
||||
let (tx, _rx) = mpsc::channel();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err();
|
||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
|
||||
assert!(err.to_string().contains("not found"));
|
||||
}
|
||||
|
||||
@@ -369,7 +401,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -399,7 +431,7 @@ mod tests {
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
let log = log.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log)
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -426,7 +458,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
@@ -457,7 +489,7 @@ mod tests {
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
||||
});
|
||||
|
||||
// Let the reader fully drain the initial 10 bytes.
|
||||
@@ -522,6 +554,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -569,6 +602,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -624,6 +658,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -668,6 +703,7 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
@@ -706,8 +742,59 @@ mod tests {
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
);
|
||||
try_collect(rx);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_file_at_the_final_name_is_not_the_rename() {
|
||||
// A leftover cap.pcap (an earlier capture reusing the name) used to
|
||||
// pass for the rename the moment tailing started, so the first lull
|
||||
// longer than the grace period finalized the copy short.
|
||||
let dir = tempdir().unwrap();
|
||||
let partial_path = dir.path().join("cap.pcap.partial");
|
||||
let final_path = dir.path().join("cap.pcap");
|
||||
std::fs::write(&final_path, b"yesterday's capture").unwrap();
|
||||
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
|
||||
stream.extend_from_slice(&record_bytes(&[2u8; 10], 1_700_000_001));
|
||||
let mut f = File::create(&partial_path).unwrap();
|
||||
f.write_all(&stream).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let handle = std::thread::spawn({
|
||||
let partial_path = partial_path.clone();
|
||||
let final_path = final_path.clone();
|
||||
move || {
|
||||
run_tail(
|
||||
partial_path,
|
||||
true,
|
||||
final_path,
|
||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||
tx,
|
||||
Arc::new(Logger::none()),
|
||||
no_stop(),
|
||||
)
|
||||
}
|
||||
});
|
||||
|
||||
// A writer lull longer than the grace period, then more packets and
|
||||
// the real rename -- over the stale file.
|
||||
std::thread::sleep(FINALIZE_GRACE_PERIOD + Duration::from_millis(500));
|
||||
let rec3 = record_bytes(&[3u8; 10], 1_700_000_002);
|
||||
f.write_all(&rec3).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
stream.extend_from_slice(&rec3);
|
||||
drop(f);
|
||||
std::fs::rename(&partial_path, &final_path).unwrap();
|
||||
|
||||
handle.join().unwrap().unwrap();
|
||||
let (data, eof) = collect(rx);
|
||||
assert_eq!(data, stream);
|
||||
assert!(eof);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user