Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6b68deb047 | ||
|
|
a7417dcdd6 |
No files matched your search
+14
-19
@@ -1,13 +1,12 @@
|
|||||||
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
||||||
use anyhow::{Result, bail};
|
use anyhow::{Result, bail};
|
||||||
use std::num::NonZeroUsize;
|
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
pub const GLOBAL_HEADER_LEN: usize = 24;
|
pub const GLOBAL_HEADER_LEN: usize = 24;
|
||||||
pub const RECORD_HEADER_LEN: usize = 16;
|
pub const RECORD_HEADER_LEN: usize = 16;
|
||||||
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
|
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
|
||||||
/// Sanity bound on a record's declared payload length, per
|
/// Sanity bound on a record's declared payload length, per
|
||||||
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
||||||
/// sources -- see `RecordAlignedChunker::partial`.
|
/// sources -- see `RecordAlignedChunker::partial`.
|
||||||
@@ -179,7 +178,7 @@ pub struct RecordAlignedChunker {
|
|||||||
/// bytes is what keeps `flush_after` an actual latency bound rather
|
/// bytes is what keeps `flush_after` an actual latency bound rather
|
||||||
/// than an idle-only one.
|
/// than an idle-only one.
|
||||||
kept_since: Option<Instant>,
|
kept_since: Option<Instant>,
|
||||||
target: NonZeroUsize,
|
target: usize,
|
||||||
flush_after: Duration,
|
flush_after: Duration,
|
||||||
/// Whether this source is a still-growing (partial) file. Gates every
|
/// Whether this source is a still-growing (partial) file. Gates every
|
||||||
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
||||||
@@ -196,12 +195,12 @@ impl RecordAlignedChunker {
|
|||||||
/// same record-confirmation logic. Production goes through
|
/// same record-confirmation logic. Production goes through
|
||||||
/// `with_filter`; this is the convenience the tests use.
|
/// `with_filter`; this is the convenience the tests use.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
|
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
|
||||||
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn with_filter(
|
pub fn with_filter(
|
||||||
target: NonZeroUsize,
|
target: usize,
|
||||||
flush_after: Duration,
|
flush_after: Duration,
|
||||||
partial: bool,
|
partial: bool,
|
||||||
filter: PacketFilter,
|
filter: PacketFilter,
|
||||||
@@ -401,7 +400,7 @@ impl Chunker for RecordAlignedChunker {
|
|||||||
// must produce no chunk at all, rather than an empty one. An empty
|
// must produce no chunk at all, rather than an empty one. An empty
|
||||||
// chunk would become a zero-content zstd frame, which
|
// chunk would become a zero-content zstd frame, which
|
||||||
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
||||||
if self.kept.len() >= self.target.get() {
|
if self.kept.len() >= self.target {
|
||||||
return Some(self.take_kept());
|
return Some(self.take_kept());
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
@@ -510,10 +509,6 @@ mod tests {
|
|||||||
h
|
h
|
||||||
}
|
}
|
||||||
|
|
||||||
fn nz(n: usize) -> NonZeroUsize {
|
|
||||||
NonZeroUsize::new(n).unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
||||||
record_bytes_with_ts(payload, big_endian, 0)
|
record_bytes_with_ts(payload, big_endian, 0)
|
||||||
}
|
}
|
||||||
@@ -576,7 +571,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
||||||
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
|
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||||
let mut stream = header_bytes(0xA1B2C3D4);
|
let mut stream = header_bytes(0xA1B2C3D4);
|
||||||
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
||||||
for i in 0..20u8 {
|
for i in 0..20u8 {
|
||||||
@@ -611,7 +606,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
||||||
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
|
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
|
||||||
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
||||||
c.ready_chunk(); // drain header
|
c.ready_chunk(); // drain header
|
||||||
|
|
||||||
@@ -918,7 +913,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn keeps_only_matching_records_and_keeps_them_whole() {
|
fn keeps_only_matching_records_and_keeps_them_whole() {
|
||||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
let hit_src = ip_record(WANTED, OTHER, 60);
|
let hit_src = ip_record(WANTED, OTHER, 60);
|
||||||
let hit_dst = ip_record(OTHER, WANTED, 60);
|
let hit_dst = ip_record(OTHER, WANTED, 60);
|
||||||
let miss = ip_record(OTHER, THIRD, 60);
|
let miss = ip_record(OTHER, THIRD, 60);
|
||||||
@@ -951,7 +946,7 @@ mod tests {
|
|||||||
let hit = ip_record(WANTED, OTHER, 100);
|
let hit = ip_record(WANTED, OTHER, 100);
|
||||||
let miss = ip_record(OTHER, THIRD, 100);
|
let miss = ip_record(OTHER, THIRD, 100);
|
||||||
let target = 10 * hit.len();
|
let target = 10 * hit.len();
|
||||||
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(target, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
|
|
||||||
c.feed(ð_header_bytes()).unwrap();
|
c.feed(ð_header_bytes()).unwrap();
|
||||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||||
@@ -1006,7 +1001,7 @@ mod tests {
|
|||||||
// An empty chunk would be compressed into a zero-content zstd frame,
|
// An empty chunk would be compressed into a zero-content zstd frame,
|
||||||
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
||||||
let miss = ip_record(OTHER, THIRD, 100);
|
let miss = ip_record(OTHER, THIRD, 100);
|
||||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
c.feed(ð_header_bytes()).unwrap();
|
c.feed(ð_header_bytes()).unwrap();
|
||||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||||
for _ in 0..20 {
|
for _ in 0..20 {
|
||||||
@@ -1031,9 +1026,9 @@ mod tests {
|
|||||||
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
|
let mut all = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||||
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut some = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
let mut none = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||||
for c in [&mut all, &mut some, &mut none] {
|
for c in [&mut all, &mut some, &mut none] {
|
||||||
c.feed(&stream).unwrap();
|
c.feed(&stream).unwrap();
|
||||||
}
|
}
|
||||||
@@ -1057,7 +1052,7 @@ mod tests {
|
|||||||
// exactly once even though the bytes around it are re-read.
|
// exactly once even though the bytes around it are re-read.
|
||||||
let hit = ip_record(WANTED, OTHER, 60);
|
let hit = ip_record(WANTED, OTHER, 60);
|
||||||
let miss = ip_record(OTHER, THIRD, 60);
|
let miss = ip_record(OTHER, THIRD, 60);
|
||||||
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||||
|
|
||||||
let mut stream = eth_header_bytes();
|
let mut stream = eth_header_bytes();
|
||||||
stream.extend_from_slice(&miss);
|
stream.extend_from_slice(&miss);
|
||||||
|
|||||||
+1
-113
@@ -7,8 +7,7 @@ use crate::protocol::{MessageSink, MessageSource, WireSink, WireSource};
|
|||||||
use crate::ssh::{build_server_command, spawn_remote_server};
|
use crate::ssh::{build_server_command, spawn_remote_server};
|
||||||
use crate::tracelog::Logger;
|
use crate::tracelog::Logger;
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{bail, Result};
|
||||||
use std::os::unix::fs::MetadataExt;
|
use std::path::PathBuf;
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::sync::{mpsc, Arc};
|
use std::sync::{mpsc, Arc};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
|
|
||||||
@@ -64,10 +63,6 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
|||||||
|
|
||||||
match (&source_target, &dest_target) {
|
match (&source_target, &dest_target) {
|
||||||
(Target::Local(src), Target::Local(_)) => {
|
(Target::Local(src), Target::Local(_)) => {
|
||||||
refuse_overlap(
|
|
||||||
[Path::new(src), Path::new(&final_source_path)],
|
|
||||||
[Path::new(&dest.temp_path), Path::new(&dest.final_path)],
|
|
||||||
)?;
|
|
||||||
let send_opts = SendOptions {
|
let send_opts = SendOptions {
|
||||||
source_path: PathBuf::from(src),
|
source_path: PathBuf::from(src),
|
||||||
is_partial,
|
is_partial,
|
||||||
@@ -171,13 +166,6 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
|||||||
};
|
};
|
||||||
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
let source: Box<dyn MessageSource> = Box::new(WireSource(stdout));
|
||||||
let recv_result = run_recv(source, recv_opts);
|
let recv_result = run_recv(source, recv_opts);
|
||||||
if recv_result.is_err() {
|
|
||||||
// The remote sender only notices we're gone when its next
|
|
||||||
// write fails, which on an idle capture may be never. Don't
|
|
||||||
// wait for that: drop the connection, so its next write
|
|
||||||
// fails and it stops.
|
|
||||||
let _ = child.kill();
|
|
||||||
}
|
|
||||||
let status = child.wait()?;
|
let status = child.wait()?;
|
||||||
recv_result?;
|
recv_result?;
|
||||||
if !status.success() {
|
if !status.success() {
|
||||||
@@ -190,103 +178,3 @@ pub fn run(cli: ClientCli) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Refuses a local copy whose destination lands on the source. The receiver
|
|
||||||
/// truncates its in-flight file on open and renames over its final one, so
|
|
||||||
/// either collision destroys the capture being read. Copying a capture into
|
|
||||||
/// its own directory is enough to hit it: `dir/cap.pcap.partial` to `dir/`
|
|
||||||
/// resolves to `dir/cap.pcap`, written via `dir/cap.pcap.partial`.
|
|
||||||
///
|
|
||||||
/// `sources` is the source as given plus its post-rename final path, since a
|
|
||||||
/// growing source moves from one to the other mid-transfer.
|
|
||||||
fn refuse_overlap(sources: [&Path; 2], dests: [&Path; 2]) -> Result<()> {
|
|
||||||
for source in sources {
|
|
||||||
for dest in dests {
|
|
||||||
if same_location(source, dest) {
|
|
||||||
bail!(
|
|
||||||
"destination {} is the source {}: copying would overwrite the capture \
|
|
||||||
being read. Copy it to a different directory or name.",
|
|
||||||
dest.display(),
|
|
||||||
source.display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether two paths name the same file, whether or not it exists yet: the
|
|
||||||
/// same inode when both exist (catching hard links and symlinks), otherwise
|
|
||||||
/// the same canonical parent directory and file name.
|
|
||||||
fn same_location(a: &Path, b: &Path) -> bool {
|
|
||||||
if let (Ok(ma), Ok(mb)) = (std::fs::metadata(a), std::fs::metadata(b)) {
|
|
||||||
return ma.dev() == mb.dev() && ma.ino() == mb.ino();
|
|
||||||
}
|
|
||||||
match (resolve(a), resolve(b)) {
|
|
||||||
(Some(ra), Some(rb)) => ra == rb,
|
|
||||||
_ => false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The canonical form of `path`, or for a file that doesn't exist yet, its
|
|
||||||
/// canonical parent joined with its name. `None` when even the parent
|
|
||||||
/// doesn't exist, in which case nothing can collide with it.
|
|
||||||
fn resolve(path: &Path) -> Option<PathBuf> {
|
|
||||||
if let Ok(canonical) = std::fs::canonicalize(path) {
|
|
||||||
return Some(canonical);
|
|
||||||
}
|
|
||||||
let parent = match path.parent() {
|
|
||||||
Some(p) if !p.as_os_str().is_empty() => p,
|
|
||||||
_ => Path::new("."),
|
|
||||||
};
|
|
||||||
Some(std::fs::canonicalize(parent).ok()?.join(path.file_name()?))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use tempfile::tempdir;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn copying_a_growing_capture_into_its_own_directory_is_refused() {
|
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let partial = dir.path().join("cap.pcap.partial");
|
|
||||||
std::fs::write(&partial, b"live").unwrap();
|
|
||||||
let final_src = dir.path().join("cap.pcap");
|
|
||||||
// What `resolve_dest_paths` makes of DEST = the source's directory.
|
|
||||||
let err = refuse_overlap([&partial, &final_src], [&partial, &final_src]).unwrap_err();
|
|
||||||
assert!(err.to_string().contains("overwrite the capture"), "{err}");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_symlinked_or_relative_spelling_of_the_source_is_still_caught() {
|
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let real = dir.path().join("real");
|
|
||||||
std::fs::create_dir(&real).unwrap();
|
|
||||||
let src = real.join("cap.pcap");
|
|
||||||
std::fs::write(&src, b"done").unwrap();
|
|
||||||
let link = dir.path().join("link");
|
|
||||||
std::os::unix::fs::symlink(&real, &link).unwrap();
|
|
||||||
let via_link = link.join("cap.pcap");
|
|
||||||
let via_dotdot = real.join("..").join("real").join("cap.pcap.partial");
|
|
||||||
assert!(refuse_overlap([&src, &src], [&via_dotdot, &via_link]).is_err());
|
|
||||||
// Neither exists yet: compared by canonical parent + name.
|
|
||||||
let not_yet = real.join("other.pcap");
|
|
||||||
assert!(refuse_overlap([¬_yet, ¬_yet], [&link.join("other.pcap"), &via_dotdot]).is_err());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_different_destination_is_allowed() {
|
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let src = dir.path().join("cap.pcap.partial");
|
|
||||||
std::fs::write(&src, b"live").unwrap();
|
|
||||||
let out = dir.path().join("out");
|
|
||||||
std::fs::create_dir(&out).unwrap();
|
|
||||||
let final_src = dir.path().join("cap.pcap");
|
|
||||||
let dests = [out.join("cap.pcap.partial"), out.join("cap.pcap")];
|
|
||||||
assert!(refuse_overlap([&src, &final_src], [&dests[0], &dests[1]]).is_ok());
|
|
||||||
// A destination under a directory that doesn't exist can't collide.
|
|
||||||
let missing = dir.path().join("nope").join("cap.pcap");
|
|
||||||
assert!(refuse_overlap([&src, &final_src], [&missing, &missing]).is_ok());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+1
-2
@@ -50,8 +50,7 @@ pub struct PacketFilter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl PacketFilter {
|
impl PacketFilter {
|
||||||
/// The no-`--ip`/`--mac` default: keeps everything, matches without
|
/// The no-`--ip` default: keeps everything, matches without inspecting.
|
||||||
/// inspecting.
|
|
||||||
pub fn match_all() -> Self {
|
pub fn match_all() -> Self {
|
||||||
Self { match_all: true, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() }
|
Self { match_all: true, v4: Vec::new(), v6: Vec::new(), macs: Vec::new() }
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-17
@@ -1,5 +1,4 @@
|
|||||||
use anyhow::{bail, Result};
|
use anyhow::{bail, Result};
|
||||||
use std::num::NonZeroUsize;
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
@@ -112,9 +111,7 @@ pub fn resolve_dest_paths(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
||||||
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
|
pub fn parse_size(s: &str) -> Result<usize> {
|
||||||
/// a zero target would cut empty chunks without end.
|
|
||||||
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
|
|
||||||
let s = s.trim();
|
let s = s.trim();
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
bail!("empty size");
|
bail!("empty size");
|
||||||
@@ -129,8 +126,7 @@ pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
|
|||||||
.trim()
|
.trim()
|
||||||
.parse()
|
.parse()
|
||||||
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
||||||
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
|
Ok(n * mult)
|
||||||
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
||||||
@@ -152,8 +148,7 @@ pub fn parse_duration(s: &str) -> Result<Duration> {
|
|||||||
.trim()
|
.trim()
|
||||||
.parse()
|
.parse()
|
||||||
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
||||||
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
|
Ok(Duration::from_millis(n * mult_ms))
|
||||||
Ok(Duration::from_millis(ms))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
||||||
@@ -287,21 +282,19 @@ mod tests {
|
|||||||
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
||||||
// 0 is legal: cut a chunk as soon as anything is kept.
|
// 0 is legal: cut a chunk as soon as anything is kept.
|
||||||
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
||||||
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
|
for bad in ["", "abc", "0.1s", "100m", "-5"] {
|
||||||
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn size_parsing() {
|
fn size_parsing() {
|
||||||
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
|
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
|
||||||
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
|
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
|
||||||
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
|
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
|
||||||
assert_eq!(parse_size("512").unwrap().get(), 512);
|
assert_eq!(parse_size("512").unwrap(), 512);
|
||||||
// 0 would cut empty chunks forever; overflow must not wrap.
|
assert!(parse_size("").is_err());
|
||||||
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
|
assert!(parse_size("abc").is_err());
|
||||||
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+4
-152
@@ -7,9 +7,7 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
|||||||
use anyhow::{bail, Result};
|
use anyhow::{bail, Result};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::num::NonZeroUsize;
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::sync::{mpsc, Arc};
|
use std::sync::{mpsc, Arc};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::thread;
|
use std::thread;
|
||||||
@@ -25,7 +23,7 @@ pub struct SendOptions {
|
|||||||
pub is_partial: bool,
|
pub is_partial: bool,
|
||||||
pub final_source_path: PathBuf,
|
pub final_source_path: PathBuf,
|
||||||
pub compress: bool,
|
pub compress: bool,
|
||||||
pub chunk_target: NonZeroUsize,
|
pub chunk_target: usize,
|
||||||
/// How long kept-but-undersized output may wait before being sent
|
/// How long kept-but-undersized output may wait before being sent
|
||||||
/// anyway. See `Chunker::should_flush`.
|
/// anyway. See `Chunker::should_flush`.
|
||||||
pub flush_after: Duration,
|
pub flush_after: Duration,
|
||||||
@@ -81,19 +79,12 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
let (tail_tx, tail_rx) = mpsc::sync_channel::<SourceEvent>(PIPELINE_DEPTH);
|
||||||
// Raised once the send loop below is done, for whatever reason. After a
|
|
||||||
// clean EOF or a source error the reader has already returned and never
|
|
||||||
// looks; after a failed send it is how the reader learns to quit instead
|
|
||||||
// of polling the source until it next has something to send, which on an
|
|
||||||
// idle capture or a filter matching nothing may be never.
|
|
||||||
let stop = Arc::new(AtomicBool::new(false));
|
|
||||||
let reader = thread::spawn({
|
let reader = thread::spawn({
|
||||||
let source_path = opts.source_path.clone();
|
let source_path = opts.source_path.clone();
|
||||||
let is_partial = opts.is_partial;
|
let is_partial = opts.is_partial;
|
||||||
let final_source_path = opts.final_source_path.clone();
|
let final_source_path = opts.final_source_path.clone();
|
||||||
let log = opts.log.clone();
|
let log = opts.log.clone();
|
||||||
let stop = stop.clone();
|
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
||||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log, stop)
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if opts.compress {
|
if opts.compress {
|
||||||
@@ -148,7 +139,6 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
stop.store(true, Ordering::Relaxed);
|
|
||||||
let reader_result = reader.join().unwrap();
|
let reader_result = reader.join().unwrap();
|
||||||
let compressor_result = compressor.join().unwrap();
|
let compressor_result = compressor.join().unwrap();
|
||||||
if let Some(e) = send_err {
|
if let Some(e) = send_err {
|
||||||
@@ -184,7 +174,6 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
stop.store(true, Ordering::Relaxed);
|
|
||||||
let reader_result = reader.join().unwrap();
|
let reader_result = reader.join().unwrap();
|
||||||
if let Some(e) = send_err {
|
if let Some(e) = send_err {
|
||||||
return Err(e);
|
return Err(e);
|
||||||
@@ -368,7 +357,7 @@ mod tests {
|
|||||||
is_partial: false,
|
is_partial: false,
|
||||||
final_source_path: src.clone(),
|
final_source_path: src.clone(),
|
||||||
compress: true,
|
compress: true,
|
||||||
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
|
chunk_target: 512, // small target to force multiple frames
|
||||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||||
filter: PacketFilter::match_all(),
|
filter: PacketFilter::match_all(),
|
||||||
@@ -496,7 +485,7 @@ mod tests {
|
|||||||
is_partial: false,
|
is_partial: false,
|
||||||
final_source_path: src.clone(),
|
final_source_path: src.clone(),
|
||||||
compress: true,
|
compress: true,
|
||||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
chunk_target: 512,
|
||||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||||
filter: ip_filter(&["1.1.1.1"]),
|
filter: ip_filter(&["1.1.1.1"]),
|
||||||
@@ -541,141 +530,4 @@ mod tests {
|
|||||||
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
run_pipe(opts, dest_final.clone(), dest_temp).unwrap();
|
||||||
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
assert_eq!(std::fs::read(&dest_final).unwrap(), eth_header_bytes());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A record holding an Ethernet frame from `src` to `dst` with
|
|
||||||
/// `ethertype`, padded to `len` captured bytes.
|
|
||||||
fn mac_record(dst: [u8; 6], src: [u8; 6], ethertype: u16, len: usize, ts_sec: u32) -> Vec<u8> {
|
|
||||||
let mut pkt = vec![0u8; len.max(14)];
|
|
||||||
pkt[0..6].copy_from_slice(&dst);
|
|
||||||
pkt[6..12].copy_from_slice(&src);
|
|
||||||
pkt[12..14].copy_from_slice(ðertype.to_be_bytes());
|
|
||||||
record_bytes(&pkt, ts_sec)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn mac_filtered_growing_capture_carries_only_the_upstream_link() {
|
|
||||||
// The scenario --mac exists for: a capture still being written, from
|
|
||||||
// which only the traffic on one switch connection is wanted. The
|
|
||||||
// source grows in uneven appends that tear records mid-header and
|
|
||||||
// mid-payload, so the filter must only ever see confirmed records.
|
|
||||||
const UPSTREAM: [u8; 6] = [0x02, 0, 0, 0, 0, 0xaa];
|
|
||||||
const OTHER: [u8; 6] = [0x02, 0, 0, 0, 0, 0xbb];
|
|
||||||
const BROADCAST: [u8; 6] = [0xff; 6];
|
|
||||||
|
|
||||||
let mut source = eth_header_bytes();
|
|
||||||
let mut expected = eth_header_bytes();
|
|
||||||
for i in 0..60u32 {
|
|
||||||
let ts = 1_700_000_000 + i;
|
|
||||||
let len = 64 + i as usize;
|
|
||||||
let (record, kept) = match i % 4 {
|
|
||||||
0 => (mac_record(OTHER, UPSTREAM, 0x0800, len, ts), true), // from the switch
|
|
||||||
1 => (mac_record(UPSTREAM, OTHER, 0x86DD, len, ts), true), // to the switch
|
|
||||||
2 => (mac_record(BROADCAST, UPSTREAM, 0x0806, len, ts), true), // ARP from the switch
|
|
||||||
_ => (mac_record(BROADCAST, OTHER, 0x0806, len, ts), false), // someone else's ARP
|
|
||||||
};
|
|
||||||
source.extend_from_slice(&record);
|
|
||||||
if kept {
|
|
||||||
expected.extend_from_slice(&record);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let partial = dir.path().join("cap.pcap.partial");
|
|
||||||
let final_src = dir.path().join("cap.pcap");
|
|
||||||
std::fs::write(&partial, &source[..10]).unwrap();
|
|
||||||
let dest_final = dir.path().join("out.pcap.zst");
|
|
||||||
let dest_temp = dir.path().join("out.pcap.zst.partial");
|
|
||||||
|
|
||||||
let opts = SendOptions {
|
|
||||||
source_path: partial.clone(),
|
|
||||||
is_partial: true,
|
|
||||||
final_source_path: final_src.clone(),
|
|
||||||
compress: true,
|
|
||||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
|
||||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
||||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
||||||
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
|
||||||
log: Arc::new(Logger::none()),
|
|
||||||
};
|
|
||||||
|
|
||||||
let (tx, rx) = mpsc::channel::<Message>();
|
|
||||||
let recv_handle = thread::spawn({
|
|
||||||
let dest_final = dest_final.clone();
|
|
||||||
let dest_temp = dest_temp.clone();
|
|
||||||
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
|
||||||
});
|
|
||||||
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
|
||||||
|
|
||||||
// Odd-sized appends (an awkward stride relative to the ~80-140 byte
|
|
||||||
// records) so the visible end lands inside headers and payloads.
|
|
||||||
let mut written = 10;
|
|
||||||
for step in [37usize, 501, 13, 977, 250].iter().cycle() {
|
|
||||||
if written == source.len() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let end = (written + step).min(source.len());
|
|
||||||
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
|
||||||
f.write_all(&source[written..end]).unwrap();
|
|
||||||
f.sync_all().unwrap();
|
|
||||||
written = end;
|
|
||||||
thread::sleep(std::time::Duration::from_millis(5));
|
|
||||||
}
|
|
||||||
std::fs::rename(&partial, &final_src).unwrap();
|
|
||||||
|
|
||||||
send_handle.join().unwrap().unwrap();
|
|
||||||
recv_handle.join().unwrap().unwrap();
|
|
||||||
|
|
||||||
let out_bytes = std::fs::read(&dest_final).unwrap();
|
|
||||||
assert_eq!(zstd::stream::decode_all(out_bytes.as_slice()).unwrap(), expected);
|
|
||||||
assert!(expected.len() < source.len());
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A peer that has gone away: every send fails.
|
|
||||||
struct GoneSink;
|
|
||||||
|
|
||||||
impl MessageSink for GoneSink {
|
|
||||||
fn send_data(&mut self, _payload: &[u8]) -> Result<()> {
|
|
||||||
bail!("peer gone")
|
|
||||||
}
|
|
||||||
fn send_finalize(&mut self) -> Result<()> {
|
|
||||||
bail!("peer gone")
|
|
||||||
}
|
|
||||||
fn send_error(&mut self, _msg: &str) -> Result<()> {
|
|
||||||
bail!("peer gone")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_failed_send_returns_even_though_the_source_never_finishes() {
|
|
||||||
// The reader used to learn the peer was gone only on its next send.
|
|
||||||
// A growing capture that stops growing (and is never renamed) never
|
|
||||||
// gives it one, so run_send waited on the reader forever.
|
|
||||||
for compress in [false, true] {
|
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let partial = dir.path().join("cap.pcap.partial");
|
|
||||||
let mut stream = eth_header_bytes();
|
|
||||||
stream.extend_from_slice(&ip_record([1, 1, 1, 1], [9, 9, 9, 9], 60));
|
|
||||||
std::fs::write(&partial, &stream).unwrap();
|
|
||||||
|
|
||||||
let opts = SendOptions {
|
|
||||||
source_path: partial.clone(),
|
|
||||||
is_partial: true,
|
|
||||||
final_source_path: dir.path().join("cap.pcap"),
|
|
||||||
compress,
|
|
||||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
|
||||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
|
||||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
|
||||||
filter: PacketFilter::match_all(),
|
|
||||||
log: Arc::new(Logger::none()),
|
|
||||||
};
|
|
||||||
let (done_tx, done_rx) = mpsc::channel();
|
|
||||||
thread::spawn(move || {
|
|
||||||
let _ = done_tx.send(run_send(opts, Box::new(GoneSink)));
|
|
||||||
});
|
|
||||||
let result = done_rx
|
|
||||||
.recv_timeout(Duration::from_secs(5))
|
|
||||||
.unwrap_or_else(|_| panic!("run_send hung after a failed send (compress={compress})"));
|
|
||||||
assert!(result.unwrap_err().to_string().contains("peer gone"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
+1
-1
@@ -39,7 +39,7 @@ fn run_send_server(cli: ServerCli) -> Result<()> {
|
|||||||
let flush_after = naming::parse_duration(&cli.flush_interval)?;
|
let flush_after = naming::parse_duration(&cli.flush_interval)?;
|
||||||
// The sending side is the only side that filters -- dropping packets
|
// The sending side is the only side that filters -- dropping packets
|
||||||
// before they hit the wire is the whole point. `run_recv` never parses
|
// before they hit the wire is the whole point. `run_recv` never parses
|
||||||
// what it writes, so --recv has no --ip/--mac.
|
// what it writes, so --recv has no --ip.
|
||||||
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
|
let filter = PacketFilter::parse(&cli.ip, &cli.mac)?;
|
||||||
|
|
||||||
let opts = SendOptions {
|
let opts = SendOptions {
|
||||||
|
|||||||
+9
-96
@@ -2,9 +2,8 @@ use crate::chunker::{Chunker, ChunkKind};
|
|||||||
use crate::tracelog::Logger;
|
use crate::tracelog::Logger;
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{bail, Result};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::os::unix::fs::{FileExt, MetadataExt};
|
use std::os::unix::fs::FileExt;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
@@ -59,12 +58,6 @@ impl EventSink for std::sync::mpsc::SyncSender<SourceEvent> {
|
|||||||
///
|
///
|
||||||
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
/// Per the confirmed design: if `source_path` doesn't exist at all when this
|
||||||
/// is called, that's a hard error (no polling for creation).
|
/// is called, that's a hard error (no polling for creation).
|
||||||
///
|
|
||||||
/// `stop` is how the consumer says it has given up (its peer is gone). The
|
|
||||||
/// loop checks it every poll, because otherwise it would only find out on
|
|
||||||
/// its next send -- and a growing source that stops growing, or a filter
|
|
||||||
/// that matches nothing, may never produce one, leaving whoever joins this
|
|
||||||
/// thread waiting forever.
|
|
||||||
pub fn run_tail(
|
pub fn run_tail(
|
||||||
source_path: PathBuf,
|
source_path: PathBuf,
|
||||||
is_partial: bool,
|
is_partial: bool,
|
||||||
@@ -72,7 +65,6 @@ pub fn run_tail(
|
|||||||
mut chunker: Box<dyn Chunker>,
|
mut chunker: Box<dyn Chunker>,
|
||||||
events: impl EventSink,
|
events: impl EventSink,
|
||||||
log: Arc<Logger>,
|
log: Arc<Logger>,
|
||||||
stop: Arc<AtomicBool>,
|
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
if !source_path.exists() {
|
if !source_path.exists() {
|
||||||
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
log.log(format_args!("tail: source not found: {}", source_path.display()));
|
||||||
@@ -83,7 +75,7 @@ pub fn run_tail(
|
|||||||
source_path.display(),
|
source_path.display(),
|
||||||
final_path.display()
|
final_path.display()
|
||||||
));
|
));
|
||||||
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log, &stop);
|
let result = drain(&source_path, is_partial, &final_path, chunker.as_mut(), &events, &log);
|
||||||
match &result {
|
match &result {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
log.log(format_args!("tail: done, sending EOF"));
|
log.log(format_args!("tail: done, sending EOF"));
|
||||||
@@ -104,7 +96,6 @@ fn drain(
|
|||||||
chunker: &mut dyn Chunker,
|
chunker: &mut dyn Chunker,
|
||||||
events: &impl EventSink,
|
events: &impl EventSink,
|
||||||
log: &Logger,
|
log: &Logger,
|
||||||
stop: &AtomicBool,
|
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let file = File::open(source_path)?;
|
let file = File::open(source_path)?;
|
||||||
let mut complete_signal = !is_partial;
|
let mut complete_signal = !is_partial;
|
||||||
@@ -132,9 +123,6 @@ fn drain(
|
|||||||
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
let mut last_stuck_peek: Option<Vec<u8>> = None;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
if stop.load(Ordering::Relaxed) {
|
|
||||||
bail!("stopped: the receiving side is gone");
|
|
||||||
}
|
|
||||||
let file_len = file.metadata()?.len();
|
let file_len = file.metadata()?.len();
|
||||||
// Whether this iteration did real work -- if so, loop again
|
// Whether this iteration did real work -- if so, loop again
|
||||||
// immediately rather than sleeping, so a large already-available
|
// immediately rather than sleeping, so a large already-available
|
||||||
@@ -219,7 +207,7 @@ fn drain(
|
|||||||
send_chunk(events, kind, data)?;
|
send_chunk(events, kind, data)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
if !complete_signal && renamed_to(final_path, &file)? {
|
if !complete_signal && final_path.try_exists()? {
|
||||||
// Rename observed -- signal now, well before finalization, per
|
// Rename observed -- signal now, well before finalization, per
|
||||||
// section 7.4's repoint-then-drain-then-finalize ordering.
|
// section 7.4's repoint-then-drain-then-finalize ordering.
|
||||||
complete_signal = true;
|
complete_signal = true;
|
||||||
@@ -281,22 +269,6 @@ fn drain(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether `final_path` now names the very file being read, i.e. the writer's
|
|
||||||
/// rename has happened. This is stricter than section 7.5's "the final path
|
|
||||||
/// exists": names get reused (section 6), so a file left at the final name by
|
|
||||||
/// an earlier capture or copy would otherwise pass for the rename the moment
|
|
||||||
/// tailing starts, and the transfer would finish early, short, at the first
|
|
||||||
/// lull in the writer. Comparing identity also still works when the writer
|
|
||||||
/// renames over such a leftover.
|
|
||||||
fn renamed_to(final_path: &Path, file: &File) -> Result<bool> {
|
|
||||||
let ours = file.metadata()?;
|
|
||||||
match std::fs::metadata(final_path) {
|
|
||||||
Ok(theirs) => Ok(theirs.dev() == ours.dev() && theirs.ino() == ours.ino()),
|
|
||||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
|
|
||||||
Err(e) => Err(e.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emit_ready(chunker: &mut dyn Chunker, events: &impl EventSink) -> Result<()> {
|
fn emit_ready(chunker: &mut dyn Chunker, events: &impl EventSink) -> Result<()> {
|
||||||
while let Some((kind, data)) = chunker.ready_chunk() {
|
while let Some((kind, data)) = chunker.ready_chunk() {
|
||||||
send_chunk(events, kind, data)?;
|
send_chunk(events, kind, data)?;
|
||||||
@@ -344,10 +316,6 @@ mod tests {
|
|||||||
(data, got_eof)
|
(data, got_eof)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn no_stop() -> Arc<AtomicBool> {
|
|
||||||
Arc::new(AtomicBool::new(false))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
fn header_bytes(magic_le: u32) -> Vec<u8> {
|
||||||
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
let mut h = vec![0u8; GLOBAL_HEADER_LEN];
|
||||||
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
h[0..4].copy_from_slice(&magic_le.to_le_bytes());
|
||||||
@@ -372,7 +340,7 @@ mod tests {
|
|||||||
std::fs::write(&path, b"hello world").unwrap();
|
std::fs::write(&path, b"hello world").unwrap();
|
||||||
|
|
||||||
let (tx, rx) = mpsc::channel();
|
let (tx, rx) = mpsc::channel();
|
||||||
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap();
|
run_tail(path.clone(), false, path.clone(), Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap();
|
||||||
let (data, eof) = collect(rx);
|
let (data, eof) = collect(rx);
|
||||||
assert_eq!(data, b"hello world");
|
assert_eq!(data, b"hello world");
|
||||||
assert!(eof);
|
assert!(eof);
|
||||||
@@ -384,7 +352,7 @@ mod tests {
|
|||||||
let path = dir.path().join("nope.pcap.partial");
|
let path = dir.path().join("nope.pcap.partial");
|
||||||
let final_path = dir.path().join("nope.pcap");
|
let final_path = dir.path().join("nope.pcap");
|
||||||
let (tx, _rx) = mpsc::channel();
|
let (tx, _rx) = mpsc::channel();
|
||||||
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop()).unwrap_err();
|
let err = run_tail(path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none())).unwrap_err();
|
||||||
assert!(err.to_string().contains("not found"));
|
assert!(err.to_string().contains("not found"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -401,7 +369,7 @@ mod tests {
|
|||||||
let handle = std::thread::spawn({
|
let handle = std::thread::spawn({
|
||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||||
});
|
});
|
||||||
|
|
||||||
std::thread::sleep(Duration::from_millis(60));
|
std::thread::sleep(Duration::from_millis(60));
|
||||||
@@ -431,7 +399,7 @@ mod tests {
|
|||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
let log = log.clone();
|
let log = log.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log, no_stop())
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, log)
|
||||||
});
|
});
|
||||||
|
|
||||||
std::thread::sleep(Duration::from_millis(60));
|
std::thread::sleep(Duration::from_millis(60));
|
||||||
@@ -458,7 +426,7 @@ mod tests {
|
|||||||
let handle = std::thread::spawn({
|
let handle = std::thread::spawn({
|
||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||||
});
|
});
|
||||||
|
|
||||||
std::thread::sleep(Duration::from_millis(60));
|
std::thread::sleep(Duration::from_millis(60));
|
||||||
@@ -489,7 +457,7 @@ mod tests {
|
|||||||
let handle = std::thread::spawn({
|
let handle = std::thread::spawn({
|
||||||
let partial_path = partial_path.clone();
|
let partial_path = partial_path.clone();
|
||||||
let final_path = final_path.clone();
|
let final_path = final_path.clone();
|
||||||
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()), no_stop())
|
move || run_tail(partial_path, true, final_path, Box::new(RawChunker::new()), tx, Arc::new(Logger::none()))
|
||||||
});
|
});
|
||||||
|
|
||||||
// Let the reader fully drain the initial 10 bytes.
|
// Let the reader fully drain the initial 10 bytes.
|
||||||
@@ -554,7 +522,6 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
no_stop(),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -602,7 +569,6 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
no_stop(),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -658,7 +624,6 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
no_stop(),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -703,7 +668,6 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
no_stop(),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -742,59 +706,8 @@ mod tests {
|
|||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false)),
|
||||||
tx,
|
tx,
|
||||||
Arc::new(Logger::none()),
|
Arc::new(Logger::none()),
|
||||||
no_stop(),
|
|
||||||
);
|
);
|
||||||
try_collect(rx);
|
try_collect(rx);
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_stale_file_at_the_final_name_is_not_the_rename() {
|
|
||||||
// A leftover cap.pcap (an earlier capture reusing the name) used to
|
|
||||||
// pass for the rename the moment tailing started, so the first lull
|
|
||||||
// longer than the grace period finalized the copy short.
|
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let partial_path = dir.path().join("cap.pcap.partial");
|
|
||||||
let final_path = dir.path().join("cap.pcap");
|
|
||||||
std::fs::write(&final_path, b"yesterday's capture").unwrap();
|
|
||||||
|
|
||||||
let mut stream = header_bytes(0xA1B2C3D4);
|
|
||||||
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
|
|
||||||
stream.extend_from_slice(&record_bytes(&[2u8; 10], 1_700_000_001));
|
|
||||||
let mut f = File::create(&partial_path).unwrap();
|
|
||||||
f.write_all(&stream).unwrap();
|
|
||||||
f.sync_all().unwrap();
|
|
||||||
|
|
||||||
let (tx, rx) = mpsc::channel();
|
|
||||||
let handle = std::thread::spawn({
|
|
||||||
let partial_path = partial_path.clone();
|
|
||||||
let final_path = final_path.clone();
|
|
||||||
move || {
|
|
||||||
run_tail(
|
|
||||||
partial_path,
|
|
||||||
true,
|
|
||||||
final_path,
|
|
||||||
Box::new(RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true)),
|
|
||||||
tx,
|
|
||||||
Arc::new(Logger::none()),
|
|
||||||
no_stop(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// A writer lull longer than the grace period, then more packets and
|
|
||||||
// the real rename -- over the stale file.
|
|
||||||
std::thread::sleep(FINALIZE_GRACE_PERIOD + Duration::from_millis(500));
|
|
||||||
let rec3 = record_bytes(&[3u8; 10], 1_700_000_002);
|
|
||||||
f.write_all(&rec3).unwrap();
|
|
||||||
f.sync_all().unwrap();
|
|
||||||
stream.extend_from_slice(&rec3);
|
|
||||||
drop(f);
|
|
||||||
std::fs::rename(&partial_path, &final_path).unwrap();
|
|
||||||
|
|
||||||
handle.join().unwrap().unwrap();
|
|
||||||
let (data, eof) = collect(rx);
|
|
||||||
assert_eq!(data, stream);
|
|
||||||
assert!(eof);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user