Make the chunk size non-zero by type; reject overflowing sizes
--chunk 0 made every chunk 'full' while empty, so the chunker emitted empty chunks (and empty zstd frames) forever -- 11 MB in 3 s from a 424-byte source. parse_size now returns NonZeroUsize, which flows through SendOptions into RecordAlignedChunker, so a zero target can't reach the cut logic. parse_size and parse_duration also use checked_mul instead of wrapping on absurd inputs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6a80aaf461
commit
b46d6a936c
3 files changed
+41
-28
No files matched your search
+19
-14
@@ -1,12 +1,13 @@
|
||||
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
||||
use anyhow::{Result, bail};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub const GLOBAL_HEADER_LEN: usize = 24;
|
||||
pub const RECORD_HEADER_LEN: usize = 16;
|
||||
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
||||
#[allow(dead_code)]
|
||||
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
|
||||
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
|
||||
/// Sanity bound on a record's declared payload length, per
|
||||
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
||||
/// sources -- see `RecordAlignedChunker::partial`.
|
||||
@@ -178,7 +179,7 @@ pub struct RecordAlignedChunker {
|
||||
/// bytes is what keeps `flush_after` an actual latency bound rather
|
||||
/// than an idle-only one.
|
||||
kept_since: Option<Instant>,
|
||||
target: usize,
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
/// Whether this source is a still-growing (partial) file. Gates every
|
||||
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
||||
@@ -195,12 +196,12 @@ impl RecordAlignedChunker {
|
||||
/// same record-confirmation logic. Production goes through
|
||||
/// `with_filter`; this is the convenience the tests use.
|
||||
#[allow(dead_code)]
|
||||
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
|
||||
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
|
||||
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
||||
}
|
||||
|
||||
pub fn with_filter(
|
||||
target: usize,
|
||||
target: NonZeroUsize,
|
||||
flush_after: Duration,
|
||||
partial: bool,
|
||||
filter: PacketFilter,
|
||||
@@ -400,7 +401,7 @@ impl Chunker for RecordAlignedChunker {
|
||||
// must produce no chunk at all, rather than an empty one. An empty
|
||||
// chunk would become a zero-content zstd frame, which
|
||||
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
||||
if self.kept.len() >= self.target {
|
||||
if self.kept.len() >= self.target.get() {
|
||||
return Some(self.take_kept());
|
||||
}
|
||||
None
|
||||
@@ -509,6 +510,10 @@ mod tests {
|
||||
h
|
||||
}
|
||||
|
||||
fn nz(n: usize) -> NonZeroUsize {
|
||||
NonZeroUsize::new(n).unwrap()
|
||||
}
|
||||
|
||||
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
||||
record_bytes_with_ts(payload, big_endian, 0)
|
||||
}
|
||||
@@ -571,7 +576,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
||||
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
||||
for i in 0..20u8 {
|
||||
@@ -606,7 +611,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
||||
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
|
||||
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
|
||||
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
||||
c.ready_chunk(); // drain header
|
||||
|
||||
@@ -913,7 +918,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn keeps_only_matching_records_and_keeps_them_whole() {
|
||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let hit_src = ip_record(WANTED, OTHER, 60);
|
||||
let hit_dst = ip_record(OTHER, WANTED, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
@@ -946,7 +951,7 @@ mod tests {
|
||||
let hit = ip_record(WANTED, OTHER, 100);
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let target = 10 * hit.len();
|
||||
let mut c = RecordAlignedChunker::with_filter(target, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
@@ -1001,7 +1006,7 @@ mod tests {
|
||||
// An empty chunk would be compressed into a zero-content zstd frame,
|
||||
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
||||
let miss = ip_record(OTHER, THIRD, 100);
|
||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
c.feed(ð_header_bytes()).unwrap();
|
||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||
for _ in 0..20 {
|
||||
@@ -1026,9 +1031,9 @@ mod tests {
|
||||
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
||||
}
|
||||
|
||||
let mut all = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||
let mut some = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut none = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||
for c in [&mut all, &mut some, &mut none] {
|
||||
c.feed(&stream).unwrap();
|
||||
}
|
||||
@@ -1052,7 +1057,7 @@ mod tests {
|
||||
// exactly once even though the bytes around it are re-read.
|
||||
let hit = ip_record(WANTED, OTHER, 60);
|
||||
let miss = ip_record(OTHER, THIRD, 60);
|
||||
let mut c = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||
|
||||
let mut stream = eth_header_bytes();
|
||||
stream.extend_from_slice(&miss);
|
||||
|
||||
+17
-10
@@ -1,4 +1,5 @@
|
||||
use anyhow::{bail, Result};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::Duration;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -111,7 +112,9 @@ pub fn resolve_dest_paths(
|
||||
}
|
||||
|
||||
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
||||
pub fn parse_size(s: &str) -> Result<usize> {
|
||||
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
|
||||
/// a zero target would cut empty chunks without end.
|
||||
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
bail!("empty size");
|
||||
@@ -126,7 +129,8 @@ pub fn parse_size(s: &str) -> Result<usize> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
||||
Ok(n * mult)
|
||||
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
|
||||
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
|
||||
}
|
||||
|
||||
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
||||
@@ -148,7 +152,8 @@ pub fn parse_duration(s: &str) -> Result<Duration> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
||||
Ok(Duration::from_millis(n * mult_ms))
|
||||
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
|
||||
Ok(Duration::from_millis(ms))
|
||||
}
|
||||
|
||||
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
||||
@@ -282,19 +287,21 @@ mod tests {
|
||||
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
||||
// 0 is legal: cut a chunk as soon as anything is kept.
|
||||
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
||||
for bad in ["", "abc", "0.1s", "100m", "-5"] {
|
||||
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
|
||||
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_parsing() {
|
||||
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap(), 512);
|
||||
assert!(parse_size("").is_err());
|
||||
assert!(parse_size("abc").is_err());
|
||||
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
|
||||
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
|
||||
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
|
||||
assert_eq!(parse_size("512").unwrap().get(), 512);
|
||||
// 0 would cut empty chunks forever; overflow must not wrap.
|
||||
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
|
||||
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+5
-4
@@ -7,6 +7,7 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
@@ -23,7 +24,7 @@ pub struct SendOptions {
|
||||
pub is_partial: bool,
|
||||
pub final_source_path: PathBuf,
|
||||
pub compress: bool,
|
||||
pub chunk_target: usize,
|
||||
pub chunk_target: NonZeroUsize,
|
||||
/// How long kept-but-undersized output may wait before being sent
|
||||
/// anyway. See `Chunker::should_flush`.
|
||||
pub flush_after: Duration,
|
||||
@@ -357,7 +358,7 @@ mod tests {
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512, // small target to force multiple frames
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::match_all(),
|
||||
@@ -485,7 +486,7 @@ mod tests {
|
||||
is_partial: false,
|
||||
final_source_path: src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: ip_filter(&["1.1.1.1"]),
|
||||
@@ -580,7 +581,7 @@ mod tests {
|
||||
is_partial: true,
|
||||
final_source_path: final_src.clone(),
|
||||
compress: true,
|
||||
chunk_target: 512,
|
||||
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
||||
|
||||
Reference in new issue
Block a user