Make the chunk size non-zero by type; reject overflowing sizes

--chunk 0 made every chunk 'full' while empty, so the chunker emitted
empty chunks (and empty zstd frames) forever -- 11 MB in 3 s from a
424-byte source. parse_size now returns NonZeroUsize, which flows through
SendOptions into RecordAlignedChunker, so a zero target can't reach the
cut logic. parse_size and parse_duration also use checked_mul instead of
wrapping on absurd inputs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ericandClaude Opus 5.5 committed 2026-11-13 16:11:17 -05:00
1 parent 6a80aaf461
commit b46d6a936c
3 files changed
+41 -28

No files matched your search

+19 -14
View File
@@ -1,12 +1,13 @@
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
use anyhow::{Result, bail};
use std::num::NonZeroUsize;
use std::time::{Duration, Instant};
pub const GLOBAL_HEADER_LEN: usize = 24;
pub const RECORD_HEADER_LEN: usize = 16;
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
#[allow(dead_code)]
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
/// Sanity bound on a record's declared payload length, per
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
/// sources -- see `RecordAlignedChunker::partial`.
@@ -178,7 +179,7 @@ pub struct RecordAlignedChunker {
/// bytes is what keeps `flush_after` an actual latency bound rather
/// than an idle-only one.
kept_since: Option<Instant>,
target: usize,
target: NonZeroUsize,
flush_after: Duration,
/// Whether this source is a still-growing (partial) file. Gates every
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
@@ -195,12 +196,12 @@ impl RecordAlignedChunker {
/// same record-confirmation logic. Production goes through
/// `with_filter`; this is the convenience the tests use.
#[allow(dead_code)]
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
}
pub fn with_filter(
target: usize,
target: NonZeroUsize,
flush_after: Duration,
partial: bool,
filter: PacketFilter,
@@ -400,7 +401,7 @@ impl Chunker for RecordAlignedChunker {
// must produce no chunk at all, rather than an empty one. An empty
// chunk would become a zero-content zstd frame, which
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
if self.kept.len() >= self.target {
if self.kept.len() >= self.target.get() {
return Some(self.take_kept());
}
None
@@ -509,6 +510,10 @@ mod tests {
h
}
fn nz(n: usize) -> NonZeroUsize {
NonZeroUsize::new(n).unwrap()
}
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
record_bytes_with_ts(payload, big_endian, 0)
}
@@ -571,7 +576,7 @@ mod tests {
#[test]
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
let mut stream = header_bytes(0xA1B2C3D4);
let mut expected_records: Vec<Vec<u8>> = Vec::new();
for i in 0..20u8 {
@@ -606,7 +611,7 @@ mod tests {
#[test]
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
c.ready_chunk(); // drain header
@@ -913,7 +918,7 @@ mod tests {
#[test]
fn keeps_only_matching_records_and_keeps_them_whole() {
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let hit_src = ip_record(WANTED, OTHER, 60);
let hit_dst = ip_record(OTHER, WANTED, 60);
let miss = ip_record(OTHER, THIRD, 60);
@@ -946,7 +951,7 @@ mod tests {
let hit = ip_record(WANTED, OTHER, 100);
let miss = ip_record(OTHER, THIRD, 100);
let target = 10 * hit.len();
let mut c = RecordAlignedChunker::with_filter(target, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
c.feed(&eth_header_bytes()).unwrap();
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
@@ -1001,7 +1006,7 @@ mod tests {
// An empty chunk would be compressed into a zero-content zstd frame,
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
let miss = ip_record(OTHER, THIRD, 100);
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
c.feed(&eth_header_bytes()).unwrap();
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
for _ in 0..20 {
@@ -1026,9 +1031,9 @@ mod tests {
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
}
let mut all = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, PacketFilter::match_all());
let mut some = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut none = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
for c in [&mut all, &mut some, &mut none] {
c.feed(&stream).unwrap();
}
@@ -1052,7 +1057,7 @@ mod tests {
// exactly once even though the bytes around it are re-read.
let hit = ip_record(WANTED, OTHER, 60);
let miss = ip_record(OTHER, THIRD, 60);
let mut c = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
let mut stream = eth_header_bytes();
stream.extend_from_slice(&miss);
+17 -10
View File
@@ -1,4 +1,5 @@
use anyhow::{bail, Result};
use std::num::NonZeroUsize;
use std::time::Duration;
use std::path::Path;
@@ -111,7 +112,9 @@ pub fn resolve_dest_paths(
}
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
pub fn parse_size(s: &str) -> Result<usize> {
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
/// a zero target would cut empty chunks without end.
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
let s = s.trim();
if s.is_empty() {
bail!("empty size");
@@ -126,7 +129,8 @@ pub fn parse_size(s: &str) -> Result<usize> {
.trim()
.parse()
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
Ok(n * mult)
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
}
/// Parses a flush/idle interval: a bare number is milliseconds, or an
@@ -148,7 +152,8 @@ pub fn parse_duration(s: &str) -> Result<Duration> {
.trim()
.parse()
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
Ok(Duration::from_millis(n * mult_ms))
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
Ok(Duration::from_millis(ms))
}
/// Single-quote a string for safe inclusion as one argument in a remote shell
@@ -282,19 +287,21 @@ mod tests {
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
// 0 is legal: cut a chunk as soon as anything is kept.
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
for bad in ["", "abc", "0.1s", "100m", "-5"] {
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
}
}
#[test]
fn size_parsing() {
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
assert_eq!(parse_size("512").unwrap(), 512);
assert!(parse_size("").is_err());
assert!(parse_size("abc").is_err());
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
assert_eq!(parse_size("512").unwrap().get(), 512);
// 0 would cut empty chunks forever; overflow must not wrap.
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
}
}
#[test]
+5 -4
View File
@@ -7,6 +7,7 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
use anyhow::{bail, Result};
use std::fs::File;
use std::io::Write;
use std::num::NonZeroUsize;
use std::path::PathBuf;
use std::sync::{mpsc, Arc};
use std::time::Duration;
@@ -23,7 +24,7 @@ pub struct SendOptions {
pub is_partial: bool,
pub final_source_path: PathBuf,
pub compress: bool,
pub chunk_target: usize,
pub chunk_target: NonZeroUsize,
/// How long kept-but-undersized output may wait before being sent
/// anyway. See `Chunker::should_flush`.
pub flush_after: Duration,
@@ -357,7 +358,7 @@ mod tests {
is_partial: false,
final_source_path: src.clone(),
compress: true,
chunk_target: 512, // small target to force multiple frames
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: PacketFilter::match_all(),
@@ -485,7 +486,7 @@ mod tests {
is_partial: false,
final_source_path: src.clone(),
compress: true,
chunk_target: 512,
chunk_target: NonZeroUsize::new(512).unwrap(),
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: ip_filter(&["1.1.1.1"]),
@@ -580,7 +581,7 @@ mod tests {
is_partial: true,
final_source_path: final_src.clone(),
compress: true,
chunk_target: 512,
chunk_target: NonZeroUsize::new(512).unwrap(),
flush_after: crate::chunker::DEFAULT_FLUSH,
level: crate::zstd_frame::DEFAULT_LEVEL,
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),