Make the chunk size non-zero by type; reject overflowing sizes
--chunk 0 made every chunk 'full' while empty, so the chunker emitted empty chunks (and empty zstd frames) forever -- 11 MB in 3 s from a 424-byte source. parse_size now returns NonZeroUsize, which flows through SendOptions into RecordAlignedChunker, so a zero target can't reach the cut logic. parse_size and parse_duration also use checked_mul instead of wrapping on absurd inputs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6a80aaf461
commit
b46d6a936c
3 files changed
+41
-28
No files matched your search
+19
-14
@@ -1,12 +1,13 @@
|
|||||||
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
use crate::filter::{PacketFilter, LINKTYPE_ETHERNET};
|
||||||
use anyhow::{Result, bail};
|
use anyhow::{Result, bail};
|
||||||
|
use std::num::NonZeroUsize;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
pub const GLOBAL_HEADER_LEN: usize = 24;
|
pub const GLOBAL_HEADER_LEN: usize = 24;
|
||||||
pub const RECORD_HEADER_LEN: usize = 16;
|
pub const RECORD_HEADER_LEN: usize = 16;
|
||||||
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
|
pub const DEFAULT_TARGET_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024).unwrap();
|
||||||
/// Sanity bound on a record's declared payload length, per
|
/// Sanity bound on a record's declared payload length, per
|
||||||
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
|
||||||
/// sources -- see `RecordAlignedChunker::partial`.
|
/// sources -- see `RecordAlignedChunker::partial`.
|
||||||
@@ -178,7 +179,7 @@ pub struct RecordAlignedChunker {
|
|||||||
/// bytes is what keeps `flush_after` an actual latency bound rather
|
/// bytes is what keeps `flush_after` an actual latency bound rather
|
||||||
/// than an idle-only one.
|
/// than an idle-only one.
|
||||||
kept_since: Option<Instant>,
|
kept_since: Option<Instant>,
|
||||||
target: usize,
|
target: NonZeroUsize,
|
||||||
flush_after: Duration,
|
flush_after: Duration,
|
||||||
/// Whether this source is a still-growing (partial) file. Gates every
|
/// Whether this source is a still-growing (partial) file. Gates every
|
||||||
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
/// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
|
||||||
@@ -195,12 +196,12 @@ impl RecordAlignedChunker {
|
|||||||
/// same record-confirmation logic. Production goes through
|
/// same record-confirmation logic. Production goes through
|
||||||
/// `with_filter`; this is the convenience the tests use.
|
/// `with_filter`; this is the convenience the tests use.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
|
pub fn new(target: NonZeroUsize, flush_after: Duration, partial: bool) -> Self {
|
||||||
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
Self::with_filter(target, flush_after, partial, PacketFilter::match_all())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn with_filter(
|
pub fn with_filter(
|
||||||
target: usize,
|
target: NonZeroUsize,
|
||||||
flush_after: Duration,
|
flush_after: Duration,
|
||||||
partial: bool,
|
partial: bool,
|
||||||
filter: PacketFilter,
|
filter: PacketFilter,
|
||||||
@@ -400,7 +401,7 @@ impl Chunker for RecordAlignedChunker {
|
|||||||
// must produce no chunk at all, rather than an empty one. An empty
|
// must produce no chunk at all, rather than an empty one. An empty
|
||||||
// chunk would become a zero-content zstd frame, which
|
// chunk would become a zero-content zstd frame, which
|
||||||
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
// STREAMING_ZSTD_FORMAT.md §3 does not allow for.
|
||||||
if self.kept.len() >= self.target {
|
if self.kept.len() >= self.target.get() {
|
||||||
return Some(self.take_kept());
|
return Some(self.take_kept());
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
@@ -509,6 +510,10 @@ mod tests {
|
|||||||
h
|
h
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn nz(n: usize) -> NonZeroUsize {
|
||||||
|
NonZeroUsize::new(n).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
|
||||||
record_bytes_with_ts(payload, big_endian, 0)
|
record_bytes_with_ts(payload, big_endian, 0)
|
||||||
}
|
}
|
||||||
@@ -571,7 +576,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
|
||||||
let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
|
let mut c = RecordAlignedChunker::new(nz(50), DEFAULT_FLUSH, false); // tiny target to force cuts
|
||||||
let mut stream = header_bytes(0xA1B2C3D4);
|
let mut stream = header_bytes(0xA1B2C3D4);
|
||||||
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
let mut expected_records: Vec<Vec<u8>> = Vec::new();
|
||||||
for i in 0..20u8 {
|
for i in 0..20u8 {
|
||||||
@@ -606,7 +611,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
|
||||||
let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
|
let mut c = RecordAlignedChunker::new(nz(15), DEFAULT_FLUSH, false);
|
||||||
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
|
||||||
c.ready_chunk(); // drain header
|
c.ready_chunk(); // drain header
|
||||||
|
|
||||||
@@ -913,7 +918,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn keeps_only_matching_records_and_keeps_them_whole() {
|
fn keeps_only_matching_records_and_keeps_them_whole() {
|
||||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
let hit_src = ip_record(WANTED, OTHER, 60);
|
let hit_src = ip_record(WANTED, OTHER, 60);
|
||||||
let hit_dst = ip_record(OTHER, WANTED, 60);
|
let hit_dst = ip_record(OTHER, WANTED, 60);
|
||||||
let miss = ip_record(OTHER, THIRD, 60);
|
let miss = ip_record(OTHER, THIRD, 60);
|
||||||
@@ -946,7 +951,7 @@ mod tests {
|
|||||||
let hit = ip_record(WANTED, OTHER, 100);
|
let hit = ip_record(WANTED, OTHER, 100);
|
||||||
let miss = ip_record(OTHER, THIRD, 100);
|
let miss = ip_record(OTHER, THIRD, 100);
|
||||||
let target = 10 * hit.len();
|
let target = 10 * hit.len();
|
||||||
let mut c = RecordAlignedChunker::with_filter(target, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(nz(target), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
|
|
||||||
c.feed(ð_header_bytes()).unwrap();
|
c.feed(ð_header_bytes()).unwrap();
|
||||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||||
@@ -1001,7 +1006,7 @@ mod tests {
|
|||||||
// An empty chunk would be compressed into a zero-content zstd frame,
|
// An empty chunk would be compressed into a zero-content zstd frame,
|
||||||
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
// which STREAMING_ZSTD_FORMAT.md §3 does not provide for.
|
||||||
let miss = ip_record(OTHER, THIRD, 100);
|
let miss = ip_record(OTHER, THIRD, 100);
|
||||||
let mut c = RecordAlignedChunker::with_filter(1, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(nz(1), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
c.feed(ð_header_bytes()).unwrap();
|
c.feed(ð_header_bytes()).unwrap();
|
||||||
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
assert_eq!(c.ready_chunk().unwrap().0, ChunkKind::Header);
|
||||||
for _ in 0..20 {
|
for _ in 0..20 {
|
||||||
@@ -1026,9 +1031,9 @@ mod tests {
|
|||||||
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
stream.extend_from_slice(&ip_record(src, THIRD, 60 + i as usize));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut all = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, PacketFilter::match_all());
|
let mut all = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, PacketFilter::match_all());
|
||||||
let mut some = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
let mut some = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["1.1.1.1"]));
|
||||||
let mut none = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
let mut none = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, false, filter_for(&["203.0.113.7"]));
|
||||||
for c in [&mut all, &mut some, &mut none] {
|
for c in [&mut all, &mut some, &mut none] {
|
||||||
c.feed(&stream).unwrap();
|
c.feed(&stream).unwrap();
|
||||||
}
|
}
|
||||||
@@ -1052,7 +1057,7 @@ mod tests {
|
|||||||
// exactly once even though the bytes around it are re-read.
|
// exactly once even though the bytes around it are re-read.
|
||||||
let hit = ip_record(WANTED, OTHER, 60);
|
let hit = ip_record(WANTED, OTHER, 60);
|
||||||
let miss = ip_record(OTHER, THIRD, 60);
|
let miss = ip_record(OTHER, THIRD, 60);
|
||||||
let mut c = RecordAlignedChunker::with_filter(1 << 20, DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
let mut c = RecordAlignedChunker::with_filter(nz(1 << 20), DEFAULT_FLUSH, true, filter_for(&["1.1.1.1"]));
|
||||||
|
|
||||||
let mut stream = eth_header_bytes();
|
let mut stream = eth_header_bytes();
|
||||||
stream.extend_from_slice(&miss);
|
stream.extend_from_slice(&miss);
|
||||||
|
|||||||
+17
-10
@@ -1,4 +1,5 @@
|
|||||||
use anyhow::{bail, Result};
|
use anyhow::{bail, Result};
|
||||||
|
use std::num::NonZeroUsize;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
@@ -111,7 +112,9 @@ pub fn resolve_dest_paths(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
/// Parse a human chunk-size string like "256k", "1m", or a bare byte count.
|
||||||
pub fn parse_size(s: &str) -> Result<usize> {
|
/// Zero is rejected here, once: a chunk is cut when it reaches this size, so
|
||||||
|
/// a zero target would cut empty chunks without end.
|
||||||
|
pub fn parse_size(s: &str) -> Result<NonZeroUsize> {
|
||||||
let s = s.trim();
|
let s = s.trim();
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
bail!("empty size");
|
bail!("empty size");
|
||||||
@@ -126,7 +129,8 @@ pub fn parse_size(s: &str) -> Result<usize> {
|
|||||||
.trim()
|
.trim()
|
||||||
.parse()
|
.parse()
|
||||||
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
.map_err(|_| anyhow::anyhow!("invalid size: {s:?}"))?;
|
||||||
Ok(n * mult)
|
let bytes = n.checked_mul(mult).ok_or_else(|| anyhow::anyhow!("size too large: {s:?}"))?;
|
||||||
|
NonZeroUsize::new(bytes).ok_or_else(|| anyhow::anyhow!("size must be greater than zero: {s:?}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
/// Parses a flush/idle interval: a bare number is milliseconds, or an
|
||||||
@@ -148,7 +152,8 @@ pub fn parse_duration(s: &str) -> Result<Duration> {
|
|||||||
.trim()
|
.trim()
|
||||||
.parse()
|
.parse()
|
||||||
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
.map_err(|_| anyhow::anyhow!("invalid interval: {s:?} (try \"100ms\" or \"1s\")"))?;
|
||||||
Ok(Duration::from_millis(n * mult_ms))
|
let ms = n.checked_mul(mult_ms).ok_or_else(|| anyhow::anyhow!("interval too large: {s:?}"))?;
|
||||||
|
Ok(Duration::from_millis(ms))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
/// Single-quote a string for safe inclusion as one argument in a remote shell
|
||||||
@@ -282,19 +287,21 @@ mod tests {
|
|||||||
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
assert_eq!(parse_duration(" 250ms ").unwrap(), Duration::from_millis(250));
|
||||||
// 0 is legal: cut a chunk as soon as anything is kept.
|
// 0 is legal: cut a chunk as soon as anything is kept.
|
||||||
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
assert_eq!(parse_duration("0").unwrap(), Duration::ZERO);
|
||||||
for bad in ["", "abc", "0.1s", "100m", "-5"] {
|
for bad in ["", "abc", "0.1s", "100m", "-5", "18446744073709551615s"] {
|
||||||
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
assert!(parse_duration(bad).is_err(), "{bad:?} should be rejected");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn size_parsing() {
|
fn size_parsing() {
|
||||||
assert_eq!(parse_size("256k").unwrap(), 256 * 1024);
|
assert_eq!(parse_size("256k").unwrap().get(), 256 * 1024);
|
||||||
assert_eq!(parse_size("1m").unwrap(), 1024 * 1024);
|
assert_eq!(parse_size("1m").unwrap().get(), 1024 * 1024);
|
||||||
assert_eq!(parse_size("2M").unwrap(), 2 * 1024 * 1024);
|
assert_eq!(parse_size("2M").unwrap().get(), 2 * 1024 * 1024);
|
||||||
assert_eq!(parse_size("512").unwrap(), 512);
|
assert_eq!(parse_size("512").unwrap().get(), 512);
|
||||||
assert!(parse_size("").is_err());
|
// 0 would cut empty chunks forever; overflow must not wrap.
|
||||||
assert!(parse_size("abc").is_err());
|
for bad in ["", "abc", "0", "0k", "18446744073709551615g"] {
|
||||||
|
assert!(parse_size(bad).is_err(), "{bad:?} should be rejected");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+5
-4
@@ -7,6 +7,7 @@ use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
|||||||
use anyhow::{bail, Result};
|
use anyhow::{bail, Result};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
|
use std::num::NonZeroUsize;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::{mpsc, Arc};
|
use std::sync::{mpsc, Arc};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -23,7 +24,7 @@ pub struct SendOptions {
|
|||||||
pub is_partial: bool,
|
pub is_partial: bool,
|
||||||
pub final_source_path: PathBuf,
|
pub final_source_path: PathBuf,
|
||||||
pub compress: bool,
|
pub compress: bool,
|
||||||
pub chunk_target: usize,
|
pub chunk_target: NonZeroUsize,
|
||||||
/// How long kept-but-undersized output may wait before being sent
|
/// How long kept-but-undersized output may wait before being sent
|
||||||
/// anyway. See `Chunker::should_flush`.
|
/// anyway. See `Chunker::should_flush`.
|
||||||
pub flush_after: Duration,
|
pub flush_after: Duration,
|
||||||
@@ -357,7 +358,7 @@ mod tests {
|
|||||||
is_partial: false,
|
is_partial: false,
|
||||||
final_source_path: src.clone(),
|
final_source_path: src.clone(),
|
||||||
compress: true,
|
compress: true,
|
||||||
chunk_target: 512, // small target to force multiple frames
|
chunk_target: NonZeroUsize::new(512).unwrap(), // small target to force multiple frames
|
||||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||||
filter: PacketFilter::match_all(),
|
filter: PacketFilter::match_all(),
|
||||||
@@ -485,7 +486,7 @@ mod tests {
|
|||||||
is_partial: false,
|
is_partial: false,
|
||||||
final_source_path: src.clone(),
|
final_source_path: src.clone(),
|
||||||
compress: true,
|
compress: true,
|
||||||
chunk_target: 512,
|
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||||
filter: ip_filter(&["1.1.1.1"]),
|
filter: ip_filter(&["1.1.1.1"]),
|
||||||
@@ -580,7 +581,7 @@ mod tests {
|
|||||||
is_partial: true,
|
is_partial: true,
|
||||||
final_source_path: final_src.clone(),
|
final_source_path: final_src.clone(),
|
||||||
compress: true,
|
compress: true,
|
||||||
chunk_target: 512,
|
chunk_target: NonZeroUsize::new(512).unwrap(),
|
||||||
flush_after: crate::chunker::DEFAULT_FLUSH,
|
flush_after: crate::chunker::DEFAULT_FLUSH,
|
||||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||||
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
filter: PacketFilter::parse(&[], &["02:00:00:00:00:aa".to_string()]).unwrap(),
|
||||||
|
|||||||
Reference in new issue
Block a user