use anyhow::{Result, bail}; use std::time::{Duration, Instant}; pub const GLOBAL_HEADER_LEN: usize = 24; pub const RECORD_HEADER_LEN: usize = 16; pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100); #[allow(dead_code)] pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024; /// Sanity bound on a record's declared payload length, per /// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing) /// sources -- see `RecordAlignedChunker::partial`. pub const MAX_INCL_LEN: usize = 262_144; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ChunkKind { Header, Data, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Endianness { Little, Big, } fn detect_endianness(header: &[u8]) -> Result { let magic = u32::from_le_bytes(header[0..4].try_into().unwrap()); match magic { 0xA1B2C3D4 | 0xA1B23C4D => Ok(Endianness::Little), 0xD4C3B2A1 | 0x4D3CB2A1 => Ok(Endianness::Big), _ => bail!("unrecognized pcap global header magic: {magic:#010x}"), } } fn read_u32(bytes: &[u8], e: Endianness) -> u32 { let arr: [u8; 4] = bytes.try_into().unwrap(); match e { Endianness::Little => u32::from_le_bytes(arr), Endianness::Big => u32::from_be_bytes(arr), } } /// Turns a stream of raw source bytes into chunks ready to hand to the wire /// protocol. `RawChunker` is a passthrough for the non-compress, non-partial /// path; `RecordAlignedChunker` cuts on pcap record boundaries for the /// compress path and/or any partial (still-growing) source. pub trait Chunker: Send { fn feed(&mut self, bytes: &[u8]) -> Result<()>; /// Pop a chunk that's ready by size/target. Call repeatedly until `None`. fn ready_chunk(&mut self) -> Option<(ChunkKind, Vec)>; /// Whether a flush-timer-triggered short chunk should be cut right now. fn should_flush(&self, now: Instant) -> bool; /// Force-cut whatever's pending (still respecting record alignment). fn flush(&mut self) -> Option<(ChunkKind, Vec)>; /// Called once, after the source is confirmed fully drained (per the /// caller's own `confirmed_len() == real final length` check -- this no /// longer re-derives that itself). Only errors if a complete global /// header was never read. fn finish(&mut self) -> Result)>>; /// Next source offset the caller should read from to get new bytes via /// `feed`. Monotonically non-decreasing. fn read_from(&self) -> u64; /// Cumulative leading source bytes confirmed to form a complete, /// plausible global header and/or pcap records. Monotonically /// non-decreasing. Equal to `read_from()` for a chunker with no /// "unconfirmed tail" concept. fn confirmed_len(&self) -> u64; /// Re-validating feed for a source that may still be overwriting /// not-yet-confirmed bytes in place (a preallocating writer). `bytes` /// MUST be a fresh disk read starting exactly at `confirmed_len()`; it /// replaces whatever was buffered beyond the last confirmed boundary and /// re-validates from scratch. Only called by tail.rs's partial-source /// path. /// /// `trust_trailing_record`: normally a candidate record is only /// confirmed once the *next* record's header has also been observed and /// looks real (proof the writer moved past it). When the caller is /// certain no more bytes will EVER arrive (rename observed and file /// length stable across repeated checks), it passes `true` to relax /// that requirement for exactly the trailing record: a record that is /// itself plausible AND is followed by nothing at all gets confirmed /// without needing a next-record proof, since there provably isn't one. /// /// Default delegates to `feed` -- correct for a chunker with no /// unconfirmed-tail/next-record concept (RawChunker). fn refresh_tail(&mut self, offset: u64, bytes: &[u8], trust_trailing_record: bool) -> Result<()> { let _ = (offset, trust_trailing_record); self.feed(bytes) } } #[derive(Default)] pub struct RawChunker { pending: Vec, total_fed: u64, } impl RawChunker { pub fn new() -> Self { Self::default() } } impl Chunker for RawChunker { fn feed(&mut self, bytes: &[u8]) -> Result<()> { self.pending.extend_from_slice(bytes); self.total_fed += bytes.len() as u64; Ok(()) } fn ready_chunk(&mut self) -> Option<(ChunkKind, Vec)> { if self.pending.is_empty() { None } else { Some((ChunkKind::Data, std::mem::take(&mut self.pending))) } } fn should_flush(&self, _now: Instant) -> bool { false } fn flush(&mut self) -> Option<(ChunkKind, Vec)> { None } fn finish(&mut self) -> Result)>> { Ok(self.ready_chunk()) } fn read_from(&self) -> u64 { self.total_fed } fn confirmed_len(&self) -> u64 { self.total_fed } } pub struct RecordAlignedChunker { header_buf: Vec, header_done: bool, header_ready: bool, endianness: Option, snaplen: u32, pending: Vec, complete_len: usize, emitted_len: u64, last_growth: Instant, target: usize, flush_after: Duration, /// Whether this source is a still-growing (partial) file. Gates every /// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len /// bound, next-record proof requirement) -- a non-partial source is /// stable and final, so none of these preallocation-writer defenses are /// needed or safe to apply (they could reject genuinely final content). partial: bool, } impl RecordAlignedChunker { pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self { Self { header_buf: Vec::with_capacity(GLOBAL_HEADER_LEN), header_done: false, header_ready: false, endianness: None, snaplen: 0, pending: Vec::new(), complete_len: 0, emitted_len: 0, last_growth: Instant::now(), target, flush_after, partial, } } fn max_incl_len(&self) -> usize { (self.snaplen as usize).max(MAX_INCL_LEN) } fn validate_header(&mut self) -> Result<()> { // On a partial source, a header that hasn't been written yet reads // back as zero bytes if the writer preallocated the file before // writing into it -- treat that as "not yet", not a hard error. if self.partial && self.header_buf.iter().all(|&b| b == 0) { return Ok(()); } let e = detect_endianness(&self.header_buf)?; self.snaplen = read_u32(&self.header_buf[16..20], e); self.endianness = Some(e); self.header_done = true; self.header_ready = true; Ok(()) } fn rescan(&mut self, trust_trailing_record: bool) { let e = match self.endianness { Some(e) => e, None => return, }; loop { let remaining = &self.pending[self.complete_len..]; if remaining.len() < RECORD_HEADER_LEN { break; } let ts_sec = read_u32(&remaining[0..4], e); let incl_len = read_u32(&remaining[8..12], e) as usize; let record_total = RECORD_HEADER_LEN + incl_len; if remaining.len() < record_total { break; } if self.partial { // Heuristic: a live capture's timestamp is never the Unix // epoch. An all-zero-looking ts_sec is essentially // certainly unwritten/preallocated disk space, not a real // record -- treat it exactly like "not enough bytes yet" // (pause, no error), so a later re-peek can pick up real // content once the writer's cursor reaches this offset. if ts_sec == 0 { break; } if incl_len > self.max_incl_len() { break; } // Never trust a record's own header alone: a writer need // not flush header and payload atomically, so a // fully-plausible header can still be sitting in front of a // payload that's still mid-write (or still zero padding). // Require positive proof the writer has moved past this // record -- the start of the NEXT record's header, itself // looking real -- before confirming it. This makes // complete_len always sit (at least) one record behind // whatever's currently visible, by construction, for every // record, not just "the last one" as a special case. let after = &remaining[record_total..]; let next_is_proven = after.len() >= RECORD_HEADER_LEN && read_u32(&after[0..4], e) != 0 && (read_u32(&after[8..12], e) as usize) <= self.max_incl_len(); // Escape hatch (only when the caller is certain nothing // more will EVER arrive): a record that is itself plausible // AND has truly nothing after it (current_len ends exactly // here) is confirmed without a next-record proof, since // there provably isn't a next record to prove it with. Any // other leftover (a nonempty `after` shorter than a full // header, or one that fails its own plausibility check) // still does NOT confirm -- that's a genuinely torn/corrupt // tail, correctly left unconfirmed so tail.rs's finalize // check reports it as such. let trailing_and_trusted = trust_trailing_record && after.is_empty(); if !next_is_proven && !trailing_and_trusted { break; } } self.complete_len += record_total; } } fn drain_complete(&mut self) -> (ChunkKind, Vec) { let chunk: Vec = self.pending.drain(..self.complete_len).collect(); self.emitted_len += self.complete_len as u64; self.complete_len = 0; (ChunkKind::Data, chunk) } } impl Chunker for RecordAlignedChunker { fn feed(&mut self, bytes: &[u8]) -> Result<()> { if bytes.is_empty() { return Ok(()); } let mut rest = bytes; if !self.header_done { let need = GLOBAL_HEADER_LEN - self.header_buf.len(); let take = need.min(rest.len()); self.header_buf.extend_from_slice(&rest[..take]); rest = &rest[take..]; if self.header_buf.len() == GLOBAL_HEADER_LEN { self.validate_header()?; if !self.header_done { // Partial source, all-zero header -- not written yet. return Ok(()); } } else { // Still accumulating the header; nothing else to do. return Ok(()); } } if !rest.is_empty() { self.pending.extend_from_slice(rest); self.rescan(false); self.last_growth = Instant::now(); } Ok(()) } fn ready_chunk(&mut self) -> Option<(ChunkKind, Vec)> { if self.header_ready { self.header_ready = false; return Some((ChunkKind::Header, std::mem::take(&mut self.header_buf))); } if self.complete_len >= self.target { return Some(self.drain_complete()); } None } fn should_flush(&self, now: Instant) -> bool { self.complete_len > 0 && now.duration_since(self.last_growth) >= self.flush_after } fn flush(&mut self) -> Option<(ChunkKind, Vec)> { if self.complete_len > 0 { Some(self.drain_complete()) } else { None } } fn finish(&mut self) -> Result)>> { if !self.header_done { bail!("source ended before a complete pcap global header was read"); } if self.complete_len > 0 { Ok(Some(self.drain_complete())) } else { Ok(None) } } fn read_from(&self) -> u64 { if !self.header_done { self.header_buf.len() as u64 } else { GLOBAL_HEADER_LEN as u64 + self.emitted_len + self.pending.len() as u64 } } fn confirmed_len(&self) -> u64 { if !self.header_done { // While not yet validated, nothing is confirmed -- even if // header_buf is "full" but stuck on an all-zero (partial-only) // header. This must be 0, not header_buf.len(), so tail.rs's // partial-branch read anchor re-reads offset 0 instead of // treating the stuck zero bytes as already consumed. 0 } else { GLOBAL_HEADER_LEN as u64 + self.emitted_len + self.complete_len as u64 } } fn refresh_tail(&mut self, offset: u64, bytes: &[u8], trust_trailing_record: bool) -> Result<()> { if offset != self.confirmed_len() { bail!( "chunker refresh_tail offset mismatch: got {offset}, expected {}", self.confirmed_len() ); } if !self.header_done { if bytes.len() < GLOBAL_HEADER_LEN { self.header_buf = bytes.to_vec(); return Ok(()); } self.header_buf = bytes[..GLOBAL_HEADER_LEN].to_vec(); self.validate_header()?; if !self.header_done { return Ok(()); } // header_done just flipped true, so complete_len is still 0 here // (nothing beyond the header has been considered yet) -- // pending is simply the fresh post-header bytes. self.pending = bytes[GLOBAL_HEADER_LEN..].to_vec(); self.rescan(trust_trailing_record); self.last_growth = Instant::now(); return Ok(()); } // `pending[..complete_len]` is CONFIRMED content that just hasn't // been drained (emitted) yet -- e.g. because it's still under // `target`. `bytes` starts exactly at `confirmed_len()` (checked // above), i.e. exactly where that confirmed prefix ends, so it must // be appended after it, never used to replace the whole buffer -- // doing that would silently discard already-confirmed, already // real, not-yet-emitted content on every single re-peek, which is // catastrophic for a long-running growing transfer (each poll would // re-derive from scratch instead of building on prior confirmed // progress, and once the fresh read window is too small to // re-establish the same progress alone -- e.g. capped by // READ_WINDOW -- confirmed_len can never recover). self.pending.truncate(self.complete_len); self.pending.extend_from_slice(bytes); self.rescan(trust_trailing_record); self.last_growth = Instant::now(); Ok(()) } } #[cfg(test)] mod tests { use super::*; fn header_bytes(magic_le: u32) -> Vec { let mut h = vec![0u8; GLOBAL_HEADER_LEN]; h[0..4].copy_from_slice(&magic_le.to_le_bytes()); h } fn record_bytes(payload: &[u8], big_endian: bool) -> Vec { record_bytes_with_ts(payload, big_endian, 0) } fn record_bytes_with_ts(payload: &[u8], big_endian: bool, ts_sec: u32) -> Vec { let mut r = Vec::with_capacity(RECORD_HEADER_LEN + payload.len()); if big_endian { r.extend_from_slice(&ts_sec.to_be_bytes()); } else { r.extend_from_slice(&ts_sec.to_le_bytes()); } r.extend_from_slice(&[0u8; 4]); // ts_usec let incl_len = payload.len() as u32; if big_endian { r.extend_from_slice(&incl_len.to_be_bytes()); } else { r.extend_from_slice(&incl_len.to_le_bytes()); } r.extend_from_slice(&incl_len.to_le_bytes()); // orig_len, value doesn't matter r.extend_from_slice(payload); r } #[test] fn raw_chunker_passthrough() { let mut c = RawChunker::new(); c.feed(b"abc").unwrap(); c.feed(b"def").unwrap(); let (kind, data) = c.ready_chunk().unwrap(); assert_eq!(kind, ChunkKind::Data); assert_eq!(data, b"abcdef"); assert!(c.ready_chunk().is_none()); } #[test] fn header_emitted_as_its_own_chunk() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); let (kind, data) = c.ready_chunk().unwrap(); assert_eq!(kind, ChunkKind::Header); assert_eq!(data.len(), GLOBAL_HEADER_LEN); assert!(c.ready_chunk().is_none()); } #[test] fn insufficient_header_bytes_is_not_an_error() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false); // Only 10 bytes so far -- not enough to validate the magic. c.feed(&[0u8; 10]).unwrap(); assert!(c.ready_chunk().is_none()); } #[test] fn bad_magic_is_an_error() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false); let mut bad = vec![0xFFu8; GLOBAL_HEADER_LEN]; bad[0..4].copy_from_slice(&0xDEADBEEFu32.to_le_bytes()); assert!(c.feed(&bad).is_err()); } #[test] fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() { let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts let mut stream = header_bytes(0xA1B2C3D4); let mut expected_records: Vec> = Vec::new(); for i in 0..20u8 { let rec = record_bytes(&[i; 10], false); expected_records.push(rec.clone()); stream.extend_from_slice(&rec); } // Feed in arbitrary tiny increments (1-3 bytes at a time). let mut collected: Vec = Vec::new(); let mut i = 0; let mut step = 1usize; while i < stream.len() { let n = step.min(stream.len() - i); c.feed(&stream[i..i + n]).unwrap(); i += n; step = (step % 3) + 1; while let Some((_, data)) = c.ready_chunk() { collected.extend_from_slice(&data); } } if let Some((_, data)) = c.finish().unwrap() { collected.extend_from_slice(&data); } // Reassembled bytes must equal the original stream, and every // intermediate chunk boundary must have landed on a whole record -- // verified implicitly since decoding never errored and the // concatenation is byte-exact. assert_eq!(collected, stream); } #[test] fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() { let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); // drain header // A single record whose payload alone overshoots the tiny target. let rec = record_bytes(&[1u8; 40], false); c.feed(&rec).unwrap(); let (kind, data) = c.ready_chunk().unwrap(); assert_eq!(kind, ChunkKind::Data); assert_eq!(data, rec); // whole record, even though it overshoots target=15 } #[test] fn flush_predicate_and_flush_are_time_independent_of_sleep() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, Duration::from_millis(100), false); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); let rec = record_bytes(&[9u8; 5], false); c.feed(&rec).unwrap(); let just_now = Instant::now(); assert!(!c.should_flush(just_now)); let later = just_now + Duration::from_millis(150); assert!(c.should_flush(later)); let (kind, data) = c.flush().unwrap(); assert_eq!(kind, ChunkKind::Data); assert_eq!(data, rec); assert!(c.flush().is_none()); } #[test] fn torn_trailing_record_is_left_unconfirmed_not_an_error() { // finish()'s job narrowed to "was a header ever read" -- detecting a // genuinely unresolvable torn tail at true EOF is tail.rs's job now // (it has the file-length/rename context needed to tell "still // resolving" from "will never resolve"). See tail.rs's // `non_partial_torn_trailing_record_is_an_error` and // `torn_trailing_record_after_rename_is_an_error_not_a_hang`. let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); let rec = record_bytes(&[1u8; 20], false); // Feed everything except the last 3 bytes of the record. c.feed(&rec[..rec.len() - 3]).unwrap(); assert_eq!(c.finish().unwrap(), None); } #[test] fn big_endian_magic_reads_incl_len_as_big_endian() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false); c.feed(&header_bytes(0xD4C3B2A1)).unwrap(); c.ready_chunk(); let rec = record_bytes(&[1u8; 30], true); c.feed(&rec).unwrap(); let (_, data) = c.finish().unwrap().unwrap(); assert_eq!(data, rec); } #[test] fn finish_before_any_header_bytes_is_an_error() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false); assert!(c.finish().is_err()); } #[test] fn all_zero_ts_sec_is_treated_as_not_yet_written() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000); let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_001); c.feed(&rec1).unwrap(); c.feed(&rec2).unwrap(); // rec1 confirms (proven by rec2's real header); rec2 itself is held // back (nothing proves it yet). assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64); // Now simulate trailing preallocated padding: a large all-zero // block. confirmed_len must not advance into it. let padding = vec![0u8; 4096]; c.feed(&padding).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64); } #[test] fn refresh_tail_reexamines_previously_zero_region_once_it_fills_in() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64); let zero_block = vec![0u8; 64]; c.refresh_tail(c.confirmed_len(), &zero_block, false).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64); // Writer has since filled this offset in with two real records. let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000); let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_001); let mut real = rec1.clone(); real.extend_from_slice(&rec2); c.refresh_tail(c.confirmed_len(), &real, false).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64); } #[test] fn all_zero_global_header_is_treated_as_not_yet_written() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); let zero_header = vec![0u8; GLOBAL_HEADER_LEN]; c.refresh_tail(0, &zero_header, false).unwrap(); assert_eq!(c.confirmed_len(), 0); c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64); } #[test] fn nonzero_bad_magic_is_still_a_hard_error_even_when_partial() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); let mut bad = vec![0xFFu8; GLOBAL_HEADER_LEN]; bad[0..4].copy_from_slice(&0xDEADBEEFu32.to_le_bytes()); assert!(c.feed(&bad).is_err()); } #[test] fn incl_len_exceeding_bound_is_rejected() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); // incl_len far exceeds MAX_INCL_LEN, but we only need the header's // 16 bytes physically present for the check to fire. let mut rec = vec![0u8; RECORD_HEADER_LEN]; rec[0..4].copy_from_slice(&1_700_000_000u32.to_le_bytes()); rec[8..12].copy_from_slice(&(MAX_INCL_LEN as u32 + 1).to_le_bytes()); c.feed(&rec).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64); } #[test] fn record_not_confirmed_without_proof_of_next_record() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); let rec = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000); c.feed(&rec).unwrap(); // A single fully-present, individually-plausible record with // nothing after it: ordinary feed() (trust=false) must not confirm // it -- no proof yet that the writer has moved past it. assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64); } #[test] fn record_confirmed_once_next_record_header_is_real() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000); let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_001); c.feed(&rec1).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64); c.feed(&rec2).unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64); } #[test] fn torn_payload_followed_by_zero_padding_is_not_confirmed() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.feed(&header_bytes(0xA1B2C3D4)).unwrap(); c.ready_chunk(); // A record whose own header looks plausible (real ts_sec, sane // incl_len) but whose declared payload reaches into all-zero bytes. let rec = record_bytes_with_ts(&[0u8; 10], false, 1_700_000_000); c.feed(&rec).unwrap(); c.feed(&[0u8; 64]).unwrap(); // more zero padding after it assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64); // Real content lands there instead. let rec2 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_001); // Replace: simulate the writer having overwritten what followed rec // by feeding a fresh, corrected view via refresh_tail. c.refresh_tail(c.confirmed_len(), &[&rec[..], &rec2[..]].concat(), false) .unwrap(); assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec.len() as u64); } #[test] fn trailing_record_confirmed_only_when_trust_granted_and_truly_last() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap(); let rec = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000); let offset = c.confirmed_len(); c.refresh_tail(offset, &rec, false).unwrap(); assert_eq!(c.confirmed_len(), offset, "must not confirm without trust"); c.refresh_tail(offset, &rec, true).unwrap(); assert_eq!(c.confirmed_len(), offset + rec.len() as u64, "must confirm once trusted"); } #[test] fn trust_does_not_confirm_a_genuinely_torn_tail() { let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap(); let rec = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000); let mut bytes = rec.clone(); // Nonempty garbage after the record -- not a full header, not empty. bytes.extend_from_slice(&[0xAB, 0xCD, 0xEF]); let offset = c.confirmed_len(); c.refresh_tail(offset, &bytes, true).unwrap(); assert_eq!(c.confirmed_len(), offset, "trust must not rescue a nonempty torn fragment"); } #[test] fn refresh_tail_never_discards_already_confirmed_undrained_content() { // Regression test for a real production incident: refresh_tail used // to unconditionally replace `pending` with the fresh read, which // silently discarded already-confirmed-but-not-yet-drained content // (anything below `target`, i.e. never emitted via ready_chunk) any // time a LATER refresh_tail call's fresh window started past it and // didn't happen to re-derive the same progress on its own. This // caused confirmed_len to oscillate or, worse, get permanently // stuck once the fresh window was too small (e.g. capped by // READ_WINDOW in tail.rs) to ever re-establish the lost progress. // // Use a target far bigger than the data so nothing ever drains via // ready_chunk -- everything must survive purely via the confirmed // prefix being preserved across refresh_tail calls. let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true); c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap(); let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_001); let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_002); let rec3 = record_bytes_with_ts(&[3u8; 10], false, 1_700_000_003); // First call: feed rec1+rec2 together. rec1 confirms (proven by // rec2's header); rec2 itself is held back (nothing proves it yet). let offset = c.confirmed_len(); let mut first_batch = rec1.clone(); first_batch.extend_from_slice(&rec2); c.refresh_tail(offset, &first_batch, false).unwrap(); assert_eq!( c.confirmed_len(), offset + rec1.len() as u64, "rec1 should confirm, proven by rec2's header" ); let after_first = c.confirmed_len(); // Second call: a FRESH read starting exactly at confirmed_len() // (per the trait contract) -- this does NOT include rec1, which is // already confirmed but still sitting undrained in `pending`. Only // rec2's own bytes (already seen) plus rec3 (new) are read this // time, simulating a subsequent, narrower re-peek. let mut second_batch = rec2.clone(); second_batch.extend_from_slice(&rec3); c.refresh_tail(after_first, &second_batch, false).unwrap(); // rec1's confirmation must NOT have been discarded, and rec2 should // now also confirm (proven by rec3's header). assert_eq!( c.confirmed_len(), offset + rec1.len() as u64 + rec2.len() as u64, "confirmed_len must never regress below what was already established" ); } }