scpcap/src/chunker.rs

use anyhow::{Result, bail};
use std::time::{Duration, Instant};

pub const GLOBAL_HEADER_LEN: usize = 24;
pub const RECORD_HEADER_LEN: usize = 16;
pub const DEFAULT_FLUSH: Duration = Duration::from_millis(100);
#[allow(dead_code)]
pub const DEFAULT_TARGET_BYTES: usize = 256 * 1024;
/// Sanity bound on a record's declared payload length, per
/// STREAMING_ZSTD_FORMAT.md §8/§9. Only enforced for partial (still-growing)
/// sources -- see `RecordAlignedChunker::partial`.
pub const MAX_INCL_LEN: usize = 262_144;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChunkKind {
    Header,
    Data,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Endianness {
    Little,
    Big,
}

fn detect_endianness(header: &[u8]) -> Result<Endianness> {
    let magic = u32::from_le_bytes(header[0..4].try_into().unwrap());
    match magic {
        0xA1B2C3D4 | 0xA1B23C4D => Ok(Endianness::Little),
        0xD4C3B2A1 | 0x4D3CB2A1 => Ok(Endianness::Big),
        _ => bail!("unrecognized pcap global header magic: {magic:#010x}"),
    }
}

fn read_u32(bytes: &[u8], e: Endianness) -> u32 {
    let arr: [u8; 4] = bytes.try_into().unwrap();
    match e {
        Endianness::Little => u32::from_le_bytes(arr),
        Endianness::Big => u32::from_be_bytes(arr),
    }
}

/// Turns a stream of raw source bytes into chunks ready to hand to the wire
/// protocol. `RawChunker` is a passthrough for the non-compress, non-partial
/// path; `RecordAlignedChunker` cuts on pcap record boundaries for the
/// compress path and/or any partial (still-growing) source.
pub trait Chunker: Send {
    fn feed(&mut self, bytes: &[u8]) -> Result<()>;
    /// Pop a chunk that's ready by size/target. Call repeatedly until `None`.
    fn ready_chunk(&mut self) -> Option<(ChunkKind, Vec<u8>)>;
    /// Whether a flush-timer-triggered short chunk should be cut right now.
    fn should_flush(&self, now: Instant) -> bool;
    /// Force-cut whatever's pending (still respecting record alignment).
    fn flush(&mut self) -> Option<(ChunkKind, Vec<u8>)>;
    /// Called once, after the source is confirmed fully drained (per the
    /// caller's own `confirmed_len() == real final length` check -- this no
    /// longer re-derives that itself). Only errors if a complete global
    /// header was never read.
    fn finish(&mut self) -> Result<Option<(ChunkKind, Vec<u8>)>>;

    /// Next source offset the caller should read from to get new bytes via
    /// `feed`. Monotonically non-decreasing.
    fn read_from(&self) -> u64;

    /// Cumulative leading source bytes confirmed to form a complete,
    /// plausible global header and/or pcap records. Monotonically
    /// non-decreasing. Equal to `read_from()` for a chunker with no
    /// "unconfirmed tail" concept.
    fn confirmed_len(&self) -> u64;

    /// Re-validating feed for a source that may still be overwriting
    /// not-yet-confirmed bytes in place (a preallocating writer). `bytes`
    /// MUST be a fresh disk read starting exactly at `confirmed_len()`; it
    /// replaces whatever was buffered beyond the last confirmed boundary and
    /// re-validates from scratch. Only called by tail.rs's partial-source
    /// path.
    ///
    /// `trust_trailing_record`: normally a candidate record is only
    /// confirmed once the *next* record's header has also been observed and
    /// looks real (proof the writer moved past it). When the caller is
    /// certain no more bytes will EVER arrive (rename observed and file
    /// length stable across repeated checks), it passes `true` to relax
    /// that requirement for exactly the trailing record: a record that is
    /// itself plausible AND is followed by nothing at all gets confirmed
    /// without needing a next-record proof, since there provably isn't one.
    ///
    /// Default delegates to `feed` -- correct for a chunker with no
    /// unconfirmed-tail/next-record concept (RawChunker).
    fn refresh_tail(&mut self, offset: u64, bytes: &[u8], trust_trailing_record: bool) -> Result<()> {
        let _ = (offset, trust_trailing_record);
        self.feed(bytes)
    }
}

#[derive(Default)]
pub struct RawChunker {
    pending: Vec<u8>,
    total_fed: u64,
}

impl RawChunker {
    pub fn new() -> Self {
        Self::default()
    }
}

impl Chunker for RawChunker {
    fn feed(&mut self, bytes: &[u8]) -> Result<()> {
        self.pending.extend_from_slice(bytes);
        self.total_fed += bytes.len() as u64;
        Ok(())
    }

    fn ready_chunk(&mut self) -> Option<(ChunkKind, Vec<u8>)> {
        if self.pending.is_empty() {
            None
        } else {
            Some((ChunkKind::Data, std::mem::take(&mut self.pending)))
        }
    }

    fn should_flush(&self, _now: Instant) -> bool {
        false
    }

    fn flush(&mut self) -> Option<(ChunkKind, Vec<u8>)> {
        None
    }

    fn finish(&mut self) -> Result<Option<(ChunkKind, Vec<u8>)>> {
        Ok(self.ready_chunk())
    }

    fn read_from(&self) -> u64 {
        self.total_fed
    }

    fn confirmed_len(&self) -> u64 {
        self.total_fed
    }
}

pub struct RecordAlignedChunker {
    header_buf: Vec<u8>,
    header_done: bool,
    header_ready: bool,
    endianness: Option<Endianness>,
    snaplen: u32,
    pending: Vec<u8>,
    complete_len: usize,
    emitted_len: u64,
    last_growth: Instant,
    target: usize,
    flush_after: Duration,
    /// Whether this source is a still-growing (partial) file. Gates every
    /// heuristic below (zero-header wait, zero-ts_sec rejection, incl_len
    /// bound, next-record proof requirement) -- a non-partial source is
    /// stable and final, so none of these preallocation-writer defenses are
    /// needed or safe to apply (they could reject genuinely final content).
    partial: bool,
}

impl RecordAlignedChunker {
    pub fn new(target: usize, flush_after: Duration, partial: bool) -> Self {
        Self {
            header_buf: Vec::with_capacity(GLOBAL_HEADER_LEN),
            header_done: false,
            header_ready: false,
            endianness: None,
            snaplen: 0,
            pending: Vec::new(),
            complete_len: 0,
            emitted_len: 0,
            last_growth: Instant::now(),
            target,
            flush_after,
            partial,
        }
    }

    fn max_incl_len(&self) -> usize {
        (self.snaplen as usize).max(MAX_INCL_LEN)
    }

    fn validate_header(&mut self) -> Result<()> {
        // On a partial source, a header that hasn't been written yet reads
        // back as zero bytes if the writer preallocated the file before
        // writing into it -- treat that as "not yet", not a hard error.
        if self.partial && self.header_buf.iter().all(|&b| b == 0) {
            return Ok(());
        }
        let e = detect_endianness(&self.header_buf)?;
        self.snaplen = read_u32(&self.header_buf[16..20], e);
        self.endianness = Some(e);
        self.header_done = true;
        self.header_ready = true;
        Ok(())
    }

    fn rescan(&mut self, trust_trailing_record: bool) {
        let e = match self.endianness {
            Some(e) => e,
            None => return,
        };
        loop {
            let remaining = &self.pending[self.complete_len..];
            if remaining.len() < RECORD_HEADER_LEN {
                break;
            }
            let ts_sec = read_u32(&remaining[0..4], e);
            let incl_len = read_u32(&remaining[8..12], e) as usize;
            let record_total = RECORD_HEADER_LEN + incl_len;
            if remaining.len() < record_total {
                break;
            }

            if self.partial {
                // Heuristic: a live capture's timestamp is never the Unix
                // epoch. An all-zero-looking ts_sec is essentially
                // certainly unwritten/preallocated disk space, not a real
                // record -- treat it exactly like "not enough bytes yet"
                // (pause, no error), so a later re-peek can pick up real
                // content once the writer's cursor reaches this offset.
                if ts_sec == 0 {
                    break;
                }
                if incl_len > self.max_incl_len() {
                    break;
                }

                // Never trust a record's own header alone: a writer need
                // not flush header and payload atomically, so a
                // fully-plausible header can still be sitting in front of a
                // payload that's still mid-write (or still zero padding).
                // Require positive proof the writer has moved past this
                // record -- the start of the NEXT record's header, itself
                // looking real -- before confirming it. This makes
                // complete_len always sit (at least) one record behind
                // whatever's currently visible, by construction, for every
                // record, not just "the last one" as a special case.
                let after = &remaining[record_total..];
                let next_is_proven = after.len() >= RECORD_HEADER_LEN
                    && read_u32(&after[0..4], e) != 0
                    && (read_u32(&after[8..12], e) as usize) <= self.max_incl_len();

                // Escape hatch (only when the caller is certain nothing
                // more will EVER arrive): a record that is itself plausible
                // AND has truly nothing after it (current_len ends exactly
                // here) is confirmed without a next-record proof, since
                // there provably isn't a next record to prove it with. Any
                // other leftover (a nonempty `after` shorter than a full
                // header, or one that fails its own plausibility check)
                // still does NOT confirm -- that's a genuinely torn/corrupt
                // tail, correctly left unconfirmed so tail.rs's finalize
                // check reports it as such.
                let trailing_and_trusted = trust_trailing_record && after.is_empty();

                if !next_is_proven && !trailing_and_trusted {
                    break;
                }
            }
            self.complete_len += record_total;
        }
    }

    fn drain_complete(&mut self) -> (ChunkKind, Vec<u8>) {
        let chunk: Vec<u8> = self.pending.drain(..self.complete_len).collect();
        self.emitted_len += self.complete_len as u64;
        self.complete_len = 0;
        (ChunkKind::Data, chunk)
    }
}

impl Chunker for RecordAlignedChunker {
    fn feed(&mut self, bytes: &[u8]) -> Result<()> {
        if bytes.is_empty() {
            return Ok(());
        }
        let mut rest = bytes;
        if !self.header_done {
            let need = GLOBAL_HEADER_LEN - self.header_buf.len();
            let take = need.min(rest.len());
            self.header_buf.extend_from_slice(&rest[..take]);
            rest = &rest[take..];
            if self.header_buf.len() == GLOBAL_HEADER_LEN {
                self.validate_header()?;
                if !self.header_done {
                    // Partial source, all-zero header -- not written yet.
                    return Ok(());
                }
            } else {
                // Still accumulating the header; nothing else to do.
                return Ok(());
            }
        }
        if !rest.is_empty() {
            self.pending.extend_from_slice(rest);
            self.rescan(false);
            self.last_growth = Instant::now();
        }
        Ok(())
    }

    fn ready_chunk(&mut self) -> Option<(ChunkKind, Vec<u8>)> {
        if self.header_ready {
            self.header_ready = false;
            return Some((ChunkKind::Header, std::mem::take(&mut self.header_buf)));
        }
        if self.complete_len >= self.target {
            return Some(self.drain_complete());
        }
        None
    }

    fn should_flush(&self, now: Instant) -> bool {
        self.complete_len > 0 && now.duration_since(self.last_growth) >= self.flush_after
    }

    fn flush(&mut self) -> Option<(ChunkKind, Vec<u8>)> {
        if self.complete_len > 0 {
            Some(self.drain_complete())
        } else {
            None
        }
    }

    fn finish(&mut self) -> Result<Option<(ChunkKind, Vec<u8>)>> {
        if !self.header_done {
            bail!("source ended before a complete pcap global header was read");
        }
        if self.complete_len > 0 {
            Ok(Some(self.drain_complete()))
        } else {
            Ok(None)
        }
    }

    fn read_from(&self) -> u64 {
        if !self.header_done {
            self.header_buf.len() as u64
        } else {
            GLOBAL_HEADER_LEN as u64 + self.emitted_len + self.pending.len() as u64
        }
    }

    fn confirmed_len(&self) -> u64 {
        if !self.header_done {
            // While not yet validated, nothing is confirmed -- even if
            // header_buf is "full" but stuck on an all-zero (partial-only)
            // header. This must be 0, not header_buf.len(), so tail.rs's
            // partial-branch read anchor re-reads offset 0 instead of
            // treating the stuck zero bytes as already consumed.
            0
        } else {
            GLOBAL_HEADER_LEN as u64 + self.emitted_len + self.complete_len as u64
        }
    }

    fn refresh_tail(&mut self, offset: u64, bytes: &[u8], trust_trailing_record: bool) -> Result<()> {
        if offset != self.confirmed_len() {
            bail!(
                "chunker refresh_tail offset mismatch: got {offset}, expected {}",
                self.confirmed_len()
            );
        }
        if !self.header_done {
            if bytes.len() < GLOBAL_HEADER_LEN {
                self.header_buf = bytes.to_vec();
                return Ok(());
            }
            self.header_buf = bytes[..GLOBAL_HEADER_LEN].to_vec();
            self.validate_header()?;
            if !self.header_done {
                return Ok(());
            }
            // header_done just flipped true, so complete_len is still 0 here
            // (nothing beyond the header has been considered yet) --
            // pending is simply the fresh post-header bytes.
            self.pending = bytes[GLOBAL_HEADER_LEN..].to_vec();
            self.rescan(trust_trailing_record);
            self.last_growth = Instant::now();
            return Ok(());
        }
        // `pending[..complete_len]` is CONFIRMED content that just hasn't
        // been drained (emitted) yet -- e.g. because it's still under
        // `target`. `bytes` starts exactly at `confirmed_len()` (checked
        // above), i.e. exactly where that confirmed prefix ends, so it must
        // be appended after it, never used to replace the whole buffer --
        // doing that would silently discard already-confirmed, already
        // real, not-yet-emitted content on every single re-peek, which is
        // catastrophic for a long-running growing transfer (each poll would
        // re-derive from scratch instead of building on prior confirmed
        // progress, and once the fresh read window is too small to
        // re-establish the same progress alone -- e.g. capped by
        // READ_WINDOW -- confirmed_len can never recover).
        self.pending.truncate(self.complete_len);
        self.pending.extend_from_slice(bytes);
        self.rescan(trust_trailing_record);
        self.last_growth = Instant::now();
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn header_bytes(magic_le: u32) -> Vec<u8> {
        let mut h = vec![0u8; GLOBAL_HEADER_LEN];
        h[0..4].copy_from_slice(&magic_le.to_le_bytes());
        h
    }

    fn record_bytes(payload: &[u8], big_endian: bool) -> Vec<u8> {
        record_bytes_with_ts(payload, big_endian, 0)
    }

    fn record_bytes_with_ts(payload: &[u8], big_endian: bool, ts_sec: u32) -> Vec<u8> {
        let mut r = Vec::with_capacity(RECORD_HEADER_LEN + payload.len());
        if big_endian {
            r.extend_from_slice(&ts_sec.to_be_bytes());
        } else {
            r.extend_from_slice(&ts_sec.to_le_bytes());
        }
        r.extend_from_slice(&[0u8; 4]); // ts_usec
        let incl_len = payload.len() as u32;
        if big_endian {
            r.extend_from_slice(&incl_len.to_be_bytes());
        } else {
            r.extend_from_slice(&incl_len.to_le_bytes());
        }
        r.extend_from_slice(&incl_len.to_le_bytes()); // orig_len, value doesn't matter
        r.extend_from_slice(payload);
        r
    }

    #[test]
    fn raw_chunker_passthrough() {
        let mut c = RawChunker::new();
        c.feed(b"abc").unwrap();
        c.feed(b"def").unwrap();
        let (kind, data) = c.ready_chunk().unwrap();
        assert_eq!(kind, ChunkKind::Data);
        assert_eq!(data, b"abcdef");
        assert!(c.ready_chunk().is_none());
    }

    #[test]
    fn header_emitted_as_its_own_chunk() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        let (kind, data) = c.ready_chunk().unwrap();
        assert_eq!(kind, ChunkKind::Header);
        assert_eq!(data.len(), GLOBAL_HEADER_LEN);
        assert!(c.ready_chunk().is_none());
    }

    #[test]
    fn insufficient_header_bytes_is_not_an_error() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false);
        // Only 10 bytes so far -- not enough to validate the magic.
        c.feed(&[0u8; 10]).unwrap();
        assert!(c.ready_chunk().is_none());
    }

    #[test]
    fn bad_magic_is_an_error() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false);
        let mut bad = vec![0xFFu8; GLOBAL_HEADER_LEN];
        bad[0..4].copy_from_slice(&0xDEADBEEFu32.to_le_bytes());
        assert!(c.feed(&bad).is_err());
    }

    #[test]
    fn chunks_always_end_on_record_boundary_when_fed_in_small_increments() {
        let mut c = RecordAlignedChunker::new(50, DEFAULT_FLUSH, false); // tiny target to force cuts
        let mut stream = header_bytes(0xA1B2C3D4);
        let mut expected_records: Vec<Vec<u8>> = Vec::new();
        for i in 0..20u8 {
            let rec = record_bytes(&[i; 10], false);
            expected_records.push(rec.clone());
            stream.extend_from_slice(&rec);
        }

        // Feed in arbitrary tiny increments (1-3 bytes at a time).
        let mut collected: Vec<u8> = Vec::new();
        let mut i = 0;
        let mut step = 1usize;
        while i < stream.len() {
            let n = step.min(stream.len() - i);
            c.feed(&stream[i..i + n]).unwrap();
            i += n;
            step = (step % 3) + 1;
            while let Some((_, data)) = c.ready_chunk() {
                collected.extend_from_slice(&data);
            }
        }
        if let Some((_, data)) = c.finish().unwrap() {
            collected.extend_from_slice(&data);
        }

        // Reassembled bytes must equal the original stream, and every
        // intermediate chunk boundary must have landed on a whole record --
        // verified implicitly since decoding never errored and the
        // concatenation is byte-exact.
        assert_eq!(collected, stream);
    }

    #[test]
    fn target_is_a_floor_not_a_ceiling_cuts_at_next_boundary() {
        let mut c = RecordAlignedChunker::new(15, DEFAULT_FLUSH, false);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk(); // drain header

        // A single record whose payload alone overshoots the tiny target.
        let rec = record_bytes(&[1u8; 40], false);
        c.feed(&rec).unwrap();
        let (kind, data) = c.ready_chunk().unwrap();
        assert_eq!(kind, ChunkKind::Data);
        assert_eq!(data, rec); // whole record, even though it overshoots target=15
    }

    #[test]
    fn flush_predicate_and_flush_are_time_independent_of_sleep() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, Duration::from_millis(100), false);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk();
        let rec = record_bytes(&[9u8; 5], false);
        c.feed(&rec).unwrap();

        let just_now = Instant::now();
        assert!(!c.should_flush(just_now));

        let later = just_now + Duration::from_millis(150);
        assert!(c.should_flush(later));

        let (kind, data) = c.flush().unwrap();
        assert_eq!(kind, ChunkKind::Data);
        assert_eq!(data, rec);
        assert!(c.flush().is_none());
    }

    #[test]
    fn torn_trailing_record_is_left_unconfirmed_not_an_error() {
        // finish()'s job narrowed to "was a header ever read" -- detecting a
        // genuinely unresolvable torn tail at true EOF is tail.rs's job now
        // (it has the file-length/rename context needed to tell "still
        // resolving" from "will never resolve"). See tail.rs's
        // `non_partial_torn_trailing_record_is_an_error` and
        // `torn_trailing_record_after_rename_is_an_error_not_a_hang`.
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk();
        let rec = record_bytes(&[1u8; 20], false);
        // Feed everything except the last 3 bytes of the record.
        c.feed(&rec[..rec.len() - 3]).unwrap();
        assert_eq!(c.finish().unwrap(), None);
    }

    #[test]
    fn big_endian_magic_reads_incl_len_as_big_endian() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false);
        c.feed(&header_bytes(0xD4C3B2A1)).unwrap();
        c.ready_chunk();
        let rec = record_bytes(&[1u8; 30], true);
        c.feed(&rec).unwrap();
        let (_, data) = c.finish().unwrap().unwrap();
        assert_eq!(data, rec);
    }

    #[test]
    fn finish_before_any_header_bytes_is_an_error() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, false);
        assert!(c.finish().is_err());
    }

    #[test]
    fn all_zero_ts_sec_is_treated_as_not_yet_written() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk();
        let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000);
        let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_001);
        c.feed(&rec1).unwrap();
        c.feed(&rec2).unwrap();
        // rec1 confirms (proven by rec2's real header); rec2 itself is held
        // back (nothing proves it yet).
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64);

        // Now simulate trailing preallocated padding: a large all-zero
        // block. confirmed_len must not advance into it.
        let padding = vec![0u8; 4096];
        c.feed(&padding).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64);
    }

    #[test]
    fn refresh_tail_reexamines_previously_zero_region_once_it_fills_in() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64);

        let zero_block = vec![0u8; 64];
        c.refresh_tail(c.confirmed_len(), &zero_block, false).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64);

        // Writer has since filled this offset in with two real records.
        let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000);
        let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_001);
        let mut real = rec1.clone();
        real.extend_from_slice(&rec2);
        c.refresh_tail(c.confirmed_len(), &real, false).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64);
    }

    #[test]
    fn all_zero_global_header_is_treated_as_not_yet_written() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        let zero_header = vec![0u8; GLOBAL_HEADER_LEN];
        c.refresh_tail(0, &zero_header, false).unwrap();
        assert_eq!(c.confirmed_len(), 0);

        c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64);
    }

    #[test]
    fn nonzero_bad_magic_is_still_a_hard_error_even_when_partial() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        let mut bad = vec![0xFFu8; GLOBAL_HEADER_LEN];
        bad[0..4].copy_from_slice(&0xDEADBEEFu32.to_le_bytes());
        assert!(c.feed(&bad).is_err());
    }

    #[test]
    fn incl_len_exceeding_bound_is_rejected() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk();
        // incl_len far exceeds MAX_INCL_LEN, but we only need the header's
        // 16 bytes physically present for the check to fire.
        let mut rec = vec![0u8; RECORD_HEADER_LEN];
        rec[0..4].copy_from_slice(&1_700_000_000u32.to_le_bytes());
        rec[8..12].copy_from_slice(&(MAX_INCL_LEN as u32 + 1).to_le_bytes());
        c.feed(&rec).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64);
    }

    #[test]
    fn record_not_confirmed_without_proof_of_next_record() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk();
        let rec = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000);
        c.feed(&rec).unwrap();
        // A single fully-present, individually-plausible record with
        // nothing after it: ordinary feed() (trust=false) must not confirm
        // it -- no proof yet that the writer has moved past it.
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64);
    }

    #[test]
    fn record_confirmed_once_next_record_header_is_real() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk();
        let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000);
        let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_001);
        c.feed(&rec1).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64);
        c.feed(&rec2).unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec1.len() as u64);
    }

    #[test]
    fn torn_payload_followed_by_zero_padding_is_not_confirmed() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.feed(&header_bytes(0xA1B2C3D4)).unwrap();
        c.ready_chunk();
        // A record whose own header looks plausible (real ts_sec, sane
        // incl_len) but whose declared payload reaches into all-zero bytes.
        let rec = record_bytes_with_ts(&[0u8; 10], false, 1_700_000_000);
        c.feed(&rec).unwrap();
        c.feed(&[0u8; 64]).unwrap(); // more zero padding after it
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64);

        // Real content lands there instead.
        let rec2 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_001);
        // Replace: simulate the writer having overwritten what followed rec
        // by feeding a fresh, corrected view via refresh_tail.
        c.refresh_tail(c.confirmed_len(), &[&rec[..], &rec2[..]].concat(), false)
            .unwrap();
        assert_eq!(c.confirmed_len(), GLOBAL_HEADER_LEN as u64 + rec.len() as u64);
    }

    #[test]
    fn trailing_record_confirmed_only_when_trust_granted_and_truly_last() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap();
        let rec = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000);
        let offset = c.confirmed_len();
        c.refresh_tail(offset, &rec, false).unwrap();
        assert_eq!(c.confirmed_len(), offset, "must not confirm without trust");

        c.refresh_tail(offset, &rec, true).unwrap();
        assert_eq!(c.confirmed_len(), offset + rec.len() as u64, "must confirm once trusted");
    }

    #[test]
    fn trust_does_not_confirm_a_genuinely_torn_tail() {
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap();
        let rec = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_000);
        let mut bytes = rec.clone();
        // Nonempty garbage after the record -- not a full header, not empty.
        bytes.extend_from_slice(&[0xAB, 0xCD, 0xEF]);
        let offset = c.confirmed_len();
        c.refresh_tail(offset, &bytes, true).unwrap();
        assert_eq!(c.confirmed_len(), offset, "trust must not rescue a nonempty torn fragment");
    }

    #[test]
    fn refresh_tail_never_discards_already_confirmed_undrained_content() {
        // Regression test for a real production incident: refresh_tail used
        // to unconditionally replace `pending` with the fresh read, which
        // silently discarded already-confirmed-but-not-yet-drained content
        // (anything below `target`, i.e. never emitted via ready_chunk) any
        // time a LATER refresh_tail call's fresh window started past it and
        // didn't happen to re-derive the same progress on its own. This
        // caused confirmed_len to oscillate or, worse, get permanently
        // stuck once the fresh window was too small (e.g. capped by
        // READ_WINDOW in tail.rs) to ever re-establish the lost progress.
        //
        // Use a target far bigger than the data so nothing ever drains via
        // ready_chunk -- everything must survive purely via the confirmed
        // prefix being preserved across refresh_tail calls.
        let mut c = RecordAlignedChunker::new(DEFAULT_TARGET_BYTES, DEFAULT_FLUSH, true);
        c.refresh_tail(0, &header_bytes(0xA1B2C3D4), false).unwrap();

        let rec1 = record_bytes_with_ts(&[1u8; 10], false, 1_700_000_001);
        let rec2 = record_bytes_with_ts(&[2u8; 10], false, 1_700_000_002);
        let rec3 = record_bytes_with_ts(&[3u8; 10], false, 1_700_000_003);

        // First call: feed rec1+rec2 together. rec1 confirms (proven by
        // rec2's header); rec2 itself is held back (nothing proves it yet).
        let offset = c.confirmed_len();
        let mut first_batch = rec1.clone();
        first_batch.extend_from_slice(&rec2);
        c.refresh_tail(offset, &first_batch, false).unwrap();
        assert_eq!(
            c.confirmed_len(),
            offset + rec1.len() as u64,
            "rec1 should confirm, proven by rec2's header"
        );
        let after_first = c.confirmed_len();

        // Second call: a FRESH read starting exactly at confirmed_len()
        // (per the trait contract) -- this does NOT include rec1, which is
        // already confirmed but still sitting undrained in `pending`. Only
        // rec2's own bytes (already seen) plus rec3 (new) are read this
        // time, simulating a subsequent, narrower re-peek.
        let mut second_batch = rec2.clone();
        second_batch.extend_from_slice(&rec3);
        c.refresh_tail(after_first, &second_batch, false).unwrap();

        // rec1's confirmation must NOT have been discarded, and rec2 should
        // now also confirm (proven by rec3's header).
        assert_eq!(
            c.confirmed_len(),
            offset + rec1.len() as u64 + rec2.len() as u64,
            "confirmed_len must never regress below what was already established"
        );
    }
}