Track confirmed pcap content instead of raw file length for growth/finalize
A preallocating capture writer (ftruncate-extend, zero-filled) reports the full reserved file size immediately and fills real data into it in place, without file length ever reflecting real progress until a final truncate at rotation. The tailer trusted raw file length for both "is there new data" and "are we done", so it read (and, in raw mode, forwarded) preallocated zero padding as real packet data, then could never satisfy pos == file_len once the file was truncated down at rotation -- a permanent hang, and worse, a corrupted destination even when the hang didn't bite. Replace file-length-based tracking with confirmed_len(), derived from actually walking pcap records (RecordAlignedChunker), for any partial (still-growing) source -- non-compress non-partial transfers are untouched, preserving today's "arbitrary content" guarantee there. Growth detection now re-reads from the last confirmed boundary every poll rather than only ever reading forward past what's already been read, since a preallocating writer can flip a byte from zero to real content without file length ever changing. Two correctness properties enforced by the new record scanner, both gated to partial sources only: - A record's header being fully present and plausible is not proof its payload is real (header and payload aren't necessarily written atomically) -- a candidate record is only confirmed once the *next* record's header has also been observed and looks real, proof the writer moved past it. This holds complete_len one record behind by construction. - That rule alone would starve the true last record of any capture forever, so a narrow escape hatch trusts the trailing record on its own plausibility once the rename has been observed and the file's length has been stable across continuous re-checks for a grace period -- backed by the writer's own "I'm done" signal (the rename), not a timing guess alone. Also: an all-zero global header on a partial source is now treated as "not written yet" rather than a hard error, for the same preallocation reason. Verified against a live preallocation simulation over ssh (locl.sh): a source truncated to a padded size well beyond its real content, with a writer catch-up (in-place record write with no length change) before the final truncate-and-rename, transfers with no hang and a byte-exact destination -- confirmed_len stalls precisely at the real/padding boundary and only advances once content, not file length, proves growth.
This commit is contained in:
1 parent
b3e7248ac5
commit
aa374bc66e
3 files changed
+750
-77
No files matched your search
+25
-7
@@ -39,8 +39,8 @@ enum CompressedItem {
|
||||
/// serializing -- see the plan's Performance section. Non-compress mode
|
||||
/// collapses to 2 stages since there's no compression work to overlap.
|
||||
pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()> {
|
||||
let chunker: Box<dyn Chunker> = if opts.compress {
|
||||
Box::new(RecordAlignedChunker::new(opts.chunk_target, DEFAULT_FLUSH))
|
||||
let chunker: Box<dyn Chunker> = if opts.compress || opts.is_partial {
|
||||
Box::new(RecordAlignedChunker::new(opts.chunk_target, DEFAULT_FLUSH, opts.is_partial))
|
||||
} else {
|
||||
Box::new(RawChunker::new())
|
||||
};
|
||||
@@ -205,9 +205,10 @@ mod tests {
|
||||
h
|
||||
}
|
||||
|
||||
fn record_bytes(payload: &[u8]) -> Vec<u8> {
|
||||
fn record_bytes(payload: &[u8], ts_sec: u32) -> Vec<u8> {
|
||||
let mut r = Vec::new();
|
||||
r.extend_from_slice(&[0u8; 8]);
|
||||
r.extend_from_slice(&ts_sec.to_le_bytes());
|
||||
r.extend_from_slice(&[0u8; 4]); // ts_usec
|
||||
let incl_len = payload.len() as u32;
|
||||
r.extend_from_slice(&incl_len.to_le_bytes());
|
||||
r.extend_from_slice(&incl_len.to_le_bytes());
|
||||
@@ -251,10 +252,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn non_compress_partial_source_waits_for_rename() {
|
||||
// Under the new chunker-selection (compress || is_partial), a raw
|
||||
// partial transfer now routes through RecordAlignedChunker, which
|
||||
// requires genuine pcap structure -- see the plan's "behavior
|
||||
// change requiring sign-off". A real global header plus a real
|
||||
// record is required; the second record is appended before rename
|
||||
// so the first one gets its next-record proof quickly rather than
|
||||
// waiting out the trailing-record grace period.
|
||||
let dir = tempdir().unwrap();
|
||||
let partial = dir.path().join("cap.pcap.partial");
|
||||
let final_src = dir.path().join("cap.pcap");
|
||||
std::fs::write(&partial, b"growing-data").unwrap();
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
stream.extend_from_slice(&record_bytes(&[1u8; 10], 1_700_000_000));
|
||||
std::fs::write(&partial, &stream).unwrap();
|
||||
let dest_final = dir.path().join("out.pcap");
|
||||
let dest_temp = dir.path().join("out.pcap.partial");
|
||||
|
||||
@@ -276,13 +286,21 @@ mod tests {
|
||||
});
|
||||
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
||||
|
||||
thread::sleep(std::time::Duration::from_millis(60));
|
||||
let rec2 = record_bytes(&[2u8; 10], 1_700_000_001);
|
||||
{
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(&partial).unwrap();
|
||||
f.write_all(&rec2).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
}
|
||||
stream.extend_from_slice(&rec2);
|
||||
thread::sleep(std::time::Duration::from_millis(60));
|
||||
std::fs::rename(&partial, &final_src).unwrap();
|
||||
|
||||
send_handle.join().unwrap().unwrap();
|
||||
recv_handle.join().unwrap().unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), b"growing-data");
|
||||
assert_eq!(std::fs::read(&dest_final).unwrap(), stream);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -291,7 +309,7 @@ mod tests {
|
||||
let src = dir.path().join("cap.pcap");
|
||||
let mut stream = header_bytes(0xA1B2C3D4);
|
||||
for i in 0..50u8 {
|
||||
stream.extend_from_slice(&record_bytes(&[i; 100]));
|
||||
stream.extend_from_slice(&record_bytes(&[i; 100], 0));
|
||||
}
|
||||
std::fs::write(&src, &stream).unwrap();
|
||||
|
||||
|
||||
Reference in new issue
Block a user