Add --log/--remote-log diagnostic tracing for stalled transfers

Adds an optional timestamped trace of transfer-relevant events (tail
position vs. file length, rename detection, finalize, protocol
send/recv) to both the client and server sides, so a stalled transfer
can be diagnosed from correlated local/remote logs instead of guessing.

Notably logs when a source's final size ends up smaller than what was
already read (pos > current_len after rename) -- this can never
satisfy the pos == current_len finalize condition, which is a real
deadlock risk for writers that preallocate a fixed-size file and
truncate down to the actual capture length before renaming.

--log <path> on either the client or `--server` process traces that
process's own events. --log on ClientCli is local-only; --remote-log
<path> is forwarded as the spawned --server process's --log so both
sides of a remote transfer leave a trace, tied together by timestamp.
This commit is contained in:
eric committed 2026-08-27 17:06:00 -04:00
1 parent 17e9e5e361
commit b3e7248ac5
7 files changed
+189 -14

No files matched your search

+37 -6
View File
@@ -1,12 +1,13 @@
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker, DEFAULT_FLUSH};
use crate::protocol::{Message, MessageSink, MessageSource};
use crate::tail::{run_tail, SourceEvent};
use crate::tracelog::Logger;
use crate::zstd_frame::{compress_frame, MARKER_FRAME};
use anyhow::{bail, Result};
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::sync::mpsc;
use std::sync::{mpsc, Arc};
use std::thread;
/// Bound on the reader->compressor and compressor->sender channels: enough to
@@ -22,6 +23,7 @@ pub struct SendOptions {
pub compress: bool,
pub chunk_target: usize,
pub level: i32,
pub log: Arc<Logger>,
}
enum CompressedItem {
@@ -48,7 +50,8 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
let source_path = opts.source_path.clone();
let is_partial = opts.is_partial;
let final_source_path = opts.final_source_path.clone();
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx)
let log = opts.log.clone();
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
});
if opts.compress {
@@ -112,21 +115,26 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
compressor_result?;
} else {
let mut send_err: Option<anyhow::Error> = None;
let mut sent: u64 = 0;
for event in tail_rx {
match event {
SourceEvent::Chunk(_kind, data) => {
sent += data.len() as u64;
if let Err(e) = sink.send_data(&data) {
opts.log.log(format_args!("send: send_data failed after {sent} bytes: {e}"));
send_err = Some(e);
break;
}
}
SourceEvent::Eof => {
opts.log.log(format_args!("send: tail EOF, {sent} bytes sent, sending FINALIZE"));
if let Err(e) = sink.send_finalize() {
send_err = Some(e);
}
break;
}
SourceEvent::Error(e) => {
opts.log.log(format_args!("send: tail error after {sent} bytes: {e}"));
let _ = sink.send_error(&e);
send_err = Some(anyhow::anyhow!(e));
break;
@@ -145,6 +153,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
pub struct RecvOptions {
pub dest_final: PathBuf,
pub dest_temp: PathBuf,
pub log: Arc<Logger>,
}
/// Writes whatever bytes arrive verbatim to a temp path, then atomically
@@ -153,15 +162,33 @@ pub struct RecvOptions {
/// error rather than renaming.
pub fn run_recv(mut source: Box<dyn MessageSource>, opts: RecvOptions) -> Result<()> {
let mut f = File::create(&opts.dest_temp)?;
let mut received: u64 = 0;
opts.log.log(format_args!(
"recv: writing to {} (final: {})",
opts.dest_temp.display(),
opts.dest_final.display()
));
loop {
match source.recv()? {
Some(Message::Data(payload)) => f.write_all(&payload)?,
Some(Message::Data(payload)) => {
received += payload.len() as u64;
f.write_all(&payload)?;
}
Some(Message::Finalize) => {
f.sync_all()?;
std::fs::rename(&opts.dest_temp, &opts.dest_final)?;
opts.log.log(format_args!(
"recv: FINALIZE after {received} bytes, renamed to {}",
opts.dest_final.display()
));
return Ok(());
}
None => bail!("transfer aborted: stream ended without FINALIZE"),
None => {
opts.log.log(format_args!(
"recv: stream ended without FINALIZE after {received} bytes"
));
bail!("transfer aborted: stream ended without FINALIZE")
}
}
}
}
@@ -190,7 +217,7 @@ mod tests {
fn run_pipe(opts: SendOptions, dest_final: PathBuf, dest_temp: PathBuf) -> Result<()> {
let (tx, rx) = mpsc::channel::<Message>();
let recv_opts = RecvOptions { dest_final, dest_temp };
let recv_opts = RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) };
let recv_handle = thread::spawn(move || run_recv(Box::new(ChannelSource(rx)), recv_opts));
run_send(opts, Box::new(ChannelSink(tx)))?;
recv_handle.join().unwrap()
@@ -211,6 +238,7 @@ mod tests {
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
level: crate::zstd_frame::DEFAULT_LEVEL,
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
@@ -237,13 +265,14 @@ mod tests {
compress: false,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
level: crate::zstd_frame::DEFAULT_LEVEL,
log: Arc::new(Logger::none()),
};
let (tx, rx) = mpsc::channel::<Message>();
let recv_handle = thread::spawn({
let dest_final = dest_final.clone();
let dest_temp = dest_temp.clone();
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp })
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
});
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
@@ -275,6 +304,7 @@ mod tests {
compress: true,
chunk_target: 512, // small target to force multiple frames
level: crate::zstd_frame::DEFAULT_LEVEL,
log: Arc::new(Logger::none()),
};
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
@@ -310,6 +340,7 @@ mod tests {
compress: true,
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
level: crate::zstd_frame::DEFAULT_LEVEL,
log: Arc::new(Logger::none()),
};
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
assert!(!dest_final.exists());