Add --log/--remote-log diagnostic tracing for stalled transfers
Adds an optional timestamped trace of transfer-relevant events (tail position vs. file length, rename detection, finalize, protocol send/recv) to both the client and server sides, so a stalled transfer can be diagnosed from correlated local/remote logs instead of guessing. Notably logs when a source's final size ends up smaller than what was already read (pos > current_len after rename) -- this can never satisfy the pos == current_len finalize condition, which is a real deadlock risk for writers that preallocate a fixed-size file and truncate down to the actual capture length before renaming. --log <path> on either the client or `--server` process traces that process's own events. --log on ClientCli is local-only; --remote-log <path> is forwarded as the spawned --server process's --log so both sides of a remote transfer leave a trace, tied together by timestamp.
This commit is contained in:
1 parent
17e9e5e361
commit
b3e7248ac5
7 files changed
+189
-14
No files matched your search
+37
-6
@@ -1,12 +1,13 @@
|
||||
use crate::chunker::{Chunker, RawChunker, RecordAlignedChunker, DEFAULT_FLUSH};
|
||||
use crate::protocol::{Message, MessageSink, MessageSource};
|
||||
use crate::tail::{run_tail, SourceEvent};
|
||||
use crate::tracelog::Logger;
|
||||
use crate::zstd_frame::{compress_frame, MARKER_FRAME};
|
||||
use anyhow::{bail, Result};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc;
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::thread;
|
||||
|
||||
/// Bound on the reader->compressor and compressor->sender channels: enough to
|
||||
@@ -22,6 +23,7 @@ pub struct SendOptions {
|
||||
pub compress: bool,
|
||||
pub chunk_target: usize,
|
||||
pub level: i32,
|
||||
pub log: Arc<Logger>,
|
||||
}
|
||||
|
||||
enum CompressedItem {
|
||||
@@ -48,7 +50,8 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
let source_path = opts.source_path.clone();
|
||||
let is_partial = opts.is_partial;
|
||||
let final_source_path = opts.final_source_path.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx)
|
||||
let log = opts.log.clone();
|
||||
move || run_tail(source_path, is_partial, final_source_path, chunker, tail_tx, log)
|
||||
});
|
||||
|
||||
if opts.compress {
|
||||
@@ -112,21 +115,26 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
compressor_result?;
|
||||
} else {
|
||||
let mut send_err: Option<anyhow::Error> = None;
|
||||
let mut sent: u64 = 0;
|
||||
for event in tail_rx {
|
||||
match event {
|
||||
SourceEvent::Chunk(_kind, data) => {
|
||||
sent += data.len() as u64;
|
||||
if let Err(e) = sink.send_data(&data) {
|
||||
opts.log.log(format_args!("send: send_data failed after {sent} bytes: {e}"));
|
||||
send_err = Some(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
SourceEvent::Eof => {
|
||||
opts.log.log(format_args!("send: tail EOF, {sent} bytes sent, sending FINALIZE"));
|
||||
if let Err(e) = sink.send_finalize() {
|
||||
send_err = Some(e);
|
||||
}
|
||||
break;
|
||||
}
|
||||
SourceEvent::Error(e) => {
|
||||
opts.log.log(format_args!("send: tail error after {sent} bytes: {e}"));
|
||||
let _ = sink.send_error(&e);
|
||||
send_err = Some(anyhow::anyhow!(e));
|
||||
break;
|
||||
@@ -145,6 +153,7 @@ pub fn run_send(opts: SendOptions, mut sink: Box<dyn MessageSink>) -> Result<()>
|
||||
pub struct RecvOptions {
|
||||
pub dest_final: PathBuf,
|
||||
pub dest_temp: PathBuf,
|
||||
pub log: Arc<Logger>,
|
||||
}
|
||||
|
||||
/// Writes whatever bytes arrive verbatim to a temp path, then atomically
|
||||
@@ -153,15 +162,33 @@ pub struct RecvOptions {
|
||||
/// error rather than renaming.
|
||||
pub fn run_recv(mut source: Box<dyn MessageSource>, opts: RecvOptions) -> Result<()> {
|
||||
let mut f = File::create(&opts.dest_temp)?;
|
||||
let mut received: u64 = 0;
|
||||
opts.log.log(format_args!(
|
||||
"recv: writing to {} (final: {})",
|
||||
opts.dest_temp.display(),
|
||||
opts.dest_final.display()
|
||||
));
|
||||
loop {
|
||||
match source.recv()? {
|
||||
Some(Message::Data(payload)) => f.write_all(&payload)?,
|
||||
Some(Message::Data(payload)) => {
|
||||
received += payload.len() as u64;
|
||||
f.write_all(&payload)?;
|
||||
}
|
||||
Some(Message::Finalize) => {
|
||||
f.sync_all()?;
|
||||
std::fs::rename(&opts.dest_temp, &opts.dest_final)?;
|
||||
opts.log.log(format_args!(
|
||||
"recv: FINALIZE after {received} bytes, renamed to {}",
|
||||
opts.dest_final.display()
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
None => bail!("transfer aborted: stream ended without FINALIZE"),
|
||||
None => {
|
||||
opts.log.log(format_args!(
|
||||
"recv: stream ended without FINALIZE after {received} bytes"
|
||||
));
|
||||
bail!("transfer aborted: stream ended without FINALIZE")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -190,7 +217,7 @@ mod tests {
|
||||
|
||||
fn run_pipe(opts: SendOptions, dest_final: PathBuf, dest_temp: PathBuf) -> Result<()> {
|
||||
let (tx, rx) = mpsc::channel::<Message>();
|
||||
let recv_opts = RecvOptions { dest_final, dest_temp };
|
||||
let recv_opts = RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) };
|
||||
let recv_handle = thread::spawn(move || run_recv(Box::new(ChannelSource(rx)), recv_opts));
|
||||
run_send(opts, Box::new(ChannelSink(tx)))?;
|
||||
recv_handle.join().unwrap()
|
||||
@@ -211,6 +238,7 @@ mod tests {
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
|
||||
@@ -237,13 +265,14 @@ mod tests {
|
||||
compress: false,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel::<Message>();
|
||||
let recv_handle = thread::spawn({
|
||||
let dest_final = dest_final.clone();
|
||||
let dest_temp = dest_temp.clone();
|
||||
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp })
|
||||
move || run_recv(Box::new(ChannelSource(rx)), RecvOptions { dest_final, dest_temp, log: Arc::new(Logger::none()) })
|
||||
});
|
||||
let send_handle = thread::spawn(move || run_send(opts, Box::new(ChannelSink(tx))));
|
||||
|
||||
@@ -275,6 +304,7 @@ mod tests {
|
||||
compress: true,
|
||||
chunk_target: 512, // small target to force multiple frames
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
run_pipe(opts, dest_final.clone(), dest_temp.clone()).unwrap();
|
||||
|
||||
@@ -310,6 +340,7 @@ mod tests {
|
||||
compress: true,
|
||||
chunk_target: crate::chunker::DEFAULT_TARGET_BYTES,
|
||||
level: crate::zstd_frame::DEFAULT_LEVEL,
|
||||
log: Arc::new(Logger::none()),
|
||||
};
|
||||
assert!(run_pipe(opts, dest_final.clone(), dest_temp).is_err());
|
||||
assert!(!dest_final.exists());
|
||||
|
||||
Reference in new issue
Block a user