5a24289898fd
log_writer.rs
//! The thread that writes the catalog log for [`crate::FsStore`] (natively).
//!
//! An append hands its bytes over and returns; the thread writes them in order and makes each batch
//! durable (`sync_data`) before the next. The caller (the UI thread, for the app) never waits for
//! the drive: on Windows a write to a file waits for a sync of that file in flight, so with the log
//! written and synced on the UI thread every flag, rating and step waited for the drive, 30–90 ms
//! on a busy disk in a test and over a second on a photographer's machine while previews were
//! written.
//!
//! **Nothing handed over is ever dropped.** The store's log is the file's first `good` bytes (written
//! and synced) followed by `held`, the bytes handed over and not yet synced. A write or sync that
//! fails keeps `held` and retries it (on every later message, and every [`RETRY`] meanwhile), first
//! cutting the file back to `good` so a fragment of the failed write never stays in it. While a
//! failure stands, [`LogWriter::append`] refuses new bytes with the failure (the journal keeps those
//! records queued, as for any failed append) and [`LogWriter::unwritten`] says how many records are
//! held, so the app counts them as not saved and its quit prompt warns before anything could be
//! lost. Cutting the log back, replacing it and reading it act on that logical content, after
//! everything handed over before; the caller waits for them. Dropping the writer waits for
//! everything handed over and tries a last time.
//!
//! **Every write asks the [`OpenedFolder`] first**, on this thread (a `stat` never makes the caller
//! wait for the drive): when the log's path no longer leads to the folder the store opened (a share
//! mounted over it, issue #3240) nothing is written. The bytes are held like those of any other
//! write that failed, and written once the folder that was opened is back at the path.
//!
//! **Successful batches reopen the log by path.** A failed batch keeps its descriptor only to
//! roll back its own bytes, never to append into a moved-away folder. A copy of the library's folder
//! put at the path counts as the opened one (it holds the lock's note) and has a log of its own: a
//! file kept open from before would go on writing into the folder that was moved away. A log that
//! isn't as long as what this thread wrote (a fragment of its own failed write is cut first) is not
//! the one it wrote, and appending to it would leave a gap: what it was handed is held like a failed
//! write (reported; later appends are taken meanwhile, since retrying can't heal it), and the
//! snapshot the app takes for held records, which holds every change, replaces the log.
use std::io::{self, Seek, Write};
use std::path::PathBuf;
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use crate::opened_folder::OpenedFolder;
/// How long a caller waits for the thread at most (a cut, a replace, a read, a flush).
const WAIT: Duration = Duration::from_secs(60);
/// How often held bytes are retried while a failure stands and nothing else arrives.
pub(crate) const RETRY: Duration = Duration::from_millis(500);
enum Msg {
Append(Vec<u8>),
Truncate(u64, Sender<io::Result<()>>),
Replace(Vec<u8>, Sender<io::Result<()>>),
Read(Sender<io::Result<Option<Vec<u8>>>>),
/// Reply once everything before is written and synced (or why it isn't).
Flush(Sender<io::Result<()>>),
}
/// What the caller can see without waiting.
#[derive(Default)]
struct Shared {
/// Why the held bytes couldn't be written (until they are, or the log is cut or replaced).
failed: Option<String>,
/// Records (lines) handed over and not yet written and synced, while `failed` stands.
held_records: usize,
/// The file isn't the log the thread wrote: only a replace helps, so appends are taken.
foreign: bool,
}
pub(crate) struct LogWriter {
pub(crate) name: String,
tx: Option<Sender<Msg>>,
shared: Arc<Mutex<Shared>>,
thread: Option<std::thread::JoinHandle<()>>,
}
fn stopped() -> io::Error {
io::Error::other("the catalog log writer stopped")
}
fn not_written(why: &str) -> io::Error {
io::Error::other(format!("the catalog log could not be written: {why}"))
}
impl LogWriter {
/// A writer for `name` in the folder (`None` when no thread can start: the store then writes itself).
pub(crate) fn start(folder: OpenedFolder, name: &str, expected: Option<u64>) -> Option<LogWriter> {
let (tx, rx) = channel::<Msg>();
let shared = Arc::new(Mutex::new(Shared::default()));
let sh = shared.clone();
let mut w = Writer::new(folder, name);
w.good = expected;
let thread = std::thread::Builder::new().name("catalog-log".into()).spawn(move || w.run(&rx, &sh)).ok()?;
Some(LogWriter { name: name.to_string(), tx: Some(tx), shared, thread: Some(thread) })
}
fn send(&self, m: Msg) -> io::Result<()> {
self.tx.as_ref().ok_or_else(stopped)?.send(m).map_err(|_| stopped())
}
fn ask<T>(&self, make: impl FnOnce(Sender<io::Result<T>>) -> Msg) -> io::Result<T> {
let (tx, rx) = channel();
self.send(make(tx))?;
rx.recv_timeout(WAIT).map_err(|e| match e {
RecvTimeoutError::Timeout => io::Error::new(io::ErrorKind::TimedOut, "the drive took over a minute to write the catalog log"),
RecvTimeoutError::Disconnected => stopped(),
})?
}
/// Hand `data` over (never waits for the drive). Refused while bytes handed over before can't
/// be written: the caller keeps `data` and appends it again later.
pub(crate) fn append(&self, data: &[u8]) -> io::Result<()> {
let sh = self.shared.lock().unwrap_or_else(PoisonError::into_inner);
if let Some(why) = sh.failed.as_deref().filter(|_| !sh.foreign) {
return Err(not_written(why));
}
drop(sh);
self.send(Msg::Append(data.to_vec()))
}
/// Cut the log to `len` bytes, after everything handed over; waits for it.
pub(crate) fn truncate(&self, len: u64) -> io::Result<()> {
self.ask(|r| Msg::Truncate(len, r))
}
/// Replace the log with `data` (atomically, durably), after everything handed over; waits.
pub(crate) fn replace(&self, data: &[u8]) -> io::Result<()> {
self.ask(|r| Msg::Replace(data.to_vec(), r))
}
/// The log as appended so far (written or still held), after everything handed over.
pub(crate) fn read(&self) -> io::Result<Option<Vec<u8>>> {
self.ask(Msg::Read)
}
/// Wait until everything handed over is written and synced; an error says why it isn't (the
/// bytes are still held and retried).
pub(crate) fn flush(&self) -> io::Result<()> {
self.ask(Msg::Flush)
}
/// Records handed over that can't be written yet, and why (`None`: nothing is held back by a
/// failure). Never waits.
pub(crate) fn unwritten(&self) -> Option<(usize, String)> {
let sh = self.shared.lock().unwrap_or_else(PoisonError::into_inner);
sh.failed.clone().map(|why| (sh.held_records.max(1), why))
}
}
impl Drop for LogWriter {
fn drop(&mut self) {
drop(self.tx.take());
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
/// The thread's side: how much of the file is written, and what is held for it.
struct Writer {
dir: PathBuf,
path: PathBuf,
tmp: PathBuf,
/// The folder the store opened: asked before anything is written.
folder: OpenedFolder,
/// Bytes read by the journal or written whole and synced (`None` until first observed).
good: Option<u64>,
/// Handed over, not yet written and synced.
held: Vec<u8>,
failed: Option<String>,
/// The exact file whose append failed; only this descriptor can have its torn bytes removed.
torn: Option<std::fs::File>,
/// The file isn't the log this thread wrote ([`Writer::open`]).
foreign: bool,
}
fn lines(b: &[u8]) -> usize {
b.iter().filter(|c| **c == b'\n').count()
}
impl Writer {
fn new(folder: OpenedFolder, name: &str) -> Writer {
let dir = folder.path().to_path_buf();
let path = dir.join(name);
let tmp = dir.join(format!("{name}.tmp"));
Writer { dir, path, tmp, folder, good: None, held: Vec::new(), failed: None, torn: None, foreign: false }
}
fn hook(&self, step: crate::fault::Step) -> io::Result<()> {
crate::fault::check(&self.dir, step)
}
/// Open the file by its path, positioned at `good`, and say where that is. A fragment a failed
/// write left past `good` is cut; any other length is not the log this thread wrote (a copy was
/// put at the path, or it was changed): refused.
fn open(&mut self) -> io::Result<(std::fs::File, u64)> {
self.folder.check()?;
if let Some(good) = self.good {
crate::store::rollback_torn(&mut self.torn, good)?;
}
// (a link or a device planted at the name meanwhile is refused, as at the first append)
crate::store::check_regular_directory_entry(&self.path)?;
let mut f = std::fs::OpenOptions::new().write(true).create(true).truncate(false).open(&self.path)?;
let len = f.metadata()?.len();
let good = match self.good {
None => len,
Some(g) if len == g => g,
Some(g) => {
self.foreign = true;
return Err(io::Error::other(format!(
"{} holds {len} bytes, the log this session wrote has {g}: it is another file, so nothing is appended to it and a snapshot replaces it",
self.path.display()
)));
}
};
f.seek(io::SeekFrom::Start(good))?;
self.good = Some(good);
self.foreign = false;
Ok((f, good))
}
/// Write and sync what is held; a failure leaves `good` as it was, and the retry cuts what was left past it.
fn write_held(&mut self) -> io::Result<()> {
if self.held.is_empty() {
return Ok(());
}
self.hook(crate::fault::Step::Write)?;
let (mut f, good) = self.open()?;
let written = f.write_all(&self.held).and_then(|()| self.hook(crate::fault::Step::Sync)).and_then(|()| f.sync_data());
self.torn = written.is_err().then_some(f);
written?;
self.good = Some(good.saturating_add(self.held.len() as u64));
self.held.clear();
Ok(())
}
/// Cut the logical log (file then held) to `len` bytes.
fn truncate(&mut self, len: u64) -> io::Result<()> {
let good = match self.good {
Some(g) => g,
None => match self.open() {
Ok((_, g)) => g,
Err(e) if e.kind() == io::ErrorKind::NotFound => 0,
Err(e) => return Err(e),
},
};
// (to exactly `good` with bytes held: the file is cut, since a write that failed may have
// left some of them past `good`)
if len > good || (len == good && self.held.is_empty()) {
// within what is held: nothing on the drive changes (what a failed write left past
// `good` is cut before the next write)
let keep = usize::try_from(len - good).unwrap_or(usize::MAX).min(self.held.len());
self.held.truncate(keep);
return Ok(());
}
self.folder.check()?;
crate::store::rollback_torn(&mut self.torn, good)?;
crate::store::check_regular_directory_entry(&self.path)?;
let f = std::fs::OpenOptions::new().write(true).open(&self.path)?;
if f.metadata()?.len() != good {
self.foreign = true;
return Err(io::Error::other("the log changed before truncate: a snapshot must replace it"));
}
f.set_len(len)?;
f.sync_all()?;
self.good = Some(len);
self.held.clear();
Ok(())
}
fn replace(&mut self, data: &[u8]) -> io::Result<()> {
self.folder.check()?;
self.hook(crate::fault::Step::Write)?;
let mut file = crate::store::create_staging(&self.tmp)?;
let written = (|| {
file.write_all(data)?;
self.hook(crate::fault::Step::Sync)?;
file.sync_all()?;
crate::store::check_staging(&self.folder, &self.tmp, &file)?;
std::fs::rename(&self.tmp, &self.path)
})();
match written {
Ok(()) => {
// what was held is part of what the caller replaced the log with (a snapshot holds
// the records before it, `data` the ones after)
self.good = Some(data.len() as u64);
self.held.clear();
self.failed = None;
self.torn = None;
self.foreign = false;
Ok(())
}
Err(e) => {
if crate::store::check_staging(&self.folder, &self.tmp, &file).is_ok() {
let _ = std::fs::remove_file(&self.tmp);
}
Err(e)
}
}
}
fn read(&mut self) -> io::Result<Option<Vec<u8>>> {
let mut bytes = match std::fs::read(&self.path) {
Ok(b) => b,
Err(e) if e.kind() == io::ErrorKind::NotFound && self.held.is_empty() => return Ok(None),
Err(e) if e.kind() == io::ErrorKind::NotFound => Vec::new(),
Err(e) => return Err(e),
};
if let Some(g) = self.good {
bytes.truncate(usize::try_from(g).unwrap_or(usize::MAX));
}
bytes.extend_from_slice(&self.held);
Ok(Some(bytes))
}
fn publish(&self, shared: &Mutex<Shared>) {
let mut sh = shared.lock().unwrap_or_else(PoisonError::into_inner);
sh.failed = self.failed.clone().filter(|_| !self.held.is_empty());
sh.foreign = self.foreign && sh.failed.is_some();
sh.held_records = if sh.failed.is_some() { lines(&self.held) } else { 0 };
}
fn run(&mut self, rx: &Receiver<Msg>, shared: &Mutex<Shared>) {
loop {
let first = if self.failed.is_some() && !self.held.is_empty() {
match rx.recv_timeout(RETRY) {
Ok(m) => Some(m),
Err(RecvTimeoutError::Timeout) => None,
Err(RecvTimeoutError::Disconnected) => break,
}
} else {
match rx.recv() {
Ok(m) => Some(m),
Err(_) => break,
}
};
let mut batch: Vec<Msg> = first.into_iter().collect();
while let Ok(m) = rx.try_recv() {
batch.push(m);
}
let mut flushes = Vec::new();
for m in batch {
match m {
Msg::Append(data) => self.held.extend_from_slice(&data),
// (published before the reply: the caller reads what it left at once)
Msg::Truncate(len, reply) => {
let r = self.truncate(len);
self.publish(shared);
let _ = reply.send(r);
}
Msg::Replace(data, reply) => {
let r = self.replace(&data);
self.publish(shared);
let _ = reply.send(r);
}
Msg::Read(reply) => {
let _ = reply.send(self.read());
}
Msg::Flush(reply) => flushes.push(reply),
}
}
self.step();
self.publish(shared);
for r in flushes {
let _ = r.send(match &self.failed {
Some(why) if !self.held.is_empty() => Err(not_written(why)),
_ => Ok(()),
});
}
}
// the store is closing: everything handed over gets one more try
self.step();
self.publish(shared);
if !self.held.is_empty() {
log::error!("catalog: {} record(s) could not be written to {} (the closing snapshot holds them)", lines(&self.held), self.path.display());
}
}
/// Write what is held; a failure is kept (and logged when it is new).
fn step(&mut self) {
match self.write_held() {
Ok(()) => {
if self.failed.take().is_some() {
log::info!("catalog: {} written again", self.path.display());
}
}
Err(e) => {
let why = e.to_string();
if self.failed.as_deref() != Some(why.as_str()) {
log::error!("catalog: {}: {why} ({} record(s) held, retried)", self.path.display(), lines(&self.held));
}
self.failed = Some(why);
}
}
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.