5a24289898fd
slow_drive.rs
//! PERF-STEP-STALL: culling on a slow or failing drive. A flag or rating never waits for the drive
//! (the catalog log is written and synced by a thread of its own), and nothing made meanwhile is
//! lost: it is on disk once the save lands, after a clean quit, after a quit with no close, and,
//! when the drive fails, held, retried and reported as unsaved until it is written.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use lightcraft_catalog::fault::{self, Step};
use lightcraft_catalog::{Flag, PhotoId};
use serde_json::json;
use crate::Session;
fn temp_dir(tag: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("lc-engine-slow-drive-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
d
}
fn open(dir: &std::path::Path) -> Session {
let mut s = Session::new();
s.open_library(dir, true).unwrap();
s
}
fn ids(s: &Session) -> Vec<PhotoId> {
let mut v: Vec<PhotoId> = s.catalog.photos().map(|p| p.id).collect();
v.sort();
v
}
/// Every sync of the log in `dir` takes `sync` (a busy disk: Windows makes a write wait for a sync
/// of the same file in flight); `fail` makes every write fail while it is set.
fn slow_drive(dir: &std::path::Path, sync: Duration, fail: Arc<AtomicBool>) {
fault::install(
dir,
Arc::new(move |step| {
if fail.load(Ordering::SeqCst) {
return Err(std::io::Error::other("the drive is gone (test)"));
}
if step == Step::Sync {
std::thread::sleep(sync);
}
Ok(())
}),
);
}
/// The flag or rating each photo got, as `(id, flag, rating)`.
fn culled(s: &Session) -> Vec<(PhotoId, Flag, u8)> {
ids(s).into_iter().filter_map(|id| s.catalog.photo(id).map(|p| (id, p.flag, p.rating))).collect()
}
/// Flag and rate every photo the way a photographer culls: pick / reject in turn, a rating on
/// every other one. Returns the slowest command, in ms.
fn cull(s: &mut Session, rounds: usize) -> f64 {
let all = ids(s);
let mut slowest = 0.0f64;
for k in 0..rounds {
for (i, id) in all.iter().enumerate() {
let flag = if (i + k) % 2 == 0 { "pick" } else { "reject" };
let t = Instant::now();
s.execute("photo.flag", &json!({"flag": flag, "ids": [id.0]})).unwrap();
slowest = slowest.max(t.elapsed().as_secs_f64() * 1000.0);
if i % 2 == 0 {
let t = Instant::now();
s.execute("photo.rate", &json!({"rating": ((i + k) % 5) as u8 + 1, "ids": [id.0]})).unwrap();
slowest = slowest.max(t.elapsed().as_secs_f64() * 1000.0);
}
}
}
slowest
}
/// (a) With every sync of the log taking 300 ms, a flag or rating returns long before one sync.
/// The same culling with the log written on the caller's thread (as before PERF-STEP-STALL) waits
/// for a sync on every key, which is what this test would see without the fix.
#[test]
fn a_flag_never_waits_for_a_slow_drive() {
let sync = Duration::from_millis(300);
let mut seen = Vec::new();
for on_caller in [true, false] {
let dir = temp_dir(if on_caller { "wait-before" } else { "wait" });
fault::write_on_caller(&dir, on_caller);
let mut s = open(&dir);
slow_drive(&dir, sync, Arc::new(AtomicBool::new(false)));
let t = Instant::now();
let slowest = cull(&mut s, 1);
let total = t.elapsed().as_secs_f64() * 1000.0;
let n = s.library.as_ref().map_or(0, |l| l.journal().stats().appends);
eprintln!(
"{} drive syncing in 300 ms: slowest command {slowest:.1} ms, {total:.0} ms for the culling",
if on_caller { "before (log written on the caller):" } else { "after (log on its own thread):" }
);
seen.push((slowest, total, n));
drop(s);
fault::remove(&dir);
fault::write_on_caller(&dir, false);
let _ = std::fs::remove_dir_all(&dir);
}
let ((before, _, _), (after, _, _)) = (seen[0], seen[1]);
assert!(before >= 300.0, "the caller waited for its sync before: {before:.1} ms");
// a debug build of the whole workspace's tests at once leaves room; one sync is 300 ms
assert!(after < 120.0, "a command waited for the drive: {after:.1} ms");
}
/// (b) Everything flagged and rated while slow saves are in flight is on disk once they land
/// (a flush), after a quit with no close (the process ends normally: the log's thread finishes),
/// and after a clean quit (`close_library`).
#[test]
fn every_change_made_during_a_slow_save_is_on_disk() {
for how in ["flush", "drop", "close"] {
let dir = temp_dir(&format!("kept-{how}"));
let mut s = open(&dir);
slow_drive(&dir, Duration::from_millis(150), Arc::new(AtomicBool::new(false)));
cull(&mut s, 3);
let want = culled(&s);
let expect = s.catalog.to_snapshot();
match how {
"flush" => {
s.library.as_mut().unwrap().journal_mut().flush().unwrap();
// a copy of the files taken now (the next crash) holds everything
let copy = temp_dir("kept-copy");
std::fs::create_dir_all(©).unwrap();
for f in [lightcraft_catalog::journal::SNAPSHOT, lightcraft_catalog::journal::LOG] {
if dir.join(f).exists() {
std::fs::copy(dir.join(f), copy.join(f)).unwrap();
}
}
let (_, c, _) = lightcraft_catalog::Journal::open(Box::new(lightcraft_catalog::store::FsStore::open(©).unwrap())).unwrap();
assert_eq!(c.to_snapshot(), expect, "{how}");
let _ = std::fs::remove_dir_all(©);
}
"close" => s.close_library().unwrap(),
_ => {}
}
assert!(s.unsaved().is_none(), "{how}: {:?}", s.unsaved());
drop(s);
fault::remove(&dir);
let again = open(&dir);
assert_eq!(culled(&again), want, "{how}");
assert_eq!(again.catalog.to_snapshot(), expect, "{how}");
drop(again);
let _ = std::fs::remove_dir_all(&dir);
}
}
/// Wait (bounded) until `done` holds, calling the frame loop's save meanwhile.
fn frames_until(s: &mut Session, what: &str, mut done: impl FnMut(&Session) -> bool) {
let t = Instant::now();
while !done(s) {
assert!(t.elapsed() < Duration::from_secs(20), "{what}: {:?}", s.unsaved());
s.persist_if_dirty();
std::thread::sleep(Duration::from_millis(20));
}
}
/// (c) A drive that fails: the flags handed over before the failure are held and retried, never
/// dropped; the failure is reported (`unsaved`, the quit prompt's source) and later flags fail
/// with `NotSaved` and stay queued; once the drive is back, all of it is written, in order, with
/// no gap in the log.
#[test]
fn a_failed_write_is_reported_and_retried_never_dropped() {
let dir = temp_dir("fail");
let mut s = open(&dir);
let fail = Arc::new(AtomicBool::new(false));
slow_drive(&dir, Duration::from_millis(5), fail.clone());
let all = ids(&s);
fail.store(true, Ordering::SeqCst);
// handed over before the thread found out: accepted, then held
s.execute("photo.flag", &json!({"flag": "pick", "ids": [all[0].0]})).unwrap();
// the quit check reads it at once, not as of the last frame (review of #3557)
let t = Instant::now();
while s.library.as_ref().unwrap().journal().unwritten().is_none() {
assert!(t.elapsed() < Duration::from_secs(20), "the log thread never reported the failure");
std::thread::sleep(Duration::from_millis(10));
}
s.refresh_unsaved();
assert!(s.unsaved().is_some(), "refreshed for the quit check");
let (n, why) = s.unsaved().map(|(n, w)| (n, w.to_string())).unwrap();
assert!(n >= 1 && why.contains("the drive is gone"), "{n} {why}");
// later flags: refused with the reason, kept in memory and queued
let r = s.execute("photo.flag", &json!({"flag": "reject", "ids": [all[1].0]}));
assert!(matches!(r, Err(crate::EngineError::NotSaved(_))), "{r:?}");
assert_eq!(s.catalog.photo(all[1]).unwrap().flag, Flag::Reject, "applied in memory");
let _ = s.execute("photo.rate", &json!({"rating": 4, "ids": [all[2].0]}));
assert!(s.unsaved().is_some_and(|(n, _)| n >= 3), "{:?}", s.unsaved());
// the drive is back: the held record and the queued ones are written by the frame loop
fail.store(false, Ordering::SeqCst);
frames_until(&mut s, "written once the drive is back", |s| s.unsaved().is_none() && s.pending_log.is_empty());
let want = culled(&s);
let expect = s.catalog.to_snapshot();
// a crash now: the files alone hold everything, and the log has no gap (no damage on load)
s.library.as_mut().unwrap().journal_mut().flush().unwrap();
let copy = temp_dir("fail-copy");
std::fs::create_dir_all(©).unwrap();
for f in [lightcraft_catalog::journal::SNAPSHOT, lightcraft_catalog::journal::LOG] {
if dir.join(f).exists() {
std::fs::copy(dir.join(f), copy.join(f)).unwrap();
}
}
let (_, c, report) = lightcraft_catalog::Journal::open(Box::new(lightcraft_catalog::store::FsStore::open(©).unwrap())).unwrap();
assert!(report.damaged.is_none(), "{report:?}");
assert_eq!(c.to_snapshot(), expect);
let _ = std::fs::remove_dir_all(©);
fault::remove(&dir);
drop(s);
assert_eq!(culled(&open(&dir)), want);
let _ = std::fs::remove_dir_all(&dir);
}
/// (c) The drive stays broken for the log until the quit: the quit prompt is told (unsaved), and
/// the closing snapshot (a fresh file) saves the held flags, so they are there on the next open.
#[test]
fn a_log_that_stays_broken_is_saved_by_the_closing_snapshot() {
let dir = temp_dir("broken");
let mut s = open(&dir);
let all = ids(&s);
let fail = Arc::new(AtomicBool::new(false));
slow_drive(&dir, Duration::from_millis(5), fail.clone());
fail.store(true, Ordering::SeqCst);
s.execute("photo.flag", &json!({"flag": "reject", "ids": [all[0].0]})).unwrap();
frames_until(&mut s, "the failure is reported", |s| s.unsaved().is_some());
let _ = s.execute("photo.flag", &json!({"flag": "pick", "ids": [all[1].0]}));
let want = culled(&s);
// (the snapshot itself is written by the store, which the hook leaves alone; the log's reset
// after it fails, so close reports it, but the snapshot holds everything)
let _ = s.close_library();
// the snapshot holds them: the quit prompt must not say they would be lost (review of #3557)
assert!(s.unsaved().is_none(), "{:?}", s.unsaved());
drop(s);
fault::remove(&dir);
let again = open(&dir);
assert!(again.library.as_ref().unwrap().report.damaged.is_none());
assert_eq!(culled(&again), want);
drop(again);
let _ = std::fs::remove_dir_all(&dir);
}
/// The same library opened again in the same session (Settings, Libraries; a reopen after an
/// error): everything the old session's log thread was handed is in the files before the new one
/// reads them (review of #3557).
#[test]
fn reopening_the_same_library_reads_what_its_log_thread_was_handed() {
let dir = temp_dir("reopen");
let mut s = open(&dir);
fault::install(
&dir,
Arc::new(|step| {
if step == Step::Write {
std::thread::sleep(Duration::from_millis(300));
}
Ok(())
}),
);
let all = ids(&s);
s.execute("photo.flag", &json!({"flag": "reject", "ids": [all[0].0]})).unwrap();
s.open_library(&dir, true).unwrap();
fault::remove(&dir);
assert_eq!(s.catalog.photo(all[0]).unwrap().flag, Flag::Reject);
drop(s);
let _ = std::fs::remove_dir_all(&dir);
}
/// The same library opened again while its log's thread can't write what it was handed: the open
/// fails and the current library stays open with the change (held, retried), instead of a new
/// catalog read from files that miss it (review of #3557). Once the drive is back, the reopen
/// reads it.
#[test]
fn a_reopen_waits_for_what_the_log_thread_could_not_write() {
let dir = temp_dir("reopen-fail");
let mut s = open(&dir);
let all = ids(&s);
let fail = Arc::new(AtomicBool::new(false));
slow_drive(&dir, Duration::from_millis(5), fail.clone());
fail.store(true, Ordering::SeqCst);
s.execute("photo.flag", &json!({"flag": "reject", "ids": [all[0].0]})).unwrap();
let r = s.open_library(&dir, true);
assert!(matches!(r, Err(crate::EngineError::NotSaved(_))), "{r:?}");
assert_eq!(s.catalog.photo(all[0]).unwrap().flag, Flag::Reject, "the open library keeps the change");
assert!(s.unsaved().is_some(), "and says it isn't saved yet");
fail.store(false, Ordering::SeqCst);
let t = Instant::now();
while s.library.as_mut().unwrap().journal_mut().flush().is_err() {
assert!(t.elapsed() < Duration::from_secs(20), "never written");
std::thread::sleep(Duration::from_millis(50));
}
s.open_library(&dir, true).unwrap();
fault::remove(&dir);
assert_eq!(s.catalog.photo(all[0]).unwrap().flag, Flag::Reject);
drop(s);
let _ = std::fs::remove_dir_all(&dir);
}
/// A reopen while the log still can't be written, but after a snapshot that holds every change
/// (the log's reset after it failed): nothing is missing from the files, so the open goes ahead
/// (review of #3557).
#[test]
fn a_reopen_after_a_snapshot_that_holds_everything_goes_ahead() {
let dir = temp_dir("reopen-snap");
let mut s = open(&dir);
let all = ids(&s);
let fail = Arc::new(AtomicBool::new(false));
slow_drive(&dir, Duration::from_millis(5), fail.clone());
fail.store(true, Ordering::SeqCst);
s.execute("photo.flag", &json!({"flag": "reject", "ids": [all[0].0]})).unwrap();
let _ = s.close_library();
assert!(s.snapshot_holds_everything());
s.open_library(&dir, true).unwrap();
assert_eq!(s.catalog.photo(all[0]).unwrap().flag, Flag::Reject);
fault::remove(&dir);
drop(s);
assert_eq!(open(&dir).catalog.photo(all[0]).unwrap().flag, Flag::Reject);
let _ = std::fs::remove_dir_all(&dir);
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.