856f865cf98d
scratch_slow_folder.rs
//! A scratch folder that stops answering (a dead hard-mounted network share) must not freeze
//! anything: status reads stay prompt, opening and measuring wait a bounded time, retries do not
//! pile up threads, leaving the folder works, and a late answer applies only to the folder it
//! was about.
//!
//! One test function: the spill manager is process-wide, so its steps must not interleave.
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Condvar, Mutex};
use std::time::{Duration, Instant};
use photocraft_color::{ColorMode, PixelFormat, SampleType};
use photocraft_geom::{Rect, TILE_SIZE};
use photocraft_raster::{Surface, spill};
const TILE_BYTES: usize = (TILE_SIZE * TILE_SIZE) as usize * 4;
/// Folders whose name starts with "hung" stall their queries while this is on.
static STALL: AtomicBool = AtomicBool::new(false);
/// Queries that reached the stall since the last reset.
static ENTERED: AtomicUsize = AtomicUsize::new(0);
/// `true` once the stalled queries may go on.
static GATE: (Mutex<bool>, Condvar) = (Mutex::new(false), Condvar::new());
fn stall(dir: &Path) {
let hung = dir.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with("hung"));
if !STALL.load(Ordering::SeqCst) || !hung {
return;
}
ENTERED.fetch_add(1, Ordering::SeqCst);
let mut open = GATE.0.lock().unwrap();
while !*open {
open = GATE.1.wait(open).unwrap();
}
}
fn arm() {
ENTERED.store(0, Ordering::SeqCst);
*GATE.0.lock().unwrap() = false;
STALL.store(true, Ordering::SeqCst);
}
fn release() {
STALL.store(false, Ordering::SeqCst);
*GATE.0.lock().unwrap() = true;
GATE.1.notify_all();
}
fn poll_until(what: &str, cond: impl Fn() -> bool) {
let start = Instant::now();
while !cond() {
assert!(start.elapsed() < Duration::from_secs(30), "timed out waiting for {what}: {:?}", spill::stats());
std::thread::sleep(Duration::from_millis(10));
}
}
/// Run `f` and require that it returns well before a query's wait.
fn prompt<T>(what: &str, f: impl FnOnce() -> T) -> T {
let start = Instant::now();
let out = f();
assert!(start.elapsed() < Duration::from_millis(1500), "{what} took {:?}", start.elapsed());
out
}
/// A query's 2 s wait plus a scheduling allowance.
const QUERY_DEADLINE: Duration = Duration::from_secs(6);
/// Run `f` and require that it returns within [`QUERY_DEADLINE`].
fn bounded<T>(what: &str, f: impl FnOnce() -> T) -> T {
let start = Instant::now();
let out = f();
assert!(start.elapsed() < QUERY_DEADLINE, "{what} took {:?}", start.elapsed());
out
}
fn rgba8() -> PixelFormat {
PixelFormat { mode: ColorMode::Rgb, sample: SampleType::U8, alpha: true }
}
/// Incompressible bytes, different per seed.
fn noise(seed: u32, len: usize) -> Vec<u8> {
let mut x = seed.wrapping_mul(2_654_435_761).max(1);
(0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
x as u8
})
.collect()
}
fn surface(seed: u32, n: i32) -> Surface {
let mut s = Surface::new(rgba8());
for i in 0..n {
let r = Rect::from_xywh(i * TILE_SIZE, 0, TILE_SIZE as u32, TILE_SIZE as u32);
s.write_interleaved(r, &noise(seed + i as u32, TILE_BYTES));
}
s
}
#[test]
fn a_folder_that_stops_answering_never_freezes_the_app() {
let _scratch = crate::lock(&crate::SCRATCH);
let root = std::env::temp_dir().join(format!("photocraft-scratch-slow-{}", std::process::id()));
let good = root.join("good");
let good2 = root.join("good2");
let hung1 = root.join("hung1");
let hung2 = root.join("hung2");
let hung3 = root.join("hung3");
spill::set_query_hook_for_tests(Some(stall));
// A. A healthy folder.
spill::configure(Some(&good), 1 << 20).unwrap();
assert!(spill::enabled() && !spill::paused());
let measurable = spill::scratch_capacity().is_some();
// B. Configuring a folder that never answers: status reads stay prompt meanwhile (they
// blocked on the config lock before), and configure gives up after its wait.
arm();
let began = Instant::now();
let configuring = {
let hung1 = hung1.clone();
std::thread::spawn(move || spill::configure(Some(&hung1), 1 << 20))
};
poll_until("the first query to start", || ENTERED.load(Ordering::SeqCst) == 1);
prompt("last_error", spill::last_error);
prompt("check_integrity", spill::check_integrity).ok();
prompt("stats", spill::stats);
prompt("free_space_unmeasured", spill::free_space_unmeasured);
prompt("paused", spill::paused);
let result = configuring.join().unwrap();
assert!(began.elapsed() < QUERY_DEADLINE, "configure of a hung folder took {:?}", began.elapsed());
assert!(result.as_ref().err().is_some_and(|e| e.contains("not responding")), "{result:?}");
assert!(spill::paused() && !spill::enabled());
assert_eq!(spill::stats().dir.as_deref(), Some(hung1.as_path()));
// C. Retries neither wait nor start more threads: the question is already being asked.
for _ in 0..5 {
spill::retry_now_for_tests();
prompt("maintain_budget", spill::maintain_budget);
}
assert_eq!(ENTERED.load(Ordering::SeqCst), 1, "a retry started a second query");
assert!(spill::paused());
// D. Leaving the dead folder works while its query is stuck.
prompt("configure away from a hung folder", || spill::configure(Some(&good2), 1 << 20)).unwrap();
assert!(spill::enabled() && !spill::paused());
// E. The stuck query's late answer is about a folder that is no longer current: discarded.
release();
poll_until("the stuck query to settle", || spill::query_threads_for_tests() == 0);
let st = spill::stats();
assert_eq!(st.dir.as_deref(), Some(good2.as_path()), "{st:?}");
assert!(spill::enabled() && !spill::paused() && st.last_error.is_none(), "{st:?}");
// F. A late answer for the current folder applies: spilling resumes by itself.
arm();
let result = bounded("configure of a second hung folder", || spill::configure(Some(&hung2), 1 << 20));
assert!(result.is_err());
assert!(spill::paused());
release();
poll_until("the late answer to resume spilling", || spill::enabled() && !spill::paused() && spill::last_error().is_none());
assert_eq!(spill::stats().dir.as_deref(), Some(hung2.as_path()));
// G. The open folder stops answering when measured: no answer counts as unknown, spilling
// goes on, and asking again costs nothing.
spill::configure(Some(&hung3), 1 << 20).unwrap();
arm();
assert_eq!(bounded("scratch_capacity", spill::scratch_capacity), None);
assert!(spill::free_space_unmeasured());
assert!(spill::stats().free_space_unmeasured);
assert!(spill::enabled() && !spill::paused());
for _ in 0..5 {
assert_eq!(prompt("scratch_capacity", spill::scratch_capacity), None);
}
assert_eq!(ENTERED.load(Ordering::SeqCst), 1, "asking again started a second query");
release();
if measurable {
poll_until("the late answer to measure the volume", || !spill::free_space_unmeasured());
}
poll_until("the previous query to finish", || spill::query_threads_for_tests() == 0);
// H. The growth check never stalls eviction: tiles still spill while it is unanswered.
spill::set_budget(8 * TILE_BYTES);
arm();
spill::set_size_cap(Some(1 << 50));
let trigger_tiles = surface(1, 32);
poll_until("the growth query to reach the stall", || ENTERED.load(Ordering::SeqCst) >= 1);
let evictions = spill::stats().evictions;
let tiles = surface(33, 32);
prompt("settle", spill::settle);
assert!(spill::stats().evictions > evictions, "{:?}", spill::stats());
poll_until("the unanswered growth check to count as unknown", spill::free_space_unmeasured);
release();
if measurable {
poll_until("the growth check to be answered", || !spill::free_space_unmeasured());
}
spill::set_size_cap(None);
// Cleanup.
spill::set_query_hook_for_tests(None);
drop(tiles);
drop(trigger_tiles);
spill::configure(None, 0).unwrap();
let _ = std::fs::remove_dir_all(&root);
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.