d84914ef1e5a
entries.rs
//! Writing the zip entries of a project file.
//!
//! Compressing the baked maps dominates a save (deflate level 6 over gigabytes of `f32`), and every
//! entry is independent, so on native targets the entries are compressed on several threads. Each
//! worker builds a one-entry zip in memory with the same options the serial writer uses; the writing
//! thread copies those entries into the real file in the original order with
//! [`ZipWriter::raw_copy_file`]. The compressed bytes of every entry and the entry order are
//! therefore the same as in a serial save.
//!
//! Workers share an estimated byte budget for entries started but not yet written. Stored entries
//! and entries larger than that budget stream directly into the destination, without a whole-entry
//! buffer. The estimate excludes allocator and compressor overhead; it is not a process memory cap.
use std::io::{Seek, Write};
use zip::ZipWriter;
use zip::write::SimpleFileOptions;
use super::zip_err;
use crate::Result;
/// Produces the uncompressed bytes of one entry into the zip writer.
pub(super) type Body<'a> = Box<dyn FnOnce(&mut dyn Write) -> std::io::Result<()> + Send + 'a>;
pub(super) struct Entry<'a> {
pub name: String,
pub options: SimpleFileOptions,
/// Estimated entry size in bytes, based on its uncompressed payload.
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
pub weight: u64,
pub body: Body<'a>,
}
/// How much of the machine a save may use. Native targets only; the browser saves on one thread.
#[cfg(not(target_arch = "wasm32"))]
mod limits {
use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
const FALLBACK_AVAILABLE_RAM_BYTES: u64 = 8 << 30;
/// Cores a save leaves to the UI (an autosave runs while the user keeps painting).
static RESERVED_CORES: AtomicUsize = AtomicUsize::new(DEFAULT_RESERVED_CORES);
/// Share of the available RAM, in percent, that entries in flight may hold.
static RAM_PERCENT: AtomicU8 = AtomicU8::new(DEFAULT_RAM_PERCENT);
pub const DEFAULT_RESERVED_CORES: usize = 1;
pub const DEFAULT_RAM_PERCENT: u8 = 50;
pub fn set(reserved_cores: usize, ram_percent: u8) {
RESERVED_CORES.store(reserved_cores, Ordering::Relaxed);
RAM_PERCENT.store(ram_percent.clamp(1, 100), Ordering::Relaxed);
}
pub fn get() -> (usize, u8) {
(RESERVED_CORES.load(Ordering::Relaxed), RAM_PERCENT.load(Ordering::Relaxed))
}
/// RAM the system can give out now, in bytes (the total when the system does not say it; 8 GiB
/// when the system tells nothing).
pub fn available_ram() -> u64 {
let Ok(mut sys) = sysinfo::System::new() else { return FALLBACK_AVAILABLE_RAM_BYTES };
sys.refresh_memory();
let available_bytes = reported_ram(sys.available_memory(), sys.total_memory());
sys.cgroup_limits().map_or(available_bytes, |limits| available_bytes.min(limits.free_memory))
}
fn reported_ram(available: u64, total: u64) -> u64 {
if available != 0 {
available
} else if total != 0 {
total
} else {
FALLBACK_AVAILABLE_RAM_BYTES
}
}
#[cfg(test)]
mod tests {
use super::reported_ram;
#[test]
fn missing_memory_readings_use_fallback() {
assert_eq!(reported_ram(0, 0), 8 << 30);
assert_eq!(reported_ram(0, 12 << 30), 12 << 30);
assert_eq!(reported_ram(3 << 30, 12 << 30), 3 << 30);
}
}
}
#[cfg(not(target_arch = "wasm32"))]
pub use limits::{DEFAULT_RAM_PERCENT, DEFAULT_RESERVED_CORES};
/// Set what saves may use: cores left to the UI (any number; at least one worker always runs) and
/// the percentage (1 to 100) of available RAM used as the estimated in-flight entry budget.
#[cfg(not(target_arch = "wasm32"))]
pub fn set_save_resources(reserved_cores: usize, ram_percent: u8) {
limits::set(reserved_cores, ram_percent);
}
/// Logical CPUs and currently available RAM in bytes, using the same readings as project saves.
#[cfg(not(target_arch = "wasm32"))]
pub fn available_save_resources() -> (usize, u64) {
let cores = std::thread::available_parallelism().map_or(1, usize::from);
(cores, limits::available_ram())
}
/// Estimated in-flight entry budget, shared by the save planner and the Settings readout.
#[cfg(not(target_arch = "wasm32"))]
pub fn save_ram_budget(available_ram_bytes: u64, ram_percent: u8) -> u64 {
u64::try_from(u128::from(available_ram_bytes) * u128::from(ram_percent.clamp(1, 100)) / 100).unwrap_or(u64::MAX)
}
/// The configured Settings share of currently available RAM, in bytes.
#[cfg(not(target_arch = "wasm32"))]
pub fn configured_ram_budget() -> u64 {
save_ram_budget(limits::available_ram(), limits::get().1)
}
/// Worker threads and in-flight byte budget for a save of `entries` entries.
#[cfg(not(target_arch = "wasm32"))]
pub(super) fn plan(cpus: usize, reserved_cores: usize, entries: usize, available_ram: u64, ram_percent: u8) -> (usize, u64) {
let workers = cpus.saturating_sub(reserved_cores).max(1).min(entries);
(workers, save_ram_budget(available_ram, ram_percent))
}
/// Write `entries` in order: in parallel where threads exist, else one after the other.
pub(super) fn write_entries<W: Write + Seek>(zip: &mut ZipWriter<W>, entries: Vec<Entry<'_>>) -> Result<()> {
#[cfg(not(target_arch = "wasm32"))]
{
let (cores, available_ram_bytes) = available_save_resources();
let (reserved_cores, ram_percent) = limits::get();
let (workers, budget) = plan(cores, reserved_cores, entries.len(), available_ram_bytes, ram_percent);
if workers > 1 {
return parallel::write(zip, entries, workers, budget);
}
}
write_serial(zip, entries)
}
/// One entry after the other straight into `zip`.
pub(super) fn write_serial<W: Write + Seek>(zip: &mut ZipWriter<W>, entries: Vec<Entry<'_>>) -> Result<()> {
for e in entries {
zip.start_file(e.name, e.options).map_err(zip_err)?;
(e.body)(zip)?;
}
Ok(())
}
#[cfg(not(target_arch = "wasm32"))]
pub(super) mod parallel {
use std::collections::BTreeMap;
use std::io::{Cursor, Seek, Write};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::{Condvar, Mutex, MutexGuard};
use zip::{CompressionMethod, ZipArchive, ZipWriter};
use super::Entry;
use crate::project_file::zip_err;
use crate::{FormatError, Result};
struct State {
/// Next entry to start.
next: usize,
/// Estimated bytes of the entries started and not yet written.
in_flight: u64,
/// Finished entries (one-entry zips) waiting for their turn.
ready: BTreeMap<usize, Result<Vec<u8>>>,
/// The writer failed: start nothing more.
stop: bool,
}
fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
fn wait<'a>(cv: &Condvar, g: MutexGuard<'a, State>) -> MutexGuard<'a, State> {
cv.wait(g).unwrap_or_else(|e| e.into_inner())
}
/// Compress one entry into a zip of its own.
fn compress(e: Entry<'_>) -> Result<Vec<u8>> {
let mut z = ZipWriter::new(Cursor::new(Vec::new()));
z.start_file(e.name, e.options).map_err(zip_err)?;
(e.body)(&mut z)?;
Ok(z.finish().map_err(zip_err)?.into_inner())
}
fn worker(slots: &[Mutex<Option<Entry<'_>>>], weights: &[u64], budget: u64, state: &Mutex<State>, cv: &Condvar) {
loop {
let entry_index = {
let mut state_guard = lock(state);
loop {
if state_guard.stop || state_guard.next >= slots.len() {
return;
}
let weight = weights.get(state_guard.next).copied().unwrap_or(0);
// Oversized entries stream separately, so every queued weight fits the budget
if state_guard.in_flight <= budget.saturating_sub(weight) {
let entry_index = state_guard.next;
state_guard.next += 1;
state_guard.in_flight = state_guard.in_flight.saturating_add(weight);
break entry_index;
}
state_guard = wait(cv, state_guard);
}
};
let entry = slots.get(entry_index).and_then(|slot| lock(slot).take());
let result = match entry {
Some(entry) => catch_unwind(AssertUnwindSafe(|| compress(entry)))
.unwrap_or_else(|_| Err(FormatError::Invalid("compressing a project entry failed".into()))),
None => Err(FormatError::Invalid("project entry missing".into())),
};
lock(state).ready.insert(entry_index, result);
cv.notify_all();
}
}
/// Copy the compressed entry of a one-entry zip into `zip` without recompressing.
fn copy_entry<W: Write + Seek>(zip: &mut ZipWriter<W>, bytes: Vec<u8>) -> Result<()> {
let mut a = ZipArchive::new(Cursor::new(bytes)).map_err(zip_err)?;
let f = a.by_index_raw(0).map_err(zip_err)?;
zip.raw_copy_file(f).map_err(zip_err)
}
fn streams_directly(entry: &Entry<'_>, budget: u64) -> bool {
let stored_options = entry.options.compression_method(CompressionMethod::Stored);
entry.options == stored_options || entry.weight > budget
}
pub(in crate::project_file) fn write<W: Write + Seek>(zip: &mut ZipWriter<W>, entries: Vec<Entry<'_>>, workers: usize, budget: u64) -> Result<()> {
if workers == 0 {
return super::write_serial(zip, entries);
}
let mut remaining = entries.into_iter().peekable();
while let Some(entry) = remaining.next() {
// Storing bytes gains nothing from a worker; oversized entries must not get a whole-entry buffer
if streams_directly(&entry, budget) {
zip.start_file(entry.name, entry.options).map_err(zip_err)?;
(entry.body)(zip)?;
continue;
}
let mut batch = vec![entry];
while remaining.peek().is_some_and(|entry| !streams_directly(entry, budget)) {
if let Some(entry) = remaining.next() {
batch.push(entry);
}
}
// Finish the workers before direct compression, keeping the CPU allowance shared
write_buffered(zip, batch, workers, budget)?;
}
Ok(())
}
fn write_buffered<W: Write + Seek>(zip: &mut ZipWriter<W>, entries: Vec<Entry<'_>>, workers: usize, budget: u64) -> Result<()> {
if entries.len() == 1 {
return super::write_serial(zip, entries);
}
let weights: Vec<u64> = entries.iter().map(|entry| entry.weight).collect();
let slots: Vec<Mutex<Option<Entry<'_>>>> = entries.into_iter().map(|entry| Mutex::new(Some(entry))).collect();
let state = Mutex::new(State { next: 0, in_flight: 0, ready: BTreeMap::new(), stop: false });
let cv = Condvar::new();
std::thread::scope(|scope| {
let mut started = 0;
for _ in 0..workers.min(slots.len()) {
// If thread creation fails, the workers already started drain the queue
if std::thread::Builder::new().spawn_scoped(scope, || worker(&slots, &weights, budget, &state, &cv)).is_ok() {
started += 1;
}
}
let result = (|| -> Result<()> {
for (entry_index, weight) in weights.iter().enumerate() {
if started == 0 {
// If no workers started, stream directly into the destination
let entry =
slots.get(entry_index).and_then(|slot| lock(slot).take()).ok_or_else(|| FormatError::Invalid("project entry missing".into()))?;
zip.start_file(entry.name, entry.options).map_err(zip_err)?;
(entry.body)(zip)?;
continue;
}
let done = {
let mut state_guard = lock(&state);
loop {
if let Some(result) = state_guard.ready.remove(&entry_index) {
break result;
}
state_guard = wait(&cv, state_guard);
}
};
let copied = done.and_then(|bytes| copy_entry(zip, bytes));
// Release the entry's share only once its bytes are gone
{
let mut state_guard = lock(&state);
state_guard.in_flight = state_guard.in_flight.saturating_sub(*weight);
}
cv.notify_all();
copied?;
}
Ok(())
})();
if result.is_err() {
lock(&state).stop = true;
cv.notify_all();
}
result
})
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.