856f865cf98d
scratch.rs
//! Process-wide, capped pool of large scratch buffers.
//!
//! With glibc's mmap threshold pinned (`MALLOC_TRIM_THRESHOLD_`, which is how PhotoCraft returns
//! memory to the OS), every allocation over 128 KiB is a fresh `mmap` that is page-faulted in and
//! `munmap`ed on free. Tiles, compositor bands and filter scratch are all that size, so a single
//! viewport render took 350× the page faults. This pool keeps freed buffers (at least
//! [`MIN_POOLED_BYTES`] of capacity) on per-type free lists and hands them back out, so the
//! steady state allocates nothing.
//!
//! Retention is capped by [`set_budget`] (default [`DEFAULT_BUDGET`]); [`release`] drops
//! everything, which is what returns the memory to the OS. The engine releases when a document
//! closes; off wasm a background thread also releases once the pool has gone unused for
//! [`IDLE_RELEASE_SECS`], so every host (app, CLI, tests) gives memory back the same way.
use std::collections::{BTreeMap, VecDeque};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Mutex, PoisonError};
/// Buffers with less capacity than this are cheap from the heap and are never retained.
pub const MIN_POOLED_BYTES: usize = 64 * 1024;
/// Default cap on the bytes the pool retains: about the working set of a 36 MP filter (tiles,
/// halo images and result bands); a browser's wasm heap never shrinks, so there it is small.
pub const DEFAULT_BUDGET: usize = if cfg!(target_arch = "wasm32") { 256 << 20 } else { 2 << 30 };
/// Pooled bytes a miss never evicts. A filter's output tiles are all misses (the old tiles stay
/// in the undo history); evicting every byte they need also dropped the scratch planes the next
/// tile reuses. On a 36 MP Gaussian Blur with the trim threshold set, 256 MiB keeps the timings
/// at glibc's defaults (128 MiB: r 40 ~17% slower) for ~10% more peak memory (0: ~20% slower).
pub const DEFAULT_MISS_FLOOR: usize = if cfg!(target_arch = "wasm32") { 64 << 20 } else { 256 << 20 };
static BUDGET: AtomicUsize = AtomicUsize::new(DEFAULT_BUDGET);
static MISS_FLOOR: AtomicUsize = AtomicUsize::new(DEFAULT_MISS_FLOOR);
static RETAINED: AtomicUsize = AtomicUsize::new(0);
static USES: AtomicU64 = AtomicU64::new(0);
static IDLE_RELEASE: AtomicBool = AtomicBool::new(true);
/// Seconds without a [`take`] or [`give`] after which the idle thread releases the pool.
pub const IDLE_RELEASE_SECS: u64 = 2;
/// Starts the idle-release thread once (off wasm, where there are threads). If the thread
/// can't be spawned the pool still works; it is then released only by [`release`].
fn start_idle_release() {
#[cfg(not(target_arch = "wasm32"))]
{
static STARTED: std::sync::Once = std::sync::Once::new();
STARTED.call_once(|| {
let _ = std::thread::Builder::new().name("scratch-idle".into()).spawn(|| {
let mut seen = uses();
loop {
std::thread::sleep(std::time::Duration::from_secs(IDLE_RELEASE_SECS));
let now = uses();
if now == seen {
// Decided under the lock, so `set_idle_release(false)` can't return
// while a release it should have stopped is about to start.
let old = {
let mut pool = lock();
// A `give` or `take` since `now` was read means the pool is in use.
(uses() == now && retained_bytes() > 0 && IDLE_RELEASE.load(Ordering::Relaxed)).then(|| empty(&mut pool))
};
drop(old);
}
seen = now;
}
});
});
}
}
/// One free list: buffers keyed by capacity (in elements), each with the sequence number it
/// was returned at (oldest first within a capacity).
#[doc(hidden)]
pub type FreeList<T> = BTreeMap<usize, VecDeque<(u64, Vec<T>)>>;
/// All the free lists behind one lock, so the oldest buffer of any type can be evicted.
/// `order` maps every pooled buffer's sequence number to its type and capacity, so the oldest
/// is found in O(log n); best-fit lookups are a range query on the capacity key.
#[doc(hidden)]
pub struct Pool {
seq: u64,
order: BTreeMap<u64, (Tag, usize)>,
u8s: FreeList<u8>,
u16s: FreeList<u16>,
f32s: FreeList<f32>,
px4: FreeList<[f32; 4]>,
}
static POOL: Mutex<Pool> =
Mutex::new(Pool { seq: 0, order: BTreeMap::new(), u8s: BTreeMap::new(), u16s: BTreeMap::new(), f32s: BTreeMap::new(), px4: BTreeMap::new() });
/// Which free list a buffer lives on.
#[doc(hidden)]
#[derive(Clone, Copy)]
pub enum Tag {
U8,
U16,
F32,
Px4,
}
/// A buffer evicted from the pool, dropped by the caller once the lock is released.
#[allow(dead_code)]
enum Evicted {
U8(Vec<u8>),
U16(Vec<u16>),
F32(Vec<f32>),
Px4(Vec<[f32; 4]>),
}
/// Element types that have a free list. Implemented for the buffers the hot paths use.
pub trait Pooled: Copy + Send + 'static {
#[doc(hidden)]
const TAG: Tag;
#[doc(hidden)]
fn list(pool: &mut Pool) -> &mut FreeList<Self>;
}
macro_rules! pooled {
($($t:ty => $field:ident, $tag:ident),*) => {$(
impl Pooled for $t {
const TAG: Tag = Tag::$tag;
fn list(pool: &mut Pool) -> &mut FreeList<Self> {
&mut pool.$field
}
}
)*};
}
pooled!(u8 => u8s, U8, u16 => u16s, U16, f32 => f32s, F32, [f32; 4] => px4, Px4);
fn bytes_of<T>(v: &Vec<T>) -> usize {
v.capacity().saturating_mul(std::mem::size_of::<T>())
}
fn lock() -> std::sync::MutexGuard<'static, Pool> {
POOL.lock().unwrap_or_else(PoisonError::into_inner)
}
/// Remove the oldest buffer of capacity `cap` from `list`.
fn drop_oldest<T>(list: &mut FreeList<T>, cap: usize) -> Option<Vec<T>> {
let queue = list.get_mut(&cap)?;
let v = queue.pop_front()?.1;
if queue.is_empty() {
list.remove(&cap);
}
Some(v)
}
impl Pool {
/// Remove the buffer returned longest ago, of any type. `None` when the pool is empty.
fn evict_oldest(&mut self) -> Option<Evicted> {
let (_, (tag, cap)) = self.order.pop_first()?;
let evicted = match tag {
Tag::U8 => drop_oldest(&mut self.u8s, cap).map(Evicted::U8),
Tag::U16 => drop_oldest(&mut self.u16s, cap).map(Evicted::U16),
Tag::F32 => drop_oldest(&mut self.f32s, cap).map(Evicted::F32),
Tag::Px4 => drop_oldest(&mut self.px4, cap).map(Evicted::Px4),
};
if let Some(e) = &evicted {
let freed = match e {
Evicted::U8(v) => bytes_of(v),
Evicted::U16(v) => bytes_of(v),
Evicted::F32(v) => bytes_of(v),
Evicted::Px4(v) => bytes_of(v),
};
RETAINED.fetch_sub(freed, Ordering::Relaxed);
}
evicted
}
}
/// A `Vec` of `len` copies of `fill`, reusing a pooled buffer when one fits: the one whose
/// capacity equals `len`, else the smallest with capacity in `len..=2*len`.
pub fn take<T: Pooled>(len: usize, fill: T) -> Vec<T> {
take_with(len, fill, 2)
}
/// An empty `Vec` with room for at least `cap` elements, reusing a pooled buffer when one fits.
pub fn take_cap<T: Pooled>(cap: usize) -> Vec<T> {
USES.fetch_add(1, Ordering::Relaxed);
match pop::<T>(cap, 2) {
Some(mut v) => {
v.clear();
v
}
None => Vec::with_capacity(cap),
}
}
/// Like [`take`] but only reuses a buffer whose capacity is exactly `len`, so
/// `into_boxed_slice` never reallocates.
pub fn take_exact<T: Pooled>(len: usize, fill: T) -> Vec<T> {
take_with(len, fill, 1)
}
/// Pop the best-fitting free buffer: capacity `len`, else the smallest up to `len * slack`.
///
/// On a miss the caller allocates `len` fresh, so the oldest free buffers of at least that many
/// bytes are dropped first, down to [`set_miss_floor`]: memory the pool can't reuse for this request
/// goes back instead of piling up beside the new buffer, so pool plus live buffers exceed what
/// the program would hold without the pool by at most the floor (#10479).
fn pop<T: Pooled>(len: usize, slack: usize) -> Option<Vec<T>> {
let need = len.saturating_mul(std::mem::size_of::<T>());
if need < MIN_POOLED_BYTES {
return None;
}
let mut evicted = Vec::new();
let hit = {
let mut pool = lock();
let max = len.saturating_mul(slack);
let list = T::list(&mut pool);
let cap = list.range(len..=max).next().map(|(&cap, _)| cap);
// The newest buffer of that capacity is the one most likely still in cache.
let found = cap.and_then(|cap| {
let queue = list.get_mut(&cap)?;
let entry = queue.pop_back();
if queue.is_empty() {
list.remove(&cap);
}
entry
});
match found {
Some((seq, v)) => {
pool.order.remove(&seq);
RETAINED.fetch_sub(bytes_of(&v), Ordering::Relaxed);
Some(v)
}
None => {
let target = RETAINED.load(Ordering::Relaxed).saturating_sub(need).max(MISS_FLOOR.load(Ordering::Relaxed));
while RETAINED.load(Ordering::Relaxed) > target {
match pool.evict_oldest() {
Some(e) => evicted.push(e),
None => break,
}
}
None
}
}
};
// Freed outside the lock.
drop(evicted);
hit
}
fn take_with<T: Pooled>(len: usize, fill: T, slack: usize) -> Vec<T> {
USES.fetch_add(1, Ordering::Relaxed);
if len == 0 {
return Vec::new();
}
match pop::<T>(len, slack) {
Some(mut v) => {
v.clear();
v.resize(len, fill);
v
}
None => vec![fill; len],
}
}
/// Fallible [`take`]: on a pool miss the heap allocation is `try_reserve_exact`, so `None` means
/// the allocation failed rather than aborting. The pool lookup keeps `take`'s existing 2x slack.
pub fn try_take<T: Pooled>(len: usize, fill: T) -> Option<Vec<T>> {
try_take_with_slack(len, fill, 2)
}
/// Fallible [`take_exact`]: on a pool miss the heap allocation is `try_reserve_exact`, so
/// `None` means the allocation failed rather than aborting. Only exact-capacity buffers are reused.
pub fn try_take_exact<T: Pooled>(len: usize, fill: T) -> Option<Vec<T>> {
try_take_with_slack(len, fill, 1)
}
fn try_take_with_slack<T: Pooled>(len: usize, fill: T, slack: usize) -> Option<Vec<T>> {
USES.fetch_add(1, Ordering::Relaxed);
let mut v = match pop::<T>(len, slack) {
Some(v) => v,
None => {
let mut v = Vec::new();
if v.try_reserve_exact(len).is_err() {
// The pool may hold up to the miss floor of buffers this request couldn't use:
// give them back, then try once more before failing the caller.
release();
v.try_reserve_exact(len).ok()?;
}
v
}
};
v.clear();
v.resize(len, fill);
Some(v)
}
/// Return a buffer to the pool. Small buffers, and buffers bigger than the whole budget, are
/// dropped; otherwise the buffers returned longest ago (of any type) make room for it.
pub fn give<T: Pooled>(v: Vec<T>) {
USES.fetch_add(1, Ordering::Relaxed);
let bytes = bytes_of(&v);
if bytes < MIN_POOLED_BYTES {
return;
}
let mut evicted = Vec::new();
{
let mut pool = lock();
// Read under the lock, which `set_budget` also takes, so no insert uses a stale budget.
let budget = BUDGET.load(Ordering::Relaxed);
if bytes > budget {
return;
}
start_idle_release();
while RETAINED.load(Ordering::Relaxed).saturating_add(bytes) > budget {
match pool.evict_oldest() {
Some(e) => evicted.push(e),
None => break,
}
}
RETAINED.fetch_add(bytes, Ordering::Relaxed);
pool.seq += 1;
let seq = pool.seq;
let cap = v.capacity();
pool.order.insert(seq, (T::TAG, cap));
T::list(&mut pool).entry(cap).or_default().push_back((seq, v));
}
// Freed outside the lock.
drop(evicted);
}
/// Set the cap on retained bytes (existing entries beyond it go on the next [`give`]).
pub fn set_budget(bytes: usize) {
let _pool = lock();
BUDGET.store(bytes, Ordering::Relaxed);
}
/// Set the pooled bytes a miss never evicts (default [`DEFAULT_MISS_FLOOR`]).
pub fn set_miss_floor(bytes: usize) {
MISS_FLOOR.store(bytes, Ordering::Relaxed);
}
/// Let the idle thread release the pool (the default) or not. Tests that check
/// [`retained_bytes`] turn it off, so a slow runner can't have the pool emptied under them.
pub fn set_idle_release(on: bool) {
// Under the lock: once this returns, the idle thread can't be midway into a release.
let _pool = lock();
IDLE_RELEASE.store(on, Ordering::Relaxed);
}
/// Bytes currently held by the pool.
pub fn retained_bytes() -> usize {
RETAINED.load(Ordering::Relaxed)
}
/// Number of [`take`]/[`give`] calls so far; a changing value means the pool is in use.
pub fn uses() -> u64 {
USES.load(Ordering::Relaxed)
}
/// Drop every pooled buffer, handing the memory back to the OS.
pub fn release() {
let old = empty(&mut lock());
// Freed outside the lock.
drop(old);
}
/// Everything `pool` holds, swapped out of it (to be freed once the lock is dropped).
fn empty(pool: &mut Pool) -> Pool {
RETAINED.store(0, Ordering::Relaxed);
let seq = pool.seq;
std::mem::replace(pool, Pool { seq, order: BTreeMap::new(), u8s: BTreeMap::new(), u16s: BTreeMap::new(), f32s: BTreeMap::new(), px4: BTreeMap::new() })
}
/// A pooled buffer that derefs to a slice and returns itself to the pool on drop.
pub struct Buf<T: Pooled>(Vec<T>);
impl<T: Pooled> Buf<T> {
/// `len` copies of `fill` from the pool.
pub fn new(len: usize, fill: T) -> Self {
Buf(take(len, fill))
}
}
impl<T: Pooled> std::ops::Deref for Buf<T> {
type Target = [T];
fn deref(&self) -> &[T] {
&self.0
}
}
impl<T: Pooled> std::ops::DerefMut for Buf<T> {
fn deref_mut(&mut self) -> &mut [T] {
&mut self.0
}
}
impl<T: Pooled> Drop for Buf<T> {
fn drop(&mut self) {
give(std::mem::take(&mut self.0));
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.