GitCorner

Code

ossartcraftfilmcraft🌐
f67d38b35cc5
//! Recycled plane buffers.
//!
//! A decoder allocates three planes per picture and the caches free them a few hundred frames
//! later. At megabytes per plane that traffic leaves the system allocator holding gigabytes of
//! freed pages (measured: 2.5 GB retained next to 2.5 GB live after twelve clips). Decoders take
//! their planes here instead ([`take_u8`], [`take_u16`]) and caches hand evicted frames back
//! ([`recycle`]), so steady-state decoding allocates nothing.
//!
//! The compositor's working images (premultiplied linear `f32`, 33 MB at 1080p) get the same
//! treatment ([`take_f32_overwritten`], [`recycle_f32`]): allocated and freed once per layer per
//! frame they cost a zero-fill and fresh page faults every time.
//!
//! The shelves are sized for the busiest work, so they are a lot to keep once it is over: whoever
//! finishes such work (a standalone export) calls [`trim_f32`] and [`trim_planes`].

use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};

use crate::{PixelData, VideoFrame};

/// Bytes of idle buffers kept per sample type; a buffer that does not fit is freed.
const MAX_BYTES: usize = 192 << 20;

/// Smaller buffers are not worth keeping (the allocator recycles those well).
const MIN_BYTES: usize = 64 << 10;

struct Shelf<T> {
    bufs: Vec<Vec<T>>,
    bytes: usize,
}

impl<T> Shelf<T> {
    const fn new() -> Self {
        Self { bufs: Vec::new(), bytes: 0 }
    }

    fn size(b: &Vec<T>) -> usize {
        b.capacity().saturating_mul(std::mem::size_of::<T>())
    }

    /// The smallest idle buffer that holds `len` samples without wasting over a quarter of it.
    fn take(&mut self, len: usize) -> Option<Vec<T>> {
        let k = self
            .bufs
            .iter()
            .enumerate()
            .filter(|(_, b)| b.capacity() >= len && b.capacity() - len <= len / 4)
            .min_by_key(|(_, b)| b.capacity())
            .map(|(k, _)| k)?;
        let b = self.bufs.swap_remove(k);
        self.bytes = self.bytes.saturating_sub(Self::size(&b));
        Some(b)
    }

    fn give(&mut self, mut b: Vec<T>, max_bytes: usize) {
        let n = Self::size(&b);
        if n < MIN_BYTES || self.bytes.saturating_add(n) > max_bytes {
            return;
        }
        b.clear();
        self.bytes += n;
        self.bufs.push(b);
    }
}

static U8: Mutex<Shelf<u8>> = Mutex::new(Shelf::new());
static U16: Mutex<Shelf<u16>> = Mutex::new(Shelf::new());
static REUSED: AtomicU64 = AtomicU64::new(0);

/// Idle float images. Unlike the plane shelves a buffer keeps its length *and its old contents*:
/// whoever overwrites every element gets it back without a zero-fill pass over tens of megabytes.
static F32: Mutex<Vec<Vec<f32>>> = Mutex::new(Vec::new());

/// Bytes of idle float images kept; a buffer that does not fit is freed.
const F32_MAX_BYTES: usize = 320 << 20;

fn take<T>(shelf: &Mutex<Shelf<T>>, len: usize) -> Vec<T> {
    let found = shelf.lock().unwrap_or_else(PoisonError::into_inner).take(len);
    match found {
        Some(b) => {
            REUSED.fetch_add(1, Ordering::Relaxed);
            b
        }
        None => Vec::with_capacity(len),
    }
}

fn give<T>(shelf: &Mutex<Shelf<T>>, plane: Arc<Vec<T>>) {
    // a plane something else still shows (a monitor, a GPU upload in flight) is theirs
    if let Ok(b) = Arc::try_unwrap(plane) {
        shelf.lock().unwrap_or_else(PoisonError::into_inner).give(b, MAX_BYTES);
    }
}

/// An empty buffer with room for `len` samples: a recycled one when one fits.
pub fn take_u8(len: usize) -> Vec<u8> {
    take(&U8, len)
}

/// [`take_u8`] for 16-bit planes.
pub fn take_u16(len: usize) -> Vec<u16> {
    take(&U16, len)
}

/// A buffer of exactly `len` floats whose contents are unspecified (whatever it held last, or zeros
/// when none was idle): the caller must write every element before it reads any. Handing it back
/// ([`recycle_f32`]) is optional; a buffer that is dropped is simply freed.
pub fn take_f32_overwritten(len: usize) -> Vec<f32> {
    if len.saturating_mul(4) >= MIN_BYTES {
        let mut idle = F32.lock().unwrap_or_else(PoisonError::into_inner);
        if let Some(k) = idle.iter().position(|b| b.len() == len) {
            REUSED.fetch_add(1, Ordering::Relaxed);
            return idle.swap_remove(k);
        }
    }
    vec![0.0; len]
}

/// Keep a float image nobody needs any more for the next [`take_f32_overwritten`] of its length.
pub fn recycle_f32(buf: Vec<f32>) {
    let bytes = buf.len().saturating_mul(4);
    if bytes < MIN_BYTES {
        return;
    }
    let mut idle = F32.lock().unwrap_or_else(PoisonError::into_inner);
    let held: usize = idle.iter().map(|b| b.len().saturating_mul(4)).sum();
    if held.saturating_add(bytes) <= F32_MAX_BYTES {
        idle.push(buf);
    }
}

/// Hand back a frame nobody needs any more: its planes are kept for the next [`take_u8`] /
/// [`take_u16`] unless something else still holds the frame or a plane.
pub fn recycle(frame: Arc<VideoFrame>) {
    let Ok(f) = Arc::try_unwrap(frame) else { return };
    match f.data {
        PixelData::Rgba8(d) => give(&U8, d),
        PixelData::RgbaF32(_) => {}
        PixelData::Yuv8 { planes, alpha, .. } => planes.into_iter().chain(alpha).for_each(|p| give(&U8, p)),
        PixelData::Yuv16 { planes, alpha, .. } => planes.into_iter().chain(alpha).for_each(|p| give(&U16, p)),
    }
}

/// Free the idle float images. A standalone export leaves that shelf full (up to [`F32_MAX_BYTES`]
/// of images, each the size of that export's frames) and nothing else asks for those sizes again,
/// so they would sit idle until the next export. Images still in use are not touched. The 8- and
/// 16-bit plane shelves are left alone: playback recycles decoded frames through them all the time
/// (see [`trim_planes`] for the end of such work).
pub fn trim_f32() {
    // taken under the lock, freed after it: giving back hundreds of megabytes takes a moment
    let floats = std::mem::take(&mut *F32.lock().unwrap_or_else(PoisonError::into_inner));
    drop(floats);
}

/// Free the idle 8- and 16-bit plane buffers (up to [`MAX_BYTES`] each). A standalone export leaves
/// both shelves full of planes the size of its sources' pictures, and nothing trims them. Planes
/// still in use are not touched, and the shelves fill again as soon as decoding resumes.
pub fn trim_planes() {
    // taken under the lock, freed after it: giving back hundreds of megabytes takes a moment
    let u8s = std::mem::replace(&mut *U8.lock().unwrap_or_else(PoisonError::into_inner), Shelf::new());
    let u16s = std::mem::replace(&mut *U16.lock().unwrap_or_else(PoisonError::into_inner), Shelf::new());
    drop((u8s, u16s));
}

/// Idle buffers held and buffers handed out again so far (`perf.stats`).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct PoolStats {
    pub idle_bytes: usize,
    /// The part of `idle_bytes` held as float images ([`recycle_f32`]).
    pub f32_idle_bytes: usize,
    pub reused: u64,
}

pub fn stats() -> PoolStats {
    let idle = |n: usize, m: usize| n.saturating_add(m);
    let a = U8.lock().unwrap_or_else(PoisonError::into_inner).bytes;
    let b = U16.lock().unwrap_or_else(PoisonError::into_inner).bytes;
    let c: usize = F32.lock().unwrap_or_else(PoisonError::into_inner).iter().map(|b| b.len().saturating_mul(4)).sum();
    PoolStats { idle_bytes: idle(idle(a, b), c), f32_idle_bytes: c, reused: REUSED.load(Ordering::Relaxed) }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::Chroma;

    /// The plane shelves may hold up to this much alongside the float shelf in a test binary.
    const U8_BYTES_UPPER_BOUND: usize = 2 * MAX_BYTES;

    #[test]
    fn shelf_hands_back_a_fitting_buffer_and_stays_within_its_budget() {
        let mut s = Shelf::<u8>::new();
        s.give(Vec::with_capacity(1 << 20), 3 << 20);
        s.give(Vec::with_capacity(2 << 20), 3 << 20);
        assert_eq!((s.bufs.len(), s.bytes), (2, 3 << 20));
        // over the budget and too small to keep: freed
        s.give(Vec::with_capacity(1 << 20), 3 << 20);
        s.give(Vec::with_capacity(1024), 3 << 20);
        assert_eq!(s.bufs.len(), 2);
        // nothing fits a far smaller or a larger request
        assert!(s.take(100 << 10).is_none());
        assert!(s.take(3 << 20).is_none());
        // the smallest buffer that fits, with its contents gone
        let b = s.take((1 << 20) - 4096).expect("fits");
        assert_eq!((b.len(), b.capacity()), (0, 1 << 20));
        assert_eq!((s.bufs.len(), s.bytes), (1, 2 << 20));
        assert!(s.take(1 << 20).is_none(), "2 MiB wastes too much for 1 MiB");
    }

    #[test]
    fn float_images_come_back_whole_and_unchanged_within_their_budget() {
        // a length nothing else in this test binary asks for
        let len = 1_000_003usize;
        let mut b = take_f32_overwritten(len);
        assert_eq!(b.len(), len);
        assert!(b.iter().all(|v| *v == 0.0), "a fresh buffer is zeroed");
        b[17] = 4.5;
        let at = b.as_ptr() as usize;
        recycle_f32(b);
        // the same allocation, length and contents (the caller overwrites it)
        let again = take_f32_overwritten(len);
        assert_eq!((again.as_ptr() as usize, again.len(), again[17]), (at, len, 4.5));
        // another length does not match, and is not handed the idle one
        recycle_f32(again);
        let other = take_f32_overwritten(len + 1);
        assert_eq!(other.len(), len + 1);
        assert_ne!(other.as_ptr() as usize, at);
        // small buffers are not kept, and nothing exceeds the budget
        recycle_f32(vec![0.0; 100]);
        let budget_len = F32_MAX_BYTES / 4;
        recycle_f32(vec![0.0; budget_len + 1]);
        assert!(stats().idle_bytes <= F32_MAX_BYTES + (U8_BYTES_UPPER_BOUND), "idle {}", stats().idle_bytes);
        // leave nothing behind for other tests
        drop(take_f32_overwritten(len));
    }

    #[test]
    fn recycled_frames_feed_the_next_take_unless_still_shared() {
        // sizes nothing else in this test binary asks for
        let (n, c) = (777_001usize, 333_003usize);
        let yuv = |fill: u8| {
            Arc::new(VideoFrame {
                width: 1,
                height: 1,
                data: PixelData::Yuv8 {
                    planes: [Arc::new(vec![fill; n]), Arc::new(vec![fill; c]), Arc::new(vec![fill; c])],
                    chroma: Chroma::C420,
                    alpha: None,
                },
                color: filmcraft_color::ColorInfo::SRGB_FULL,
                par: (1, 1),
                pts: filmcraft_time::Tick::ZERO,
            })
        };
        let plane_ptrs = |frame: &VideoFrame| match &frame.data {
            PixelData::Yuv8 { planes, .. } => planes.each_ref().map(|plane| plane.as_ptr() as usize),
            _ => panic!("fixture has three byte planes"),
        };
        // A frame someone else still holds is left alone. The reuse statistic is global:
        // concurrent float-buffer tests can change it without touching these planes.
        let shared = yuv(1);
        let held = shared.clone();
        let held_ptrs = plane_ptrs(&held);
        recycle(shared);
        let fresh = take_u8(n);
        assert!(fresh.capacity() >= n);
        assert!(!held_ptrs.contains(&(fresh.as_ptr() as usize)), "a shared plane cannot be handed out");
        assert_eq!(held.byte_size(), n + 2 * c);
        // An unshared frame's exact allocations come back, empty. Equal-size chroma
        // planes may be taken in either order.
        let unshared = yuv(2);
        let returned_ptrs = plane_ptrs(&unshared);
        recycle(unshared);
        let (y, u, v) = (take_u8(n), take_u8(c), take_u8(c));
        assert!(y.is_empty() && u.is_empty() && v.is_empty());
        assert!(y.capacity() >= n && u.capacity() >= c && v.capacity() >= c);
        assert_eq!(y.as_ptr() as usize, returned_ptrs[0]);
        let mut expected_chroma = [returned_ptrs[1], returned_ptrs[2]];
        let mut actual_chroma = [u.as_ptr() as usize, v.as_ptr() as usize];
        expected_chroma.sort_unstable();
        actual_chroma.sort_unstable();
        assert_eq!(actual_chroma, expected_chroma);
        // The returned allocations remain owned by y/u/v and cannot be handed out twice.
        let another = take_u8(n);
        assert!(another.capacity() >= n);
        assert!(!returned_ptrs.contains(&(another.as_ptr() as usize)));
    }
}