GitCorner

Code

ossartcraftlightcraft🌐
5a24289898fd
use super::settings_hash::SettingsHashes;
use super::source::{DecodedSource, DenoiseSpec, PairLoader, RENDERED_MEM_BYTES, SourceLevel, SourceRef, THUMB_SOURCE_BYTES};
use super::{Arc, DcpProfile, DevelopSettings, HashMap, Hasher128, Lru, MediaCache, Photo, PhotoId, PreviewCache, Rgb32f, Source, SourceInfo};
use lightcraft_preview::WriteGate;

/// The photos that have a cached denoised picture for the denoise model in use. Only memory: nothing here touches the
/// file system, so it can be asked on the UI thread.
#[derive(Default)]
pub struct DenoiseIndex {
    ready: std::collections::HashMap<PhotoId, DenoiseSpec>,
    /// Reads the pictures (`None`: this build cannot, and nothing is ever ready).
    pub loader: Option<PairLoader>,
}

impl DenoiseIndex {
    /// What to load for `id` besides its plain picture, and how.
    pub fn get(&self, id: PhotoId) -> Option<(&DenoiseSpec, &PairLoader)> {
        self.ready.get(&id).zip(self.loader.as_ref())
    }

    pub fn spec(&self, id: PhotoId) -> Option<&DenoiseSpec> {
        self.ready.get(&id)
    }

    pub fn set(&mut self, id: PhotoId, spec: DenoiseSpec) {
        self.ready.insert(id, spec);
    }

    pub fn remove(&mut self, id: PhotoId) -> bool {
        self.ready.remove(&id).is_some()
    }

    pub fn clear(&mut self) {
        self.ready.clear();
    }

    pub fn len(&self) -> usize {
        self.ready.len()
    }

    pub fn is_empty(&self) -> bool {
        self.ready.is_empty()
    }

    pub fn ids(&self) -> Vec<PhotoId> {
        let mut v: Vec<PhotoId> = self.ready.keys().copied().collect();
        v.sort();
        v
    }
}

impl Default for MediaCache {
    fn default() -> Self {
        let budget = crate::memory::cache_share(crate::memory::budget());
        MediaCache {
            settings_hashes: SettingsHashes::default(),
            thumbs: Lru::new(THUMB_SOURCE_BYTES.min(budget)),
            previews: Vec::new(),
            full: Vec::new(),
            full_capacity: 1,
            on_screen: None,
            preview_capacity: 0,
            budget,
            file_loader: None,
            reduced_loader: None,
            file_probe: None,
            preview_loader: None,
            file_bytes: None,
            smart_dir: None,
            availability: Default::default(),
            scenes: Vec::new(),
            rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))),
            denoise: DenoiseIndex::default(),
            camera_profiles: HashMap::new(),
        }
    }
}

/// Memory for rendered previews out of the cache share: a quarter of it, so a big budget keeps
/// a whole shoot's previews (a 20 GB budget holds about 230 standard previews, or 1 500 loupe
/// renders at 1 200 px), and at least [`RENDERED_MEM_BYTES`] where the share allows.
pub(crate) fn rendered_budget(share: usize) -> usize {
    (share / 4).max(RENDERED_MEM_BYTES.min(share / 2))
}

impl MediaCache {
    /// Keep rendered thumbnails on disk in `dir` as well. The cache this replaces is retired
    /// (its files stay): render jobs still holding it can't write through it any more, even
    /// into a directory a later clear emptied (`dir` may be the same one). `gate` is asked before
    /// the cache changes its folder (`lightcraft_preview::DiskCache::set_gate`).
    pub fn attach_disk_cache(&mut self, dir: &std::path::Path, disk_bytes: u64, gate: Option<WriteGate>) {
        self.rendered.retire();
        let cache = PreviewCache::with_disk(rendered_budget(self.budget), dir, disk_bytes);
        if let (Some(disk), Some(gate)) = (cache.disk(), gate) {
            disk.set_gate(gate);
        }
        self.rendered = Arc::new(cache);
    }

    /// Change the disk budget of the cache in `dir`: in place when that cache is attached (its
    /// thumbnails are keyed by content, so they and the textures shown from them stay valid,
    /// and the same cache object keeps its generation), else attach it (with `gate`).
    pub fn set_disk_cache_bytes(&mut self, dir: &std::path::Path, disk_bytes: u64, gate: Option<WriteGate>) {
        match self.rendered.disk().filter(|d| d.dir() == dir) {
            Some(d) => d.set_budget(disk_bytes),
            None => self.attach_disk_cache(dir, disk_bytes, gate),
        }
    }

    /// Forget every decoded source (photo ids changed meaning, e.g. another library was opened).
    pub fn clear_sources(&mut self) {
        self.settings_hashes = SettingsHashes::default();
        self.thumbs = Lru::new(THUMB_SOURCE_BYTES.min(self.budget));
        self.previews.clear();
        self.full.clear();
    }

    /// How many full-resolution originals to keep at once (at least one). Compare and Reference
    /// views zoomed to 1:1 cut windows from two photos' originals: with room for one, every window
    /// render of one side would evict the other side's original and decode its own again. Lowering
    /// it drops the least recently used originals at once.
    pub fn set_full_capacity(&mut self, n: usize) {
        self.full_capacity = n.max(1);
        self.trim_full();
    }

    /// The photo the loupe shows (the frontend sets it every frame; `None` in other views). Its original stays
    /// even over budget: a 64 MP original (797 MB as linear float) alone exceeds the default share, and evicting it
    /// made every pan of a zoomed view decode the raw again. Showing another photo lets it go at once.
    pub fn set_on_screen(&mut self, id: Option<PhotoId>) {
        if self.on_screen != id {
            self.on_screen = id;
            self.enforce_budget();
        }
    }

    fn trim_full(&mut self) {
        while self.full.len() > self.full_capacity {
            let oldest = self.full.iter().enumerate().min_by_key(|(_, e)| e.2).map(|(i, _)| i).unwrap_or(0);
            self.full.remove(oldest);
        }
    }

    /// Bytes the caches may hold together.
    pub fn budget(&self) -> usize {
        self.budget
    }

    /// Change the bytes the caches may hold together, evicting at once if needed.
    pub fn set_budget(&mut self, bytes: usize) {
        self.budget = bytes;
        self.thumbs.set_budget(THUMB_SOURCE_BYTES.min(bytes));
        self.rendered.set_mem_budget(rendered_budget(bytes));
        self.enforce_budget();
    }

    /// Bytes held by decoded sources and rendered previews in memory.
    pub fn held(&self) -> usize {
        let (t, p, f) = self.usage();
        t.bytes + p.bytes + f.bytes + self.rendered.mem_usage().1
    }

    /// Evict least recently used entries across the source caches and the rendered previews
    /// until they fit the budget. The newest preview-level source (the photo on screen, or the
    /// one just decoded) is never evicted here, nor is the original of the photo on screen
    /// ([`Self::set_on_screen`]).
    pub fn enforce_budget(&mut self) {
        // The on-screen original is never evicted here, so it does not count either: counted, a 64 MP original (760 MiB)
        // nearly fills the default share (768 MiB) alone and everything else went instead — the photo's preview and
        // thumbnail sources and the neighbours' were decoded again from the raw on every edit, zoom-out and step.
        let pinned: usize = self.full.iter().filter(|e| Some(e.0) == self.on_screen).map(|e| e.1.bytes()).sum();
        let mut held = self.held().saturating_sub(pinned);
        while held > self.budget {
            let newest_preview = self.previews.iter().map(|e| e.2).max();
            let candidates = [
                self.thumbs.oldest_tick().map(|t| (t, 0)),
                self.previews.iter().filter(|e| Some(e.2) != newest_preview).map(|e| e.2).min().map(|t| (t, 1)),
                self.full.iter().filter(|e| Some(e.0) != self.on_screen).map(|e| e.2).min().map(|t| (t, 2)),
                self.rendered.oldest_tick().map(|t| (t, 3)),
            ];
            let Some((tick, which)) = candidates.into_iter().flatten().min() else { break };
            let freed = match which {
                0 => self.thumbs.pop_oldest(),
                1 => self.previews.iter().position(|e| e.2 == tick).map(|i| self.previews.remove(i).1.bytes()),
                2 => self.full.iter().position(|e| e.2 == tick).map(|i| self.full.remove(i).1.bytes()),
                _ => self.rendered.evict_oldest(),
            };
            match freed {
                Some(b) if b > 0 => held = held.saturating_sub(b),
                _ => break,
            }
        }
    }

    pub fn get(&mut self, id: PhotoId, level: SourceLevel) -> Option<Arc<Rgb32f>> {
        self.get_source(id, level).map(|s| s.image)
    }

    pub(super) fn get_source(&mut self, id: PhotoId, level: SourceLevel) -> Option<DecodedSource> {
        match level {
            SourceLevel::Thumb => self.thumbs.get(&id).cloned(),
            SourceLevel::Preview => self.previews.iter_mut().find(|e| e.0 == id).map(|e| {
                e.2 = lightcraft_preview::next_tick();
                e.1.clone()
            }),
            SourceLevel::Full => self.full.iter_mut().find(|e| e.0 == id).map(|e| {
                e.2 = lightcraft_preview::next_tick();
                e.1.clone()
            }),
        }
    }

    /// The facts of photo `id`'s cached sources: the decoder's when one has them, else `header`
    /// (with a smart preview's stored camera tone curve). Only sources developed with `profile` (the photo's imported
    /// camera profile tag, [`Self::camera_profile_of`], or none) count: one made with another profile has its facts.
    pub(super) fn source_facts(&self, id: PhotoId, header: SourceInfo, profile: Option<&str>) -> SourceInfo {
        let cached = || {
            self.thumbs
                .peek(&id)
                .into_iter()
                .chain(self.previews.iter().filter(|e| e.0 == id).map(|e| &e.1))
                .chain(self.full.iter().filter(|e| e.0 == id).map(|e| &e.1))
                .filter(|s| s.camera_profile.as_deref() == profile)
        };
        match cached().find(|s| s.info.is_some()).or_else(|| cached().next()) {
            Some(s) => s.info_or(header),
            None => header,
        }
    }

    pub fn insert(&mut self, id: PhotoId, level: SourceLevel, img: Arc<Rgb32f>) {
        self.insert_source(id, level, DecodedSource::new(img, None));
    }

    /// A decoded source with what the decoder reported about it (tests: photos of another as-shot white).
    #[cfg(test)]
    pub(crate) fn insert_with_info(&mut self, id: PhotoId, level: SourceLevel, img: Arc<Rgb32f>, info: lightcraft_pipeline::SourceInfo) {
        self.insert_source(id, level, DecodedSource::new(img, Some(info)));
    }

    pub(crate) fn insert_source(&mut self, id: PhotoId, level: SourceLevel, img: DecodedSource) {
        let tick = lightcraft_preview::next_tick();
        match level {
            SourceLevel::Thumb => {
                let cost = img.bytes();
                self.thumbs.insert(id, img, cost);
            }
            SourceLevel::Preview => {
                self.previews.retain(|e| e.0 != id);
                self.previews.push((id, img, tick));
                let cap = if self.preview_capacity == 0 { 4 } else { self.preview_capacity };
                while self.previews.len() > cap {
                    let oldest = self.previews.iter().enumerate().min_by_key(|(_, e)| e.2).map(|(i, _)| i).unwrap_or(0);
                    self.previews.remove(oldest);
                }
            }
            SourceLevel::Full => {
                self.full.retain(|e| e.0 != id);
                self.full.push((id, img, tick));
                self.trim_full();
            }
        }
        self.enforce_budget();
    }

    /// Forget a photo's decoded sources (e.g. after its file changed).
    pub fn forget(&mut self, id: PhotoId) {
        self.thumbs.remove(&id);
        self.previews.retain(|e| e.0 != id);
        self.full.retain(|e| e.0 != id);
    }

    /// (decoded thumbnail sources, bytes).
    pub fn source_usage(&self) -> (usize, usize) {
        (self.thumbs.len(), self.thumbs.cost())
    }

    /// Whether the decoded source of `id` at `level` is held.
    pub fn has_source(&self, id: PhotoId, level: SourceLevel) -> bool {
        match level {
            SourceLevel::Thumb => self.thumbs.contains(&id),
            SourceLevel::Preview => self.previews.iter().any(|e| e.0 == id),
            SourceLevel::Full => self.full.iter().any(|e| e.0 == id),
        }
    }

    /// Decoded sources held: (thumbnail level, preview level, full size).
    pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) {
        use crate::memory::Usage;
        let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| e.1.bytes()).sum());
        let full = Usage::new(self.full.len(), self.full.iter().map(|e| e.1.bytes()).sum());
        (Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full)
    }

    /// What a render of photo `id` with settings `s` adds to its identity for the denoised picture it will use: 0 unless
    /// it uses one (the Denoise amount is above 0 and the photo has a cached denoised picture).
    pub fn denoise_salt(&self, id: PhotoId, s: &DevelopSettings) -> u64 {
        match self.denoise.spec(id) {
            Some(spec) if s.denoise_amount() > 0.0 => Hasher128::new().str(&spec.key).finish().0 as u64 | 1,
            _ => 0,
        }
    }

    /// How to load photo `p` at `level`: from the cache when its source is there, else from its file (with its denoised
    /// picture beside it when it has one), its smart preview or the demo scene.
    pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
        self.source_ref_with(p, level, true)
    }

    /// [`Self::source_ref`] that never takes a decoded source from memory: for a job that needs something the cached
    /// one lacks (an export, whose denoised picture is made when it runs).
    pub fn source_ref_uncached(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
        self.source_ref_with(p, level, false)
    }

    /// [`Self::source_ref`] without the photo's imported camera profile (keeping its denoised picture): a smart
    /// preview keeps the file's own colour, and a cached source developed with the profile doesn't stand in for it.
    #[cfg(not(target_arch = "wasm32"))]
    pub fn source_ref_without_profile(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
        self.source_ref_for(p, level, true, None)
    }

    fn source_ref_with(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {
        let wanted = self.camera_profile_of(p);
        self.source_ref_for(p, level, cached, wanted)
    }

    /// [`Self::source_ref_with`] developed with camera profile `wanted` (none: the file's own colour).
    fn source_ref_for(&mut self, p: &Photo, level: SourceLevel, cached: bool, wanted: Option<(String, Arc<DcpProfile>)>) -> SourceRef {
        let mut r = self.source_ref_plain(p, level, cached, wanted);
        if let (SourceRef::File { denoise, .. }, Some((spec, loader))) = (&mut r, self.denoise.get(p.id)) {
            *denoise = Some((spec.clone(), loader.clone()));
        }
        r
    }

    /// Is photo `p`'s original known to be missing (cached answer only)?
    fn original_offline(&self, p: &Photo) -> bool {
        matches!(&p.source, Source::File { path } if self.availability.is_offline(path))
    }

    fn source_ref_plain(&mut self, p: &Photo, level: SourceLevel, cached: bool, wanted: Option<(String, Arc<DcpProfile>)>) -> SourceRef {
        let outliers = p.develop.removes_hot_pixels();
        // (a source developed with another camera profile, or none, doesn't stand in; a smart preview
        // has no camera profile, and decoding it again gives the same picture, so it stands in while the
        // original is offline)
        if cached
            && let Some(a) = self.get_source(p.id, level).filter(|a| a.fits_outliers(outliers))
            && (a.camera_profile == wanted.as_ref().map(|(id, _)| id.clone()) || (a.from_proxy && self.original_offline(p)))
        {
            return SourceRef::Loaded(Box::new(a));
        }
        // a thumbnail of a photo whose preview or full source is decoded (the one in the loupe) is cut from it; not
        // with a denoised picture, which the thumbnail would lose, nor from one decoded with another camera profile
        if cached
            && level == SourceLevel::Thumb
            && self.denoise.spec(p.id).is_none()
            && let Some(big) = [SourceLevel::Preview, SourceLevel::Full].into_iter().find_map(|level| {
                self.get_source(p.id, level).filter(|b| {
                    b.fits_outliers(outliers)
                        && (b.camera_profile.as_deref() == wanted.as_ref().map(|(tag, _)| tag.as_str()) || (b.from_proxy && self.original_offline(p)))
                })
            })
        {
            return SourceRef::Fit { from: Box::new(big), max_edge: level.max_edge() };
        }
        // Procedural scenes have a nominal size: "full" is that size, not unbounded.
        let max_edge = level.max_edge().min(match (&p.source, level) {
            (Source::Demo { .. }, SourceLevel::Full) => p.width.max(p.height).max(1) as usize,
            _ => usize::MAX,
        });
        // the original is offline: its smart preview, when there is one. Only cached answers are
        // used here (no file-system call on the UI thread in the app); an original not known to
        // be offline is tried first and its smart preview is the render worker's fallback.
        #[cfg(not(target_arch = "wasm32"))]
        if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir) {
            let sharded = crate::smart::sharded_file_path(dir, p);
            let legacy = dir.join(crate::smart::file_name(p));
            let sp = if self.availability.exists(&sharded.to_string_lossy()) == Some(true) { sharded } else { legacy };
            // (a proxy cut short fails to load, like the offline original; the worker's fallback
            // in `SourceRef::load` only uses a proxy that passes `smart::is_valid`)
            if self.availability.is_offline(path) && self.availability.exists(&sp.to_string_lossy()) == Some(true) {
                return SourceRef::Smart { path: sp };
            }
            let mut r = self.origin_ref(&p.source, max_edge);
            if let SourceRef::File { fallback, camera_profile, outliers: o, .. } = &mut r {
                *fallback = Some(sp);
                *camera_profile = wanted;
                *o = outliers;
            }
            return r;
        }
        let mut r = self.origin_ref(&p.source, max_edge);
        if let SourceRef::File { camera_profile, outliers: o, .. } = &mut r {
            *camera_profile = wanted;
            *o = outliers;
        }
        r
    }

    /// The imported camera profile photo `p` develops with, when its profile setting names one this library has: what a
    /// source decoded with it is tagged with (its id and the file's fingerprint), and the profile.
    pub fn camera_profile_of(&self, p: &Photo) -> Option<(String, Arc<DcpProfile>)> {
        let id = &p.develop.profile.id;
        if !crate::cmd::dcp_profiles::is_dcp_id(id) {
            return None;
        }
        self.camera_profiles.get(id).map(|d| (format!("{id}#{:016x}", d.fingerprint), d.profile.clone()))
    }

    /// What a render with profile `profile_id` adds to its cache keys: the imported camera profile's fingerprint, 0
    /// for any other profile and for a camera profile this library no longer has (it renders without it).
    pub fn camera_profile_salt(&self, profile_id: &str) -> u64 {
        if !crate::cmd::dcp_profiles::is_dcp_id(profile_id) {
            return 0;
        }
        self.camera_profiles.get(profile_id).map_or(0, |d| d.fingerprint)
    }

    /// How to load `origin` at most `max_edge` pixels long, ignoring decoded sources in memory (a
    /// render worker in another wasm instance builds its sources from this).
    pub fn origin_ref(&mut self, origin: &Source, max_edge: usize) -> SourceRef {
        match origin {
            Source::Demo { scene } => {
                if self.scenes.is_empty() {
                    self.scenes = lightcraft_scenes::demo_library();
                }
                match self.scenes.iter().find(|s| s.id == *scene) {
                    Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge },
                    None => SourceRef::File {
                        path: format!("demo:{scene}"),
                        max_edge,
                        loader: None,
                        fallback: None,
                        denoise: None,
                        camera_profile: None,
                        outliers: false,
                        reduced: None,
                    },
                }
            }
            Source::File { path } => SourceRef::File {
                path: path.clone(),
                max_edge,
                loader: self.file_loader.clone(),
                fallback: None,
                denoise: None,
                camera_profile: None,
                outliers: false,
                reduced: None,
            },
        }
    }
}