GitCorner

Code

ossartcraftlightcraft๐ŸŒ
5a24289898fd
use super::commit::{How, Plan, Slot, Transfer, commit_prepared, name_family, run_transfers, system_clock};
use super::files::{Seen, expand, probe_all, reusable_probes, sidecars_of};
use super::types::{FileFacts, ImportCandidate, ImportMode, ImportOptions, ImportReport, Kept, OnDeleted};
use super::{HashMap, MediaKind, Path, PathBuf, Photo, PhotoId, ProbeInfo, Session, Source};

/// Import files/folders. See the module docs.
pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result<ImportReport> {
    import_with(s, paths, &ImportOptions { mode, ..Default::default() })
}

/// Import files/folders with options (preset, keywords). See the module docs.
///
/// The file-system half ([`ImportJob::prepare`]: expanding folders, probing, reading sidecars,
/// copying / placing files) and the catalog half ([`commit_prepared`]) are separate so the app can
/// run the first on a worker thread; here both run in turn.
pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> crate::Result<ImportReport> {
    let mut job = ImportJob::new(s, opts.clone())?;
    let prepared = job.prepare(paths, &std::sync::atomic::AtomicBool::new(false));
    // probes a scan left for files not imported now stay for a later import
    s.import_probes.extend(std::mem::take(&mut job.cache));
    commit_prepared(s, &job.opts, &job.now, prepared)
}

/// Everything the file-system half of an import needs, detached from the session so it can run
/// on another thread (a network share or a card can take minutes). Built by [`ImportJob::new`];
/// [`ImportJob::prepare`] may be called for several batches in turn (renamed copies keep counting,
/// duplicates are found across batches); each batch is then [`commit_prepared`] on the session.
pub struct ImportJob {
    pub opts: ImportOptions,
    /// The mode in effect (Copy / Move become Add in a library without its own folder).
    mode: ImportMode,
    lib_dir: Option<PathBuf>,
    copy_root: Option<PathBuf>,
    /// The library as it was when the import was set up, until it is looked up
    /// ([`ImportJob::look_up_library`], with the first files): setting an import up is the
    /// session's thread, and a table of a library of millions of photos takes seconds. With it,
    /// whether a photo in Recently Deleted is passed over (an import that replaces such photos,
    /// as the options said when the import was set up).
    pub(super) library: Option<(lightcraft_catalog::Catalog, bool)>,
    /// path โ†’ (photo, a Local browse record?)
    pub(super) by_path: HashMap<String, (PhotoId, bool)>,
    /// content hash โ†’ the photo that has it (`None`: a file earlier in this import)
    pub(super) by_hash: HashMap<String, Option<PhotoId>>,
    pub(super) probe: Option<crate::media::FileProbe>,
    file_bytes: Option<crate::merge::ByteReader>,
    /// Probes from a preceding scan (not read again).
    pub(super) cache: HashMap<String, ProbeInfo>,
    naming: crate::sidecar::SidecarNaming,
    now: String,
    /// Next rename number; `None` once the final representable value has been reserved.
    seq: Option<usize>,
    /// Files found so far (folders expanded) and handled so far, for progress.
    pub total: std::sync::Arc<std::sync::atomic::AtomicUsize>,
    pub done: std::sync::Arc<std::sync::atomic::AtomicUsize>,
    /// Of those, the files that have to be read first (not probed by a preceding scan) and the
    /// ones read so far: the step before they are handled.
    pub read_total: std::sync::Arc<std::sync::atomic::AtomicUsize>,
    pub read_done: std::sync::Arc<std::sync::atomic::AtomicUsize>,
}

/// The outcome of [`ImportJob::prepare`] for one batch, in file order.
#[derive(Debug, Default)]
pub struct Prepared {
    pub scanned: usize,
    pub(super) items: Vec<PreparedItem>,
}

#[derive(Debug)]
pub(super) enum PreparedItem {
    /// A Local (browsed) record imported for real.
    Promote(PhotoId),
    Duplicate {
        path: String,
        existing: Option<PhotoId>,
        reason: &'static str,
        hash: Option<String>,
        /// What the file itself says, when it was read ([`Duplicate::file`]).
        file: Option<Box<FileFacts>>,
    },
    Failed(String, String),
    Kept(Kept),
    Ready(Box<ReadyFile>),
}

#[derive(Debug)]
pub(super) struct ReadyFile {
    pub(super) path: String,
    pub(super) stored: String,
    pub(super) info: ProbeInfo,
    pub(super) sidecar: Option<crate::sidecar::SidecarData>,
    pub(super) placed: Option<crate::import_move::Placed>,
}

impl Prepared {
    /// Undo what the batch did on disk that only a commit would make permanent: files placed by a
    /// Move (their sources were never touched). Copies stay, like copies of a failed commit.
    pub fn rollback(&self) {
        for it in &self.items {
            if let PreparedItem::Ready(r) = it
                && let Some(pl) = &r.placed
            {
                crate::import_move::rollback(pl);
            }
        }
    }

    /// Entries in the batch (after expanding folders).
    pub fn len(&self) -> usize {
        self.items.len()
    }

    pub fn is_empty(&self) -> bool {
        self.items.is_empty()
    }

    /// Recheck Add-only entries against the owner catalog immediately before commit.
    /// Preparation may have run while another import added the same path or content.
    pub fn revalidate_add(&mut self, s: &Session) {
        let mut paths = HashMap::new();
        let mut hashes = HashMap::new();
        for photo in s.catalog.photos() {
            if let Source::File { path } = &photo.source {
                paths.insert(normalize_import_path(path), photo.id);
            }
            if let Some(hash) = &photo.content_hash {
                hashes.insert(hash.clone(), photo.id);
            }
        }
        let mut items = Vec::with_capacity(self.items.len());
        for item in std::mem::take(&mut self.items) {
            let PreparedItem::Ready(ready) = item else {
                items.push(item);
                continue;
            };
            let ReadyFile { path, stored, info, sidecar, placed } = *ready;
            let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied());
            let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied());
            if let Some(existing) = existing_path.or(existing_hash) {
                if let Some(placed) = &placed {
                    crate::import_move::rollback(placed);
                }
                items.push(PreparedItem::Duplicate {
                    path,
                    existing: Some(existing),
                    reason: if existing_path.is_some() { "path" } else { "content" },
                    hash: info.content_hash,
                    file: Some(Box::new(FileFacts { meta: info.meta, captured: info.captured })),
                });
            } else {
                items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed })));
            }
        }
        self.items = items;
    }
}

fn normalize_import_path(path: &str) -> String {
    let path = path.replace('\\', "/");
    if cfg!(windows) { path.to_lowercase() } else { path }
}

impl ImportJob {
    /// Snapshot what an import with `opts` needs from the session (no file-system calls). Takes
    /// the probes a preceding scan left ([`Session::import_probes`]).
    pub fn new(s: &mut Session, opts: ImportOptions) -> crate::Result<Self> {
        // a recipe is checked before any file is read or placed, not when the photos are committed
        crate::cmd::check_recipe(&opts.recipe).map_err(|e| crate::cmd::bad("library.import", e))?;
        let mode = opts.mode;
        // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of
        // every added file in its own storage already, so "copy" there means "add".
        let transfer = matches!(mode, ImportMode::Copy | ImportMode::Move);
        let mode = if transfer && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode };
        if mode == ImportMode::Move && opts.local {
            return Err(crate::EngineError::Other("browsing a folder never moves its files".into()));
        }
        let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
        // where copies go: the chosen folder, else the library's Originals/
        let copy_root = opts
            .destination
            .as_deref()
            .filter(|d| !d.trim().is_empty())
            .map(std::path::PathBuf::from)
            .or_else(|| lib_dir.as_ref().map(|l| l.join("Originals")));
        if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() {
            return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into()));
        }
        let fresh = opts.on_deleted == OnDeleted::Fresh;
        Ok(ImportJob {
            seq: Some(opts.rename_start.max(1)),
            opts,
            mode,
            lib_dir,
            copy_root,
            library: Some((s.catalog.clone(), fresh)),
            by_path: HashMap::new(),
            by_hash: HashMap::new(),
            probe: s.media.file_probe.clone(),
            file_bytes: s.media.file_bytes.clone(),
            cache: std::mem::take(&mut s.import_probes),
            naming: s.xmp.naming,
            now: (s.clock)(),
            total: Default::default(),
            done: Default::default(),
            read_total: Default::default(),
            read_done: Default::default(),
        })
    }

    /// The file-system half of importing `paths`: expand folders, skip known paths, probe, skip
    /// duplicates by content, read sidecars, copy or place (Move) the files โ€” several at once,
    /// with the names and outcomes of one after another. Nothing in the catalog changes. Stops
    /// before the next file once `cancel` is set.
    pub fn prepare(&mut self, paths: &[String], cancel: &std::sync::atomic::AtomicBool) -> Prepared {
        let files = self.expand(paths);
        self.prepare_files(files, cancel)
    }

    /// Expand `paths` (folders recursively, the library's own folder skipped) into the files an
    /// import would handle; counted in [`ImportJob::total`].
    pub fn expand(&self, paths: &[String]) -> Vec<String> {
        let files = expand(paths, self.lib_dir.as_deref());
        self.total.fetch_add(files.len(), std::sync::atomic::Ordering::Relaxed);
        files
    }

    /// When the import started ([`Session::clock`]), for [`commit_prepared`].
    pub fn now(&self) -> &str {
        &self.now
    }

    /// What the library held when the import was set up, by path and by content: made once, with
    /// the first files (on the thread that prepares them), from the library as it was then. The
    /// snapshot comes back: the files the import is adding are ranked against it for the ownership
    /// of the shared stem sidecars they read ([`crate::sidecar::ImportStems`]).
    fn look_up_library(&mut self) -> Option<lightcraft_catalog::Catalog> {
        let (library, fresh) = self.library.take()?;
        for p in library.photos() {
            // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate
            if p.deleted && fresh {
                continue;
            }
            if let Source::File { path } = &p.source {
                self.by_path.insert(path.clone(), (p.id, p.local));
            }
            if let Some(h) = &p.content_hash {
                self.by_hash.insert(h.clone(), Some(p.id));
            }
        }
        Some(library)
    }

    /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned.
    pub fn prepare_files(&mut self, files: Vec<String>, cancel: &std::sync::atomic::AtomicBool) -> Prepared {
        use std::sync::atomic::Ordering::Relaxed;
        // the library as it was, kept for the ownership of the shared stem sidecars below
        let library = self.look_up_library();
        let opts = self.opts.clone();
        let mode = self.mode;
        let mut out = Prepared { scanned: files.len(), items: Vec::new() };
        let mut todo = Vec::new();
        for f in files {
            match self.by_path.get(f.as_str()).copied() {
                Some((id, true)) if !opts.local => {
                    // a file that was only browsed (Local) joins the library when it is imported for real
                    self.by_path.insert(f, (id, false));
                    out.items.push(PreparedItem::Promote(id));
                    self.done.fetch_add(1, Relaxed);
                }
                Some((id, _)) => {
                    out.items.push(PreparedItem::Duplicate { path: f, existing: Some(id), reason: "path", hash: None, file: None });
                    self.done.fetch_add(1, Relaxed);
                }
                None => todo.push(f),
            }
        }

        // files a preceding scan already probed are not read again (a network share is slow to read),
        // as long as they are still there, as they were: a file gone or changed since is probed again
        let cached = reusable_probes(&todo, &mut self.cache, cancel, |seen, info| {
            // (the web's files are not std::fs files: kept as probed)
            cfg!(target_arch = "wasm32") || matches!(seen, Seen::At { is_file: true, len } if len == info.file_size)
        });
        let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
        // Reading the files is the slow part of adding them where they are: it is counted on its
        // own, and `cancel` stops it too (a large library takes long enough to want both).
        self.read_total.fetch_add(missing.len(), Relaxed);
        let mut fresh = probe_all(self.probe.as_ref(), &missing, &self.read_done, cancel).into_iter();
        let probed: Vec<Result<ProbeInfo, String>> =
            cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
        crate::memory::release();
        // Who owns the shared stem sidecar (`IMG_0001.xmp`) among the library's files and the ones
        // arriving with this import: a JPEG or a TIFF arriving beside its raw reads its own sidecar
        // and never the raw's packet (issue #1905).
        let stems = crate::sidecar::ImportStems::for_import(
            library.as_ref(),
            todo.iter().zip(&probed).filter_map(|(path, info)| info.as_ref().ok().map(|i| (path.as_str(), i.kind))),
        );
        // The sidecar of every file that was probed, looked up for the batch at once: each is a few
        // names asked of the disk, which on a network share made a folder of thousands of photos wait
        // a round trip per name, one name after another.
        let asked: Vec<&str> = todo.iter().zip(&probed).filter(|(_, p)| p.is_ok()).map(|(f, _)| f.as_str()).collect();
        // (a file whose content the library has is skipped below as a duplicate: it asks the disk nothing)
        let known: std::collections::HashSet<&str> = todo
            .iter()
            .zip(&probed)
            .filter(|(_, p)| !opts.local && p.as_ref().is_ok_and(|i| i.content_hash.as_ref().is_some_and(|h| self.by_hash.contains_key(h))))
            .map(|(f, _)| f.as_str())
            .collect();
        let mut found =
            sidecars_of(&asked, cancel, |f| if known.contains(f) { None } else { crate::sidecar::find_sidecar_for_import(&stems, self.naming, f) })
                .into_iter();
        // Files are planned one by one, in order (duplicates, sidecars, names, folders); their
        // copies or moves โ€” the slow part, reading and writing every byte and verifying it โ€” are
        // collected and run on several threads (issue #367), then their outcomes take their places.
        let mut plan = Plan::default();
        for (path, info) in todo.into_iter().zip(probed) {
            if cancel.load(Relaxed) {
                break;
            }
            let mut info = match info {
                Ok(i) => i,
                Err(e) => {
                    log::warn!("import {path}: {e}");
                    self.done.fetch_add(1, Relaxed);
                    plan.slots.push(Slot::Item(PreparedItem::Failed(path, e)));
                    continue;
                }
            };
            // (this file's place in the lookups above: files that failed to probe have none)
            let found_sidecar = found.next().flatten();
            // a copy of a file whose transfer is still pending: a duplicate only if that one succeeds
            if !opts.local
                && let Some(h) = &info.content_hash
                && plan.pending_hashes.contains(h)
            {
                self.run_plan(&mut plan, &mut out, cancel);
                if cancel.load(Relaxed) {
                    break;
                }
            }
            // same bytes as a library photo: skipped below, once its capture time is known
            let duplicate_of =
                info.content_hash.as_ref().filter(|_| !opts.local).and_then(|h| self.by_hash.get(h).map(|existing| (h.clone(), *existing)));
            // the XMP sidecar (or a raw's embedded XMP), read before copying: its capture time files
            // and names the copy when the file itself has none (a duplicate needs only that time)
            let raw = info.kind == MediaKind::Raw;
            let packet = if duplicate_of.is_some() && info.captured.is_some() {
                None
            } else {
                found_sidecar.and_then(|f| std::fs::read_to_string(f).ok()).or_else(|| info.xmp.clone().filter(|_| raw))
            };
            let target = crate::crs::Target::for_file(info.kind, &info.format, info.preview_only.is_some(), &info.meta.camera);
            let sidecar = packet.and_then(|x| match crate::sidecar::parse_sidecar(&x, target) {
                Ok(sc) => Some(sc),
                Err(e) => {
                    log::warn!("import {path}: XMP: {e}");
                    None
                }
            });
            if info.captured.is_none() {
                info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone());
            }
            // no date in the file or its sidecar: the file's own date, as Lightroom does
            if info.captured.is_none() {
                info.captured = info.file_modified.clone();
            }
            if let Some((hash, existing)) = duplicate_of {
                self.done.fetch_add(1, Relaxed);
                plan.slots.push(Slot::Item(PreparedItem::Duplicate {
                    path,
                    existing,
                    reason: "content",
                    hash: Some(hash),
                    file: Some(Box::new(FileFacts { meta: info.meta, captured: info.captured })),
                }));
                continue;
            }
            let stored = match (mode, &self.copy_root) {
                (ImportMode::Copy | ImportMode::Move, Some(root))
                    if !crate::import_move::inside(Path::new(&path), root)
                        && !self.lib_dir.as_ref().is_some_and(|l| crate::import_move::inside(Path::new(&path), l)) =>
                {
                    // the templates see the photo as it will be catalogued (undated: the import time)
                    let own = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
                    let mut q = Photo::new(PhotoId(0), Source::File { path: path.clone() }, &own, &info.format, 0, 0, &self.now);
                    q.captured = info.captured.clone();
                    q.meta = info.meta.clone();
                    let name = if let Some(template) = opts.rename.as_deref().filter(|t| !t.trim().is_empty()) {
                        let Some(seq) = self.seq else {
                            self.done.fetch_add(1, Relaxed);
                            plan.slots.push(Slot::Item(PreparedItem::Failed(
                                path,
                                "rename sequence exhausted; choose a lower renameStart or leave rename blank".into(),
                            )));
                            continue;
                        };
                        self.seq = seq.checked_add(1);
                        Some(crate::rename::expand(template, &q, seq))
                    } else {
                        None
                    };
                    let folders = opts.organize.folders(&q);
                    let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f));
                    // the name this file gets depends on what a pending transfer of a like-named file
                    // into the same folder ends up with: that one goes first
                    let as_named = name.as_deref().unwrap_or(if own.is_empty() { "photo" } else { own.as_str() });
                    let family = (dir.to_string_lossy().to_lowercase(), name_family(as_named));
                    if plan.pending_families.contains(&family) {
                        self.run_plan(&mut plan, &mut out, cancel);
                        if cancel.load(Relaxed) {
                            break;
                        }
                    }
                    let how = if mode == ImportMode::Move && !crate::import_move::is_symlink(Path::new(&path)) {
                        How::Place { name: name.unwrap_or(own) }
                    } else {
                        if mode == ImportMode::Move {
                            let reason = "a symbolic link: the file it points to was copied, the link and its target stay";
                            plan.slots.push(Slot::Item(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() })));
                        }
                        How::Copy { name }
                    };
                    // Copy as DNG: the copied raw becomes a DNG (the copy is ours to replace)
                    let dng = opts.convert_dng && mode == ImportMode::Copy && raw && !info.format.eq_ignore_ascii_case("DNG");
                    plan.slots.push(Slot::Transfer);
                    // (transfers of one folder and name family can take each other's names, so they never run together)
                    plan.pending_families.insert(family);
                    plan.pending_hashes.extend(info.content_hash.clone());
                    plan.transfers.push(Transfer { path, info, sidecar, dir, how, dng });
                    continue;
                }
                (ImportMode::Move, _) => {
                    let reason = "already inside the destination or the library: added where it is";
                    plan.slots.push(Slot::Item(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() })));
                    path.clone()
                }
                _ => path.clone(),
            };
            if let Some(h) = &info.content_hash {
                self.by_hash.insert(h.clone(), None);
            }
            self.done.fetch_add(1, Relaxed);
            plan.slots.push(Slot::Item(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed: None }))));
        }
        self.run_plan(&mut plan, &mut out, cancel);
        out
    }

    /// Run the transfers `plan` collected (on several threads) and append its items to `out` in
    /// file order. Transfers not started because `cancel` was set leave no item.
    fn run_plan(&mut self, plan: &mut Plan, out: &mut Prepared, cancel: &std::sync::atomic::AtomicBool) {
        plan.pending_hashes.clear();
        plan.pending_families.clear();
        let mut done = run_transfers(std::mem::take(&mut plan.transfers), self.file_bytes.as_ref(), cancel, &self.done).into_iter();
        for slot in std::mem::take(&mut plan.slots) {
            let item = match slot {
                Slot::Item(it) => it,
                Slot::Transfer => match done.next().flatten() {
                    Some(it) => it,
                    None => continue,
                },
            };
            if let PreparedItem::Ready(r) = &item
                && let Some(h) = &r.info.content_hash
            {
                self.by_hash.insert(h.clone(), None);
            }
            out.items.push(item);
        }
    }
}

/// Identity of a candidate preview at the bounded requested edge, without constructing a job.
pub fn candidate_thumb_key(c: &ImportCandidate, edge: usize) -> u64 {
    lightcraft_preview::Hasher128::new().str(&c.path).u64(c.file_size).u64(edge.clamp(64, 1024) as u64).finish().0 as u64
}

impl Session {
    /// A thumbnail for an import candidate (not in the catalog): a raw's embedded preview, else a
    /// small render of the file. `slot` makes the job's photo id unique (`u64::MAX - slot`).
    pub fn candidate_thumb_job(&mut self, c: &ImportCandidate, edge: usize, slot: u64) -> Option<crate::media::QuickJob> {
        if c.error.is_some() {
            return None;
        }
        let id = PhotoId(u64::MAX - slot);
        let mut p = Photo::new(id, Source::File { path: c.path.clone() }, &c.name, &c.format, c.width.max(1), c.height.max(1), "");
        p.kind = c.kind;
        if let Some(info) = self.import_probes.get(&c.path) {
            p.as_shot_wb = info.as_shot_wb;
            p.embedded_lens = info.embedded_lens;
            p.preview_only = info.preview_only.clone();
        }
        p.develop = std::sync::Arc::new(p.import_defaults());
        let edge = edge.clamp(64, 1024);
        let level = crate::media::SourceLevel::for_size(edge);
        let source = self.media.origin_ref(&p.source, level.max_edge());
        let key = candidate_thumb_key(c, edge);
        // A small original already supplies all its detail; do not enlarge it to the bucket.
        let native_edge = c.width.max(c.height) as usize;
        let render_edge = if native_edge > 0 { edge.min(native_edge) } else { edge };
        let small = crate::media::RenderJob {
            request_id: 0,
            cache_generation: self.media.rendered.generation(),
            source_key: None,
            photo: id,
            level,
            source,
            origin: p.source.clone(),
            info: crate::media::source_info(&p),
            settings: p.develop.clone(),
            request: lightcraft_pipeline::RenderRequest::fit(render_edge, render_edge),
            key,
            cache: None,
            stages: None,
            view_cache: None,
            display: None,
            on_screen: false,
            slide: None,
            preview_cut: None,
            display_fit: None,
        };
        let embedded = match (&self.media.preview_loader, c.kind) {
            (Some(l), MediaKind::Raw) => Some((c.path.clone(), l.clone(), edge)),
            _ => None,
        };
        let minimum_embedded_edge = (c.preview_only.is_none() && p.preview_only.is_none()).then_some(render_edge);
        Some(crate::media::QuickJob {
            request_id: 0,
            photo: id,
            key,
            cached: Vec::new(),
            embedded,
            embedded_cache: None,
            embedded_on_disk: false,
            minimum_embedded_edge,
            small: Some(Box::new(small)),
        })
    }

    /// Use the system clock for import/edit times (native hosts).
    pub fn with_system_clock(mut self) -> Self {
        self.clock = Box::new(system_clock);
        self
    }
}

#[cfg(test)]
mod tests;