GitCorner

Code

ossartcraftlightcraft🌐
5a24289898fd
//! Detached Lightroom catalog preparation and owner-thread commit.
//!
//! `LightroomJob::new` takes only an in-memory session snapshot.  `prepare` owns all catalog
//! reading, probing and archive staging, while `commit_prepared` only applies catalog operations
//! to the session that created the job.  Archive index bytes are returned for a worker to finish
//! after the owner has accepted the catalog operation.

use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::{
    Arc,
    atomic::{AtomicBool, AtomicUsize, Ordering},
};

use serde_json::Value;

use crate::import::{ImportJob, ImportMode, ImportOptions, Prepared};
use crate::lightroom_catalog::{self, CatalogImport, ImportIndex};

const COMMAND: &str = "library.importLightroom";
const MAX_INDEX_BYTES: usize = 16 * 1024 * 1024;

fn error(message: impl Into<String>) -> crate::EngineError {
    crate::EngineError::BadParams { cmd: COMMAND.into(), msg: message.into() }
}

/// The import was cancelled ([`crate::ErrorKind::Cancelled`]: not a failure). The message is the
/// one it always had, so control-channel and MCP clients see the same `error` text.
fn cancelled() -> crate::EngineError {
    crate::EngineError::Cancelled(error("Lightroom import cancelled").to_string())
}

fn stage_archive(archive_dir: Option<&Path>, data: &mut CatalogImport) -> crate::Result<Option<PathBuf>> {
    let archive_path = archive_dir.map(|dir| crate::lightroom_archive::store_best_effort(dir, data).map_err(error)).transpose()?.flatten();
    if archive_dir.is_some() && archive_path.is_none() {
        data.warnings.push("Lightroom source archive exceeded 32 MiB; archive skipped".into());
    }
    Ok(archive_path)
}

/// Memory identity of the library which owns a prepared completion.
///
/// Each successful open/close refreshes session identity, including reopening the same path.
/// The owner check deliberately does not compare catalog revisions: ordinary edits while preparation
/// runs must survive and are merged by the owner commit.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LightroomLibraryToken {
    library_dir: Option<PathBuf>,
    identity: Arc<()>,
}

impl LightroomLibraryToken {
    fn capture(s: &crate::Session) -> Self {
        Self {
            library_dir: s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()),
            identity: Arc::clone(&s.library_identity),
        }
    }

    /// Whether `s` still refers to the library which started preparation.
    pub fn matches(&self, s: &crate::Session) -> bool {
        Arc::ptr_eq(&self.identity, &s.library_identity)
            && self.library_dir == s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone())
    }

    pub fn library_dir(&self) -> Option<&Path> {
        self.library_dir.as_deref()
    }
}

/// Filesystem-free result of catalog preparation.
pub struct PreparedLightroom {
    pub(crate) data: CatalogImport,
    pub(crate) normal: Prepared,
    pub(crate) index: ImportIndex,
    pub(crate) archive_path: Option<PathBuf>,
    pub(crate) token: LightroomLibraryToken,
    pub(crate) missing_sources: HashSet<String>,
    pub(crate) update_existing: bool,
    pub(crate) now: String,
    cancelled: bool,
}

impl PreparedLightroom {
    pub fn source(&self) -> &str {
        &self.data.source
    }

    pub fn photo_count(&self) -> usize {
        self.data.photos.len()
    }

    pub fn token(&self) -> &LightroomLibraryToken {
        &self.token
    }

    pub fn is_cancelled(&self) -> bool {
        self.cancelled
    }

    /// Roll back any normal-import filesystem placement held by this prepared value.
    pub fn rollback(&self) {
        self.normal.rollback();
    }
}

/// Index write returned by owner commit.  `finish` is intended for a worker after the owner has
/// returned, and performs the only archive-index filesystem operation in this pipeline.
pub struct LightroomArchiveFinalization {
    path: Option<PathBuf>,
    bytes: Option<Vec<u8>>,
}

impl LightroomArchiveFinalization {
    pub fn finish(self) -> crate::Result<()> {
        let (Some(path), Some(bytes)) = (self.path, self.bytes) else { return Ok(()) };
        if bytes.len() > MAX_INDEX_BYTES {
            return Err(error("Lightroom import index exceeds 16 MiB"));
        }
        lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))
    }

    pub fn path(&self) -> Option<&Path> {
        self.path.as_deref()
    }
}

/// Owner-thread catalog result.  The report is ready for the UI immediately; index finalization
/// can remain on the worker while its task stays alive.
pub struct LightroomCompletion {
    pub report: Value,
    pub finalization: LightroomArchiveFinalization,
}

/// Alias used by hosts that call the result a commit.
pub type LightroomCommit = LightroomCompletion;

/// The step of preparing a Lightroom import that is under way ([`LightroomProgress::step`]).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LightroomStep {
    /// Reading the catalog's tables.
    Catalog,
    /// Reading the photos the catalog names (headers, content hash): nearly all of the time.
    ReadPhotos,
    /// Readying the photos that were read for the catalog.
    AddPhotos,
}

/// Progress of [`LightroomJob::prepare`] step by step, shared with the thread that runs it.
#[derive(Clone, Default)]
pub struct LightroomProgress {
    /// Catalog tables to read and read so far ([`lightroom_catalog::read_with_progress`]).
    pub catalog_total: Arc<AtomicUsize>,
    pub catalog_done: Arc<AtomicUsize>,
    read_total: Arc<AtomicUsize>,
    read_done: Arc<AtomicUsize>,
    add_total: Arc<AtomicUsize>,
    add_done: Arc<AtomicUsize>,
    /// The catalog is read: the photos are next.
    photos: Arc<AtomicBool>,
}

impl LightroomProgress {
    /// Reading the photos is nearly all of an import's time; the catalog and readying the photos
    /// take the rest of [`LightroomProgress::fraction`].
    const CATALOG_SHARE: f32 = 0.05;
    const READ_SHARE: f32 = 0.85;

    /// The step under way, with how much of it is done and its total (0 while not yet known).
    pub fn step(&self) -> (LightroomStep, usize, usize) {
        let load = |n: &AtomicUsize| n.load(Ordering::Relaxed);
        if !self.photos.load(Ordering::Relaxed) {
            return (LightroomStep::Catalog, load(&self.catalog_done), load(&self.catalog_total));
        }
        let (read_done, read_total) = (load(&self.read_done), load(&self.read_total));
        let (add_done, add_total) = (load(&self.add_done), load(&self.add_total));
        // nothing to read (every photo is already in the library) goes straight to adding
        if read_done < read_total || (read_total == 0 && add_done == 0) {
            (LightroomStep::ReadPhotos, read_done, read_total)
        } else {
            (LightroomStep::AddPhotos, add_done, add_total)
        }
    }

    /// The whole preparation, 0..=1.
    pub fn fraction(&self) -> f32 {
        let (step, done, total) = self.step();
        let part = if total == 0 { 0.0 } else { (done as f32 / total as f32).clamp(0.0, 1.0) };
        match step {
            LightroomStep::Catalog => Self::CATALOG_SHARE * part,
            LightroomStep::ReadPhotos => Self::CATALOG_SHARE + Self::READ_SHARE * part,
            LightroomStep::AddPhotos => Self::CATALOG_SHARE + Self::READ_SHARE + (1.0 - Self::CATALOG_SHARE - Self::READ_SHARE) * part,
        }
    }
}

/// A Lightroom import detached from the owner thread.
pub struct LightroomJob {
    path: PathBuf,
    update_existing: bool,
    import: ImportJob,
    token: LightroomLibraryToken,
    archive_dir: Option<PathBuf>,
    progress: LightroomProgress,
}

impl LightroomJob {
    /// Snapshot session state only.  This constructor performs no filesystem work.
    pub fn new(s: &mut crate::Session, path: PathBuf, update_existing: bool) -> crate::Result<Self> {
        // a photo deleted here stays deleted: the catalog's record is matched, not restored
        let opts = ImportOptions { mode: ImportMode::Add, on_deleted: crate::import::OnDeleted::Skip, ..ImportOptions::default() };
        let import = ImportJob::new(s, opts)?;
        let archive_dir = s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.join("Interop"));
        let progress = LightroomProgress {
            read_total: Arc::clone(&import.read_total),
            read_done: Arc::clone(&import.read_done),
            add_total: Arc::clone(&import.total),
            add_done: Arc::clone(&import.done),
            ..LightroomProgress::default()
        };
        Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir, progress })
    }

    /// The job's progress, to watch from another thread while [`LightroomJob::prepare`] runs.
    pub fn progress(&self) -> LightroomProgress {
        self.progress.clone()
    }

    pub fn token(&self) -> &LightroomLibraryToken {
        &self.token
    }

    /// Finish an owner commit's deferred archive-index write on a worker.
    pub fn finish_archive(finalization: LightroomArchiveFinalization) -> crate::Result<()> {
        finalization.finish()
    }

    /// Read/probe/stage all source data off the owner thread.
    pub fn prepare(&mut self, cancel: &AtomicBool) -> crate::Result<PreparedLightroom> {
        if cancel.load(Ordering::Relaxed) {
            return Err(cancelled());
        }
        // (reading stops with a message when `cancel` is set: the flag says it was cancelled)
        let mut data = lightroom_catalog::read_source(
            &self.path,
            self.archive_dir.as_deref(),
            cancel,
            &self.progress.catalog_total,
            &self.progress.catalog_done,
        )
        .map_err(|e| if cancel.load(Ordering::Relaxed) { cancelled() } else { error(e) })?;
        lightroom_catalog::validate_for_job(&data).map_err(error)?;
        if cancel.load(Ordering::Relaxed) {
            return Err(cancelled());
        }
        let (paths, missing_sources) = original_paths(&data, cancel)?;
        // Catalog entries name files, never folders. Missing originals already have enough
        // catalog metadata for placeholders; probing them again only adds failed I/O and errors.
        // Complete the catalog step before the separate read/add photo progress starts.
        self.progress.catalog_done.fetch_add(1, Ordering::Relaxed);
        self.progress.photos.store(true, Ordering::Relaxed);
        self.import.total.fetch_add(paths.len(), Ordering::Relaxed);
        let normal = self.import.prepare_files(paths, cancel);
        if cancel.load(Ordering::Relaxed) {
            normal.rollback();
            return Err(cancelled());
        }
        let archive_path = stage_archive(self.archive_dir.as_deref(), &mut data)?;
        let index_path = self.archive_dir.as_ref().map(|dir| dir.join("lightroom-index.json"));
        let index = lightroom_catalog::load_index(index_path.as_deref())?;
        Ok(PreparedLightroom {
            data,
            normal,
            index,
            archive_path,
            token: self.token.clone(),
            missing_sources,
            update_existing: self.update_existing,
            now: self.import.now().to_string(),
            cancelled: cancel.load(Ordering::Relaxed),
        })
    }
}

/// Apply prepared data on the owner thread.  No source, archive or index filesystem work occurs.
pub fn commit_prepared(s: &mut crate::Session, mut prepared: PreparedLightroom) -> crate::Result<LightroomCompletion> {
    if prepared.cancelled {
        prepared.normal.rollback();
        return Err(cancelled());
    }
    if !prepared.token.matches(s) {
        prepared.normal.rollback();
        return Err(error("Lightroom import belongs to a different open library"));
    }
    // A command-scoped Lightroom import must never rewrite source XMP sidecars.  Direct depth-0
    // callers do not set this transient flag, so they cannot affect a later command.
    if s.depth > 0 {
        s.skip_auto_write = true;
    }

    let undo0 = s.undo.len();
    prepared.normal.revalidate_add(s);
    let options = ImportOptions { mode: ImportMode::Add, on_deleted: crate::import::OnDeleted::Skip, ..ImportOptions::default() };
    let now = prepared.now.clone();
    let files = crate::import::commit_prepared(s, &options, &now, prepared.normal)?;
    let applied = lightroom_catalog::apply_prepared(
        s,
        prepared.data,
        lightroom_catalog::ApplyPrepared {
            update_existing: prepared.update_existing,
            files,
            index: &mut prepared.index,
            archive_path: prepared.archive_path,
            missing_sources: &prepared.missing_sources,
            undo0,
        },
    )?;
    let path = prepared.token.library_dir().map(|dir| dir.join("Interop/lightroom-index.json"));
    let bytes = if path.is_some() { Some(serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?) } else { None };
    Ok(LightroomCompletion { report: applied.report, finalization: LightroomArchiveFinalization { path, bytes } })
}

fn original_paths(data: &CatalogImport, cancel: &AtomicBool) -> crate::Result<(Vec<String>, HashSet<String>)> {
    let mut paths = Vec::new();
    let mut missing = HashSet::new();
    let mut seen = HashSet::new();
    for photo in &data.photos {
        if cancel.load(Ordering::Relaxed) {
            return Err(cancelled());
        }
        if lightroom_catalog::number_for_job(&photo.image, "masterImage") != 0 {
            continue;
        }
        let key = normalize(&photo.path);
        if !seen.insert(key.clone()) {
            continue;
        }
        if photo.original_is_file() {
            paths.push(photo.path.clone());
        } else {
            missing.insert(key);
        }
    }
    Ok((paths, missing))
}

fn normalize(path: &str) -> String {
    let path = path.replace('\\', "/");
    if cfg!(windows) { path.to_lowercase() } else { path }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn empty_data() -> CatalogImport {
        CatalogImport {
            source: "fixture.lrcat".into(),
            catalog_id: None,
            catalog_path: None,
            photos: Vec::new(),
            collections: Vec::new(),
            members: Vec::new(),
            collection_content: Vec::new(),
            warnings: Vec::new(),
            skipped: Vec::new(),
        }
    }

    fn cancelled_prepared(s: &crate::Session) -> PreparedLightroom {
        PreparedLightroom {
            data: empty_data(),
            normal: Prepared::default(),
            index: ImportIndex::default(),
            archive_path: None,
            token: LightroomLibraryToken::capture(s),
            missing_sources: HashSet::new(),
            update_existing: false,
            now: "2026-01-01T00:00:00".into(),
            cancelled: true,
        }
    }

    fn prepared_for(s: &crate::Session, path: &str, rating: i64) -> PreparedLightroom {
        let mut image = std::collections::BTreeMap::new();
        image.insert("id_local".into(), serde_json::json!(1));
        image.insert("rating".into(), serde_json::json!(rating));
        image.insert("pick".into(), serde_json::json!(0));
        image.insert("fileFormat".into(), serde_json::json!("JPG"));
        image.insert("fileWidth".into(), serde_json::json!(1));
        image.insert("fileHeight".into(), serde_json::json!(1));
        PreparedLightroom {
            data: CatalogImport {
                source: "fixture.lrcat".into(),
                catalog_id: None,
                catalog_path: None,
                photos: vec![lightroom_catalog::CatalogPhoto {
                    source_id: 1,
                    uuid: "fixture-photo".into(),
                    path: path.into(),
                    image,
                    settings: String::new(),
                    xmp: String::new(),
                    keywords: Vec::new(),
                    history: Vec::new(),
                    snapshots: Vec::new(),
                    original: lightroom_catalog::Original::Local,
                }],
                collections: Vec::new(),
                members: Vec::new(),
                collection_content: Vec::new(),
                warnings: Vec::new(),
                skipped: Vec::new(),
            },
            normal: Prepared::default(),
            index: ImportIndex::default(),
            archive_path: None,
            token: LightroomLibraryToken::capture(s),
            missing_sources: [normalize(path)].into_iter().collect(),
            update_existing: false,
            now: "2026-01-01T00:00:00".into(),
            cancelled: false,
        }
    }

    /// One count for the whole import read "14 of 24554" while the photos were read, then twice
    /// the number of photos: each step has its own count, and the whole only moves forward.
    #[test]
    fn progress_names_each_step_with_its_own_count() {
        let p = LightroomProgress::default();
        let set = |n: &AtomicUsize, v: usize| n.store(v, Ordering::Relaxed);
        assert_eq!(p.step(), (LightroomStep::Catalog, 0, 0));
        assert_eq!(p.fraction(), 0.0);
        set(&p.catalog_total, 14);
        set(&p.catalog_done, 7);
        assert_eq!(p.step(), (LightroomStep::Catalog, 7, 14));
        let mut last = p.fraction();
        assert!(last > 0.0);
        let mut forward = |p: &LightroomProgress| {
            let f = p.fraction();
            assert!(f >= last && f <= 1.0, "{f} after {last}");
            last = f;
        };
        set(&p.catalog_done, 14);
        p.photos.store(true, Ordering::Relaxed);
        assert_eq!(p.step(), (LightroomStep::ReadPhotos, 0, 0), "the photos are not counted yet");
        forward(&p);
        set(&p.add_total, 100);
        set(&p.read_total, 100);
        set(&p.read_done, 40);
        assert_eq!(p.step(), (LightroomStep::ReadPhotos, 40, 100));
        forward(&p);
        set(&p.read_done, 100);
        set(&p.add_done, 25);
        assert_eq!(p.step(), (LightroomStep::AddPhotos, 25, 100));
        forward(&p);
        set(&p.add_done, 100);
        forward(&p);
        assert!(p.fraction() > 0.999);
        // every photo already in the library: nothing to read
        let known = LightroomProgress::default();
        known.photos.store(true, Ordering::Relaxed);
        set(&known.add_total, 5);
        set(&known.add_done, 5);
        assert_eq!(known.step(), (LightroomStep::AddPhotos, 5, 5));
    }

    #[test]
    fn cancellation_before_commit_leaves_owner_catalog_unchanged() {
        let mut session = crate::Session::new();
        let before = session.catalog.to_snapshot();
        let prepared = cancelled_prepared(&session);
        let Err(e) = commit_prepared(&mut session, prepared) else { panic!("a cancelled import commits nothing") };
        assert_eq!(e.kind(), crate::ErrorKind::Cancelled, "{e}");
        assert_eq!(session.catalog.to_snapshot(), before);
        assert!(session.undo.is_empty());
    }

    /// Cancelling while the catalog is read is [`crate::ErrorKind::Cancelled`] whatever the message
    /// says, and a failure that is not a cancellation is not one, even when its text says "cancelled".
    #[test]
    fn a_cancelled_read_is_cancelled_by_kind() {
        let mut session = crate::Session::new();
        let mut job = LightroomJob::new(&mut session, PathBuf::from("/nonexistent/cancelled.lrcat"), false).unwrap();
        let Err(e) = job.prepare(&AtomicBool::new(true)) else { panic!("cancelled before it started") };
        assert_eq!(e.kind(), crate::ErrorKind::Cancelled, "{e}");
        // the `error` text clients see is unchanged
        assert_eq!(e.to_string(), "invalid parameters for `library.importLightroom`: Lightroom import cancelled");
        let Err(e) = job.prepare(&AtomicBool::new(false)) else { panic!("no such catalog") };
        assert_ne!(e.kind(), crate::ErrorKind::Cancelled, "{e}");
    }

    #[test]
    fn library_identity_rejects_late_completion_after_reopen() {
        let mut session = crate::Session::new();
        let token = LightroomLibraryToken::capture(&session);
        assert!(token.matches(&session));
        session.library_identity = Arc::new(());
        assert!(!token.matches(&session));
    }

    #[test]
    fn prepared_reimport_preserves_current_edit_and_uses_one_undo_per_import() {
        let mut session = crate::Session::new();
        let path = "/missing/lightroom-job-edit.jpg";
        let prepared = prepared_for(&session, path, 4);
        let first = commit_prepared(&mut session, prepared).unwrap();
        assert_eq!(first.report["mapping"]["1"].as_u64(), Some(1));
        assert_eq!(session.undo.len(), 1);
        let id = lightcraft_catalog::PhotoId(1);
        let mut personal = lightcraft_develop::DevelopSettings::default();
        personal.light.exposure = 1.5;
        session.set_develop(id, personal, "Personal").unwrap();
        let before = session.catalog.photo(id).map(|photo| photo.develop.clone());
        let prepared = prepared_for(&session, path, 5);
        commit_prepared(&mut session, prepared).unwrap();
        assert_eq!(session.catalog.photo(id).map(|photo| photo.develop.clone()), before);
        assert_eq!(session.catalog.photo(id).map(|photo| photo.rating), Some(5));
        assert_eq!(session.undo.len(), 3);
    }

    #[test]
    fn oversized_history_archive_keeps_prepared_import_successful_with_warning() {
        let archive_dir = std::env::temp_dir().join(format!("lightcraft-lr-prepared-scale-{}", std::process::id()));
        let mut data = empty_data();
        data.photos = (0..5_000)
            .map(|source_id| {
                let image = std::collections::BTreeMap::from([
                    ("id_local".into(), serde_json::json!(source_id)),
                    ("fileFormat".into(), serde_json::json!("JPG")),
                    ("fileWidth".into(), serde_json::json!(1)),
                    ("fileHeight".into(), serde_json::json!(1)),
                ]);
                lightroom_catalog::CatalogPhoto {
                    source_id,
                    uuid: format!("scale-{source_id}"),
                    path: format!("/missing/scale-{source_id}.jpg"),
                    image,
                    settings: String::new(),
                    xmp: String::new(),
                    keywords: Vec::new(),
                    history: vec![std::collections::BTreeMap::from([(
                        "text".into(),
                        serde_json::json!(format!("history-{source_id}-{}", "x".repeat(8 * 1024))),
                    )])],
                    snapshots: Vec::new(),
                    original: lightroom_catalog::Original::Local,
                }
            })
            .collect();
        let archive_path = stage_archive(Some(&archive_dir), &mut data).unwrap();
        assert!(archive_path.is_none());
        let mut session = crate::Session::new();
        let token = LightroomLibraryToken::capture(&session);
        let prepared = PreparedLightroom {
            data,
            normal: Prepared::default(),
            index: ImportIndex::default(),
            archive_path,
            token,
            missing_sources: HashSet::new(),
            update_existing: false,
            now: "2026-01-01T00:00:00".into(),
            cancelled: false,
        };
        let completion = commit_prepared(&mut session, prepared).unwrap();
        assert_eq!(completion.report["photos"], 5_000);
        assert!(
            completion.report["warnings"]
                .as_array()
                .is_some_and(|warnings| warnings.iter().any(|warning| warning == "Lightroom source archive exceeded 32 MiB; archive skipped"))
        );
        assert_eq!(session.catalog.len(), 5_000);
        std::fs::remove_dir_all(archive_dir).unwrap();
    }

    #[test]
    fn execute_fn_import_does_not_rewrite_source_sidecar() {
        let dir = std::env::temp_dir().join(format!("lightcraft-lr-job-sidecar-{}", std::process::id()));
        let path = dir.join("source.jpg");
        let sidecar = dir.join("source.xmp");
        std::fs::create_dir_all(&dir).unwrap();
        std::fs::write(&path, b"fixture").unwrap();
        std::fs::write(&sidecar, b"original sidecar").unwrap();
        let original = std::fs::read(&sidecar).unwrap();
        let mut session = crate::Session::new();
        session.xmp.auto_write = true;
        let result = session.execute_fn("library.importLightroom", |s| {
            let prepared = prepared_for(s, &path.to_string_lossy(), 5);
            let completion = commit_prepared(s, prepared)?;
            Ok(completion.report)
        });
        assert!(result.is_ok());
        assert_eq!(std::fs::read(&sidecar).unwrap(), original);
        std::fs::remove_dir_all(dir).unwrap();
    }
}