5a24289898fd
lightroom_catalog.rs
//! Direct, read-only Lightroom Classic catalog import. SQLite pages are read in Rust;
//! Lightroom, Python and a C SQLite runtime are not needed. Original paths stay in place.
//! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders.
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::{
Arc,
atomic::{AtomicBool, AtomicUsize, Ordering},
};
use crate::lightroom_sqlite::{Database, FileSource, LiveTable, Value as SqlValue};
use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source};
use lightcraft_geom::{Affine, Orientation, Point, Rect};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
type Row = BTreeMap<String, Value>;
const COMMAND: &str = "library.importLightroom";
/// Largest catalog database or WAL read into memory (4 GiB). The whole file is held in RAM, so the cap stays finite.
const MAX_FILE: u64 = 1 << 32;
fn error(why: impl Into<String>) -> crate::EngineError {
crate::EngineError::BadParams { cmd: COMMAND.into(), msg: why.into() }
}
fn text<'a>(r: &'a Row, k: &str) -> &'a str {
r.get(k).and_then(Value::as_str).unwrap_or("")
}
fn number(r: &Row, k: &str) -> i64 {
r.get(k).and_then(Value::as_f64).filter(|n| n.is_finite()).unwrap_or(0.0) as i64
}
pub(crate) fn number_for_job(r: &Row, k: &str) -> i64 {
number(r, k)
}
fn key(path: &str) -> String {
let p = path.replace('\\', "/");
if cfg!(windows) { p.to_lowercase() } else { p }
}
fn source_path(p: &Photo) -> Option<&str> {
match &p.source {
Source::File { path } => Some(path),
_ => None,
}
}
struct Reader {
db: Database,
tables: Vec<LiveTable>,
}
impl Reader {
fn table(&self, name: &str, required: bool) -> Result<Vec<Row>, String> {
let Some(t) = self.tables.iter().find(|t| t.name == name) else {
return if required { Err(format!("unsupported Lightroom catalog: missing {name}")) } else { Ok(Vec::new()) };
};
let columns = t.column_names.as_ref().ok_or_else(|| format!("cannot read {name} columns"))?;
let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("{name}: {e:?}"))?;
if rows.len() > 1_000_000 {
return Err(format!("{name}: more than one million records"));
}
Ok(rows
.into_iter()
.map(|r| {
columns
.iter()
.cloned()
.zip(r.values.into_iter().map(|v| match v {
SqlValue::Null => Some(Value::Null),
SqlValue::Integer(n) => Some(json!(n)),
SqlValue::Real(n) => Some(json!(n)),
SqlValue::Text(s) => Some(json!(s)),
// Opaque Lightroom payloads never become JSON byte arrays.
SqlValue::Blob(_) => None,
}))
.filter_map(|(column, value)| value.map(|value| (column, value)))
.collect()
})
.collect())
}
/// The catalog's `Adobe_storeProviderID` (a UUID), kept by Lightroom in its backups; `None`
/// when the table or the value is missing or doesn't look like an id.
fn catalog_id(&self) -> Option<String> {
let rows = self.table("Adobe_variablesTable", false).ok()?;
let row = rows.iter().find(|r| text(r, "name") == "Adobe_storeProviderID")?;
let id = text(row, "value").trim();
(!id.is_empty() && id.len() <= 128 && id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')).then(|| id.to_string())
}
fn optional_table(&self, name: &str, warnings: &mut Vec<String>) -> Result<Vec<Row>, String> {
match self.table(name, false) {
Ok(rows) => Ok(rows),
Err(error) => {
warnings.push(format!("skipping optional Lightroom table {name}: {error}"));
Ok(Vec::new())
}
}
}
fn xmp_table(&self) -> Result<(Vec<Row>, Vec<String>), String> {
let Some(t) = self.tables.iter().find(|t| t.name == "Adobe_AdditionalMetadata") else { return Ok((Vec::new(), Vec::new())) };
let columns = t.column_names.as_ref().ok_or_else(|| "cannot read Adobe_AdditionalMetadata columns".to_string())?;
let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("Adobe_AdditionalMetadata: {e:?}"))?;
if rows.len() > 1_000_000 {
return Err("Adobe_AdditionalMetadata: more than one million records".into());
}
let mut warnings = Vec::new();
let mut out = Vec::with_capacity(rows.len());
for row in rows {
let image = row
.values
.iter()
.zip(columns)
.find_map(|(value, column)| {
(column == "image").then_some(match value {
SqlValue::Integer(n) => *n,
_ => 0,
})
})
.unwrap_or(0);
let mut mapped = Row::new();
for (column, value) in columns.iter().zip(row.values) {
let value = match value {
SqlValue::Null => Some(Value::Null),
SqlValue::Integer(n) => Some(json!(n)),
SqlValue::Real(n) => Some(json!(n)),
SqlValue::Text(s) => Some(json!(s)),
SqlValue::Blob(bytes) if column == "xmp" => match decode_xmp_bytes(&bytes) {
Ok(packet) => Some(Value::String(packet)),
Err(error) => {
warnings.push(format!("image {image}: {error}"));
None
}
},
SqlValue::Blob(_) => None,
};
if let Some(value) = value {
mapped.insert(column.clone(), value);
}
}
out.push(mapped);
}
Ok((out, warnings))
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CatalogPhoto {
pub source_id: i64,
pub uuid: String,
pub path: String,
pub image: Row,
pub settings: String,
pub xmp: String,
pub keywords: Vec<String>,
pub history: Vec<Row>,
pub snapshots: Vec<Row>,
/// Whether `path` may be looked at (derived when the catalog is read; not archived).
#[serde(skip)]
pub original: Original,
}
/// Whether the original a catalog lists may be looked at. Its path is resolved with the catalog
/// as the document ([`crate::doc_path::resolve`]).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub enum Original {
/// `path` is a local file path, or one on the network share the catalog is on.
Local,
/// `path` is not a path on this computer (a catalog made on another system): the photo is
/// kept as an offline record for later relinking, and the path is not looked at.
#[default]
NotHere,
/// `path` is refused (why): it is on another computer or a device, or its folder or file name
/// is an absolute path. The photo is reported missing and not added to the library.
Refused(String),
}
impl CatalogPhoto {
/// Whether the original is a file here; only a [`Original::Local`] path is looked at.
pub fn original_is_file(&self) -> bool {
self.original == Original::Local && Path::new(&self.path).is_file()
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CatalogImport {
pub source: String,
/// The catalog's own id (`Adobe_storeProviderID` in `Adobe_variablesTable`), the same in the
/// catalog and in every backup of it, so a backup's collections match those of an earlier import
/// of the catalog (issue #2592). `None` when the catalog has none.
#[serde(default)]
pub catalog_id: Option<String>,
/// Where the catalog was when it isn't `source`: for a backup zip, the `.lrcat` it was backed up
/// from. Collections imported by earlier releases are keyed on that path, so a backup still finds
/// them. `None` when the catalog was read where it is.
#[serde(default)]
pub catalog_path: Option<String>,
pub photos: Vec<CatalogPhoto>,
pub collections: Vec<Row>,
pub members: Vec<Row>,
pub collection_content: Vec<Row>,
pub warnings: Vec<String>,
/// Catalog images left out because their records are incomplete; reported as data (not text)
/// so each front end can say so in its own language.
#[serde(default)]
pub skipped: Vec<SkippedImage>,
}
/// A catalog image that was not imported, and why.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SkippedImage {
/// The image's `Adobe_images.id_local`.
pub image: i64,
pub reason: SkipReason,
}
/// Why a catalog image was not imported. Lightroom leaves such records behind and ignores them.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum SkipReason {
/// Its `AgLibraryFile` record is gone.
MissingFile,
/// Its file's `AgLibraryFolder` record is gone.
MissingFolder,
/// Its folder's `AgLibraryRootFolder` record is gone.
MissingRoot,
/// Its file name or root folder path is empty.
IncompletePath,
/// A virtual copy whose master was skipped.
MasterSkipped,
}
#[derive(Clone, Default, Serialize, Deserialize)]
pub(crate) struct ImportIndex {
pub(crate) photos: BTreeMap<String, u64>,
pub(crate) collections: BTreeMap<String, u64>,
/// The catalog id (`Adobe_storeProviderID`) each collection key was recorded for, when the
/// import had one: a path key of one catalog never matches another catalog that later sits at
/// the same path. Keys without an owner (earlier releases, catalogs without an id) match any.
#[serde(default)]
pub(crate) collection_owners: BTreeMap<String, String>,
}
pub(crate) fn load_index(path: Option<&Path>) -> crate::Result<ImportIndex> {
match path {
Some(path) => {
let file = match std::fs::File::open(path) {
Ok(file) => file,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(ImportIndex::default()),
Err(e) => return Err(error(e.to_string())),
};
if file.metadata().map_err(|e| error(e.to_string()))?.len() > 16 * 1024 * 1024 {
return Err(error("Lightroom import index exceeds 16 MiB"));
}
let mut bytes = Vec::new();
use std::io::Read;
file.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| error(e.to_string()))?;
if bytes.len() > 16 * 1024 * 1024 {
return Err(error("Lightroom import index exceeds 16 MiB"));
}
serde_json::from_slice(&bytes).map_err(|e| error(format!("invalid Lightroom import index: {e}")))
}
None => Ok(ImportIndex::default()),
}
}
pub(crate) struct AppliedImport {
pub(crate) report: Value,
pub(crate) index: ImportIndex,
}
fn identity(data: &CatalogImport, photo: &CatalogPhoto) -> String {
// (a backup's photos are recognised by the catalog it was made from, as the catalog's own are)
if photo.uuid.is_empty() {
format!("{}:{}", key(data.catalog_path.as_deref().unwrap_or(&data.source)), photo.source_id)
} else {
photo.uuid.clone()
}
}
fn validate(data: &CatalogImport) -> Result<(), String> {
let photos: HashMap<_, _> = data.photos.iter().map(|p| (p.source_id, p)).collect();
if photos.len() != data.photos.len() {
return Err("duplicate source photo ids".into());
}
for photo in &data.photos {
let master = number(&photo.image, "masterImage");
if master != 0
&& (master == photo.source_id
|| !photos.contains_key(&master)
|| photos.get(&master).is_some_and(|p| number(&p.image, "masterImage") != 0))
{
return Err(format!("invalid virtual copy master for {}", photo.source_id));
}
}
let collections: HashMap<_, _> = data.collections.iter().map(|r| (number(r, "id_local"), r)).collect();
for id in collections.keys() {
let mut seen = HashSet::new();
let mut at = *id;
while let Some(r) = collections.get(&at) {
if seen.len() >= 64 || !seen.insert(at) {
return Err("cyclic or overly deep collection hierarchy".into());
}
at = number(r, "parent");
if at == 0 {
break;
}
}
}
Ok(())
}
pub(crate) fn validate_for_job(data: &CatalogImport) -> Result<(), String> {
validate(data)
}
fn bytes(path: &Path) -> Result<Vec<u8>, String> {
use std::io::Read;
let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
let len = f.metadata().map_err(|e| e.to_string())?.len();
if len > MAX_FILE {
return Err("catalog or WAL exceeds 4 GiB import limit".into());
}
let mut out = Vec::new();
// Reserve the exact size up front: growing by doubling would briefly need about twice the catalog's size.
let wanted = usize::try_from(len).map_err(|_| "catalog is too large for this platform".to_string())?;
out.try_reserve_exact(wanted).map_err(|_| "not enough memory to read the catalog".to_string())?;
f.take(MAX_FILE + 1).read_to_end(&mut out).map_err(|e| e.to_string())?;
if out.len() as u64 > MAX_FILE {
return Err("catalog grew beyond import limit".into());
}
Ok(out)
}
fn xmp(v: Option<&Value>) -> Result<String, String> {
match v {
Some(Value::String(s)) => Ok(s.clone()),
Some(Value::Array(a)) if !a.is_empty() => {
let b: Vec<u8> = a.iter().filter_map(|n| n.as_u64().and_then(|n| u8::try_from(n).ok())).collect();
let size = b.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?;
if size > 16 << 20 {
return Err("XMP packet exceeds 16 MiB".into());
}
let compressed = b.get(4..).ok_or("truncated XMP")?;
let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(compressed, 16 << 20).map_err(|_| "invalid compressed XMP")?;
if decoded.len() != size as usize {
return Err("XMP packet length mismatch".into());
}
String::from_utf8(decoded).map_err(|e| e.to_string())
}
_ => Ok(String::new()),
}
}
fn decode_xmp_bytes(bytes: &[u8]) -> Result<String, String> {
let size = bytes.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?;
if size > 16 << 20 {
return Err("XMP packet exceeds 16 MiB".into());
}
let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(bytes.get(4..).ok_or("truncated XMP")?, 16 << 20)
.map_err(|_| "invalid compressed XMP")?;
if decoded.len() != size as usize {
return Err("XMP packet length mismatch".into());
}
String::from_utf8(decoded).map_err(|e| e.to_string())
}
fn be_word(bytes: &[u8], at: usize) -> Result<u32, String> {
let word = bytes.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or("truncated WAL")?;
Ok(u32::from_be_bytes(word))
}
fn wal_checksum(bytes: &[u8], big: bool, mut sum: (u32, u32)) -> (u32, u32) {
for pair in bytes.as_chunks::<8>().0 {
let first = [pair[0], pair[1], pair[2], pair[3]];
let second = [pair[4], pair[5], pair[6], pair[7]];
let (first, second) =
if big { (u32::from_be_bytes(first), u32::from_be_bytes(second)) } else { (u32::from_le_bytes(first), u32::from_le_bytes(second)) };
sum.0 = sum.0.wrapping_add(first).wrapping_add(sum.1);
sum.1 = sum.1.wrapping_add(second).wrapping_add(sum.0);
}
sum
}
// SQLite file format Β§4: validate the live checksum chain in constant space, then overlay
// only its last committed prefix. The forensic timeline materializes EVERY historical
// database snapshot; it must never be used to validate a large, actively used catalog.
fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result<usize, String> {
if wal.is_empty() {
return Ok(0);
}
let header = wal.get(..32).ok_or("truncated WAL header")?;
let magic = be_word(header, 0)?;
if !matches!(magic, 0x377f0682 | 0x377f0683) || be_word(header, 4)? != 3_007_000 {
return Err("invalid WAL header".into());
}
let size = main.get(16..18).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(u16::from_be_bytes).ok_or("truncated SQLite header")?;
let size = if size == 1 { 65536 } else { u32::from(size) };
if !(512..=65536).contains(&size) || !size.is_power_of_two() || be_word(header, 8)? != size {
return Err("WAL page size mismatch".into());
}
let big = magic == 0x377f0683;
let mut sum = wal_checksum(&header[..24], big, (0, 0));
if sum != (be_word(header, 24)?, be_word(header, 28)?) {
return Err("invalid WAL header checksum".into());
}
let salt = (be_word(header, 16)?, be_word(header, 20)?);
let mut committed = 0;
let stride = size as usize + 24;
for (i, frame) in wal[32..].chunks_exact(stride).enumerate() {
if be_word(frame, 0)? == 0 || salt != (be_word(frame, 8)?, be_word(frame, 12)?) {
break;
}
sum = wal_checksum(&frame[..8], big, sum);
sum = wal_checksum(&frame[24..], big, sum);
if sum != (be_word(frame, 16)?, be_word(frame, 20)?) {
break;
}
if be_word(frame, 4)? != 0 {
committed = 32 + (i + 1) * stride;
}
}
Ok(committed)
}
/// Where a root folder is when its `absolutePath` isn't a folder here (the catalog came from
/// another computer): `relativePathFromCatalog`, which Lightroom records for roots on the
/// catalog's volume, resolved against the catalog's folder, if that is a folder. Both paths are
/// looked at only when [`crate::doc_path::resolve`] follows them.
fn relocated_root(absolute: &str, relative: &str, catalog: &Path) -> Option<PathBuf> {
if relative.is_empty() || !Path::new(relative).is_relative() || crate::doc_path::resolve(absolute, Some(catalog)).is_ok_and(|p| p.is_dir()) {
return None;
}
crate::doc_path::resolve(relative, Some(catalog)).ok().filter(|dir| dir.is_dir())
}
/// Whether a folder path or file name from the catalog is absolute: `Path::join` would put it in
/// place of the root folder.
fn is_absolute_part(part: &str) -> bool {
let bytes = part.as_bytes();
part.starts_with(['/', '\\'])
|| (bytes.first().is_some_and(u8::is_ascii_alphabetic) && bytes.get(1) == Some(&b':'))
|| Path::new(part).is_absolute()
}
/// The path of an original under `root` and whether it may be looked at, resolved with the
/// catalog at `catalog` as the document.
fn original_path(root: &str, folder: &str, name: &str, catalog: &Path) -> (String, Original) {
let stored = Path::new(root).join(folder).join(name).to_string_lossy().to_string();
if is_absolute_part(folder) || is_absolute_part(name) {
return (stored, Original::Refused("its folder or file name is an absolute path".into()));
}
match crate::doc_path::resolve(&stored, Some(catalog)) {
Ok(found) => (found.to_string_lossy().to_string(), Original::Local),
Err(crate::doc_path::NotFollowed::NotHere) => (stored, Original::NotHere),
Err(crate::doc_path::NotFollowed::Refused(why)) => (stored, Original::Refused(why.to_string())),
}
}
/// Inspect `.lrcat` plus committed WAL pages without modifying either source file.
pub fn read(path: &Path) -> Result<CatalogImport, String> {
let cancel = AtomicBool::new(false);
let total = AtomicUsize::new(0);
let done = AtomicUsize::new(0);
read_with_progress(path, &cancel, &total, &done)
}
/// Read a Lightroom Classic catalog for import, counting the tables read in `total` / `done` and
/// stopping when `cancel` is set. Images whose catalog records are incomplete are left out and
/// listed in [`CatalogImport::skipped`] instead of failing the read.
pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, done: &AtomicUsize) -> Result<CatalogImport, String> {
read_source(path, None, cancel, total, done)
}
/// Where a Lightroom backup's catalog is unpacked while it is read: the open library's `Interop/`
/// folder (on the library's disk), or `None` for the system's temporary folder.
pub fn backup_temp_root(s: &crate::Session) -> Option<PathBuf> {
s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"))
}
/// [`read_with_progress`] of a catalog or of a Lightroom catalog backup (`.zip`, issue #2592): a
/// backup's catalog is unpacked into a private folder under `temp_root` (else the system's temporary
/// folder), read, and removed again ([`crate::lightroom_backup`]). Paths the catalog records
/// relative to itself are taken from where the catalog was before it was backed up.
pub fn read_source(
path: &Path,
temp_root: Option<&Path>,
cancel: &AtomicBool,
total: &AtomicUsize,
done: &AtomicUsize,
) -> Result<CatalogImport, String> {
// 13 catalog tables plus one validation stage; ImportJob adds one unit per original path.
total.store(14, Ordering::Relaxed);
done.store(0, Ordering::Relaxed);
ensure_read_active(cancel)?;
if crate::lightroom_backup::is_backup(path) {
let root = temp_root.map_or_else(std::env::temp_dir, Path::to_path_buf);
let extracted = crate::lightroom_backup::extract(path, &root, cancel)?;
let (document, known) = crate::lightroom_backup::original_catalog(path, &extracted.entry);
let mut data = read_file(&extracted.catalog, &document, path, cancel, done)?;
if !known {
// only a guess: it resolves relative paths, but never names the catalog
data.catalog_path = None;
}
return Ok(data);
}
read_file(path, path, path, cancel, done)
}
/// Read the catalog file `path`, resolving the paths it records with `document` as the catalog's
/// place and reporting `source` as where it came from.
fn read_file(path: &Path, document: &Path, source: &Path, cancel: &AtomicBool, done: &AtomicUsize) -> Result<CatalogImport, String> {
let before = std::fs::metadata(path).map_err(|e| e.to_string())?;
let wal_path = PathBuf::from(format!("{}-wal", path.display()));
let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
let main = FileSource::open(path)?;
let wal = match std::fs::metadata(&wal_path) {
Ok(_) => bytes(&wal_path)?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(e) => return Err(e.to_string()),
};
let wal_len = committed_wal_len(main.header(), &wal)?;
ensure_read_active(cancel)?;
let db = Database::open_file_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?;
let after = std::fs::metadata(path).map_err(|e| e.to_string())?;
let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after {
return Err("catalog changed during reading; close Lightroom and retry".into());
}
let reader = Reader { tables: db.live_tables()?, db };
let mut warnings = Vec::new();
let table = |name: &str, required: bool| -> Result<Vec<Row>, String> {
ensure_read_active(cancel)?;
let rows = reader.table(name, required)?;
done.fetch_add(1, Ordering::Relaxed);
Ok(rows)
};
let optional_table = |name: &str, warnings: &mut Vec<String>| -> Result<Vec<Row>, String> {
ensure_read_active(cancel)?;
let rows = reader.optional_table(name, warnings)?;
done.fetch_add(1, Ordering::Relaxed);
Ok(rows)
};
let roots: HashMap<_, _> = table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
let folders: HashMap<_, _> = table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
let files: HashMap<_, _> = table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
let develops: HashMap<_, _> = optional_table("Adobe_imageDevelopSettings", &mut warnings)?
.into_iter()
.map(|r| (number(&r, "image"), text(&r, "text").to_string()))
.collect();
ensure_read_active(cancel)?;
let (metadata_rows, metadata_warnings) = match reader.xmp_table() {
Ok(result) => result,
Err(error) => {
warnings.push(format!("skipping optional Lightroom table Adobe_AdditionalMetadata: {error}"));
(Vec::new(), Vec::new())
}
};
warnings.extend(metadata_warnings);
done.fetch_add(1, Ordering::Relaxed);
let metadata: HashMap<_, _> = metadata_rows.into_iter().map(|r| (number(&r, "image"), r)).collect();
let keyword_rows: HashMap<_, _> = optional_table("AgLibraryKeyword", &mut warnings)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
let mut keywords: HashMap<i64, Vec<String>> = HashMap::new();
for r in optional_table("AgLibraryKeywordImage", &mut warnings)? {
ensure_read_active(cancel)?;
let mut parts = Vec::new();
let mut at = number(&r, "tag");
let mut seen = HashSet::new();
while at != 0 {
if parts.len() >= 64 || !seen.insert(at) {
return Err("cyclic or overly deep keyword hierarchy".into());
}
let Some(k) = keyword_rows.get(&at) else { break };
let name = text(k, "name");
if !name.is_empty() {
parts.push(name.to_string());
}
at = number(k, "parent");
}
parts.reverse();
if !parts.is_empty() {
keywords.entry(number(&r, "image")).or_default().push(parts.join("|"));
}
}
let group = |rows: Vec<Row>| {
let mut out: HashMap<i64, Vec<Row>> = HashMap::new();
for r in rows {
out.entry(number(&r, "image")).or_default().push(r);
}
out
};
let mut root_paths = HashMap::new();
let mut refused_roots = HashSet::new();
let mut root_ids: Vec<_> = roots.keys().copied().collect();
root_ids.sort_unstable();
for (id, root) in root_ids.iter().filter_map(|id| Some((*id, roots.get(id)?))) {
let absolute = text(root, "absolutePath");
let resolved = match relocated_root(absolute, text(root, "relativePathFromCatalog"), document) {
Some(found) => {
let found = found.to_string_lossy().to_string();
warnings.push(format!("root folder {}: {absolute} not found; using {found} (relative to the catalog)", text(root, "name")));
found
}
None => absolute.to_string(),
};
if let Err(crate::doc_path::NotFollowed::Refused(why)) = crate::doc_path::resolve(&resolved, Some(document)) {
warnings
.push(format!("root folder {}: {resolved} is not opened: {why}; its photos are listed as missing and not added", text(root, "name")));
refused_roots.insert(id);
}
root_paths.insert(id, resolved);
}
let mut history = group(optional_table("Adobe_libraryImageDevelopHistoryStep", &mut warnings)?);
let mut snapshots = group(optional_table("Adobe_libraryImageDevelopSnapshot", &mut warnings)?);
let mut out = CatalogImport {
source: source.to_string_lossy().into(),
catalog_id: reader.catalog_id(),
catalog_path: (document != source).then(|| document.to_string_lossy().into()),
photos: Vec::new(),
collections: optional_table("AgLibraryCollection", &mut warnings)?,
members: optional_table("AgLibraryCollectionImage", &mut warnings)?,
collection_content: optional_table("AgLibraryCollectionContent", &mut warnings)?,
warnings,
skipped: Vec::new(),
};
// images whose file, folder or root record is gone, or whose path is empty (Lightroom leaves
// these behind and ignores them); they and their virtual copies are skipped and reported
// instead of failing the import
let mut orphans = HashSet::new();
for image in table("Adobe_images", true)? {
ensure_read_active(cancel)?;
let id = number(&image, "id_local");
let record = files.get(&number(&image, "rootFile")).ok_or(SkipReason::MissingFile).and_then(|file| {
let folder = folders.get(&number(file, "folder")).ok_or(SkipReason::MissingFolder)?;
let root = roots.get(&number(folder, "rootFolder")).ok_or(SkipReason::MissingRoot)?;
Ok((file, folder, root))
});
let (file, folder, root) = match record {
Ok(found) => found,
Err(reason) => {
out.skipped.push(SkippedImage { image: id, reason });
orphans.insert(id);
continue;
}
};
let root_id = number(root, "id_local");
let root_path = root_paths.get(&root_id).map_or("", String::as_str);
let folder_path = text(folder, "pathFromRoot");
let name = text(file, "idx_filename");
if root_path.is_empty() || name.is_empty() {
out.skipped.push(SkippedImage { image: id, reason: SkipReason::IncompletePath });
orphans.insert(id);
continue;
}
let (photo_path, original) = original_path(root_path, folder_path, name, document);
if let Original::Refused(why) = &original
&& !refused_roots.contains(&root_id)
{
out.warnings.push(format!("image {id}: {photo_path} is not opened: {why}; listed as missing and not added"));
}
let packet = match xmp(metadata.get(&id).and_then(|r| r.get("xmp"))) {
Ok(s) => s,
Err(e) => {
out.warnings.push(format!("image {id}: {e}"));
String::new()
}
};
out.photos.push(CatalogPhoto {
source_id: id,
uuid: text(&image, "id_global").into(),
path: photo_path,
image,
settings: develops.get(&id).cloned().unwrap_or_default(),
xmp: packet,
keywords: keywords.remove(&id).unwrap_or_default(),
history: history.remove(&id).unwrap_or_default(),
snapshots: snapshots.remove(&id).unwrap_or_default(),
original,
});
}
drop_copies_of(&mut out, &orphans);
refuse_non_media(&mut out);
ensure_read_active(cancel)?;
reader.db.ensure_unchanged()?;
let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
if wal_before != wal_after {
return Err("catalog changed during reading; close Lightroom and retry".into());
}
Ok(out)
}
/// Drop the virtual copies whose master was skipped, adding them to the skipped images: a copy has
/// no original of its own.
fn drop_copies_of(data: &mut CatalogImport, masters: &HashSet<i64>) {
if masters.is_empty() {
return;
}
let mut skipped = Vec::new();
data.photos.retain(|p| {
let master = number(&p.image, "masterImage");
// 0 marks a master, never a copy
if master == 0 || !masters.contains(&master) {
return true;
}
skipped.push(SkippedImage { image: p.source_id, reason: SkipReason::MasterSkipped });
false
});
data.skipped.extend(skipped);
}
/// A video the catalog lists with the format `VIDEO` and a video extension. Videos aren't image
/// types, but they become video records (never opened by the system opener), so they pass.
fn is_catalog_video(p: &CatalogPhoto) -> bool {
const EXTENSIONS: [&str; 12] = ["mp4", "mov", "m4v", "avi", "mts", "m2ts", "3gp", "mkv", "webm", "wmv", "mpg", "mpeg"];
text(&p.image, "fileFormat").eq_ignore_ascii_case("VIDEO")
&& Path::new(&p.path).extension().and_then(|e| e.to_str()).is_some_and(|e| EXTENSIONS.iter().any(|x| e.eq_ignore_ascii_case(x)))
}
/// Drop the catalog's images whose original is not a file type LightCraft reads, saying so in the
/// import's warnings.
///
/// The catalog decides these paths, and a record it leaves behind is one click from the system
/// opener (Photo βΈ Edit In βΈ Open Original in Other Editor). A record for `payload.exe`,
/// `start.lnk` or `Evil.app` would make a `.lrcat` someone was sent a way to have a program run
/// (issue #2135), so such an image never becomes a photo: not imported, not relinkable, not
/// openable. A virtual copy of a refused original goes with it β it has no original of its own.
///
/// Called where the catalog is read and again where its photos become records, so neither route
/// can keep one; the second pass finds nothing left to say.
pub(crate) fn refuse_non_media(data: &mut CatalogImport) {
// masters first: a virtual copy may come before its master in the table
let refused: HashSet<i64> = data
.photos
.iter()
.filter(|p| number(&p.image, "masterImage") == 0 && !crate::open_path::is_media(Path::new(&p.path)) && !is_catalog_video(p))
.map(|p| p.source_id)
.collect();
if refused.is_empty() {
return;
}
let mut warnings = Vec::new();
data.photos.retain(|p| {
let master = number(&p.image, "masterImage");
// `masterImage` is 0 for a master, so a refused id of 0 (a malformed catalog) must not
// take every master of the catalog with it
if master != 0 {
if !refused.contains(&master) {
return true;
}
warnings.push(format!("image {}: not imported; its original was not imported", p.source_id));
return false;
}
if !refused.contains(&p.source_id) {
return true;
}
warnings.push(format!("image {}: not imported; {} isn't one of the file types LightCraft reads", p.source_id, p.path));
false
});
data.warnings.extend(warnings);
}
fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {
if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }
}
fn mapped_settings(
text: &str,
target: crate::crs::Target,
aspect: f64,
orientation: Orientation,
stored_aspect: f64,
user: Orientation,
) -> Result<(Value, Vec<String>), String> {
let mut root = crate::preset_import::parse_lua(text)?;
let deferred = strip_lua_sentinels(&mut root);
let (mut props, mut values) = crate::preset_import::lua_settings_props(&root)?;
// Catalog Lua names the component inversion `Inverted` while XMP's is `MaskInverted`;
// rename it so an inverted mask keeps its inversion instead of importing as normal.
for v in values.values_mut() {
alias_catalog_inverted(v);
}
// A catalog's develop settings carry the crop edges and angle but no `HasCrop` (XMP has it):
// any crop field means the photo is cropped.
const CROP: [&str; 5] = ["crs:CropLeft", "crs:CropTop", "crs:CropRight", "crs:CropBottom", "crs:CropAngle"];
if !props.contains_key("crs:HasCrop") && CROP.iter().any(|k| props.contains_key(*k)) {
props.insert("crs:HasCrop".into(), vec!["True".into()]);
}
// Mask edges are fractions of the stored image, like the crop's, so radii go to long-edge units
// with its aspect: `aspect` is the photo's in LightCraft, which is the shown one for a photo
// already in the library and only stands in when the catalog has no file size.
let mask_aspect = if stored_aspect.is_finite() && stored_aspect > 0.0 { stored_aspect } else { aspect };
let (mut partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), target, mask_aspect);
catalog_crop(&mut partial, orientation, stored_aspect);
catalog_masks(&mut partial, in_photo_frame(orientation, user));
if deferred > 0 {
unknown.push("Deferred Lightroom adjustments (-999999)".into());
}
Ok((partial, unknown))
}
/// Lightroom's `Adobe_images.orientation`: where the stored image's top-left and top-right corners
/// land in the displayed photo, naming the displayed corners A, B, C, D clockwise from the top left
/// (`AB` upright, `BC` a quarter turn clockwise, `DA` a quarter turn counter-clockwise, `BA` mirrored).
fn lightroom_orientation(code: &str) -> Option<Orientation> {
const ALL: [Orientation; 8] = [
Orientation::Normal,
Orientation::FlipH,
Orientation::Rotate180,
Orientation::FlipV,
Orientation::Transpose,
Orientation::Rotate90,
Orientation::Transverse,
Orientation::Rotate270,
];
let corner = |o: Orientation, x: f64, y: f64| match o.map(x, y, 1.0, 1.0) {
(u, v) if u < 0.5 && v < 0.5 => 'A',
(_, v) if v < 0.5 => 'B',
(u, _) if u >= 0.5 => 'C',
_ => 'D',
};
ALL.into_iter().find(|&o| code.chars().eq([corner(o, 0.0, 0.0), corner(o, 1.0, 0.0)]))
}
/// Lightroom's crop in LightCraft's frame. `crs` gives the crop's top-left (`CropLeft`, `CropTop`)
/// and bottom-right (`CropRight`, `CropBottom`) corners on the stored image (before orientation and
/// straightening), normalized to its size, plus a straighten angle that turns the other way to
/// LightCraft's. LightCraft's crop is a rectangle in the oriented image rotated by `angle` about its
/// centre, normalized to the oriented image's size, so the corners go through the orientation and
/// then the rotation; `stored_aspect` (width / height of the stored image) matters once the
/// rectangle turns. On a real catalog this reproduced the frame aspect of all 223 straightened crops
/// in Lightroom's exports within 1%.
fn catalog_crop(partial: &mut Value, orientation: Orientation, stored_aspect: f64) {
let Some(geometry) = partial.pointer_mut("/crop/geometry").and_then(Value::as_object_mut) else { return };
let edge = |k: &str| geometry.get("rect").and_then(|r| r.get(k)).and_then(Value::as_f64);
let (Some(x0), Some(y0), Some(x1), Some(y1)) = (edge("x0"), edge("y0"), edge("x1"), edge("y1")) else { return };
let lightroom_angle = geometry.get("angle").and_then(Value::as_f64).unwrap_or(0.0);
if orientation == Orientation::Normal && lightroom_angle == 0.0 {
return;
}
let w = if stored_aspect.is_finite() && stored_aspect > 0.0 { stored_aspect } else { 1.0 };
let (ow, oh) = if orientation.swaps_axes() { (1.0, w) } else { (w, 1.0) };
// a mirror turns the straighten the other way too
let angle = if orientation.to_parts().0 { lightroom_angle } else { -lightroom_angle };
let to_rotated = Affine::rotate_about(angle.to_radians(), Point::new(ow / 2.0, oh / 2.0));
let corner = |x: f64, y: f64| {
let (u, v) = orientation.map(x * w, y, w, 1.0);
to_rotated.apply(Point::new(u, v))
};
let r = Rect::from_points(corner(x0, y0), corner(x1, y1));
geometry.insert("rect".into(), json!({"x0": r.x0 / ow, "y0": r.y0 / oh, "x1": r.x1 / ow, "y1": r.y1 / oh}));
geometry.insert("angle".into(), json!(angle));
}
/// The catalog's orientation (the stored image to the photo as Lightroom shows it), then the
/// photo's own Rotate Left / Right and Flip in LightCraft (`user`), which an update of a photo
/// already in the library keeps: the frame LightCraft draws masks in.
fn in_photo_frame(catalog: Orientation, user: Orientation) -> Orientation {
let (flip, turns) = user.to_parts();
let o = if flip { catalog.flipped_h() } else { catalog };
(0..turns).fold(o, |o, _| o.rotated(true))
}
/// Rename a catalog Lua component's `Inverted` to XMP's `MaskInverted` (see [`mapped_settings`]);
/// a `crs:MaskInverted` already present wins, and this does not touch `Flipped`.
fn alias_catalog_inverted(v: &mut lightcraft_meta::XmpValue) {
use lightcraft_meta::XmpValue;
match v {
XmpValue::Array(items) => items.iter_mut().for_each(alias_catalog_inverted),
XmpValue::Struct(map) => {
if let Some(inverted) = map.remove("crs:Inverted")
&& !map.contains_key("crs:MaskInverted")
{
map.insert("crs:MaskInverted".into(), inverted);
}
map.values_mut().for_each(alias_catalog_inverted);
}
XmpValue::Text(_) => {}
}
}
/// Lightroom's masks in LightCraft's frame. Like the crop's corners, their positions are on the
/// stored image (before orientation), normalized to its size; LightCraft's are on the oriented
/// image. So the points go through the orientation, a radial's radii (in long-edge units, from the
/// stored image's aspect) swap when the axes do, and a mirror turns its angle the other way. A
/// quarter turn keeps the angle: the ellipse turned 90Β° is the same ellipse with its radii swapped.
fn catalog_masks(partial: &mut Value, orientation: Orientation) {
if orientation == Orientation::Normal {
return;
}
let point = |p: Option<&mut Value>| {
let Some(p) = p.and_then(Value::as_object_mut) else { return };
let (Some(x), Some(y)) = (p.get("x").and_then(Value::as_f64), p.get("y").and_then(Value::as_f64)) else { return };
let (u, v) = orientation.map(x, y, 1.0, 1.0);
p.insert("x".into(), json!(u));
p.insert("y".into(), json!(v));
};
let Some(masks) = partial.get_mut("masks").and_then(Value::as_array_mut) else { return };
let components = masks.iter_mut().filter_map(|m| m.get_mut("components").and_then(Value::as_array_mut)).flatten();
for shape in components.filter_map(|c| c.get_mut("shape").and_then(Value::as_object_mut)) {
point(shape.get_mut("start"));
point(shape.get_mut("end"));
point(shape.get_mut("center"));
let strokes = shape.get_mut("strokes").and_then(Value::as_array_mut).into_iter().flatten();
for points in strokes.filter_map(|s| s.get_mut("points").and_then(Value::as_array_mut)) {
points.iter_mut().for_each(|p| point(Some(p)));
}
if shape.get("kind").and_then(Value::as_str) == Some("radial") {
if orientation.swaps_axes()
&& let (Some(rx), Some(ry)) = (shape.get("rx").cloned(), shape.get("ry").cloned())
{
shape.insert("rx".into(), ry);
shape.insert("ry".into(), rx);
}
if orientation.to_parts().0
&& let Some(angle) = shape.get("angle").and_then(Value::as_f64)
{
shape.insert("angle".into(), json!(-angle));
}
}
}
}
/// The catalog's XMP masks in LightCraft's frame, as [`catalog_masks`] puts the develop row's: they
/// are on the same stored image, and they stay when the row has no masks of its own. The packet's
/// radials come fitted to the sidecar default aspect and `sidecar::merge_into` re-fits them to the
/// photo's size, so they are fitted to the stored image to be turned (a quarter turn swaps the
/// radii) and handed back in the default aspect, for that re-fit to undo.
fn catalog_xmp_masks(partial: &mut Value, orientation: Orientation, stored_aspect: f64, photo: &Photo) {
use crate::crs_masks::{DEFAULT_ASPECT, refit_radials};
let merged = if photo.width > 0 && photo.height > 0 { photo.width as f64 / photo.height as f64 } else { DEFAULT_ASPECT };
let fit = if stored_aspect.is_finite() && stored_aspect > 0.0 { stored_aspect } else { merged };
refit_radials(partial, DEFAULT_ASPECT, fit);
catalog_masks(partial, orientation);
refit_radials(partial, merged, DEFAULT_ASPECT);
}
fn strip_lua_sentinels(value: &mut crate::preset_import::Lua) -> usize {
use crate::preset_import::Lua;
match value {
Lua::Num(n) if *n == -999999.0 => {
*value = Lua::Nil;
1
}
Lua::Table(array, fields) => {
let mut removed = array.iter_mut().map(strip_lua_sentinels).sum::<usize>();
fields.retain_mut(|(_, value)| {
removed += strip_lua_sentinels(value);
!matches!(value, Lua::Nil)
});
removed
}
_ => 0,
}
}
fn strip_xmp_sentinels(value: &mut Value) {
match value {
Value::Object(fields) => fields.retain(|_, value| {
if value.as_f64() == Some(-999999.0) {
return false;
}
strip_xmp_sentinels(value);
true
}),
Value::Array(values) => {
if values.iter().any(|v| v.as_f64() == Some(-999999.0)) {
values.clear();
} else {
for v in values {
strip_xmp_sentinels(v);
}
}
}
_ => {}
}
}
/// Import into the current library. Existing edited records are preserved unless explicitly
/// requested; missing originals remain catalogued for later relinking. The source catalog is read-only.
pub fn import(s: &mut crate::Session, path: &Path, update_existing: bool) -> crate::Result<Value> {
let mut job = crate::lightroom_job::LightroomJob::new(s, path.to_path_buf(), update_existing)?;
let cancel = AtomicBool::new(false);
let prepared = job.prepare(&cancel)?;
let completion = crate::lightroom_job::commit_prepared(s, prepared)?;
if s.library.as_ref().is_some_and(|library| library.on_disk) {
s.persist()?;
}
completion.finalization.finish()?;
Ok(completion.report)
}
#[cfg(test)]
fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> crate::Result<Value> {
let archive_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"));
let index_path = archive_dir.as_ref().map(|p| p.join("lightroom-index.json"));
let mut index = load_index(index_path.as_deref())?;
let archive_path = archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?;
let undo0 = s.undo.len();
let existing: HashMap<_, _> =
s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();
let paths: Vec<_> = data
.photos
.iter()
.filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && p.original_is_file())
.map(|p| p.path.clone())
.collect();
let missing_sources: HashSet<String> =
data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !p.original_is_file()).map(|p| key(&p.path)).collect();
let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?;
let applied =
apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?;
if let Some(path) = index_path {
s.persist()?;
let bytes = serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?;
lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?;
}
Ok(applied.report)
}
/// Apply prepared normal-file results and Lightroom metadata in one owner-thread operation.
/// This function does no filesystem work; callers finish its returned index separately.
pub(crate) struct ApplyPrepared<'a> {
pub(crate) update_existing: bool,
pub(crate) files: crate::import::ImportReport,
pub(crate) index: &'a mut ImportIndex,
pub(crate) archive_path: Option<PathBuf>,
pub(crate) missing_sources: &'a HashSet<String>,
pub(crate) undo0: usize,
}
/// The catalog's collections with each id once, in catalog order. A row repeated as it is drops quietly; a different
/// collection with an id an earlier one has is left out with a warning naming both, since two albums can't have one id
/// (#1588: one bad Lightroom record costs that record, not the import). The photos listed under the id go to the first.
fn first_of_each_collection<'a>(data: &'a CatalogImport, warnings: &mut Vec<String>) -> Vec<&'a Row> {
let mut first: HashMap<i64, &Row> = HashMap::new();
let mut rows = Vec::new();
for collection in &data.collections {
let id = number(collection, "id_local");
let Some(kept) = first.get(&id) else {
first.insert(id, collection);
rows.push(collection);
continue;
};
if *kept != collection {
let photos = if data.members.iter().any(|m| number(m, "collection") == id) {
format!("; the photos listed under that id go to β{}β", text(kept, "name"))
} else {
String::new()
};
warnings.push(format!(
"collection β{}β: not imported, it has the id of collection β{}β{photos}",
text(collection, "name"),
text(kept, "name")
));
}
}
rows
}
fn stage_collections(
s: &mut crate::Session,
data: &CatalogImport,
index: &mut ImportIndex,
ids: &HashMap<i64, PhotoId>,
ops: &mut Vec<Op>,
warnings: &mut Vec<String>,
) -> crate::Result<usize> {
let rows = first_of_each_collection(data, warnings);
let mut albums = HashMap::new();
// Two collections never share an album. An index that names one album for two gives the second a new one; the
// counter saturates, so a damaged one hands out an id that's taken, and that stops the import instead.
let mut assigned = HashSet::new();
for &collection in &rows {
if number(collection, "systemOnly") != 0 {
continue;
}
// keyed on the catalog's own id (the same in its backups); imports made before that was
// read are keyed on the catalog's path, and are still found by it, also from a backup (the
// path of the catalog it was backed up from)
let path_key = format!("{}:{}", key(&data.source), number(collection, "id_local"));
let original_key = data.catalog_path.as_ref().map(|p| format!("{}:{}", key(p), number(collection, "id_local")));
let collection_key = data.catalog_id.as_ref().map_or_else(|| path_key.clone(), |c| format!("lrcat:{c}:{}", number(collection, "id_local")));
// A path can be reused by a different catalog. Without an ID we cannot prove that an
// owned path key belongs to this input, so refuse instead of silently mixing albums.
if data.catalog_id.is_none()
&& [Some(&path_key), original_key.as_ref()].into_iter().flatten().any(|k| index.collection_owners.contains_key(k))
{
return Err(error(
"the Lightroom catalog has no stable identity and its path was previously associated with a catalog; cannot safely match collections",
));
}
// a key recorded for another catalog (by its id) is not this one, whatever its path
let mine = |k: &String| match (data.catalog_id.as_ref(), index.collection_owners.get(k)) {
(Some(c), Some(owner)) => owner == c,
_ => true,
};
let id = [Some(&collection_key), Some(&path_key), original_key.as_ref()]
.into_iter()
.flatten()
.filter(|k| mine(k))
.filter_map(|k| index.collections.get(k).copied())
.map(lightcraft_catalog::AlbumId)
.find(|id| s.catalog.album(*id).is_some() && !assigned.contains(id));
let id = match id {
Some(id) => id,
None => {
let id = s.catalog.alloc_album_id();
if s.catalog.album(id).is_some() || assigned.contains(&id) {
return Err(error(format!(
"collection β{}β: the library has no unused album id left for it (its album numbering is exhausted or damaged)",
text(collection, "name")
)));
}
id
}
};
assigned.insert(id);
// recorded under its id and the live catalog's path (a backup's: the catalog it was made
// from), so a later import of the same catalog finds it in either order; a zip's own path
// is not kept when the catalog's is known (every dated backup has a new one)
let path_kept = if data.catalog_path.is_some() { None } else { Some(path_key) };
for k in [Some(collection_key), path_kept, original_key].into_iter().flatten() {
if let Some(c) = &data.catalog_id {
index.collection_owners.insert(k.clone(), c.clone());
}
index.collections.insert(k, id.0);
}
albums.insert(number(collection, "id_local"), id);
}
let parents: HashMap<_, _> = rows.iter().map(|r| (number(r, "id_local"), number(r, "parent"))).collect();
let collections: HashMap<_, _> = rows.iter().map(|r| (number(r, "id_local"), *r)).collect();
let mut grouped_members: HashMap<i64, Vec<&Row>> = HashMap::new();
for member in &data.members {
grouped_members.entry(number(member, "collection")).or_default().push(member);
}
for members in grouped_members.values_mut() {
members.sort_by(|a, b| text(a, "positionInCollection").cmp(text(b, "positionInCollection")));
}
let mut ordered = rows.clone();
ordered.sort_by_key(|r| {
let mut depth = 0;
let mut at = number(r, "parent");
while at != 0 && depth < 64 {
depth += 1;
at = parents.get(&at).copied().unwrap_or(0);
}
depth
});
// Collections whose albums are folders: a collection set, or an earlier import's album when it is a folder.
let sets: HashSet<i64> = albums
.iter()
.filter(|(lr, id)| match s.catalog.album(**id) {
Some(album) => album.folder,
None => collections.get(*lr).is_some_and(|r| text(r, "creationId").contains("group")),
})
.map(|(lr, _)| *lr)
.collect();
let mut added_folders = HashSet::new();
for collection in ordered {
let Some(id) = albums.get(&number(collection, "id_local")).copied() else { continue };
let mut album = Album::new(id, text(collection, "name"));
album.folder = text(collection, "creationId").contains("group");
if text(collection, "creationId").contains("smart") {
warnings.push(format!("{}: smart rules preserved in archive; imported current membership", album.name));
}
if !album.folder {
album.photos = grouped_members
.get(&number(collection, "id_local"))
.into_iter()
.flatten()
.filter_map(|r| ids.get(&number(r, "image")).copied())
.collect();
}
if s.catalog.album(id).is_some() {
// Reimport preserves the user's album name/hierarchy, merges only new membership.
if let Some(old) = s.catalog.album(id).filter(|a| !a.folder && !a.is_smart()) {
let mut photos = old.photos.clone();
let mut seen: HashSet<_> = photos.iter().copied().collect();
for photo in album.photos {
if seen.insert(photo) {
photos.push(photo);
}
}
ops.push(Op::SetAlbumPhotos { id, photos });
}
} else {
let (set, moved) = collection_parent(collection, &collections, &sets);
album.parent = set.and_then(|set| albums.get(&set).copied());
// `AddAlbum` rejects a parent that isn't a folder yet, and that would roll back the whole batch: every
// album and every catalog-only photo. `collection_parent` already prevents it; this keeps a bug there
// (or a cycle `validate` missed) from doing so.
if let Some(parent) = album.parent
&& !added_folders.contains(&parent)
&& !s.catalog.album(parent).is_some_and(|a| a.folder)
{
album.parent = None;
warnings.push(format!(
"{}: its collection set could not be created first; imported at the top level (Lightroom collection {})",
album.name,
number(collection, "id_local")
));
} else if moved {
let lr_parent = number(collection, "parent");
let why = match collections.get(&lr_parent) {
None => "its parent collection isn't in the catalog".to_string(),
Some(r) => match albums.get(&lr_parent).and_then(|id| s.catalog.album(*id)) {
// a set imported earlier whose album is no longer a folder
Some(a) if text(r, "creationId").contains("group") => {
format!("its collection set β{}β is the album β{}β in this library, which isn't a folder", text(r, "name"), a.name)
}
_ => format!("Lightroom keeps it inside β{}β, which isn't a collection set", text(r, "name")),
},
};
// the set's name in this library: the user may have renamed it since an earlier import
let to = match (album.parent.and_then(|id| s.catalog.album(id)), set.and_then(|set| collections.get(&set))) {
(Some(a), _) => format!("into β{}β", a.name),
(None, Some(r)) => format!("into β{}β", text(r, "name")),
(None, None) => "at the top level".to_string(),
};
// the Lightroom id tells apart collections that share a name (#2165)
warnings.push(format!("{}: {why}; imported {to} (Lightroom collection {})", album.name, number(collection, "id_local")));
}
if album.folder {
added_folders.insert(id);
}
ops.push(Op::AddAlbum { album });
}
}
Ok(albums.len())
}
/// Turn the read catalog into library records in one undo step and return the import report
/// (added, missing, failed, warnings and skipped images).
pub(crate) fn apply_prepared(s: &mut crate::Session, mut data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {
let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context;
// nothing but a file type LightCraft reads becomes a record, whichever route got here (#2135)
refuse_non_media(&mut data);
validate(&data).map_err(error)?;
let mut warnings = data.warnings.clone();
let duplicates: HashMap<_, _> =
files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), (PhotoId(id), d.file.as_deref())))).collect();
let mut by_path: HashMap<_, _> =
s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();
let mut ids = HashMap::new();
let mut ops = Vec::new();
let now = (s.clock)();
let mut missing = Vec::new();
// Normal import counts probed originals; catalog import also adds offline records,
// distinct originals with identical bytes, and virtual copies.
let mut imported = files.imported.len();
let mut promoted = HashSet::new();
let mut unmapped = BTreeMap::new();
let mut preserved = 0;
// Library photos in Recently Deleted that a catalog photo matched: they stay there (only an
// import of the files themselves brings a photo back, see `OnDeleted`), and the report says so.
let mut in_trash = HashSet::new();
// Masters precede their virtual copies.
let mut photos: Vec<_> = data.photos.iter().collect();
photos.sort_by_key(|p| number(&p.image, "masterImage") != 0);
let copy_masters: HashSet<_> = data.photos.iter().map(|p| number(&p.image, "masterImage")).filter(|id| *id != 0).collect();
let refused: HashSet<_> = data.photos.iter().filter(|p| matches!(p.original, Original::Refused(_))).map(|p| p.source_id).collect();
let mut planned: HashMap<PhotoId, Photo> = HashMap::new();
for src in photos {
let master = number(&src.image, "masterImage");
if master == 0 && missing_sources.contains(&key(&src.path)) {
missing.push(src.path.clone());
}
// A refused path must not become a library record the app would look at later; its
// virtual copies go with it.
if refused.contains(&src.source_id) || refused.contains(&master) {
continue;
}
let indexed = index
.photos
.get(&identity(&data, src))
.copied()
.map(PhotoId)
.filter(|id| s.catalog.photo(*id).is_some_and(|p| source_path(p).is_some_and(|path| key(path) == key(&src.path))));
let existing_id = indexed.or_else(|| if master == 0 { by_path.get(&key(&src.path)).copied() } else { None });
let mut p = if let Some(id) = existing_id.and_then(|id| s.catalog.photo(id)) {
if id.deleted {
in_trash.insert(id.id);
}
(**id).clone()
} else if master != 0 {
let master_id = ids.get(&master).copied().ok_or_else(|| error(format!("virtual copy {}: missing master", src.source_id)))?;
let mut p = planned
.get(&master_id)
.cloned()
.or_else(|| s.catalog.photo(master_id).map(|p| (**p).clone()))
.ok_or_else(|| error("virtual copy master is unavailable"))?;
p.id = s.catalog.alloc_photo_id();
p.copy_of = Some(master_id);
p.copy_name = Some(text(&src.image, "copyName").to_string());
// a new copy of a master in Recently Deleted is imported, not trashed with it
p.deleted = false;
p
} else if let Some((duplicate, file)) =
duplicates.get(&key(&src.path)).and_then(|(id, meta)| s.catalog.photo(*id).cloned().map(|p| (p, *meta)))
{
// Catalogs can contain distinct originals with identical bytes and independent edits.
// The new record takes only what the bytes determine from the photo with the same
// bytes, never that photo's own state: not its Recently Deleted (#2589), virtual copy,
// edits, history, versions, ratings, capture time or the metadata a person writes. Its
// metadata and capture time are what its own file gives it, as a file import reads them
// (copyright, creatorβ¦); the catalog's capture time, when it has one, applies on top.
let name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| src.path.clone());
let mut p = Photo::new(
s.catalog.alloc_photo_id(),
Source::File { path: src.path.clone() },
&name,
&duplicate.format,
duplicate.width,
duplicate.height,
&now,
);
p.kind = duplicate.kind;
p.file_size = duplicate.file_size;
p.duration = duplicate.duration;
p.as_shot_wb = duplicate.as_shot_wb;
p.content_hash = duplicate.content_hash.clone();
p.embedded_lens = duplicate.embedded_lens;
p.preview_only = duplicate.preview_only.clone();
p.preview_checked = duplicate.preview_checked.clone();
if let Some(file) = file {
p.meta = file.meta.clone();
p.captured = file.captured.clone();
} else {
// (the file wasn't read: only the exposure description, which no command edits;
// not GPS, which placing a photo on the map does, nor the capture time, which
// Edit Capture Time does)
let m = &duplicate.meta;
p.meta.camera = m.camera.clone();
p.meta.lens = m.lens.clone();
p.meta.focal_mm = m.focal_mm;
p.meta.aperture = m.aperture;
p.meta.shutter = m.shutter.clone();
p.meta.iso = m.iso;
p.meta.focus = m.focus.clone();
}
// the settings a file import gives it (as-shot white balance, embedded lens
// corrections, the default preset); the catalog's settings apply on top
crate::import::apply_import_defaults(s, &mut p);
p
} else {
let id = s.catalog.alloc_photo_id();
let name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| src.path.clone());
let mut p = Photo::new(
id,
Source::File { path: src.path.clone() },
&name,
text(&src.image, "fileFormat"),
number(&src.image, "fileWidth").max(0) as u32,
number(&src.image, "fileHeight").max(0) as u32,
&now,
);
if ["RAW", "DNG", "ARW", "CR2", "CR3", "CRW", "NEF", "NRW", "RAF", "ORF", "RW2", "PEF"].contains(&p.format.to_uppercase().as_str()) {
p.kind = lightcraft_catalog::MediaKind::Raw;
}
p
};
// Offline videos bypass file probing; Lightroom's format still identifies their kind.
if text(&src.image, "fileFormat").eq_ignore_ascii_case("VIDEO") {
p.kind = lightcraft_catalog::MediaKind::Video;
}
ids.insert(src.source_id, p.id);
index.photos.insert(identity(&data, src), p.id.0);
let keep_develop = existing_id.is_some() && p.edited.is_some() && !update_existing;
let previous_develop = (p.develop.clone(), p.edited.clone());
if keep_develop {
preserved += 1;
}
let orientation = lightroom_orientation(text(&src.image, "orientation")).unwrap_or_default();
let stored_aspect = number(&src.image, "fileWidth") as f64 / number(&src.image, "fileHeight") as f64;
if !src.xmp.is_empty() {
match crate::sidecar::parse_sidecar(&src.xmp, crate::crs::Target::for_photo(&p)) {
Ok(mut sc) => {
if let Some(crate::sidecar::DevelopPatch::Partial(partial)) = &mut sc.develop {
strip_xmp_sentinels(partial);
catalog_xmp_masks(partial, in_photo_frame(orientation, p.develop.orientation), stored_aspect, &p);
}
crate::sidecar::merge_into(&mut p, &sc, &now);
}
Err(e) => warnings.push(format!("image {} XMP: {e}", src.source_id)),
}
}
p.rating = number(&src.image, "rating").clamp(0, 5) as u8;
p.flag = match number(&src.image, "pick") {
1 => Flag::Pick,
-1 => Flag::Reject,
_ => Flag::None,
};
p.label = s.catalog.label_from_name(text(&src.image, "colorLabels"));
if !text(&src.image, "captureTime").is_empty() {
p.captured = Some(text(&src.image, "captureTime").into());
}
if !src.keywords.is_empty() {
p.meta.keywords = src.keywords.clone();
}
if !src.settings.is_empty() && !keep_develop {
match mapped_settings(
&src.settings,
crate::crs::Target::for_photo(&p),
p.width.max(1) as f64 / p.height.max(1) as f64,
orientation,
stored_aspect,
p.develop.orientation,
) {
Ok((partial, unknown)) => {
p.develop = Arc::new(lightcraft_develop::apply_partial(&p.develop, &partial, 1.0));
p.edited = Some(now.clone());
if !unknown.is_empty() {
unmapped.insert(src.source_id, unknown);
}
}
Err(e) => warnings.push(format!("image {} settings: {e}", src.source_id)),
}
}
if keep_develop {
(p.develop, p.edited) = previous_develop;
}
// Missing files bypass normal-file preparation, including its Local promotion.
// Import still joins the library, and a virtual copy must inherit that membership.
let was_local = p.local;
p.local = false;
let photo_id = p.id;
// Only virtual-copy masters need a second owned photo while operations are staged.
// Keeping every photo here doubled the new-record working set on large catalogs.
if copy_masters.contains(&src.source_id) {
planned.insert(photo_id, p.clone());
}
if let Some(old) = s.catalog.photo(p.id) {
if old.kind != p.kind {
// Repair older imports with existing reversible operations. Removing just the
// record leaves album/stack membership intact; the enclosing batch is atomic.
let mut repaired = (**old).clone();
repaired.kind = p.kind;
ops.extend([Op::RemovePhoto { id: p.id }, Op::AddPhoto { photo: Box::new(repaired) }]);
}
if was_local && promoted.insert(p.id) {
imported += 1;
ops.push(Op::SetLocal { id: p.id, local: false });
}
ops.extend([
Op::SetRating { id: p.id, rating: p.rating },
Op::SetFlag { id: p.id, flag: p.flag },
Op::SetLabel { id: p.id, label: p.label },
Op::SetMeta { id: p.id, meta: Box::new(p.meta.clone()) },
Op::SetCaptured { id: p.id, captured: p.captured.clone() },
]);
ops.push(Op::SetDevelop { id: p.id, settings: p.develop.clone(), label: "Lightroom catalog".into(), edited: p.edited.clone() });
} else {
imported += 1;
ops.push(Op::AddPhoto { photo: Box::new(p) });
}
by_path.entry(key(&src.path)).or_insert(photo_id);
}
let album_count = stage_collections(s, &data, index, &ids, &mut ops, &mut warnings)?;
if !ops.is_empty() {
s.commit("Import Lightroom Catalog", Op::Batch { ops })?;
}
s.merge_undo(s.undo.len().saturating_sub(undo0), "Import Lightroom Catalog");
Ok(AppliedImport {
report: json!({"source":data.source,"photos":ids.len(),"imported":imported,"library":s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.to_string_lossy()),"collections":album_count,"preservedExistingEdits":preserved,"inRecentlyDeleted":in_trash.len(),"missing":missing,"failed":files.failed,"warnings":warnings,"skipped":data.skipped,"unmapped":unmapped,"archive":archive_path,"mapping":ids}),
index: index.clone(),
})
}
/// The collection set a Lightroom collection's album goes into (`None`: the top level), and whether that isn't its
/// Lightroom parent. A LightCraft album nests only in a folder, but Lightroom also keeps a print, book, slideshow or
/// web gallery (or, in an odd catalog, a collection) inside a plain or smart collection, and a parent can be
/// system-only (not imported) or gone from the catalog. Such an album joins the nearest collection set above it.
fn collection_parent(collection: &Row, collections: &HashMap<i64, &Row>, sets: &HashSet<i64>) -> (Option<i64>, bool) {
let mut at = number(collection, "parent");
let mut seen = HashSet::from([number(collection, "id_local")]);
while at != 0 && !sets.contains(&at) {
// `validate` rejects cycles and deep chains; stop at the top level if one got here anyway.
if seen.len() > 64 || !seen.insert(at) {
return (None, true);
}
at = collections.get(&at).map_or(0, |r| number(r, "parent"));
}
((at != 0).then_some(at), at != number(collection, "parent"))
}
#[cfg(test)]
mod tests;
No file is open. Choose one in the tree, or press Ctrl+O (βO on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.