diff --git a/ROADMAP.md b/ROADMAP.md index b691b9c53..73891acc0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -191,6 +191,18 @@ Not decoded yet — preview only (no permissively licensed description; black-bo matrices stored in the files themselves (Olympus ImageProcessing `ColorMatrix`, Pentax equivalents; Panasonic RW2 files carry none) and our own chart-based calibration (M11.4). Adobe matrices are never used. +## Lightroom Classic catalog migration + +File → Import Lightroom Catalog… reads `.lrcat` plus committed WAL pages directly through a pure-Rust +SQLite reader. Originals stay in place; ratings, flags, labels, XMP metadata, hierarchical keywords, +collections/sets, virtual copies and supported develop settings migrate into LightCraft. Existing edits +are preserved by default, reimport identities are persistent, and source settings/history/snapshots are +archived before catalog mutation. Missing originals remain available for relinking. + +Rendering is approximate: unsupported Adobe profiles/AI/process settings are reported and archived; +history/snapshots remain source data, and smart collections import current membership. The Lightroom +database is read-only; LightCraft owns subsequent edits. See `docs/lightroom-catalog-import.md`. + ## Log - 2026-09-30: roadmap created; M0 in progress; research docs (Lightroom reference, Rust imaging ecosystem) complete. - 2026-09-30 (later): app running with the full Lightroom-style UI; pipeline v0; DNG/CR2/ARW; README showcase. ≈ 8 h elapsed. diff --git a/apps/lightcraft-web/src/web.rs b/apps/lightcraft-web/src/web.rs index 41e64c024..ede1de0df 100644 --- a/apps/lightcraft-web/src/web.rs +++ b/apps/lightcraft-web/src/web.rs @@ -207,6 +207,7 @@ fn services(originals: Originals, backend: Option, files: Files, frozen // Preset files: browser pickers are asynchronous; not wired on the web yet. pick_preset_files: None, pick_tracklog: None, + pick_lightroom_catalog: None, save_preset_file: None, pick_curve_preset_files: None, save_curve_preset_file: None, diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index 363141766..37b17f37c 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -283,6 +283,13 @@ fn windows_open_url_command(url: &str) -> std::process::Command { fn services() -> Services { Services { + pick_lightroom_catalog: Some(Box::new(|| { + rfd::FileDialog::new() + .set_title(lightcraft_ui_egui::i18n::tr("Import Lightroom Catalog")) + .add_filter(lightcraft_ui_egui::i18n::tr("Lightroom Classic Catalog"), &["lrcat"]) + .pick_file() + .map(|p| p.to_string_lossy().to_string()) + })), pick_folder: Some(Box::new(|| { rfd::FileDialog::new().set_title(lightcraft_ui_egui::i18n::tr("Open Library")).pick_folder().map(|p| p.to_string_lossy().to_string()) })), diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index c37404b53..436ad123b 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -1062,6 +1062,25 @@ pub fn specs() -> Vec { ok() }), // ---- import + cmd!(query "library.inspectLightroom", "Inspect Lightroom Catalog", [], None, + "{path: .lrcat} → {photos, collections, missing, warnings}; read-only, no Lightroom required", + always, |_, p| { + let path = str_param(p,"path").ok_or_else(|| bad("library.inspectLightroom","missing path"))?; + let data = crate::lightroom_catalog::read(std::path::Path::new(path)).map_err(|e| bad("library.inspectLightroom",e))?; + Ok(json!({"photos":data.photos.len(),"collections":data.collections.iter().filter(|r|r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0)==0.0).count(),"missing":data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p|&p.path).collect::>(),"warnings":data.warnings})) + }), + cmd!( + "library.importLightroom", + "Import Lightroom Catalog", + [], + None, + "{path: .lrcat, updateExisting?: false}; direct import of paths, ratings, flags, metadata, keywords, collections, virtual copies & mapped edits. Archives source settings/history first. Existing edited photos are preserved unless updateExisting=true; source catalog is read-only.", + always, + |s, p| { + let path = str_param(p, "path").ok_or_else(|| bad("library.importLightroom", "missing path"))?; + crate::lightroom_catalog::import(s, std::path::Path::new(path), bool_or(p, "updateExisting", false)) + } + ), cmd!( query "library.importPreview", "Review Import", diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 5df949887..e8a95f0f0 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -641,6 +641,50 @@ impl Prepared { pub fn is_empty(&self) -> bool { self.items.is_empty() } + + /// Recheck Add-only entries against the owner catalog immediately before commit. + /// Preparation may have run while another import added the same path or content. + pub fn revalidate_add(&mut self, s: &Session) { + let mut paths = HashMap::new(); + let mut hashes = HashMap::new(); + for photo in s.catalog.photos() { + if let Source::File { path } = &photo.source { + paths.insert(normalize_import_path(path), photo.id); + } + if let Some(hash) = &photo.content_hash { + hashes.insert(hash.clone(), photo.id); + } + } + let mut items = Vec::with_capacity(self.items.len()); + for item in std::mem::take(&mut self.items) { + let PreparedItem::Ready(ready) = item else { + items.push(item); + continue; + }; + let ReadyFile { path, stored, info, sidecar, placed } = *ready; + let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied()); + let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied()); + if let Some(existing) = existing_path.or(existing_hash) { + if let Some(placed) = &placed { + crate::import_move::rollback(placed); + } + items.push(PreparedItem::Duplicate { + path, + existing: Some(existing), + reason: if existing_path.is_some() { "path" } else { "content" }, + hash: info.content_hash, + }); + } else { + items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed }))); + } + } + self.items = items; + } +} + +fn normalize_import_path(path: &str) -> String { + let path = path.replace('\\', "/"); + if cfg!(windows) { path.to_lowercase() } else { path } } impl ImportJob { @@ -1093,3 +1137,31 @@ impl Session { self } } + +#[cfg(test)] +mod prepared_tests { + use super::*; + + #[test] + fn revalidate_add_only_converts_matching_ready_item() { + let mut s = Session::new(); + let existing_path = std::env::temp_dir().join("lightcraft-revalidate-existing.jpg"); + let other_path = std::env::temp_dir().join("lightcraft-revalidate-other.jpg"); + let existing = Photo::new(PhotoId(1), Source::File { path: existing_path.to_string_lossy().into() }, "existing.jpg", "JPG", 1, 1, "now"); + s.commit("existing", Op::AddPhoto { photo: Box::new(existing) }).unwrap(); + let ready = |path: &Path| ReadyFile { + path: path.to_string_lossy().into(), + stored: path.to_string_lossy().into(), + info: ProbeInfo { format: "JPG".into(), ..ProbeInfo::default() }, + sidecar: None, + placed: None, + }; + let mut prepared = Prepared { + scanned: 2, + items: vec![PreparedItem::Ready(Box::new(ready(&existing_path))), PreparedItem::Ready(Box::new(ready(&other_path)))], + }; + prepared.revalidate_add(&s); + assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. }))); + assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_)))); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index fab178964..0db9ea62f 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -25,6 +25,10 @@ pub mod guard; pub mod import; mod import_move; pub mod library; +mod lightroom_archive; +pub mod lightroom_catalog; +pub mod lightroom_job; +mod lightroom_sqlite; pub mod logging; pub mod media; pub mod memory; @@ -186,6 +190,8 @@ pub struct Session { pub active_spot: Option, /// The persistent library this session writes to (`None` = in-memory only). pub library: Option, + /// Identity of current library opening; refreshed only after a successful open. + pub(crate) library_identity: Arc<()>, /// XMP sidecar preferences (persisted with the library). pub xmp: sidecar::XmpPrefs, /// Parameters of the last export (`app.export` params, minus targets), persisted in prefs.json. @@ -274,6 +280,7 @@ impl Session { segmenter: segment::Segmenter::default(), active_spot: None, library: None, + library_identity: Arc::new(()), xmp: sidecar::XmpPrefs::default(), last_export: None, export_presets: Vec::new(), diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs index 7f902843e..c1b91c4cd 100644 --- a/crates/engine/src/library.rs +++ b/crates/engine/src/library.rs @@ -412,6 +412,7 @@ impl Session { lib.forgot_local = Some(plan); } } + self.library_identity = std::sync::Arc::new(()); Ok(()) } @@ -504,12 +505,16 @@ impl Session { lib.retry_at = None; lib.last_error = None; } - match (persisted, snapshot) { + let result = match (persisted, snapshot) { (_, Ok(())) => Ok(()), // neither the log nor the snapshot took the queued ops (Err(EngineError::NotSaved(e)), Err(s)) if lib.journal.seq() == before => Err(EngineError::NotSaved(format!("{e}; snapshot: {s}"))), (_, Err(s)) => Err(s.into()), + }; + if result.is_ok() { + self.library_identity = std::sync::Arc::new(()); } + result } fn view_json(&self) -> Vec { diff --git a/crates/engine/src/lightroom_archive.rs b/crates/engine/src/lightroom_archive.rs new file mode 100644 index 000000000..0dceb421d --- /dev/null +++ b/crates/engine/src/lightroom_archive.rs @@ -0,0 +1,685 @@ +//! Bounded, deduplicated persistence for Lightroom import source data. +//! +//! Archives use a deterministic JSON representation wrapped in zlib. The JSON +//! writer streams through a bounded compressor, so an import cannot first build +//! an unbounded archive buffer. Blob columns are omitted by the catalog reader; +//! decoded XMP, +//! settings, history, snapshots, identities, paths, collections & membership +//! remain available for recovery. + +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; + +use miniz_oxide::deflate::{ + core::{CompressorOxide, create_comp_flags_from_zip_params}, + stream::deflate, +}; +use miniz_oxide::{MZFlush, MZStatus}; +use serde::ser::{Serialize, SerializeMap, SerializeSeq, SerializeStruct, Serializer}; +use serde_json::Value; + +use crate::lightroom_catalog::{CatalogImport, CatalogPhoto}; + +/// Maximum size of one complete archive on disk, including format header. +pub const MAX_ARCHIVE_BYTES: u64 = 32 * 1024 * 1024; +/// Maximum total size retained in managed archives. +pub const MAX_TOTAL_ARCHIVE_BYTES: u64 = 128 * 1024 * 1024; +/// Maximum number of managed archives retained. +pub const MAX_MANAGED_ARCHIVES: usize = 8; + +const FORMAT_VERSION: u8 = 1; +const HEADER: &[u8] = b"LC-LRARCH\0\x01Z"; +const NAME_PREFIX: &str = "lightroom-import-"; +const NAME_SUFFIX: &str = ".lca"; +const MAX_SCAN: usize = 1024; +const IO_BUFFER: usize = 16 * 1024; + +/// Store compact source data below `directory` and return its managed archive. +/// +/// The operation is safe to call before catalog mutation. It rejects a +/// symlink directory, writes only `lightroom-import-.lca`, compares the +/// complete bounded byte stream with recent managed archives, and retains at +/// most [`MAX_MANAGED_ARCHIVES`] and [`MAX_TOTAL_ARCHIVE_BYTES`]. Legacy +/// `lightroom-import-*.json` files are deliberately ignored and preserved for +/// an explicit caller warning. +pub fn store(directory: &Path, data: &CatalogImport) -> Result { + prepare_directory(directory)?; + let mut managed = managed_files(directory)?; + managed.sort_by_key(|entry| entry.index); + + let temporary = temporary_path(directory)?; + write_archive(&temporary, data).inspect_err(|_| { + let _ = remove_managed_file(&temporary); + })?; + + let recent = managed.iter().rev().take(MAX_MANAGED_ARCHIVES); + for entry in recent { + let same = match same_bounded(&temporary, &entry.path) { + Ok(same) => same, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error.to_string()); + } + }; + if same { + remove_managed_file(&temporary).map_err(|e| e.to_string())?; + let matched = entry.path.clone(); + let mut all = managed.clone(); + prune_existing(directory, &mut all, Some(&matched))?; + return Ok(matched); + } + } + + let index = managed.last().map_or(0, |entry| entry.index.saturating_add(1)); + if managed.last().is_some_and(|entry| entry.index == u64::MAX) { + let _ = remove_managed_file(&temporary); + return Err("managed Lightroom archive index is exhausted".into()); + } + let destination = match next_destination(directory, index) { + Ok(path) => path, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error); + } + }; + let size = match fs::metadata(&temporary) { + Ok(meta) => meta.len(), + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error.to_string()); + } + }; + let new_entry = Managed { index, path: destination.clone(), size }; + let mut after = managed; + after.push(new_entry); + after.sort_by_key(|entry| entry.index); + let retirees = match retirement_set(&after) { + Ok(retirees) => retirees, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error); + } + }; + let quarantined = match quarantine(&retirees) { + Ok(quarantined) => quarantined, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error); + } + }; + + if let Err(error) = publish_new(&temporary, &destination) { + rollback_quarantine(&quarantined); + let _ = remove_managed_file(&temporary); + return Err(error.to_string()); + } + if let Err(error) = remove_managed_file(&temporary) { + return Err(format!("archive published but temporary cleanup failed: {error}")); + } + for (old, held) in quarantined { + if let Err(error) = remove_managed_file(&held) { + return Err(format!("archive published but could not retire {}: {error}", old.display())); + } + } + Ok(destination) +} + +/// Store source data when possible. An archive that reaches its bounded size is +/// omitted with a warning so catalog import can still complete. +pub fn store_best_effort(directory: &Path, data: &CatalogImport) -> Result, String> { + match store(directory, data) { + Ok(path) => Ok(Some(path)), + Err(error) if error.contains("Lightroom archive exceeds 32 MiB") => Ok(None), + Err(error) => Err(error), + } +} + +#[derive(Clone, Debug)] +struct Managed { + index: u64, + path: PathBuf, + size: u64, +} + +fn prepare_directory(directory: &Path) -> Result<(), String> { + match fs::symlink_metadata(directory) { + Ok(meta) => { + if meta.file_type().is_symlink() || !meta.is_dir() { + return Err(format!("archive directory is not a real directory: {}", directory.display())); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + fs::create_dir_all(directory).map_err(|e| e.to_string())?; + let meta = fs::symlink_metadata(directory).map_err(|e| e.to_string())?; + if meta.file_type().is_symlink() || !meta.is_dir() { + return Err(format!("archive directory is not a real directory: {}", directory.display())); + } + } + Err(error) => return Err(error.to_string()), + } + Ok(()) +} + +fn managed_files(directory: &Path) -> Result, String> { + let mut files = Vec::new(); + for (seen, result) in fs::read_dir(directory).map_err(|e| e.to_string())?.enumerate() { + if seen >= MAX_SCAN { + return Err(format!("too many entries in archive directory (limit {MAX_SCAN})")); + } + let entry = result.map_err(|e| e.to_string())?; + let Some(index) = parse_managed_name(&entry.file_name()) else { continue }; + let meta = fs::symlink_metadata(entry.path()).map_err(|e| e.to_string())?; + if meta.file_type().is_symlink() || !meta.is_file() { + continue; + } + files.push(Managed { index, path: entry.path(), size: meta.len() }); + } + Ok(files) +} + +fn parse_managed_name(name: &std::ffi::OsStr) -> Option { + let name = name.to_str()?; + let digits = name.strip_prefix(NAME_PREFIX)?.strip_suffix(NAME_SUFFIX)?; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + digits.parse().ok() +} + +fn temporary_path(directory: &Path) -> Result { + let pid = std::process::id(); + for slot in 0..MAX_SCAN { + let path = directory.join(format!(".lightroom-import-{pid}-{slot}.tmp")); + match fs::symlink_metadata(&path) { + Ok(_) => continue, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path), + Err(error) => return Err(error.to_string()), + } + } + Err("no safe temporary archive name available".into()) +} + +fn next_destination(directory: &Path, mut index: u64) -> Result { + for _ in 0..MAX_SCAN { + let path = directory.join(format!("{NAME_PREFIX}{index}{NAME_SUFFIX}")); + match fs::symlink_metadata(&path) { + Ok(_) => index = index.checked_add(1).ok_or("managed archive index is exhausted")?, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path), + Err(error) => return Err(error.to_string()), + } + } + Err("no safe managed archive name available".into()) +} + +fn write_archive(path: &Path, data: &CatalogImport) -> Result<(), String> { + lightcraft_catalog::safe_file::write_atomic_with(path, &mut |raw| { + let mut bounded = BoundedWriter { inner: raw, total: 0 }; + bounded.write_all(HEADER)?; + let mut compressor = ZlibWriter::new(&mut bounded); + serde_json::to_writer(&mut compressor, &ArchiveRef(data)).map_err(json_io)?; + compressor.finish()?; + Ok(()) + }) + .map(|_| ()) + .map_err(|error| format!("could not write Lightroom archive: {error}")) +} + +fn json_io(error: serde_json::Error) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error.to_string()) +} + +struct BoundedWriter<'a> { + inner: &'a mut dyn Write, + total: u64, +} + +impl Write for BoundedWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let next = self.total.checked_add(bytes.len() as u64).ok_or_else(limit_error)?; + if next > MAX_ARCHIVE_BYTES { + return Err(limit_error()); + } + self.inner.write_all(bytes)?; + self.total = next; + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +fn limit_error() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "Lightroom archive exceeds 32 MiB") +} + +struct ZlibWriter<'a, 'b> { + sink: &'a mut BoundedWriter<'b>, + compressor: CompressorOxide, + plain: u64, +} + +impl<'a, 'b> ZlibWriter<'a, 'b> { + fn new(sink: &'a mut BoundedWriter<'b>) -> ZlibWriter<'a, 'b> { + let flags = create_comp_flags_from_zip_params(6, 15, 0); + ZlibWriter { sink, compressor: CompressorOxide::new(flags), plain: 0 } + } + + fn run(&mut self, input: &[u8], flush: MZFlush) -> io::Result<()> { + let mut rest = input; + loop { + let mut output = [0u8; IO_BUFFER]; + let result = deflate(&mut self.compressor, rest, &mut output, flush); + self.sink.write_all(&output[..result.bytes_written])?; + match result.status { + Ok(MZStatus::StreamEnd) => return Ok(()), + Ok(MZStatus::Ok) => { + let consumed = result.bytes_consumed; + if consumed > rest.len() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressor progress")); + } + rest = &rest[consumed..]; + if rest.is_empty() && flush == MZFlush::None { + return Ok(()); + } + if consumed == 0 && result.bytes_written == 0 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "compressor made no progress")); + } + } + Ok(status) => { + return Err(io::Error::new(io::ErrorKind::InvalidData, format!("unsupported compressor status: {status:?}"))); + } + Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, format!("compression failed: {error:?}"))), + } + } + } + + fn finish(&mut self) -> io::Result<()> { + self.run(&[], MZFlush::Finish) + } +} + +impl Write for ZlibWriter<'_, '_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let next = self.plain.checked_add(bytes.len() as u64).ok_or_else(limit_error)?; + if next > MAX_ARCHIVE_BYTES { + return Err(limit_error()); + } + self.run(bytes, MZFlush::None)?; + self.plain = next; + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn same_bounded(left: &Path, right: &Path) -> io::Result { + let left_meta = fs::symlink_metadata(left)?; + let right_meta = fs::symlink_metadata(right)?; + if left_meta.file_type().is_symlink() || right_meta.file_type().is_symlink() || !left_meta.is_file() || !right_meta.is_file() { + return Ok(false); + } + let left_len = left_meta.len(); + let right_len = right_meta.len(); + if left_len != right_len || left_len > MAX_ARCHIVE_BYTES { + return Ok(false); + } + let mut left = File::open(left)?; + let mut right = File::open(right)?; + let mut left_buf = [0u8; IO_BUFFER]; + let mut right_buf = [0u8; IO_BUFFER]; + loop { + let left_n = left.read(&mut left_buf)?; + let right_n = right.read(&mut right_buf)?; + if left_n != right_n { + return Ok(false); + } + if left_n == 0 { + return Ok(true); + } + if left_buf[..left_n] != right_buf[..right_n] { + return Ok(false); + } + } +} + +fn retirement_set(entries: &[Managed]) -> Result, String> { + retirement_set_protected(entries, None) +} + +fn retirement_set_protected(entries: &[Managed], protected: Option<&Path>) -> Result, String> { + let mut ordered = entries.to_vec(); + ordered.sort_by_key(|entry| entry.index); + let mut total = ordered.iter().try_fold(0u64, |sum, entry| sum.checked_add(entry.size)).ok_or("archive size overflow")?; + let mut retirees = Vec::new(); + while ordered.len() > MAX_MANAGED_ARCHIVES || total > MAX_TOTAL_ARCHIVE_BYTES { + let Some(position) = ordered.iter().position(|entry| protected.is_none_or(|path| entry.path != path)) else { + return Err("cannot satisfy archive retention limit".into()); + }; + let oldest = ordered.remove(position); + total = total.saturating_sub(oldest.size); + retirees.push(oldest); + } + Ok(retirees) +} + +fn prune_existing(directory: &Path, entries: &mut [Managed], protected: Option<&Path>) -> Result<(), String> { + let retirees = retirement_set_protected(entries, protected)?; + let quarantined = quarantine(&retirees)?; + for (old, held) in quarantined { + if let Err(error) = remove_managed_file(&held) { + return Err(format!("could not retire {} from {}: {error}", old.display(), directory.display())); + } + } + Ok(()) +} + +fn quarantine(entries: &[Managed]) -> Result, String> { + let mut moved = Vec::new(); + let pid = std::process::id(); + for (slot, entry) in entries.iter().enumerate() { + let held = entry.path.with_file_name(format!(".lightroom-retire-{pid}-{slot}.tmp")); + match fs::symlink_metadata(&held) { + Ok(_) => { + rollback_quarantine(&moved); + return Err(format!("retention temporary path already exists: {}", held.display())); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + rollback_quarantine(&moved); + return Err(error.to_string()); + } + } + let meta = fs::symlink_metadata(&entry.path).map_err(|error| { + rollback_quarantine(&moved); + error.to_string() + })?; + if meta.file_type().is_symlink() || !meta.is_file() { + rollback_quarantine(&moved); + return Err(format!("managed archive changed into non-file: {}", entry.path.display())); + } + if let Err(error) = fs::rename(&entry.path, &held) { + rollback_quarantine(&moved); + return Err(error.to_string()); + } + moved.push((entry.path.clone(), held)); + } + Ok(moved) +} + +fn rollback_quarantine(moved: &[(PathBuf, PathBuf)]) { + for (original, held) in moved.iter().rev() { + let _ = fs::rename(held, original); + } +} + +fn publish_new(temporary: &Path, destination: &Path) -> io::Result<()> { + match fs::hard_link(temporary, destination) { + Ok(()) => Ok(()), + Err(error) => Err(io::Error::new(error.kind(), format!("cannot atomically publish archive: {error}"))), + } +} + +fn remove_managed_file(path: &Path) -> io::Result<()> { + let meta = fs::symlink_metadata(path)?; + if meta.file_type().is_symlink() || !meta.is_file() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to remove non-regular archive path")); + } + fs::remove_file(path) +} + +struct ArchiveRef<'a>(&'a CatalogImport); + +impl Serialize for ArchiveRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_struct("CatalogImportArchive", 7)?; + out.serialize_field("version", &FORMAT_VERSION)?; + out.serialize_field("source", &self.0.source)?; + out.serialize_field("photos", &PhotoSeq(&self.0.photos))?; + out.serialize_field("collections", &RowSeq(&self.0.collections))?; + out.serialize_field("members", &RowSeq(&self.0.members))?; + out.serialize_field("collection_content", &RowSeq(&self.0.collection_content))?; + out.serialize_field("warnings", &self.0.warnings)?; + out.end() + } +} + +struct PhotoSeq<'a>(&'a [CatalogPhoto]); +impl Serialize for PhotoSeq<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_seq(Some(self.0.len()))?; + for photo in self.0 { + out.serialize_element(&PhotoRef(photo))?; + } + out.end() + } +} + +struct PhotoRef<'a>(&'a CatalogPhoto); +impl Serialize for PhotoRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let p = self.0; + let mut out = serializer.serialize_struct("CatalogPhotoArchive", 8)?; + out.serialize_field("source_id", &p.source_id)?; + out.serialize_field("uuid", &p.uuid)?; + out.serialize_field("path", &p.path)?; + out.serialize_field("image", &RowRef(&p.image))?; + out.serialize_field("settings", &p.settings)?; + out.serialize_field("xmp", &p.xmp)?; + out.serialize_field("keywords", &p.keywords)?; + out.serialize_field("history", &RowSeq(&p.history))?; + out.serialize_field("snapshots", &RowSeq(&p.snapshots))?; + out.end() + } +} + +struct RowSeq<'a>(&'a [BTreeMap]); +impl Serialize for RowSeq<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_seq(Some(self.0.len()))?; + for row in self.0 { + out.serialize_element(&RowRef(row))?; + } + out.end() + } +} + +struct RowRef<'a>(&'a BTreeMap); +impl Serialize for RowRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_map(None)?; + for (key, value) in self.0 { + out.serialize_entry(key, value)?; + } + out.end() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lightroom_catalog::{CatalogImport, CatalogPhoto}; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT: AtomicU64 = AtomicU64::new(0); + + fn tempdir() -> PathBuf { + let path = std::env::temp_dir().join(format!("lightcraft-lr-archive-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); + fs::create_dir_all(&path).unwrap(); + path + } + + fn data(source: String) -> CatalogImport { + CatalogImport { + source, + photos: Vec::new(), + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: Vec::new(), + } + } + + fn rich_data() -> CatalogImport { + let photo = CatalogPhoto { + source_id: 7, + uuid: "uuid-7".into(), + path: "/photos/one.raw".into(), + image: BTreeMap::from([("rating".into(), serde_json::json!(5)), ("opaque_blob".into(), serde_json::json!([1, 2, 3, 4]))]), + settings: "s = { Exposure2012 = 0.75 }".into(), + xmp: "decoded".into(), + keywords: vec!["Trips|Goa".into()], + history: vec![BTreeMap::from([("text".into(), serde_json::json!("step")), ("curve".into(), serde_json::json!([0, 128, 255]))])], + snapshots: vec![BTreeMap::from([("name".into(), serde_json::json!("before"))])], + }; + CatalogImport { + source: "source.lrcat".into(), + photos: vec![photo], + collections: vec![BTreeMap::from([("name".into(), serde_json::json!("Keepers"))])], + members: vec![BTreeMap::from([("image".into(), serde_json::json!(7))])], + collection_content: vec![BTreeMap::from([("rule".into(), serde_json::json!("rating > 3"))])], + warnings: vec!["one warning".into()], + } + } + + fn managed_count(path: &Path) -> usize { + fs::read_dir(path).unwrap().filter_map(Result::ok).filter(|entry| parse_managed_name(&entry.file_name()).is_some()).count() + } + + #[test] + fn deduplicates_identical_deterministic_archives() { + let path = tempdir(); + let input = data("catalog.lrcat".into()); + let first = store(&path, &input).unwrap(); + let second = store(&path, &input).unwrap(); + assert_eq!(first, second); + assert_eq!(managed_count(&path), 1); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn dedup_returns_archive_that_retention_keeps() { + let path = tempdir(); + let input = data("catalog.lrcat".into()); + let first = store(&path, &input).unwrap(); + for index in 1..MAX_MANAGED_ARCHIVES { + let filler = path.join(format!("lightroom-import-{index}.lca")); + File::create(filler).unwrap().set_len(19 * 1024 * 1024).unwrap(); + } + let reused = store(&path, &input).unwrap(); + assert_eq!(reused, first); + assert!(reused.is_file()); + assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn caps_managed_archive_count_and_preserves_unrelated_files() { + let path = tempdir(); + let human = path.join("notes.txt"); + let legacy = path.join("lightroom-import-0.json"); + fs::write(&human, b"keep").unwrap(); + fs::write(&legacy, b"legacy").unwrap(); + for index in 0..(MAX_MANAGED_ARCHIVES + 3) { + let input = data(format!("catalog-{index}.lrcat")); + store(&path, &input).unwrap(); + } + assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES); + assert_eq!(fs::read(&human).unwrap(), b"keep"); + assert_eq!(fs::read(&legacy).unwrap(), b"legacy"); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn rejects_oversize_before_creating_archive() { + let path = tempdir(); + let input = data("x".repeat((MAX_ARCHIVE_BYTES as usize) + 1)); + assert!(store(&path, &input).is_err()); + assert_eq!(managed_count(&path), 0); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn large_catalog_history_is_best_effort_and_does_not_abort_import() { + let path = tempdir(); + let mut input = data("catalog.lrcat".into()); + input.photos = (0..5_000) + .map(|index| CatalogPhoto { + source_id: index, + uuid: format!("photo-{index}"), + path: format!("/photos/{index}.raw"), + image: BTreeMap::new(), + settings: String::new(), + xmp: String::new(), + keywords: Vec::new(), + history: vec![BTreeMap::from([("text".into(), serde_json::json!(format!("history-{index}-{}", "x".repeat(8 * 1024))))])], + snapshots: Vec::new(), + }) + .collect(); + assert_eq!(store_best_effort(&path, &input).unwrap(), None); + assert_eq!(managed_count(&path), 0); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn archive_roundtrips_zlib_payload_and_preserves_numeric_arrays() { + let path = tempdir(); + let archive = store(&path, &rich_data()).unwrap(); + let bytes = fs::read(archive).unwrap(); + assert!(bytes.starts_with(HEADER)); + let json = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(&bytes[HEADER.len()..], MAX_ARCHIVE_BYTES as usize).unwrap(); + let decoded: Value = serde_json::from_slice(&json).unwrap(); + assert_eq!(decoded["version"], serde_json::json!(FORMAT_VERSION)); + assert_eq!(decoded["source"], serde_json::json!("source.lrcat")); + assert_eq!(decoded["photos"].as_array().unwrap().len(), 1); + assert_eq!(decoded["photos"][0]["source_id"], serde_json::json!(7)); + assert_eq!(decoded["photos"][0]["uuid"], serde_json::json!("uuid-7")); + assert_eq!(decoded["photos"][0]["path"], serde_json::json!("/photos/one.raw")); + assert_eq!(decoded["photos"][0]["settings"], serde_json::json!("s = { Exposure2012 = 0.75 }")); + assert_eq!(decoded["photos"][0]["xmp"], serde_json::json!("decoded")); + assert_eq!(decoded["photos"][0]["keywords"], serde_json::json!(["Trips|Goa"])); + assert_eq!(decoded["photos"][0]["image"]["rating"], serde_json::json!(5)); + assert_eq!(decoded["photos"][0]["image"]["opaque_blob"], serde_json::json!([1, 2, 3, 4])); + assert_eq!(decoded["photos"][0]["history"].as_array().unwrap().len(), 1); + assert_eq!(decoded["photos"][0]["history"][0]["curve"], serde_json::json!([0, 128, 255])); + assert_eq!(decoded["photos"][0]["snapshots"].as_array().unwrap().len(), 1); + assert_eq!(decoded["collections"].as_array().unwrap().len(), 1); + assert_eq!(decoded["collections"][0]["name"], serde_json::json!("Keepers")); + assert_eq!(decoded["members"].as_array().unwrap().len(), 1); + assert_eq!(decoded["members"][0]["image"], serde_json::json!(7)); + assert_eq!(decoded["collection_content"].as_array().unwrap().len(), 1); + assert_eq!(decoded["warnings"], serde_json::json!(["one warning"])); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn retention_enforces_aggregate_byte_cap_with_small_synthetic_entries() { + let mut entries: Vec<_> = (0..MAX_MANAGED_ARCHIVES as u64) + .map(|index| Managed { index, path: PathBuf::from(format!("lightroom-import-{index}.lca")), size: 20 * 1024 * 1024 }) + .collect(); + entries.push(Managed { index: 8, path: PathBuf::from("lightroom-import-8.lca"), size: 20 * 1024 * 1024 }); + let retirees = retirement_set(&entries).unwrap(); + assert_eq!(retirees.len(), 3); + assert_eq!(retirees.iter().map(|entry| entry.index).collect::>(), vec![0, 1, 2]); + } + + #[test] + fn retention_failure_preserves_old_archive() { + let path = tempdir(); + let old = path.join("lightroom-import-0.lca"); + let held = path.join(format!(".lightroom-retire-{}-0.tmp", std::process::id())); + fs::write(&old, b"old archive").unwrap(); + fs::write(&held, b"occupied").unwrap(); + let entry = Managed { index: 0, path: old.clone(), size: 11 }; + assert!(quarantine(&[entry]).is_err()); + assert_eq!(fs::read(&old).unwrap(), b"old archive"); + fs::remove_dir_all(path).unwrap(); + } +} diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs new file mode 100644 index 000000000..3c510fded --- /dev/null +++ b/crates/engine/src/lightroom_catalog.rs @@ -0,0 +1,1090 @@ +//! Direct, read-only Lightroom Classic catalog import. SQLite pages are read in Rust; +//! Lightroom, Python and a C SQLite runtime are not needed. Original paths stay in place. +//! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders. +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue}; +use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +type Row = BTreeMap; +const COMMAND: &str = "library.importLightroom"; +const MAX_FILE: u64 = 1 << 30; + +fn error(why: impl Into) -> crate::EngineError { + crate::EngineError::BadParams { cmd: COMMAND.into(), msg: why.into() } +} + +fn text<'a>(r: &'a Row, k: &str) -> &'a str { + r.get(k).and_then(Value::as_str).unwrap_or("") +} +fn number(r: &Row, k: &str) -> i64 { + r.get(k).and_then(Value::as_f64).filter(|n| n.is_finite()).unwrap_or(0.0) as i64 +} + +pub(crate) fn number_for_job(r: &Row, k: &str) -> i64 { + number(r, k) +} +fn key(path: &str) -> String { + let p = path.replace('\\', "/"); + if cfg!(windows) { p.to_lowercase() } else { p } +} +fn source_path(p: &Photo) -> Option<&str> { + match &p.source { + Source::File { path } => Some(path), + _ => None, + } +} + +struct Reader { + db: Database, + tables: Vec, +} +impl Reader { + fn table(&self, name: &str, required: bool) -> Result, String> { + let Some(t) = self.tables.iter().find(|t| t.name == name) else { + return if required { Err(format!("unsupported Lightroom catalog: missing {name}")) } else { Ok(Vec::new()) }; + }; + let columns = t.column_names.as_ref().ok_or_else(|| format!("cannot read {name} columns"))?; + let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("{name}: {e:?}"))?; + if rows.len() > 1_000_000 { + return Err(format!("{name}: more than one million records")); + } + Ok(rows + .into_iter() + .map(|r| { + columns + .iter() + .cloned() + .zip(r.values.into_iter().map(|v| match v { + SqlValue::Null => Some(Value::Null), + SqlValue::Integer(n) => Some(json!(n)), + SqlValue::Real(n) => Some(json!(n)), + SqlValue::Text(s) => Some(json!(s)), + // Opaque Lightroom payloads never become JSON byte arrays. + SqlValue::Blob(_) => None, + })) + .filter_map(|(column, value)| value.map(|value| (column, value))) + .collect() + }) + .collect()) + } + + fn optional_table(&self, name: &str, warnings: &mut Vec) -> Result, String> { + match self.table(name, false) { + Ok(rows) => Ok(rows), + Err(error) => { + warnings.push(format!("skipping optional Lightroom table {name}: {error}")); + Ok(Vec::new()) + } + } + } + + fn xmp_table(&self) -> Result<(Vec, Vec), String> { + let Some(t) = self.tables.iter().find(|t| t.name == "Adobe_AdditionalMetadata") else { return Ok((Vec::new(), Vec::new())) }; + let columns = t.column_names.as_ref().ok_or_else(|| "cannot read Adobe_AdditionalMetadata columns".to_string())?; + let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("Adobe_AdditionalMetadata: {e:?}"))?; + if rows.len() > 1_000_000 { + return Err("Adobe_AdditionalMetadata: more than one million records".into()); + } + let mut warnings = Vec::new(); + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let image = row + .values + .iter() + .zip(columns) + .find_map(|(value, column)| { + (column == "image").then_some(match value { + SqlValue::Integer(n) => *n, + _ => 0, + }) + }) + .unwrap_or(0); + let mut mapped = Row::new(); + for (column, value) in columns.iter().zip(row.values) { + let value = match value { + SqlValue::Null => Some(Value::Null), + SqlValue::Integer(n) => Some(json!(n)), + SqlValue::Real(n) => Some(json!(n)), + SqlValue::Text(s) => Some(json!(s)), + SqlValue::Blob(bytes) if column == "xmp" => match decode_xmp_bytes(&bytes) { + Ok(packet) => Some(Value::String(packet)), + Err(error) => { + warnings.push(format!("image {image}: {error}")); + None + } + }, + SqlValue::Blob(_) => None, + }; + if let Some(value) = value { + mapped.insert(column.clone(), value); + } + } + out.push(mapped); + } + Ok((out, warnings)) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CatalogPhoto { + pub source_id: i64, + pub uuid: String, + pub path: String, + pub image: Row, + pub settings: String, + pub xmp: String, + pub keywords: Vec, + pub history: Vec, + pub snapshots: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CatalogImport { + pub source: String, + pub photos: Vec, + pub collections: Vec, + pub members: Vec, + pub collection_content: Vec, + pub warnings: Vec, +} + +#[derive(Clone, Default, Serialize, Deserialize)] +pub(crate) struct ImportIndex { + pub(crate) photos: BTreeMap, + pub(crate) collections: BTreeMap, +} + +pub(crate) fn load_index(path: Option<&Path>) -> crate::Result { + match path { + Some(path) => { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(ImportIndex::default()), + Err(e) => return Err(error(e.to_string())), + }; + if file.metadata().map_err(|e| error(e.to_string()))?.len() > 16 * 1024 * 1024 { + return Err(error("Lightroom import index exceeds 16 MiB")); + } + let mut bytes = Vec::new(); + use std::io::Read; + file.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| error(e.to_string()))?; + if bytes.len() > 16 * 1024 * 1024 { + return Err(error("Lightroom import index exceeds 16 MiB")); + } + serde_json::from_slice(&bytes).map_err(|e| error(format!("invalid Lightroom import index: {e}"))) + } + None => Ok(ImportIndex::default()), + } +} + +pub(crate) struct AppliedImport { + pub(crate) report: Value, + pub(crate) index: ImportIndex, +} + +fn identity(data: &CatalogImport, photo: &CatalogPhoto) -> String { + if photo.uuid.is_empty() { format!("{}:{}", key(&data.source), photo.source_id) } else { photo.uuid.clone() } +} + +fn validate(data: &CatalogImport) -> Result<(), String> { + let photos: HashMap<_, _> = data.photos.iter().map(|p| (p.source_id, p)).collect(); + if photos.len() != data.photos.len() { + return Err("duplicate source photo ids".into()); + } + for photo in &data.photos { + let master = number(&photo.image, "masterImage"); + if master != 0 + && (master == photo.source_id + || !photos.contains_key(&master) + || photos.get(&master).is_some_and(|p| number(&p.image, "masterImage") != 0)) + { + return Err(format!("invalid virtual copy master for {}", photo.source_id)); + } + } + let collections: HashMap<_, _> = data.collections.iter().map(|r| (number(r, "id_local"), r)).collect(); + for id in collections.keys() { + let mut seen = HashSet::new(); + let mut at = *id; + while let Some(r) = collections.get(&at) { + if seen.len() >= 64 || !seen.insert(at) { + return Err("cyclic or overly deep collection hierarchy".into()); + } + at = number(r, "parent"); + if at == 0 { + break; + } + } + } + Ok(()) +} + +pub(crate) fn validate_for_job(data: &CatalogImport) -> Result<(), String> { + validate(data) +} + +fn bytes(path: &Path) -> Result, String> { + use std::io::Read; + let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + if f.metadata().map_err(|e| e.to_string())?.len() > MAX_FILE { + return Err("catalog or WAL exceeds 1 GiB import limit".into()); + } + let mut out = Vec::new(); + f.take(MAX_FILE + 1).read_to_end(&mut out).map_err(|e| e.to_string())?; + if out.len() as u64 > MAX_FILE { + return Err("catalog grew beyond import limit".into()); + } + Ok(out) +} + +fn xmp(v: Option<&Value>) -> Result { + match v { + Some(Value::String(s)) => Ok(s.clone()), + Some(Value::Array(a)) if !a.is_empty() => { + let b: Vec = a.iter().filter_map(|n| n.as_u64().and_then(|n| u8::try_from(n).ok())).collect(); + let size = b.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?; + if size > 16 << 20 { + return Err("XMP packet exceeds 16 MiB".into()); + } + let compressed = b.get(4..).ok_or("truncated XMP")?; + let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(compressed, 16 << 20).map_err(|_| "invalid compressed XMP")?; + if decoded.len() != size as usize { + return Err("XMP packet length mismatch".into()); + } + String::from_utf8(decoded).map_err(|e| e.to_string()) + } + _ => Ok(String::new()), + } +} + +fn decode_xmp_bytes(bytes: &[u8]) -> Result { + let size = bytes.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?; + if size > 16 << 20 { + return Err("XMP packet exceeds 16 MiB".into()); + } + let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(bytes.get(4..).ok_or("truncated XMP")?, 16 << 20) + .map_err(|_| "invalid compressed XMP")?; + if decoded.len() != size as usize { + return Err("XMP packet length mismatch".into()); + } + String::from_utf8(decoded).map_err(|e| e.to_string()) +} + +fn be_word(bytes: &[u8], at: usize) -> Result { + let word = bytes.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or("truncated WAL")?; + Ok(u32::from_be_bytes(word)) +} + +fn wal_checksum(bytes: &[u8], big: bool, mut sum: (u32, u32)) -> (u32, u32) { + for pair in bytes.as_chunks::<8>().0 { + let first = [pair[0], pair[1], pair[2], pair[3]]; + let second = [pair[4], pair[5], pair[6], pair[7]]; + let (first, second) = + if big { (u32::from_be_bytes(first), u32::from_be_bytes(second)) } else { (u32::from_le_bytes(first), u32::from_le_bytes(second)) }; + sum.0 = sum.0.wrapping_add(first).wrapping_add(sum.1); + sum.1 = sum.1.wrapping_add(second).wrapping_add(sum.0); + } + sum +} + +// SQLite file format §4: validate the live checksum chain in constant space, then overlay +// only its last committed prefix. The forensic timeline materializes EVERY historical +// database snapshot; it must never be used to validate a large, actively used catalog. +fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result { + if wal.is_empty() { + return Ok(0); + } + let header = wal.get(..32).ok_or("truncated WAL header")?; + let magic = be_word(header, 0)?; + if !matches!(magic, 0x377f0682 | 0x377f0683) || be_word(header, 4)? != 3_007_000 { + return Err("invalid WAL header".into()); + } + let size = main.get(16..18).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(u16::from_be_bytes).ok_or("truncated SQLite header")?; + let size = if size == 1 { 65536 } else { u32::from(size) }; + if !(512..=65536).contains(&size) || !size.is_power_of_two() || be_word(header, 8)? != size { + return Err("WAL page size mismatch".into()); + } + let big = magic == 0x377f0683; + let mut sum = wal_checksum(&header[..24], big, (0, 0)); + if sum != (be_word(header, 24)?, be_word(header, 28)?) { + return Err("invalid WAL header checksum".into()); + } + let salt = (be_word(header, 16)?, be_word(header, 20)?); + let mut committed = 0; + let stride = size as usize + 24; + for (i, frame) in wal[32..].chunks_exact(stride).enumerate() { + if be_word(frame, 0)? == 0 || salt != (be_word(frame, 8)?, be_word(frame, 12)?) { + break; + } + sum = wal_checksum(&frame[..8], big, sum); + sum = wal_checksum(&frame[24..], big, sum); + if sum != (be_word(frame, 16)?, be_word(frame, 20)?) { + break; + } + if be_word(frame, 4)? != 0 { + committed = 32 + (i + 1) * stride; + } + } + Ok(committed) +} + +/// Inspect `.lrcat` plus committed WAL pages without modifying either source file. +pub fn read(path: &Path) -> Result { + let cancel = AtomicBool::new(false); + let total = AtomicUsize::new(0); + let done = AtomicUsize::new(0); + read_with_progress(path, &cancel, &total, &done) +} + +pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, done: &AtomicUsize) -> Result { + // 13 catalog tables plus one validation stage; ImportJob adds one unit per original path. + total.store(14, Ordering::Relaxed); + done.store(0, Ordering::Relaxed); + ensure_read_active(cancel)?; + let before = std::fs::metadata(path).map_err(|e| e.to_string())?; + let wal_path = PathBuf::from(format!("{}-wal", path.display())); + let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); + let main = bytes(path)?; + let wal = match std::fs::metadata(&wal_path) { + Ok(_) => bytes(&wal_path)?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(e.to_string()), + }; + let wal_len = committed_wal_len(&main, &wal)?; + ensure_read_active(cancel)?; + let db = Database::open_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?; + let after = std::fs::metadata(path).map_err(|e| e.to_string())?; + let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); + if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after { + return Err("catalog changed during reading; close Lightroom and retry".into()); + } + let reader = Reader { tables: db.live_tables()?, db }; + let mut warnings = Vec::new(); + let table = |name: &str, required: bool| -> Result, String> { + ensure_read_active(cancel)?; + let rows = reader.table(name, required)?; + done.fetch_add(1, Ordering::Relaxed); + Ok(rows) + }; + let optional_table = |name: &str, warnings: &mut Vec| -> Result, String> { + ensure_read_active(cancel)?; + let rows = reader.optional_table(name, warnings)?; + done.fetch_add(1, Ordering::Relaxed); + Ok(rows) + }; + let roots: HashMap<_, _> = table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + let folders: HashMap<_, _> = table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + let files: HashMap<_, _> = table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + let develops: HashMap<_, _> = optional_table("Adobe_imageDevelopSettings", &mut warnings)? + .into_iter() + .map(|r| (number(&r, "image"), text(&r, "text").to_string())) + .collect(); + ensure_read_active(cancel)?; + let (metadata_rows, metadata_warnings) = match reader.xmp_table() { + Ok(result) => result, + Err(error) => { + warnings.push(format!("skipping optional Lightroom table Adobe_AdditionalMetadata: {error}")); + (Vec::new(), Vec::new()) + } + }; + warnings.extend(metadata_warnings); + done.fetch_add(1, Ordering::Relaxed); + let metadata: HashMap<_, _> = metadata_rows.into_iter().map(|r| (number(&r, "image"), r)).collect(); + let keyword_rows: HashMap<_, _> = optional_table("AgLibraryKeyword", &mut warnings)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + let mut keywords: HashMap> = HashMap::new(); + for r in optional_table("AgLibraryKeywordImage", &mut warnings)? { + ensure_read_active(cancel)?; + let mut parts = Vec::new(); + let mut at = number(&r, "tag"); + let mut seen = HashSet::new(); + while at != 0 { + if parts.len() >= 64 || !seen.insert(at) { + return Err("cyclic or overly deep keyword hierarchy".into()); + } + let Some(k) = keyword_rows.get(&at) else { break }; + let name = text(k, "name"); + if !name.is_empty() { + parts.push(name.to_string()); + } + at = number(k, "parent"); + } + parts.reverse(); + if !parts.is_empty() { + keywords.entry(number(&r, "image")).or_default().push(parts.join("|")); + } + } + let group = |rows: Vec| { + let mut out: HashMap> = HashMap::new(); + for r in rows { + out.entry(number(&r, "image")).or_default().push(r); + } + out + }; + let mut history = group(optional_table("Adobe_libraryImageDevelopHistoryStep", &mut warnings)?); + let mut snapshots = group(optional_table("Adobe_libraryImageDevelopSnapshot", &mut warnings)?); + let mut out = CatalogImport { + source: path.to_string_lossy().into(), + photos: Vec::new(), + collections: optional_table("AgLibraryCollection", &mut warnings)?, + members: optional_table("AgLibraryCollectionImage", &mut warnings)?, + collection_content: optional_table("AgLibraryCollectionContent", &mut warnings)?, + warnings, + }; + for image in table("Adobe_images", true)? { + ensure_read_active(cancel)?; + let id = number(&image, "id_local"); + let file = files.get(&number(&image, "rootFile")).ok_or_else(|| format!("image {id}: missing file record"))?; + let folder = folders.get(&number(file, "folder")).ok_or_else(|| format!("image {id}: missing folder record"))?; + let root = roots.get(&number(folder, "rootFolder")).ok_or_else(|| format!("image {id}: missing root record"))?; + let root_path = text(root, "absolutePath"); + let folder_path = text(folder, "pathFromRoot"); + let name = text(file, "idx_filename"); + if root_path.is_empty() || name.is_empty() { + return Err(format!("image {id}: incomplete original path")); + } + let path = Path::new(root_path).join(folder_path).join(name).to_string_lossy().to_string(); + let packet = match xmp(metadata.get(&id).and_then(|r| r.get("xmp"))) { + Ok(s) => s, + Err(e) => { + out.warnings.push(format!("image {id}: {e}")); + String::new() + } + }; + out.photos.push(CatalogPhoto { + source_id: id, + uuid: text(&image, "id_global").into(), + path, + image, + settings: develops.get(&id).cloned().unwrap_or_default(), + xmp: packet, + keywords: keywords.remove(&id).unwrap_or_default(), + history: history.remove(&id).unwrap_or_default(), + snapshots: snapshots.remove(&id).unwrap_or_default(), + }); + } + ensure_read_active(cancel)?; + Ok(out) +} + +fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> { + if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) } +} + +fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec), String> { + let mut root = crate::preset_import::parse_lua(text)?; + let deferred = strip_lua_sentinels(&mut root); + let (props, values) = crate::preset_import::lua_settings_props(&root)?; + let (partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), Some(raw), aspect); + if deferred > 0 { + unknown.push("Deferred Lightroom adjustments (-999999)".into()); + } + Ok((partial, unknown)) +} + +fn strip_lua_sentinels(value: &mut crate::preset_import::Lua) -> usize { + use crate::preset_import::Lua; + match value { + Lua::Num(n) if *n == -999999.0 => { + *value = Lua::Nil; + 1 + } + Lua::Table(array, fields) => { + let mut removed = array.iter_mut().map(strip_lua_sentinels).sum::(); + fields.retain_mut(|(_, value)| { + removed += strip_lua_sentinels(value); + !matches!(value, Lua::Nil) + }); + removed + } + _ => 0, + } +} + +fn strip_xmp_sentinels(value: &mut Value) { + match value { + Value::Object(fields) => fields.retain(|_, value| { + if value.as_f64() == Some(-999999.0) { + return false; + } + strip_xmp_sentinels(value); + true + }), + Value::Array(values) => { + if values.iter().any(|v| v.as_f64() == Some(-999999.0)) { + values.clear(); + } else { + for v in values { + strip_xmp_sentinels(v); + } + } + } + _ => {} + } +} + +/// Import into the current library. Existing edited records are preserved unless explicitly +/// requested; missing originals remain catalogued for later relinking. The source catalog is read-only. +pub fn import(s: &mut crate::Session, path: &Path, update_existing: bool) -> crate::Result { + let mut job = crate::lightroom_job::LightroomJob::new(s, path.to_path_buf(), update_existing)?; + let cancel = AtomicBool::new(false); + let prepared = job.prepare(&cancel)?; + let completion = crate::lightroom_job::commit_prepared(s, prepared)?; + if s.library.as_ref().is_some_and(|library| library.on_disk) { + s.persist()?; + } + completion.finalization.finish()?; + Ok(completion.report) +} + +#[cfg(test)] +fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> crate::Result { + let archive_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop")); + let index_path = archive_dir.as_ref().map(|p| p.join("lightroom-index.json")); + let mut index = load_index(index_path.as_deref())?; + let archive_path = archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?; + let undo0 = s.undo.len(); + let existing: HashMap<_, _> = + s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect(); + let paths: Vec<_> = data + .photos + .iter() + .filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && Path::new(&p.path).is_file()) + .map(|p| p.path.clone()) + .collect(); + let missing_sources: HashSet = + data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !Path::new(&p.path).is_file()).map(|p| key(&p.path)).collect(); + let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?; + let applied = + apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?; + if let Some(path) = index_path { + s.persist()?; + let bytes = serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?; + lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?; + } + Ok(applied.report) +} + +/// Apply prepared normal-file results and Lightroom metadata in one owner-thread operation. +/// This function does no filesystem work; callers finish its returned index separately. +pub(crate) struct ApplyPrepared<'a> { + pub(crate) update_existing: bool, + pub(crate) files: crate::import::ImportReport, + pub(crate) index: &'a mut ImportIndex, + pub(crate) archive_path: Option, + pub(crate) missing_sources: &'a HashSet, + pub(crate) undo0: usize, +} + +pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result { + let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context; + validate(&data).map_err(error)?; + let mut warnings = data.warnings.clone(); + let duplicates: HashMap<_, _> = files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), PhotoId(id)))).collect(); + let mut by_path: HashMap<_, _> = + s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect(); + let mut ids = HashMap::new(); + let mut ops = Vec::new(); + let now = (s.clock)(); + let mut missing = Vec::new(); + let mut unmapped = BTreeMap::new(); + let mut preserved = 0; + // Masters precede their virtual copies. + let mut photos: Vec<_> = data.photos.iter().collect(); + photos.sort_by_key(|p| number(&p.image, "masterImage") != 0); + let mut planned: HashMap = HashMap::new(); + for src in photos { + let master = number(&src.image, "masterImage"); + let indexed = index + .photos + .get(&identity(&data, src)) + .copied() + .map(PhotoId) + .filter(|id| s.catalog.photo(*id).is_some_and(|p| source_path(p).is_some_and(|path| key(path) == key(&src.path)))); + let existing_id = indexed.or_else(|| if master == 0 { by_path.get(&key(&src.path)).copied() } else { None }); + let mut p = if let Some(id) = existing_id.and_then(|id| s.catalog.photo(id)) { + (**id).clone() + } else if master != 0 { + let master_id = ids.get(&master).copied().ok_or_else(|| error(format!("virtual copy {}: missing master", src.source_id)))?; + let mut p = planned + .get(&master_id) + .cloned() + .or_else(|| s.catalog.photo(master_id).map(|p| (**p).clone())) + .ok_or_else(|| error("virtual copy master is unavailable"))?; + p.id = s.catalog.alloc_photo_id(); + p.copy_of = Some(master_id); + p.copy_name = Some(text(&src.image, "copyName").to_string()); + p + } else if let Some(duplicate) = duplicates.get(&key(&src.path)).and_then(|id| s.catalog.photo(*id)) { + // Catalogs can contain distinct originals with identical bytes and independent edits. + let mut p = (**duplicate).clone(); + p.id = s.catalog.alloc_photo_id(); + p.source = Source::File { path: src.path.clone() }; + p.file_name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().into()).unwrap_or_else(|| src.path.clone()); + p + } else { + let id = s.catalog.alloc_photo_id(); + let name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| src.path.clone()); + if missing_sources.contains(&key(&src.path)) { + missing.push(src.path.clone()); + } + let mut p = Photo::new( + id, + Source::File { path: src.path.clone() }, + &name, + text(&src.image, "fileFormat"), + number(&src.image, "fileWidth").max(0) as u32, + number(&src.image, "fileHeight").max(0) as u32, + &now, + ); + if ["RAW", "DNG", "ARW", "CR2", "CR3", "NEF", "NRW", "RAF", "ORF", "RW2", "PEF"].contains(&p.format.to_uppercase().as_str()) { + p.kind = lightcraft_catalog::MediaKind::Raw; + } + p + }; + ids.insert(src.source_id, p.id); + index.photos.insert(identity(&data, src), p.id.0); + let keep_develop = existing_id.is_some() && p.edited.is_some() && !update_existing; + let previous_develop = (p.develop.clone(), p.edited.clone()); + if keep_develop { + preserved += 1; + } + if !src.xmp.is_empty() { + match crate::sidecar::parse_sidecar(&src.xmp, p.kind == lightcraft_catalog::MediaKind::Raw) { + Ok(mut sc) => { + if let Some(crate::sidecar::DevelopPatch::Partial(partial)) = &mut sc.develop { + strip_xmp_sentinels(partial); + } + crate::sidecar::merge_into(&mut p, &sc, &now); + } + Err(e) => warnings.push(format!("image {} XMP: {e}", src.source_id)), + } + } + p.rating = number(&src.image, "rating").clamp(0, 5) as u8; + p.flag = match number(&src.image, "pick") { + 1 => Flag::Pick, + -1 => Flag::Reject, + _ => Flag::None, + }; + p.label = s.catalog.label_from_name(text(&src.image, "colorLabels")); + if !text(&src.image, "captureTime").is_empty() { + p.captured = Some(text(&src.image, "captureTime").into()); + } + if !src.keywords.is_empty() { + p.meta.keywords = src.keywords.clone(); + } + if !src.settings.is_empty() && !keep_develop { + match mapped_settings(&src.settings, p.kind == lightcraft_catalog::MediaKind::Raw, p.width.max(1) as f64 / p.height.max(1) as f64) { + Ok((partial, unknown)) => { + p.develop = Arc::new(lightcraft_develop::apply_partial(&p.develop, &partial, 1.0)); + p.edited = Some(now.clone()); + if !unknown.is_empty() { + unmapped.insert(src.source_id, unknown); + } + } + Err(e) => warnings.push(format!("image {} settings: {e}", src.source_id)), + } + } + if keep_develop { + (p.develop, p.edited) = previous_develop; + } + if s.catalog.photo(p.id).is_some() { + ops.extend([ + Op::SetRating { id: p.id, rating: p.rating }, + Op::SetFlag { id: p.id, flag: p.flag }, + Op::SetLabel { id: p.id, label: p.label }, + Op::SetMeta { id: p.id, meta: Box::new(p.meta.clone()) }, + Op::SetCaptured { id: p.id, captured: p.captured.clone() }, + ]); + ops.push(Op::SetDevelop { id: p.id, settings: p.develop.clone(), label: "Lightroom catalog".into(), edited: p.edited.clone() }); + } else { + ops.push(Op::AddPhoto { photo: Box::new(p.clone()) }); + } + by_path.entry(key(&src.path)).or_insert(p.id); + planned.insert(p.id, p); + } + let mut albums = HashMap::new(); + for collection in &data.collections { + if number(collection, "systemOnly") != 0 { + continue; + } + let collection_key = format!("{}:{}", key(&data.source), number(collection, "id_local")); + let id = index + .collections + .get(&collection_key) + .copied() + .map(lightcraft_catalog::AlbumId) + .filter(|id| s.catalog.album(*id).is_some()) + .unwrap_or_else(|| s.catalog.alloc_album_id()); + index.collections.insert(collection_key, id.0); + albums.insert(number(collection, "id_local"), id); + } + let parents: HashMap<_, _> = data.collections.iter().map(|r| (number(r, "id_local"), number(r, "parent"))).collect(); + let mut grouped_members: HashMap> = HashMap::new(); + for member in &data.members { + grouped_members.entry(number(member, "collection")).or_default().push(member); + } + for members in grouped_members.values_mut() { + members.sort_by(|a, b| text(a, "positionInCollection").cmp(text(b, "positionInCollection"))); + } + let mut ordered: Vec<_> = data.collections.iter().collect(); + ordered.sort_by_key(|r| { + let mut depth = 0; + let mut at = number(r, "parent"); + while at != 0 && depth < 64 { + depth += 1; + at = parents.get(&at).copied().unwrap_or(0); + } + depth + }); + for collection in ordered { + let Some(id) = albums.get(&number(collection, "id_local")).copied() else { continue }; + let mut album = Album::new(id, text(collection, "name")); + album.parent = albums.get(&number(collection, "parent")).copied(); + album.folder = text(collection, "creationId").contains("group"); + if text(collection, "creationId").contains("smart") { + warnings.push(format!("{}: smart rules preserved in archive; imported current membership", album.name)); + } + if !album.folder { + album.photos = grouped_members + .get(&number(collection, "id_local")) + .into_iter() + .flatten() + .filter_map(|r| ids.get(&number(r, "image")).copied()) + .collect(); + } + if s.catalog.album(id).is_some() { + // Reimport preserves the user's album name/hierarchy, merges only new membership. + if let Some(old) = s.catalog.album(id).filter(|a| !a.folder && !a.is_smart()) { + let mut photos = old.photos.clone(); + let mut seen: HashSet<_> = photos.iter().copied().collect(); + for photo in album.photos { + if seen.insert(photo) { + photos.push(photo); + } + } + ops.push(Op::SetAlbumPhotos { id, photos }); + } + } else { + ops.push(Op::AddAlbum { album }); + } + } + if !ops.is_empty() { + s.commit("Import Lightroom Catalog", Op::Batch { ops })?; + } + s.merge_undo(s.undo.len().saturating_sub(undo0), "Import Lightroom Catalog"); + Ok(AppliedImport { + report: json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}), + index: index.clone(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + fn row(v: Value) -> Row { + serde_json::from_value(v).unwrap() + } + fn sample() -> CatalogImport { + let mut photo = CatalogPhoto { + source_id: 1, + uuid: "test-master".into(), + path: "/missing/test.ARW".into(), + image: row(json!({"id_local":1,"rating":5,"pick":1,"fileFormat":"ARW","fileWidth":20,"fileHeight":10})), + settings: "s = { Exposure2012 = 1.25, Contrast2012 = 17, CameraProfile = \"Example\" }".into(), + xmp: String::new(), + keywords: vec!["Travel|Goa".into()], + history: vec![row(json!({"text":"s = { Exposure2012 = 0.5 }"}))], + snapshots: Vec::new(), + }; + let mut copy = photo.clone(); + copy.source_id = 2; + copy.uuid = "test-copy".into(); + copy.image = row(json!({"id_local":2,"masterImage":1,"copyName":"Mono","rating":2,"pick":-1})); + copy.settings = "s = { ConvertToGrayscale = true, Exposure2012 = -0.5 }".into(); + photo.image.insert("captureTime".into(), json!("2024-01-16T12:34:56")); + CatalogImport { + source: "test.lrcat".into(), + photos: vec![copy, photo], + collections: vec![ + row(json!({"id_local":9,"name":"Travel","creationId":"collection.group"})), + row(json!({"id_local":10,"parent":9,"name":"Keepers","creationId":"collection"})), + ], + members: vec![ + row(json!({"collection":10,"image":2,"positionInCollection":"a"})), + row(json!({"collection":10,"image":1,"positionInCollection":"b"})), + ], + collection_content: Vec::new(), + warnings: Vec::new(), + } + } + #[test] + fn imports_copies_collections_ratings_and_mapped_edits_in_one_undo_step() { + let mut s = crate::Session::new(); + let data = sample(); + let r = apply(&mut s, data.clone(), false).unwrap(); + assert_eq!(r["photos"], 2); + assert_eq!(r["missing"].as_array().unwrap().len(), 1); + let master = s.catalog.photo(PhotoId(r["mapping"]["1"].as_u64().unwrap())).unwrap(); + assert_eq!(master.rating, 5); + assert_eq!(master.flag, Flag::Pick); + assert_eq!(master.meta.keywords, vec!["Travel|Goa"]); + assert_eq!(master.develop.light.exposure, 1.25); + let copy = s.catalog.photos().find(|p| p.copy_of.is_some()).unwrap(); + assert_eq!(copy.copy_of, Some(master.id)); + assert_eq!(copy.rating, 2); + assert_eq!(copy.flag, Flag::Reject); + assert_eq!(copy.develop.light.exposure, -0.5); + let album = s.catalog.albums().find(|a| a.name == "Keepers").unwrap(); + assert_eq!(album.photos, vec![copy.id, master.id]); + assert!(album.parent.is_some()); + assert!(r["unmapped"]["1"].as_array().unwrap().iter().any(|v| v.as_str() == Some("CameraProfile"))); + assert_eq!(s.undo.len(), 1); + s.undo_step().unwrap(); + assert!(s.catalog.is_empty()); + assert!(s.catalog.albums().next().is_none()); + assert!(data.photos[1].settings.contains("CameraProfile")); // unmapped source stays intact + } + #[test] + fn preserves_existing_edits_and_rejects_invalid_structure_before_mutation() { + let mut s = crate::Session::new(); + let data = sample(); + apply(&mut s, data.clone(), false).unwrap(); + let id = s.catalog.photos().find(|p| p.copy_of.is_none()).unwrap().id; + s.set_develop(id, lightcraft_develop::DevelopSettings::default(), "Personal edit").unwrap(); + let r = apply(&mut s, data.clone(), false).unwrap(); + assert!(r["preservedExistingEdits"].as_u64().unwrap() > 0); + assert_eq!(s.catalog.photo(id).unwrap().develop.light.exposure, 0.0); + let before = s.catalog.to_snapshot(); + let mut cyclic = data; + cyclic.collections[0].insert("parent".into(), json!(10)); + assert!(apply(&mut s, cyclic, false).is_err()); + assert_eq!(s.catalog.to_snapshot(), before); + assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0).is_err()); + } + #[test] + fn compressed_xmp_is_bounded_and_settings_are_data_only() { + let packet = b"example"; + let mut encoded = (packet.len() as u32).to_be_bytes().to_vec(); + encoded.extend(miniz_oxide::deflate::compress_to_vec_zlib(packet, 6)); + assert_eq!(xmp(Some(&json!(encoded))).unwrap(), String::from_utf8_lossy(packet)); + encoded[0] = 0xff; + assert!(xmp(Some(&json!(encoded))).is_err()); + let (partial, _) = mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5).unwrap(); + assert_eq!(partial["light"]["exposure"], 0.75); + } + + #[test] + fn deferred_auto_tone_sentinels_never_become_real_slider_values() { + let mut data = sample(); + data.photos.truncate(1); + data.photos[0].image.remove("masterImage"); + data.photos[0].settings = + "s = { AutoTone = true, Exposure2012 = -999999, Contrast2012 = -999999, Shadows2012 = 65, Temperature = 6150 }".into(); + data.photos[0].xmp = r#""#.into(); + let mut s = crate::Session::new(); + let report = apply(&mut s, data, false).unwrap(); + let p = s.catalog.photos().next().unwrap(); + assert_eq!(p.develop.light.exposure, 0.0); + assert_eq!(p.develop.light.contrast, 0.0); + assert_eq!(p.develop.light.shadows, 65.0); + assert_eq!(p.develop.light.blacks, -22.0); + assert!(report["unmapped"]["2"].as_array().unwrap().iter().any(|v| v.as_str().is_some_and(|v| v.contains("Deferred")))); + let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0).unwrap(); + assert_eq!(partial["light"]["exposure"], -5.0); + assert_eq!(partial["light"]["contrast"], -100.0); + } + + #[test] + fn wal_checksums_cover_commits_without_materializing_historical_databases() { + let mut main = vec![0; 512]; + main[16..18].copy_from_slice(&512u16.to_be_bytes()); + assert_eq!(wal_checksum(&[1, 0, 0, 0, 2, 0, 0, 0], false, (0, 0)), (1, 3)); + for big in [false, true] { + let mut wal = vec![0; 32]; + wal[0..4].copy_from_slice(&(if big { 0x377f0683u32 } else { 0x377f0682u32 }).to_be_bytes()); + wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes()); + wal[8..12].copy_from_slice(&512u32.to_be_bytes()); + wal[16..20].copy_from_slice(&123u32.to_be_bytes()); + let mut sum = wal_checksum(&wal[..24], big, (0, 0)); + wal[24..28].copy_from_slice(&sum.0.to_be_bytes()); + wal[28..32].copy_from_slice(&sum.1.to_be_bytes()); + for _ in 0..128 { + let mut frame = vec![0; 536]; + frame[0..4].copy_from_slice(&1u32.to_be_bytes()); + frame[4..8].copy_from_slice(&1u32.to_be_bytes()); + frame[8..12].copy_from_slice(&123u32.to_be_bytes()); + sum = wal_checksum(&frame[..8], big, sum); + sum = wal_checksum(&frame[24..], big, sum); + frame[16..20].copy_from_slice(&sum.0.to_be_bytes()); + frame[20..24].copy_from_slice(&sum.1.to_be_bytes()); + wal.extend(frame); + } + assert_eq!(committed_wal_len(&main, &wal).unwrap(), wal.len()); + let end = wal.len(); + wal[end - 1] ^= 1; + assert_eq!(committed_wal_len(&main, &wal).unwrap(), end - 536); + wal[24] ^= 1; + assert!(committed_wal_len(&main, &wal).is_err()); + main[17] = 0; + assert!(committed_wal_len(&main, &wal).is_err()); + main[16..18].copy_from_slice(&512u16.to_be_bytes()); + } + } + + #[test] + fn persistent_reimport_keeps_copy_and_collection_ids_and_capture_time() { + let dir = std::env::temp_dir().join(format!("lightcraft-lrcat-reimport-{}", std::process::id())); + let mut s = crate::Session::new(); + s.open_library(&dir, false).unwrap(); + let mut data = sample(); + data.collections.reverse(); // Parent must be committed before its child regardless of row order. + let first = apply(&mut s, data.clone(), false).unwrap(); + s.close_library().unwrap(); + s.open_library(&dir, false).unwrap(); + let id = PhotoId(first["mapping"]["1"].as_u64().unwrap()); + s.commit("Changed time", Op::SetCaptured { id, captured: None }).unwrap(); + let second = apply(&mut s, data, true).unwrap(); + assert_eq!(first["mapping"], second["mapping"]); + assert_eq!(s.catalog.len(), 2); + assert_eq!(s.catalog.albums().count(), 2); + assert_eq!(s.catalog.photo(id).unwrap().captured.as_deref(), Some("2024-01-16T12:34:56")); + s.close_library().unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + } + + // Construct a small SQLite fixture from public file-format records, never private media. + fn varint(mut n: u64) -> Vec { + let mut out = vec![(n & 127) as u8]; + n >>= 7; + while n > 0 { + out.push((n & 127) as u8 | 128); + n >>= 7; + } + out.reverse(); + out + } + fn record(values: &[SqlValue]) -> Vec { + let mut header = Vec::new(); + let mut body = Vec::new(); + for value in values { + let serial = match value { + SqlValue::Null => 0, + SqlValue::Integer(n) => { + body.extend(n.to_be_bytes()); + 6 + } + SqlValue::Text(s) => { + body.extend(s.as_bytes()); + 13 + 2 * s.len() as u64 + } + _ => unreachable!(), + }; + header.extend(varint(serial)); + } + let mut out = varint(1 + header.len() as u64); + out.extend(header); + out.extend(body); + out + } + fn leaf(page: &mut [u8], offset: usize, rows: &[Vec]) { + page[offset] = 13; + page[offset + 3..offset + 5].copy_from_slice(&(rows.len() as u16).to_be_bytes()); + let mut end = page.len(); + for (i, values) in rows.iter().enumerate() { + let payload = record(values); + let mut cell = varint(payload.len() as u64); + cell.extend(varint(i as u64 + 1)); + cell.extend(payload); + end -= cell.len(); + page[end..end + cell.len()].copy_from_slice(&cell); + page[offset + 8 + i * 2..offset + 10 + i * 2].copy_from_slice(&(end as u16).to_be_bytes()); + } + page[offset + 5..offset + 7].copy_from_slice(&(end as u16).to_be_bytes()); + } + #[test] + fn reads_native_sqlite_tables_without_lightroom_or_external_processes() { + use SqlValue::{Integer as I, Null as N, Text as T}; + let tables = [ + ("AgLibraryRootFolder", "id_local INTEGER PRIMARY KEY, absolutePath", vec![N, T("/catalog/".into())]), + ("AgLibraryFolder", "id_local INTEGER PRIMARY KEY, rootFolder, pathFromRoot", vec![N, I(1), T("Photos/".into())]), + ("AgLibraryFile", "id_local INTEGER PRIMARY KEY, folder, idx_filename", vec![N, I(1), T("test.ARW".into())]), + ( + "Adobe_images", + "id_local INTEGER PRIMARY KEY, id_global, rootFile, fileFormat, fileWidth, fileHeight, rating, pick", + vec![N, T("fixture-image".into()), I(1), T("ARW".into()), I(20), I(10), I(5), I(1)], + ), + ]; + let mut bytes = vec![0u8; 5 * 2048]; + bytes[..16].copy_from_slice(b"SQLite format 3\0"); + bytes[16..18].copy_from_slice(&2048u16.to_be_bytes()); + bytes[18] = 1; + bytes[19] = 1; + bytes[21] = 64; + bytes[22] = 32; + bytes[23] = 32; + bytes[28..32].copy_from_slice(&5u32.to_be_bytes()); + bytes[44..48].copy_from_slice(&4u32.to_be_bytes()); + bytes[56..60].copy_from_slice(&1u32.to_be_bytes()); + let mut schema = Vec::new(); + for (i, (name, columns, row)) in tables.iter().enumerate() { + schema.push(vec![ + T("table".into()), + T((*name).into()), + T((*name).into()), + I(i as i64 + 2), + T(format!("CREATE TABLE {name} ({columns})")), + ]); + leaf(&mut bytes[(i + 1) * 2048..(i + 2) * 2048], 0, std::slice::from_ref(row)); + } + leaf(&mut bytes[..2048], 100, &schema); + let path = std::env::temp_dir().join(format!("lightcraft-native-catalog-{}.lrcat", std::process::id())); + std::fs::write(&path, &bytes).unwrap(); + let catalog = read(&path).unwrap(); + assert_eq!(catalog.photos.len(), 1); + assert_eq!(catalog.photos[0].uuid, "fixture-image"); + assert_eq!(number(&catalog.photos[0].image, "rating"), 5); + assert!(catalog.photos[0].path.ends_with("test.ARW")); + std::fs::write(&path, b"bad catalog").unwrap(); + assert!(read(&path).is_err()); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn oversized_optional_table_warns_and_is_skipped() { + let mut main = vec![0u8; 2 * 512]; + main[..16].copy_from_slice(b"SQLite format 3\0"); + main[16..18].copy_from_slice(&512u16.to_be_bytes()); + main[18] = 1; + main[19] = 1; + main[21] = 64; + main[22] = 32; + main[23] = 32; + main[28..32].copy_from_slice(&2u32.to_be_bytes()); + main[44..48].copy_from_slice(&4u32.to_be_bytes()); + main[56..60].copy_from_slice(&1u32.to_be_bytes()); + main[100] = 0x0d; + let page = &mut main[512..]; + page[0] = 0x0d; + page[3..5].copy_from_slice(&1u16.to_be_bytes()); + let mut cell = varint((64 * 1024 * 1024 + 1) as u64); + cell.extend(varint(1)); + let start = 500 - cell.len(); + page[start..start + cell.len()].copy_from_slice(&cell); + page[8..10].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + page[5..7].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + let db = Database::open_with_wal(main, &[]).unwrap(); + let reader = + Reader { db, tables: vec![LiveTable { name: "optional_history".into(), rootpage: 2, column_names: Some(vec!["payload".into()]) }] }; + let mut warnings = Vec::new(); + let rows = reader.optional_table("optional_history", &mut warnings).unwrap(); + assert!(rows.is_empty()); + assert!(warnings.iter().any(|warning| warning.contains("optional_history"))); + } +} diff --git a/crates/engine/src/lightroom_job.rs b/crates/engine/src/lightroom_job.rs new file mode 100644 index 000000000..fd939341d --- /dev/null +++ b/crates/engine/src/lightroom_job.rs @@ -0,0 +1,450 @@ +//! Detached Lightroom catalog preparation and owner-thread commit. +//! +//! `LightroomJob::new` takes only an in-memory session snapshot. `prepare` owns all catalog +//! reading, probing and archive staging, while `commit_prepared` only applies catalog operations +//! to the session that created the job. Archive index bytes are returned for a worker to finish +//! after the owner has accepted the catalog operation. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +use serde_json::Value; + +use crate::import::{ImportJob, ImportMode, ImportOptions, Prepared}; +use crate::lightroom_catalog::{self, CatalogImport, ImportIndex}; + +const COMMAND: &str = "library.importLightroom"; +const MAX_INDEX_BYTES: usize = 16 * 1024 * 1024; + +fn error(message: impl Into) -> crate::EngineError { + crate::EngineError::BadParams { cmd: COMMAND.into(), msg: message.into() } +} + +fn stage_archive(archive_dir: Option<&Path>, data: &mut CatalogImport) -> crate::Result> { + let archive_path = archive_dir.map(|dir| crate::lightroom_archive::store_best_effort(dir, data).map_err(error)).transpose()?.flatten(); + if archive_dir.is_some() && archive_path.is_none() { + data.warnings.push("Lightroom source archive exceeded 32 MiB; archive skipped".into()); + } + Ok(archive_path) +} + +/// Memory identity of the library which owns a prepared completion. +/// +/// Each successful open/close refreshes session identity, including reopening the same path. +/// The owner check deliberately does not compare catalog revisions: ordinary edits while preparation +/// runs must survive and are merged by the owner commit. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LightroomLibraryToken { + library_dir: Option, + identity: Arc<()>, +} + +impl LightroomLibraryToken { + fn capture(s: &crate::Session) -> Self { + Self { + library_dir: s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()), + identity: Arc::clone(&s.library_identity), + } + } + + /// Whether `s` still refers to the library which started preparation. + pub fn matches(&self, s: &crate::Session) -> bool { + Arc::ptr_eq(&self.identity, &s.library_identity) + && self.library_dir == s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()) + } + + pub fn library_dir(&self) -> Option<&Path> { + self.library_dir.as_deref() + } +} + +/// Filesystem-free result of catalog preparation. +pub struct PreparedLightroom { + pub(crate) data: CatalogImport, + pub(crate) normal: Prepared, + pub(crate) index: ImportIndex, + pub(crate) archive_path: Option, + pub(crate) token: LightroomLibraryToken, + pub(crate) missing_sources: HashSet, + pub(crate) update_existing: bool, + pub(crate) now: String, + cancelled: bool, +} + +impl PreparedLightroom { + pub fn source(&self) -> &str { + &self.data.source + } + + pub fn photo_count(&self) -> usize { + self.data.photos.len() + } + + pub fn token(&self) -> &LightroomLibraryToken { + &self.token + } + + pub fn is_cancelled(&self) -> bool { + self.cancelled + } + + /// Roll back any normal-import filesystem placement held by this prepared value. + pub fn rollback(&self) { + self.normal.rollback(); + } +} + +/// Index write returned by owner commit. `finish` is intended for a worker after the owner has +/// returned, and performs the only archive-index filesystem operation in this pipeline. +pub struct LightroomArchiveFinalization { + path: Option, + bytes: Option>, +} + +impl LightroomArchiveFinalization { + pub fn finish(self) -> crate::Result<()> { + let (Some(path), Some(bytes)) = (self.path, self.bytes) else { return Ok(()) }; + if bytes.len() > MAX_INDEX_BYTES { + return Err(error("Lightroom import index exceeds 16 MiB")); + } + lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string())) + } + + pub fn path(&self) -> Option<&Path> { + self.path.as_deref() + } +} + +/// Owner-thread catalog result. The report is ready for the UI immediately; index finalization +/// can remain on the worker while its task stays alive. +pub struct LightroomCompletion { + pub report: Value, + pub finalization: LightroomArchiveFinalization, +} + +/// Alias used by hosts that call the result a commit. +pub type LightroomCommit = LightroomCompletion; + +/// A Lightroom import detached from the owner thread. +pub struct LightroomJob { + path: PathBuf, + update_existing: bool, + import: ImportJob, + token: LightroomLibraryToken, + archive_dir: Option, +} + +impl LightroomJob { + /// Snapshot session state only. This constructor performs no filesystem work. + pub fn new(s: &mut crate::Session, path: PathBuf, update_existing: bool) -> crate::Result { + let opts = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }; + let import = ImportJob::new(s, opts)?; + let archive_dir = s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.join("Interop")); + Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir }) + } + + pub fn total(&self) -> usize { + self.import.total.load(Ordering::Relaxed) + } + + pub fn done(&self) -> usize { + self.import.done.load(Ordering::Relaxed) + } + + pub fn progress(&self) -> (usize, usize) { + (self.done(), self.total()) + } + + pub fn token(&self) -> &LightroomLibraryToken { + &self.token + } + + pub fn total_atomic(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.import.total) + } + + pub fn done_atomic(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.import.done) + } + + /// Finish an owner commit's deferred archive-index write on a worker. + pub fn finish_archive(finalization: LightroomArchiveFinalization) -> crate::Result<()> { + finalization.finish() + } + + /// Read/probe/stage all source data off the owner thread. + pub fn prepare(&mut self, cancel: &AtomicBool) -> crate::Result { + if cancel.load(Ordering::Relaxed) { + return Err(error("Lightroom import cancelled")); + } + self.import.total.store(1, Ordering::Relaxed); + self.import.done.store(0, Ordering::Relaxed); + let mut data = lightroom_catalog::read_with_progress(&self.path, cancel, &self.import.total, &self.import.done).map_err(error)?; + lightroom_catalog::validate_for_job(&data).map_err(error)?; + if cancel.load(Ordering::Relaxed) { + return Err(error("Lightroom import cancelled")); + } + self.import.done.fetch_add(1, Ordering::Relaxed); + let paths: Vec = data + .photos + .iter() + .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0) + .map(|photo| photo.path.clone()) + .collect(); + let missing_sources = data + .photos + .iter() + .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0) + .filter(|photo| !Path::new(&photo.path).is_file()) + .map(|photo| normalize(&photo.path)) + .collect(); + let normal = self.import.prepare(&paths, cancel); + if cancel.load(Ordering::Relaxed) { + normal.rollback(); + return Err(error("Lightroom import cancelled")); + } + let archive_path = stage_archive(self.archive_dir.as_deref(), &mut data)?; + let index_path = self.archive_dir.as_ref().map(|dir| dir.join("lightroom-index.json")); + let index = lightroom_catalog::load_index(index_path.as_deref())?; + Ok(PreparedLightroom { + data, + normal, + index, + archive_path, + token: self.token.clone(), + missing_sources, + update_existing: self.update_existing, + now: self.import.now().to_string(), + cancelled: cancel.load(Ordering::Relaxed), + }) + } +} + +/// Apply prepared data on the owner thread. No source, archive or index filesystem work occurs. +pub fn commit_prepared(s: &mut crate::Session, mut prepared: PreparedLightroom) -> crate::Result { + if prepared.cancelled { + prepared.normal.rollback(); + return Err(error("Lightroom import cancelled")); + } + if !prepared.token.matches(s) { + prepared.normal.rollback(); + return Err(error("Lightroom import belongs to a different open library")); + } + // A command-scoped Lightroom import must never rewrite source XMP sidecars. Direct depth-0 + // callers do not set this transient flag, so they cannot affect a later command. + if s.depth > 0 { + s.skip_auto_write = true; + } + + let undo0 = s.undo.len(); + prepared.normal.revalidate_add(s); + let options = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }; + let now = prepared.now.clone(); + let files = crate::import::commit_prepared(s, &options, &now, prepared.normal)?; + let applied = lightroom_catalog::apply_prepared( + s, + prepared.data, + lightroom_catalog::ApplyPrepared { + update_existing: prepared.update_existing, + files, + index: &mut prepared.index, + archive_path: prepared.archive_path, + missing_sources: &prepared.missing_sources, + undo0, + }, + )?; + let path = prepared.token.library_dir().map(|dir| dir.join("Interop/lightroom-index.json")); + let bytes = if path.is_some() { Some(serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?) } else { None }; + Ok(LightroomCompletion { report: applied.report, finalization: LightroomArchiveFinalization { path, bytes } }) +} + +fn normalize(path: &str) -> String { + let path = path.replace('\\', "/"); + if cfg!(windows) { path.to_lowercase() } else { path } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_data() -> CatalogImport { + CatalogImport { + source: "fixture.lrcat".into(), + photos: Vec::new(), + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: Vec::new(), + } + } + + fn cancelled_prepared(s: &crate::Session) -> PreparedLightroom { + PreparedLightroom { + data: empty_data(), + normal: Prepared::default(), + index: ImportIndex::default(), + archive_path: None, + token: LightroomLibraryToken::capture(s), + missing_sources: HashSet::new(), + update_existing: false, + now: "2026-01-01T00:00:00".into(), + cancelled: true, + } + } + + fn prepared_for(s: &crate::Session, path: &str, rating: i64) -> PreparedLightroom { + let mut image = std::collections::BTreeMap::new(); + image.insert("id_local".into(), serde_json::json!(1)); + image.insert("rating".into(), serde_json::json!(rating)); + image.insert("pick".into(), serde_json::json!(0)); + image.insert("fileFormat".into(), serde_json::json!("JPG")); + image.insert("fileWidth".into(), serde_json::json!(1)); + image.insert("fileHeight".into(), serde_json::json!(1)); + PreparedLightroom { + data: CatalogImport { + source: "fixture.lrcat".into(), + photos: vec![lightroom_catalog::CatalogPhoto { + source_id: 1, + uuid: "fixture-photo".into(), + path: path.into(), + image, + settings: String::new(), + xmp: String::new(), + keywords: Vec::new(), + history: Vec::new(), + snapshots: Vec::new(), + }], + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: Vec::new(), + }, + normal: Prepared::default(), + index: ImportIndex::default(), + archive_path: None, + token: LightroomLibraryToken::capture(s), + missing_sources: [normalize(path)].into_iter().collect(), + update_existing: false, + now: "2026-01-01T00:00:00".into(), + cancelled: false, + } + } + + #[test] + fn cancellation_before_commit_leaves_owner_catalog_unchanged() { + let mut session = crate::Session::new(); + let before = session.catalog.to_snapshot(); + let prepared = cancelled_prepared(&session); + assert!(commit_prepared(&mut session, prepared).is_err()); + assert_eq!(session.catalog.to_snapshot(), before); + assert!(session.undo.is_empty()); + } + + #[test] + fn library_identity_rejects_late_completion_after_reopen() { + let mut session = crate::Session::new(); + let token = LightroomLibraryToken::capture(&session); + assert!(token.matches(&session)); + session.library_identity = Arc::new(()); + assert!(!token.matches(&session)); + } + + #[test] + fn prepared_reimport_preserves_current_edit_and_uses_one_undo_per_import() { + let mut session = crate::Session::new(); + let path = "/missing/lightroom-job-edit.jpg"; + let prepared = prepared_for(&session, path, 4); + let first = commit_prepared(&mut session, prepared).unwrap(); + assert_eq!(first.report["mapping"]["1"].as_u64(), Some(1)); + assert_eq!(session.undo.len(), 1); + let id = lightcraft_catalog::PhotoId(1); + let mut personal = lightcraft_develop::DevelopSettings::default(); + personal.light.exposure = 1.5; + session.set_develop(id, personal, "Personal").unwrap(); + let before = session.catalog.photo(id).map(|photo| photo.develop.clone()); + let prepared = prepared_for(&session, path, 5); + commit_prepared(&mut session, prepared).unwrap(); + assert_eq!(session.catalog.photo(id).map(|photo| photo.develop.clone()), before); + assert_eq!(session.catalog.photo(id).map(|photo| photo.rating), Some(5)); + assert_eq!(session.undo.len(), 3); + } + + #[test] + fn oversized_history_archive_keeps_prepared_import_successful_with_warning() { + let archive_dir = std::env::temp_dir().join(format!("lightcraft-lr-prepared-scale-{}", std::process::id())); + let mut data = empty_data(); + data.photos = (0..5_000) + .map(|source_id| { + let image = std::collections::BTreeMap::from([ + ("id_local".into(), serde_json::json!(source_id)), + ("fileFormat".into(), serde_json::json!("JPG")), + ("fileWidth".into(), serde_json::json!(1)), + ("fileHeight".into(), serde_json::json!(1)), + ]); + lightroom_catalog::CatalogPhoto { + source_id, + uuid: format!("scale-{source_id}"), + path: format!("/missing/scale-{source_id}.jpg"), + image, + settings: String::new(), + xmp: String::new(), + keywords: Vec::new(), + history: vec![std::collections::BTreeMap::from([( + "text".into(), + serde_json::json!(format!("history-{source_id}-{}", "x".repeat(8 * 1024))), + )])], + snapshots: Vec::new(), + } + }) + .collect(); + let archive_path = stage_archive(Some(&archive_dir), &mut data).unwrap(); + assert!(archive_path.is_none()); + let mut session = crate::Session::new(); + let token = LightroomLibraryToken::capture(&session); + let prepared = PreparedLightroom { + data, + normal: Prepared::default(), + index: ImportIndex::default(), + archive_path, + token, + missing_sources: HashSet::new(), + update_existing: false, + now: "2026-01-01T00:00:00".into(), + cancelled: false, + }; + let completion = commit_prepared(&mut session, prepared).unwrap(); + assert_eq!(completion.report["photos"], 5_000); + assert!( + completion.report["warnings"] + .as_array() + .is_some_and(|warnings| warnings.iter().any(|warning| warning == "Lightroom source archive exceeded 32 MiB; archive skipped")) + ); + assert_eq!(session.catalog.len(), 5_000); + std::fs::remove_dir_all(archive_dir).unwrap(); + } + + #[test] + fn execute_fn_import_does_not_rewrite_source_sidecar() { + let dir = std::env::temp_dir().join(format!("lightcraft-lr-job-sidecar-{}", std::process::id())); + let path = dir.join("source.jpg"); + let sidecar = dir.join("source.xmp"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(&path, b"fixture").unwrap(); + std::fs::write(&sidecar, b"original sidecar").unwrap(); + let original = std::fs::read(&sidecar).unwrap(); + let mut session = crate::Session::new(); + session.xmp.auto_write = true; + let result = session.execute_fn("library.importLightroom", |s| { + let prepared = prepared_for(s, &path.to_string_lossy(), 5); + let completion = commit_prepared(s, prepared)?; + Ok(completion.report) + }); + assert!(result.is_ok()); + assert_eq!(std::fs::read(&sidecar).unwrap(), original); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/engine/src/lightroom_sqlite.rs b/crates/engine/src/lightroom_sqlite.rs new file mode 100644 index 000000000..900fe4a26 --- /dev/null +++ b/crates/engine/src/lightroom_sqlite.rs @@ -0,0 +1,1148 @@ +//! Small, read-only SQLite reader used by Lightroom catalog import. +//! +//! This is a clean-room implementation based on SQLite's public file-format +//! description: (sections 1.6, 1.7, +//! 2.1, 2.2, 2.3, 2.4 and 2.5). It is deliberately a page reader, rather +//! than a SQL engine. It never writes its input and has no SQLite runtime +//! dependency. + +use std::collections::{HashMap, HashSet}; + +const MAX_PAGE_ID: u32 = 16_777_216; +const MAX_COLUMNS: usize = 16_384; +const MAX_ROWS: usize = 1_000_000; +const MAX_CELLS: usize = 4_000_000; +const MAX_PAYLOAD: usize = 64 * 1024 * 1024; +const MAX_TABLE_PAYLOAD: usize = 256 * 1024 * 1024; +const MAX_OVERFLOW_PAGES: usize = 1_048_576; +const MAX_BTREE_DEPTH: usize = 64; + +/// Values supported by SQLite table records. +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + Null, + Integer(i64), + Real(f64), + Text(String), + Blob(Vec), +} + +/// A row returned by [`Database::read_table`]. +#[derive(Clone, Debug, PartialEq)] +pub struct Record { + pub values: Vec, +} + +/// A table discovered in `sqlite_schema`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiveTable { + pub name: String, + pub rootpage: u32, + pub column_names: Option>, +} + +/// Read-only SQLite database image, with a committed WAL overlay if supplied. +pub struct Database { + main: Vec, + overlay: HashMap>, + page_size: usize, + reserved: usize, + encoding: TextEncoding, + page_count: u32, + tables: Vec, + rowid_aliases: HashMap, +} + +#[derive(Clone, Copy)] +enum TextEncoding { + Utf8, + Utf16Le, + Utf16Be, +} + +struct WalkState { + seen: HashSet, + out: Vec, + payload_total: usize, +} + +struct WalOverlay { + page_size: usize, + frame_count: usize, + pages: HashMap>, + page_count: Option, +} + +struct SchemaInfo { + tables: Vec, + aliases: HashMap, +} + +impl Database { + /// Open a main database image and an already checksum-validated WAL prefix. + /// + /// The WAL argument must include its 32-byte header and only complete + /// frames through a commit. Frames are overlaid by page number, with the + /// final commit's database-size field defining the visible page count. + pub fn open_with_wal(main: Vec, committed_wal: &[u8]) -> Result { + let main_header = main.get(..100).ok_or("truncated SQLite header")?; + if main_header.get(..16) != Some(b"SQLite format 3\0") { + return Err("invalid SQLite header signature".into()); + } + let page_size = sqlite_page_size(be_u16(main_header, 16)?)?; + validate_header_fields(main_header, page_size)?; + if main.len() < page_size || !main.len().is_multiple_of(page_size) { + return Err("SQLite image is not a whole number of pages".into()); + } + let main_pages = u32::try_from(main.len() / page_size).map_err(|_| "SQLite image is too large")?; + if main_pages == 0 || main_pages > MAX_PAGE_ID { + return Err("invalid SQLite image page count".into()); + } + + let mut overlay = HashMap::new(); + let mut wal_page_count = None; + if !committed_wal.is_empty() { + let parsed = parse_wal(committed_wal, page_size)?; + if parsed.page_size != page_size { + return Err("WAL page size mismatch".into()); + } + if parsed.frame_count > MAX_CELLS { + return Err("WAL contains too many frames".into()); + } + overlay = parsed.pages; + wal_page_count = parsed.page_count; + } + let effective_header = overlay.get(&1).map(Vec::as_slice).unwrap_or(main_header); + if effective_header.get(..16) != Some(b"SQLite format 3\0") { + return Err("invalid SQLite header page in WAL".into()); + } + let effective_size = sqlite_page_size(be_u16(effective_header, 16)?)?; + if effective_size != page_size { + return Err("WAL page 1 changes SQLite page size".into()); + } + let reserved = validate_header_fields(effective_header, page_size)?; + let header_pages = be_u32(effective_header, 28)?; + let header_valid = be_u32(effective_header, 24)? != 0 && be_u32(effective_header, 24)? == be_u32(effective_header, 92)?; + if header_valid && header_pages > main_pages && wal_page_count.is_none() { + return Err("SQLite header page count exceeds image".into()); + } + let mut page_count = if header_valid && header_pages != 0 { header_pages } else { main_pages }; + if let Some(count) = wal_page_count { + page_count = count; + } + if page_count == 0 || page_count > MAX_PAGE_ID { + return Err("invalid visible SQLite page count".into()); + } + let encoding = match be_u32(effective_header, 56)? { + 0 | 1 => TextEncoding::Utf8, + 2 => TextEncoding::Utf16Le, + 3 => TextEncoding::Utf16Be, + _ => return Err("unsupported SQLite text encoding".into()), + }; + + let mut db = Self { main, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() }; + let schema = db.parse_schema()?; + db.tables = schema.tables; + db.rowid_aliases = schema.aliases; + Ok(db) + } + + /// Return tables from `sqlite_schema` in schema order. + pub fn live_tables(&self) -> Result, String> { + Ok(self.tables.clone()) + } + + /// Read a table B-tree, decoding up to `ncols` values per record. + pub fn read_table(&self, rootpage: u32, ncols: usize) -> Result, String> { + if !(1..=MAX_PAGE_ID).contains(&rootpage) || rootpage > self.page_count { + return Err("table root page is outside database".into()); + } + if ncols > MAX_COLUMNS { + return Err("table has too many columns".into()); + } + let alias = self.rowid_aliases.get(&rootpage).copied(); + let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 }; + self.visit_table(rootpage, ncols, alias, 0, &mut state)?; + Ok(state.out) + } + + fn page(&self, page_id: u32) -> Result<&[u8], String> { + if page_id == 0 || page_id > self.page_count { + return Err(format!("page {page_id} is outside database")); + } + if let Some(page) = self.overlay.get(&page_id) { + return Ok(page); + } + let id = usize::try_from(page_id).map_err(|_| "page number does not fit host")?; + let start = id.checked_sub(1).and_then(|n| n.checked_mul(self.page_size)).ok_or("page offset overflow")?; + let end = start.checked_add(self.page_size).ok_or("page offset overflow")?; + self.main.get(start..end).ok_or_else(|| format!("page {page_id} is truncated")) + } + + fn visit_table(&self, page_id: u32, ncols: usize, alias: Option, depth: usize, state: &mut WalkState) -> Result<(), String> { + if depth > MAX_BTREE_DEPTH { + return Err("SQLite B-tree is too deep".into()); + } + if !state.seen.insert(page_id) { + return Err("SQLite B-tree contains a cycle or shared page".into()); + } + let page = self.page(page_id)?; + let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?; + let base = if page_id == 1 { 100 } else { 0 }; + let content = page.get(..usable).ok_or("truncated usable SQLite page")?; + let kind = *content.get(base).ok_or("truncated B-tree header")?; + let cells = usize::from(be_u16_at(content, base + 3)?); + if cells > MAX_CELLS || state.out.len().saturating_add(cells) > MAX_ROWS.saturating_add(MAX_CELLS) { + return Err("SQLite B-tree contains too many cells".into()); + } + match kind { + 0x0d => { + let ptr_start = base.checked_add(8).ok_or("B-tree header overflow")?; + let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?; + let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?; + if ptr_end > usable { + return Err("truncated B-tree pointer array".into()); + } + for i in 0..cells { + let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?; + let cell = usize::from(be_u16_at(content, p)?); + if cell < ptr_end || cell >= usable { + return Err("invalid B-tree cell pointer".into()); + } + let mut at = cell; + let payload_len = usize_from_u64(read_varint(content, &mut at)?, "payload length")?; + if payload_len > MAX_PAYLOAD { + return Err("SQLite record payload is too large".into()); + } + let rowid = i64::from_be_bytes(read_varint(content, &mut at)?.to_be_bytes()); + state.payload_total = state.payload_total.checked_add(payload_len).ok_or("SQLite table payload total overflow")?; + if state.payload_total > MAX_TABLE_PAYLOAD { + return Err("SQLite table payload exceeds bounded import size".into()); + } + let payload = self.payload(page, at, payload_len)?; + let mut values = decode_record(&payload, ncols, self.encoding)?; + if let Some(index) = alias { + if index >= values.len() { + return Err("INTEGER PRIMARY KEY column is missing from record".into()); + } + if let Some(value @ Value::Null) = values.get_mut(index) { + *value = Value::Integer(rowid); + } + } + if state.out.len() >= MAX_ROWS { + return Err("SQLite table contains more than one million records".into()); + } + state.out.push(Record { values }); + } + Ok(()) + } + 0x05 => { + let ptr_start = base.checked_add(12).ok_or("B-tree header overflow")?; + let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?; + let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?; + if ptr_end > usable { + return Err("truncated interior B-tree pointer array".into()); + } + for i in 0..cells { + let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?; + let cell = usize::from(be_u16_at(content, p)?); + if cell < ptr_end || cell >= usable { + return Err("invalid interior B-tree cell pointer".into()); + } + let child = be_u32_at(content, cell)?; + if child == 0 { + return Err("interior B-tree cell has null child".into()); + } + let mut key_at = cell.checked_add(4).ok_or("interior cell overflow")?; + let _ = read_varint(content, &mut key_at)?; + self.visit_table(child, ncols, alias, depth + 1, state)?; + } + let right = be_u32_at(content, base + 8)?; + if right == 0 { + return Err("interior B-tree has null right child".into()); + } + self.visit_table(right, ncols, alias, depth + 1, state) + } + 0x02 | 0x0a => Err("SQLite index B-trees are unsupported".into()), + other => Err(format!("unsupported SQLite B-tree page type 0x{other:02x}")), + } + } + + fn payload(&self, page: &[u8], cell_at: usize, payload_len: usize) -> Result, String> { + let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?; + if usable < 8 { + return Err("SQLite usable page size is too small".into()); + } + let max_local = usable.checked_sub(35).ok_or("SQLite usable page size is too small")?; + let min_local = usable + .checked_sub(12) + .ok_or("SQLite usable page size is too small")? + .saturating_mul(32) + .checked_div(255) + .ok_or("SQLite local payload calculation failed")? + .saturating_sub(23); + let local = if payload_len <= max_local { + payload_len + } else { + let span = usable.checked_sub(4).ok_or("SQLite usable page size is too small")?; + let candidate = min_local.saturating_add(payload_len.checked_sub(min_local).ok_or("SQLite payload length is too small")? % span); + if candidate > max_local { min_local } else { candidate } + }; + let local_end = cell_at.checked_add(local).ok_or("SQLite cell payload overflow")?; + if local_end > usable { + return Err("truncated SQLite cell payload".into()); + } + let overflow_bytes = self + .page_count + .saturating_sub(1) + .try_into() + .unwrap_or(usize::MAX) + .min(MAX_OVERFLOW_PAGES) + .checked_mul(usable.checked_sub(4).ok_or("SQLite usable page size is too small")?) + .ok_or("SQLite overflow capacity overflow")?; + let capacity = local.checked_add(overflow_bytes).ok_or("SQLite payload capacity overflow")?.min(payload_len); + let mut out = Vec::with_capacity(capacity); + out.extend_from_slice(page.get(cell_at..local_end).ok_or("truncated SQLite cell payload")?); + let mut remaining = payload_len.checked_sub(local).ok_or("SQLite payload length is too small")?; + if remaining == 0 { + return Ok(out); + } + let overflow_at = local_end; + match overflow_at.checked_add(4) { + Some(end) if end <= usable => {} + _ => return Err("SQLite overflow pointer is in reserved page bytes".into()), + } + let first = be_u32_at(page, overflow_at)?; + let mut next = first; + let mut seen = HashSet::new(); + while remaining > 0 { + if next == 0 || !seen.insert(next) || seen.len() > MAX_OVERFLOW_PAGES { + return Err("invalid or cyclic SQLite overflow chain".into()); + } + let overflow = self.page(next)?; + let take = remaining.min(usable.checked_sub(4).ok_or("SQLite usable page size is too small")?); + let end = 4usize.checked_add(take).ok_or("overflow payload offset")?; + out.extend_from_slice(overflow.get(4..end).ok_or("truncated SQLite overflow page")?); + remaining -= take; + if remaining > 0 { + next = be_u32_at(overflow, 0)?; + } + } + Ok(out) + } + + fn parse_schema(&self) -> Result { + let mut tables = Vec::new(); + let mut aliases = HashMap::new(); + let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 }; + self.visit_table(1, 5, None, 0, &mut state)?; + for record in state.out { + if record.values.len() < 5 { + return Err("sqlite_schema row has too few columns".into()); + } + let kind = match record.values.first() { + Some(Value::Text(s)) => s.as_str(), + _ => return Err("sqlite_schema row has invalid type".into()), + }; + let name = match record.values.get(1) { + Some(Value::Text(s)) => s.clone(), + _ => return Err("sqlite_schema row has invalid name".into()), + }; + if kind.eq_ignore_ascii_case("virtual") { + continue; + } + if !kind.eq_ignore_ascii_case("table") { + continue; + } + let rootpage = match record.values.get(3) { + Some(Value::Integer(n)) if *n > 0 => u32::try_from(*n).map_err(|_| "sqlite_schema root page is invalid")?, + _ => continue, + }; + let sql = match record.values.get(4) { + Some(Value::Text(s)) => s, + _ => continue, + }; + let Ok((column_names, alias)) = parse_create_table(sql) else { continue }; + if column_names.len() > MAX_COLUMNS { + continue; + } + if tables.iter().any(|table: &LiveTable| table.rootpage == rootpage) { + continue; + } + if let Some(index) = alias { + aliases.insert(rootpage, index); + } + tables.push(LiveTable { name, rootpage, column_names: Some(column_names) }); + } + Ok(SchemaInfo { tables, aliases }) + } +} + +fn validate_header_fields(header: &[u8], page_size: usize) -> Result { + let write_version = *header.get(18).ok_or("truncated SQLite header")?; + let read_version = *header.get(19).ok_or("truncated SQLite header")?; + if !matches!(write_version, 1 | 2) || !matches!(read_version, 1 | 2) { + return Err("unsupported SQLite file format version".into()); + } + if *header.get(21).ok_or("truncated SQLite header")? != 64 + || *header.get(22).ok_or("truncated SQLite header")? != 32 + || *header.get(23).ok_or("truncated SQLite header")? != 32 + { + return Err("invalid SQLite payload fractions".into()); + } + let reserved = usize::from(*header.get(20).ok_or("truncated SQLite header")?); + let usable = page_size.checked_sub(reserved).ok_or("invalid SQLite reserved-byte count")?; + if usable < 480 { + return Err("SQLite usable page size is below 480 bytes".into()); + } + let schema_format = be_u32(header, 44)?; + if !(1..=4).contains(&schema_format) { + return Err("unsupported SQLite schema format".into()); + } + Ok(reserved) +} + +fn parse_wal(wal: &[u8], page_size: usize) -> Result { + let header = wal.get(..32).ok_or("truncated WAL header")?; + let magic = be_u32(header, 0)?; + if magic != 0x377f0682 && magic != 0x377f0683 { + return Err("invalid WAL magic".into()); + } + if be_u32(header, 4)? != 3_007_000 || be_u32(header, 8)? != u32::try_from(page_size).map_err(|_| "page size overflow")? { + return Err("invalid WAL page size or format version".into()); + } + let stride = page_size.checked_add(24).ok_or("WAL frame size overflow")?; + let body = wal.get(32..).ok_or("truncated WAL body")?; + if body.len() % stride != 0 { + return Err("WAL ends with a partial frame".into()); + } + let salt = (be_u32(header, 16)?, be_u32(header, 20)?); + let frame_count = body.len() / stride; + if frame_count > MAX_CELLS { + return Err("WAL has too many frames".into()); + } + let mut overlay = HashMap::new(); + let mut page_count = None; + for frame_no in 0..frame_count { + let start = frame_no.checked_mul(stride).ok_or("WAL frame offset overflow")?; + let end = start.checked_add(stride).ok_or("WAL frame offset overflow")?; + let frame = body.get(start..end).ok_or("truncated WAL frame")?; + let page_id = be_u32(frame, 0)?; + if page_id == 0 || page_id > MAX_PAGE_ID || salt != (be_u32(frame, 8)?, be_u32(frame, 12)?) { + return Err("invalid WAL frame page or salt".into()); + } + let commit = be_u32(frame, 4)?; + if commit != 0 { + if commit > MAX_PAGE_ID { + return Err("WAL commit database size is too large".into()); + } + page_count = Some(commit); + } + overlay.insert(page_id, frame.get(24..).ok_or("truncated WAL page")?.to_vec()); + } + Ok(WalOverlay { page_size, frame_count, pages: overlay, page_count }) +} + +fn decode_record(payload: &[u8], ncols: usize, encoding: TextEncoding) -> Result, String> { + let mut at = 0; + let header_len = usize_from_u64(read_varint(payload, &mut at)?, "record header length")?; + if header_len < at || header_len > payload.len() { + return Err("invalid SQLite record header length".into()); + } + let mut serials = Vec::new(); + while at < header_len { + if serials.len() >= ncols { + return Err("SQLite record has more columns than table schema".into()); + } + serials.push(read_varint(payload, &mut at)?); + } + if at != header_len { + return Err("SQLite record header is malformed".into()); + } + let mut body = header_len; + let mut values = Vec::with_capacity(ncols); + for serial in serials { + let (value, size) = serial_value(payload, &mut body, serial, encoding)?; + values.push(value); + if values.len() > MAX_COLUMNS { + return Err("SQLite record has too many columns".into()); + } + let _ = size; + } + if body > payload.len() { + return Err("SQLite record body is truncated".into()); + } + values.resize(ncols, Value::Null); + Ok(values) +} + +fn serial_value(payload: &[u8], body: &mut usize, serial: u64, encoding: TextEncoding) -> Result<(Value, usize), String> { + let (kind, len) = match serial { + 0 => (0, 0), + 1 => (1, 1), + 2 => (2, 2), + 3 => (3, 3), + 4 => (4, 4), + 5 => (5, 6), + 6 => (6, 8), + 7 => (7, 8), + 8 => (8, 0), + 9 => (9, 0), + 10 | 11 => return Err("reserved SQLite serial type".into()), + n if n >= 12 => { + let len = usize_from_u64((n - if n % 2 == 0 { 12 } else { 13 }) / 2, "SQLite field length")?; + (if n % 2 == 0 { 12 } else { 13 }, len) + } + _ => return Err("invalid SQLite serial type".into()), + }; + let end = body.checked_add(len).ok_or("SQLite record body offset overflow")?; + let bytes = payload.get(*body..end).ok_or("SQLite record body is truncated")?; + let value = match kind { + 0 => Value::Null, + 1 => Value::Integer(i64::from(i8::from_be_bytes([*bytes.first().ok_or("truncated integer")?]))), + 2 => Value::Integer(i64::from(i16::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))), + 3 => { + let first = *bytes.first().ok_or("truncated integer")?; + let second = *bytes.get(1).ok_or("truncated integer")?; + let third = *bytes.get(2).ok_or("truncated integer")?; + let n = (u32::from(first) << 16) | (u32::from(second) << 8) | u32::from(third); + let signed = if n & 0x80_0000 != 0 { n | 0xff00_0000 } else { n }; + Value::Integer(i64::from(i32::from_be_bytes(signed.to_be_bytes()))) + } + 4 => Value::Integer(i64::from(i32::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))), + 5 => { + let n = bytes.iter().fold(0u64, |acc, b| (acc << 8) | u64::from(*b)); + let signed = if n & (1 << 47) != 0 { n | 0xffff_0000_0000_0000 } else { n }; + Value::Integer(i64::from_be_bytes(signed.to_be_bytes())) + } + 6 => Value::Integer(i64::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?)), + 7 => { + let n = f64::from_be_bytes(bytes.try_into().map_err(|_| "truncated real")?); + if !n.is_finite() { + return Err("non-finite SQLite real value".into()); + } + Value::Real(n) + } + 8 => Value::Integer(0), + 9 => Value::Integer(1), + 12 => Value::Blob(bytes.to_vec()), + 13 => Value::Text(decode_text(bytes, encoding)?), + _ => return Err("invalid SQLite field type".into()), + }; + *body = end; + Ok((value, len)) +} + +fn decode_text(bytes: &[u8], encoding: TextEncoding) -> Result { + match encoding { + TextEncoding::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|_| "invalid UTF-8 SQLite text".into()), + TextEncoding::Utf16Le | TextEncoding::Utf16Be => { + if !bytes.len().is_multiple_of(2) { + return Err("odd-length UTF-16 SQLite text".into()); + } + let units: Result, String> = bytes + .as_chunks::<2>() + .0 + .iter() + .map(|pair| { + let first = *pair.first().ok_or("truncated UTF-16 SQLite text")?; + let second = *pair.get(1).ok_or("truncated UTF-16 SQLite text")?; + Ok(match encoding { + TextEncoding::Utf16Le => u16::from_le_bytes([first, second]), + TextEncoding::Utf16Be => u16::from_be_bytes([first, second]), + TextEncoding::Utf8 => return Err("invalid UTF-16 encoding state".into()), + }) + }) + .collect(); + String::from_utf16(&units?).map_err(|_| "invalid UTF-16 SQLite text".into()) + } + } +} + +fn parse_create_table(sql: &str) -> Result<(Vec, Option), String> { + let tokens = tokenize_sql(sql)?; + if !tokens.first().is_some_and(|t| t.word_is("CREATE")) { + return Err("malformed CREATE TABLE SQL".into()); + } + let mut kind_at = 1usize; + if tokens.get(kind_at).is_some_and(|t| t.word_is("TEMPORARY") || t.word_is("TEMP")) { + kind_at += 1; + } + if tokens.get(kind_at).is_some_and(|t| t.word_is("VIRTUAL")) { + return Err("virtual tables are unsupported".into()); + } + if !tokens.get(kind_at).is_some_and(|t| t.word_is("TABLE")) { + return Err("malformed CREATE TABLE SQL".into()); + } + let open = tokens.iter().position(|t| t.is_symbol(b'(')).ok_or("CREATE TABLE has no column list")?; + let mut depth = 0usize; + let mut close = None; + for (i, token) in tokens.iter().enumerate().skip(open) { + if token.is_symbol(b'(') { + depth += 1; + } else if token.is_symbol(b')') { + depth = depth.checked_sub(1).ok_or("unbalanced CREATE TABLE parentheses")?; + if depth == 0 { + close = Some(i); + break; + } + } + } + let close = close.ok_or("unterminated CREATE TABLE column list")?; + if tokens + .get(close + 1..) + .ok_or("invalid CREATE TABLE suffix")? + .windows(2) + .any(|w| w.first().is_some_and(|t| t.word_is("WITHOUT")) && w.get(1).is_some_and(|t| t.word_is("ROWID"))) + { + return Err("WITHOUT ROWID tables are unsupported".into()); + } + let body = tokens.get(open + 1..close).ok_or("invalid CREATE TABLE column list")?; + let mut columns = Vec::new(); + let mut alias = None; + let mut integer_columns = HashSet::new(); + let mut table_primary = None; + let mut start = 0usize; + let mut nested = 0usize; + for i in 0..=body.len() { + let comma = i == body.len() || (nested == 0 && body.get(i).is_some_and(|t| t.is_symbol(b','))); + if comma { + let part = body.get(start..i).ok_or("invalid CREATE TABLE column range")?; + if part.is_empty() { + return Err("empty CREATE TABLE column definition".into()); + } + if is_table_constraint(part) { + if table_primary.is_none() { + table_primary = table_primary_column(part); + } + } else { + let name = part.first().and_then(Token::identifier).ok_or("column name is not an identifier")?; + if columns.iter().any(|old| old == name) { + return Err("duplicate SQLite column name".into()); + } + let index = columns.len(); + if exact_integer_decl(part) { + integer_columns.insert(name.to_ascii_lowercase()); + } + if integer_primary_key_alias(part) { + alias = Some(index); + } + columns.push(name.to_string()); + } + start = i + 1; + } else if body.get(i).is_some_and(|token| token.is_symbol(b'(')) { + nested += 1; + } else if body.get(i).is_some_and(|token| token.is_symbol(b')')) { + nested = nested.checked_sub(1).ok_or("unbalanced CREATE TABLE column definition")?; + } + } + if columns.is_empty() { + return Err("CREATE TABLE has no columns".into()); + } + if alias.is_none() + && let Some(primary) = table_primary + { + let key = primary.to_ascii_lowercase(); + if integer_columns.contains(&key) { + alias = columns.iter().position(|column| column.eq_ignore_ascii_case(&primary)); + } + } + Ok((columns, alias)) +} + +fn is_table_constraint(part: &[Token]) -> bool { + part.first().is_some_and(|token| ["CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN"].iter().any(|word| token.word_is(word))) +} + +fn exact_integer_decl(part: &[Token]) -> bool { + if !part.get(1).is_some_and(|token| token.word_is("INTEGER")) { + return false; + } + match part.get(2) { + None => true, + Some(token) if token.is_symbol(b'(') => false, + Some(token) => ["PRIMARY", "NOT", "NULL", "UNIQUE", "CHECK", "DEFAULT", "COLLATE", "REFERENCES", "CONSTRAINT", "GENERATED"] + .iter() + .any(|word| token.word_is(word)), + } +} + +fn table_primary_column(part: &[Token]) -> Option { + let primary = part.iter().position(|token| token.word_is("PRIMARY"))?; + if !part.get(primary + 1).is_some_and(|token| token.word_is("KEY")) { + return None; + } + let open = primary + 2; + if !part.get(open).is_some_and(|token| token.is_symbol(b'(')) { + return None; + } + let close = (open + 1..part.len()).find(|index| part.get(*index).is_some_and(|token| token.is_symbol(b')')))?; + let terms = part.get(open + 1..close)?; + if terms.len() == 1 || (terms.len() == 2 && terms.get(1).is_some_and(|token| token.word_is("ASC") || token.word_is("DESC"))) { + return terms.first().and_then(Token::identifier).map(str::to_owned); + } + None +} + +fn integer_primary_key_alias(part: &[Token]) -> bool { + if !exact_integer_decl(part) { + return false; + } + let mut primary = None; + let mut depth = 0usize; + for i in 2..part.len() { + if part.get(i).is_some_and(|token| token.is_symbol(b'(')) { + depth += 1; + } else if part.get(i).is_some_and(|token| token.is_symbol(b')')) { + depth = depth.saturating_sub(1); + } else if depth == 0 && part.get(i).is_some_and(|token| token.word_is("PRIMARY")) && part.get(i + 1).is_some_and(|t| t.word_is("KEY")) { + primary = Some(i); + break; + } + } + let Some(i) = primary else { return false }; + !part.get(i + 2).is_some_and(|token| token.word_is("DESC")) && !part.get(i + 2).is_some_and(|token| token.is_symbol(b'(')) +} + +#[derive(Clone, Debug)] +enum Token { + Word(String), + Quoted(String), + StringLiteral, + Symbol(u8), +} + +impl Token { + fn is_symbol(&self, symbol: u8) -> bool { + matches!(self, Token::Symbol(value) if *value == symbol) + } + + fn word_is(&self, wanted: &str) -> bool { + matches!(self, Token::Word(word) if word.eq_ignore_ascii_case(wanted)) + } + + fn identifier(&self) -> Option<&str> { + match self { + Token::Word(word) | Token::Quoted(word) => Some(word), + _ => None, + } + } +} + +fn tokenize_sql(sql: &str) -> Result, String> { + let bytes = sql.as_bytes(); + let mut out = Vec::new(); + let mut at = 0usize; + while at < bytes.len() { + if bytes.get(at).is_some_and(u8::is_ascii_whitespace) { + at += 1; + } else if bytes.get(at) == Some(&b'-') && bytes.get(at + 1) == Some(&b'-') { + at += 2; + while at < bytes.len() && bytes.get(at) != Some(&b'\n') { + at += 1; + } + } else if bytes.get(at) == Some(&b'/') && bytes.get(at + 1) == Some(&b'*') { + at += 2; + let mut closed = false; + while at + 1 < bytes.len() { + if bytes.get(at) == Some(&b'*') && bytes.get(at + 1) == Some(&b'/') { + at += 2; + closed = true; + break; + } + at += 1; + } + if !closed { + return Err("unterminated SQL comment".into()); + } + } else if matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';')) { + out.push(Token::Symbol(*bytes.get(at).ok_or("truncated SQL punctuation")?)); + at += 1; + } else if matches!(bytes.get(at), Some(b'"' | b'`' | b'[')) { + let quote = *bytes.get(at).ok_or("truncated SQL quote")?; + let end_quote = if quote == b'[' { b']' } else { quote }; + at += 1; + let mut value = Vec::new(); + let mut closed = false; + while at < bytes.len() { + if bytes.get(at) == Some(&end_quote) { + if bytes.get(at + 1) == Some(&end_quote) { + value.push(end_quote); + at += 2; + } else { + at += 1; + closed = true; + break; + } + } else { + value.push(*bytes.get(at).ok_or("truncated SQL identifier")?); + at += 1; + } + } + if !closed { + return Err("unterminated quoted SQL identifier".into()); + } + out.push(Token::Quoted(String::from_utf8(value).map_err(|_| "invalid SQL identifier encoding")?)); + } else if bytes.get(at) == Some(&b'\'') { + at += 1; + let mut closed = false; + while at < bytes.len() { + if bytes.get(at) == Some(&b'\'') { + if bytes.get(at + 1) == Some(&b'\'') { + at += 2; + } else { + at += 1; + closed = true; + break; + } + } else { + at += 1; + } + } + if !closed { + return Err("unterminated SQL string".into()); + } + out.push(Token::StringLiteral); + } else { + let start = at; + while at < bytes.len() + && !bytes.get(at).is_some_and(u8::is_ascii_whitespace) + && !matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';' | b'\'' | b'"' | b'`' | b'[')) + { + at += 1; + } + if start == at { + return Err("unsupported SQL punctuation".into()); + } + out.push(Token::Word(sql.get(start..at).ok_or("invalid SQL token boundary")?.to_string())); + } + } + Ok(out) +} + +fn sqlite_page_size(value: u16) -> Result { + let size = if value == 1 { 65_536 } else { usize::from(value) }; + if !(512..=65_536).contains(&size) || !size.is_power_of_two() { + return Err("invalid SQLite page size".into()); + } + Ok(size) +} + +fn read_varint(bytes: &[u8], at: &mut usize) -> Result { + let mut value = 0u64; + for i in 0..9 { + let byte = *bytes.get(*at).ok_or("truncated SQLite varint")?; + *at = (*at).checked_add(1).ok_or("SQLite varint offset overflow")?; + if i == 8 { + value = (value << 8) | u64::from(byte); + return Ok(value); + } + value = (value << 7) | u64::from(byte & 0x7f); + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err("invalid SQLite varint".into()) +} + +fn usize_from_u64(value: u64, label: &str) -> Result { + usize::try_from(value).map_err(|_| format!("{label} exceeds host size")) +} + +fn be_u16(bytes: &[u8], at: usize) -> Result { + let end = at.checked_add(2).ok_or("SQLite integer offset overflow")?; + bytes.get(at..end).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(u16::from_be_bytes).ok_or("truncated SQLite integer".into()) +} + +fn be_u16_at(bytes: &[u8], at: usize) -> Result { + be_u16(bytes, at) +} + +fn be_u32(bytes: &[u8], at: usize) -> Result { + let end = at.checked_add(4).ok_or("SQLite integer offset overflow")?; + bytes.get(at..end).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated SQLite integer".into()) +} + +fn be_u32_at(bytes: &[u8], at: usize) -> Result { + be_u32(bytes, at) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn varint(mut value: u64) -> Vec { + let mut out = vec![u8::try_from(value & 0x7f).unwrap_or(0)]; + value >>= 7; + while value != 0 { + out.push(u8::try_from(value & 0x7f).unwrap_or(0) | 0x80); + value >>= 7; + } + out.reverse(); + out + } + + fn leaf(page: &mut [u8], base: usize, rowid: u64, payload: &[u8]) { + page[base] = 0x0d; + page[base + 3..base + 5].copy_from_slice(&1u16.to_be_bytes()); + let mut cell = varint(payload.len() as u64); + cell.extend(varint(rowid)); + cell.extend(payload); + let start = page.len() - cell.len(); + page[start..].copy_from_slice(&cell); + page[base + 8..base + 10].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + page[base + 5..base + 7].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + } + + fn leaf_rows(page: &mut [u8], base: usize, rows: &[(u64, &[u8])]) { + page[base] = 0x0d; + page[base + 3..base + 5].copy_from_slice(&u16::try_from(rows.len()).unwrap_or(0).to_be_bytes()); + let mut end = page.len(); + for (index, (rowid, payload)) in rows.iter().enumerate() { + let mut cell = varint(payload.len() as u64); + cell.extend(varint(*rowid)); + cell.extend(*payload); + end -= cell.len(); + page[end..end + cell.len()].copy_from_slice(&cell); + let pointer = base + 8 + index * 2; + page[pointer..pointer + 2].copy_from_slice(&u16::try_from(end).unwrap_or(0).to_be_bytes()); + } + page[base + 5..base + 7].copy_from_slice(&u16::try_from(end).unwrap_or(0).to_be_bytes()); + } + + fn schema_record(name: &str, root: u8, sql: &str) -> Vec { + let mut header = vec![23]; + let text_serial = |text: &str| 13 + 2 * text.len() as u64; + header.extend(varint(text_serial(name))); + header.extend(varint(text_serial(name))); + header.push(1); + header.extend(varint(text_serial(sql))); + let mut body = b"table".to_vec(); + body.extend(name.as_bytes()); + body.extend(name.as_bytes()); + body.push(root); + body.extend(sql.as_bytes()); + let mut record = varint((1 + header.len()) as u64); + record.extend(header); + record.extend(body); + record + } + + #[test] + fn schema_names_and_integer_primary_key_alias_are_decoded() { + let sql = r#"CREATE TABLE "t" ("id_local" INTEGER PRIMARY KEY, "odd,name" TEXT, PRIMARY KEY ("id_local"))"#; + let (names, alias) = parse_create_table(sql).unwrap(); + assert_eq!(names, vec!["id_local", "odd,name"]); + assert_eq!(alias, Some(0)); + let (_, no_alias) = parse_create_table("CREATE TABLE t (id INTEGER PRIMARY KEY DESC, value TEXT)").unwrap(); + assert_eq!(no_alias, None); + let (table_names, table_alias) = parse_create_table("CREATE TABLE t (id INTEGER, value TEXT, PRIMARY KEY(id))").unwrap(); + assert_eq!(table_names, vec!["id", "value"]); + assert_eq!(table_alias, Some(0)); + } + + #[test] + fn unsupported_unrelated_schema_entries_are_skipped() { + let mut main = vec![0u8; 2 * 512]; + main[..16].copy_from_slice(b"SQLite format 3\0"); + main[16..18].copy_from_slice(&512u16.to_be_bytes()); + main[18] = 1; + main[19] = 1; + main[21] = 64; + main[22] = 32; + main[23] = 32; + main[28..32].copy_from_slice(&2u32.to_be_bytes()); + main[44..48].copy_from_slice(&4u32.to_be_bytes()); + main[56..60].copy_from_slice(&1u32.to_be_bytes()); + let virtual_sql = "CREATE VIRTUAL TABLE unrelated USING fts5(text)"; + let supported_sql = "CREATE TABLE supported (id INTEGER PRIMARY KEY)"; + let virtual_record = schema_record("unrelated", 2, virtual_sql); + let supported_record = schema_record("supported", 2, supported_sql); + leaf_rows(&mut main[..512], 100, &[(1, &virtual_record), (2, &supported_record)]); + let db = Database::open_with_wal(main, &[]).unwrap(); + let tables = db.live_tables().unwrap(); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].name, "supported"); + } + + #[test] + fn synthetic_btree_substitutes_rowid_and_reads_schema() { + let mut main = vec![0u8; 2 * 512]; + main[..16].copy_from_slice(b"SQLite format 3\0"); + main[16..18].copy_from_slice(&512u16.to_be_bytes()); + main[18] = 1; + main[19] = 1; + main[21] = 64; + main[22] = 32; + main[23] = 32; + main[28..32].copy_from_slice(&2u32.to_be_bytes()); + main[44..48].copy_from_slice(&4u32.to_be_bytes()); + main[56..60].copy_from_slice(&1u32.to_be_bytes()); + let sql = "CREATE TABLE \"t\" (\"id_local\" INTEGER PRIMARY KEY, \"name\" TEXT)"; + // rootpage is an integer field, so replace fourth text value with a compact record. + let mut header = vec![23, 15, 15, 1]; + header.extend(varint((13 + 2 * sql.len()) as u64)); + let mut body = b"tablett".to_vec(); + body.push(2); + body.extend(sql.as_bytes()); + let mut record = varint((1 + header.len()) as u64); + record.append(&mut header); + record.extend(body); + let schema = record; + leaf(&mut main[512..], 0, 1, &[3, 0, 15, b'x']); + leaf(&mut main[..512], 100, 1, &schema); + let db = Database::open_with_wal(main, &[]).unwrap(); + let tables = db.live_tables().unwrap(); + assert!(tables.iter().any(|table| table.name == "t")); + let rows = db.read_table(2, 2).unwrap(); + assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(1))); + } + + #[test] + fn interior_table_pointer_array_uses_cell_offsets() { + let mut main = vec![0u8; 4 * 512]; + let mut interior = vec![0u8; 512]; + interior[0] = 0x05; + interior[3..5].copy_from_slice(&1u16.to_be_bytes()); + interior[8..12].copy_from_slice(&4u32.to_be_bytes()); + interior[12..14].copy_from_slice(&500u16.to_be_bytes()); + interior[500..504].copy_from_slice(&3u32.to_be_bytes()); + interior[504] = 1; + main[512..1024].copy_from_slice(&interior); + leaf(&mut main[1024..1536], 0, 1, &[2, 0]); + leaf(&mut main[1536..], 0, 2, &[2, 0]); + let db = Database { + main, + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 4, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + assert_eq!(db.read_table(2, 1).unwrap().len(), 2); + } + + #[test] + fn negative_rowid_varint_is_signed() { + let mut main = vec![0u8; 512]; + let page = &mut main[..]; + page[100] = 0x0d; + page[103..105].copy_from_slice(&1u16.to_be_bytes()); + let start = 500usize; + page[start] = 2; + for offset in 1..10 { + if let Some(byte) = page.get_mut(start + offset) { + *byte = 0xff; + } + } + page[start + 10..start + 12].copy_from_slice(&[2, 0]); + page[108..110].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + page[105..107].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + let mut aliases = HashMap::new(); + aliases.insert(1, 0); + let db = Database { + main, + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 1, + tables: Vec::new(), + rowid_aliases: aliases, + }; + let rows = db.read_table(1, 1).unwrap(); + assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(-1))); + } + + #[test] + fn reserved_page_trailer_cannot_supply_cells() { + let mut main = vec![0u8; 512]; + main[100] = 0x0d; + main[103..105].copy_from_slice(&1u16.to_be_bytes()); + main[108..110].copy_from_slice(&490u16.to_be_bytes()); + let db = Database { + main, + overlay: HashMap::new(), + page_size: 512, + reserved: 32, + encoding: TextEncoding::Utf8, + page_count: 1, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + assert!(db.read_table(1, 1).is_err()); + } + + #[test] + fn utf16_and_overflow_payloads_are_supported() { + let mut payload = vec![2, 17, b'A', 0]; + assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Le).unwrap(), vec![Value::Text("A".into())]); + payload.clear(); + payload.extend([2, 17, 0, b'A']); + assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Be).unwrap(), vec![Value::Text("A".into())]); + + let mut main = vec![0u8; 2 * 512]; + let db = Database { + main: std::mem::take(&mut main), + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 2, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + let mut cell_page = vec![b'x'; 512]; + // For a 512-byte table leaf & 600-byte payload, SQLite stores 92 + // payload bytes locally, followed by the overflow page pointer. + cell_page[92..96].copy_from_slice(&2u32.to_be_bytes()); + let inline_page = vec![b'i'; 512]; + assert_eq!(db.payload(&inline_page, 0, 110).unwrap(), vec![b'i'; 110]); + let mut overflow_page = vec![b'y'; 512]; + overflow_page[..4].copy_from_slice(&0u32.to_be_bytes()); + let mut db = db; + db.main[512..].copy_from_slice(&overflow_page); + let mut with_overflow = db.overlay; + with_overflow.insert(2, overflow_page); + db.overlay = with_overflow; + let bytes = db.payload(&cell_page, 0, 600).unwrap(); + assert_eq!(bytes.len(), 600); + assert_eq!(&bytes[..92], &[b'x'; 92]); + assert_eq!(&bytes[92..], &[b'y'; 508]); + } + + #[test] + fn malformed_varint_and_wal_cycle_fail_cleanly() { + let mut at = 0; + assert!(read_varint(&[0x80; 8], &mut at).is_err()); + let mut wal = vec![0u8; 32 + 24 + 512]; + wal[..4].copy_from_slice(&0x377f0682u32.to_be_bytes()); + wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes()); + wal[8..12].copy_from_slice(&512u32.to_be_bytes()); + wal[16..20].copy_from_slice(&7u32.to_be_bytes()); + wal[20..24].copy_from_slice(&8u32.to_be_bytes()); + wal[32..36].copy_from_slice(&1u32.to_be_bytes()); + wal[36..40].copy_from_slice(&1u32.to_be_bytes()); + wal[40..44].copy_from_slice(&7u32.to_be_bytes()); + wal[44..48].copy_from_slice(&8u32.to_be_bytes()); + let parsed = parse_wal(&wal, 512).unwrap(); + assert_eq!(parsed.frame_count, 1); + assert_eq!(parsed.page_count, Some(1)); + assert_eq!(parsed.pages.get(&1).map(Vec::len), Some(512)); + let mut page = vec![0u8; 512]; + page[16..18].copy_from_slice(&512u16.to_be_bytes()); + let db = Database { + main: vec![0; 512], + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 1, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + assert!(db.payload(&page, 0, 600).is_err()); + } +} diff --git a/crates/engine/src/preset_import.rs b/crates/engine/src/preset_import.rs index 3f9498cc9..040d46c3b 100644 --- a/crates/engine/src/preset_import.rs +++ b/crates/engine/src/preset_import.rs @@ -100,6 +100,7 @@ impl Lua { struct LuaParser<'a> { s: &'a [u8], i: usize, + depth: usize, } impl LuaParser<'_> { @@ -210,6 +211,15 @@ impl LuaParser<'_> { std::str::from_utf8(&self.s[start..self.i]).ok().and_then(|t| t.parse().ok()).map_or_else(|| self.err("bad number"), Ok) } fn value(&mut self) -> Result { + if self.depth >= 64 { + return self.err("Lua settings exceed nesting limit"); + } + self.depth += 1; + let result = self.value_inner(); + self.depth -= 1; + result + } + fn value_inner(&mut self) -> Result { self.skip_ws(); let Some(&c) = self.s.get(self.i) else { return self.err("unexpected end") }; match c { @@ -315,7 +325,10 @@ impl LuaParser<'_> { /// Parse a Lua table literal, optionally preceded by `name =` or `return`. pub fn parse_lua(text: &str) -> Result { - let mut p = LuaParser { s: text.as_bytes(), i: 0 }; + if text.len() > 16 << 20 { + return Err("Lua settings exceed 16 MiB".into()); + } + let mut p = LuaParser { s: text.as_bytes(), i: 0, depth: 0 }; p.skip_ws(); let save = p.i; match p.ident().as_deref() { @@ -347,6 +360,12 @@ pub fn lrtemplate_props(text: &str) -> Result<(Option, Props, crate::crs let root = parse_lua(text)?; let title = root.get("title").or_else(|| root.get("internalName")).and_then(Lua::str).map(delocalize).filter(|t| !t.trim().is_empty()); let settings = root.get("value").and_then(|v| v.get("settings")).ok_or("no develop settings in this template")?; + let (props, values) = lua_settings_props(settings)?; + Ok((title, props, values)) +} + +/// Data-only Lua settings shared by preset and native catalog import. Never executes Lua. +pub(crate) fn lua_settings_props(settings: &Lua) -> Result<(Props, crate::crs_masks::Values), String> { let Lua::Table(_, fields) = settings else { return Err("no develop settings in this template".into()) }; let mut props = Props::new(); let mut values = crate::crs_masks::Values::new(); @@ -374,7 +393,7 @@ pub fn lrtemplate_props(text: &str) -> Result<(Option, Props, crate::crs }; props.insert(key, vals); } - Ok((title, props, values)) + Ok((props, values)) } // ------------------------------------------------------------------------------- XMP in files diff --git a/crates/ui-egui/locales/ja-formats.json b/crates/ui-egui/locales/ja-formats.json index c0c5505bd..33adde1e2 100644 --- a/crates/ui-egui/locales/ja-formats.json +++ b/crates/ui-egui/locales/ja-formats.json @@ -50,6 +50,7 @@ "Proof Preview · {}": "校正プレビュー・{}", "Quick Develop · {n} photos": "クイック現像・{n}枚", "Rated {}": "評価：{}", + "Reading Lightroom catalog… {done} of {total}": "Reading Lightroom catalog… {done} of {total}", "Reading photos… {done} of {total}": "写真を読み込み中：{total}枚中{done}枚", "Redo {}": "やり直す：{}", "Remove “{}” from Favorites": "「{}」をお気に入りから削除", @@ -116,6 +117,12 @@ "Build Smart Previews ({scope})": "スマートプレビューを作成（{scope}）", "Build Standard-Sized Previews ({scope})": "標準プレビューを作成（{scope}）", "Discard Smart Previews ({scope})": "スマートプレビューを破棄（{scope}）", + "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing": "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing", + "Lightroom import complete: {imported} photos added": "Lightroom import complete: {imported} photos added", + "Lightroom import complete: {imported} photos added, {failed} failed": "Lightroom import complete: {imported} photos added, {failed} failed", + "Lightroom import complete: {imported} photos added; index warning: {warning}": "Lightroom import complete: {imported} photos added; index warning: {warning}", + "Lightroom import failed: {e}{warning}": "Lightroom import failed: {e}{warning}", + "Lightroom inspection failed: {e}{warning}": "Lightroom inspection failed: {e}{warning}", "Label Set: {name}": "ラベルセット：{name}", "Quick Develop: {name}": "クイック現像: {name}", "Reset {name}": "{name}をリセット", diff --git a/crates/ui-egui/locales/pt-br-formats.json b/crates/ui-egui/locales/pt-br-formats.json index 73b609c13..2a59467f8 100644 --- a/crates/ui-egui/locales/pt-br-formats.json +++ b/crates/ui-egui/locales/pt-br-formats.json @@ -45,6 +45,12 @@ "Imported {n} preset{}": "Importadas {n} predefinição{:.0}", "Imported {n} preset{}, {f} file{} not readable": "Importadas {n} predefinição{:.0}, {f} arquivo{:.0} não legível", "Imported {} photo{}": "Importadas {} foto{:.0}", + "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing": "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing", + "Lightroom import complete: {imported} photos added": "Lightroom import complete: {imported} photos added", + "Lightroom import complete: {imported} photos added, {failed} failed": "Lightroom import complete: {imported} photos added, {failed} failed", + "Lightroom import complete: {imported} photos added; index warning: {warning}": "Lightroom import complete: {imported} photos added; index warning: {warning}", + "Lightroom import failed: {e}{warning}": "Lightroom import failed: {e}{warning}", + "Lightroom inspection failed: {e}{warning}": "Lightroom inspection failed: {e}{warning}", "Label Set: {name}": "Conjunto de rótulos: {name}", "LightCraft could not open its window: {e}": "O LightCraft não conseguiu abrir sua janela: {e}", "LightCraft could not open its window: {e}\n\nThe log file has the details: {path}": "O LightCraft não conseguiu abrir sua janela: {e}\n\nO arquivo de log tem os detalhes: {path}", @@ -62,6 +68,7 @@ "Quick Develop · {n} photos": "Revelação rápida · {n} fotos", "Quick Develop: {name}": "Revelação rápida: {name}", "Rated {}": "Avaliada {}", + "Reading Lightroom catalog… {done} of {total}": "Reading Lightroom catalog… {done} of {total}", "Reading photos… {done} of {total}": "Lendo fotos… {done} de {total}", "Reading photos… {} of {}": "Lendo fotos… {} de {}", "Redo {}": "Refazer {}", diff --git a/crates/ui-egui/locales/zh-hans-formats.json b/crates/ui-egui/locales/zh-hans-formats.json index cbd77d128..906d90378 100644 --- a/crates/ui-egui/locales/zh-hans-formats.json +++ b/crates/ui-egui/locales/zh-hans-formats.json @@ -50,6 +50,7 @@ "Proof Preview · {}": "校样预览 · {0}", "Quick Develop · {n} photos": "快速编辑 · {n} 张照片", "Rated {}": "已评级 {0}", + "Reading Lightroom catalog… {done} of {total}": "Reading Lightroom catalog… {done} of {total}", "Reading photos… {done} of {total}": "正在读取照片… {total} 张中的 {done} 张", "Redo {}": "重做：{0}", "Remove “{}” from Favorites": "从收藏夹中移除“{0}”", @@ -116,6 +117,12 @@ "Build Smart Previews ({scope})": "生成智能预览（{scope}）", "Build Standard-Sized Previews ({scope})": "生成标准尺寸预览（{scope}）", "Discard Smart Previews ({scope})": "丢弃智能预览（{scope}）", + "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing": "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing", + "Lightroom import complete: {imported} photos added": "Lightroom import complete: {imported} photos added", + "Lightroom import complete: {imported} photos added, {failed} failed": "Lightroom import complete: {imported} photos added, {failed} failed", + "Lightroom import complete: {imported} photos added; index warning: {warning}": "Lightroom import complete: {imported} photos added; index warning: {warning}", + "Lightroom import failed: {e}{warning}": "Lightroom import failed: {e}{warning}", + "Lightroom inspection failed: {e}{warning}": "Lightroom inspection failed: {e}{warning}", "Label Set: {name}": "标签集：{name}", "Quick Develop: {name}": "快速编辑：{name}", "Reset {name}": "重置{name}", diff --git a/crates/ui-egui/locales/zh-hant-formats.json b/crates/ui-egui/locales/zh-hant-formats.json index a2f95cb1c..a51e1749d 100644 --- a/crates/ui-egui/locales/zh-hant-formats.json +++ b/crates/ui-egui/locales/zh-hant-formats.json @@ -42,6 +42,12 @@ "Imported {n} preset{}": "已匯入 {n} 個預設集{:.0}", "Imported {n} preset{}, {f} file{} not readable": "已匯入 {n} 個預設集{:.0}，{f} 個檔案無法讀取{:.0}", "Imported {} photo{}": "已匯入 {} 張照片{:.0}", + "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing": "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing", + "Lightroom import complete: {imported} photos added": "Lightroom import complete: {imported} photos added", + "Lightroom import complete: {imported} photos added, {failed} failed": "Lightroom import complete: {imported} photos added, {failed} failed", + "Lightroom import complete: {imported} photos added; index warning: {warning}": "Lightroom import complete: {imported} photos added; index warning: {warning}", + "Lightroom import failed: {e}{warning}": "Lightroom import failed: {e}{warning}", + "Lightroom inspection failed: {e}{warning}": "Lightroom inspection failed: {e}{warning}", "Label Set: {name}": "標籤集：{name}", "LightCraft could not open its window: {e}": "LightCraft 無法開啟視窗：{e}", "LightCraft could not open its window: {e}\n\nThe log file has the details: {path}": "LightCraft 無法開啟視窗：{e}\n\n詳細資訊請見記錄檔：{path}", @@ -59,6 +65,7 @@ "Quick Develop · {n} photos": "快速顯像 · {n} 張照片", "Quick Develop: {name}": "快速顯像：{name}", "Rated {}": "評分：{}", + "Reading Lightroom catalog… {done} of {total}": "Reading Lightroom catalog… {done} of {total}", "Reading photos… {done} of {total}": "正在讀取照片… {done}／{total}", "Reading photos… {} of {}": "正在讀取照片… {}／{}", "Redo {}": "重做 {}", diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 708eb4ac4..fb3712119 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -13,6 +13,7 @@ pub mod headless; pub mod i18n; pub mod icons; pub mod import; +pub mod lightroom_import; pub mod links; pub mod menubar; pub mod menus; @@ -106,6 +107,8 @@ pub struct Services { pub reveal: Option, /// Choose a folder (Settings → General → Open Library…; desktop only). pub pick_folder: Option, + /// Open a Lightroom Classic `.lrcat` catalog for read-only import. + pub pick_lightroom_catalog: Option, /// Open a web link in the browser (Help menu, About, Discord button). pub open_url: Option, /// Open a file in an external editor (Edit in External Editor; desktop only). @@ -196,6 +199,10 @@ pub struct LightcraftApp { pub import: Option, /// A folder scan in progress (feeds the import review). pub scan: Option, + /// A Lightroom catalog inspect/import in progress. + pub lightroom: Option, + /// Last terminal Lightroom result, exposed by the command's status/wait response. + pub lightroom_last: Option, /// A background export in progress. pub export: Option, /// Background file-system work of other commands (Find Missing Photos, auto import…). @@ -256,6 +263,8 @@ impl LightcraftApp { merge: merge::MergeState::default(), import: None, scan: None, + lightroom: None, + lightroom_last: None, export: None, tasks: Default::default(), last_export_result: None, @@ -655,6 +664,7 @@ impl LightcraftApp { merge::poll(self, ctx); import::poll_scan(self, ctx); import::tick(self, ctx); + lightroom_import::tick(self, ctx); tasks::poll(self, ctx); self.preview_build_status(ctx); self.save_status(ctx); @@ -676,7 +686,7 @@ impl LightcraftApp { if let Some(folder) = self.session.import_defaults.auto_folder.clone() { const LABEL: &str = "Auto Import"; let now = ctx.input(|i| i.time); - if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && !self.tasks.is_running(LABEL) { + if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && self.lightroom.is_none() && !self.tasks.is_running(LABEL) { self.ui.auto_import_at = now; let work = move || lightcraft_engine::cmd::library::list_auto_import_folder(&folder); let done = |app: &mut LightcraftApp, _ctx: &egui::Context, listing: Result, String>| { @@ -717,10 +727,12 @@ impl LightcraftApp { let _ = self.run("file.importPresets", serde_json::json!({"paths": presets})); } // read and added on a worker thread (dropped folders can be large, or on a slow drive) - if !photos.is_empty() - && let Err(e) = import::start_paths(self, photos) - { - self.toast(ctx, e); + if !photos.is_empty() { + if self.lightroom.is_some() { + self.toast(ctx, "Finish Lightroom catalog import before adding photos"); + } else if let Err(e) = import::start_paths(self, photos) { + self.toast(ctx, e); + } } } } @@ -857,6 +869,7 @@ impl LightcraftApp { panels::library_problem::show(self, &ctx); import::progress(self, &ctx); import::scan_progress(self, &ctx); + lightroom_import::progress(self, &ctx); export_task::poll(self, &ctx); panels::grid::drag_feedback(self, &ctx); panels::toast(self, &ctx); diff --git a/crates/ui-egui/src/lightroom_import.rs b/crates/ui-egui/src/lightroom_import.rs new file mode 100644 index 000000000..8cee63e39 --- /dev/null +++ b/crates/ui-egui/src/lightroom_import.rs @@ -0,0 +1,471 @@ +//! Background Lightroom catalog inspection and import. +//! +//! Reading a catalog, probing its originals and staging its archive can take minutes on a +//! removable or network drive. This task keeps that work off the owner thread, commits prepared +//! data only on the owner thread, then finishes the archive index on a worker. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; + +use egui::Align2; +use serde_json::{Value, json}; + +use crate::LightcraftApp; +use crate::theme::Tokens; +use crate::widgets::register; + +#[derive(Clone, Copy)] +enum Kind { + Inspect, + Import, +} + +enum Message { + Inspected(Result), + Prepared(Result, String>), + Finalized(Result<(), String>), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Phase { + Reading, + Committing, + Finalizing, +} + +/// A Lightroom task in flight. The receiver is polled from the egui owner thread. +pub struct LightroomTask { + kind: Kind, + path: PathBuf, + cancel: Arc, + total: Arc, + done: Arc, + rx: mpsc::Receiver, + phase: Phase, + cancelled: bool, +} + +impl LightroomTask { + #[cfg(not(target_arch = "wasm32"))] + fn status(&self) -> Value { + let kind = match self.kind { + Kind::Inspect => "inspect", + Kind::Import => "import", + }; + let phase = match self.phase { + Phase::Reading => "reading", + Phase::Committing => "committing", + Phase::Finalizing => "finalizing", + }; + json!({ + "running": true, + "kind": kind, + "path": self.path, + "phase": phase, + "done": self.done.load(Ordering::Relaxed), + "total": self.total.load(Ordering::Relaxed), + "cancelled": self.cancelled || self.cancel.load(Ordering::Relaxed), + }) + } + + fn cancel(&mut self) { + self.cancelled = true; + self.cancel.store(true, Ordering::Relaxed); + } +} + +impl Drop for LightroomTask { + fn drop(&mut self) { + self.cancel.store(true, Ordering::Relaxed); + } +} + +#[cfg(target_arch = "wasm32")] +fn unsupported_wasm() -> Result { + Err("Lightroom catalog import is unavailable in browser builds; open the catalog in the native LightCraft app".into()) +} + +/// Whether a Lightroom task currently owns the catalog transition. +pub fn is_running(app: &LightcraftApp) -> bool { + app.lightroom.is_some() +} + +#[cfg(not(target_arch = "wasm32"))] +fn busy(app: &LightcraftApp) -> Result<(), String> { + if app.lightroom.is_some() { + return Err("a Lightroom catalog task is already running".into()); + } + if app.import.is_some() || app.scan.is_some() { + return Err("an import is running".into()); + } + Ok(()) +} + +#[cfg(not(target_arch = "wasm32"))] +fn spawn_inspect( + path: PathBuf, + cancel: Arc, + total: Arc, + done: Arc, + tx: mpsc::Sender, + ctx: egui::Context, +) -> Result<(), String> { + std::thread::Builder::new() + .name("lc-lightroom-inspect".into()) + .spawn(move || { + let result = lightcraft_engine::guard::catch("Lightroom inspect", || { + lightcraft_engine::lightroom_catalog::read_with_progress(&path, &cancel, &total, &done) + }) + .and_then(|r| r.map(inspect_report)); + let _ = tx.send(Message::Inspected(result)); + ctx.request_repaint(); + }) + .map(|_| ()) + .map_err(|e| format!("could not start Lightroom inspect: {e}")) +} + +#[cfg(not(target_arch = "wasm32"))] +fn spawn_prepare( + mut job: lightcraft_engine::lightroom_job::LightroomJob, + cancel: Arc, + tx: mpsc::Sender, + ctx: egui::Context, +) -> Result<(), String> { + std::thread::Builder::new() + .name("lc-lightroom-import".into()) + .spawn(move || { + let result = + lightcraft_engine::guard::catch("Lightroom import", || job.prepare(&cancel)).and_then(|r| r.map(Box::new).map_err(|e| e.to_string())); + if let Err(mpsc::SendError(Message::Prepared(Ok(prepared)))) = tx.send(Message::Prepared(result)) { + // The owner dropped the task (for example while closing): staged copies/moves + // must be put back even though no UI receiver remains to accept the result. + prepared.rollback(); + } + ctx.request_repaint(); + }) + .map(|_| ()) + .map_err(|e| format!("could not start Lightroom import: {e}")) +} + +#[cfg(not(target_arch = "wasm32"))] +fn spawn_finalize( + finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization, + tx: mpsc::Sender, + ctx: egui::Context, +) -> Result<(), String> { + std::thread::Builder::new() + .name("lc-lightroom-finalize".into()) + .spawn(move || { + let result = lightcraft_engine::guard::catch("Lightroom archive finalization", || finalization.finish()) + .and_then(|r| r.map_err(|e| e.to_string())); + let _ = tx.send(Message::Finalized(result)); + ctx.request_repaint(); + }) + .map(|_| ()) + .map_err(|e| format!("could not start Lightroom archive finalization: {e}")) +} + +#[cfg(target_arch = "wasm32")] +fn spawn_finalize( + _finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization, + _tx: mpsc::Sender, + _ctx: egui::Context, +) -> Result<(), String> { + unsupported_wasm().map(|_| ()) +} + +#[cfg(not(target_arch = "wasm32"))] +fn inspect_report(data: lightcraft_engine::lightroom_catalog::CatalogImport) -> Value { + json!({ + "photos": data.photos.len(), + "collections": data.collections.iter().filter(|r| r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0) == 0.0).count(), + "missing": data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p| &p.path).collect::>(), + "warnings": data.warnings, + }) +} + +fn terminal(app: &mut LightcraftApp, ctx: &egui::Context, kind: Kind, value: Value) { + let message = if let Some(e) = value.get("error").and_then(Value::as_str) { + let warning = value.get("indexWarning").and_then(Value::as_str).map_or(String::new(), |w| format!("; index warning: {w}")); + if matches!(kind, Kind::Inspect) { + crate::i18n::tr_format!("Lightroom inspection failed: {e}{warning}", e = e, warning = warning) + } else { + crate::i18n::tr_format!("Lightroom import failed: {e}{warning}", e = e, warning = warning) + } + } else if value.get("cancelled").and_then(Value::as_bool).unwrap_or(false) { + crate::i18n::tr(if matches!(kind, Kind::Inspect) { "Lightroom inspection cancelled" } else { "Lightroom import cancelled" }).into() + } else if matches!(kind, Kind::Inspect) { + let photos = value["photos"].as_u64().unwrap_or(0); + let collections = value["collections"].as_u64().unwrap_or(0); + let missing = value["missing"].as_array().map_or(0, Vec::len); + crate::i18n::tr_format!( + "Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing", + photos = photos, + collections = collections, + missing = missing + ) + } else { + let report = value.get("report").unwrap_or(&value); + let imported = report["imported"].as_u64().unwrap_or(0); + let failed = report["failed"].as_array().map_or(0, Vec::len); + let warning = value.get("indexWarning").and_then(Value::as_str); + if let Some(warning) = warning { + crate::i18n::tr_format!( + "Lightroom import complete: {imported} photos added; index warning: {warning}", + imported = imported, + warning = warning + ) + } else if failed == 0 { + crate::i18n::tr_format!("Lightroom import complete: {imported} photos added", imported = imported) + } else { + crate::i18n::tr_format!("Lightroom import complete: {imported} photos added, {failed} failed", imported = imported, failed = failed) + } + }; + app.lightroom_last = Some(value); + app.lightroom = None; + app.toast_for(ctx, message, 6.0); +} + +#[cfg(not(target_arch = "wasm32"))] +fn start_inspect(app: &mut LightcraftApp, path: PathBuf, ctx: &egui::Context) -> Result { + busy(app)?; + let cancel = Arc::new(AtomicBool::new(false)); + let total = Arc::new(AtomicUsize::new(0)); + let done = Arc::new(AtomicUsize::new(0)); + let (tx, rx) = mpsc::channel(); + spawn_inspect(path.clone(), cancel.clone(), total.clone(), done.clone(), tx, ctx.clone())?; + app.lightroom_last = None; + app.lightroom = Some(LightroomTask { kind: Kind::Inspect, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false }); + Ok(json!({"running": true, "kind": "inspect", "path": path})) +} + +#[cfg(not(target_arch = "wasm32"))] +fn start_import(app: &mut LightcraftApp, path: PathBuf, update_existing: bool, ctx: &egui::Context) -> Result { + busy(app)?; + let job = lightcraft_engine::lightroom_job::LightroomJob::new(&mut app.session, path.clone(), update_existing).map_err(|e| e.to_string())?; + let cancel = Arc::new(AtomicBool::new(false)); + let total = job.total_atomic(); + let done = job.done_atomic(); + let (tx, rx) = mpsc::channel(); + spawn_prepare(job, cancel.clone(), tx, ctx.clone())?; + app.lightroom_last = None; + app.lightroom = Some(LightroomTask { kind: Kind::Import, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false }); + Ok(json!({"running": true, "kind": "import", "path": path})) +} + +/// Run Lightroom UI commands before the generic engine dispatcher. +pub fn command(app: &mut LightcraftApp, id: &str, p: &Value, ctx: &egui::Context) -> Result { + #[cfg(target_arch = "wasm32")] + { + let _ = (app, id, p, ctx); + return unsupported_wasm(); + } + #[cfg(not(target_arch = "wasm32"))] + { + let path = p.get("path").and_then(Value::as_str).map(PathBuf::from); + let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false); + let status = p.get("status").and_then(Value::as_bool).unwrap_or(false) || path.is_none(); + if status { + if p.get("cancel").and_then(Value::as_bool).unwrap_or(false) + && let Some(task) = app.lightroom.as_mut() + { + task.cancel(); + } + if wait && app.lightroom.is_some() { + return wait_for(app, ctx, std::time::Duration::from_secs(600)); + } + return Ok(app.lightroom.as_ref().map(LightroomTask::status).or_else(|| app.lightroom_last.clone()).unwrap_or(Value::Null)); + } + let Some(path) = path else { return Err("missing `path`".into()) }; + let result = if id == "library.inspectLightroom" { + start_inspect(app, path, ctx) + } else { + start_import(app, path, p.get("updateExisting").and_then(Value::as_bool).unwrap_or(false), ctx) + }?; + if wait { + return wait_for(app, ctx, std::time::Duration::from_secs(600)); + } + Ok(result) + } +} + +/// Poll owner-thread work, commit a prepared import, and finish its archive index. +pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(mut task) = app.lightroom.take() else { return }; + let message = match task.rx.try_recv() { + Ok(m) => m, + Err(mpsc::TryRecvError::Empty) => { + app.lightroom = Some(task); + return; + } + Err(mpsc::TryRecvError::Disconnected) => { + terminal(app, ctx, task.kind, json!({"error": "Lightroom worker stopped unexpectedly"})); + return; + } + }; + match message { + Message::Inspected(result) => match result { + Ok(report) if !task.cancel.load(Ordering::Relaxed) => terminal(app, ctx, task.kind, report), + Ok(_) => terminal(app, ctx, task.kind, json!({"cancelled": true})), + Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => terminal(app, ctx, task.kind, json!({"cancelled": true})), + Err(e) => terminal(app, ctx, task.kind, json!({"error": e})), + }, + Message::Prepared(result) => { + let prepared = match result { + Ok(p) => p, + Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => { + terminal(app, ctx, task.kind, json!({"cancelled": true})); + return; + } + Err(e) => { + terminal(app, ctx, task.kind, json!({"error": e})); + return; + } + }; + if task.cancel.load(Ordering::Relaxed) || prepared.is_cancelled() || !prepared.token().matches(&app.session) { + prepared.rollback(); + terminal( + app, + ctx, + task.kind, + if task.cancel.load(Ordering::Relaxed) { + json!({"cancelled": true}) + } else { + json!({"error": "Lightroom import belongs to a different open library"}) + }, + ); + return; + } + task.phase = Phase::Committing; + let mut finalization = None; + let mut report = None; + let committed = app.session.execute_fn("library.importLightroom", |s| { + let completion = lightcraft_engine::lightroom_job::commit_prepared(s, *prepared)?; + report = Some(completion.report.clone()); + finalization = Some(completion.finalization); + Ok(completion.report) + }); + let report = report.unwrap_or(Value::Null); + task.phase = Phase::Finalizing; + let (tx, rx) = mpsc::channel(); + let commit_error = committed.as_ref().err().map(ToString::to_string); + let Some(finalization) = finalization else { + terminal(app, ctx, task.kind, json!({"error": commit_error.clone().unwrap_or_else(|| "Lightroom import did not commit".into())})); + return; + }; + if let Err(e) = spawn_finalize(finalization, tx, ctx.clone()) { + let mut result = json!({"report": report, "indexWarning": e}); + if let Some(error) = commit_error { + result["error"] = json!(error); + } + terminal(app, ctx, task.kind, result); + return; + } + task.rx = rx; + task.cancelled = false; + let mut pending = json!({"running": true, "kind": "import", "report": report}); + if let Some(error) = commit_error { + pending["error"] = json!(error); + } + app.lightroom_last = Some(pending); + app.lightroom = Some(task); + } + Message::Finalized(result) => { + let pending = app.lightroom_last.take().unwrap_or(Value::Null); + let report = pending.get("report").cloned().unwrap_or(Value::Null); + match result { + Ok(()) if pending.get("error").is_none() => terminal(app, ctx, task.kind, report), + Ok(()) => terminal(app, ctx, task.kind, json!({"report": report, "error": pending["error"].clone()})), + Err(e) => { + let mut value = json!({"report": report, "indexWarning": e}); + if let Some(error) = pending.get("error") { + value["error"] = error.clone(); + } + terminal(app, ctx, task.kind, value); + } + } + } + } + ctx.request_repaint(); +} + +/// Wait for a task in control/headless mode while still applying owner-thread completion. +#[cfg(not(target_arch = "wasm32"))] +pub fn wait_for(app: &mut LightcraftApp, ctx: &egui::Context, timeout: std::time::Duration) -> Result { + let start = std::time::Instant::now(); + while app.lightroom.is_some() { + tick(app, ctx); + if start.elapsed() >= timeout { + return Err("timed out waiting for Lightroom task".into()); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + Ok(app.lightroom_last.clone().unwrap_or(Value::Null)) +} + +#[cfg(target_arch = "wasm32")] +pub fn wait_for(_app: &mut LightcraftApp, _ctx: &egui::Context, _timeout: std::time::Duration) -> Result { + unsupported_wasm() +} + +/// The progress window for inspect/import, with cancellation. +pub fn progress(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(task) = app.lightroom.as_ref() else { return }; + let total = task.total.load(Ordering::Relaxed); + let done = task.done.load(Ordering::Relaxed); + let title = match task.kind { + Kind::Inspect => crate::i18n::tr("Inspecting Lightroom Catalog"), + Kind::Import => crate::i18n::tr("Importing Lightroom Catalog"), + }; + let text = match task.phase { + Phase::Reading if total == 0 => crate::i18n::tr("Reading Lightroom catalog…").into(), + Phase::Reading => crate::i18n::tr_format!("Reading Lightroom catalog… {done} of {total}", done = done, total = total), + Phase::Committing => crate::i18n::tr("Adding Lightroom photos…").into(), + Phase::Finalizing => crate::i18n::tr("Saving Lightroom import index…").into(), + }; + let mut cancel = false; + let t = Tokens::get(ctx); + egui::Window::new(title).title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([360.0, 92.0]).show( + ctx, + |ui| { + ui.label(egui::RichText::new(text).color(t.text)); + ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(340.0)); + let r = + ui.add_enabled(task.phase == Phase::Reading && !task.cancel.load(Ordering::Relaxed), egui::Button::new(crate::i18n::tr("Cancel"))); + register(ui.ctx(), "button:lightroomCancel", r.rect); + cancel = r.clicked(); + }, + ); + if cancel { + if let Some(task) = app.lightroom.as_mut() { + task.cancel(); + } + ctx.request_repaint(); + } + if app.lightroom.is_some() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } +} + +#[cfg(test)] +mod tests { + use super::inspect_report; + use lightcraft_engine::lightroom_catalog::CatalogImport; + + #[test] + fn inspect_report_keeps_empty_catalog_shape() { + let report = inspect_report(CatalogImport { + source: "synthetic.lrcat".into(), + photos: Vec::new(), + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: vec!["warning".into()], + }); + assert_eq!(report["photos"], 0); + assert_eq!(report["collections"], 0); + assert_eq!(report["missing"], serde_json::json!([])); + assert_eq!(report["warnings"], serde_json::json!(["warning"])); + } +} diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 6b95fd576..609107276 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -148,6 +148,7 @@ pub const UI_COMMANDS: &[UiCommand] = &[ ("merge.hdrPanoramaLast", "HDR Panorama with Last Settings", None, "Photo>Photo Merge"), ("file.addPhotos", "Import Photos…", Some("Cmd+Shift+I"), "File"), ("file.addFolder", "Import from Folder…", None, "File"), + ("file.importLightroom", "Import Lightroom Catalog…", None, "File"), ("file.addFromDevice", "Import from Device", None, ""), ("file.findMissing", "Find Missing Photos…", None, "File"), ("file.backupLibrary", "Back Up Library…", None, "File"), @@ -242,6 +243,10 @@ pub fn parse_rgb(v: &Value) -> Option<[u8; 3]> { /// Handle UI commands; `None` means "not a UI command — send it to the engine". pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option> { + if matches!(id, "library.inspectLightroom" | "library.importLightroom") { + let ctx = egui::Context::default(); + return Some(crate::lightroom_import::command(app, id, p, &ctx)); + } if let Some(language) = language_from_command(id) { app.ui.language = language; // Immediately, not on the next frame: the reply and anything else run this frame @@ -984,11 +989,17 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option { + if crate::lightroom_import::is_running(app) { + return Some(Err("finish Lightroom catalog import before browsing folders".into())); + } // listed and read in the background (see `import::browse`) let path = p.get("path").and_then(Value::as_str)?; crate::import::browse(app, path, p.get("subfolders").and_then(Value::as_bool)) } "file.addPhotos" => { + if crate::lightroom_import::is_running(app) { + return Some(Err("finish Lightroom catalog import before adding photos".into())); + } let paths = match p.get("paths").and_then(Value::as_array) { Some(a) => a.iter().filter_map(Value::as_str).map(str::to_string).collect(), None => app.services.pick_files.as_mut().map(|f| f()).unwrap_or_default(), @@ -999,6 +1010,14 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option { + let path = + p.get("path").and_then(Value::as_str).map(str::to_string).or_else(|| app.services.pick_lightroom_catalog.as_mut().and_then(|f| f())); + match path { + Some(path) => app.run("library.importLightroom", json!({"path":path})), + None => Ok(Value::Null), + } + } "app.quit" => { app.ui.quit = true; Ok(Value::Null) @@ -1326,7 +1345,7 @@ pub fn ui_enabled(app: &LightcraftApp, id: &str) -> bool { "file.exportCurvePresets" => !app.session.curve_presets.is_empty(), "view.compare" => app.session.catalog.len() > 1, "view.fullScreenPreview" | "view.infoOverlay" | "view.navigator" => app.session.active().is_some() || app.ui.fullscreen, - "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some(), + "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some() && !crate::lightroom_import::is_running(app), "file.backupLibrary" => app.services.backup_library.is_some(), "file.restoreLibrary" => app.services.restore_library.is_some(), "compare.swap" | "compare.makeSelect" => app.ui.view == ViewMode::Compare, diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index b33d41ae5..e16d88154 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -543,6 +543,9 @@ fn interface_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) { /// `app.openLibrary {path?}`: close the current library and open (or create) the one at `path`, /// or a folder chosen in a dialog. Remembered as the library to open at launch. pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result { + if crate::lightroom_import::is_running(app) { + return Err("wait for Lightroom catalog import to finish before switching libraries".into()); + } let path = match p.get("path").and_then(Value::as_str) { Some(x) => x.to_string(), None => match app.services.pick_folder.as_mut() { @@ -555,6 +558,7 @@ pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result }; app.session.close_library().map_err(|e| e.to_string())?; app.session.open_library(&path, false).map_err(|e| e.to_string())?; + app.lightroom_last = None; app.renderer.forget_all(); app.ui.compare = None; app.ui.settings.library_path = path.clone(); diff --git a/docs/lightroom-catalog-import.md b/docs/lightroom-catalog-import.md new file mode 100644 index 000000000..d3d1940e9 --- /dev/null +++ b/docs/lightroom-catalog-import.md @@ -0,0 +1,59 @@ +# Lightroom Classic catalog import + +File → Import Lightroom Catalog… opens `.lrcat` directly. Lightroom, Python, DNG conversion +and a C SQLite runtime are not required. The source database is read-only, including committed +`-wal` pages; a catalog that changes during the read is rejected. LightCraft owns subsequent edits. + +Equivalent commands: + +```text +lightcraft-cli run --library "LightCraft Library" library.inspectLightroom path="Lightroom Catalog.lrcat" +lightcraft-cli run --library "LightCraft Library" library.importLightroom path="Lightroom Catalog.lrcat" +``` + +Original files are referenced in place. Ratings, picks/rejects, color labels, XMP metadata, +hierarchical keywords, regular collections/collection sets, virtual copies and mapped develop +settings are imported. Missing originals remain catalogued for relinking. Existing LightCraft +edits are preserved by default; `updateExisting=true` explicitly replaces them. Persistent import +identity prevents reimport from duplicating virtual copies and collections. One undo step reverses +catalog changes; any saved recovery archives remain available. + +Before changing records, the importer attempts to save source image records, decoded XMP, +verbatim develop settings, history, snapshots and collection content to compressed +`Interop/lightroom-import-N.lca` recovery archives in the LightCraft library. Archiving is +best-effort: a size limit or write failure produces a warning and does not prevent photo import. +Unchanged source data reuses its existing archive. Opaque SQL BLOBs are skipped. Each archive is +capped at 32 MiB (uncompressed and on disk), with at most eight archives and 128 MiB total +retained. Oldest managed archives are retired; legacy `.json` archives remain untouched. +`Interop/lightroom-index.json` records imported identities. Source catalogs and originals are +never overwritten. + +Native import is bounded to a 1 GiB database/WAL, 16 MiB XMP packets and one million rows per +table. The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format +specification; it introduces no dependency or C runtime. Table payloads are capped at 256 MiB. +Optional history, keyword and collection tables that exceed limits are skipped with warnings; +required photo tables still report errors. Unrelated unsupported schemas are ignored. A needed +WITHOUT ROWID or virtual table produces an explicit error, handled as a warning when optional. + +Develop settings reuse the existing XMP/preset mapper. Supported sliders, curves and supported +mask structures remain editable, but this is approximate rendering: camera profiles, Adobe AI +models, some masking/retouch fields and process-version algorithms are not reproduced. Unmapped +fields are reported. When a recovery archive is saved, it retains source settings/history/snapshots +and original smart-collection rules. Smart collections become regular albums with current +membership. Archived history and snapshots are source data, not native LightCraft history yet. +Lightroom's `-999999` deferred-adjustment sentinel is omitted from both catalog and XMP mappings; +it is reported, included in any saved archive, and never clamped into a real slider value. Deferred Adobe Auto Tone +is not evaluated by the importer; LightCraft's Auto control remains available after migration. + +Native desktop inspection/import runs on a cancellable background worker with progress. Only +prepared catalog operations commit on the owner thread; archive index writing stays on a worker. +Late results from a closed or switched library are rejected, including reopening the same path. +Ordinary edits during preparation remain intact. CLI commands complete synchronously using the +same preparation/commit pipeline; browser builds reject native catalog import. + +Archive version 1 begins with the 12 bytes `LC-LRARCH\0\x01Z`, followed by zlib-compressed JSON. +Recovery readers must cap decompression at 32 MiB. JSON retains source identities, photo paths, +settings, decoded XMP, keywords, history, snapshots, collection definitions and membership. + +The importer uses shared Rust code on macOS, Windows and Linux. If originals move between +computers or volumes, use LightCraft's missing-photo relinking tools to update their paths. diff --git a/docs/parity.md b/docs/parity.md index b6126ea5d..8b7f2cbae 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -731,6 +731,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06- | LRC-LIB-SURVEY | Survey view | P2 | ✅ | `cmd:view.survey` | N | | LRC-LIB-REFVIEW | Reference view | P2 | ✅ | `cmd:view.reference`, `cmd:photo.setReference`, `crates/ui-egui/src/panels/compare.rs` (`show_reference`) | View ▸ Reference View (⇧R): the reference photo beside the active one (edited with the Edit panel); photo menu ▸ Set as Reference Photo | | LRC-LIB-CATALOG-PANEL | Catalog sets | P1 | ✅ | `cmd:library.source` (`missing`) | all, recently added, picks, recently deleted, missing photos (when any) | +| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-tree read-only Rust SQLite B-tree reader; background inspection/import with progress/cancellation, per-open library identity checks; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Best-effort deduplicated recovery archives retain settings/history/snapshots/collection rules (32 MiB each, eight archives/128 MiB aggregate); archive limits/write failures and optional-table limits warn without blocking photo import; unrelated unsupported tables and opaque SQL BLOBs skipped. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs | | LRC-LIB-CATALOG-LOCK | Catalog open in one program at a time | P1 | ✅ | `cmd:app.openLibrary`, `crates/catalog/src/lock.rs`, `crates/engine/src/library.rs` (`open_library`) | exclusive OS lock on `catalog.lock` for the session (app and `lightcraft-cli --library`); a second opener is refused with who has it (`catalog.lock.owner`: program, process, computer); released by the OS on a crash, so never stale; file systems without locks open unprotected with a warning (the app's notices, `library.info` → `settingsWarnings`, and the CLI's stderr; issue #171); no read-only mode; the browser build (OPFS) has no guard across tabs yet. A library that can't be opened at launch (locked, unreadable, newer format, drive missing) shows a window — Try Again, Choose Another Library…, Continue Without Saving (then a banner; nothing written), Quit — instead of a silent demo session (`crates/ui-egui/src/panels/library_problem.rs`) | | LRC-LIB-FOLDERS | Disk folder tree | P1 | ✅ | `cmd:library.browse`, `cmd:folder.rename`, `cmd:folder.move`, `crates/ui-egui/src/panels/left.rs` (`local_section`, `folder_tree`) | standard places + the browsed folder + Browse Folder…, each an expandable tree of subfolders, breadcrumb navigation; context menu: Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked; one undo step each: undo / redo move the folder back and forth, refused when the destination is taken) | | LRC-LIB-LIBFOLDERS | Library folders | P1 | ✅ | `cmd:library.folders`, `cmd:library.source` (`libraryFolder`), `cmd:library.filter` (`libraryFolder`), `cmd:library.removeFolder`, `cmd:folder.rename`, `cmd:folder.move`, `crates/catalog/src/folders.rs`, `crates/ui-egui/src/panels/left.rs` (`folders_section`, `folder_menu_for_library`, `row_menu`), `crates/ui-egui/src/tests_panels.rs` | sidebar Folders: the folders the library's photos were imported from, with photo counts (subfolders included), as another way to explore the library next to By Date and Keywords; Local stays the way to browse any folder on disk. Each disk is a row of its own and starts open: a drive letter, a Windows share (`\\server\share`, `\\?\UNC\…`, `\\wsl$\…`), a macOS share or disk (`/Volumes/tokyo`), a Linux mount (`/media`, `/run/media`, `/mnt`, gvfs `smb-share:…`, `/net/`), NFS (`/Network/Servers/`), else “This Computer” — told apart from the stored paths alone, so an offline share lists like a mounted one and the sidebar never waits on a network. Only folders that hold imported photos are listed (browsed, deleted and demo photos don't count); every folder is a row of its own (`Users` → `me` → `Pictures`), and a folder that holds nothing itself and leads to one folder starts open; every level indents one step more than its parent however deep it goes: the whole sidebar (Albums, Local, By Date, Folders, Keywords) scrolls sideways when its widest row is wider than the panel (rows are measured one frame behind; Create Album and the counts stay at the visible edge); a name that does not fit ends in an ellipsis at the count (the full name on hover; a row asks for room for at most 140 pt of its name, so depth, not one long name, makes the sidebar scroll), as does the grid title; selection and hover bars end at the panel's edge; click makes the folder the source, like an album or a Local folder (its photos and those of the folders inside it fill the grid, titled “photos/travel”; the filter bar narrows it further; `library.source` `libraryFolder` + `path`; kept with the library's view), and its row is always opened into view; the shown folder follows a rename or move and their undo (engine-tested; the sidebar test covers rename), is dropped back to All Photos when it is removed or no longer holds a photo, and a smart album made from the view keeps it, also when its rules are edited (the editor has no folder field). Only a folder that shows exactly its own photos can be shown (not the startup disk, not `/Volumes`-like folders above disks, not an empty one). A row that would cover other disks' photos too (the startup disk, `/Volumes`, `/mnt`…) only opens. Right-click (row or triangle): Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked, as in Local) and Remove from Library… (asks; moves the folder's photos to Recently Deleted, one undo step, no file touched); a disk row offers Remove Disk from Library… (the command needs `disk: true` for a whole disk or a folder spanning several). Limits: names are compared as written (macOS `/Volumes/tokyo` vs `/Volumes/TOKYO` are two disks; `/Volumes/Macintosh HD/…` and `/…` two rows); no disk capacity / online status; no “include subfolders” switch (a folder always includes the ones inside it); creating a folder and moving photos between folders are not here | diff --git a/docs/whats-new.md b/docs/whats-new.md index 2381cb4c1..81366dc9e 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -27,6 +27,12 @@ - Sony A7R II (and other) raws whose camera JPEG is lens-corrected no longer open grey and too dark (issue #232): the starting look is fitted to the camera JPEG away from edges when the misaligned edges spoil the fit on all pixels. +### Lightroom Classic catalogs +- File → Import Lightroom Catalog… opens `.lrcat` directly, with originals referenced in place. + Ratings, flags, labels, keywords, collections/sets, virtual copies and supported edits migrate; + existing LightCraft edits are preserved by default. Source settings/history are archived, unsupported + fields are reported, and the original Lightroom database stays read-only. Rendering is approximate. + ### Presets and profiles - Import presets from other editors: XMP presets, classic `.lrtemplate` files, "DNG presets" from mobile apps and `.zip` bundles of any of these — whole folders at once, grouped by pack. Masks inside presets come along.