diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index ef838fe02..9d6d1006f 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -31,6 +31,7 @@ mod prefs; mod preset_files; pub mod previews; mod query; +mod sync; mod xmp; use serde::Serialize; @@ -142,6 +143,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(before::specs()); v.extend(browse::specs()); v.extend(missing::specs()); + v.extend(sync::specs()); v.extend(metadata::specs()); v.extend(filters::specs()); v.extend(denoise::specs()); diff --git a/crates/engine/src/cmd/sync.rs b/crates/engine/src/cmd/sync.rs new file mode 100644 index 000000000..1aefe94b7 --- /dev/null +++ b/crates/engine/src/cmd/sync.rs @@ -0,0 +1,53 @@ +//! Synchronize Folder commands (see [`crate::sync`]). + +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, bool_or, cmd, str_param}; +use crate::sync::{SyncChoice, scan, synchronize}; +use crate::{Result, Session}; + +fn path(p: &Value, c: &str) -> Result { + str_param(p, "path").filter(|d| !d.trim().is_empty()).map(str::to_string).ok_or_else(|| bad(c, "missing `path`")) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + query "folder.scanChanges", + "Scan Folder for Changes", + [], + None, + "{path} — compare a folder of the library (and the folders inside it) with the disk; changes nothing → {path, new: [{path, name, format, …}] files the library doesn't have, duplicates: files whose content it already has, unreadable: [{path, error}], missing: [{id, path}] photos whose file is gone, metadata: [{id, path}] photos whose XMP sidecar was saved after they came in or were last edited here and says something the library doesn't}", + always, + |s, p| { + const C: &str = "folder.scanChanges"; + let path = path(p, C)?; + let changes = scan(s, &path).map_err(|e| bad(C, e.to_string()))?; + Ok(serde_json::to_value(changes).unwrap_or_default()) + } + ), + cmd!( + "folder.synchronize", + "Synchronize Folder", + [], + None, + "{path, importNew?: true, removeMissing?: false, readMetadata?: false} — bring a folder of the library up to date with the disk (see folder.scanChanges): import its new files in place, move photos whose file is gone to Recently Deleted, read XMP sidecars saved by other apps (the sidecar wins); one undo step, no file is touched → {imported, removed, read, failed: [[path, error]]}", + always, + sync + ), + ] +} + +fn sync(s: &mut Session, p: &Value) -> Result { + const C: &str = "folder.synchronize"; + let path = path(p, C)?; + let d = SyncChoice::default(); + let choice = SyncChoice { + import_new: bool_or(p, "importNew", d.import_new), + remove_missing: bool_or(p, "removeMissing", d.remove_missing), + read_metadata: bool_or(p, "readMetadata", d.read_metadata), + }; + let changes = scan(s, &path).map_err(|e| bad(C, e.to_string()))?; + let r = synchronize(s, changes, choice)?; + Ok(json!({"imported": r.imported, "removed": r.removed, "read": r.read, "failed": r.failed})) +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a6c5f6420..feeeccc60 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -53,6 +53,7 @@ pub mod rename; pub mod segment; pub mod sidecar; pub mod smart; +pub mod sync; mod view; pub mod walk; @@ -877,4 +878,6 @@ mod tests_settings_files; #[cfg(test)] mod tests_spots; #[cfg(test)] +mod tests_sync; +#[cfg(test)] mod tests_xmp; diff --git a/crates/engine/src/sync.rs b/crates/engine/src/sync.rs new file mode 100644 index 000000000..c91e47fa0 --- /dev/null +++ b/crates/engine/src/sync.rs @@ -0,0 +1,232 @@ +//! Synchronize Folder: bring a folder of the library up to date with what is on disk, as +//! Lightroom Classic's Synchronize Folder does. +//! +//! A [scan](scan_with) compares the folder (and the folders inside it) with the library and finds +//! three kinds of change, touching nothing: +//! +//! - **new** files: supported files the library doesn't have, by path or by content (the import +//! scan's duplicate rules, [`crate::import::scan_with`]); +//! - **missing** photos: library photos in the folder whose file is gone (the Missing Photos rule, +//! [`crate::cmd::missing::checked_path`]); +//! - **metadata** updates: photos whose XMP sidecar file was saved after the photo came into the +//! library or was last edited here, and says something the library doesn't (rating, flag, +//! label, metadata, capture time or develop settings). Reading one makes the sidecar win, as +//! Read Metadata from File does. Embedded XMP in a raw is not a sidecar and is not looked at. +//! +//! [`synchronize`] then imports the new files (in place), moves the missing photos to Recently +//! Deleted and reads the metadata updates — each only when chosen — as one undo step. +//! +//! The scan runs without the session ([`SyncInput`] + [`scan_with`]), so the app can run it on a +//! worker thread: a folder on a sleeping network share must never stall a frame. +//! +//! Limits: a sidecar the library itself wrote (auto-write on) agrees with it and is never an +//! update; one written here before auto-write was turned off, followed by changes made only in +//! the library, is listed as an update even though the library is newer (there is no record of +//! when a photo's metadata last changed here). Reading updates is therefore never the default. + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use lightcraft_catalog::{Catalog, Filter, Op, Photo, PhotoId, Sort, Source}; +use serde::Serialize; + +use crate::import::{ImportCandidate, ImportMode, ImportOptions, ScanInput, ScanProgress}; +use crate::media::ProbeInfo; +use crate::sidecar::SidecarNaming; +use crate::{EngineError, Result, Session}; + +/// The largest XMP sidecar read when looking for metadata updates; a bigger one is skipped. +const MAX_SIDECAR_BYTES: u64 = 16 * 1024 * 1024; + +/// What a scan of a folder needs, taken from the session so it can run on a worker thread. +pub struct SyncInput { + folder: String, + scan: ScanInput, + /// The library's photos in the folder, each with how its sidecar is named. + photos: Vec<(Arc, SidecarNaming)>, + /// For the colour label names a sidecar may use. + labels: Catalog, +} + +/// What changed in a folder since it came into the library. +#[derive(Clone, Debug, Default, Serialize)] +pub struct FolderChanges { + /// The folder, as asked for. + pub path: String, + /// Files to import. + pub new: Vec, + /// Files whose content the library already has (under another path). + pub duplicates: usize, + /// Supported files that couldn't be read. + pub unreadable: Vec, + /// Photos whose file is gone. + pub missing: Vec, + /// Photos whose sidecar has news. + pub metadata: Vec, + /// The probes of the new files, kept for the import that follows. + #[serde(skip)] + probes: HashMap, +} + +/// A photo of the folder and its file. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct FolderPhoto { + pub id: u64, + pub path: String, +} + +/// What [`synchronize`] does with the changes found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SyncChoice { + pub import_new: bool, + pub remove_missing: bool, + pub read_metadata: bool, +} + +impl Default for SyncChoice { + /// Lightroom Classic's defaults: import the new files, leave the rest to be looked at. + fn default() -> Self { + SyncChoice { import_new: true, remove_missing: false, read_metadata: false } + } +} + +/// What [`synchronize`] did. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SyncReport { + pub imported: usize, + pub removed: usize, + pub read: usize, + /// (path or photo, why) for what couldn't be done. + pub failed: Vec<(String, String)>, +} + +fn invalid(msg: impl Into) -> EngineError { + EngineError::Other(msg.into()) +} + +impl SyncInput { + /// The scan of folder `path`: a folder the library holds photos in, not the startup disk. + pub fn new(s: &mut Session, path: &str) -> Result { + let path = path.trim(); + if path.is_empty() { + return Err(invalid("missing `path`")); + } + if lightcraft_catalog::folders::is_startup_disk(path) { + return Err(invalid("choose a folder, not the whole startup disk")); + } + let f = Filter { library_folder: Some(path.to_string()), ..Default::default() }; + let ids = s.catalog.query(&f, &Sort::default()); + if ids.is_empty() { + return Err(invalid(format!("{path}: no photo in the library was imported from it"))); + } + let photos = ids.iter().filter_map(|id| s.catalog.photo(*id).map(|p| (Arc::clone(p), s.sidecar_naming(*id)))).collect(); + let (scan, _) = ScanInput::new(s, &[path.to_string()]); + Ok(SyncInput { folder: path.to_string(), scan, photos, labels: s.catalog.clone() }) + } +} + +/// Find what changed in the folder (see the module docs). Reads the disk, never the session. +pub fn scan_with(input: SyncInput, progress: &ScanProgress) -> FolderChanges { + let SyncInput { folder, scan, photos, labels } = input; + let out = crate::import::scan_with(scan, std::slice::from_ref(&folder), progress); + let mut changes = FolderChanges { path: folder, probes: out.probes, ..Default::default() }; + for c in out.candidates { + match (&c.duplicate, &c.error) { + (Some(d), _) if d == "path" => {} + (Some(_), _) => changes.duplicates += 1, + (None, Some(_)) => changes.unreadable.push(c), + (None, None) => changes.new.push(c), + } + } + for (p, naming) in &photos { + let Some(file) = crate::cmd::missing::checked_path(p) else { continue }; + if cfg!(target_arch = "wasm32") { + continue; + } + if !Path::new(file).exists() { + changes.missing.push(FolderPhoto { id: p.id.0, path: file.to_string() }); + } else if p.copy_of.is_none() && sidecar_has_news(&labels, p, *naming) { + changes.metadata.push(FolderPhoto { id: p.id.0, path: file.to_string() }); + } + } + changes +} + +/// [`scan_with`] on the calling thread. +pub fn scan(s: &mut Session, path: &str) -> Result { + let input = SyncInput::new(s, path)?; + Ok(scan_with(input, &ScanProgress::default())) +} + +/// Whether `p`'s XMP sidecar file was saved after the library last had its say about the photo +/// (imported, or edited here) and says something the library doesn't. +fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming) -> bool { + let Source::File { path } = &p.source else { return false }; + let Some(file) = crate::sidecar::find_sidecar(path, naming) else { return false }; + let Ok(meta) = std::fs::metadata(&file) else { return false }; + if meta.len() > MAX_SIDECAR_BYTES { + return false; + } + let saved = meta.modified().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map(|d| d.as_secs()); + let Some(saved) = saved.and_then(|s| i64::try_from(s).ok()) else { return false }; + let seen = [Some(p.imported.as_str()), p.edited.as_deref()].into_iter().flatten().filter_map(lightcraft_catalog::stacks::iso_seconds).max(); + if seen.is_some_and(|t| saved <= t) { + return false; + } + let Ok(text) = std::fs::read_to_string(&file) else { return false }; + let Ok(sc) = crate::sidecar::parse_sidecar(&text, p.kind == lightcraft_catalog::MediaKind::Raw) else { return false }; + let sc = sc.resolve_label(labels); + let mut q = p.clone(); + crate::sidecar::merge_into(&mut q, &sc, ""); + (q.rating, q.flag, q.label, &q.meta, &q.captured, &q.develop) != (p.rating, p.flag, p.label, &p.meta, &p.captured, &p.develop) +} + +/// Act on `changes` as `choice` says, as one undo step ("Synchronize Folder"). A missing photo +/// whose file is back by now is left alone. +pub fn synchronize(s: &mut Session, changes: FolderChanges, choice: SyncChoice) -> Result { + let mark = s.undo.len(); + let mut report = SyncReport::default(); + let FolderChanges { new, missing, metadata, probes, .. } = changes; + if choice.import_new && !new.is_empty() { + s.import_probes = probes; + let files: Vec = new.into_iter().map(|c| c.path).collect(); + let r = crate::import::import_with(s, &files, &ImportOptions { mode: ImportMode::Add, ..Default::default() })?; + report.imported = r.imported.len(); + report.failed.extend(r.failed); + } + if choice.remove_missing { + let gone: Vec = missing.iter().filter(|m| !Path::new(&m.path).exists()).map(|m| PhotoId(m.id)).collect(); + let ops: Vec = gone + .iter() + .filter(|id| s.catalog.photo(**id).is_some_and(|p| p.in_library())) + .map(|id| Op::SetDeleted { id: *id, deleted: true }) + .collect(); + if !ops.is_empty() { + report.removed = ops.len(); + s.commit("Remove Missing Photos", Op::Batch { ops })?; + } + } + if choice.read_metadata { + let mut ops = Vec::new(); + for m in &metadata { + match s.read_sidecar_op(PhotoId(m.id)) { + Ok(Some((op, _))) => { + ops.push(op); + report.read += 1; + } + Ok(None) => report.failed.push((m.path.clone(), "no XMP sidecar".into())), + Err(e) => report.failed.push((m.path.clone(), e.to_string())), + } + } + if !ops.is_empty() { + s.commit("Read Metadata from File", Op::Batch { ops })?; + } + } + let steps = s.undo.len().saturating_sub(mark); + s.merge_undo(steps, "Synchronize Folder"); + if let Some(last) = s.undo.last_mut().filter(|_| steps == 1) { + last.label = "Synchronize Folder".into(); + } + Ok(report) +} diff --git a/crates/engine/src/tests_sync.rs b/crates/engine/src/tests_sync.rs new file mode 100644 index 000000000..4822f1172 --- /dev/null +++ b/crates/engine/src/tests_sync.rs @@ -0,0 +1,211 @@ +//! Synchronize Folder: bring a library folder up to date with what is on disk (see +//! [`crate::sync`]). +//! +//! Scenarios, in the words of someone whose folder changed outside LightCraft: +//! +//! * Given a folder of the library, when files were added to it (or to a folder inside it), the +//! scan lists them as new; files the library already has, by path or by content, are not new. +//! * When a photo's file was deleted or moved away, the scan lists the photo as missing. +//! * When another app saved a photo's XMP sidecar after the photo came into the library, and the +//! sidecar says something the library doesn't, the scan lists a metadata update; a sidecar older +//! than that, or one that agrees with the library, is not an update. +//! * Scanning changes nothing. +//! * Synchronizing imports the new files; it removes missing photos (to Recently Deleted) and +//! reads metadata updates only when asked. Whatever it does is one undo step. +//! * Only a folder of the library can be synchronized, never the whole startup disk. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use serde_json::{Value, json}; + +use crate::Session; + +/// A scratch folder that goes away with the test, however it ends. +struct Scratch(PathBuf); + +impl Scratch { + fn new(tag: &str) -> Scratch { + let dir = std::env::temp_dir().join(format!("lc-sync-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + Scratch(dir) + } + fn path(&self, rel: &str) -> String { + self.0.join(rel).to_string_lossy().to_string() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// A small procedural PNG (distinct per seed). +fn write_png(path: &str, seed: u8) { + let (w, h) = (24usize, 16usize); + let data: Vec<[u8; 4]> = (0..w * h).map(|i| [(i % w * 9) as u8, (i / w * 13) as u8, seed, 255]).collect(); + let img = lightcraft_raster::Rgba8 { width: w, height: h, data }; + let bytes = lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap(); + std::fs::create_dir_all(Path::new(path).parent().unwrap()).unwrap(); + std::fs::write(path, bytes).unwrap(); +} + +fn sidecar(rating: u8) -> String { + format!( + r#" + +"# + ) +} + +/// Write `a.xmp` beside `a.png`, dated `age` before now. +fn write_sidecar(png: &str, rating: u8, age: Duration) { + let xmp = Path::new(png).with_extension("xmp"); + std::fs::write(&xmp, sidecar(rating)).unwrap(); + let f = std::fs::File::options().write(true).open(&xmp).unwrap(); + f.set_modified(SystemTime::now() - age).unwrap(); +} + +/// A library holding `trip/a.png` and `trip/day1/b.png`, the session clock at a fixed past time. +fn library(dir: &Scratch) -> Session { + write_png(&dir.path("trip/a.png"), 1); + write_png(&dir.path("trip/day1/b.png"), 2); + let mut s = Session::new().with_fs(); + s.execute("library.import", &json!({"paths": [dir.path("trip")]})).unwrap(); + assert_eq!(s.catalog.len(), 2); + s +} + +fn paths(v: &Value, key: &str) -> Vec { + let mut out: Vec = v[key].as_array().unwrap().iter().map(|c| c["path"].as_str().unwrap().to_string()).collect(); + out.sort(); + out +} + +/// What the library holds: each photo's file, whether it is in the library, and its rating. +fn holdings(s: &Session) -> Vec<(String, bool, u8)> { + let mut v: Vec<(String, bool, u8)> = s.catalog.photos().map(|p| (p.file_name.clone(), p.in_library(), p.rating)).collect(); + v.sort(); + v +} + +fn scan(s: &mut Session, path: &str) -> Value { + s.execute("folder.scanChanges", &json!({"path": path})).unwrap() +} + +#[test] +fn an_unchanged_folder_has_no_changes() { + let dir = Scratch::new("none"); + let mut s = library(&dir); + let r = scan(&mut s, &dir.path("trip")); + assert!(paths(&r, "new").is_empty() && paths(&r, "missing").is_empty() && paths(&r, "metadata").is_empty(), "{r}"); +} + +#[test] +fn files_added_to_the_folder_or_a_folder_inside_it_are_new() { + let dir = Scratch::new("new"); + let mut s = library(&dir); + write_png(&dir.path("trip/c.png"), 3); + write_png(&dir.path("trip/day2/d.png"), 4); + // the same bytes as a photo the library has: not new + std::fs::copy(dir.path("trip/a.png"), dir.path("trip/a-copy.png")).unwrap(); + let before = s.catalog.to_snapshot(); + let r = scan(&mut s, &dir.path("trip")); + assert_eq!(paths(&r, "new"), vec![dir.path("trip/c.png"), dir.path("trip/day2/d.png")], "{r}"); + assert_eq!(r["duplicates"], 1, "{r}"); + assert_eq!(s.catalog.to_snapshot(), before, "scanning changes nothing"); +} + +#[test] +fn a_photo_whose_file_is_gone_is_missing() { + let dir = Scratch::new("missing"); + let mut s = library(&dir); + std::fs::remove_file(dir.path("trip/day1/b.png")).unwrap(); + let r = scan(&mut s, &dir.path("trip")); + assert_eq!(paths(&r, "missing"), vec![dir.path("trip/day1/b.png")], "{r}"); + assert!(r["missing"][0]["id"].is_u64()); +} + +#[test] +fn a_sidecar_saved_by_another_app_is_a_metadata_update() { + let dir = Scratch::new("meta"); + let mut s = library(&dir); + // saved now: after the photo came in (the session clock says 2026-09-30) + write_sidecar(&dir.path("trip/a.png"), 4, Duration::ZERO); + let r = scan(&mut s, &dir.path("trip")); + assert_eq!(paths(&r, "metadata"), vec![dir.path("trip/a.png")], "{r}"); +} + +#[test] +fn an_old_sidecar_or_one_that_agrees_is_no_update() { + let dir = Scratch::new("meta-old"); + let mut s = library(&dir); + // older than the import: whatever it says, the library has had its say since + write_sidecar(&dir.path("trip/a.png"), 4, Duration::from_secs(400 * 24 * 3600)); + // new, but says what the library already knows + write_sidecar(&dir.path("trip/day1/b.png"), 0, Duration::ZERO); + let r = scan(&mut s, &dir.path("trip")); + assert!(paths(&r, "metadata").is_empty(), "{r}"); +} + +#[test] +fn synchronizing_imports_new_files_and_leaves_the_rest_unless_asked() { + let dir = Scratch::new("sync"); + let mut s = library(&dir); + write_png(&dir.path("trip/c.png"), 3); + std::fs::remove_file(dir.path("trip/day1/b.png")).unwrap(); + write_sidecar(&dir.path("trip/a.png"), 5, Duration::ZERO); + let r = s.execute("folder.synchronize", &json!({"path": dir.path("trip")})).unwrap(); + assert_eq!((r["imported"].as_u64(), r["removed"].as_u64(), r["read"].as_u64()), (Some(1), Some(0), Some(0)), "{r}"); + assert_eq!(s.catalog.photos().filter(|p| p.in_library()).count(), 3, "the missing photo stays"); + let a = s.catalog.photos().find(|p| p.file_name == "a.png").unwrap(); + assert_eq!(a.rating, 0, "metadata is read only when asked"); +} + +#[test] +fn synchronizing_everything_is_one_undo_step() { + let dir = Scratch::new("sync-all"); + let mut s = library(&dir); + let before = holdings(&s); + write_png(&dir.path("trip/c.png"), 3); + std::fs::remove_file(dir.path("trip/day1/b.png")).unwrap(); + write_sidecar(&dir.path("trip/a.png"), 5, Duration::ZERO); + let r = + s.execute("folder.synchronize", &json!({"path": dir.path("trip"), "importNew": true, "removeMissing": true, "readMetadata": true})).unwrap(); + assert_eq!((r["imported"].as_u64(), r["removed"].as_u64(), r["read"].as_u64()), (Some(1), Some(1), Some(1)), "{r}"); + let library: Vec = s.catalog.photos().filter(|p| p.in_library()).map(|p| p.file_name.clone()).collect(); + assert_eq!(library.len(), 2, "b went to Recently Deleted: {library:?}"); + assert_eq!(s.catalog.photos().find(|p| p.file_name == "a.png").unwrap().rating, 5, "the sidecar was read"); + let undo = s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(undo["undone"], "Synchronize Folder"); + assert_eq!(holdings(&s), before, "one undo step puts everything back"); +} + +#[test] +fn a_folder_with_no_changes_synchronizes_without_an_undo_step() { + let dir = Scratch::new("sync-none"); + let mut s = library(&dir); + let steps = s.undo.len(); + let r = s.execute("folder.synchronize", &json!({"path": dir.path("trip"), "removeMissing": true, "readMetadata": true})).unwrap(); + assert_eq!((r["imported"].as_u64(), r["removed"].as_u64(), r["read"].as_u64()), (Some(0), Some(0), Some(0)), "{r}"); + assert_eq!(s.undo.len(), steps); +} + +#[test] +fn only_a_folder_of_the_library_is_synchronized() { + let dir = Scratch::new("refused"); + let mut s = library(&dir); + std::fs::create_dir_all(dir.0.join("elsewhere")).unwrap(); + for (cmd, p, why) in [ + ("folder.scanChanges", json!({}), "missing path"), + ("folder.scanChanges", json!({"path": " "}), "blank path"), + ("folder.scanChanges", json!({"path": dir.path("elsewhere")}), "no photo was imported from it"), + ("folder.scanChanges", json!({"path": "/"}), "the startup disk"), + ("folder.synchronize", json!({"path": dir.path("elsewhere")}), "no photo was imported from it"), + ("folder.synchronize", json!({"path": "/"}), "the startup disk"), + ] { + assert!(s.execute(cmd, &p).is_err(), "{cmd}: {why}"); + } +}