From 2783e5de9b3424d83859490d91972b695c027ccf Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Tue, 13 Oct 2026 10:52:56 -0400 Subject: [PATCH] PERF-IMPORT: a scan, an import and a sidecar write no longer make a table of the whole library first Setting up Synchronize Folder, an import review or an import made a table of every photo in the library on the session's thread before the first file was looked at, and naming one photo's sidecar worked out who owns every shared sidecar name in the library. In a library of 3.5 million photos (`scan_setup_bench`, release, a folder of 500): 5.8 s before a Synchronize scan, 2.6 s before an import review, 1.7 s before an import, 2.4 s for each sidecar written. - A scan keeps the library as it is (photos are shared, not copied) and looks up the files it finds (`library_files`) and the contents it probes (`library_hashes`) on its own thread. - An import looks the library up with its first files (`ImportJob::look_up_library`), from the library and the options as they were when it was set up. - The owners of shared sidecar names are worked out for the photos asked about (`StemOwners::among`): all five callers name only those photos. - Synchronize Folder names its photos' sidecars in the scan, after it has seen that the folder is there. Now 0.9 s, 0.05 s, 0.05 s and under 1 s. What is found, and how sidecars are named, is unchanged: the tests hold the owners against the table of the whole library as it was made before, and the key made without allocating against the key, for every string of up to six of the characters that decide it. Co-Authored-By: Claude Opus 5.5 --- crates/engine/examples/scan_setup_bench.rs | 63 +++++++ crates/engine/src/cmd/xmp.rs | 5 +- crates/engine/src/import.rs | 147 +++++++++++----- crates/engine/src/import/tests_known.rs | 134 ++++++++++++++ crates/engine/src/sidecar.rs | 99 +++++++++-- .../engine/src/sidecar/tests_stem_owners.rs | 163 ++++++++++++++++++ crates/engine/src/sync.rs | 26 ++- docs/whats-new.md | 10 ++ 8 files changed, 578 insertions(+), 69 deletions(-) create mode 100644 crates/engine/examples/scan_setup_bench.rs create mode 100644 crates/engine/src/import/tests_known.rs create mode 100644 crates/engine/src/sidecar/tests_stem_owners.rs diff --git a/crates/engine/examples/scan_setup_bench.rs b/crates/engine/examples/scan_setup_bench.rs new file mode 100644 index 000000000..e9d8f2957 --- /dev/null +++ b/crates/engine/examples/scan_setup_bench.rs @@ -0,0 +1,63 @@ +//! What a scan, an import and Synchronize Folder cost before they read the disk, and what +//! naming one photo's sidecar costs, by the size of the library (`N`, default 100,000 photos; the +//! folder looked at holds 500 of them). +//! +//! `N=3500000 cargo run --release -p lightcraft-engine --example scan_setup_bench` +use std::time::Instant; + +use lightcraft_catalog::{Op, Photo, Source}; +use lightcraft_engine::Session; + +fn main() { + let n: u64 = std::env::var("N").ok().and_then(|v| v.parse().ok()).unwrap_or(100_000); + let mut s = Session::new(); + let t = Instant::now(); + for chunk in (0..n).collect::>().chunks(10_000) { + for i in chunk { + let id = s.catalog.alloc_photo_id(); + let name = format!("IMG_{i:08}.jpg"); + let path = format!("/Volumes/Photos/Library/Archive/2019/2019-07 Summer/Trip to the coast/Day {:05}/extra/{name}", i / 500); + let mut p = Photo::new(id, Source::File { path }, &name, "JPEG", 4032, 3024, "2026-09-30T12:00:00"); + p.file_size = 3_400_000 + i; + p.captured = Some(format!("2019-07-{:02}T10:{:02}:{:02}", 1 + i % 28, i % 60, (i / 60) % 60)); + p.content_hash = Some(format!("{:032x}", i.wrapping_mul(0x9e37_79b9_7f4a_7c15))); + s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + } + } + eprintln!("{n} photos built in {:.2} s", t.elapsed().as_secs_f64()); + let folder = "/Volumes/Photos/Library/Archive/2019/2019-07 Summer/Trip to the coast/Day 00000".to_string(); + for run in 0..3 { + let ms = |t: Instant| t.elapsed().as_secs_f64() * 1e3; + let t = Instant::now(); + let f = lightcraft_catalog::Filter { library_folder: Some(folder.clone()), ..Default::default() }; + let ids = s.catalog.query(&f, &lightcraft_catalog::Sort::default()); + let query = ms(t); + let t = Instant::now(); + let namings = s.sidecar_namings(&ids); + let naming = ms(t); + let t = Instant::now(); + let one = ids.first().map(|id| s.sidecar_naming(*id)); + let naming_one = ms(t); + let t = Instant::now(); + let (input, paths) = lightcraft_engine::import::ScanInput::new(&mut s, std::slice::from_ref(&folder)); + let scan = ms(t); + // (the folder is not on this disk: the scan finds no file, and its cost is what it makes of the library) + let t = Instant::now(); + let out = lightcraft_engine::import::scan_with(input, &paths, &Default::default()); + let scanned = ms(t); + let t = Instant::now(); + let job = lightcraft_engine::import::ImportJob::new(&mut s, Default::default()); + let import = ms(t); + let t = Instant::now(); + let sync = lightcraft_engine::sync::SyncInput::new(&mut s, &folder, false); + let whole = ms(t); + println!( + "N={n} run {run}: query {query:.0} ms ({} ids) | sidecar_namings {naming:.0} ms | sidecar_naming of one {naming_one:.0} ms ({one:?}) | ScanInput::new {scan:.0} ms + scan_with {scanned:.0} ms ({} found) | ImportJob::new {import:.0} ms (ok {}) | SyncInput::new {whole:.0} ms (ok {})", + ids.len(), + out.candidates.len(), + job.is_ok(), + sync.is_ok() + ); + let _ = namings; + } +} diff --git a/crates/engine/src/cmd/xmp.rs b/crates/engine/src/cmd/xmp.rs index 6af62637f..cd89aa896 100644 --- a/crates/engine/src/cmd/xmp.rs +++ b/crates/engine/src/cmd/xmp.rs @@ -12,8 +12,9 @@ fn save(s: &mut Session, p: &Value) -> Result { let mut merged = Vec::new(); let mut backups = Vec::new(); let mut failed = Vec::new(); - let owners = crate::sidecar::StemOwners::of(&s.catalog); - for id in s.targets(p) { + let ids = s.targets(p); + let owners = crate::sidecar::StemOwners::among(&s.catalog, &ids); + for id in ids { match s.save_sidecar_with(id, &owners) { Ok(r) => { let path = r.path.display().to_string(); diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 1f008e766..418fbca66 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -548,12 +548,62 @@ fn copy_into(dir: &Path, src: &str, name: Option<&str>, probe_hash: Option<&str> pub struct ScanInput { probe: Option, skip: Option, - /// path → (photo, its summary) for files already in the library. - by_path: HashMap, - by_hash: HashMap, + /// The library as it is now (photos are shared with the session, not copied): looked up for + /// the files the scan finds ([`library_files`]) and the contents it probes + /// ([`library_hashes`]), on the scan's thread. + library: std::sync::Arc, cache: HashMap, } +/// The library's photos of `files`: path → (photo, its summary as a candidate that is there +/// already). One pass over the library, and a record only for a file that was found: a folder of +/// a hundred files in a library of millions costs a hundred records. +fn library_files(library: &lightcraft_catalog::Catalog, files: &[String]) -> HashMap { + let found: std::collections::HashSet<&Path> = files.iter().map(Path::new).collect(); + let mut by_path = HashMap::new(); + if found.is_empty() { + return by_path; + } + for p in library.photos() { + if let Source::File { path } = &p.source + && found.contains(Path::new(path)) + { + let c = ImportCandidate { + path: path.clone(), + format: p.format.clone(), + kind: p.kind, + width: p.width, + height: p.height, + file_size: p.file_size, + captured: p.captured.clone(), + duplicate: Some("path".into()), + existing: Some(p.id.0), + ..Default::default() + }; + by_path.insert(PathBuf::from(path), (p.id, c)); + } + } + by_path +} + +/// Which photo of the library has each of the contents `hashes` (the later one where several +/// have it). One pass over the library, and an entry only for a content that was asked about. +fn library_hashes<'a>(library: &lightcraft_catalog::Catalog, hashes: impl Iterator) -> HashMap { + let wanted: std::collections::HashSet<&str> = hashes.collect(); + let mut by_hash = HashMap::new(); + if wanted.is_empty() { + return by_hash; + } + for p in library.photos() { + if let Some(h) = &p.content_hash + && wanted.contains(h.as_str()) + { + by_hash.insert(h.clone(), p.id); + } + } + by_hash +} + /// Progress and cancellation of a running [`scan_with`]. #[derive(Default)] pub struct ScanProgress { @@ -573,31 +623,17 @@ pub struct ScanOutput { impl ScanInput { pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec) { let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); - let mut by_path = HashMap::new(); - let mut by_hash = HashMap::new(); - for p in s.catalog.photos() { - if let Source::File { path } = &p.source { - let c = ImportCandidate { - path: path.clone(), - format: p.format.clone(), - kind: p.kind, - width: p.width, - height: p.height, - file_size: p.file_size, - captured: p.captured.clone(), - duplicate: Some("path".into()), - existing: Some(p.id.0), - ..Default::default() - }; - by_path.insert(PathBuf::from(path), (p.id, c)); - } - if let Some(h) = &p.content_hash { - by_hash.insert(h.clone(), p.id); - } - } - let input = ScanInput { probe: s.media.file_probe.clone(), skip, by_path, by_hash, cache: std::mem::take(&mut s.import_probes) }; + // (no table of the library here: this is the session's thread, and the library may hold + // millions of photos. The scan looks up what it finds) + let library = std::sync::Arc::new(s.catalog.clone()); + let input = ScanInput { probe: s.media.file_probe.clone(), skip, library, cache: std::mem::take(&mut s.import_probes) }; (input, paths.to_vec()) } + + /// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one). + pub(crate) fn library(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.library) + } } /// Find and probe what importing `paths` would add, marking duplicates (by path, or by content @@ -615,7 +651,8 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec { pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput { use std::sync::atomic::Ordering::Relaxed; let files = expand(paths, input.skip.as_deref()); - let todo: Vec = files.iter().filter(|f| !input.by_path.contains_key(Path::new(f))).cloned().collect(); + let by_path = library_files(&input.library, &files); + let todo: Vec = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect(); progress.total.store(todo.len(), Relaxed); // probes from a preceding `scan` are reused when the file is unchanged (same size) let cached: Vec> = todo @@ -633,11 +670,14 @@ pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); crate::memory::release(); let mut probes: HashMap> = todo.into_iter().zip(probed).collect(); - let mut seen_hash: HashMap> = input.by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect(); + // (who in the library has a content that was probed: asked once the probes are in) + let probed_hashes = probes.values().filter_map(|r| r.as_ref().ok()).filter_map(|info| info.content_hash.as_deref()); + let mut seen_hash: HashMap> = + library_hashes(&input.library, probed_hashes).into_iter().map(|(h, id)| (h, Some(id.0))).collect(); let mut out = Vec::with_capacity(files.len()); let mut kept = HashMap::new(); for f in files { - if let Some((_, c)) = input.by_path.get(Path::new(&f)) { + if let Some((_, c)) = by_path.get(Path::new(&f)) { let mut c = c.clone(); c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); out.push(c); @@ -701,6 +741,12 @@ pub struct ImportJob { mode: ImportMode, lib_dir: Option, copy_root: Option, + /// The library as it was when the import was set up, until it is looked up + /// ([`ImportJob::look_up_library`], with the first files): setting an import up is the + /// session's thread, and a table of a library of millions of photos takes seconds. With it, + /// whether a photo in Recently Deleted is passed over (an import that replaces such photos, + /// as the options said when the import was set up). + library: Option<(lightcraft_catalog::Catalog, bool)>, /// path → (photo, a Local browse record?) by_path: HashMap, /// content hash → the photo that has it (`None`: a file earlier in this import) @@ -842,28 +888,16 @@ impl ImportJob { if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() { return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into())); } - let mut by_path = HashMap::new(); - let mut by_hash = HashMap::new(); - for p in s.catalog.photos() { - // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate - if p.deleted && opts.on_deleted == OnDeleted::Fresh { - continue; - } - if let Source::File { path } = &p.source { - by_path.insert(path.clone(), (p.id, p.local)); - } - if let Some(h) = &p.content_hash { - by_hash.insert(h.clone(), Some(p.id)); - } - } + let fresh = opts.on_deleted == OnDeleted::Fresh; Ok(ImportJob { seq: opts.rename_start.max(1), opts, mode, lib_dir, copy_root, - by_path, - by_hash, + library: Some((s.catalog.clone(), fresh)), + by_path: HashMap::new(), + by_hash: HashMap::new(), probe: s.media.file_probe.clone(), file_bytes: s.media.file_bytes.clone(), cache: std::mem::take(&mut s.import_probes), @@ -898,9 +932,28 @@ impl ImportJob { &self.now } + /// What the library held when the import was set up, by path and by content: made once, with + /// the first files (on the thread that prepares them), from the library as it was then. + fn look_up_library(&mut self) { + let Some((library, fresh)) = self.library.take() else { return }; + for p in library.photos() { + // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate + if p.deleted && fresh { + continue; + } + if let Source::File { path } = &p.source { + self.by_path.insert(path.clone(), (p.id, p.local)); + } + if let Some(h) = &p.content_hash { + self.by_hash.insert(h.clone(), Some(p.id)); + } + } + } + /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned. pub fn prepare_files(&mut self, files: Vec, cancel: &std::sync::atomic::AtomicBool) -> Prepared { use std::sync::atomic::Ordering::Relaxed; + self.look_up_library(); let opts = self.opts.clone(); let mode = self.mode; let mut out = Prepared { scanned: files.len(), items: Vec::new() }; @@ -1494,6 +1547,10 @@ impl Session { } } +#[cfg(test)] +#[path = "import/tests_known.rs"] +mod tests_known; + #[cfg(test)] mod prepared_tests { use super::*; diff --git a/crates/engine/src/import/tests_known.rs b/crates/engine/src/import/tests_known.rs new file mode 100644 index 000000000..9adadca8a --- /dev/null +++ b/crates/engine/src/import/tests_known.rs @@ -0,0 +1,134 @@ +//! What a scan or an import knows of the library: worked out for the files it meets, and not on +//! the session's thread. A library can hold millions of photos; looking at one folder must not +//! cost a table of them all before the first file is read. +use super::*; +use lightcraft_catalog::{Catalog, Op}; + +/// A library of `folders` folders of three photos each, the third a copy (by content) of a +/// photo in the folder before. +fn library(folders: usize) -> Catalog { + let mut c = Catalog::new(); + for folder in 0..folders { + for (n, name) in ["a.jpg", "b.jpg", "c.jpg"].into_iter().enumerate() { + let id = c.alloc_photo_id(); + let mut p = Photo::new(id, Source::File { path: format!("/pics/{folder:03}/{name}") }, name, "JPEG", 6, 4, "2026-01-01"); + p.content_hash = Some(format!("{:032x}", if n == 2 { folder.saturating_sub(1) * 3 } else { folder * 3 + n })); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + } + } + c +} + +/// Given a library of many folders, when a scan meets the files of one of them (and one the +/// library does not have), then what it knows of the library is those files' photos, each as the +/// library has it, and nothing of the other folders. +#[test] +fn a_scan_knows_the_library_photos_of_the_files_it_found() { + let c = library(100); + let files: Vec = ["/pics/007/a.jpg", "/pics/007/b.jpg", "/pics/007/new.jpg", "/pics/007//c.jpg"].map(String::from).to_vec(); + let known = library_files(&c, &files); + assert_eq!(known.len(), 3, "the three the library has (a path is compared as a path, not as text)"); + let (id, as_found) = known.get(Path::new("/pics/007/b.jpg")).expect("b"); + let p = c.photo(*id).unwrap(); + assert_eq!((as_found.path.as_str(), as_found.existing, as_found.duplicate.as_deref()), ("/pics/007/b.jpg", Some(id.0), Some("path"))); + assert_eq!((as_found.format.as_str(), as_found.width, as_found.height, as_found.file_size), (p.format.as_str(), p.width, p.height, p.file_size)); + assert!(library_files(&c, &[]).is_empty()); + // two photos of one file (a virtual copy): the later one is the one named, as a table of the + // whole library named it + let mut c = c; + let copy = c.alloc_photo_id(); + c.apply(Op::AddPhoto { photo: Box::new(Photo::new(copy, Source::File { path: "/pics/007/a.jpg".into() }, "a.jpg", "JPEG", 6, 4, "2026-01-02")) }) + .unwrap(); + assert_eq!(library_files(&c, &files).get(Path::new("/pics/007/a.jpg")).map(|(id, _)| *id), Some(copy)); +} + +/// Given a library and a folder on disk that holds a file the library has by its path, one it +/// has by its content under another name, and a new one, when the folder is scanned (the whole +/// scan, as the import review runs it), then each is found for what it is. +#[test] +fn a_scan_finds_what_the_library_has_by_path_and_by_content() { + let dir = std::env::temp_dir().join(format!("lc-known-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let at = |name: &str| dir.join(name).to_string_lossy().to_string(); + for name in ["here.jpg", "copy.jpg", "new.jpg"] { + std::fs::write(dir.join(name), name.as_bytes()).unwrap(); + } + let mut s = Session::new(); + s.catalog = library(20); + let here = s.catalog.alloc_photo_id(); + s.catalog + .apply(Op::AddPhoto { photo: Box::new(Photo::new(here, Source::File { path: at("here.jpg") }, "here.jpg", "JPEG", 6, 4, "2026-01-01")) }) + .unwrap(); + // (the probe: every file is a picture whose content is its name; `copy.jpg` has the content of a library photo) + let had = s.catalog.photos().find(|p| p.file_name == "b.jpg").map(|p| (p.id, p.content_hash.clone().unwrap())).unwrap(); + let shared = had.1.clone(); + s.media.file_probe = Some(std::sync::Arc::new(move |path: &str| { + let name = Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let content_hash = Some(if name == "copy.jpg" { shared.clone() } else { format!("hash-of-{name}") }); + Ok(ProbeInfo { format: "JPEG".into(), width: 6, height: 4, file_size: 8, content_hash, ..Default::default() }) + })); + let found = scan(&mut s, &[dir.to_string_lossy().to_string()]); + let of = |name: &str| found.iter().find(|c| c.name == name).map(|c| (c.duplicate.as_deref(), c.existing)).unwrap(); + assert_eq!(of("here.jpg"), (Some("path"), Some(here.0))); + assert_eq!(of("copy.jpg"), (Some("content"), Some(had.0.0))); + assert_eq!(of("new.jpg"), (None, None)); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Given the hashes of the files a scan probed, when the library is asked who has them, then it +/// answers for those hashes only; and where two photos have one content, the same one of them +/// as a table of the whole library would name (the later one). +#[test] +fn a_scan_knows_who_has_the_contents_it_probed() { + let c = library(100); + // (the content of /pics/006/a.jpg, which /pics/007/c.jpg has too; and one nobody has) + let shared = format!("{:032x}", 6 * 3); + let nobody = format!("{:032x}", 999_999); + let known = library_hashes(&c, [shared.as_str(), nobody.as_str()].into_iter()); + assert_eq!(known.len(), 1); + let whole: HashMap = c.photos().filter_map(|p| p.content_hash.clone().map(|h| (h, p.id))).collect(); + assert_eq!(known.get(&shared), whole.get(&shared)); + assert!(matches!(&c.photo(known[&shared]).unwrap().source, Source::File { path } if path == "/pics/007/c.jpg")); +} + +/// Given a session with a library, when an import is set up, then no table of the library is +/// made on the spot (that is the session's thread); it is made when the first files are +/// prepared, and then finds a file the library has by its path and one it has by its content. +#[test] +fn an_import_looks_the_library_up_when_it_prepares_files() { + let mut s = Session::new(); + s.catalog = library(50); + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap(); + assert!(job.by_path.is_empty() && job.by_hash.is_empty() && job.library.is_some(), "nothing is looked up yet"); + let out = job.prepare_files(vec!["/pics/007/a.jpg".into()], &std::sync::atomic::AtomicBool::new(false)); + assert!(job.library.is_none(), "looked up once"); + assert_eq!((job.by_path.len(), job.by_hash.len()), (150, 100)); + assert!(matches!(out.items.as_slice(), [PreparedItem::Duplicate { reason: "path", existing: Some(_), .. }]), "found by its path"); + // what the options said when the import was set up decides, whatever is made of them after + let id = s.catalog.photos().next().unwrap().id; + s.catalog.apply(Op::SetDeleted { id, deleted: true }).unwrap(); + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, on_deleted: OnDeleted::Fresh, ..ImportOptions::default() }).unwrap(); + job.opts.on_deleted = OnDeleted::Skip; + job.prepare_files(Vec::new(), &std::sync::atomic::AtomicBool::new(false)); + assert_eq!(job.by_path.len(), 149, "the trashed photo was passed over, as set up"); +} + +/// Given a photo in Recently Deleted and an import that replaces such photos with a fresh +/// import, when the library is looked up, then that photo is no duplicate (as before the lookup +/// moved to the first files). +#[test] +fn a_trashed_photo_is_not_looked_up_when_imports_replace_it() { + let mut s = Session::new(); + s.catalog = library(2); + // (a photo whose content is its own: nobody else in this library has it) + let id = s.catalog.photos().find(|p| p.file_name == "b.jpg").unwrap().id; + s.catalog.apply(Op::SetDeleted { id, deleted: true }).unwrap(); + for (on_deleted, expected) in [(OnDeleted::Fresh, 5), (OnDeleted::Skip, 6)] { + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, on_deleted, ..ImportOptions::default() }).unwrap(); + job.prepare_files(Vec::new(), &std::sync::atomic::AtomicBool::new(false)); + assert_eq!(job.by_path.len(), expected, "{on_deleted:?}"); + let its_content = s.catalog.photo(id).and_then(|p| p.content_hash.clone()).unwrap(); + assert_eq!(job.by_hash.contains_key(&its_content), on_deleted == OnDeleted::Skip, "{on_deleted:?}"); + } +} diff --git a/crates/engine/src/sidecar.rs b/crates/engine/src/sidecar.rs index 9aaf7e937..c67a2c84f 100644 --- a/crates/engine/src/sidecar.rs +++ b/crates/engine/src/sidecar.rs @@ -450,6 +450,33 @@ fn stem_key(original: &str) -> String { sidecar_path(original, SidecarNaming::Stem).to_string_lossy().to_lowercase() } +/// Where a plain path is cut to make its [`stem_key`]: at the dot of its extension, which is +/// what `with_extension` replaces. A plain path is ASCII and ends in its file's name with an +/// extension, which is what a library's paths are; `None` for any other shape. +fn plain_cut(original: &str) -> Option { + let path = Path::new(original); + let (name, stem, ext) = (path.file_name()?.to_str()?, path.file_stem()?.to_str()?, path.extension()?.to_str()?); + // (the path ends in its file's name, as it stands: no `/.` or separator after it; and the + // name has a stem and an extension of its own, which `..jpg` and `a.` have not) + let plain = original.is_ascii() && original.ends_with(name) && !ext.is_empty() && !stem.is_empty() && stem != "." && stem != ".."; + let cut = original.len().checked_sub(ext.len().saturating_add(1))?; + (plain && original.as_bytes().get(cut) == Some(&b'.')).then_some(cut) +} + +/// [`stem_key`] written into `key`, which is reused: nothing is allocated for a plain path +/// ([`plain_cut`]). Any other shape goes the long way. +fn stem_key_into(key: &mut String, original: &str) { + key.clear(); + match plain_cut(original) { + Some(cut) => { + key.push_str(original.get(..cut).unwrap_or_default()); + key.make_ascii_lowercase(); + key.push_str(".xmp"); + } + None => key.push_str(&stem_key(original)), + } +} + fn file_path(p: &Photo) -> Option<&str> { match &p.source { Source::File { path } => Some(path), @@ -490,8 +517,11 @@ pub(crate) fn op_photos(op: &Op, out: &mut Vec) { pub(crate) struct StemOwners(HashMap); impl StemOwners { + /// The owners of every shared stem in the library, each file's key made the long way: what + /// [`StemOwners::among`] must agree with for the photos it is asked about (this is how the + /// owners were worked out before, for every write). + #[cfg(test)] pub(crate) fn of(cat: &lightcraft_catalog::Catalog) -> StemOwners { - // per stem: (raw first, then file name, then id) of the best candidate; and how many let mut best: HashMap = HashMap::new(); for p in cat.photos().filter(|p| p.copy_of.is_none()) { let Some(path) = file_path(p) else { continue }; @@ -505,6 +535,39 @@ impl StemOwners { StemOwners(best.into_iter().filter(|(_, v)| v.2 > 1).map(|(k, v)| (k, v.1)).collect()) } + /// The owners of the stems photos `ids` map to: all that naming those photos needs. The + /// library is passed over once, and only files of those stems are looked at closer, so a + /// sidecar written for one photo does not cost a table of the whole library. + pub(crate) fn among(cat: &lightcraft_catalog::Catalog, ids: &[PhotoId]) -> StemOwners { + let wanted: std::collections::HashSet = + ids.iter().filter_map(|id| cat.photo(*id)).filter_map(|p| file_path(p)).map(stem_key).collect(); + if wanted.is_empty() { + return StemOwners(HashMap::new()); + } + Self::where_wanted(cat, &wanted) + } + + /// The owners of the shared stems in `wanted`. + fn where_wanted(cat: &lightcraft_catalog::Catalog, wanted: &std::collections::HashSet) -> StemOwners { + // per stem: (raw first, then file name, then id) of the best candidate; and how many + let mut best: HashMap = HashMap::new(); + let mut key = String::new(); + for p in cat.photos().filter(|p| p.copy_of.is_none()) { + let Some(path) = file_path(p) else { continue }; + stem_key_into(&mut key, path); + if !wanted.contains(key.as_str()) { + continue; + } + let rank = (p.kind != MediaKind::Raw, path.to_lowercase(), p.id.0); + let e = best.entry(key.clone()).or_insert_with(|| (rank.clone(), p.id, 0)); + e.2 += 1; + if rank < e.0 { + (e.0, e.1) = (rank, p.id); + } + } + StemOwners(best.into_iter().filter(|(_, v)| v.2 > 1).map(|(k, v)| (k, v.1)).collect()) + } + /// `naming` for photo `p`, except that a stem sidecar another file owns becomes Full. fn naming(&self, p: &Photo, naming: SidecarNaming) -> SidecarNaming { match (naming, file_path(p)) { @@ -514,33 +577,39 @@ impl StemOwners { } } +/// How the sidecar of each of photos `ids` is named in library `cat`, whose naming is `naming` +/// ([`Session::sidecar_namings`], for a thread that has the library and not the session). +pub(crate) fn namings_in(cat: &lightcraft_catalog::Catalog, naming: SidecarNaming, ids: &[PhotoId]) -> Vec { + let owners = (naming == SidecarNaming::Stem).then(|| StemOwners::among(cat, ids)); + ids.iter() + .map(|id| match (cat.photo(*id), &owners) { + (Some(p), Some(o)) => o.naming(p, naming), + _ => naming, + }) + .collect() +} + impl Session { /// The sidecar naming for photo `id`: the library's, except under Stem naming when another /// catalogued file shares the stem (`IMG_0001.CR3` + `IMG_0001.JPG`) and owns the stem /// sidecar — a raw first, else the first by file name. The others use Full naming /// (`IMG_0001.JPG.xmp`), so their metadata never overwrites each other. - /// [`Session::sidecar_naming`] for many photos, the catalog's sidecar owners worked out once - /// (per photo that is a pass over the whole catalog). + /// [`Session::sidecar_naming`] for many photos, their stems' owners worked out once (per + /// photo that is a pass over the whole catalog). pub fn sidecar_namings(&self, ids: &[PhotoId]) -> Vec { - let owners = (self.xmp.naming == SidecarNaming::Stem).then(|| StemOwners::of(&self.catalog)); - ids.iter() - .map(|id| match (self.catalog.photo(*id), &owners) { - (Some(p), Some(o)) => o.naming(p, self.xmp.naming), - _ => self.xmp.naming, - }) - .collect() + namings_in(&self.catalog, self.xmp.naming, ids) } pub fn sidecar_naming(&self, id: PhotoId) -> SidecarNaming { match self.catalog.photo(id) { - Some(p) if self.xmp.naming == SidecarNaming::Stem => StemOwners::of(&self.catalog).naming(p, self.xmp.naming), + Some(p) if self.xmp.naming == SidecarNaming::Stem => StemOwners::among(&self.catalog, &[id]).naming(p, self.xmp.naming), _ => self.xmp.naming, } } /// Write the photo's XMP sidecar, merged into an existing one (see the module docs). pub fn save_sidecar(&mut self, id: PhotoId) -> Result { - self.save_sidecar_with(id, &StemOwners::of(&self.catalog)) + self.save_sidecar_with(id, &StemOwners::among(&self.catalog, &[id])) } pub(crate) fn save_sidecar_with(&mut self, id: PhotoId, owners: &StemOwners) -> Result { @@ -642,7 +711,7 @@ impl Session { if ids.is_empty() { return; } - let owners = StemOwners::of(&self.catalog); + let owners = StemOwners::among(&self.catalog, &ids); for id in ids { if self.catalog.photo(id).is_some_and(|p| file_path(p).is_some() && p.copy_of.is_none()) && let Err(e) = self.save_sidecar_with(id, &owners) @@ -764,3 +833,7 @@ mod tests_repair_companions; #[cfg(test)] #[path = "sidecar/tests_repair_provenance.rs"] mod tests_repair_provenance; + +#[cfg(test)] +#[path = "sidecar/tests_stem_owners.rs"] +mod tests_stem_owners; diff --git a/crates/engine/src/sidecar/tests_stem_owners.rs b/crates/engine/src/sidecar/tests_stem_owners.rs new file mode 100644 index 000000000..43170f4f6 --- /dev/null +++ b/crates/engine/src/sidecar/tests_stem_owners.rs @@ -0,0 +1,163 @@ +//! Who owns a stem sidecar, worked out for the photos asked about only (`StemOwners::among`). +//! +//! A library can hold millions of photos; a sidecar that is written for one of them, or a folder +//! that is synchronized, must not cost a table of them all. +use super::*; +use lightcraft_catalog::{Catalog, Op, Photo, Source}; + +/// Paths that try the key: case in folders, names and endings, no ending, a dot file, endings +/// inside folder names, non-ASCII letters (one that lower-cases to ASCII: the Kelvin sign). +const PATHS: [&str; 26] = [ + "/pics/trip/IMG_1.CR3", + "/pics/trip/IMG_1.JPG", + "/pics/trip/img_1.jpeg", + "/Pics/Trip/IMG_1.tif", + "/pics/trip/IMG_2.JPG", + "/pics/trip/IMG_3.CR3", + "/pics/trip/IMG_3.cr3.JPG", + "/pics/trip/IMG_3.cr3", + "/pics/other/IMG_1.JPG", + "/pics/other/IMG_1", + "/pics/other/.hidden", + "/pics/other/.hidden.jpg", + "/pics/v1.2/a.b.JPG", + "/pics/v1.2/a.b.cr3", + "/pics/v1.2/a", + "/pics/straße/Größe.JPG", + "/pics/STRASSE/GRÖSSE.cr3", + "/pics/straße/größe.CR3", + "/pics/kelvin/\u{212a}1.JPG", + "/pics/kelvin/k1.cr3", + "/pics/ΟΔΟΣ/ΟΔΟΣ.JPG", + "/pics/οδος/οδος.CR3", + // (names that are all dots before their ending: the ending is no extension to replace) + "/pics/dots/..cr3", + "/pics/dots/..jpg", + "/pics/dots/a./.", + "/pics/dots/a.jpg", +]; + +fn library() -> (Catalog, Vec) { + let mut c = Catalog::new(); + let mut ids = Vec::new(); + for path in PATHS { + let id = c.alloc_photo_id(); + let name = path.rsplit('/').next().unwrap_or(path); + let mut p = Photo::new(id, Source::File { path: path.to_string() }, name, "X", 6, 4, "2026-01-01"); + if path.to_lowercase().ends_with("cr3") { + p.kind = MediaKind::Raw; + } + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + ids.push(id); + } + // a virtual copy of the first (it owns nothing) and a photo that is no file + let copy = c.alloc_photo_id(); + let mut p = Photo::new(copy, Source::File { path: PATHS[0].to_string() }, "IMG_1.CR3", "X", 6, 4, "2026-01-01"); + p.copy_of = Some(ids[0]); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + ids.push(copy); + let demo = c.alloc_photo_id(); + c.apply(Op::AddPhoto { photo: Box::new(Photo::new(demo, Source::Demo { scene: 1 }, "demo", "X", 6, 4, "2026-01-01")) }).unwrap(); + ids.push(demo); + (c, ids) +} + +fn namings(owners: &StemOwners, c: &Catalog, ids: &[PhotoId]) -> Vec { + ids.iter().map(|id| owners.naming(c.photo(*id).unwrap(), SidecarNaming::Stem)).collect() +} + +/// Given a library where files share stems in every way a name can, when the owners are worked +/// out for some of its photos only, then each of those photos is named exactly as the table of +/// the whole library names it: whichever photos are asked about, alone or together. +#[test] +fn owners_among_some_photos_name_them_as_the_whole_library_does() { + let (c, ids) = library(); + let whole = StemOwners::of(&c); + let expected = namings(&whole, &c, &ids); + assert!(expected.contains(&SidecarNaming::Full) && expected.contains(&SidecarNaming::Stem), "both cases are there: {expected:?}"); + // all at once + assert_eq!(namings(&StemOwners::among(&c, &ids), &c, &ids), expected); + // one at a time, and every pair + for (i, a) in ids.iter().enumerate() { + assert_eq!(namings(&StemOwners::among(&c, &[*a]), &c, &[*a]), [expected[i]], "{:?}", PATHS.get(i)); + for (j, b) in ids.iter().enumerate() { + assert_eq!(namings(&StemOwners::among(&c, &[*a, *b]), &c, &[*a, *b]), [expected[i], expected[j]]); + } + } + // an id that is not in the library is passed over + assert_eq!(StemOwners::among(&c, &[PhotoId(9_999)]).0.len(), 0); +} + +/// Given a library of many folders, when the owners are worked out for the photos of one, then +/// the table holds that folder's shared stems and nothing of the others: what it costs follows +/// the photos asked about, not the library. +#[test] +fn owners_among_some_photos_hold_nothing_of_the_rest() { + let mut c = Catalog::new(); + let mut first = Vec::new(); + for folder in 0..200 { + for name in ["IMG_1.CR3", "IMG_1.JPG", "IMG_2.JPG"] { + let id = c.alloc_photo_id(); + let mut p = Photo::new(id, Source::File { path: format!("/pics/{folder:03}/{name}") }, name, "X", 6, 4, "2026-01-01"); + if name.ends_with("CR3") { + p.kind = MediaKind::Raw; + } + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + if folder == 0 { + first.push(id); + } + } + } + assert_eq!(StemOwners::of(&c).0.len(), 200, "one shared stem a folder"); + let some = StemOwners::among(&c, &first); + assert_eq!(some.0.len(), 1, "the one shared stem of the folder asked about"); + assert_eq!(namings(&some, &c, &first), [SidecarNaming::Stem, SidecarNaming::Full, SidecarNaming::Stem]); +} + +/// The key written into a buffer (no allocation for a plain path) is the key: for every path +/// above, for shapes a path from a file can have, and for every string of up to six of the +/// characters that decide a key (a letter in each case, a dot, both separators, a letter that is +/// not ASCII). +#[test] +fn the_key_in_a_buffer_is_the_key() { + let odd = [ + "", + "/", + "a", + "a.", + ".a", + "..a", + "...a", + "a.b", + "a/b.c/", + "a/b.c//", + "a/.b", + "a/..b", + "a/b.", + "a/b..c", + "a./.", + "a\\b.c", + "C:\\x\\y.JPG", + "a/b.c/.", + "a/b.c/..", + "x.JPG ", + "É.JPG", + ]; + let mut buf = String::new(); + for path in PATHS.iter().chain(&odd) { + stem_key_into(&mut buf, path); + assert_eq!(buf, stem_key(path), "{path:?}"); + } + let letters = ['a', 'B', '.', '/', '\\', 'é']; + let mut tried = 0usize; + let mut todo: Vec = vec![String::new()]; + while let Some(path) = todo.pop() { + stem_key_into(&mut buf, &path); + assert_eq!(buf, stem_key(&path), "{path:?}"); + tried += 1; + if path.chars().count() < 6 { + todo.extend(letters.iter().map(|c| format!("{path}{c}"))); + } + } + assert_eq!(tried, (0..=6).map(|n| 6usize.pow(n)).sum::()); +} diff --git a/crates/engine/src/sync.rs b/crates/engine/src/sync.rs index 0a00a59ab..4342ec6ad 100644 --- a/crates/engine/src/sync.rs +++ b/crates/engine/src/sync.rs @@ -74,10 +74,14 @@ impl SyncProgress { pub struct SyncInput { folder: String, scan: ScanInput, - /// The library's photos in the folder, each with how its sidecar is named. - photos: Vec<(Arc, SidecarNaming)>, - /// For the colour label names a sidecar may use. - labels: Catalog, + /// The library's photos in the folder. + ids: Vec, + /// How the library names sidecars; how each photo's is named is worked out by the scan (a + /// pass over the library, which the session's thread does not wait for). + naming: SidecarNaming, + /// The library as it is now (the one the scan looks files up in): for those photos, their + /// sidecars' names and the colour label names a sidecar may use. + labels: Arc, } /// What changed in a folder since it came into the library. @@ -176,10 +180,9 @@ impl SyncInput { if !disk && (lightcraft_catalog::folders::is_disk_root(path) || crate::cmd::library::covers_other_disks(s, path, &ids)) { return Err(invalid(format!("{path} is a whole disk or holds disks: pass `disk: true` to synchronize it"))); } - let namings = s.sidecar_namings(&ids); - let photos = ids.iter().zip(namings).filter_map(|(id, naming)| s.catalog.photo(*id).map(|p| (Arc::clone(p), naming))).collect(); let (scan, _) = ScanInput::new(s, &[path.to_string()]); - Ok(SyncInput { folder: path.to_string(), scan, photos, labels: s.catalog.clone() }) + let labels = scan.library(); + Ok(SyncInput { folder: path.to_string(), scan, ids, naming: s.xmp.naming, labels }) } } @@ -190,11 +193,16 @@ impl SyncInput { /// library's photos, which run on several threads too. Stops early when `progress.files.cancel` /// is set. pub fn scan_with(input: SyncInput, progress: &SyncProgress) -> FolderChanges { - let SyncInput { folder, scan, photos, labels } = input; - let seen: Vec> = photos.iter().map(|(p, _)| Arc::clone(p)).collect(); + let SyncInput { folder, scan, ids, naming, labels } = input; + let seen: Vec> = ids.iter().filter_map(|id| labels.photo(*id)).map(Arc::clone).collect(); if !cfg!(target_arch = "wasm32") && !Path::new(&folder).is_dir() { return FolderChanges { path: folder, seen, offline: true, ..Default::default() }; } + // (how each photo's sidecar is named: a pass over the library, made here and not while the + // session waits, and not at all for a folder that is not there) + let namings = crate::sidecar::namings_in(&labels, naming, &ids); + let photos: Vec<(Arc, SidecarNaming)> = + ids.iter().zip(namings).filter_map(|(id, naming)| labels.photo(*id).map(|p| (Arc::clone(p), naming))).collect(); // (no threads on the web: one after the other there) #[cfg(target_arch = "wasm32")] let (out, checks) = (crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files), check_photos(&photos, &labels, progress)); diff --git a/docs/whats-new.md b/docs/whats-new.md index b5c2ac68c..d85a760f0 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -2,6 +2,16 @@ ## 2026-10-13 +### Large libraries: Synchronize Folder, import and sidecars start sooner +- Opening Synchronize Folder, an import review or an import no longer makes a table of every photo in the library + before it looks at the first file. In a library of 3.5 million photos that held the window for about 6 s before + a Synchronize scan began (now under 1 s), 2.6 s before an import review (now 0.05 s) and 1.7 s before an import + (now 0.05 s). The scan looks up only the files it finds and the contents it reads, on its own thread. +- Writing one photo's XMP sidecar (Save Metadata to File, or automatically after an edit) no longer works out who + owns every shared sidecar name in the library: 2.4 s a write in that library, now under 1 s. +- Nothing else changes: the same files are found new, already there, moved or missing, and sidecars get the same + names. + ### Slider drags on large raws - Dragging a slider no longer re-renders the edited photo's filmstrip thumbnail on every frame; it follows once you let go. On a 64 MP raw the first of those renders decoded the whole file again, which held up the loupe's