diff --git a/crates/engine/examples/scan_setup_bench.rs b/crates/engine/examples/scan_setup_bench.rs new file mode 100644 index 000000000..e9d8f2957 --- /dev/null +++ b/crates/engine/examples/scan_setup_bench.rs @@ -0,0 +1,63 @@ +//! What a scan, an import and Synchronize Folder cost before they read the disk, and what +//! naming one photo's sidecar costs, by the size of the library (`N`, default 100,000 photos; the +//! folder looked at holds 500 of them). +//! +//! `N=3500000 cargo run --release -p lightcraft-engine --example scan_setup_bench` +use std::time::Instant; + +use lightcraft_catalog::{Op, Photo, Source}; +use lightcraft_engine::Session; + +fn main() { + let n: u64 = std::env::var("N").ok().and_then(|v| v.parse().ok()).unwrap_or(100_000); + let mut s = Session::new(); + let t = Instant::now(); + for chunk in (0..n).collect::>().chunks(10_000) { + for i in chunk { + let id = s.catalog.alloc_photo_id(); + let name = format!("IMG_{i:08}.jpg"); + let path = format!("/Volumes/Photos/Library/Archive/2019/2019-07 Summer/Trip to the coast/Day {:05}/extra/{name}", i / 500); + let mut p = Photo::new(id, Source::File { path }, &name, "JPEG", 4032, 3024, "2026-09-30T12:00:00"); + p.file_size = 3_400_000 + i; + p.captured = Some(format!("2019-07-{:02}T10:{:02}:{:02}", 1 + i % 28, i % 60, (i / 60) % 60)); + p.content_hash = Some(format!("{:032x}", i.wrapping_mul(0x9e37_79b9_7f4a_7c15))); + s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + } + } + eprintln!("{n} photos built in {:.2} s", t.elapsed().as_secs_f64()); + let folder = "/Volumes/Photos/Library/Archive/2019/2019-07 Summer/Trip to the coast/Day 00000".to_string(); + for run in 0..3 { + let ms = |t: Instant| t.elapsed().as_secs_f64() * 1e3; + let t = Instant::now(); + let f = lightcraft_catalog::Filter { library_folder: Some(folder.clone()), ..Default::default() }; + let ids = s.catalog.query(&f, &lightcraft_catalog::Sort::default()); + let query = ms(t); + let t = Instant::now(); + let namings = s.sidecar_namings(&ids); + let naming = ms(t); + let t = Instant::now(); + let one = ids.first().map(|id| s.sidecar_naming(*id)); + let naming_one = ms(t); + let t = Instant::now(); + let (input, paths) = lightcraft_engine::import::ScanInput::new(&mut s, std::slice::from_ref(&folder)); + let scan = ms(t); + // (the folder is not on this disk: the scan finds no file, and its cost is what it makes of the library) + let t = Instant::now(); + let out = lightcraft_engine::import::scan_with(input, &paths, &Default::default()); + let scanned = ms(t); + let t = Instant::now(); + let job = lightcraft_engine::import::ImportJob::new(&mut s, Default::default()); + let import = ms(t); + let t = Instant::now(); + let sync = lightcraft_engine::sync::SyncInput::new(&mut s, &folder, false); + let whole = ms(t); + println!( + "N={n} run {run}: query {query:.0} ms ({} ids) | sidecar_namings {naming:.0} ms | sidecar_naming of one {naming_one:.0} ms ({one:?}) | ScanInput::new {scan:.0} ms + scan_with {scanned:.0} ms ({} found) | ImportJob::new {import:.0} ms (ok {}) | SyncInput::new {whole:.0} ms (ok {})", + ids.len(), + out.candidates.len(), + job.is_ok(), + sync.is_ok() + ); + let _ = namings; + } +} diff --git a/crates/engine/src/cmd/xmp.rs b/crates/engine/src/cmd/xmp.rs index 6af62637f..cd89aa896 100644 --- a/crates/engine/src/cmd/xmp.rs +++ b/crates/engine/src/cmd/xmp.rs @@ -12,8 +12,9 @@ fn save(s: &mut Session, p: &Value) -> Result { let mut merged = Vec::new(); let mut backups = Vec::new(); let mut failed = Vec::new(); - let owners = crate::sidecar::StemOwners::of(&s.catalog); - for id in s.targets(p) { + let ids = s.targets(p); + let owners = crate::sidecar::StemOwners::among(&s.catalog, &ids); + for id in ids { match s.save_sidecar_with(id, &owners) { Ok(r) => { let path = r.path.display().to_string(); diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 1f008e766..418fbca66 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -548,12 +548,62 @@ fn copy_into(dir: &Path, src: &str, name: Option<&str>, probe_hash: Option<&str> pub struct ScanInput { probe: Option, skip: Option, - /// path → (photo, its summary) for files already in the library. - by_path: HashMap, - by_hash: HashMap, + /// The library as it is now (photos are shared with the session, not copied): looked up for + /// the files the scan finds ([`library_files`]) and the contents it probes + /// ([`library_hashes`]), on the scan's thread. + library: std::sync::Arc, cache: HashMap, } +/// The library's photos of `files`: path → (photo, its summary as a candidate that is there +/// already). One pass over the library, and a record only for a file that was found: a folder of +/// a hundred files in a library of millions costs a hundred records. +fn library_files(library: &lightcraft_catalog::Catalog, files: &[String]) -> HashMap { + let found: std::collections::HashSet<&Path> = files.iter().map(Path::new).collect(); + let mut by_path = HashMap::new(); + if found.is_empty() { + return by_path; + } + for p in library.photos() { + if let Source::File { path } = &p.source + && found.contains(Path::new(path)) + { + let c = ImportCandidate { + path: path.clone(), + format: p.format.clone(), + kind: p.kind, + width: p.width, + height: p.height, + file_size: p.file_size, + captured: p.captured.clone(), + duplicate: Some("path".into()), + existing: Some(p.id.0), + ..Default::default() + }; + by_path.insert(PathBuf::from(path), (p.id, c)); + } + } + by_path +} + +/// Which photo of the library has each of the contents `hashes` (the later one where several +/// have it). One pass over the library, and an entry only for a content that was asked about. +fn library_hashes<'a>(library: &lightcraft_catalog::Catalog, hashes: impl Iterator) -> HashMap { + let wanted: std::collections::HashSet<&str> = hashes.collect(); + let mut by_hash = HashMap::new(); + if wanted.is_empty() { + return by_hash; + } + for p in library.photos() { + if let Some(h) = &p.content_hash + && wanted.contains(h.as_str()) + { + by_hash.insert(h.clone(), p.id); + } + } + by_hash +} + /// Progress and cancellation of a running [`scan_with`]. #[derive(Default)] pub struct ScanProgress { @@ -573,31 +623,17 @@ pub struct ScanOutput { impl ScanInput { pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec) { let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); - let mut by_path = HashMap::new(); - let mut by_hash = HashMap::new(); - for p in s.catalog.photos() { - if let Source::File { path } = &p.source { - let c = ImportCandidate { - path: path.clone(), - format: p.format.clone(), - kind: p.kind, - width: p.width, - height: p.height, - file_size: p.file_size, - captured: p.captured.clone(), - duplicate: Some("path".into()), - existing: Some(p.id.0), - ..Default::default() - }; - by_path.insert(PathBuf::from(path), (p.id, c)); - } - if let Some(h) = &p.content_hash { - by_hash.insert(h.clone(), p.id); - } - } - let input = ScanInput { probe: s.media.file_probe.clone(), skip, by_path, by_hash, cache: std::mem::take(&mut s.import_probes) }; + // (no table of the library here: this is the session's thread, and the library may hold + // millions of photos. The scan looks up what it finds) + let library = std::sync::Arc::new(s.catalog.clone()); + let input = ScanInput { probe: s.media.file_probe.clone(), skip, library, cache: std::mem::take(&mut s.import_probes) }; (input, paths.to_vec()) } + + /// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one). + pub(crate) fn library(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.library) + } } /// Find and probe what importing `paths` would add, marking duplicates (by path, or by content @@ -615,7 +651,8 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec { pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput { use std::sync::atomic::Ordering::Relaxed; let files = expand(paths, input.skip.as_deref()); - let todo: Vec = files.iter().filter(|f| !input.by_path.contains_key(Path::new(f))).cloned().collect(); + let by_path = library_files(&input.library, &files); + let todo: Vec = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect(); progress.total.store(todo.len(), Relaxed); // probes from a preceding `scan` are reused when the file is unchanged (same size) let cached: Vec> = todo @@ -633,11 +670,14 @@ pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); crate::memory::release(); let mut probes: HashMap> = todo.into_iter().zip(probed).collect(); - let mut seen_hash: HashMap> = input.by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect(); + // (who in the library has a content that was probed: asked once the probes are in) + let probed_hashes = probes.values().filter_map(|r| r.as_ref().ok()).filter_map(|info| info.content_hash.as_deref()); + let mut seen_hash: HashMap> = + library_hashes(&input.library, probed_hashes).into_iter().map(|(h, id)| (h, Some(id.0))).collect(); let mut out = Vec::with_capacity(files.len()); let mut kept = HashMap::new(); for f in files { - if let Some((_, c)) = input.by_path.get(Path::new(&f)) { + if let Some((_, c)) = by_path.get(Path::new(&f)) { let mut c = c.clone(); c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); out.push(c); @@ -701,6 +741,12 @@ pub struct ImportJob { mode: ImportMode, lib_dir: Option, copy_root: Option, + /// The library as it was when the import was set up, until it is looked up + /// ([`ImportJob::look_up_library`], with the first files): setting an import up is the + /// session's thread, and a table of a library of millions of photos takes seconds. With it, + /// whether a photo in Recently Deleted is passed over (an import that replaces such photos, + /// as the options said when the import was set up). + library: Option<(lightcraft_catalog::Catalog, bool)>, /// path → (photo, a Local browse record?) by_path: HashMap, /// content hash → the photo that has it (`None`: a file earlier in this import) @@ -842,28 +888,16 @@ impl ImportJob { if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() { return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into())); } - let mut by_path = HashMap::new(); - let mut by_hash = HashMap::new(); - for p in s.catalog.photos() { - // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate - if p.deleted && opts.on_deleted == OnDeleted::Fresh { - continue; - } - if let Source::File { path } = &p.source { - by_path.insert(path.clone(), (p.id, p.local)); - } - if let Some(h) = &p.content_hash { - by_hash.insert(h.clone(), Some(p.id)); - } - } + let fresh = opts.on_deleted == OnDeleted::Fresh; Ok(ImportJob { seq: opts.rename_start.max(1), opts, mode, lib_dir, copy_root, - by_path, - by_hash, + library: Some((s.catalog.clone(), fresh)), + by_path: HashMap::new(), + by_hash: HashMap::new(), probe: s.media.file_probe.clone(), file_bytes: s.media.file_bytes.clone(), cache: std::mem::take(&mut s.import_probes), @@ -898,9 +932,28 @@ impl ImportJob { &self.now } + /// What the library held when the import was set up, by path and by content: made once, with + /// the first files (on the thread that prepares them), from the library as it was then. + fn look_up_library(&mut self) { + let Some((library, fresh)) = self.library.take() else { return }; + for p in library.photos() { + // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate + if p.deleted && fresh { + continue; + } + if let Source::File { path } = &p.source { + self.by_path.insert(path.clone(), (p.id, p.local)); + } + if let Some(h) = &p.content_hash { + self.by_hash.insert(h.clone(), Some(p.id)); + } + } + } + /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned. pub fn prepare_files(&mut self, files: Vec, cancel: &std::sync::atomic::AtomicBool) -> Prepared { use std::sync::atomic::Ordering::Relaxed; + self.look_up_library(); let opts = self.opts.clone(); let mode = self.mode; let mut out = Prepared { scanned: files.len(), items: Vec::new() }; @@ -1494,6 +1547,10 @@ impl Session { } } +#[cfg(test)] +#[path = "import/tests_known.rs"] +mod tests_known; + #[cfg(test)] mod prepared_tests { use super::*; diff --git a/crates/engine/src/import/tests_known.rs b/crates/engine/src/import/tests_known.rs new file mode 100644 index 000000000..9adadca8a --- /dev/null +++ b/crates/engine/src/import/tests_known.rs @@ -0,0 +1,134 @@ +//! What a scan or an import knows of the library: worked out for the files it meets, and not on +//! the session's thread. A library can hold millions of photos; looking at one folder must not +//! cost a table of them all before the first file is read. +use super::*; +use lightcraft_catalog::{Catalog, Op}; + +/// A library of `folders` folders of three photos each, the third a copy (by content) of a +/// photo in the folder before. +fn library(folders: usize) -> Catalog { + let mut c = Catalog::new(); + for folder in 0..folders { + for (n, name) in ["a.jpg", "b.jpg", "c.jpg"].into_iter().enumerate() { + let id = c.alloc_photo_id(); + let mut p = Photo::new(id, Source::File { path: format!("/pics/{folder:03}/{name}") }, name, "JPEG", 6, 4, "2026-01-01"); + p.content_hash = Some(format!("{:032x}", if n == 2 { folder.saturating_sub(1) * 3 } else { folder * 3 + n })); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + } + } + c +} + +/// Given a library of many folders, when a scan meets the files of one of them (and one the +/// library does not have), then what it knows of the library is those files' photos, each as the +/// library has it, and nothing of the other folders. +#[test] +fn a_scan_knows_the_library_photos_of_the_files_it_found() { + let c = library(100); + let files: Vec = ["/pics/007/a.jpg", "/pics/007/b.jpg", "/pics/007/new.jpg", "/pics/007//c.jpg"].map(String::from).to_vec(); + let known = library_files(&c, &files); + assert_eq!(known.len(), 3, "the three the library has (a path is compared as a path, not as text)"); + let (id, as_found) = known.get(Path::new("/pics/007/b.jpg")).expect("b"); + let p = c.photo(*id).unwrap(); + assert_eq!((as_found.path.as_str(), as_found.existing, as_found.duplicate.as_deref()), ("/pics/007/b.jpg", Some(id.0), Some("path"))); + assert_eq!((as_found.format.as_str(), as_found.width, as_found.height, as_found.file_size), (p.format.as_str(), p.width, p.height, p.file_size)); + assert!(library_files(&c, &[]).is_empty()); + // two photos of one file (a virtual copy): the later one is the one named, as a table of the + // whole library named it + let mut c = c; + let copy = c.alloc_photo_id(); + c.apply(Op::AddPhoto { photo: Box::new(Photo::new(copy, Source::File { path: "/pics/007/a.jpg".into() }, "a.jpg", "JPEG", 6, 4, "2026-01-02")) }) + .unwrap(); + assert_eq!(library_files(&c, &files).get(Path::new("/pics/007/a.jpg")).map(|(id, _)| *id), Some(copy)); +} + +/// Given a library and a folder on disk that holds a file the library has by its path, one it +/// has by its content under another name, and a new one, when the folder is scanned (the whole +/// scan, as the import review runs it), then each is found for what it is. +#[test] +fn a_scan_finds_what_the_library_has_by_path_and_by_content() { + let dir = std::env::temp_dir().join(format!("lc-known-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let at = |name: &str| dir.join(name).to_string_lossy().to_string(); + for name in ["here.jpg", "copy.jpg", "new.jpg"] { + std::fs::write(dir.join(name), name.as_bytes()).unwrap(); + } + let mut s = Session::new(); + s.catalog = library(20); + let here = s.catalog.alloc_photo_id(); + s.catalog + .apply(Op::AddPhoto { photo: Box::new(Photo::new(here, Source::File { path: at("here.jpg") }, "here.jpg", "JPEG", 6, 4, "2026-01-01")) }) + .unwrap(); + // (the probe: every file is a picture whose content is its name; `copy.jpg` has the content of a library photo) + let had = s.catalog.photos().find(|p| p.file_name == "b.jpg").map(|p| (p.id, p.content_hash.clone().unwrap())).unwrap(); + let shared = had.1.clone(); + s.media.file_probe = Some(std::sync::Arc::new(move |path: &str| { + let name = Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let content_hash = Some(if name == "copy.jpg" { shared.clone() } else { format!("hash-of-{name}") }); + Ok(ProbeInfo { format: "JPEG".into(), width: 6, height: 4, file_size: 8, content_hash, ..Default::default() }) + })); + let found = scan(&mut s, &[dir.to_string_lossy().to_string()]); + let of = |name: &str| found.iter().find(|c| c.name == name).map(|c| (c.duplicate.as_deref(), c.existing)).unwrap(); + assert_eq!(of("here.jpg"), (Some("path"), Some(here.0))); + assert_eq!(of("copy.jpg"), (Some("content"), Some(had.0.0))); + assert_eq!(of("new.jpg"), (None, None)); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Given the hashes of the files a scan probed, when the library is asked who has them, then it +/// answers for those hashes only; and where two photos have one content, the same one of them +/// as a table of the whole library would name (the later one). +#[test] +fn a_scan_knows_who_has_the_contents_it_probed() { + let c = library(100); + // (the content of /pics/006/a.jpg, which /pics/007/c.jpg has too; and one nobody has) + let shared = format!("{:032x}", 6 * 3); + let nobody = format!("{:032x}", 999_999); + let known = library_hashes(&c, [shared.as_str(), nobody.as_str()].into_iter()); + assert_eq!(known.len(), 1); + let whole: HashMap = c.photos().filter_map(|p| p.content_hash.clone().map(|h| (h, p.id))).collect(); + assert_eq!(known.get(&shared), whole.get(&shared)); + assert!(matches!(&c.photo(known[&shared]).unwrap().source, Source::File { path } if path == "/pics/007/c.jpg")); +} + +/// Given a session with a library, when an import is set up, then no table of the library is +/// made on the spot (that is the session's thread); it is made when the first files are +/// prepared, and then finds a file the library has by its path and one it has by its content. +#[test] +fn an_import_looks_the_library_up_when_it_prepares_files() { + let mut s = Session::new(); + s.catalog = library(50); + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap(); + assert!(job.by_path.is_empty() && job.by_hash.is_empty() && job.library.is_some(), "nothing is looked up yet"); + let out = job.prepare_files(vec!["/pics/007/a.jpg".into()], &std::sync::atomic::AtomicBool::new(false)); + assert!(job.library.is_none(), "looked up once"); + assert_eq!((job.by_path.len(), job.by_hash.len()), (150, 100)); + assert!(matches!(out.items.as_slice(), [PreparedItem::Duplicate { reason: "path", existing: Some(_), .. }]), "found by its path"); + // what the options said when the import was set up decides, whatever is made of them after + let id = s.catalog.photos().next().unwrap().id; + s.catalog.apply(Op::SetDeleted { id, deleted: true }).unwrap(); + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, on_deleted: OnDeleted::Fresh, ..ImportOptions::default() }).unwrap(); + job.opts.on_deleted = OnDeleted::Skip; + job.prepare_files(Vec::new(), &std::sync::atomic::AtomicBool::new(false)); + assert_eq!(job.by_path.len(), 149, "the trashed photo was passed over, as set up"); +} + +/// Given a photo in Recently Deleted and an import that replaces such photos with a fresh +/// import, when the library is looked up, then that photo is no duplicate (as before the lookup +/// moved to the first files). +#[test] +fn a_trashed_photo_is_not_looked_up_when_imports_replace_it() { + let mut s = Session::new(); + s.catalog = library(2); + // (a photo whose content is its own: nobody else in this library has it) + let id = s.catalog.photos().find(|p| p.file_name == "b.jpg").unwrap().id; + s.catalog.apply(Op::SetDeleted { id, deleted: true }).unwrap(); + for (on_deleted, expected) in [(OnDeleted::Fresh, 5), (OnDeleted::Skip, 6)] { + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, on_deleted, ..ImportOptions::default() }).unwrap(); + job.prepare_files(Vec::new(), &std::sync::atomic::AtomicBool::new(false)); + assert_eq!(job.by_path.len(), expected, "{on_deleted:?}"); + let its_content = s.catalog.photo(id).and_then(|p| p.content_hash.clone()).unwrap(); + assert_eq!(job.by_hash.contains_key(&its_content), on_deleted == OnDeleted::Skip, "{on_deleted:?}"); + } +} diff --git a/crates/engine/src/sidecar.rs b/crates/engine/src/sidecar.rs index 9aaf7e937..c67a2c84f 100644 --- a/crates/engine/src/sidecar.rs +++ b/crates/engine/src/sidecar.rs @@ -450,6 +450,33 @@ fn stem_key(original: &str) -> String { sidecar_path(original, SidecarNaming::Stem).to_string_lossy().to_lowercase() } +/// Where a plain path is cut to make its [`stem_key`]: at the dot of its extension, which is +/// what `with_extension` replaces. A plain path is ASCII and ends in its file's name with an +/// extension, which is what a library's paths are; `None` for any other shape. +fn plain_cut(original: &str) -> Option { + let path = Path::new(original); + let (name, stem, ext) = (path.file_name()?.to_str()?, path.file_stem()?.to_str()?, path.extension()?.to_str()?); + // (the path ends in its file's name, as it stands: no `/.` or separator after it; and the + // name has a stem and an extension of its own, which `..jpg` and `a.` have not) + let plain = original.is_ascii() && original.ends_with(name) && !ext.is_empty() && !stem.is_empty() && stem != "." && stem != ".."; + let cut = original.len().checked_sub(ext.len().saturating_add(1))?; + (plain && original.as_bytes().get(cut) == Some(&b'.')).then_some(cut) +} + +/// [`stem_key`] written into `key`, which is reused: nothing is allocated for a plain path +/// ([`plain_cut`]). Any other shape goes the long way. +fn stem_key_into(key: &mut String, original: &str) { + key.clear(); + match plain_cut(original) { + Some(cut) => { + key.push_str(original.get(..cut).unwrap_or_default()); + key.make_ascii_lowercase(); + key.push_str(".xmp"); + } + None => key.push_str(&stem_key(original)), + } +} + fn file_path(p: &Photo) -> Option<&str> { match &p.source { Source::File { path } => Some(path), @@ -490,8 +517,11 @@ pub(crate) fn op_photos(op: &Op, out: &mut Vec) { pub(crate) struct StemOwners(HashMap); impl StemOwners { + /// The owners of every shared stem in the library, each file's key made the long way: what + /// [`StemOwners::among`] must agree with for the photos it is asked about (this is how the + /// owners were worked out before, for every write). + #[cfg(test)] pub(crate) fn of(cat: &lightcraft_catalog::Catalog) -> StemOwners { - // per stem: (raw first, then file name, then id) of the best candidate; and how many let mut best: HashMap = HashMap::new(); for p in cat.photos().filter(|p| p.copy_of.is_none()) { let Some(path) = file_path(p) else { continue }; @@ -505,6 +535,39 @@ impl StemOwners { StemOwners(best.into_iter().filter(|(_, v)| v.2 > 1).map(|(k, v)| (k, v.1)).collect()) } + /// The owners of the stems photos `ids` map to: all that naming those photos needs. The + /// library is passed over once, and only files of those stems are looked at closer, so a + /// sidecar written for one photo does not cost a table of the whole library. + pub(crate) fn among(cat: &lightcraft_catalog::Catalog, ids: &[PhotoId]) -> StemOwners { + let wanted: std::collections::HashSet = + ids.iter().filter_map(|id| cat.photo(*id)).filter_map(|p| file_path(p)).map(stem_key).collect(); + if wanted.is_empty() { + return StemOwners(HashMap::new()); + } + Self::where_wanted(cat, &wanted) + } + + /// The owners of the shared stems in `wanted`. + fn where_wanted(cat: &lightcraft_catalog::Catalog, wanted: &std::collections::HashSet) -> StemOwners { + // per stem: (raw first, then file name, then id) of the best candidate; and how many + let mut best: HashMap = HashMap::new(); + let mut key = String::new(); + for p in cat.photos().filter(|p| p.copy_of.is_none()) { + let Some(path) = file_path(p) else { continue }; + stem_key_into(&mut key, path); + if !wanted.contains(key.as_str()) { + continue; + } + let rank = (p.kind != MediaKind::Raw, path.to_lowercase(), p.id.0); + let e = best.entry(key.clone()).or_insert_with(|| (rank.clone(), p.id, 0)); + e.2 += 1; + if rank < e.0 { + (e.0, e.1) = (rank, p.id); + } + } + StemOwners(best.into_iter().filter(|(_, v)| v.2 > 1).map(|(k, v)| (k, v.1)).collect()) + } + /// `naming` for photo `p`, except that a stem sidecar another file owns becomes Full. fn naming(&self, p: &Photo, naming: SidecarNaming) -> SidecarNaming { match (naming, file_path(p)) { @@ -514,33 +577,39 @@ impl StemOwners { } } +/// How the sidecar of each of photos `ids` is named in library `cat`, whose naming is `naming` +/// ([`Session::sidecar_namings`], for a thread that has the library and not the session). +pub(crate) fn namings_in(cat: &lightcraft_catalog::Catalog, naming: SidecarNaming, ids: &[PhotoId]) -> Vec { + let owners = (naming == SidecarNaming::Stem).then(|| StemOwners::among(cat, ids)); + ids.iter() + .map(|id| match (cat.photo(*id), &owners) { + (Some(p), Some(o)) => o.naming(p, naming), + _ => naming, + }) + .collect() +} + impl Session { /// The sidecar naming for photo `id`: the library's, except under Stem naming when another /// catalogued file shares the stem (`IMG_0001.CR3` + `IMG_0001.JPG`) and owns the stem /// sidecar — a raw first, else the first by file name. The others use Full naming /// (`IMG_0001.JPG.xmp`), so their metadata never overwrites each other. - /// [`Session::sidecar_naming`] for many photos, the catalog's sidecar owners worked out once - /// (per photo that is a pass over the whole catalog). + /// [`Session::sidecar_naming`] for many photos, their stems' owners worked out once (per + /// photo that is a pass over the whole catalog). pub fn sidecar_namings(&self, ids: &[PhotoId]) -> Vec { - let owners = (self.xmp.naming == SidecarNaming::Stem).then(|| StemOwners::of(&self.catalog)); - ids.iter() - .map(|id| match (self.catalog.photo(*id), &owners) { - (Some(p), Some(o)) => o.naming(p, self.xmp.naming), - _ => self.xmp.naming, - }) - .collect() + namings_in(&self.catalog, self.xmp.naming, ids) } pub fn sidecar_naming(&self, id: PhotoId) -> SidecarNaming { match self.catalog.photo(id) { - Some(p) if self.xmp.naming == SidecarNaming::Stem => StemOwners::of(&self.catalog).naming(p, self.xmp.naming), + Some(p) if self.xmp.naming == SidecarNaming::Stem => StemOwners::among(&self.catalog, &[id]).naming(p, self.xmp.naming), _ => self.xmp.naming, } } /// Write the photo's XMP sidecar, merged into an existing one (see the module docs). pub fn save_sidecar(&mut self, id: PhotoId) -> Result { - self.save_sidecar_with(id, &StemOwners::of(&self.catalog)) + self.save_sidecar_with(id, &StemOwners::among(&self.catalog, &[id])) } pub(crate) fn save_sidecar_with(&mut self, id: PhotoId, owners: &StemOwners) -> Result { @@ -642,7 +711,7 @@ impl Session { if ids.is_empty() { return; } - let owners = StemOwners::of(&self.catalog); + let owners = StemOwners::among(&self.catalog, &ids); for id in ids { if self.catalog.photo(id).is_some_and(|p| file_path(p).is_some() && p.copy_of.is_none()) && let Err(e) = self.save_sidecar_with(id, &owners) @@ -764,3 +833,7 @@ mod tests_repair_companions; #[cfg(test)] #[path = "sidecar/tests_repair_provenance.rs"] mod tests_repair_provenance; + +#[cfg(test)] +#[path = "sidecar/tests_stem_owners.rs"] +mod tests_stem_owners; diff --git a/crates/engine/src/sidecar/tests_stem_owners.rs b/crates/engine/src/sidecar/tests_stem_owners.rs new file mode 100644 index 000000000..43170f4f6 --- /dev/null +++ b/crates/engine/src/sidecar/tests_stem_owners.rs @@ -0,0 +1,163 @@ +//! Who owns a stem sidecar, worked out for the photos asked about only (`StemOwners::among`). +//! +//! A library can hold millions of photos; a sidecar that is written for one of them, or a folder +//! that is synchronized, must not cost a table of them all. +use super::*; +use lightcraft_catalog::{Catalog, Op, Photo, Source}; + +/// Paths that try the key: case in folders, names and endings, no ending, a dot file, endings +/// inside folder names, non-ASCII letters (one that lower-cases to ASCII: the Kelvin sign). +const PATHS: [&str; 26] = [ + "/pics/trip/IMG_1.CR3", + "/pics/trip/IMG_1.JPG", + "/pics/trip/img_1.jpeg", + "/Pics/Trip/IMG_1.tif", + "/pics/trip/IMG_2.JPG", + "/pics/trip/IMG_3.CR3", + "/pics/trip/IMG_3.cr3.JPG", + "/pics/trip/IMG_3.cr3", + "/pics/other/IMG_1.JPG", + "/pics/other/IMG_1", + "/pics/other/.hidden", + "/pics/other/.hidden.jpg", + "/pics/v1.2/a.b.JPG", + "/pics/v1.2/a.b.cr3", + "/pics/v1.2/a", + "/pics/straße/Größe.JPG", + "/pics/STRASSE/GRÖSSE.cr3", + "/pics/straße/größe.CR3", + "/pics/kelvin/\u{212a}1.JPG", + "/pics/kelvin/k1.cr3", + "/pics/ΟΔΟΣ/ΟΔΟΣ.JPG", + "/pics/οδος/οδος.CR3", + // (names that are all dots before their ending: the ending is no extension to replace) + "/pics/dots/..cr3", + "/pics/dots/..jpg", + "/pics/dots/a./.", + "/pics/dots/a.jpg", +]; + +fn library() -> (Catalog, Vec) { + let mut c = Catalog::new(); + let mut ids = Vec::new(); + for path in PATHS { + let id = c.alloc_photo_id(); + let name = path.rsplit('/').next().unwrap_or(path); + let mut p = Photo::new(id, Source::File { path: path.to_string() }, name, "X", 6, 4, "2026-01-01"); + if path.to_lowercase().ends_with("cr3") { + p.kind = MediaKind::Raw; + } + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + ids.push(id); + } + // a virtual copy of the first (it owns nothing) and a photo that is no file + let copy = c.alloc_photo_id(); + let mut p = Photo::new(copy, Source::File { path: PATHS[0].to_string() }, "IMG_1.CR3", "X", 6, 4, "2026-01-01"); + p.copy_of = Some(ids[0]); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + ids.push(copy); + let demo = c.alloc_photo_id(); + c.apply(Op::AddPhoto { photo: Box::new(Photo::new(demo, Source::Demo { scene: 1 }, "demo", "X", 6, 4, "2026-01-01")) }).unwrap(); + ids.push(demo); + (c, ids) +} + +fn namings(owners: &StemOwners, c: &Catalog, ids: &[PhotoId]) -> Vec { + ids.iter().map(|id| owners.naming(c.photo(*id).unwrap(), SidecarNaming::Stem)).collect() +} + +/// Given a library where files share stems in every way a name can, when the owners are worked +/// out for some of its photos only, then each of those photos is named exactly as the table of +/// the whole library names it: whichever photos are asked about, alone or together. +#[test] +fn owners_among_some_photos_name_them_as_the_whole_library_does() { + let (c, ids) = library(); + let whole = StemOwners::of(&c); + let expected = namings(&whole, &c, &ids); + assert!(expected.contains(&SidecarNaming::Full) && expected.contains(&SidecarNaming::Stem), "both cases are there: {expected:?}"); + // all at once + assert_eq!(namings(&StemOwners::among(&c, &ids), &c, &ids), expected); + // one at a time, and every pair + for (i, a) in ids.iter().enumerate() { + assert_eq!(namings(&StemOwners::among(&c, &[*a]), &c, &[*a]), [expected[i]], "{:?}", PATHS.get(i)); + for (j, b) in ids.iter().enumerate() { + assert_eq!(namings(&StemOwners::among(&c, &[*a, *b]), &c, &[*a, *b]), [expected[i], expected[j]]); + } + } + // an id that is not in the library is passed over + assert_eq!(StemOwners::among(&c, &[PhotoId(9_999)]).0.len(), 0); +} + +/// Given a library of many folders, when the owners are worked out for the photos of one, then +/// the table holds that folder's shared stems and nothing of the others: what it costs follows +/// the photos asked about, not the library. +#[test] +fn owners_among_some_photos_hold_nothing_of_the_rest() { + let mut c = Catalog::new(); + let mut first = Vec::new(); + for folder in 0..200 { + for name in ["IMG_1.CR3", "IMG_1.JPG", "IMG_2.JPG"] { + let id = c.alloc_photo_id(); + let mut p = Photo::new(id, Source::File { path: format!("/pics/{folder:03}/{name}") }, name, "X", 6, 4, "2026-01-01"); + if name.ends_with("CR3") { + p.kind = MediaKind::Raw; + } + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + if folder == 0 { + first.push(id); + } + } + } + assert_eq!(StemOwners::of(&c).0.len(), 200, "one shared stem a folder"); + let some = StemOwners::among(&c, &first); + assert_eq!(some.0.len(), 1, "the one shared stem of the folder asked about"); + assert_eq!(namings(&some, &c, &first), [SidecarNaming::Stem, SidecarNaming::Full, SidecarNaming::Stem]); +} + +/// The key written into a buffer (no allocation for a plain path) is the key: for every path +/// above, for shapes a path from a file can have, and for every string of up to six of the +/// characters that decide a key (a letter in each case, a dot, both separators, a letter that is +/// not ASCII). +#[test] +fn the_key_in_a_buffer_is_the_key() { + let odd = [ + "", + "/", + "a", + "a.", + ".a", + "..a", + "...a", + "a.b", + "a/b.c/", + "a/b.c//", + "a/.b", + "a/..b", + "a/b.", + "a/b..c", + "a./.", + "a\\b.c", + "C:\\x\\y.JPG", + "a/b.c/.", + "a/b.c/..", + "x.JPG ", + "É.JPG", + ]; + let mut buf = String::new(); + for path in PATHS.iter().chain(&odd) { + stem_key_into(&mut buf, path); + assert_eq!(buf, stem_key(path), "{path:?}"); + } + let letters = ['a', 'B', '.', '/', '\\', 'é']; + let mut tried = 0usize; + let mut todo: Vec = vec![String::new()]; + while let Some(path) = todo.pop() { + stem_key_into(&mut buf, &path); + assert_eq!(buf, stem_key(&path), "{path:?}"); + tried += 1; + if path.chars().count() < 6 { + todo.extend(letters.iter().map(|c| format!("{path}{c}"))); + } + } + assert_eq!(tried, (0..=6).map(|n| 6usize.pow(n)).sum::()); +} diff --git a/crates/engine/src/sync.rs b/crates/engine/src/sync.rs index 0a00a59ab..4342ec6ad 100644 --- a/crates/engine/src/sync.rs +++ b/crates/engine/src/sync.rs @@ -74,10 +74,14 @@ impl SyncProgress { pub struct SyncInput { folder: String, scan: ScanInput, - /// The library's photos in the folder, each with how its sidecar is named. - photos: Vec<(Arc, SidecarNaming)>, - /// For the colour label names a sidecar may use. - labels: Catalog, + /// The library's photos in the folder. + ids: Vec, + /// How the library names sidecars; how each photo's is named is worked out by the scan (a + /// pass over the library, which the session's thread does not wait for). + naming: SidecarNaming, + /// The library as it is now (the one the scan looks files up in): for those photos, their + /// sidecars' names and the colour label names a sidecar may use. + labels: Arc, } /// What changed in a folder since it came into the library. @@ -176,10 +180,9 @@ impl SyncInput { if !disk && (lightcraft_catalog::folders::is_disk_root(path) || crate::cmd::library::covers_other_disks(s, path, &ids)) { return Err(invalid(format!("{path} is a whole disk or holds disks: pass `disk: true` to synchronize it"))); } - let namings = s.sidecar_namings(&ids); - let photos = ids.iter().zip(namings).filter_map(|(id, naming)| s.catalog.photo(*id).map(|p| (Arc::clone(p), naming))).collect(); let (scan, _) = ScanInput::new(s, &[path.to_string()]); - Ok(SyncInput { folder: path.to_string(), scan, photos, labels: s.catalog.clone() }) + let labels = scan.library(); + Ok(SyncInput { folder: path.to_string(), scan, ids, naming: s.xmp.naming, labels }) } } @@ -190,11 +193,16 @@ impl SyncInput { /// library's photos, which run on several threads too. Stops early when `progress.files.cancel` /// is set. pub fn scan_with(input: SyncInput, progress: &SyncProgress) -> FolderChanges { - let SyncInput { folder, scan, photos, labels } = input; - let seen: Vec> = photos.iter().map(|(p, _)| Arc::clone(p)).collect(); + let SyncInput { folder, scan, ids, naming, labels } = input; + let seen: Vec> = ids.iter().filter_map(|id| labels.photo(*id)).map(Arc::clone).collect(); if !cfg!(target_arch = "wasm32") && !Path::new(&folder).is_dir() { return FolderChanges { path: folder, seen, offline: true, ..Default::default() }; } + // (how each photo's sidecar is named: a pass over the library, made here and not while the + // session waits, and not at all for a folder that is not there) + let namings = crate::sidecar::namings_in(&labels, naming, &ids); + let photos: Vec<(Arc, SidecarNaming)> = + ids.iter().zip(namings).filter_map(|(id, naming)| labels.photo(*id).map(|p| (Arc::clone(p), naming))).collect(); // (no threads on the web: one after the other there) #[cfg(target_arch = "wasm32")] let (out, checks) = (crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files), check_photos(&photos, &labels, progress)); diff --git a/docs/whats-new.md b/docs/whats-new.md index 6abe7aa87..64b7045ca 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -48,6 +48,16 @@ from or written through such a link. ## 2026-10-13 +### Large libraries: Synchronize Folder, import and sidecars start sooner +- Opening Synchronize Folder, an import review or an import no longer makes a table of every photo in the library + before it looks at the first file. In a library of 3.5 million photos that held the window for about 6 s before + a Synchronize scan began (now under 1 s), 2.6 s before an import review (now 0.05 s) and 1.7 s before an import + (now 0.05 s). The scan looks up only the files it finds and the contents it reads, on its own thread. +- Writing one photo's XMP sidecar (Save Metadata to File, or automatically after an edit) no longer works out who + owns every shared sidecar name in the library: 2.4 s a write in that library, now under 1 s. +- Nothing else changes: the same files are found new, already there, moved or missing, and sidecars get the same + names. + ### Slider drags on large raws - Dragging a slider no longer re-renders the edited photo's filmstrip thumbnail on every frame; it follows once you let go. On a 64 MP raw the first of those renders decoded the whole file again, which held up the loupe's