Merge PR #1583

# Conflicts:
#	docs/whats-new.md

crates/engine/examples/scan_setup_bench.rs

@@ -0,0 +1,63 @@
+//! What a scan, an import and Synchronize Folder cost before they read the disk, and what
+//! naming one photo's sidecar costs, by the size of the library (`N`, default 100,000 photos; the
+//! folder looked at holds 500 of them).
+//!
+//! `N=3500000 cargo run --release -p lightcraft-engine --example scan_setup_bench`
+use std::time::Instant;
+
+use lightcraft_catalog::{Op, Photo, Source};
+use lightcraft_engine::Session;
+
+fn main() {
+    let n: u64 = std::env::var("N").ok().and_then(|v| v.parse().ok()).unwrap_or(100_000);
+    let mut s = Session::new();
+    let t = Instant::now();
+    for chunk in (0..n).collect::<Vec<_>>().chunks(10_000) {
+        for i in chunk {
+            let id = s.catalog.alloc_photo_id();
+            let name = format!("IMG_{i:08}.jpg");
+            let path = format!("/Volumes/Photos/Library/Archive/2019/2019-07 Summer/Trip to the coast/Day {:05}/extra/{name}", i / 500);
+            let mut p = Photo::new(id, Source::File { path }, &name, "JPEG", 4032, 3024, "2026-09-30T12:00:00");
+            p.file_size = 3_400_000 + i;
+            p.captured = Some(format!("2019-07-{:02}T10:{:02}:{:02}", 1 + i % 28, i % 60, (i / 60) % 60));
+            p.content_hash = Some(format!("{:032x}", i.wrapping_mul(0x9e37_79b9_7f4a_7c15)));
+            s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
+        }
+    }
+    eprintln!("{n} photos built in {:.2} s", t.elapsed().as_secs_f64());
+    let folder = "/Volumes/Photos/Library/Archive/2019/2019-07 Summer/Trip to the coast/Day 00000".to_string();
+    for run in 0..3 {
+        let ms = |t: Instant| t.elapsed().as_secs_f64() * 1e3;
+        let t = Instant::now();
+        let f = lightcraft_catalog::Filter { library_folder: Some(folder.clone()), ..Default::default() };
+        let ids = s.catalog.query(&f, &lightcraft_catalog::Sort::default());
+        let query = ms(t);
+        let t = Instant::now();
+        let namings = s.sidecar_namings(&ids);
+        let naming = ms(t);
+        let t = Instant::now();
+        let one = ids.first().map(|id| s.sidecar_naming(*id));
+        let naming_one = ms(t);
+        let t = Instant::now();
+        let (input, paths) = lightcraft_engine::import::ScanInput::new(&mut s, std::slice::from_ref(&folder));
+        let scan = ms(t);
+        // (the folder is not on this disk: the scan finds no file, and its cost is what it makes of the library)
+        let t = Instant::now();
+        let out = lightcraft_engine::import::scan_with(input, &paths, &Default::default());
+        let scanned = ms(t);
+        let t = Instant::now();
+        let job = lightcraft_engine::import::ImportJob::new(&mut s, Default::default());
+        let import = ms(t);
+        let t = Instant::now();
+        let sync = lightcraft_engine::sync::SyncInput::new(&mut s, &folder, false);
+        let whole = ms(t);
+        println!(
+            "N={n} run {run}: query {query:.0} ms ({} ids) | sidecar_namings {naming:.0} ms | sidecar_naming of one {naming_one:.0} ms ({one:?}) | ScanInput::new {scan:.0} ms + scan_with {scanned:.0} ms ({} found) | ImportJob::new {import:.0} ms (ok {}) | SyncInput::new {whole:.0} ms (ok {})",
+            ids.len(),
+            out.candidates.len(),
+            job.is_ok(),
+            sync.is_ok()
+        );
+        let _ = namings;
+    }
+}

crates/engine/src/cmd/xmp.rs

@@ -12,8 +12,9 @@ fn save(s: &mut Session, p: &Value) -> Result<Value> {
     let mut merged = Vec::new();
     let mut backups = Vec::new();
     let mut failed = Vec::new();
-    let owners = crate::sidecar::StemOwners::of(&s.catalog);
-    for id in s.targets(p) {
+    let ids = s.targets(p);
+    let owners = crate::sidecar::StemOwners::among(&s.catalog, &ids);
+    for id in ids {
         match s.save_sidecar_with(id, &owners) {
             Ok(r) => {
                 let path = r.path.display().to_string();

crates/engine/src/import.rs

@@ -548,12 +548,62 @@ fn copy_into(dir: &Path, src: &str, name: Option<&str>, probe_hash: Option<&str>
 pub struct ScanInput {
     probe: Option<crate::media::FileProbe>,
     skip: Option<PathBuf>,
-    /// path → (photo, its summary) for files already in the library.
-    by_path: HashMap<PathBuf, (PhotoId, ImportCandidate)>,
-    by_hash: HashMap<String, PhotoId>,
+    /// The library as it is now (photos are shared with the session, not copied): looked up for
+    /// the files the scan finds ([`library_files`]) and the contents it probes
+    /// ([`library_hashes`]), on the scan's thread.
+    library: std::sync::Arc<lightcraft_catalog::Catalog>,
     cache: HashMap<String, ProbeInfo>,
 }
 
+/// The library's photos of `files`: path → (photo, its summary as a candidate that is there
+/// already). One pass over the library, and a record only for a file that was found: a folder of
+/// a hundred files in a library of millions costs a hundred records.
+fn library_files(library: &lightcraft_catalog::Catalog, files: &[String]) -> HashMap<PathBuf, (PhotoId, ImportCandidate)> {
+    let found: std::collections::HashSet<&Path> = files.iter().map(Path::new).collect();
+    let mut by_path = HashMap::new();
+    if found.is_empty() {
+        return by_path;
+    }
+    for p in library.photos() {
+        if let Source::File { path } = &p.source
+            && found.contains(Path::new(path))
+        {
+            let c = ImportCandidate {
+                path: path.clone(),
+                format: p.format.clone(),
+                kind: p.kind,
+                width: p.width,
+                height: p.height,
+                file_size: p.file_size,
+                captured: p.captured.clone(),
+                duplicate: Some("path".into()),
+                existing: Some(p.id.0),
+                ..Default::default()
+            };
+            by_path.insert(PathBuf::from(path), (p.id, c));
+        }
+    }
+    by_path
+}
+
+/// Which photo of the library has each of the contents `hashes` (the later one where several
+/// have it). One pass over the library, and an entry only for a content that was asked about.
+fn library_hashes<'a>(library: &lightcraft_catalog::Catalog, hashes: impl Iterator<Item = &'a str>) -> HashMap<String, PhotoId> {
+    let wanted: std::collections::HashSet<&str> = hashes.collect();
+    let mut by_hash = HashMap::new();
+    if wanted.is_empty() {
+        return by_hash;
+    }
+    for p in library.photos() {
+        if let Some(h) = &p.content_hash
+            && wanted.contains(h.as_str())
+        {
+            by_hash.insert(h.clone(), p.id);
+        }
+    }
+    by_hash
+}
+
 /// Progress and cancellation of a running [`scan_with`].
 #[derive(Default)]
 pub struct ScanProgress {
@@ -573,31 +623,17 @@ pub struct ScanOutput {
 impl ScanInput {
     pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec<String>) {
         let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
-        let mut by_path = HashMap::new();
-        let mut by_hash = HashMap::new();
-        for p in s.catalog.photos() {
-            if let Source::File { path } = &p.source {
-                let c = ImportCandidate {
-                    path: path.clone(),
-                    format: p.format.clone(),
-                    kind: p.kind,
-                    width: p.width,
-                    height: p.height,
-                    file_size: p.file_size,
-                    captured: p.captured.clone(),
-                    duplicate: Some("path".into()),
-                    existing: Some(p.id.0),
-                    ..Default::default()
-                };
-                by_path.insert(PathBuf::from(path), (p.id, c));
-            }
-            if let Some(h) = &p.content_hash {
-                by_hash.insert(h.clone(), p.id);
-            }
-        }
-        let input = ScanInput { probe: s.media.file_probe.clone(), skip, by_path, by_hash, cache: std::mem::take(&mut s.import_probes) };
+        // (no table of the library here: this is the session's thread, and the library may hold
+        // millions of photos. The scan looks up what it finds)
+        let library = std::sync::Arc::new(s.catalog.clone());
+        let input = ScanInput { probe: s.media.file_probe.clone(), skip, library, cache: std::mem::take(&mut s.import_probes) };
         (input, paths.to_vec())
     }
+
+    /// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one).
+    pub(crate) fn library(&self) -> std::sync::Arc<lightcraft_catalog::Catalog> {
+        std::sync::Arc::clone(&self.library)
+    }
 }
 
 /// Find and probe what importing `paths` would add, marking duplicates (by path, or by content
@@ -615,7 +651,8 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
 pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
     use std::sync::atomic::Ordering::Relaxed;
     let files = expand(paths, input.skip.as_deref());
-    let todo: Vec<String> = files.iter().filter(|f| !input.by_path.contains_key(Path::new(f))).cloned().collect();
+    let by_path = library_files(&input.library, &files);
+    let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect();
     progress.total.store(todo.len(), Relaxed);
     // probes from a preceding `scan` are reused when the file is unchanged (same size)
     let cached: Vec<Option<ProbeInfo>> = todo
@@ -633,11 +670,14 @@ pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress
         cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
     crate::memory::release();
     let mut probes: HashMap<String, Result<ProbeInfo, String>> = todo.into_iter().zip(probed).collect();
-    let mut seen_hash: HashMap<String, Option<u64>> = input.by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect();
+    // (who in the library has a content that was probed: asked once the probes are in)
+    let probed_hashes = probes.values().filter_map(|r| r.as_ref().ok()).filter_map(|info| info.content_hash.as_deref());
+    let mut seen_hash: HashMap<String, Option<u64>> =
+        library_hashes(&input.library, probed_hashes).into_iter().map(|(h, id)| (h, Some(id.0))).collect();
     let mut out = Vec::with_capacity(files.len());
     let mut kept = HashMap::new();
     for f in files {
-        if let Some((_, c)) = input.by_path.get(Path::new(&f)) {
+        if let Some((_, c)) = by_path.get(Path::new(&f)) {
             let mut c = c.clone();
             c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());
             out.push(c);
@@ -701,6 +741,12 @@ pub struct ImportJob {
     mode: ImportMode,
     lib_dir: Option<PathBuf>,
     copy_root: Option<PathBuf>,
+    /// The library as it was when the import was set up, until it is looked up
+    /// ([`ImportJob::look_up_library`], with the first files): setting an import up is the
+    /// session's thread, and a table of a library of millions of photos takes seconds. With it,
+    /// whether a photo in Recently Deleted is passed over (an import that replaces such photos,
+    /// as the options said when the import was set up).
+    library: Option<(lightcraft_catalog::Catalog, bool)>,
     /// path → (photo, a Local browse record?)
     by_path: HashMap<String, (PhotoId, bool)>,
     /// content hash → the photo that has it (`None`: a file earlier in this import)
@@ -842,28 +888,16 @@ impl ImportJob {
         if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() {
             return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into()));
         }
-        let mut by_path = HashMap::new();
-        let mut by_hash = HashMap::new();
-        for p in s.catalog.photos() {
-            // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate
-            if p.deleted && opts.on_deleted == OnDeleted::Fresh {
-                continue;
-            }
-            if let Source::File { path } = &p.source {
-                by_path.insert(path.clone(), (p.id, p.local));
-            }
-            if let Some(h) = &p.content_hash {
-                by_hash.insert(h.clone(), Some(p.id));
-            }
-        }
+        let fresh = opts.on_deleted == OnDeleted::Fresh;
         Ok(ImportJob {
             seq: opts.rename_start.max(1),
             opts,
             mode,
             lib_dir,
             copy_root,
-            by_path,
-            by_hash,
+            library: Some((s.catalog.clone(), fresh)),
+            by_path: HashMap::new(),
+            by_hash: HashMap::new(),
             probe: s.media.file_probe.clone(),
             file_bytes: s.media.file_bytes.clone(),
             cache: std::mem::take(&mut s.import_probes),
@@ -898,9 +932,28 @@ impl ImportJob {
         &self.now
     }
 
+    /// What the library held when the import was set up, by path and by content: made once, with
+    /// the first files (on the thread that prepares them), from the library as it was then.
+    fn look_up_library(&mut self) {
+        let Some((library, fresh)) = self.library.take() else { return };
+        for p in library.photos() {
+            // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate
+            if p.deleted && fresh {
+                continue;
+            }
+            if let Source::File { path } = &p.source {
+                self.by_path.insert(path.clone(), (p.id, p.local));
+            }
+            if let Some(h) = &p.content_hash {
+                self.by_hash.insert(h.clone(), Some(p.id));
+            }
+        }
+    }
+
     /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned.
     pub fn prepare_files(&mut self, files: Vec<String>, cancel: &std::sync::atomic::AtomicBool) -> Prepared {
         use std::sync::atomic::Ordering::Relaxed;
+        self.look_up_library();
         let opts = self.opts.clone();
         let mode = self.mode;
         let mut out = Prepared { scanned: files.len(), items: Vec::new() };
@@ -1494,6 +1547,10 @@ impl Session {
     }
 }
 
+#[cfg(test)]
+#[path = "import/tests_known.rs"]
+mod tests_known;
+
 #[cfg(test)]
 mod prepared_tests {
     use super::*;

crates/engine/src/import/tests_known.rs

@@ -0,0 +1,134 @@
+//! What a scan or an import knows of the library: worked out for the files it meets, and not on
+//! the session's thread. A library can hold millions of photos; looking at one folder must not
+//! cost a table of them all before the first file is read.
+use super::*;
+use lightcraft_catalog::{Catalog, Op};
+
+/// A library of `folders` folders of three photos each, the third a copy (by content) of a
+/// photo in the folder before.
+fn library(folders: usize) -> Catalog {
+    let mut c = Catalog::new();
+    for folder in 0..folders {
+        for (n, name) in ["a.jpg", "b.jpg", "c.jpg"].into_iter().enumerate() {
+            let id = c.alloc_photo_id();
+            let mut p = Photo::new(id, Source::File { path: format!("/pics/{folder:03}/{name}") }, name, "JPEG", 6, 4, "2026-01-01");
+            p.content_hash = Some(format!("{:032x}", if n == 2 { folder.saturating_sub(1) * 3 } else { folder * 3 + n }));
+            c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
+        }
+    }
+    c
+}
+
+/// Given a library of many folders, when a scan meets the files of one of them (and one the
+/// library does not have), then what it knows of the library is those files' photos, each as the
+/// library has it, and nothing of the other folders.
+#[test]
+fn a_scan_knows_the_library_photos_of_the_files_it_found() {
+    let c = library(100);
+    let files: Vec<String> = ["/pics/007/a.jpg", "/pics/007/b.jpg", "/pics/007/new.jpg", "/pics/007//c.jpg"].map(String::from).to_vec();
+    let known = library_files(&c, &files);
+    assert_eq!(known.len(), 3, "the three the library has (a path is compared as a path, not as text)");
+    let (id, as_found) = known.get(Path::new("/pics/007/b.jpg")).expect("b");
+    let p = c.photo(*id).unwrap();
+    assert_eq!((as_found.path.as_str(), as_found.existing, as_found.duplicate.as_deref()), ("/pics/007/b.jpg", Some(id.0), Some("path")));
+    assert_eq!((as_found.format.as_str(), as_found.width, as_found.height, as_found.file_size), (p.format.as_str(), p.width, p.height, p.file_size));
+    assert!(library_files(&c, &[]).is_empty());
+    // two photos of one file (a virtual copy): the later one is the one named, as a table of the
+    // whole library named it
+    let mut c = c;
+    let copy = c.alloc_photo_id();
+    c.apply(Op::AddPhoto { photo: Box::new(Photo::new(copy, Source::File { path: "/pics/007/a.jpg".into() }, "a.jpg", "JPEG", 6, 4, "2026-01-02")) })
+        .unwrap();
+    assert_eq!(library_files(&c, &files).get(Path::new("/pics/007/a.jpg")).map(|(id, _)| *id), Some(copy));
+}
+
+/// Given a library and a folder on disk that holds a file the library has by its path, one it
+/// has by its content under another name, and a new one, when the folder is scanned (the whole
+/// scan, as the import review runs it), then each is found for what it is.
+#[test]
+fn a_scan_finds_what_the_library_has_by_path_and_by_content() {
+    let dir = std::env::temp_dir().join(format!("lc-known-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    let at = |name: &str| dir.join(name).to_string_lossy().to_string();
+    for name in ["here.jpg", "copy.jpg", "new.jpg"] {
+        std::fs::write(dir.join(name), name.as_bytes()).unwrap();
+    }
+    let mut s = Session::new();
+    s.catalog = library(20);
+    let here = s.catalog.alloc_photo_id();
+    s.catalog
+        .apply(Op::AddPhoto { photo: Box::new(Photo::new(here, Source::File { path: at("here.jpg") }, "here.jpg", "JPEG", 6, 4, "2026-01-01")) })
+        .unwrap();
+    // (the probe: every file is a picture whose content is its name; `copy.jpg` has the content of a library photo)
+    let had = s.catalog.photos().find(|p| p.file_name == "b.jpg").map(|p| (p.id, p.content_hash.clone().unwrap())).unwrap();
+    let shared = had.1.clone();
+    s.media.file_probe = Some(std::sync::Arc::new(move |path: &str| {
+        let name = Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+        let content_hash = Some(if name == "copy.jpg" { shared.clone() } else { format!("hash-of-{name}") });
+        Ok(ProbeInfo { format: "JPEG".into(), width: 6, height: 4, file_size: 8, content_hash, ..Default::default() })
+    }));
+    let found = scan(&mut s, &[dir.to_string_lossy().to_string()]);
+    let of = |name: &str| found.iter().find(|c| c.name == name).map(|c| (c.duplicate.as_deref(), c.existing)).unwrap();
+    assert_eq!(of("here.jpg"), (Some("path"), Some(here.0)));
+    assert_eq!(of("copy.jpg"), (Some("content"), Some(had.0.0)));
+    assert_eq!(of("new.jpg"), (None, None));
+    let _ = std::fs::remove_dir_all(&dir);
+}
+
+/// Given the hashes of the files a scan probed, when the library is asked who has them, then it
+/// answers for those hashes only; and where two photos have one content, the same one of them
+/// as a table of the whole library would name (the later one).
+#[test]
+fn a_scan_knows_who_has_the_contents_it_probed() {
+    let c = library(100);
+    // (the content of /pics/006/a.jpg, which /pics/007/c.jpg has too; and one nobody has)
+    let shared = format!("{:032x}", 6 * 3);
+    let nobody = format!("{:032x}", 999_999);
+    let known = library_hashes(&c, [shared.as_str(), nobody.as_str()].into_iter());
+    assert_eq!(known.len(), 1);
+    let whole: HashMap<String, PhotoId> = c.photos().filter_map(|p| p.content_hash.clone().map(|h| (h, p.id))).collect();
+    assert_eq!(known.get(&shared), whole.get(&shared));
+    assert!(matches!(&c.photo(known[&shared]).unwrap().source, Source::File { path } if path == "/pics/007/c.jpg"));
+}
+
+/// Given a session with a library, when an import is set up, then no table of the library is
+/// made on the spot (that is the session's thread); it is made when the first files are
+/// prepared, and then finds a file the library has by its path and one it has by its content.
+#[test]
+fn an_import_looks_the_library_up_when_it_prepares_files() {
+    let mut s = Session::new();
+    s.catalog = library(50);
+    let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap();
+    assert!(job.by_path.is_empty() && job.by_hash.is_empty() && job.library.is_some(), "nothing is looked up yet");
+    let out = job.prepare_files(vec!["/pics/007/a.jpg".into()], &std::sync::atomic::AtomicBool::new(false));
+    assert!(job.library.is_none(), "looked up once");
+    assert_eq!((job.by_path.len(), job.by_hash.len()), (150, 100));
+    assert!(matches!(out.items.as_slice(), [PreparedItem::Duplicate { reason: "path", existing: Some(_), .. }]), "found by its path");
+    // what the options said when the import was set up decides, whatever is made of them after
+    let id = s.catalog.photos().next().unwrap().id;
+    s.catalog.apply(Op::SetDeleted { id, deleted: true }).unwrap();
+    let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, on_deleted: OnDeleted::Fresh, ..ImportOptions::default() }).unwrap();
+    job.opts.on_deleted = OnDeleted::Skip;
+    job.prepare_files(Vec::new(), &std::sync::atomic::AtomicBool::new(false));
+    assert_eq!(job.by_path.len(), 149, "the trashed photo was passed over, as set up");
+}
+
+/// Given a photo in Recently Deleted and an import that replaces such photos with a fresh
+/// import, when the library is looked up, then that photo is no duplicate (as before the lookup
+/// moved to the first files).
+#[test]
+fn a_trashed_photo_is_not_looked_up_when_imports_replace_it() {
+    let mut s = Session::new();
+    s.catalog = library(2);
+    // (a photo whose content is its own: nobody else in this library has it)
+    let id = s.catalog.photos().find(|p| p.file_name == "b.jpg").unwrap().id;
+    s.catalog.apply(Op::SetDeleted { id, deleted: true }).unwrap();
+    for (on_deleted, expected) in [(OnDeleted::Fresh, 5), (OnDeleted::Skip, 6)] {
+        let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, on_deleted, ..ImportOptions::default() }).unwrap();
+        job.prepare_files(Vec::new(), &std::sync::atomic::AtomicBool::new(false));
+        assert_eq!(job.by_path.len(), expected, "{on_deleted:?}");
+        let its_content = s.catalog.photo(id).and_then(|p| p.content_hash.clone()).unwrap();
+        assert_eq!(job.by_hash.contains_key(&its_content), on_deleted == OnDeleted::Skip, "{on_deleted:?}");
+    }
+}

crates/engine/src/sidecar.rs

@@ -450,6 +450,33 @@ fn stem_key(original: &str) -> String {
     sidecar_path(original, SidecarNaming::Stem).to_string_lossy().to_lowercase()
 }
 
+/// Where a plain path is cut to make its [`stem_key`]: at the dot of its extension, which is
+/// what `with_extension` replaces. A plain path is ASCII and ends in its file's name with an
+/// extension, which is what a library's paths are; `None` for any other shape.
+fn plain_cut(original: &str) -> Option<usize> {
+    let path = Path::new(original);
+    let (name, stem, ext) = (path.file_name()?.to_str()?, path.file_stem()?.to_str()?, path.extension()?.to_str()?);
+    // (the path ends in its file's name, as it stands: no `/.` or separator after it; and the
+    // name has a stem and an extension of its own, which `..jpg` and `a.` have not)
+    let plain = original.is_ascii() && original.ends_with(name) && !ext.is_empty() && !stem.is_empty() && stem != "." && stem != "..";
+    let cut = original.len().checked_sub(ext.len().saturating_add(1))?;
+    (plain && original.as_bytes().get(cut) == Some(&b'.')).then_some(cut)
+}
+
+/// [`stem_key`] written into `key`, which is reused: nothing is allocated for a plain path
+/// ([`plain_cut`]). Any other shape goes the long way.
+fn stem_key_into(key: &mut String, original: &str) {
+    key.clear();
+    match plain_cut(original) {
+        Some(cut) => {
+            key.push_str(original.get(..cut).unwrap_or_default());
+            key.make_ascii_lowercase();
+            key.push_str(".xmp");
+        }
+        None => key.push_str(&stem_key(original)),
+    }
+}
+
 fn file_path(p: &Photo) -> Option<&str> {
     match &p.source {
         Source::File { path } => Some(path),
@@ -490,8 +517,11 @@ pub(crate) fn op_photos(op: &Op, out: &mut Vec<PhotoId>) {
 pub(crate) struct StemOwners(HashMap<String, PhotoId>);
 
 impl StemOwners {
+    /// The owners of every shared stem in the library, each file's key made the long way: what
+    /// [`StemOwners::among`] must agree with for the photos it is asked about (this is how the
+    /// owners were worked out before, for every write).
+    #[cfg(test)]
     pub(crate) fn of(cat: &lightcraft_catalog::Catalog) -> StemOwners {
-        // per stem: (raw first, then file name, then id) of the best candidate; and how many
         let mut best: HashMap<String, ((bool, String, u64), PhotoId, usize)> = HashMap::new();
         for p in cat.photos().filter(|p| p.copy_of.is_none()) {
             let Some(path) = file_path(p) else { continue };
@@ -505,6 +535,39 @@ impl StemOwners {
         StemOwners(best.into_iter().filter(|(_, v)| v.2 > 1).map(|(k, v)| (k, v.1)).collect())
     }
 
+    /// The owners of the stems photos `ids` map to: all that naming those photos needs. The
+    /// library is passed over once, and only files of those stems are looked at closer, so a
+    /// sidecar written for one photo does not cost a table of the whole library.
+    pub(crate) fn among(cat: &lightcraft_catalog::Catalog, ids: &[PhotoId]) -> StemOwners {
+        let wanted: std::collections::HashSet<String> =
+            ids.iter().filter_map(|id| cat.photo(*id)).filter_map(|p| file_path(p)).map(stem_key).collect();
+        if wanted.is_empty() {
+            return StemOwners(HashMap::new());
+        }
+        Self::where_wanted(cat, &wanted)
+    }
+
+    /// The owners of the shared stems in `wanted`.
+    fn where_wanted(cat: &lightcraft_catalog::Catalog, wanted: &std::collections::HashSet<String>) -> StemOwners {
+        // per stem: (raw first, then file name, then id) of the best candidate; and how many
+        let mut best: HashMap<String, ((bool, String, u64), PhotoId, usize)> = HashMap::new();
+        let mut key = String::new();
+        for p in cat.photos().filter(|p| p.copy_of.is_none()) {
+            let Some(path) = file_path(p) else { continue };
+            stem_key_into(&mut key, path);
+            if !wanted.contains(key.as_str()) {
+                continue;
+            }
+            let rank = (p.kind != MediaKind::Raw, path.to_lowercase(), p.id.0);
+            let e = best.entry(key.clone()).or_insert_with(|| (rank.clone(), p.id, 0));
+            e.2 += 1;
+            if rank < e.0 {
+                (e.0, e.1) = (rank, p.id);
+            }
+        }
+        StemOwners(best.into_iter().filter(|(_, v)| v.2 > 1).map(|(k, v)| (k, v.1)).collect())
+    }
+
     /// `naming` for photo `p`, except that a stem sidecar another file owns becomes Full.
     fn naming(&self, p: &Photo, naming: SidecarNaming) -> SidecarNaming {
         match (naming, file_path(p)) {
@@ -514,33 +577,39 @@ impl StemOwners {
     }
 }
 
+/// How the sidecar of each of photos `ids` is named in library `cat`, whose naming is `naming`
+/// ([`Session::sidecar_namings`], for a thread that has the library and not the session).
+pub(crate) fn namings_in(cat: &lightcraft_catalog::Catalog, naming: SidecarNaming, ids: &[PhotoId]) -> Vec<SidecarNaming> {
+    let owners = (naming == SidecarNaming::Stem).then(|| StemOwners::among(cat, ids));
+    ids.iter()
+        .map(|id| match (cat.photo(*id), &owners) {
+            (Some(p), Some(o)) => o.naming(p, naming),
+            _ => naming,
+        })
+        .collect()
+}
+
 impl Session {
     /// The sidecar naming for photo `id`: the library's, except under Stem naming when another
     /// catalogued file shares the stem (`IMG_0001.CR3` + `IMG_0001.JPG`) and owns the stem
     /// sidecar — a raw first, else the first by file name. The others use Full naming
     /// (`IMG_0001.JPG.xmp`), so their metadata never overwrites each other.
-    /// [`Session::sidecar_naming`] for many photos, the catalog's sidecar owners worked out once
-    /// (per photo that is a pass over the whole catalog).
+    /// [`Session::sidecar_naming`] for many photos, their stems' owners worked out once (per
+    /// photo that is a pass over the whole catalog).
     pub fn sidecar_namings(&self, ids: &[PhotoId]) -> Vec<SidecarNaming> {
-        let owners = (self.xmp.naming == SidecarNaming::Stem).then(|| StemOwners::of(&self.catalog));
-        ids.iter()
-            .map(|id| match (self.catalog.photo(*id), &owners) {
-                (Some(p), Some(o)) => o.naming(p, self.xmp.naming),
-                _ => self.xmp.naming,
-            })
-            .collect()
+        namings_in(&self.catalog, self.xmp.naming, ids)
     }
 
     pub fn sidecar_naming(&self, id: PhotoId) -> SidecarNaming {
         match self.catalog.photo(id) {
-            Some(p) if self.xmp.naming == SidecarNaming::Stem => StemOwners::of(&self.catalog).naming(p, self.xmp.naming),
+            Some(p) if self.xmp.naming == SidecarNaming::Stem => StemOwners::among(&self.catalog, &[id]).naming(p, self.xmp.naming),
             _ => self.xmp.naming,
         }
     }
 
     /// Write the photo's XMP sidecar, merged into an existing one (see the module docs).
     pub fn save_sidecar(&mut self, id: PhotoId) -> Result<SidecarSaved> {
-        self.save_sidecar_with(id, &StemOwners::of(&self.catalog))
+        self.save_sidecar_with(id, &StemOwners::among(&self.catalog, &[id]))
     }
 
     pub(crate) fn save_sidecar_with(&mut self, id: PhotoId, owners: &StemOwners) -> Result<SidecarSaved> {
@@ -642,7 +711,7 @@ impl Session {
         if ids.is_empty() {
             return;
         }
-        let owners = StemOwners::of(&self.catalog);
+        let owners = StemOwners::among(&self.catalog, &ids);
         for id in ids {
             if self.catalog.photo(id).is_some_and(|p| file_path(p).is_some() && p.copy_of.is_none())
                 && let Err(e) = self.save_sidecar_with(id, &owners)
@@ -764,3 +833,7 @@ mod tests_repair_companions;
 #[cfg(test)]
 #[path = "sidecar/tests_repair_provenance.rs"]
 mod tests_repair_provenance;
+
+#[cfg(test)]
+#[path = "sidecar/tests_stem_owners.rs"]
+mod tests_stem_owners;

crates/engine/src/sidecar/tests_stem_owners.rs

@@ -0,0 +1,163 @@
+//! Who owns a stem sidecar, worked out for the photos asked about only (`StemOwners::among`).
+//!
+//! A library can hold millions of photos; a sidecar that is written for one of them, or a folder
+//! that is synchronized, must not cost a table of them all.
+use super::*;
+use lightcraft_catalog::{Catalog, Op, Photo, Source};
+
+/// Paths that try the key: case in folders, names and endings, no ending, a dot file, endings
+/// inside folder names, non-ASCII letters (one that lower-cases to ASCII: the Kelvin sign).
+const PATHS: [&str; 26] = [
+    "/pics/trip/IMG_1.CR3",
+    "/pics/trip/IMG_1.JPG",
+    "/pics/trip/img_1.jpeg",
+    "/Pics/Trip/IMG_1.tif",
+    "/pics/trip/IMG_2.JPG",
+    "/pics/trip/IMG_3.CR3",
+    "/pics/trip/IMG_3.cr3.JPG",
+    "/pics/trip/IMG_3.cr3",
+    "/pics/other/IMG_1.JPG",
+    "/pics/other/IMG_1",
+    "/pics/other/.hidden",
+    "/pics/other/.hidden.jpg",
+    "/pics/v1.2/a.b.JPG",
+    "/pics/v1.2/a.b.cr3",
+    "/pics/v1.2/a",
+    "/pics/straße/Größe.JPG",
+    "/pics/STRASSE/GRÖSSE.cr3",
+    "/pics/straße/größe.CR3",
+    "/pics/kelvin/\u{212a}1.JPG",
+    "/pics/kelvin/k1.cr3",
+    "/pics/ΟΔΟΣ/ΟΔΟΣ.JPG",
+    "/pics/οδος/οδος.CR3",
+    // (names that are all dots before their ending: the ending is no extension to replace)
+    "/pics/dots/..cr3",
+    "/pics/dots/..jpg",
+    "/pics/dots/a./.",
+    "/pics/dots/a.jpg",
+];
+
+fn library() -> (Catalog, Vec<PhotoId>) {
+    let mut c = Catalog::new();
+    let mut ids = Vec::new();
+    for path in PATHS {
+        let id = c.alloc_photo_id();
+        let name = path.rsplit('/').next().unwrap_or(path);
+        let mut p = Photo::new(id, Source::File { path: path.to_string() }, name, "X", 6, 4, "2026-01-01");
+        if path.to_lowercase().ends_with("cr3") {
+            p.kind = MediaKind::Raw;
+        }
+        c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
+        ids.push(id);
+    }
+    // a virtual copy of the first (it owns nothing) and a photo that is no file
+    let copy = c.alloc_photo_id();
+    let mut p = Photo::new(copy, Source::File { path: PATHS[0].to_string() }, "IMG_1.CR3", "X", 6, 4, "2026-01-01");
+    p.copy_of = Some(ids[0]);
+    c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
+    ids.push(copy);
+    let demo = c.alloc_photo_id();
+    c.apply(Op::AddPhoto { photo: Box::new(Photo::new(demo, Source::Demo { scene: 1 }, "demo", "X", 6, 4, "2026-01-01")) }).unwrap();
+    ids.push(demo);
+    (c, ids)
+}
+
+fn namings(owners: &StemOwners, c: &Catalog, ids: &[PhotoId]) -> Vec<SidecarNaming> {
+    ids.iter().map(|id| owners.naming(c.photo(*id).unwrap(), SidecarNaming::Stem)).collect()
+}
+
+/// Given a library where files share stems in every way a name can, when the owners are worked
+/// out for some of its photos only, then each of those photos is named exactly as the table of
+/// the whole library names it: whichever photos are asked about, alone or together.
+#[test]
+fn owners_among_some_photos_name_them_as_the_whole_library_does() {
+    let (c, ids) = library();
+    let whole = StemOwners::of(&c);
+    let expected = namings(&whole, &c, &ids);
+    assert!(expected.contains(&SidecarNaming::Full) && expected.contains(&SidecarNaming::Stem), "both cases are there: {expected:?}");
+    // all at once
+    assert_eq!(namings(&StemOwners::among(&c, &ids), &c, &ids), expected);
+    // one at a time, and every pair
+    for (i, a) in ids.iter().enumerate() {
+        assert_eq!(namings(&StemOwners::among(&c, &[*a]), &c, &[*a]), [expected[i]], "{:?}", PATHS.get(i));
+        for (j, b) in ids.iter().enumerate() {
+            assert_eq!(namings(&StemOwners::among(&c, &[*a, *b]), &c, &[*a, *b]), [expected[i], expected[j]]);
+        }
+    }
+    // an id that is not in the library is passed over
+    assert_eq!(StemOwners::among(&c, &[PhotoId(9_999)]).0.len(), 0);
+}
+
+/// Given a library of many folders, when the owners are worked out for the photos of one, then
+/// the table holds that folder's shared stems and nothing of the others: what it costs follows
+/// the photos asked about, not the library.
+#[test]
+fn owners_among_some_photos_hold_nothing_of_the_rest() {
+    let mut c = Catalog::new();
+    let mut first = Vec::new();
+    for folder in 0..200 {
+        for name in ["IMG_1.CR3", "IMG_1.JPG", "IMG_2.JPG"] {
+            let id = c.alloc_photo_id();
+            let mut p = Photo::new(id, Source::File { path: format!("/pics/{folder:03}/{name}") }, name, "X", 6, 4, "2026-01-01");
+            if name.ends_with("CR3") {
+                p.kind = MediaKind::Raw;
+            }
+            c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
+            if folder == 0 {
+                first.push(id);
+            }
+        }
+    }
+    assert_eq!(StemOwners::of(&c).0.len(), 200, "one shared stem a folder");
+    let some = StemOwners::among(&c, &first);
+    assert_eq!(some.0.len(), 1, "the one shared stem of the folder asked about");
+    assert_eq!(namings(&some, &c, &first), [SidecarNaming::Stem, SidecarNaming::Full, SidecarNaming::Stem]);
+}
+
+/// The key written into a buffer (no allocation for a plain path) is the key: for every path
+/// above, for shapes a path from a file can have, and for every string of up to six of the
+/// characters that decide a key (a letter in each case, a dot, both separators, a letter that is
+/// not ASCII).
+#[test]
+fn the_key_in_a_buffer_is_the_key() {
+    let odd = [
+        "",
+        "/",
+        "a",
+        "a.",
+        ".a",
+        "..a",
+        "...a",
+        "a.b",
+        "a/b.c/",
+        "a/b.c//",
+        "a/.b",
+        "a/..b",
+        "a/b.",
+        "a/b..c",
+        "a./.",
+        "a\\b.c",
+        "C:\\x\\y.JPG",
+        "a/b.c/.",
+        "a/b.c/..",
+        "x.JPG ",
+        "É.JPG",
+    ];
+    let mut buf = String::new();
+    for path in PATHS.iter().chain(&odd) {
+        stem_key_into(&mut buf, path);
+        assert_eq!(buf, stem_key(path), "{path:?}");
+    }
+    let letters = ['a', 'B', '.', '/', '\\', 'é'];
+    let mut tried = 0usize;
+    let mut todo: Vec<String> = vec![String::new()];
+    while let Some(path) = todo.pop() {
+        stem_key_into(&mut buf, &path);
+        assert_eq!(buf, stem_key(&path), "{path:?}");
+        tried += 1;
+        if path.chars().count() < 6 {
+            todo.extend(letters.iter().map(|c| format!("{path}{c}")));
+        }
+    }
+    assert_eq!(tried, (0..=6).map(|n| 6usize.pow(n)).sum::<usize>());
+}

crates/engine/src/sync.rs

@@ -74,10 +74,14 @@ impl SyncProgress {
 pub struct SyncInput {
     folder: String,
     scan: ScanInput,
-    /// The library's photos in the folder, each with how its sidecar is named.
-    photos: Vec<(Arc<Photo>, SidecarNaming)>,
-    /// For the colour label names a sidecar may use.
-    labels: Catalog,
+    /// The library's photos in the folder.
+    ids: Vec<PhotoId>,
+    /// How the library names sidecars; how each photo's is named is worked out by the scan (a
+    /// pass over the library, which the session's thread does not wait for).
+    naming: SidecarNaming,
+    /// The library as it is now (the one the scan looks files up in): for those photos, their
+    /// sidecars' names and the colour label names a sidecar may use.
+    labels: Arc<Catalog>,
 }
 
 /// What changed in a folder since it came into the library.
@@ -176,10 +180,9 @@ impl SyncInput {
         if !disk && (lightcraft_catalog::folders::is_disk_root(path) || crate::cmd::library::covers_other_disks(s, path, &ids)) {
             return Err(invalid(format!("{path} is a whole disk or holds disks: pass `disk: true` to synchronize it")));
         }
-        let namings = s.sidecar_namings(&ids);
-        let photos = ids.iter().zip(namings).filter_map(|(id, naming)| s.catalog.photo(*id).map(|p| (Arc::clone(p), naming))).collect();
         let (scan, _) = ScanInput::new(s, &[path.to_string()]);
-        Ok(SyncInput { folder: path.to_string(), scan, photos, labels: s.catalog.clone() })
+        let labels = scan.library();
+        Ok(SyncInput { folder: path.to_string(), scan, ids, naming: s.xmp.naming, labels })
     }
 }
 
@@ -190,11 +193,16 @@ impl SyncInput {
 /// library's photos, which run on several threads too. Stops early when `progress.files.cancel`
 /// is set.
 pub fn scan_with(input: SyncInput, progress: &SyncProgress) -> FolderChanges {
-    let SyncInput { folder, scan, photos, labels } = input;
-    let seen: Vec<Arc<Photo>> = photos.iter().map(|(p, _)| Arc::clone(p)).collect();
+    let SyncInput { folder, scan, ids, naming, labels } = input;
+    let seen: Vec<Arc<Photo>> = ids.iter().filter_map(|id| labels.photo(*id)).map(Arc::clone).collect();
     if !cfg!(target_arch = "wasm32") && !Path::new(&folder).is_dir() {
         return FolderChanges { path: folder, seen, offline: true, ..Default::default() };
     }
+    // (how each photo's sidecar is named: a pass over the library, made here and not while the
+    // session waits, and not at all for a folder that is not there)
+    let namings = crate::sidecar::namings_in(&labels, naming, &ids);
+    let photos: Vec<(Arc<Photo>, SidecarNaming)> =
+        ids.iter().zip(namings).filter_map(|(id, naming)| labels.photo(*id).map(|p| (Arc::clone(p), naming))).collect();
     // (no threads on the web: one after the other there)
     #[cfg(target_arch = "wasm32")]
     let (out, checks) = (crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files), check_photos(&photos, &labels, progress));

docs/whats-new.md

@@ -48,6 +48,16 @@
   from or written through such a link.
 ## 2026-10-13
 
+### Large libraries: Synchronize Folder, import and sidecars start sooner
+- Opening Synchronize Folder, an import review or an import no longer makes a table of every photo in the library
+  before it looks at the first file. In a library of 3.5 million photos that held the window for about 6 s before
+  a Synchronize scan began (now under 1 s), 2.6 s before an import review (now 0.05 s) and 1.7 s before an import
+  (now 0.05 s). The scan looks up only the files it finds and the contents it reads, on its own thread.
+- Writing one photo's XMP sidecar (Save Metadata to File, or automatically after an edit) no longer works out who
+  owns every shared sidecar name in the library: 2.4 s a write in that library, now under 1 s.
+- Nothing else changes: the same files are found new, already there, moved or missing, and sidecars get the same
+  names.
+
 ### Slider drags on large raws
 - Dragging a slider no longer re-renders the edited photo's filmstrip thumbnail on every frame; it follows once you
   let go. On a 64 MP raw the first of those renders decoded the whole file again, which held up the loupe's