diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index bb81bd240..6fe665896 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -32,1730 +32,31 @@ use serde::{Deserialize, Serialize}; use crate::Session; use crate::media::ProbeInfo; -/// File extensions offered by import discovery and file pickers (lower case). -/// This includes preview-only raws; the file probe decides whether sensor decoding is available. -pub const EXTENSIONS: &[&str] = &[ - "jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "sr2", "srf", "raf", "orf", "rw2", "rwl", "raw", "pef", - "srw", "psd", "psb", "jxl", "gif", "bmp", - // Decoded when built with the codecs' `heif` feature; otherwise the import reports them as - // failed with the reason ("HEIC/HEIF support isn't included in this build"). - "heic", "heif", "avif", // no AV1 decoder yet: reported as failed, like a HEIC without the feature - "iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", "3fr", "fff", "mef", -]; +mod commit; +mod defaults; +mod files; +mod job; +mod scan; +mod types; -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum ImportMode { - /// Reference the files where they are. - #[default] - Add, - /// Copy the files into the library's `Originals/` folder. - Copy, - /// Move the files there: as Copy, then each original (and its sidecars) is removed from the - /// source once the copy is verified and catalogued. - Move, -} - -/// What an import does with a file the library already has in Recently Deleted. -/// -/// Someone who imports such a file wants the photo in the library again, so the default brings -/// it back (issues #298, #706: before, the import skipped it as "already in the library" while -/// the grid showed it deleted, and the user was stuck). Restoring loses nothing — the edits, -/// rating and albums come back with the photo and Reset Edits gives a clean start — where -/// importing afresh would silently drop them. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum OnDeleted { - /// Leave it there; the report says so ([`Duplicate::existing_deleted`]). - Skip, - /// Bring the trashed photo back, with its edits ([`ImportReport::restored`]); a photo whose - /// file is no longer where the library had it is relinked to the imported file. - #[default] - Restore, - /// Delete the trashed record permanently and import the file as a new photo. - Fresh, -} - -impl OnDeleted { - pub fn parse(s: &str) -> Option { - match s { - "skip" => Some(Self::Skip), - "restore" => Some(Self::Restore), - "fresh" => Some(Self::Fresh), - _ => None, - } - } -} - -/// What an import does besides adding the photos. -#[derive(Clone, Debug, Default)] -pub struct ImportOptions { - /// A file that is in Recently Deleted: restore (default), skip, or import afresh. - pub on_deleted: OnDeleted, - pub mode: ImportMode, - /// Applied to every imported photo (one History entry). - pub preset: Option, - /// Added to every imported photo. - pub keywords: Vec, - /// Browsing a folder: photos come in as `local` (not in the library), and a file with the - /// same content as one already known is still listed. - pub local: bool, - /// Copy: where the copies go (default: the library's `Originals/`). - pub destination: Option, - /// Copy: the folders inside the destination. - pub organize: Organize, - /// Copy: a file-name template for the copies ([`crate::rename::expand`] tokens: `{name}`, - /// `{seq:N}`, `{date:%Y%m%d}`, `{camera}`…), numbered from `rename_start`. - pub rename: Option, - pub rename_start: usize, - /// A metadata preset (by name) applied to every imported photo. - pub metadata_preset: Option, - /// Copy: raws are copied as DNG (Copy as DNG). - pub convert_dng: bool, -} - -/// How copies are filed in the destination. The date is the capture time, else the time of the -/// import. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub enum Organize { - /// `YYYY/YYYY-MM-DD/` by capture date. - #[default] - ByDay, - /// `YYYY/YYYY-MM/`. - ByMonth, - /// All in the destination itself. - Flat, - /// A folder template, e.g. `{date:%Y}/{date:%Y%m%d}` → `2026/20260114/`: the template's own - /// `/` make the levels, each expanded with the file-name tokens ([`crate::rename::expand_folder`]; - /// never outside the destination). - Template(String), -} - -impl Organize { - /// `date` | `month` | `flat`, or a folder template (anything with a `{`, `/` or `\`). - pub fn parse(s: &str) -> Option { - match s { - "date" | "day" | "byDay" => Some(Organize::ByDay), - "month" | "byMonth" => Some(Organize::ByMonth), - "flat" | "none" | "intoOneFolder" => Some(Organize::Flat), - t if t.contains(['{', '/', '\\']) => Some(Organize::Template(t.trim().to_string())), - _ => None, - } - } - - /// The folders (inside the destination) a copy of `p` goes to. - pub fn folders(&self, p: &Photo) -> Vec { - let date = p.date(); - let day = date.get(..10).filter(|d| d.len() == 10).unwrap_or("undated"); - let year = day.get(..4).unwrap_or("undated"); - match self { - Organize::ByDay => vec![year.into(), day.into()], - Organize::ByMonth => vec![year.into(), day.get(..7).unwrap_or("undated").into()], - Organize::Flat => Vec::new(), - Organize::Template(t) => crate::rename::expand_folder(t, p, 1), - } - } -} - -/// A file found by [`scan`], for the import review. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(default, rename_all = "camelCase")] -pub struct ImportCandidate { - pub path: String, - pub name: String, - pub format: String, - pub kind: MediaKind, - pub width: u32, - pub height: u32, - pub file_size: u64, - pub captured: Option, - /// `"path"` (already in the library), `"content"` (same bytes in the library or earlier in this - /// list): skipped by the import. - pub duplicate: Option, - /// The photo that already has it. - pub existing: Option, - /// Not readable. - pub error: Option, - /// A raw variant that can't be decoded yet (why): it imports as preview only. - #[serde(skip_serializing_if = "Option::is_none")] - pub preview_only: Option, -} - -#[derive(Clone, Debug, Default, Serialize)] -pub struct Duplicate { - pub path: String, - /// The photo that already has these bytes (or this path). - pub existing: Option, - /// `"path"` (already imported from there) or `"content"` (same bytes elsewhere). - pub reason: &'static str, - /// That photo is in Recently Deleted (see [`OnDeleted`]). - #[serde(rename = "existingDeleted", skip_serializing_if = "std::ops::Not::not")] - pub existing_deleted: bool, -} - -#[derive(Clone, Debug, Default, Serialize)] -pub struct ImportReport { - pub imported: Vec, - pub duplicates: Vec, - /// [`OnDeleted::Restore`]: photos brought back from Recently Deleted instead of imported. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub restored: Vec, - /// Move: the originals that were moved (and removed from the source). - #[serde(skip_serializing_if = "Vec::is_empty")] - pub moved: Vec, - /// Move: originals (or sidecars) left at the source, and why — the photo may still have been - /// imported (from its copy, or in place). - #[serde(skip_serializing_if = "Vec::is_empty")] - pub kept: Vec, - /// (path, error) - pub failed: Vec<(String, String)>, - /// Files found after expanding folders. - pub scanned: usize, - /// Photos whose metadata/develop settings were read from an XMP sidecar (or embedded XMP). - pub sidecars: usize, - /// Per-photo diagnostics from foreign XMP develop settings that could not be fully carried over. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub warnings: Vec, -} - -/// A structured diagnostic from foreign XMP develop settings. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ImportWarningCode { - /// A named camera profile was found but cannot be applied as Lightroom does. - CameraProfile, - /// Lightroom geometry data was found but its saved transform is not retained. - UprightGeometry, - /// One or more other XMP develop fields have no LightCraft mapping. - UnmappedXmp, -} - -/// The saved Lightroom Upright mode associated with a foreign geometry transform. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ImportUprightMode { - Off, - Auto, - Level, - Vertical, - Full, - Guided, - Unknown, -} - -/// One actionable XMP import diagnostic. `path` is filled when its photo is committed. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ImportWarning { - pub path: String, - pub code: ImportWarningCode, - #[serde(skip_serializing_if = "Option::is_none")] - pub profile: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mode: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub fields: Vec, -} - -/// A file moved by an import. -#[derive(Clone, Debug, Default, Serialize)] -pub struct Moved { - pub from: String, - pub to: String, - /// Its sidecars' new paths. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub sidecars: Vec, -} - -/// A source a Move left where it was. -#[derive(Clone, Debug, Default, Serialize)] -pub struct Kept { - pub path: String, - pub reason: String, -} - -/// The develop settings a photo gets on import: raws start from their as-shot white balance with -/// default sharpening / colour noise reduction, and file-embedded lens corrections on (as the -/// camera intended); a user default preset ([`ImportDefaults`]) goes on top. -pub fn import_defaults(p: &Photo) -> lightcraft_develop::DevelopSettings { - p.import_defaults() -} - -/// Does the photo still look as the camera rendered it (its embedded camera preview is then a -/// fair stand-in)? Not when a default preset changed it on import. -pub fn has_import_look(p: &Photo) -> bool { - *p.develop == p.camera_defaults() -} - -/// User defaults applied on import (Settings → Import; saved in the library's `prefs.json`). -/// Presets are referenced by id; a preset that no longer exists is ignored. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(default, rename_all = "camelCase")] -pub struct ImportDefaults { - /// Preset applied to raw files (`None` = the LightCraft default). - pub raw_preset: Option, - /// Use a camera's own default (below) when the photo's camera has one. - pub per_camera: bool, - /// Per-camera raw defaults, keyed by the camera's make + model as shown in Info - /// (`"Canon EOS R5"`). - pub cameras: Vec, - /// Preset applied to non-raw images (JPEG, PNG, TIFF, HEIC…; `None` = none). - pub other_preset: Option, - /// Copyright notice given to imported photos that don't carry one (empty = none). - pub copyright: String, - /// Creator given to imported photos that don't name one (empty = none). - pub creator: String, - /// Metadata preset applied to every imported photo (`metadata.*`; `None` = none). - pub metadata_preset: Option, - /// Auto import: a watched folder whose new photos are added as they arrive (`None` = off), - /// copied into the library's Originals instead of added in place when `auto_copy`, and an - /// album they go to. - pub auto_folder: Option, - pub auto_copy: bool, - pub auto_album: Option, - /// Open the import review when a camera or memory card is connected. - pub show_on_card: bool, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(default, rename_all = "camelCase")] -pub struct CameraDefault { - /// Make + model (`Meta::camera`). - pub camera: String, - /// Preset id; `None` = the LightCraft default for this camera. - pub preset: Option, -} - -impl ImportDefaults { - /// The preset id that applies to a photo of this kind and camera, if any. - pub fn preset_for(&self, raw: bool, camera: &str) -> Option<&str> { - if !raw { - return self.other_preset.as_deref(); - } - if self.per_camera - && let Some(c) = self.cameras.iter().find(|c| !camera.is_empty() && c.camera.eq_ignore_ascii_case(camera)) - { - return c.preset.as_deref(); - } - self.raw_preset.as_deref() - } -} - -/// Give a freshly imported photo its default settings: the camera defaults, then the matching -/// default preset ([`ImportDefaults::preset_for`]), remembered as the photo's import look. -pub fn apply_import_defaults(s: &Session, p: &mut Photo) { - // metadata defaults fill gaps only: the file's own copyright / creator win - let d = &s.import_defaults; - if p.meta.copyright.trim().is_empty() && !d.copyright.trim().is_empty() { - p.meta.copyright = d.copyright.trim().to_string(); - } - if p.meta.creator.trim().is_empty() && !d.creator.trim().is_empty() { - p.meta.creator = d.creator.trim().to_string(); - } - if let Some(mp) = d.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) { - crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields); - } - let base = p.camera_defaults(); - // a raw shown from its embedded JPEG gets the rendered-file default, not the raw one - let raw = p.develops_raw(); - let preset = s.import_defaults.preset_for(raw, &p.meta.camera).and_then(|id| s.presets.iter().find(|x| x.id == id)); - match preset { - Some(pr) => { - let look = std::sync::Arc::new(pr.apply(&base, 1.0)); - p.develop = look.clone(); - p.import_look = (*look != base).then_some(look); - } - None => { - p.develop = std::sync::Arc::new(base); - p.import_look = None; - } - } -} - -pub fn is_supported(path: &Path) -> bool { - path.extension().is_some_and(|e| EXTENSIONS.contains(&e.to_string_lossy().to_lowercase().as_str())) -} - -/// The web file picker's accept list, derived from [`EXTENSIONS`]. Each extension -/// is listed in both cases: browsers' `accept` matching is case-insensitive per -/// spec, but some implementations (e.g. Windows/Edge) match case-sensitively and -/// rejected uppercase `.NEF` files, so a lowercase-only list silently hid raws -/// such as CR3, NRW, RAF, ORF, RW2, RWL, RAW, PEF and SRW from the picker. -/// -/// The picker only offers files this build can actually import: `.avif` has no -/// AV1 decoder in any build, and `.heic`/`.heif` decode only when the codecs' -/// `heif` feature is enabled ([`lightcraft_codecs::Format::Heif::can_decode`]). -/// [`EXTENSIONS`] itself stays complete — files on disk are still recognised, -/// and import reports them as failed, saying why. -pub fn accept_list() -> String { - let heif_decodable = lightcraft_codecs::Format::Heif.can_decode(); - let mut out = String::new(); - for &ext in EXTENSIONS { - // Skip extensions a web user could pick but this build could not import. - let pickable = match ext { - "avif" => false, - "heic" | "heif" => heif_decodable, - _ => true, - }; - if !pickable { - continue; - } - if !out.is_empty() { - out.push(','); - } - out.push('.'); - out.push_str(ext); - out.push(','); - out.push('.'); - out.push_str(&ext.to_uppercase()); - } - out -} - -/// Expand files and folders (recursively) into supported files. `skip` (e.g. the library folder) -/// is never descended into. Each folder's walk is bounded ([`crate::walk::Limits::default`]); a -/// walk that stops at a bound is logged. -pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec { - expand_within(paths, skip, crate::walk::Limits::default()).0 -} - -/// [`expand`] with the walk bounded by `limits` → (files, whether a walk stopped at a bound). -pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk::Limits) -> (Vec, bool) { - expand_noting(paths, skip, limits, None) -} - -/// [`expand_within`], handing the listing of every folder that is walked to `note` -/// ([`crate::walk::files_in_noting`]). -pub(crate) fn expand_noting( - paths: &[String], - skip: Option<&Path>, - limits: crate::walk::Limits, - mut note: Option, -) -> (Vec, bool) { - let mut out = Vec::new(); - let mut truncated = false; - for p in paths { - let p = Path::new(p); - if skip.is_some_and(|s| p == s) { - continue; - } - if p.is_dir() { - let w = match note.as_mut() { - Some(note) => crate::walk::files_in_noting(p, skip, limits, is_supported, &mut **note), - None => crate::walk::files_in(p, skip, limits, is_supported), - }; - if w.truncated { - log::warn!( - "import: stopped looking in {} after {} entries (limits: {} entries, {} folder levels)", - p.display(), - w.entries, - limits.max_entries, - limits.max_depth - ); - } - truncated |= w.truncated; - out.extend(w.files.into_iter().map(|f| f.to_string_lossy().to_string())); - } else { - // explicitly named files are attempted even with an unknown extension (sniffed) - out.push(p.to_string_lossy().to_string()); - } - } - let mut seen = std::collections::HashSet::new(); - out.retain(|p| seen.insert(p.clone())); - (out, truncated) -} - -/// What a path given on the command line at launch becomes ([`launch_path`]). -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum LaunchPath { - /// Import this (absolute) file or folder, in the background. - Import(String), - /// Not imported: `path` (absolute) is a whole drive or the whole home folder, which is never - /// walked without being chosen in the app (issue #374). - Refused { path: String, why: String }, -} - -/// Resolve a launch argument for importing (issue #374). An empty argument is ignored (`None`); -/// a relative one (`.`, `..`, `photos`; `/` or `\` on Windows, the current drive's root) is -/// resolved against `cwd` — the working folder a launcher picked, often a drive root or -/// `C:\Windows\System32` — and refused when that leaves no folder (`cwd` unknown). A drive -/// or file-system root is replaced by its camera folder (`DCIM`, from `dcim`: a memory card) or -/// refused, and so is the home folder `home`: either would walk most of the disk. -pub fn launch_path(arg: &str, cwd: Option<&Path>, home: Option<&Path>, dcim: impl Fn(&Path) -> Option) -> Option { - if arg.trim().is_empty() { - return None; - } - let given = Path::new(arg); - let joined = if given.is_absolute() { - given.to_path_buf() - } else { - match cwd { - Some(c) => c.join(given), - None => return Some(LaunchPath::Refused { path: arg.to_string(), why: "a relative path, and the working folder is unknown".into() }), - } - }; - // `.` and `..` resolved by name (no file-system access) - let mut abs = PathBuf::new(); - for c in joined.components() { - match c { - std::path::Component::CurDir => {} - std::path::Component::ParentDir => { - abs.pop(); - } - other => abs.push(other.as_os_str()), - } - } - let shown = abs.to_string_lossy().to_string(); - if abs.parent().is_none() { - return Some(match dcim(&abs) { - Some(d) => LaunchPath::Import(d.to_string_lossy().to_string()), - None => LaunchPath::Refused { path: shown, why: "a whole drive".into() }, - }); - } - let key = lightcraft_catalog::query::folder_key; - if home.is_some_and(|h| key(&h.to_string_lossy()) == key(&shown)) { - return Some(LaunchPath::Refused { path: shown, why: "the whole home folder".into() }); - } - Some(LaunchPath::Import(shown)) -} - -/// Probe files (headers + content hash) as an import would. -pub(crate) fn probe_paths(s: &Session, paths: &[String]) -> Vec> { - let progress = ScanProgress::default(); - probe_all(s.media.file_probe.as_ref(), paths, &progress.done, &progress.cancel) -} - -/// Probe files with `probe` (a clone of the session's file probe), off the session: on a worker thread. -pub(crate) fn probe_with(probe: &crate::media::FileProbe, paths: &[String]) -> Vec> { - let progress = ScanProgress::default(); - probe_all(Some(probe), paths, &progress.done, &progress.cancel) -} - -/// Probe `paths` (several at once), counting each in `done`. Stops before the next file once -/// `cancel` is set: the files not reached come back as errors. -pub(crate) fn probe_all( - probe: Option<&crate::media::FileProbe>, - paths: &[String], - done: &std::sync::atomic::AtomicUsize, - cancel: &std::sync::atomic::AtomicBool, -) -> Vec> { - use std::sync::atomic::Ordering::Relaxed; - let Some(probe) = probe.cloned() else { - return paths - .iter() - .map(|p| { - Ok(ProbeInfo { - format: Path::new(p).extension().map(|e| e.to_string_lossy().to_uppercase()).unwrap_or_default(), - ..Default::default() - }) - }) - .collect(); - }; - #[cfg(not(target_arch = "wasm32"))] - { - let n = workers(paths.len()); - if n > 1 { - let mut results: Vec>> = vec![None; paths.len()]; - let next = std::sync::atomic::AtomicUsize::new(0); - let out = std::sync::Mutex::new(&mut results); - std::thread::scope(|sc| { - for _ in 0..n { - sc.spawn(|| { - loop { - let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if i >= paths.len() { - break; - } - if cancel.load(Relaxed) { - break; - } - let r = probe(&paths[i]); - done.fetch_add(1, Relaxed); - out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r); - } - }); - } - }); - return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect(); - } - } - paths - .iter() - .map(|p| { - if cancel.load(Relaxed) { - return Err("cancelled".into()); - } - let r = probe(p); - done.fetch_add(1, Relaxed); - r - }) - .collect() -} - -/// Copy `src` into `dir` as `name` (default: its own name), made unique with -1, -2…; returns the -/// new path. The copy is verified ([`crate::import_move::copy_new`]: a new file, synced and -/// checked against `probe_hash`, the content hash the probe computed — or compared byte for byte -/// with the source when there is none); a bad one is removed and the photo reported as failed. -fn copy_into(dir: &Path, src: &str, name: Option<&str>, probe_hash: Option<&str>) -> Result { - let name = name - .map(str::to_string) - .unwrap_or_else(|| Path::new(src).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "photo".into())); - let expect = probe_hash.and_then(lightcraft_preview::Hash128::parse); - crate::import_move::copy_new(Path::new(src), dir, &name, expect).map(|p| p.to_string_lossy().to_string()) -} - -/// What a [`scan`] needs from the session, so it can run on another thread (a folder on a network -/// share can take minutes to read). -pub struct ScanInput { - probe: Option, - skip: Option, - /// The library as it is now (photos are shared with the session, not copied): looked up for - /// the files the scan finds ([`library_files`]) and the contents it probes - /// ([`library_hashes`]), on the scan's thread. - library: std::sync::Arc, - cache: HashMap, -} - -/// The library's photos of `files`: path → (photo, its summary as a candidate that is there -/// already). One pass over the library, and a record only for a file that was found: a folder of -/// a hundred files in a library of millions costs a hundred records. -fn library_files(library: &lightcraft_catalog::Catalog, files: &[String]) -> HashMap { - let found: std::collections::HashSet<&Path> = files.iter().map(Path::new).collect(); - let mut by_path = HashMap::new(); - if found.is_empty() { - return by_path; - } - for p in library.photos() { - if let Source::File { path } = &p.source - && found.contains(Path::new(path)) - { - let c = ImportCandidate { - path: path.clone(), - format: p.format.clone(), - kind: p.kind, - width: p.width, - height: p.height, - file_size: p.file_size, - captured: p.captured.clone(), - duplicate: Some("path".into()), - existing: Some(p.id.0), - ..Default::default() - }; - by_path.insert(PathBuf::from(path), (p.id, c)); - } - } - by_path -} - -/// Which photo of the library has each of the contents `hashes` (the later one where several -/// have it). One pass over the library, and an entry only for a content that was asked about. -fn library_hashes<'a>(library: &lightcraft_catalog::Catalog, hashes: impl Iterator) -> HashMap { - let wanted: std::collections::HashSet<&str> = hashes.collect(); - let mut by_hash = HashMap::new(); - if wanted.is_empty() { - return by_hash; - } - for p in library.photos() { - if let Some(h) = &p.content_hash - && wanted.contains(h.as_str()) - { - by_hash.insert(h.clone(), p.id); - } - } - by_hash -} - -/// Progress and cancellation of a running [`scan_with`]. -#[derive(Default)] -pub struct ScanProgress { - /// Files to probe, set once the folders are expanded. - pub total: std::sync::atomic::AtomicUsize, - pub done: std::sync::atomic::AtomicUsize, - /// Shared with the scan's row in the activity stack (its ✕ sets it). - pub cancel: std::sync::Arc, -} - -/// The result of [`scan_with`]: the candidates, and the probes to keep for the import that follows. -pub struct ScanOutput { - pub candidates: Vec, - pub probes: HashMap, -} - -impl ScanInput { - pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec) { - let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); - // (no table of the library here: this is the session's thread, and the library may hold - // millions of photos. The scan looks up what it finds) - let library = std::sync::Arc::new(s.catalog.clone()); - let input = ScanInput { probe: s.media.file_probe.clone(), skip, library, cache: std::mem::take(&mut s.import_probes) }; - (input, paths.to_vec()) - } - - /// The folder a scan never enters (the library's own). - pub(crate) fn skip(&self) -> Option<&Path> { - self.skip.as_deref() - } - - /// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one). - pub(crate) fn library(&self) -> std::sync::Arc { - std::sync::Arc::clone(&self.library) - } -} - -/// Find and probe what importing `paths` would add, marking duplicates (by path, or by content -/// against the library and earlier candidates). Nothing is added; the probes are kept for the -/// import that follows. -pub fn scan(s: &mut Session, paths: &[String]) -> Vec { - let (input, paths) = ScanInput::new(s, paths); - let out = scan_with(input, &paths, &ScanProgress::default()); - s.import_probes = out.probes; - out.candidates -} - -/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it -/// has) when `progress.cancel` is set. -pub fn scan_with(input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput { - let files = expand(paths, input.skip.as_deref()); - scan_files(input, files, progress) -} - -/// [`scan_with`] for files that were found already (`paths` expanded, [`expand`]). -pub(crate) fn scan_files(mut input: ScanInput, files: Vec, progress: &ScanProgress) -> ScanOutput { - use std::sync::atomic::Ordering::Relaxed; - let by_path = library_files(&input.library, &files); - let todo: Vec = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect(); - progress.total.store(todo.len(), Relaxed); - // probes from a preceding `scan` are reused when the file is unchanged (same size) - let cached: Vec> = todo - .iter() - .map(|f| { - let info = input.cache.remove(f)?; - // (✕ set while the files' cached probes are checked: no more round trips to the disk; what is - // not reached counts as not probed, like the files after it) - if progress.cancel.load(Relaxed) { - return None; - } - let size = std::fs::metadata(f).map(|m| m.len()).ok(); - (size.is_none() || size == Some(info.file_size)).then_some(info) - }) - .collect(); - let missing: Vec = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect(); - progress.done.store(todo.len() - missing.len(), Relaxed); - let mut fresh = probe_all(input.probe.as_ref(), &missing, &progress.done, &progress.cancel).into_iter(); - let probed: Vec> = - cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); - crate::memory::release(); - let mut probes: HashMap> = todo.into_iter().zip(probed).collect(); - // (who in the library has a content that was probed: asked once the probes are in) - let probed_hashes = probes.values().filter_map(|r| r.as_ref().ok()).filter_map(|info| info.content_hash.as_deref()); - let mut seen_hash: HashMap> = - library_hashes(&input.library, probed_hashes).into_iter().map(|(h, id)| (h, Some(id.0))).collect(); - let mut out = Vec::with_capacity(files.len()); - let mut kept = HashMap::new(); - for f in files { - if let Some((_, c)) = by_path.get(Path::new(&f)) { - let mut c = c.clone(); - c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); - out.push(c); - continue; - } - let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); - let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() }; - match probes.remove(&f) { - Some(Ok(info)) => { - (c.format, c.kind, c.width, c.height, c.file_size, c.captured) = - (info.format.clone(), info.kind, info.width, info.height, info.file_size, info.captured.clone()); - // as the import will date it (a sidecar's date, when there is one, still wins there) - c.captured = c.captured.or_else(|| info.file_modified.clone()); - c.preview_only = info.preview_only.clone(); - if let Some(h) = &info.content_hash { - match seen_hash.get(h) { - Some(existing) => { - c.duplicate = Some("content".into()); - c.existing = *existing; - } - None => { - seen_hash.insert(h.clone(), None); - } - } - } - kept.insert(f, info); - } - Some(Err(e)) => c.error = Some(e), - None => c.error = Some("not probed".into()), - } - out.push(c); - } - ScanOutput { candidates: out, probes: kept } -} - -/// Import files/folders. See the module docs. -pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result { - import_with(s, paths, &ImportOptions { mode, ..Default::default() }) -} - -/// Import files/folders with options (preset, keywords). See the module docs. -/// -/// The file-system half ([`ImportJob::prepare`]: expanding folders, probing, reading sidecars, -/// copying / placing files) and the catalog half ([`commit_prepared`]) are separate so the app can -/// run the first on a worker thread; here both run in turn. -pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> crate::Result { - let mut job = ImportJob::new(s, opts.clone())?; - let prepared = job.prepare(paths, &std::sync::atomic::AtomicBool::new(false)); - // probes a scan left for files not imported now stay for a later import - s.import_probes.extend(std::mem::take(&mut job.cache)); - commit_prepared(s, &job.opts, &job.now, prepared) -} - -/// Everything the file-system half of an import needs, detached from the session so it can run -/// on another thread (a network share or a card can take minutes). Built by [`ImportJob::new`]; -/// [`ImportJob::prepare`] may be called for several batches in turn (renamed copies keep counting, -/// duplicates are found across batches); each batch is then [`commit_prepared`] on the session. -pub struct ImportJob { - pub opts: ImportOptions, - /// The mode in effect (Copy / Move become Add in a library without its own folder). - mode: ImportMode, - lib_dir: Option, - copy_root: Option, - /// The library as it was when the import was set up, until it is looked up - /// ([`ImportJob::look_up_library`], with the first files): setting an import up is the - /// session's thread, and a table of a library of millions of photos takes seconds. With it, - /// whether a photo in Recently Deleted is passed over (an import that replaces such photos, - /// as the options said when the import was set up). - library: Option<(lightcraft_catalog::Catalog, bool)>, - /// path → (photo, a Local browse record?) - by_path: HashMap, - /// content hash → the photo that has it (`None`: a file earlier in this import) - by_hash: HashMap>, - probe: Option, - file_bytes: Option, - /// Probes from a preceding scan (not read again). - cache: HashMap, - naming: crate::sidecar::SidecarNaming, - now: String, - seq: usize, - /// Files found so far (folders expanded) and handled so far, for progress. - pub total: std::sync::Arc, - pub done: std::sync::Arc, - /// Of those, the files that have to be read first (not probed by a preceding scan) and the - /// ones read so far: the step before they are handled. - pub read_total: std::sync::Arc, - pub read_done: std::sync::Arc, -} - -/// The outcome of [`ImportJob::prepare`] for one batch, in file order. -#[derive(Debug, Default)] -pub struct Prepared { - pub scanned: usize, - items: Vec, -} - -#[derive(Debug)] -enum PreparedItem { - /// A Local (browsed) record imported for real. - Promote(PhotoId), - Duplicate { - path: String, - existing: Option, - reason: &'static str, - hash: Option, - }, - Failed(String, String), - Kept(Kept), - Ready(Box), -} - -#[derive(Debug)] -struct ReadyFile { - path: String, - stored: String, - info: ProbeInfo, - sidecar: Option, - placed: Option, -} - -impl Prepared { - /// Undo what the batch did on disk that only a commit would make permanent: files placed by a - /// Move (their sources were never touched). Copies stay, like copies of a failed commit. - pub fn rollback(&self) { - for it in &self.items { - if let PreparedItem::Ready(r) = it - && let Some(pl) = &r.placed - { - crate::import_move::rollback(pl); - } - } - } - - /// Entries in the batch (after expanding folders). - pub fn len(&self) -> usize { - self.items.len() - } - - pub fn is_empty(&self) -> bool { - self.items.is_empty() - } - - /// Recheck Add-only entries against the owner catalog immediately before commit. - /// Preparation may have run while another import added the same path or content. - pub fn revalidate_add(&mut self, s: &Session) { - let mut paths = HashMap::new(); - let mut hashes = HashMap::new(); - for photo in s.catalog.photos() { - if let Source::File { path } = &photo.source { - paths.insert(normalize_import_path(path), photo.id); - } - if let Some(hash) = &photo.content_hash { - hashes.insert(hash.clone(), photo.id); - } - } - let mut items = Vec::with_capacity(self.items.len()); - for item in std::mem::take(&mut self.items) { - let PreparedItem::Ready(ready) = item else { - items.push(item); - continue; - }; - let ReadyFile { path, stored, info, sidecar, placed } = *ready; - let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied()); - let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied()); - if let Some(existing) = existing_path.or(existing_hash) { - if let Some(placed) = &placed { - crate::import_move::rollback(placed); - } - items.push(PreparedItem::Duplicate { - path, - existing: Some(existing), - reason: if existing_path.is_some() { "path" } else { "content" }, - hash: info.content_hash, - }); - } else { - items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed }))); - } - } - self.items = items; - } -} - -fn normalize_import_path(path: &str) -> String { - let path = path.replace('\\', "/"); - if cfg!(windows) { path.to_lowercase() } else { path } -} - -impl ImportJob { - /// Snapshot what an import with `opts` needs from the session (no file-system calls). Takes - /// the probes a preceding scan left ([`Session::import_probes`]). - pub fn new(s: &mut Session, opts: ImportOptions) -> crate::Result { - let mode = opts.mode; - // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of - // every added file in its own storage already, so "copy" there means "add". - let transfer = matches!(mode, ImportMode::Copy | ImportMode::Move); - let mode = if transfer && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode }; - if mode == ImportMode::Move && opts.local { - return Err(crate::EngineError::Other("browsing a folder never moves its files".into())); - } - let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); - // where copies go: the chosen folder, else the library's Originals/ - let copy_root = opts - .destination - .as_deref() - .filter(|d| !d.trim().is_empty()) - .map(std::path::PathBuf::from) - .or_else(|| lib_dir.as_ref().map(|l| l.join("Originals"))); - if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() { - return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into())); - } - let fresh = opts.on_deleted == OnDeleted::Fresh; - Ok(ImportJob { - seq: opts.rename_start.max(1), - opts, - mode, - lib_dir, - copy_root, - library: Some((s.catalog.clone(), fresh)), - by_path: HashMap::new(), - by_hash: HashMap::new(), - probe: s.media.file_probe.clone(), - file_bytes: s.media.file_bytes.clone(), - cache: std::mem::take(&mut s.import_probes), - naming: s.xmp.naming, - now: (s.clock)(), - total: Default::default(), - done: Default::default(), - read_total: Default::default(), - read_done: Default::default(), - }) - } - - /// The file-system half of importing `paths`: expand folders, skip known paths, probe, skip - /// duplicates by content, read sidecars, copy or place (Move) the files — several at once, - /// with the names and outcomes of one after another. Nothing in the catalog changes. Stops - /// before the next file once `cancel` is set. - pub fn prepare(&mut self, paths: &[String], cancel: &std::sync::atomic::AtomicBool) -> Prepared { - let files = self.expand(paths); - self.prepare_files(files, cancel) - } - - /// Expand `paths` (folders recursively, the library's own folder skipped) into the files an - /// import would handle; counted in [`ImportJob::total`]. - pub fn expand(&self, paths: &[String]) -> Vec { - let files = expand(paths, self.lib_dir.as_deref()); - self.total.fetch_add(files.len(), std::sync::atomic::Ordering::Relaxed); - files - } - - /// When the import started ([`Session::clock`]), for [`commit_prepared`]. - pub fn now(&self) -> &str { - &self.now - } - - /// What the library held when the import was set up, by path and by content: made once, with - /// the first files (on the thread that prepares them), from the library as it was then. The - /// snapshot comes back: the files the import is adding are ranked against it for the ownership - /// of the shared stem sidecars they read ([`crate::sidecar::ImportStems`]). - fn look_up_library(&mut self) -> Option { - let (library, fresh) = self.library.take()?; - for p in library.photos() { - // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate - if p.deleted && fresh { - continue; - } - if let Source::File { path } = &p.source { - self.by_path.insert(path.clone(), (p.id, p.local)); - } - if let Some(h) = &p.content_hash { - self.by_hash.insert(h.clone(), Some(p.id)); - } - } - Some(library) - } - - /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned. - pub fn prepare_files(&mut self, files: Vec, cancel: &std::sync::atomic::AtomicBool) -> Prepared { - use std::sync::atomic::Ordering::Relaxed; - // the library as it was, kept for the ownership of the shared stem sidecars below - let library = self.look_up_library(); - let opts = self.opts.clone(); - let mode = self.mode; - let mut out = Prepared { scanned: files.len(), items: Vec::new() }; - let mut todo = Vec::new(); - for f in files { - match self.by_path.get(f.as_str()).copied() { - Some((id, true)) if !opts.local => { - // a file that was only browsed (Local) joins the library when it is imported for real - self.by_path.insert(f, (id, false)); - out.items.push(PreparedItem::Promote(id)); - self.done.fetch_add(1, Relaxed); - } - Some((id, _)) => { - out.items.push(PreparedItem::Duplicate { path: f, existing: Some(id), reason: "path", hash: None }); - self.done.fetch_add(1, Relaxed); - } - None => todo.push(f), - } - } - - // files a preceding scan already probed are not read again (a network share is slow to read), - // as long as they are still there, as they were: a file gone or changed since is probed again - let cached: Vec> = todo - .iter() - .map(|f| { - // (the web's files are not std::fs files: kept as probed) - self.cache - .remove(f) - .filter(|info| cfg!(target_arch = "wasm32") || std::fs::metadata(f).is_ok_and(|m| m.is_file() && m.len() == info.file_size)) - }) - .collect(); - let missing: Vec = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect(); - // Reading the files is the slow part of adding them where they are: it is counted on its - // own, and `cancel` stops it too (a large library takes long enough to want both). - self.read_total.fetch_add(missing.len(), Relaxed); - let mut fresh = probe_all(self.probe.as_ref(), &missing, &self.read_done, cancel).into_iter(); - let probed: Vec> = - cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); - crate::memory::release(); - // Who owns the shared stem sidecar (`IMG_0001.xmp`) among the library's files and the ones - // arriving with this import: a JPEG or a TIFF arriving beside its raw reads its own sidecar - // and never the raw's packet (issue #1905). - let stems = crate::sidecar::ImportStems::for_import( - library.as_ref(), - todo.iter().zip(&probed).filter_map(|(path, info)| info.as_ref().ok().map(|i| (path.as_str(), i.kind))), - ); - // Files are planned one by one, in order (duplicates, sidecars, names, folders); their - // copies or moves — the slow part, reading and writing every byte and verifying it — are - // collected and run on several threads (issue #367), then their outcomes take their places. - let mut plan = Plan::default(); - for (path, info) in todo.into_iter().zip(probed) { - if cancel.load(Relaxed) { - break; - } - let mut info = match info { - Ok(i) => i, - Err(e) => { - log::warn!("import {path}: {e}"); - self.done.fetch_add(1, Relaxed); - plan.slots.push(Slot::Item(PreparedItem::Failed(path, e))); - continue; - } - }; - // a copy of a file whose transfer is still pending: a duplicate only if that one succeeds - if !opts.local - && let Some(h) = &info.content_hash - && plan.transfers.iter().any(|t| t.info.content_hash.as_ref() == Some(h)) - { - self.run_plan(&mut plan, &mut out, cancel); - if cancel.load(Relaxed) { - break; - } - } - if let Some(h) = &info.content_hash - && !opts.local - && let Some(existing) = self.by_hash.get(h) - { - self.done.fetch_add(1, Relaxed); - plan.slots.push(Slot::Item(PreparedItem::Duplicate { path, existing: *existing, reason: "content", hash: Some(h.clone()) })); - continue; - } - // the XMP sidecar (or a raw's embedded XMP), read before copying: its capture time files - // and names the copy when the file itself has none - let raw = info.kind == MediaKind::Raw; - let packet = crate::sidecar::find_sidecar_for_import(&stems, self.naming, &path) - .and_then(|f| std::fs::read_to_string(f).ok()) - .or_else(|| info.xmp.clone().filter(|_| raw)); - let target = crate::crs::Target::for_file(info.kind, &info.format, info.preview_only.is_some(), &info.meta.camera); - let sidecar = packet.and_then(|x| match crate::sidecar::parse_sidecar(&x, target) { - Ok(sc) => Some(sc), - Err(e) => { - log::warn!("import {path}: XMP: {e}"); - None - } - }); - if info.captured.is_none() { - info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone()); - } - // no date in the file or its sidecar: the file's own date, as Lightroom does - if info.captured.is_none() { - info.captured = info.file_modified.clone(); - } - let stored = match (mode, &self.copy_root) { - (ImportMode::Copy | ImportMode::Move, Some(root)) - if !crate::import_move::inside(Path::new(&path), root) - && !self.lib_dir.as_ref().is_some_and(|l| crate::import_move::inside(Path::new(&path), l)) => - { - // the templates see the photo as it will be catalogued (undated: the import time) - let own = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - let mut q = Photo::new(PhotoId(0), Source::File { path: path.clone() }, &own, &info.format, 0, 0, &self.now); - q.captured = info.captured.clone(); - q.meta = info.meta.clone(); - let name = opts.rename.as_deref().filter(|t| !t.trim().is_empty()).map(|t| { - let n = crate::rename::expand(t, &q, self.seq); - self.seq += 1; - n - }); - let folders = opts.organize.folders(&q); - let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f)); - // the name this file gets depends on what a pending transfer of a like-named file - // into the same folder ends up with: that one goes first - let as_named = name.as_deref().unwrap_or(if own.is_empty() { "photo" } else { own.as_str() }); - let family = (dir.to_string_lossy().to_lowercase(), name_family(as_named)); - if plan.transfers.iter().any(|t| t.family == family) { - self.run_plan(&mut plan, &mut out, cancel); - if cancel.load(Relaxed) { - break; - } - } - let how = if mode == ImportMode::Move && !crate::import_move::is_symlink(Path::new(&path)) { - How::Place { name: name.unwrap_or(own) } - } else { - if mode == ImportMode::Move { - let reason = "a symbolic link: the file it points to was copied, the link and its target stay"; - plan.slots.push(Slot::Item(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() }))); - } - How::Copy { name } - }; - // Copy as DNG: the copied raw becomes a DNG (the copy is ours to replace) - let dng = opts.convert_dng && mode == ImportMode::Copy && raw && !info.format.eq_ignore_ascii_case("DNG"); - plan.slots.push(Slot::Transfer); - plan.transfers.push(Transfer { path, info, sidecar, dir, how, dng, family }); - continue; - } - (ImportMode::Move, _) => { - let reason = "already inside the destination or the library: added where it is"; - plan.slots.push(Slot::Item(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() }))); - path.clone() - } - _ => path.clone(), - }; - if let Some(h) = &info.content_hash { - self.by_hash.insert(h.clone(), None); - } - self.done.fetch_add(1, Relaxed); - plan.slots.push(Slot::Item(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed: None })))); - } - self.run_plan(&mut plan, &mut out, cancel); - out - } - - /// Run the transfers `plan` collected (on several threads) and append its items to `out` in - /// file order. Transfers not started because `cancel` was set leave no item. - fn run_plan(&mut self, plan: &mut Plan, out: &mut Prepared, cancel: &std::sync::atomic::AtomicBool) { - let mut done = run_transfers(std::mem::take(&mut plan.transfers), self.file_bytes.as_ref(), cancel, &self.done).into_iter(); - for slot in std::mem::take(&mut plan.slots) { - let item = match slot { - Slot::Item(it) => it, - Slot::Transfer => match done.next().flatten() { - Some(it) => it, - None => continue, - }, - }; - if let PreparedItem::Ready(r) = &item - && let Some(h) = &r.info.content_hash - { - self.by_hash.insert(h.clone(), None); - } - out.items.push(item); - } - } -} - -/// A batch's items in file order while its transfers are pending ([`ImportJob::run_plan`]). -#[derive(Default)] -struct Plan { - slots: Vec, - transfers: Vec, -} - -enum Slot { - Item(PreparedItem), - /// The outcome of the next of [`Plan::transfers`]. - Transfer, -} - -/// A file to copy or move into `dir`, planned in order and run on any thread ([`run_transfer`]). -struct Transfer { - path: String, - info: ProbeInfo, - sidecar: Option, - dir: PathBuf, - how: How, - /// Copy as DNG. - dng: bool, - /// The folder (lowercase) and [`name_family`]: transfers of one family can take each other's - /// names, so they never run at the same time. - family: (String, String), -} - -enum How { - /// A verified copy, under `name` (default: the file's own). - Copy { name: Option }, - /// Move: placed (linked or copied) without touching the source. - Place { name: String }, -} - -/// The names a file called `name` could end up with, as one key: the transfers number a taken name -/// `IMG-1.JPG`, `IMG-2.JPG`…, and Copy as DNG names its DNG `IMG.dng`, `IMG-2.dng`…, so `IMG.JPG`, -/// `IMG-1.jpg` and `img.dng` are all `img` (case-insensitive: so are some file systems). -fn name_family(name: &str) -> String { - let mut stem = match name.rsplit_once('.') { - Some((s, _)) if !s.is_empty() => s, - _ => name, - }; - while let Some((head, tail)) = stem.rsplit_once('-') - && !head.is_empty() - && !tail.is_empty() - && tail.bytes().all(|b| b.is_ascii_digit()) - { - stem = head; - } - stem.to_lowercase() -} - -/// Threads for `jobs` file reads (probes, copies) at once. #[cfg(not(target_arch = "wasm32"))] -pub(crate) fn workers(jobs: usize) -> usize { - std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(1, 8).min(jobs.max(1)) -} - -/// Files per batch for a caller that prepares an import in batches ([`ImportJob::prepare_files`]): -/// enough to keep every probe and copy thread busy (a batch waits for its slowest file), few -/// enough that each batch soon joins the catalog. -pub fn batch_size() -> usize { - #[cfg(not(target_arch = "wasm32"))] - { - workers(usize::MAX).saturating_mul(2).clamp(8, 16) - } - #[cfg(target_arch = "wasm32")] - { - 8 - } -} - -/// Run `transfers` (several at once where threads exist); their outcomes in order, `None` for one -/// not started because `cancel` was set. `done` counts each one finished. -fn run_transfers( - transfers: Vec, - file_bytes: Option<&crate::merge::ByteReader>, - cancel: &std::sync::atomic::AtomicBool, - done: &std::sync::atomic::AtomicUsize, -) -> Vec> { - use std::sync::atomic::Ordering::Relaxed; - let one = |t: Transfer| { - if cancel.load(Relaxed) { - return None; - } - let path = t.path.clone(); - // last-resort guard: a panic is this file's failure, not the import's - let item = crate::guard::catch("import", || run_transfer(t, file_bytes)).unwrap_or_else(|e| PreparedItem::Failed(path, e)); - done.fetch_add(1, Relaxed); - Some(item) - }; - #[cfg(not(target_arch = "wasm32"))] - { - let n = workers(transfers.len()); - if n > 1 { - let mut results: Vec> = std::iter::repeat_with(|| None).take(transfers.len()).collect(); - let queue = std::sync::Mutex::new(transfers.into_iter().enumerate()); - let out = std::sync::Mutex::new(&mut results); - // (the tests' injected faults hold for the workers too) - #[cfg(test)] - let fault = crate::import_move::injected_fault(); - let work = &|| { - #[cfg(test)] - crate::import_move::inject(fault); - loop { - let next = queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner).next(); - let Some((i, t)) = next else { break }; - let r = one(t); - if let Some(slot) = out.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get_mut(i) { - *slot = r; - } - } - }; - std::thread::scope(|sc| { - let handles: Vec<_> = - (0..n).filter_map(|i| std::thread::Builder::new().name(format!("lc-import-copy-{i}")).spawn_scoped(sc, work).ok()).collect(); - if handles.is_empty() { - // no thread to be had: copy here, one after another - work(); - } - for h in handles { - if h.join().is_err() { - log::error!("import: a copy worker stopped unexpectedly"); - } - } - }); - return results; - } - } - transfers.into_iter().map(one).collect() -} - -/// Copy or move one planned file (and Copy as DNG); its item. -fn run_transfer(t: Transfer, file_bytes: Option<&crate::merge::ByteReader>) -> PreparedItem { - let Transfer { path, mut info, sidecar, dir, how, dng, .. } = t; - let mut placed = None; - let mut stored = match how { - How::Place { name } => match crate::import_move::place(Path::new(&path), &dir, &name) { - Ok(pl) => { - let dst = pl.dst.to_string_lossy().to_string(); - placed = Some(pl); - dst - } - Err(e) => return PreparedItem::Failed(path, e), - }, - How::Copy { name } => match copy_into(&dir, &path, name.as_deref(), info.content_hash.as_deref()) { - Ok(p) => p, - Err(e) => return PreparedItem::Failed(path, e), - }, - }; - if dng { - // decoding the raw twice and encoding it takes several times the file's size in memory - let weight = usize::try_from(info.file_size).unwrap_or(usize::MAX).saturating_mul(6); - let _permit = crate::memory::work_gate().acquire(weight); - match crate::cmd::convert::write_dng_with(file_bytes, &stored, String::new()) { - Ok(dng) => { - let _ = std::fs::remove_file(&stored); - stored = dng; - info.format = "DNG".into(); - } - Err(e) => log::warn!("import {path}: copy as DNG: {e}"), - } - } - PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed })) -} - -/// A trashed photo's file is not where the library had it (never on the web, where paths are -/// storage keys, not files). -fn trashed_file_is_gone(cat: &lightcraft_catalog::Catalog, id: PhotoId) -> bool { - !cfg!(target_arch = "wasm32") - && match cat.photo(id).map(|p| &p.source) { - Some(Source::File { path }) => !Path::new(path).exists(), - _ => false, - } -} - -/// The catalog half of an import: add the photos [`ImportJob::prepare`] readied (one undoable op; -/// a Move's sources are removed once it is saved). On a failed commit a Move's placed files are -/// taken back. -pub fn commit_prepared(s: &mut Session, opts: &ImportOptions, now: &str, prepared: Prepared) -> crate::Result { - let now = now.to_string(); - let mut report = ImportReport { scanned: prepared.scanned, ..Default::default() }; - let mut ops = Vec::new(); - // Move: placed at the destination, sources untouched until the records are committed - let mut placed: Vec = Vec::new(); - // photos added by this batch, by content (a duplicate of a file earlier in the import names it) - let mut new_hash: HashMap = HashMap::new(); - // Fresh: trashed photos a new import replaces (by content or by file), each removed once - let (mut trash_hash, mut trash_path): (HashMap, HashMap) = Default::default(); - if opts.on_deleted == OnDeleted::Fresh { - for p in s.catalog.photos().filter(|p| p.deleted) { - if let Some(h) = &p.content_hash { - trash_hash.insert(h.clone(), p.id); - } - if let Source::File { path } = &p.source { - trash_path.insert(path.clone(), p.id); - } - } - } - let mut purge: Vec = Vec::new(); - let mut purged: std::collections::HashSet = Default::default(); - for it in prepared.items { - match it { - PreparedItem::Promote(id) => { - if s.catalog.photo(id).is_some_and(|p| p.local) { - ops.push(Op::SetLocal { id, local: false }); - } - report.imported.push(id.0); - } - PreparedItem::Duplicate { path, existing, reason, hash } => { - let existing = existing.or_else(|| { - let h = hash?; - new_hash.get(&h).copied().or_else(|| s.catalog.photos().find(|p| p.content_hash.as_deref() == Some(h.as_str())).map(|p| p.id)) - }); - let trashed = existing.filter(|id| s.catalog.photo(*id).is_some_and(|p| p.deleted)); - match trashed { - // (browsing a folder in Local only looks: it never restores) - Some(id) if opts.on_deleted == OnDeleted::Restore && !opts.local => { - if !report.restored.contains(&id.0) { - ops.push(Op::SetDeleted { id, deleted: false }); - // matched by content from another path, and the file the record - // points at is gone (moved, renamed): the photo follows the file - if reason == "content" && trashed_file_is_gone(&s.catalog, id) { - ops.push(crate::cmd::missing::relink_op(id, &path)); - } - report.restored.push(id.0); - } - } - _ => report.duplicates.push(Duplicate { path, existing: existing.map(|i| i.0), reason, existing_deleted: trashed.is_some() }), - } - } - PreparedItem::Failed(path, e) => report.failed.push((path, e)), - PreparedItem::Kept(k) => report.kept.push(k), - PreparedItem::Ready(r) => { - let ReadyFile { path, stored, info, sidecar, placed: pl } = *r; - placed.extend(pl); - let id = s.catalog.alloc_photo_id(); - if let Some(h) = &info.content_hash { - new_hash.insert(h.clone(), id); - } - for old in info.content_hash.as_ref().and_then(|h| trash_hash.get(h)).into_iter().chain(trash_path.get(&stored)) { - if purged.insert(*old) { - purge.push(*old); - } - } - let sidecar = sidecar.map(|sc| sc.resolve_label(&s.catalog)).filter(|sc| *sc != crate::sidecar::SidecarData::default()); - // a copy is catalogued under its new name - let name = Path::new(&stored).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone()); - let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now); - p.kind = info.kind; - p.file_size = info.file_size; - p.captured = info.captured; - p.meta = info.meta; - p.as_shot_wb = info.as_shot_wb; - p.content_hash = info.content_hash; - p.embedded_lens = info.embedded_lens; - p.set_preview_only(info.preview_only.clone()); - apply_import_defaults(s, &mut p); - if let Some(sc) = &sidecar { - crate::sidecar::merge_into(&mut p, sc, &now); - report.sidecars += 1; - report.warnings.extend(sc.warnings.iter().cloned().map(|mut warning| { - warning.path = path.clone(); - warning - })); - } - if let Some(mp) = opts.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) { - crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields); - } - for k in &opts.keywords { - let k = lightcraft_catalog::keywords::clean(k); - if !k.is_empty() && !p.meta.keywords.iter().any(|x| x.eq_ignore_ascii_case(&k)) { - p.meta.keywords.push(k); - } - } - if let Some(preset) = &opts.preset { - let d = std::sync::Arc::new(preset.apply(&p.develop, 1.0)); - let label = format!("Preset: {}", preset.name); - p.develop = d.clone(); - p.edited = Some(now.clone()); - p.history.push(lightcraft_catalog::HistoryStep { label, settings: d }); - } - report.imported.push(id.0); - p.local = opts.local; - if opts.local { - // the state as browsed: a later change keeps the record from being forgotten - p.set_local_baseline(); - } - ops.push(Op::AddPhoto { photo: Box::new(p) }); - } - } - } - // (built against the catalog as it is: one op list for all of them, ahead of the additions) - if !purge.is_empty() { - ops.insert(0, s.catalog.delete_photos_permanently_ops(&purge)); - } - let log0 = s.pending_log.len(); - let (verb, n) = if report.imported.is_empty() && !report.restored.is_empty() { - ("Restore", report.restored.len()) - } else { - ("Add", report.imported.len().max(1)) - }; - if !ops.is_empty() - && let Err(e) = s.commit(&format!("{verb} {n} Photo{}", if n == 1 { "" } else { "s" }), Op::Batch { ops }) - { - // nothing was catalogued: take the moves back (their sources were never touched) - placed.iter().for_each(crate::import_move::rollback); - return Err(e); - } - if !placed.is_empty() { - finish_moves(s, placed, log0, &mut report); - } - Ok(report) -} - -/// Move, after the commit: save the library, then remove each source whose destination is -/// still intact. Nothing is removed when the library can't be saved. -fn finish_moves(s: &mut Session, placed: Vec, log0: usize, report: &mut ImportReport) { - // (saving here clears the pending ops, so the sidecars `execute` would auto-write are written here) - let auto: Vec = if s.xmp.auto_write { s.pending_log.get(log0..).map(<[Op]>::to_vec).unwrap_or_default() } else { Vec::new() }; - let before = s.pending_log.len(); - if let Err(e) = s.persist() { - for p in placed { - let reason = format!("the library could not be saved ({e}): the original stays; the photo uses the copy at {}", p.dst.display()); - report.kept.push(Kept { path: p.src.to_string_lossy().to_string(), reason }); - } - return; - } - for p in placed { - let from = p.src.to_string_lossy().to_string(); - match crate::import_move::finish(&p) { - Ok(kept) => { - let sidecars = p.sidecars.iter().map(|sc| sc.dst.to_string_lossy().to_string()).collect(); - report.moved.push(Moved { from, to: p.dst.to_string_lossy().to_string(), sidecars }); - report.kept.extend(kept.into_iter().map(|(path, reason)| Kept { path, reason })); - } - Err(reason) => { - log::warn!("import: move {from}: {reason}"); - report.kept.push(Kept { path: from, reason }); - } - } - } - if s.pending_log.len() < before && s.interaction.is_none() && !auto.is_empty() { - s.auto_write_sidecars(&auto); - } -} - -/// The current local time as ISO 8601 (`YYYY-MM-DDTHH:MM:SS`, UTC on targets without a clock -/// offset); for [`Session::clock`] on native hosts. -pub fn system_clock() -> String { - #[cfg(not(target_arch = "wasm32"))] - { - let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); - civil(secs) - } - #[cfg(target_arch = "wasm32")] - { - "2026-01-01T00:00:00".to_string() - } -} - -/// Unix seconds → `YYYY-MM-DDTHH:MM:SS` (UTC), proleptic Gregorian. -pub fn civil(secs: i64) -> String { - lightcraft_catalog::dates::civil(secs) -} - -impl Session { - /// A thumbnail for an import candidate (not in the catalog): a raw's embedded preview, else a - /// small render of the file. `slot` makes the job's photo id unique (`u64::MAX - slot`). - pub fn candidate_thumb_job(&mut self, c: &ImportCandidate, edge: usize, slot: u64) -> Option { - if c.error.is_some() { - return None; - } - let id = PhotoId(u64::MAX - slot); - let mut p = Photo::new(id, Source::File { path: c.path.clone() }, &c.name, &c.format, c.width.max(1), c.height.max(1), ""); - p.kind = c.kind; - if let Some(info) = self.import_probes.get(&c.path) { - p.as_shot_wb = info.as_shot_wb; - p.embedded_lens = info.embedded_lens; - p.preview_only = info.preview_only.clone(); - } - p.develop = std::sync::Arc::new(p.import_defaults()); - let edge = edge.clamp(64, crate::media::SourceLevel::Thumb.max_edge()); - let level = crate::media::SourceLevel::Thumb; - let source = self.media.origin_ref(&p.source, level.max_edge()); - let key = lightcraft_preview::Hasher128::new().str(&c.path).u64(c.file_size).u64(edge as u64).finish().0 as u64; - let small = crate::media::RenderJob { - request_id: 0, - cache_generation: self.media.rendered.generation(), - source_key: None, - photo: id, - level, - source, - origin: p.source.clone(), - info: crate::media::source_info(&p), - settings: p.develop.clone(), - request: lightcraft_pipeline::RenderRequest::fit(edge, edge), - key, - cache: None, - stages: None, - view_cache: None, - display: None, - slide: None, - preview_cut: None, - display_fit: None, - }; - let embedded = match (&self.media.preview_loader, c.kind) { - (Some(l), MediaKind::Raw) => Some((c.path.clone(), l.clone(), edge)), - _ => None, - }; - Some(crate::media::QuickJob { request_id: 0, photo: id, key, cached: Vec::new(), embedded, small: Some(Box::new(small)) }) - } - - /// Use the system clock for import/edit times (native hosts). - pub fn with_system_clock(mut self) -> Self { - self.clock = Box::new(system_clock); - self - } -} +pub(crate) use commit::workers; +pub use commit::{batch_size, civil, commit_prepared, system_clock}; +pub use defaults::{CameraDefault, ImportDefaults, apply_import_defaults, has_import_look, import_defaults}; +pub use files::{EXTENSIONS, LaunchPath, accept_list, expand, expand_within, is_supported, launch_path}; +pub(crate) use files::{expand_noting, probe_all, probe_paths, probe_with}; +#[cfg(test)] +use job::PreparedItem; +pub use job::{ImportJob, Prepared, import, import_with}; +pub(crate) use scan::scan_files; +pub use scan::{ScanInput, ScanOutput, ScanProgress, scan, scan_with}; +#[cfg(test)] +use scan::{library_files, library_hashes}; +pub use types::{ + Duplicate, ImportCandidate, ImportMode, ImportOptions, ImportReport, ImportUprightMode, ImportWarning, ImportWarningCode, Kept, Moved, OnDeleted, + Organize, +}; #[cfg(test)] #[path = "import/tests_known.rs"] mod tests_known; - -#[cfg(test)] -mod prepared_tests { - use super::*; - - /// Extensions the picker may offer: every one in [`EXTENSIONS`], minus the - /// files this build cannot import (see [`accept_list`]). - fn pickable_extensions() -> Vec<&'static str> { - let heif_decodable = lightcraft_codecs::Format::Heif.can_decode(); - EXTENSIONS - .iter() - .copied() - .filter(|ext| match *ext { - "avif" => false, - "heic" | "heif" => heif_decodable, - _ => true, - }) - .collect() - } - - #[test] - fn accept_list_covers_every_pickable_extension_in_both_cases() { - let list = accept_list(); - for ext in pickable_extensions() { - assert!(list.contains(&format!(".{ext}")), "lowercase .{ext} missing from accept list"); - assert!(list.contains(&format!(".{}", ext.to_uppercase())), "uppercase .{} missing from accept list", ext.to_uppercase()); - } - } - - /// The web picker must not offer files this build cannot import: `.avif` has - /// no AV1 decoder in any build, so it is never listed, in either case. - #[test] - fn accept_list_never_offers_avif() { - let list = accept_list(); - assert!(!list.contains(".avif"), "{list}"); - assert!(!list.contains(".AVIF"), "{list}"); - } - - /// `.heic`/`.heif` follow the codecs' `heif` feature: listed (in both cases) - /// only when this build can decode them; otherwise hidden, so a web user - /// cannot pick a file that would fail to import. - #[test] - fn accept_list_offers_heic_heif_only_when_decodable() { - let list = accept_list(); - let listed = lightcraft_codecs::Format::Heif.can_decode(); - for ext in ["heic", "heif"] { - assert_eq!(list.contains(&format!(".{ext}")), listed, "lowercase .{ext}"); - assert_eq!(list.contains(&format!(".{}", ext.to_uppercase())), listed, "uppercase .{ext}"); - } - } - - #[test] - fn revalidate_add_only_converts_matching_ready_item() { - let mut s = Session::new(); - let existing_path = std::env::temp_dir().join("lightcraft-revalidate-existing.jpg"); - let other_path = std::env::temp_dir().join("lightcraft-revalidate-other.jpg"); - let existing = Photo::new(PhotoId(1), Source::File { path: existing_path.to_string_lossy().into() }, "existing.jpg", "JPG", 1, 1, "now"); - s.commit("existing", Op::AddPhoto { photo: Box::new(existing) }).unwrap(); - let ready = |path: &Path| ReadyFile { - path: path.to_string_lossy().into(), - stored: path.to_string_lossy().into(), - info: ProbeInfo { format: "JPG".into(), ..ProbeInfo::default() }, - sidecar: None, - placed: None, - }; - let mut prepared = Prepared { - scanned: 2, - items: vec![PreparedItem::Ready(Box::new(ready(&existing_path))), PreparedItem::Ready(Box::new(ready(&other_path)))], - }; - prepared.revalidate_add(&s); - assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. }))); - assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_)))); - } - - fn unread_paths(n: usize) -> Vec { - (0..n).map(|i| std::env::temp_dir().join(format!("lightcraft-probe-progress-{i}.jpg")).to_string_lossy().into()).collect() - } - - /// Reading the files of a large import (a Lightroom catalog's originals) reported nothing: - /// the progress stood still at "14 of 24554" for as long as it took. - #[test] - fn prepare_counts_the_files_it_reads() { - use std::sync::atomic::Ordering::Relaxed; - let mut s = Session::new(); - let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let (seen_in_probe, done_in_probe) = (seen.clone(), std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0))); - let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap(); - let (done, last) = (job.read_done.clone(), done_in_probe.clone()); - job.probe = Some(std::sync::Arc::new(move |_: &str| { - seen_in_probe.fetch_add(1, Relaxed); - last.fetch_max(done.load(Relaxed), Relaxed); - Ok(ProbeInfo { format: "JPEG".into(), ..ProbeInfo::default() }) - })); - let prepared = job.prepare(&unread_paths(40), &std::sync::atomic::AtomicBool::new(false)); - assert_eq!(prepared.items.len(), 40); - assert_eq!(seen.load(Relaxed), 40); - assert!(done_in_probe.load(Relaxed) > 0, "progress moves while files are read"); - assert_eq!((job.read_done.load(Relaxed), job.read_total.load(Relaxed)), (40, 40), "every file read is counted"); - assert_eq!((job.done.load(Relaxed), job.total.load(Relaxed)), (40, 40), "handling them is counted on its own"); - } - - /// Cancel had no effect until every file had been read. - #[test] - fn cancel_stops_reading_files() { - use std::sync::atomic::Ordering::Relaxed; - let mut s = Session::new(); - let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap(); - let (seen_in_probe, cancel_in_probe) = (seen.clone(), cancel.clone()); - job.probe = Some(std::sync::Arc::new(move |_: &str| { - seen_in_probe.fetch_add(1, Relaxed); - cancel_in_probe.store(true, Relaxed); - Ok(ProbeInfo { format: "JPEG".into(), ..ProbeInfo::default() }) - })); - let prepared = job.prepare(&unread_paths(400), &cancel); - // at most the files the (up to 8) readers had already started - assert!(seen.load(Relaxed) <= 8, "read {} files after cancel", seen.load(Relaxed)); - assert!(prepared.items.is_empty()); - } -} diff --git a/crates/engine/src/import/commit.rs b/crates/engine/src/import/commit.rs new file mode 100644 index 000000000..bed8356fb --- /dev/null +++ b/crates/engine/src/import/commit.rs @@ -0,0 +1,371 @@ +use super::defaults::apply_import_defaults; +use super::files::copy_into; +use super::job::{Prepared, PreparedItem, ReadyFile}; +use super::types::{Duplicate, ImportOptions, ImportReport, Kept, Moved, OnDeleted}; +use super::*; + +/// A batch's items in file order while its transfers are pending ([`ImportJob::run_plan`]). +#[derive(Default)] +pub(super) struct Plan { + pub(super) slots: Vec, + pub(super) transfers: Vec, +} + +pub(super) enum Slot { + Item(PreparedItem), + /// The outcome of the next of [`Plan::transfers`]. + Transfer, +} + +/// A file to copy or move into `dir`, planned in order and run on any thread ([`run_transfer`]). +pub(super) struct Transfer { + pub(super) path: String, + pub(super) info: ProbeInfo, + pub(super) sidecar: Option, + pub(super) dir: PathBuf, + pub(super) how: How, + /// Copy as DNG. + pub(super) dng: bool, + /// The folder (lowercase) and [`name_family`]: transfers of one family can take each other's + /// names, so they never run at the same time. + pub(super) family: (String, String), +} + +pub(super) enum How { + /// A verified copy, under `name` (default: the file's own). + Copy { name: Option }, + /// Move: placed (linked or copied) without touching the source. + Place { name: String }, +} + +/// The names a file called `name` could end up with, as one key: the transfers number a taken name +/// `IMG-1.JPG`, `IMG-2.JPG`…, and Copy as DNG names its DNG `IMG.dng`, `IMG-2.dng`…, so `IMG.JPG`, +/// `IMG-1.jpg` and `img.dng` are all `img` (case-insensitive: so are some file systems). +pub(super) fn name_family(name: &str) -> String { + let mut stem = match name.rsplit_once('.') { + Some((s, _)) if !s.is_empty() => s, + _ => name, + }; + while let Some((head, tail)) = stem.rsplit_once('-') + && !head.is_empty() + && !tail.is_empty() + && tail.bytes().all(|b| b.is_ascii_digit()) + { + stem = head; + } + stem.to_lowercase() +} + +/// Threads for `jobs` file reads (probes, copies) at once. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn workers(jobs: usize) -> usize { + std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(1, 8).min(jobs.max(1)) +} + +/// Files per batch for a caller that prepares an import in batches ([`ImportJob::prepare_files`]): +/// enough to keep every probe and copy thread busy (a batch waits for its slowest file), few +/// enough that each batch soon joins the catalog. +pub fn batch_size() -> usize { + #[cfg(not(target_arch = "wasm32"))] + { + workers(usize::MAX).saturating_mul(2).clamp(8, 16) + } + #[cfg(target_arch = "wasm32")] + { + 8 + } +} + +/// Run `transfers` (several at once where threads exist); their outcomes in order, `None` for one +/// not started because `cancel` was set. `done` counts each one finished. +pub(super) fn run_transfers( + transfers: Vec, + file_bytes: Option<&crate::merge::ByteReader>, + cancel: &std::sync::atomic::AtomicBool, + done: &std::sync::atomic::AtomicUsize, +) -> Vec> { + use std::sync::atomic::Ordering::Relaxed; + let one = |t: Transfer| { + if cancel.load(Relaxed) { + return None; + } + let path = t.path.clone(); + // last-resort guard: a panic is this file's failure, not the import's + let item = crate::guard::catch("import", || run_transfer(t, file_bytes)).unwrap_or_else(|e| PreparedItem::Failed(path, e)); + done.fetch_add(1, Relaxed); + Some(item) + }; + #[cfg(not(target_arch = "wasm32"))] + { + let n = workers(transfers.len()); + if n > 1 { + let mut results: Vec> = std::iter::repeat_with(|| None).take(transfers.len()).collect(); + let queue = std::sync::Mutex::new(transfers.into_iter().enumerate()); + let out = std::sync::Mutex::new(&mut results); + // (the tests' injected faults hold for the workers too) + #[cfg(test)] + let fault = crate::import_move::injected_fault(); + let work = &|| { + #[cfg(test)] + crate::import_move::inject(fault); + loop { + let next = queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner).next(); + let Some((i, t)) = next else { break }; + let r = one(t); + if let Some(slot) = out.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get_mut(i) { + *slot = r; + } + } + }; + std::thread::scope(|sc| { + let handles: Vec<_> = + (0..n).filter_map(|i| std::thread::Builder::new().name(format!("lc-import-copy-{i}")).spawn_scoped(sc, work).ok()).collect(); + if handles.is_empty() { + // no thread to be had: copy here, one after another + work(); + } + for h in handles { + if h.join().is_err() { + log::error!("import: a copy worker stopped unexpectedly"); + } + } + }); + return results; + } + } + transfers.into_iter().map(one).collect() +} + +/// Copy or move one planned file (and Copy as DNG); its item. +fn run_transfer(t: Transfer, file_bytes: Option<&crate::merge::ByteReader>) -> PreparedItem { + let Transfer { path, mut info, sidecar, dir, how, dng, .. } = t; + let mut placed = None; + let mut stored = match how { + How::Place { name } => match crate::import_move::place(Path::new(&path), &dir, &name) { + Ok(pl) => { + let dst = pl.dst.to_string_lossy().to_string(); + placed = Some(pl); + dst + } + Err(e) => return PreparedItem::Failed(path, e), + }, + How::Copy { name } => match copy_into(&dir, &path, name.as_deref(), info.content_hash.as_deref()) { + Ok(p) => p, + Err(e) => return PreparedItem::Failed(path, e), + }, + }; + if dng { + // decoding the raw twice and encoding it takes several times the file's size in memory + let weight = usize::try_from(info.file_size).unwrap_or(usize::MAX).saturating_mul(6); + let _permit = crate::memory::work_gate().acquire(weight); + match crate::cmd::convert::write_dng_with(file_bytes, &stored, String::new()) { + Ok(dng) => { + let _ = std::fs::remove_file(&stored); + stored = dng; + info.format = "DNG".into(); + } + Err(e) => log::warn!("import {path}: copy as DNG: {e}"), + } + } + PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed })) +} + +/// A trashed photo's file is not where the library had it (never on the web, where paths are +/// storage keys, not files). +fn trashed_file_is_gone(cat: &lightcraft_catalog::Catalog, id: PhotoId) -> bool { + !cfg!(target_arch = "wasm32") + && match cat.photo(id).map(|p| &p.source) { + Some(Source::File { path }) => !Path::new(path).exists(), + _ => false, + } +} + +/// The catalog half of an import: add the photos [`ImportJob::prepare`] readied (one undoable op; +/// a Move's sources are removed once it is saved). On a failed commit a Move's placed files are +/// taken back. +pub fn commit_prepared(s: &mut Session, opts: &ImportOptions, now: &str, prepared: Prepared) -> crate::Result { + let now = now.to_string(); + let mut report = ImportReport { scanned: prepared.scanned, ..Default::default() }; + let mut ops = Vec::new(); + // Move: placed at the destination, sources untouched until the records are committed + let mut placed: Vec = Vec::new(); + // photos added by this batch, by content (a duplicate of a file earlier in the import names it) + let mut new_hash: HashMap = HashMap::new(); + // Fresh: trashed photos a new import replaces (by content or by file), each removed once + let (mut trash_hash, mut trash_path): (HashMap, HashMap) = Default::default(); + if opts.on_deleted == OnDeleted::Fresh { + for p in s.catalog.photos().filter(|p| p.deleted) { + if let Some(h) = &p.content_hash { + trash_hash.insert(h.clone(), p.id); + } + if let Source::File { path } = &p.source { + trash_path.insert(path.clone(), p.id); + } + } + } + let mut purge: Vec = Vec::new(); + let mut purged: std::collections::HashSet = Default::default(); + for it in prepared.items { + match it { + PreparedItem::Promote(id) => { + if s.catalog.photo(id).is_some_and(|p| p.local) { + ops.push(Op::SetLocal { id, local: false }); + } + report.imported.push(id.0); + } + PreparedItem::Duplicate { path, existing, reason, hash } => { + let existing = existing.or_else(|| { + let h = hash?; + new_hash.get(&h).copied().or_else(|| s.catalog.photos().find(|p| p.content_hash.as_deref() == Some(h.as_str())).map(|p| p.id)) + }); + let trashed = existing.filter(|id| s.catalog.photo(*id).is_some_and(|p| p.deleted)); + match trashed { + // (browsing a folder in Local only looks: it never restores) + Some(id) if opts.on_deleted == OnDeleted::Restore && !opts.local => { + if !report.restored.contains(&id.0) { + ops.push(Op::SetDeleted { id, deleted: false }); + // matched by content from another path, and the file the record + // points at is gone (moved, renamed): the photo follows the file + if reason == "content" && trashed_file_is_gone(&s.catalog, id) { + ops.push(crate::cmd::missing::relink_op(id, &path)); + } + report.restored.push(id.0); + } + } + _ => report.duplicates.push(Duplicate { path, existing: existing.map(|i| i.0), reason, existing_deleted: trashed.is_some() }), + } + } + PreparedItem::Failed(path, e) => report.failed.push((path, e)), + PreparedItem::Kept(k) => report.kept.push(k), + PreparedItem::Ready(r) => { + let ReadyFile { path, stored, info, sidecar, placed: pl } = *r; + placed.extend(pl); + let id = s.catalog.alloc_photo_id(); + if let Some(h) = &info.content_hash { + new_hash.insert(h.clone(), id); + } + for old in info.content_hash.as_ref().and_then(|h| trash_hash.get(h)).into_iter().chain(trash_path.get(&stored)) { + if purged.insert(*old) { + purge.push(*old); + } + } + let sidecar = sidecar.map(|sc| sc.resolve_label(&s.catalog)).filter(|sc| *sc != crate::sidecar::SidecarData::default()); + // a copy is catalogued under its new name + let name = Path::new(&stored).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone()); + let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now); + p.kind = info.kind; + p.file_size = info.file_size; + p.captured = info.captured; + p.meta = info.meta; + p.as_shot_wb = info.as_shot_wb; + p.content_hash = info.content_hash; + p.embedded_lens = info.embedded_lens; + p.set_preview_only(info.preview_only.clone()); + apply_import_defaults(s, &mut p); + if let Some(sc) = &sidecar { + crate::sidecar::merge_into(&mut p, sc, &now); + report.sidecars += 1; + report.warnings.extend(sc.warnings.iter().cloned().map(|mut warning| { + warning.path = path.clone(); + warning + })); + } + if let Some(mp) = opts.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) { + crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields); + } + for k in &opts.keywords { + let k = lightcraft_catalog::keywords::clean(k); + if !k.is_empty() && !p.meta.keywords.iter().any(|x| x.eq_ignore_ascii_case(&k)) { + p.meta.keywords.push(k); + } + } + if let Some(preset) = &opts.preset { + let d = std::sync::Arc::new(preset.apply(&p.develop, 1.0)); + let label = format!("Preset: {}", preset.name); + p.develop = d.clone(); + p.edited = Some(now.clone()); + p.history.push(lightcraft_catalog::HistoryStep { label, settings: d }); + } + report.imported.push(id.0); + p.local = opts.local; + if opts.local { + // the state as browsed: a later change keeps the record from being forgotten + p.set_local_baseline(); + } + ops.push(Op::AddPhoto { photo: Box::new(p) }); + } + } + } + // (built against the catalog as it is: one op list for all of them, ahead of the additions) + if !purge.is_empty() { + ops.insert(0, s.catalog.delete_photos_permanently_ops(&purge)); + } + let log0 = s.pending_log.len(); + let (verb, n) = if report.imported.is_empty() && !report.restored.is_empty() { + ("Restore", report.restored.len()) + } else { + ("Add", report.imported.len().max(1)) + }; + if !ops.is_empty() + && let Err(e) = s.commit(&format!("{verb} {n} Photo{}", if n == 1 { "" } else { "s" }), Op::Batch { ops }) + { + // nothing was catalogued: take the moves back (their sources were never touched) + placed.iter().for_each(crate::import_move::rollback); + return Err(e); + } + if !placed.is_empty() { + finish_moves(s, placed, log0, &mut report); + } + Ok(report) +} + +/// Move, after the commit: save the library, then remove each source whose destination is +/// still intact. Nothing is removed when the library can't be saved. +fn finish_moves(s: &mut Session, placed: Vec, log0: usize, report: &mut ImportReport) { + // (saving here clears the pending ops, so the sidecars `execute` would auto-write are written here) + let auto: Vec = if s.xmp.auto_write { s.pending_log.get(log0..).map(<[Op]>::to_vec).unwrap_or_default() } else { Vec::new() }; + let before = s.pending_log.len(); + if let Err(e) = s.persist() { + for p in placed { + let reason = format!("the library could not be saved ({e}): the original stays; the photo uses the copy at {}", p.dst.display()); + report.kept.push(Kept { path: p.src.to_string_lossy().to_string(), reason }); + } + return; + } + for p in placed { + let from = p.src.to_string_lossy().to_string(); + match crate::import_move::finish(&p) { + Ok(kept) => { + let sidecars = p.sidecars.iter().map(|sc| sc.dst.to_string_lossy().to_string()).collect(); + report.moved.push(Moved { from, to: p.dst.to_string_lossy().to_string(), sidecars }); + report.kept.extend(kept.into_iter().map(|(path, reason)| Kept { path, reason })); + } + Err(reason) => { + log::warn!("import: move {from}: {reason}"); + report.kept.push(Kept { path: from, reason }); + } + } + } + if s.pending_log.len() < before && s.interaction.is_none() && !auto.is_empty() { + s.auto_write_sidecars(&auto); + } +} + +/// The current local time as ISO 8601 (`YYYY-MM-DDTHH:MM:SS`, UTC on targets without a clock +/// offset); for [`Session::clock`] on native hosts. +pub fn system_clock() -> String { + #[cfg(not(target_arch = "wasm32"))] + { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); + civil(secs) + } + #[cfg(target_arch = "wasm32")] + { + "2026-01-01T00:00:00".to_string() + } +} + +/// Unix seconds → `YYYY-MM-DDTHH:MM:SS` (UTC), proleptic Gregorian. +pub fn civil(secs: i64) -> String { + lightcraft_catalog::dates::civil(secs) +} diff --git a/crates/engine/src/import/defaults.rs b/crates/engine/src/import/defaults.rs new file mode 100644 index 000000000..7ca5cecc4 --- /dev/null +++ b/crates/engine/src/import/defaults.rs @@ -0,0 +1,99 @@ +use super::*; + +/// The develop settings a photo gets on import: raws start from their as-shot white balance with +/// default sharpening / colour noise reduction, and file-embedded lens corrections on (as the +/// camera intended); a user default preset ([`ImportDefaults`]) goes on top. +pub fn import_defaults(p: &Photo) -> lightcraft_develop::DevelopSettings { + p.import_defaults() +} + +/// Does the photo still look as the camera rendered it (its embedded camera preview is then a +/// fair stand-in)? Not when a default preset changed it on import. +pub fn has_import_look(p: &Photo) -> bool { + *p.develop == p.camera_defaults() +} + +/// User defaults applied on import (Settings → Import; saved in the library's `prefs.json`). +/// Presets are referenced by id; a preset that no longer exists is ignored. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ImportDefaults { + /// Preset applied to raw files (`None` = the LightCraft default). + pub raw_preset: Option, + /// Use a camera's own default (below) when the photo's camera has one. + pub per_camera: bool, + /// Per-camera raw defaults, keyed by the camera's make + model as shown in Info + /// (`"Canon EOS R5"`). + pub cameras: Vec, + /// Preset applied to non-raw images (JPEG, PNG, TIFF, HEIC…; `None` = none). + pub other_preset: Option, + /// Copyright notice given to imported photos that don't carry one (empty = none). + pub copyright: String, + /// Creator given to imported photos that don't name one (empty = none). + pub creator: String, + /// Metadata preset applied to every imported photo (`metadata.*`; `None` = none). + pub metadata_preset: Option, + /// Auto import: a watched folder whose new photos are added as they arrive (`None` = off), + /// copied into the library's Originals instead of added in place when `auto_copy`, and an + /// album they go to. + pub auto_folder: Option, + pub auto_copy: bool, + pub auto_album: Option, + /// Open the import review when a camera or memory card is connected. + pub show_on_card: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CameraDefault { + /// Make + model (`Meta::camera`). + pub camera: String, + /// Preset id; `None` = the LightCraft default for this camera. + pub preset: Option, +} + +impl ImportDefaults { + /// The preset id that applies to a photo of this kind and camera, if any. + pub fn preset_for(&self, raw: bool, camera: &str) -> Option<&str> { + if !raw { + return self.other_preset.as_deref(); + } + if self.per_camera + && let Some(c) = self.cameras.iter().find(|c| !camera.is_empty() && c.camera.eq_ignore_ascii_case(camera)) + { + return c.preset.as_deref(); + } + self.raw_preset.as_deref() + } +} + +/// Give a freshly imported photo its default settings: the camera defaults, then the matching +/// default preset ([`ImportDefaults::preset_for`]), remembered as the photo's import look. +pub fn apply_import_defaults(s: &Session, p: &mut Photo) { + // metadata defaults fill gaps only: the file's own copyright / creator win + let d = &s.import_defaults; + if p.meta.copyright.trim().is_empty() && !d.copyright.trim().is_empty() { + p.meta.copyright = d.copyright.trim().to_string(); + } + if p.meta.creator.trim().is_empty() && !d.creator.trim().is_empty() { + p.meta.creator = d.creator.trim().to_string(); + } + if let Some(mp) = d.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) { + crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields); + } + let base = p.camera_defaults(); + // a raw shown from its embedded JPEG gets the rendered-file default, not the raw one + let raw = p.develops_raw(); + let preset = s.import_defaults.preset_for(raw, &p.meta.camera).and_then(|id| s.presets.iter().find(|x| x.id == id)); + match preset { + Some(pr) => { + let look = std::sync::Arc::new(pr.apply(&base, 1.0)); + p.develop = look.clone(); + p.import_look = (*look != base).then_some(look); + } + None => { + p.develop = std::sync::Arc::new(base); + p.import_look = None; + } + } +} diff --git a/crates/engine/src/import/files.rs b/crates/engine/src/import/files.rs new file mode 100644 index 000000000..2cbfc85e7 --- /dev/null +++ b/crates/engine/src/import/files.rs @@ -0,0 +1,372 @@ +#[cfg(not(target_arch = "wasm32"))] +use super::commit::workers; +#[cfg(test)] +use super::job::{ImportJob, Prepared, PreparedItem, ReadyFile}; +use super::scan::ScanProgress; +#[cfg(test)] +use super::types::{ImportMode, ImportOptions}; +use super::*; + +/// File extensions offered by import discovery and file pickers (lower case). +/// This includes preview-only raws; the file probe decides whether sensor decoding is available. +pub const EXTENSIONS: &[&str] = &[ + "jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "sr2", "srf", "raf", "orf", "rw2", "rwl", "raw", "pef", + "srw", "psd", "psb", "jxl", "gif", "bmp", + // Decoded when built with the codecs' `heif` feature; otherwise the import reports them as + // failed with the reason ("HEIC/HEIF support isn't included in this build"). + "heic", "heif", "avif", // no AV1 decoder yet: reported as failed, like a HEIC without the feature + "iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", "3fr", "fff", "mef", +]; + +pub fn is_supported(path: &Path) -> bool { + path.extension().is_some_and(|e| EXTENSIONS.contains(&e.to_string_lossy().to_lowercase().as_str())) +} + +/// The web file picker's accept list, derived from [`EXTENSIONS`]. Each extension +/// is listed in both cases: browsers' `accept` matching is case-insensitive per +/// spec, but some implementations (e.g. Windows/Edge) match case-sensitively and +/// rejected uppercase `.NEF` files, so a lowercase-only list silently hid raws +/// such as CR3, NRW, RAF, ORF, RW2, RWL, RAW, PEF and SRW from the picker. +/// +/// The picker only offers files this build can actually import: `.avif` has no +/// AV1 decoder in any build, and `.heic`/`.heif` decode only when the codecs' +/// `heif` feature is enabled ([`lightcraft_codecs::Format::Heif::can_decode`]). +/// [`EXTENSIONS`] itself stays complete — files on disk are still recognised, +/// and import reports them as failed, saying why. +pub fn accept_list() -> String { + let heif_decodable = lightcraft_codecs::Format::Heif.can_decode(); + let mut out = String::new(); + for &ext in EXTENSIONS { + // Skip extensions a web user could pick but this build could not import. + let pickable = match ext { + "avif" => false, + "heic" | "heif" => heif_decodable, + _ => true, + }; + if !pickable { + continue; + } + if !out.is_empty() { + out.push(','); + } + out.push('.'); + out.push_str(ext); + out.push(','); + out.push('.'); + out.push_str(&ext.to_uppercase()); + } + out +} + +/// Expand files and folders (recursively) into supported files. `skip` (e.g. the library folder) +/// is never descended into. Each folder's walk is bounded ([`crate::walk::Limits::default`]); a +/// walk that stops at a bound is logged. +pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec { + expand_within(paths, skip, crate::walk::Limits::default()).0 +} + +/// [`expand`] with the walk bounded by `limits` → (files, whether a walk stopped at a bound). +pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk::Limits) -> (Vec, bool) { + expand_noting(paths, skip, limits, None) +} + +/// [`expand_within`], handing the listing of every folder that is walked to `note` +/// ([`crate::walk::files_in_noting`]). +pub(crate) fn expand_noting( + paths: &[String], + skip: Option<&Path>, + limits: crate::walk::Limits, + mut note: Option, +) -> (Vec, bool) { + let mut out = Vec::new(); + let mut truncated = false; + for p in paths { + let p = Path::new(p); + if skip.is_some_and(|s| p == s) { + continue; + } + if p.is_dir() { + let w = match note.as_mut() { + Some(note) => crate::walk::files_in_noting(p, skip, limits, is_supported, &mut **note), + None => crate::walk::files_in(p, skip, limits, is_supported), + }; + if w.truncated { + log::warn!( + "import: stopped looking in {} after {} entries (limits: {} entries, {} folder levels)", + p.display(), + w.entries, + limits.max_entries, + limits.max_depth + ); + } + truncated |= w.truncated; + out.extend(w.files.into_iter().map(|f| f.to_string_lossy().to_string())); + } else { + // explicitly named files are attempted even with an unknown extension (sniffed) + out.push(p.to_string_lossy().to_string()); + } + } + let mut seen = std::collections::HashSet::new(); + out.retain(|p| seen.insert(p.clone())); + (out, truncated) +} + +/// What a path given on the command line at launch becomes ([`launch_path`]). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LaunchPath { + /// Import this (absolute) file or folder, in the background. + Import(String), + /// Not imported: `path` (absolute) is a whole drive or the whole home folder, which is never + /// walked without being chosen in the app (issue #374). + Refused { path: String, why: String }, +} + +/// Resolve a launch argument for importing (issue #374). An empty argument is ignored (`None`); +/// a relative one (`.`, `..`, `photos`; `/` or `\` on Windows, the current drive's root) is +/// resolved against `cwd` — the working folder a launcher picked, often a drive root or +/// `C:\Windows\System32` — and refused when that leaves no folder (`cwd` unknown). A drive +/// or file-system root is replaced by its camera folder (`DCIM`, from `dcim`: a memory card) or +/// refused, and so is the home folder `home`: either would walk most of the disk. +pub fn launch_path(arg: &str, cwd: Option<&Path>, home: Option<&Path>, dcim: impl Fn(&Path) -> Option) -> Option { + if arg.trim().is_empty() { + return None; + } + let given = Path::new(arg); + let joined = if given.is_absolute() { + given.to_path_buf() + } else { + match cwd { + Some(c) => c.join(given), + None => return Some(LaunchPath::Refused { path: arg.to_string(), why: "a relative path, and the working folder is unknown".into() }), + } + }; + // `.` and `..` resolved by name (no file-system access) + let mut abs = PathBuf::new(); + for c in joined.components() { + match c { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + abs.pop(); + } + other => abs.push(other.as_os_str()), + } + } + let shown = abs.to_string_lossy().to_string(); + if abs.parent().is_none() { + return Some(match dcim(&abs) { + Some(d) => LaunchPath::Import(d.to_string_lossy().to_string()), + None => LaunchPath::Refused { path: shown, why: "a whole drive".into() }, + }); + } + let key = lightcraft_catalog::query::folder_key; + if home.is_some_and(|h| key(&h.to_string_lossy()) == key(&shown)) { + return Some(LaunchPath::Refused { path: shown, why: "the whole home folder".into() }); + } + Some(LaunchPath::Import(shown)) +} + +/// Probe files (headers + content hash) as an import would. +pub(crate) fn probe_paths(s: &Session, paths: &[String]) -> Vec> { + let progress = ScanProgress::default(); + probe_all(s.media.file_probe.as_ref(), paths, &progress.done, &progress.cancel) +} + +/// Probe files with `probe` (a clone of the session's file probe), off the session: on a worker thread. +pub(crate) fn probe_with(probe: &crate::media::FileProbe, paths: &[String]) -> Vec> { + let progress = ScanProgress::default(); + probe_all(Some(probe), paths, &progress.done, &progress.cancel) +} + +/// Probe `paths` (several at once), counting each in `done`. Stops before the next file once +/// `cancel` is set: the files not reached come back as errors. +pub(crate) fn probe_all( + probe: Option<&crate::media::FileProbe>, + paths: &[String], + done: &std::sync::atomic::AtomicUsize, + cancel: &std::sync::atomic::AtomicBool, +) -> Vec> { + use std::sync::atomic::Ordering::Relaxed; + let Some(probe) = probe.cloned() else { + return paths + .iter() + .map(|p| { + Ok(ProbeInfo { + format: Path::new(p).extension().map(|e| e.to_string_lossy().to_uppercase()).unwrap_or_default(), + ..Default::default() + }) + }) + .collect(); + }; + #[cfg(not(target_arch = "wasm32"))] + { + let n = workers(paths.len()); + if n > 1 { + let mut results: Vec>> = vec![None; paths.len()]; + let next = std::sync::atomic::AtomicUsize::new(0); + let out = std::sync::Mutex::new(&mut results); + std::thread::scope(|sc| { + for _ in 0..n { + sc.spawn(|| { + loop { + let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if i >= paths.len() { + break; + } + if cancel.load(Relaxed) { + break; + } + let r = probe(&paths[i]); + done.fetch_add(1, Relaxed); + out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r); + } + }); + } + }); + return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect(); + } + } + paths + .iter() + .map(|p| { + if cancel.load(Relaxed) { + return Err("cancelled".into()); + } + let r = probe(p); + done.fetch_add(1, Relaxed); + r + }) + .collect() +} + +/// Copy `src` into `dir` as `name` (default: its own name), made unique with -1, -2…; returns the +/// new path. The copy is verified ([`crate::import_move::copy_new`]: a new file, synced and +/// checked against `probe_hash`, the content hash the probe computed — or compared byte for byte +/// with the source when there is none); a bad one is removed and the photo reported as failed. +pub(super) fn copy_into(dir: &Path, src: &str, name: Option<&str>, probe_hash: Option<&str>) -> Result { + let name = name + .map(str::to_string) + .unwrap_or_else(|| Path::new(src).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "photo".into())); + let expect = probe_hash.and_then(lightcraft_preview::Hash128::parse); + crate::import_move::copy_new(Path::new(src), dir, &name, expect).map(|p| p.to_string_lossy().to_string()) +} + +#[cfg(test)] +mod prepared_tests { + use super::*; + + /// Extensions the picker may offer: every one in [`EXTENSIONS`], minus the + /// files this build cannot import (see [`accept_list`]). + fn pickable_extensions() -> Vec<&'static str> { + let heif_decodable = lightcraft_codecs::Format::Heif.can_decode(); + EXTENSIONS + .iter() + .copied() + .filter(|ext| match *ext { + "avif" => false, + "heic" | "heif" => heif_decodable, + _ => true, + }) + .collect() + } + + #[test] + fn accept_list_covers_every_pickable_extension_in_both_cases() { + let list = accept_list(); + for ext in pickable_extensions() { + assert!(list.contains(&format!(".{ext}")), "lowercase .{ext} missing from accept list"); + assert!(list.contains(&format!(".{}", ext.to_uppercase())), "uppercase .{} missing from accept list", ext.to_uppercase()); + } + } + + /// The web picker must not offer files this build cannot import: `.avif` has + /// no AV1 decoder in any build, so it is never listed, in either case. + #[test] + fn accept_list_never_offers_avif() { + let list = accept_list(); + assert!(!list.contains(".avif"), "{list}"); + assert!(!list.contains(".AVIF"), "{list}"); + } + + /// `.heic`/`.heif` follow the codecs' `heif` feature: listed (in both cases) + /// only when this build can decode them; otherwise hidden, so a web user + /// cannot pick a file that would fail to import. + #[test] + fn accept_list_offers_heic_heif_only_when_decodable() { + let list = accept_list(); + let listed = lightcraft_codecs::Format::Heif.can_decode(); + for ext in ["heic", "heif"] { + assert_eq!(list.contains(&format!(".{ext}")), listed, "lowercase .{ext}"); + assert_eq!(list.contains(&format!(".{}", ext.to_uppercase())), listed, "uppercase .{ext}"); + } + } + + #[test] + fn revalidate_add_only_converts_matching_ready_item() { + let mut s = Session::new(); + let existing_path = std::env::temp_dir().join("lightcraft-revalidate-existing.jpg"); + let other_path = std::env::temp_dir().join("lightcraft-revalidate-other.jpg"); + let existing = Photo::new(PhotoId(1), Source::File { path: existing_path.to_string_lossy().into() }, "existing.jpg", "JPG", 1, 1, "now"); + s.commit("existing", Op::AddPhoto { photo: Box::new(existing) }).unwrap(); + let ready = |path: &Path| ReadyFile { + path: path.to_string_lossy().into(), + stored: path.to_string_lossy().into(), + info: ProbeInfo { format: "JPG".into(), ..ProbeInfo::default() }, + sidecar: None, + placed: None, + }; + let mut prepared = Prepared { + scanned: 2, + items: vec![PreparedItem::Ready(Box::new(ready(&existing_path))), PreparedItem::Ready(Box::new(ready(&other_path)))], + }; + prepared.revalidate_add(&s); + assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. }))); + assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_)))); + } + + fn unread_paths(n: usize) -> Vec { + (0..n).map(|i| std::env::temp_dir().join(format!("lightcraft-probe-progress-{i}.jpg")).to_string_lossy().into()).collect() + } + + /// Reading the files of a large import (a Lightroom catalog's originals) reported nothing: + /// the progress stood still at "14 of 24554" for as long as it took. + #[test] + fn prepare_counts_the_files_it_reads() { + use std::sync::atomic::Ordering::Relaxed; + let mut s = Session::new(); + let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let (seen_in_probe, done_in_probe) = (seen.clone(), std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0))); + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap(); + let (done, last) = (job.read_done.clone(), done_in_probe.clone()); + job.probe = Some(std::sync::Arc::new(move |_: &str| { + seen_in_probe.fetch_add(1, Relaxed); + last.fetch_max(done.load(Relaxed), Relaxed); + Ok(ProbeInfo { format: "JPEG".into(), ..ProbeInfo::default() }) + })); + let prepared = job.prepare(&unread_paths(40), &std::sync::atomic::AtomicBool::new(false)); + assert_eq!(prepared.items.len(), 40); + assert_eq!(seen.load(Relaxed), 40); + assert!(done_in_probe.load(Relaxed) > 0, "progress moves while files are read"); + assert_eq!((job.read_done.load(Relaxed), job.read_total.load(Relaxed)), (40, 40), "every file read is counted"); + assert_eq!((job.done.load(Relaxed), job.total.load(Relaxed)), (40, 40), "handling them is counted on its own"); + } + + /// Cancel had no effect until every file had been read. + #[test] + fn cancel_stops_reading_files() { + use std::sync::atomic::Ordering::Relaxed; + let mut s = Session::new(); + let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap(); + let (seen_in_probe, cancel_in_probe) = (seen.clone(), cancel.clone()); + job.probe = Some(std::sync::Arc::new(move |_: &str| { + seen_in_probe.fetch_add(1, Relaxed); + cancel_in_probe.store(true, Relaxed); + Ok(ProbeInfo { format: "JPEG".into(), ..ProbeInfo::default() }) + })); + let prepared = job.prepare(&unread_paths(400), &cancel); + // at most the files the (up to 8) readers had already started + assert!(seen.load(Relaxed) <= 8, "read {} files after cancel", seen.load(Relaxed)); + assert!(prepared.items.is_empty()); + } +} diff --git a/crates/engine/src/import/job.rs b/crates/engine/src/import/job.rs new file mode 100644 index 000000000..47bc0756f --- /dev/null +++ b/crates/engine/src/import/job.rs @@ -0,0 +1,486 @@ +use super::commit::{How, Plan, Slot, Transfer, commit_prepared, name_family, run_transfers, system_clock}; +use super::files::{expand, probe_all}; +use super::types::{ImportCandidate, ImportMode, ImportOptions, ImportReport, Kept, OnDeleted}; +use super::*; + +/// Import files/folders. See the module docs. +pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result { + import_with(s, paths, &ImportOptions { mode, ..Default::default() }) +} + +/// Import files/folders with options (preset, keywords). See the module docs. +/// +/// The file-system half ([`ImportJob::prepare`]: expanding folders, probing, reading sidecars, +/// copying / placing files) and the catalog half ([`commit_prepared`]) are separate so the app can +/// run the first on a worker thread; here both run in turn. +pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> crate::Result { + let mut job = ImportJob::new(s, opts.clone())?; + let prepared = job.prepare(paths, &std::sync::atomic::AtomicBool::new(false)); + // probes a scan left for files not imported now stay for a later import + s.import_probes.extend(std::mem::take(&mut job.cache)); + commit_prepared(s, &job.opts, &job.now, prepared) +} + +/// Everything the file-system half of an import needs, detached from the session so it can run +/// on another thread (a network share or a card can take minutes). Built by [`ImportJob::new`]; +/// [`ImportJob::prepare`] may be called for several batches in turn (renamed copies keep counting, +/// duplicates are found across batches); each batch is then [`commit_prepared`] on the session. +pub struct ImportJob { + pub opts: ImportOptions, + /// The mode in effect (Copy / Move become Add in a library without its own folder). + mode: ImportMode, + lib_dir: Option, + copy_root: Option, + /// The library as it was when the import was set up, until it is looked up + /// ([`ImportJob::look_up_library`], with the first files): setting an import up is the + /// session's thread, and a table of a library of millions of photos takes seconds. With it, + /// whether a photo in Recently Deleted is passed over (an import that replaces such photos, + /// as the options said when the import was set up). + pub(super) library: Option<(lightcraft_catalog::Catalog, bool)>, + /// path → (photo, a Local browse record?) + pub(super) by_path: HashMap, + /// content hash → the photo that has it (`None`: a file earlier in this import) + pub(super) by_hash: HashMap>, + pub(super) probe: Option, + file_bytes: Option, + /// Probes from a preceding scan (not read again). + cache: HashMap, + naming: crate::sidecar::SidecarNaming, + now: String, + seq: usize, + /// Files found so far (folders expanded) and handled so far, for progress. + pub total: std::sync::Arc, + pub done: std::sync::Arc, + /// Of those, the files that have to be read first (not probed by a preceding scan) and the + /// ones read so far: the step before they are handled. + pub read_total: std::sync::Arc, + pub read_done: std::sync::Arc, +} + +/// The outcome of [`ImportJob::prepare`] for one batch, in file order. +#[derive(Debug, Default)] +pub struct Prepared { + pub scanned: usize, + pub(super) items: Vec, +} + +#[derive(Debug)] +pub(super) enum PreparedItem { + /// A Local (browsed) record imported for real. + Promote(PhotoId), + Duplicate { + path: String, + existing: Option, + reason: &'static str, + hash: Option, + }, + Failed(String, String), + Kept(Kept), + Ready(Box), +} + +#[derive(Debug)] +pub(super) struct ReadyFile { + pub(super) path: String, + pub(super) stored: String, + pub(super) info: ProbeInfo, + pub(super) sidecar: Option, + pub(super) placed: Option, +} + +impl Prepared { + /// Undo what the batch did on disk that only a commit would make permanent: files placed by a + /// Move (their sources were never touched). Copies stay, like copies of a failed commit. + pub fn rollback(&self) { + for it in &self.items { + if let PreparedItem::Ready(r) = it + && let Some(pl) = &r.placed + { + crate::import_move::rollback(pl); + } + } + } + + /// Entries in the batch (after expanding folders). + pub fn len(&self) -> usize { + self.items.len() + } + + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + /// Recheck Add-only entries against the owner catalog immediately before commit. + /// Preparation may have run while another import added the same path or content. + pub fn revalidate_add(&mut self, s: &Session) { + let mut paths = HashMap::new(); + let mut hashes = HashMap::new(); + for photo in s.catalog.photos() { + if let Source::File { path } = &photo.source { + paths.insert(normalize_import_path(path), photo.id); + } + if let Some(hash) = &photo.content_hash { + hashes.insert(hash.clone(), photo.id); + } + } + let mut items = Vec::with_capacity(self.items.len()); + for item in std::mem::take(&mut self.items) { + let PreparedItem::Ready(ready) = item else { + items.push(item); + continue; + }; + let ReadyFile { path, stored, info, sidecar, placed } = *ready; + let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied()); + let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied()); + if let Some(existing) = existing_path.or(existing_hash) { + if let Some(placed) = &placed { + crate::import_move::rollback(placed); + } + items.push(PreparedItem::Duplicate { + path, + existing: Some(existing), + reason: if existing_path.is_some() { "path" } else { "content" }, + hash: info.content_hash, + }); + } else { + items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed }))); + } + } + self.items = items; + } +} + +fn normalize_import_path(path: &str) -> String { + let path = path.replace('\\', "/"); + if cfg!(windows) { path.to_lowercase() } else { path } +} + +impl ImportJob { + /// Snapshot what an import with `opts` needs from the session (no file-system calls). Takes + /// the probes a preceding scan left ([`Session::import_probes`]). + pub fn new(s: &mut Session, opts: ImportOptions) -> crate::Result { + let mode = opts.mode; + // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of + // every added file in its own storage already, so "copy" there means "add". + let transfer = matches!(mode, ImportMode::Copy | ImportMode::Move); + let mode = if transfer && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode }; + if mode == ImportMode::Move && opts.local { + return Err(crate::EngineError::Other("browsing a folder never moves its files".into())); + } + let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); + // where copies go: the chosen folder, else the library's Originals/ + let copy_root = opts + .destination + .as_deref() + .filter(|d| !d.trim().is_empty()) + .map(std::path::PathBuf::from) + .or_else(|| lib_dir.as_ref().map(|l| l.join("Originals"))); + if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() { + return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into())); + } + let fresh = opts.on_deleted == OnDeleted::Fresh; + Ok(ImportJob { + seq: opts.rename_start.max(1), + opts, + mode, + lib_dir, + copy_root, + library: Some((s.catalog.clone(), fresh)), + by_path: HashMap::new(), + by_hash: HashMap::new(), + probe: s.media.file_probe.clone(), + file_bytes: s.media.file_bytes.clone(), + cache: std::mem::take(&mut s.import_probes), + naming: s.xmp.naming, + now: (s.clock)(), + total: Default::default(), + done: Default::default(), + read_total: Default::default(), + read_done: Default::default(), + }) + } + + /// The file-system half of importing `paths`: expand folders, skip known paths, probe, skip + /// duplicates by content, read sidecars, copy or place (Move) the files — several at once, + /// with the names and outcomes of one after another. Nothing in the catalog changes. Stops + /// before the next file once `cancel` is set. + pub fn prepare(&mut self, paths: &[String], cancel: &std::sync::atomic::AtomicBool) -> Prepared { + let files = self.expand(paths); + self.prepare_files(files, cancel) + } + + /// Expand `paths` (folders recursively, the library's own folder skipped) into the files an + /// import would handle; counted in [`ImportJob::total`]. + pub fn expand(&self, paths: &[String]) -> Vec { + let files = expand(paths, self.lib_dir.as_deref()); + self.total.fetch_add(files.len(), std::sync::atomic::Ordering::Relaxed); + files + } + + /// When the import started ([`Session::clock`]), for [`commit_prepared`]. + pub fn now(&self) -> &str { + &self.now + } + + /// What the library held when the import was set up, by path and by content: made once, with + /// the first files (on the thread that prepares them), from the library as it was then. The + /// snapshot comes back: the files the import is adding are ranked against it for the ownership + /// of the shared stem sidecars they read ([`crate::sidecar::ImportStems`]). + fn look_up_library(&mut self) -> Option { + let (library, fresh) = self.library.take()?; + for p in library.photos() { + // a trashed photo is replaced by a fresh import, so it doesn't count as a duplicate + if p.deleted && fresh { + continue; + } + if let Source::File { path } = &p.source { + self.by_path.insert(path.clone(), (p.id, p.local)); + } + if let Some(h) = &p.content_hash { + self.by_hash.insert(h.clone(), Some(p.id)); + } + } + Some(library) + } + + /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned. + pub fn prepare_files(&mut self, files: Vec, cancel: &std::sync::atomic::AtomicBool) -> Prepared { + use std::sync::atomic::Ordering::Relaxed; + // the library as it was, kept for the ownership of the shared stem sidecars below + let library = self.look_up_library(); + let opts = self.opts.clone(); + let mode = self.mode; + let mut out = Prepared { scanned: files.len(), items: Vec::new() }; + let mut todo = Vec::new(); + for f in files { + match self.by_path.get(f.as_str()).copied() { + Some((id, true)) if !opts.local => { + // a file that was only browsed (Local) joins the library when it is imported for real + self.by_path.insert(f, (id, false)); + out.items.push(PreparedItem::Promote(id)); + self.done.fetch_add(1, Relaxed); + } + Some((id, _)) => { + out.items.push(PreparedItem::Duplicate { path: f, existing: Some(id), reason: "path", hash: None }); + self.done.fetch_add(1, Relaxed); + } + None => todo.push(f), + } + } + + // files a preceding scan already probed are not read again (a network share is slow to read), + // as long as they are still there, as they were: a file gone or changed since is probed again + let cached: Vec> = todo + .iter() + .map(|f| { + // (the web's files are not std::fs files: kept as probed) + self.cache + .remove(f) + .filter(|info| cfg!(target_arch = "wasm32") || std::fs::metadata(f).is_ok_and(|m| m.is_file() && m.len() == info.file_size)) + }) + .collect(); + let missing: Vec = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect(); + // Reading the files is the slow part of adding them where they are: it is counted on its + // own, and `cancel` stops it too (a large library takes long enough to want both). + self.read_total.fetch_add(missing.len(), Relaxed); + let mut fresh = probe_all(self.probe.as_ref(), &missing, &self.read_done, cancel).into_iter(); + let probed: Vec> = + cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); + crate::memory::release(); + // Who owns the shared stem sidecar (`IMG_0001.xmp`) among the library's files and the ones + // arriving with this import: a JPEG or a TIFF arriving beside its raw reads its own sidecar + // and never the raw's packet (issue #1905). + let stems = crate::sidecar::ImportStems::for_import( + library.as_ref(), + todo.iter().zip(&probed).filter_map(|(path, info)| info.as_ref().ok().map(|i| (path.as_str(), i.kind))), + ); + // Files are planned one by one, in order (duplicates, sidecars, names, folders); their + // copies or moves — the slow part, reading and writing every byte and verifying it — are + // collected and run on several threads (issue #367), then their outcomes take their places. + let mut plan = Plan::default(); + for (path, info) in todo.into_iter().zip(probed) { + if cancel.load(Relaxed) { + break; + } + let mut info = match info { + Ok(i) => i, + Err(e) => { + log::warn!("import {path}: {e}"); + self.done.fetch_add(1, Relaxed); + plan.slots.push(Slot::Item(PreparedItem::Failed(path, e))); + continue; + } + }; + // a copy of a file whose transfer is still pending: a duplicate only if that one succeeds + if !opts.local + && let Some(h) = &info.content_hash + && plan.transfers.iter().any(|t| t.info.content_hash.as_ref() == Some(h)) + { + self.run_plan(&mut plan, &mut out, cancel); + if cancel.load(Relaxed) { + break; + } + } + if let Some(h) = &info.content_hash + && !opts.local + && let Some(existing) = self.by_hash.get(h) + { + self.done.fetch_add(1, Relaxed); + plan.slots.push(Slot::Item(PreparedItem::Duplicate { path, existing: *existing, reason: "content", hash: Some(h.clone()) })); + continue; + } + // the XMP sidecar (or a raw's embedded XMP), read before copying: its capture time files + // and names the copy when the file itself has none + let raw = info.kind == MediaKind::Raw; + let packet = crate::sidecar::find_sidecar_for_import(&stems, self.naming, &path) + .and_then(|f| std::fs::read_to_string(f).ok()) + .or_else(|| info.xmp.clone().filter(|_| raw)); + let target = crate::crs::Target::for_file(info.kind, &info.format, info.preview_only.is_some(), &info.meta.camera); + let sidecar = packet.and_then(|x| match crate::sidecar::parse_sidecar(&x, target) { + Ok(sc) => Some(sc), + Err(e) => { + log::warn!("import {path}: XMP: {e}"); + None + } + }); + if info.captured.is_none() { + info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone()); + } + // no date in the file or its sidecar: the file's own date, as Lightroom does + if info.captured.is_none() { + info.captured = info.file_modified.clone(); + } + let stored = match (mode, &self.copy_root) { + (ImportMode::Copy | ImportMode::Move, Some(root)) + if !crate::import_move::inside(Path::new(&path), root) + && !self.lib_dir.as_ref().is_some_and(|l| crate::import_move::inside(Path::new(&path), l)) => + { + // the templates see the photo as it will be catalogued (undated: the import time) + let own = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let mut q = Photo::new(PhotoId(0), Source::File { path: path.clone() }, &own, &info.format, 0, 0, &self.now); + q.captured = info.captured.clone(); + q.meta = info.meta.clone(); + let name = opts.rename.as_deref().filter(|t| !t.trim().is_empty()).map(|t| { + let n = crate::rename::expand(t, &q, self.seq); + self.seq += 1; + n + }); + let folders = opts.organize.folders(&q); + let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f)); + // the name this file gets depends on what a pending transfer of a like-named file + // into the same folder ends up with: that one goes first + let as_named = name.as_deref().unwrap_or(if own.is_empty() { "photo" } else { own.as_str() }); + let family = (dir.to_string_lossy().to_lowercase(), name_family(as_named)); + if plan.transfers.iter().any(|t| t.family == family) { + self.run_plan(&mut plan, &mut out, cancel); + if cancel.load(Relaxed) { + break; + } + } + let how = if mode == ImportMode::Move && !crate::import_move::is_symlink(Path::new(&path)) { + How::Place { name: name.unwrap_or(own) } + } else { + if mode == ImportMode::Move { + let reason = "a symbolic link: the file it points to was copied, the link and its target stay"; + plan.slots.push(Slot::Item(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() }))); + } + How::Copy { name } + }; + // Copy as DNG: the copied raw becomes a DNG (the copy is ours to replace) + let dng = opts.convert_dng && mode == ImportMode::Copy && raw && !info.format.eq_ignore_ascii_case("DNG"); + plan.slots.push(Slot::Transfer); + plan.transfers.push(Transfer { path, info, sidecar, dir, how, dng, family }); + continue; + } + (ImportMode::Move, _) => { + let reason = "already inside the destination or the library: added where it is"; + plan.slots.push(Slot::Item(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() }))); + path.clone() + } + _ => path.clone(), + }; + if let Some(h) = &info.content_hash { + self.by_hash.insert(h.clone(), None); + } + self.done.fetch_add(1, Relaxed); + plan.slots.push(Slot::Item(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed: None })))); + } + self.run_plan(&mut plan, &mut out, cancel); + out + } + + /// Run the transfers `plan` collected (on several threads) and append its items to `out` in + /// file order. Transfers not started because `cancel` was set leave no item. + fn run_plan(&mut self, plan: &mut Plan, out: &mut Prepared, cancel: &std::sync::atomic::AtomicBool) { + let mut done = run_transfers(std::mem::take(&mut plan.transfers), self.file_bytes.as_ref(), cancel, &self.done).into_iter(); + for slot in std::mem::take(&mut plan.slots) { + let item = match slot { + Slot::Item(it) => it, + Slot::Transfer => match done.next().flatten() { + Some(it) => it, + None => continue, + }, + }; + if let PreparedItem::Ready(r) = &item + && let Some(h) = &r.info.content_hash + { + self.by_hash.insert(h.clone(), None); + } + out.items.push(item); + } + } +} + +impl Session { + /// A thumbnail for an import candidate (not in the catalog): a raw's embedded preview, else a + /// small render of the file. `slot` makes the job's photo id unique (`u64::MAX - slot`). + pub fn candidate_thumb_job(&mut self, c: &ImportCandidate, edge: usize, slot: u64) -> Option { + if c.error.is_some() { + return None; + } + let id = PhotoId(u64::MAX - slot); + let mut p = Photo::new(id, Source::File { path: c.path.clone() }, &c.name, &c.format, c.width.max(1), c.height.max(1), ""); + p.kind = c.kind; + if let Some(info) = self.import_probes.get(&c.path) { + p.as_shot_wb = info.as_shot_wb; + p.embedded_lens = info.embedded_lens; + p.preview_only = info.preview_only.clone(); + } + p.develop = std::sync::Arc::new(p.import_defaults()); + let edge = edge.clamp(64, crate::media::SourceLevel::Thumb.max_edge()); + let level = crate::media::SourceLevel::Thumb; + let source = self.media.origin_ref(&p.source, level.max_edge()); + let key = lightcraft_preview::Hasher128::new().str(&c.path).u64(c.file_size).u64(edge as u64).finish().0 as u64; + let small = crate::media::RenderJob { + request_id: 0, + cache_generation: self.media.rendered.generation(), + source_key: None, + photo: id, + level, + source, + origin: p.source.clone(), + info: crate::media::source_info(&p), + settings: p.develop.clone(), + request: lightcraft_pipeline::RenderRequest::fit(edge, edge), + key, + cache: None, + stages: None, + view_cache: None, + display: None, + slide: None, + preview_cut: None, + display_fit: None, + }; + let embedded = match (&self.media.preview_loader, c.kind) { + (Some(l), MediaKind::Raw) => Some((c.path.clone(), l.clone(), edge)), + _ => None, + }; + Some(crate::media::QuickJob { request_id: 0, photo: id, key, cached: Vec::new(), embedded, small: Some(Box::new(small)) }) + } + + /// Use the system clock for import/edit times (native hosts). + pub fn with_system_clock(mut self) -> Self { + self.clock = Box::new(system_clock); + self + } +} diff --git a/crates/engine/src/import/scan.rs b/crates/engine/src/import/scan.rs new file mode 100644 index 000000000..61d8d4690 --- /dev/null +++ b/crates/engine/src/import/scan.rs @@ -0,0 +1,188 @@ +use super::files::{expand, probe_all}; +use super::types::ImportCandidate; +use super::*; + +/// What a [`scan`] needs from the session, so it can run on another thread (a folder on a network +/// share can take minutes to read). +pub struct ScanInput { + probe: Option, + skip: Option, + /// The library as it is now (photos are shared with the session, not copied): looked up for + /// the files the scan finds ([`library_files`]) and the contents it probes + /// ([`library_hashes`]), on the scan's thread. + library: std::sync::Arc, + cache: HashMap, +} + +/// The library's photos of `files`: path → (photo, its summary as a candidate that is there +/// already). One pass over the library, and a record only for a file that was found: a folder of +/// a hundred files in a library of millions costs a hundred records. +pub(super) fn library_files(library: &lightcraft_catalog::Catalog, files: &[String]) -> HashMap { + let found: std::collections::HashSet<&Path> = files.iter().map(Path::new).collect(); + let mut by_path = HashMap::new(); + if found.is_empty() { + return by_path; + } + for p in library.photos() { + if let Source::File { path } = &p.source + && found.contains(Path::new(path)) + { + let c = ImportCandidate { + path: path.clone(), + format: p.format.clone(), + kind: p.kind, + width: p.width, + height: p.height, + file_size: p.file_size, + captured: p.captured.clone(), + duplicate: Some("path".into()), + existing: Some(p.id.0), + ..Default::default() + }; + by_path.insert(PathBuf::from(path), (p.id, c)); + } + } + by_path +} + +/// Which photo of the library has each of the contents `hashes` (the later one where several +/// have it). One pass over the library, and an entry only for a content that was asked about. +pub(super) fn library_hashes<'a>(library: &lightcraft_catalog::Catalog, hashes: impl Iterator) -> HashMap { + let wanted: std::collections::HashSet<&str> = hashes.collect(); + let mut by_hash = HashMap::new(); + if wanted.is_empty() { + return by_hash; + } + for p in library.photos() { + if let Some(h) = &p.content_hash + && wanted.contains(h.as_str()) + { + by_hash.insert(h.clone(), p.id); + } + } + by_hash +} + +/// Progress and cancellation of a running [`scan_with`]. +#[derive(Default)] +pub struct ScanProgress { + /// Files to probe, set once the folders are expanded. + pub total: std::sync::atomic::AtomicUsize, + pub done: std::sync::atomic::AtomicUsize, + /// Shared with the scan's row in the activity stack (its ✕ sets it). + pub cancel: std::sync::Arc, +} + +/// The result of [`scan_with`]: the candidates, and the probes to keep for the import that follows. +pub struct ScanOutput { + pub candidates: Vec, + pub probes: HashMap, +} + +impl ScanInput { + pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec) { + let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); + // (no table of the library here: this is the session's thread, and the library may hold + // millions of photos. The scan looks up what it finds) + let library = std::sync::Arc::new(s.catalog.clone()); + let input = ScanInput { probe: s.media.file_probe.clone(), skip, library, cache: std::mem::take(&mut s.import_probes) }; + (input, paths.to_vec()) + } + + /// The folder a scan never enters (the library's own). + pub(crate) fn skip(&self) -> Option<&Path> { + self.skip.as_deref() + } + + /// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one). + pub(crate) fn library(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.library) + } +} + +/// Find and probe what importing `paths` would add, marking duplicates (by path, or by content +/// against the library and earlier candidates). Nothing is added; the probes are kept for the +/// import that follows. +pub fn scan(s: &mut Session, paths: &[String]) -> Vec { + let (input, paths) = ScanInput::new(s, paths); + let out = scan_with(input, &paths, &ScanProgress::default()); + s.import_probes = out.probes; + out.candidates +} + +/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it +/// has) when `progress.cancel` is set. +pub fn scan_with(input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput { + let files = expand(paths, input.skip.as_deref()); + scan_files(input, files, progress) +} + +/// [`scan_with`] for files that were found already (`paths` expanded, [`expand`]). +pub(crate) fn scan_files(mut input: ScanInput, files: Vec, progress: &ScanProgress) -> ScanOutput { + use std::sync::atomic::Ordering::Relaxed; + let by_path = library_files(&input.library, &files); + let todo: Vec = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect(); + progress.total.store(todo.len(), Relaxed); + // probes from a preceding `scan` are reused when the file is unchanged (same size) + let cached: Vec> = todo + .iter() + .map(|f| { + let info = input.cache.remove(f)?; + // (✕ set while the files' cached probes are checked: no more round trips to the disk; what is + // not reached counts as not probed, like the files after it) + if progress.cancel.load(Relaxed) { + return None; + } + let size = std::fs::metadata(f).map(|m| m.len()).ok(); + (size.is_none() || size == Some(info.file_size)).then_some(info) + }) + .collect(); + let missing: Vec = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect(); + progress.done.store(todo.len() - missing.len(), Relaxed); + let mut fresh = probe_all(input.probe.as_ref(), &missing, &progress.done, &progress.cancel).into_iter(); + let probed: Vec> = + cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); + crate::memory::release(); + let mut probes: HashMap> = todo.into_iter().zip(probed).collect(); + // (who in the library has a content that was probed: asked once the probes are in) + let probed_hashes = probes.values().filter_map(|r| r.as_ref().ok()).filter_map(|info| info.content_hash.as_deref()); + let mut seen_hash: HashMap> = + library_hashes(&input.library, probed_hashes).into_iter().map(|(h, id)| (h, Some(id.0))).collect(); + let mut out = Vec::with_capacity(files.len()); + let mut kept = HashMap::new(); + for f in files { + if let Some((_, c)) = by_path.get(Path::new(&f)) { + let mut c = c.clone(); + c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); + out.push(c); + continue; + } + let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); + let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() }; + match probes.remove(&f) { + Some(Ok(info)) => { + (c.format, c.kind, c.width, c.height, c.file_size, c.captured) = + (info.format.clone(), info.kind, info.width, info.height, info.file_size, info.captured.clone()); + // as the import will date it (a sidecar's date, when there is one, still wins there) + c.captured = c.captured.or_else(|| info.file_modified.clone()); + c.preview_only = info.preview_only.clone(); + if let Some(h) = &info.content_hash { + match seen_hash.get(h) { + Some(existing) => { + c.duplicate = Some("content".into()); + c.existing = *existing; + } + None => { + seen_hash.insert(h.clone(), None); + } + } + } + kept.insert(f, info); + } + Some(Err(e)) => c.error = Some(e), + None => c.error = Some("not probed".into()), + } + out.push(c); + } + ScanOutput { candidates: out, probes: kept } +} diff --git a/crates/engine/src/import/types.rs b/crates/engine/src/import/types.rs new file mode 100644 index 000000000..4a848015b --- /dev/null +++ b/crates/engine/src/import/types.rs @@ -0,0 +1,230 @@ +use super::*; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ImportMode { + /// Reference the files where they are. + #[default] + Add, + /// Copy the files into the library's `Originals/` folder. + Copy, + /// Move the files there: as Copy, then each original (and its sidecars) is removed from the + /// source once the copy is verified and catalogued. + Move, +} + +/// What an import does with a file the library already has in Recently Deleted. +/// +/// Someone who imports such a file wants the photo in the library again, so the default brings +/// it back (issues #298, #706: before, the import skipped it as "already in the library" while +/// the grid showed it deleted, and the user was stuck). Restoring loses nothing — the edits, +/// rating and albums come back with the photo and Reset Edits gives a clean start — where +/// importing afresh would silently drop them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum OnDeleted { + /// Leave it there; the report says so ([`Duplicate::existing_deleted`]). + Skip, + /// Bring the trashed photo back, with its edits ([`ImportReport::restored`]); a photo whose + /// file is no longer where the library had it is relinked to the imported file. + #[default] + Restore, + /// Delete the trashed record permanently and import the file as a new photo. + Fresh, +} + +impl OnDeleted { + pub fn parse(s: &str) -> Option { + match s { + "skip" => Some(Self::Skip), + "restore" => Some(Self::Restore), + "fresh" => Some(Self::Fresh), + _ => None, + } + } +} + +/// What an import does besides adding the photos. +#[derive(Clone, Debug, Default)] +pub struct ImportOptions { + /// A file that is in Recently Deleted: restore (default), skip, or import afresh. + pub on_deleted: OnDeleted, + pub mode: ImportMode, + /// Applied to every imported photo (one History entry). + pub preset: Option, + /// Added to every imported photo. + pub keywords: Vec, + /// Browsing a folder: photos come in as `local` (not in the library), and a file with the + /// same content as one already known is still listed. + pub local: bool, + /// Copy: where the copies go (default: the library's `Originals/`). + pub destination: Option, + /// Copy: the folders inside the destination. + pub organize: Organize, + /// Copy: a file-name template for the copies ([`crate::rename::expand`] tokens: `{name}`, + /// `{seq:N}`, `{date:%Y%m%d}`, `{camera}`…), numbered from `rename_start`. + pub rename: Option, + pub rename_start: usize, + /// A metadata preset (by name) applied to every imported photo. + pub metadata_preset: Option, + /// Copy: raws are copied as DNG (Copy as DNG). + pub convert_dng: bool, +} + +/// How copies are filed in the destination. The date is the capture time, else the time of the +/// import. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum Organize { + /// `YYYY/YYYY-MM-DD/` by capture date. + #[default] + ByDay, + /// `YYYY/YYYY-MM/`. + ByMonth, + /// All in the destination itself. + Flat, + /// A folder template, e.g. `{date:%Y}/{date:%Y%m%d}` → `2026/20260114/`: the template's own + /// `/` make the levels, each expanded with the file-name tokens ([`crate::rename::expand_folder`]; + /// never outside the destination). + Template(String), +} + +impl Organize { + /// `date` | `month` | `flat`, or a folder template (anything with a `{`, `/` or `\`). + pub fn parse(s: &str) -> Option { + match s { + "date" | "day" | "byDay" => Some(Organize::ByDay), + "month" | "byMonth" => Some(Organize::ByMonth), + "flat" | "none" | "intoOneFolder" => Some(Organize::Flat), + t if t.contains(['{', '/', '\\']) => Some(Organize::Template(t.trim().to_string())), + _ => None, + } + } + + /// The folders (inside the destination) a copy of `p` goes to. + pub fn folders(&self, p: &Photo) -> Vec { + let date = p.date(); + let day = date.get(..10).filter(|d| d.len() == 10).unwrap_or("undated"); + let year = day.get(..4).unwrap_or("undated"); + match self { + Organize::ByDay => vec![year.into(), day.into()], + Organize::ByMonth => vec![year.into(), day.get(..7).unwrap_or("undated").into()], + Organize::Flat => Vec::new(), + Organize::Template(t) => crate::rename::expand_folder(t, p, 1), + } + } +} + +/// A file found by [`scan`], for the import review. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ImportCandidate { + pub path: String, + pub name: String, + pub format: String, + pub kind: MediaKind, + pub width: u32, + pub height: u32, + pub file_size: u64, + pub captured: Option, + /// `"path"` (already in the library), `"content"` (same bytes in the library or earlier in this + /// list): skipped by the import. + pub duplicate: Option, + /// The photo that already has it. + pub existing: Option, + /// Not readable. + pub error: Option, + /// A raw variant that can't be decoded yet (why): it imports as preview only. + #[serde(skip_serializing_if = "Option::is_none")] + pub preview_only: Option, +} + +#[derive(Clone, Debug, Default, Serialize)] +pub struct Duplicate { + pub path: String, + /// The photo that already has these bytes (or this path). + pub existing: Option, + /// `"path"` (already imported from there) or `"content"` (same bytes elsewhere). + pub reason: &'static str, + /// That photo is in Recently Deleted (see [`OnDeleted`]). + #[serde(rename = "existingDeleted", skip_serializing_if = "std::ops::Not::not")] + pub existing_deleted: bool, +} + +#[derive(Clone, Debug, Default, Serialize)] +pub struct ImportReport { + pub imported: Vec, + pub duplicates: Vec, + /// [`OnDeleted::Restore`]: photos brought back from Recently Deleted instead of imported. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub restored: Vec, + /// Move: the originals that were moved (and removed from the source). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub moved: Vec, + /// Move: originals (or sidecars) left at the source, and why — the photo may still have been + /// imported (from its copy, or in place). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub kept: Vec, + /// (path, error) + pub failed: Vec<(String, String)>, + /// Files found after expanding folders. + pub scanned: usize, + /// Photos whose metadata/develop settings were read from an XMP sidecar (or embedded XMP). + pub sidecars: usize, + /// Per-photo diagnostics from foreign XMP develop settings that could not be fully carried over. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub warnings: Vec, +} + +/// A structured diagnostic from foreign XMP develop settings. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ImportWarningCode { + /// A named camera profile was found but cannot be applied as Lightroom does. + CameraProfile, + /// Lightroom geometry data was found but its saved transform is not retained. + UprightGeometry, + /// One or more other XMP develop fields have no LightCraft mapping. + UnmappedXmp, +} + +/// The saved Lightroom Upright mode associated with a foreign geometry transform. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ImportUprightMode { + Off, + Auto, + Level, + Vertical, + Full, + Guided, + Unknown, +} + +/// One actionable XMP import diagnostic. `path` is filled when its photo is committed. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportWarning { + pub path: String, + pub code: ImportWarningCode, + #[serde(skip_serializing_if = "Option::is_none")] + pub profile: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mode: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub fields: Vec, +} + +/// A file moved by an import. +#[derive(Clone, Debug, Default, Serialize)] +pub struct Moved { + pub from: String, + pub to: String, + /// Its sidecars' new paths. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub sidecars: Vec, +} + +/// A source a Move left where it was. +#[derive(Clone, Debug, Default, Serialize)] +pub struct Kept { + pub path: String, + pub reason: String, +}