mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-24 10:09:01 -06:00
Nathaniel LedfordandClaude Opus 5.5
d1d3dbbb0e
439ab04c2 fixed the checklist row but missed the module docs.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
892 lines
41 KiB
Rust
892 lines
41 KiB
Rust
//! Synchronize Folder: bring a folder of the library up to date with what is on disk, as
|
||
//! Lightroom Classic's Synchronize Folder does.
|
||
//!
|
||
//! A [scan](scan_with) compares the folder (and the folders inside it) with the library and finds
|
||
//! three kinds of change, touching nothing:
|
||
//!
|
||
//! - **new** files: supported files the library doesn't have, by path or by content (the import
|
||
//! scan's duplicate rules, [`crate::import::scan_with`]);
|
||
//! - **missing** photos: library photos in the folder whose file is gone (the Missing Photos rule,
|
||
//! [`crate::cmd::missing::checked_path`]);
|
||
//! - **metadata** updates: photos whose XMP sidecar file was saved after the photo came into the
|
||
//! library or was last edited here, and says something the library doesn't (rating, flag,
|
||
//! label, metadata, capture time or develop settings). Reading one makes the sidecar win, as
|
||
//! Read Metadata from File does. Embedded XMP in a raw is not a sidecar and is not looked at.
|
||
//!
|
||
//! [`synchronize`] then imports the new files (in place), relinks the moved photos, moves the
|
||
//! missing photos to Recently Deleted and reads the metadata updates — each only when chosen — as
|
||
//! one undo step. It runs in halves ([`SyncJob`]) so that, in the app, every file it reads is
|
||
//! read on a worker thread and only the catalog changes happen on the UI thread — with what every
|
||
//! command does after them there: the library's durable save and, with XMP auto-write on, the
|
||
//! sidecars of the photos whose metadata was read.
|
||
//!
|
||
//! The scan runs without the session ([`SyncInput`] + [`scan_with`]), so the app can run it on a
|
||
//! worker thread: a folder on a sleeping network share must never stall a frame.
|
||
//!
|
||
//! Limits of telling an update from a sidecar that is merely older:
|
||
//! - a sidecar the library itself wrote (auto-write on) agrees with it and is never an update;
|
||
//! - the library records when a photo came in and when its develop settings last changed, not
|
||
//! when its rating, flag, label or metadata did: a sidecar another app saved, followed by such
|
||
//! a change made here (with auto-write off), is listed as an update, and reading it puts the
|
||
//! older value back. Reading updates is therefore never the default;
|
||
//! - times are compared as the clock the session runs with; headless sessions (CLI, MCP) start
|
||
//! with a fixed clock, so their own times say little;
|
||
//! - a sidecar is found as Read Metadata from File finds it, including the other naming
|
||
//! scheme's file (`IMG_1.xmp` for `IMG_1.JPG` beside `IMG_1.CR3`).
|
||
//!
|
||
//! The walk for new files stops at the import's limits on depth and entries (logged), or when the
|
||
//! scan is cancelled.
|
||
|
||
use std::collections::HashMap;
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::Arc;
|
||
|
||
use lightcraft_catalog::{Catalog, Filter, Op, Photo, PhotoId, Sort, Source};
|
||
use serde::Serialize;
|
||
|
||
use crate::import::{ImportCandidate, ImportMode, ImportOptions, ScanInput, ScanProgress};
|
||
use crate::media::ProbeInfo;
|
||
use crate::sidecar::SidecarNaming;
|
||
use crate::{EngineError, Result, Session};
|
||
|
||
/// The largest XMP sidecar read when looking for metadata updates; a bigger one is skipped.
|
||
const MAX_SIDECAR_BYTES: u64 = 16 * 1024 * 1024;
|
||
|
||
/// How far a scan is: the folders and files listed so far (the walk, which comes first and on a
|
||
/// network share can take minutes), the folder's files probed ([`ScanProgress`], whose `cancel`
|
||
/// stops the whole scan, walk included) and the library's photos checked.
|
||
#[derive(Default)]
|
||
pub struct SyncProgress {
|
||
pub files: ScanProgress,
|
||
pub checked: std::sync::atomic::AtomicUsize,
|
||
pub to_check: std::sync::atomic::AtomicUsize,
|
||
/// Folders and files (photos or not) the walk has listed.
|
||
pub folders_listed: std::sync::atomic::AtomicUsize,
|
||
pub files_listed: std::sync::atomic::AtomicUsize,
|
||
/// The walk is over (or there was nothing to walk).
|
||
pub listed: std::sync::atomic::AtomicBool,
|
||
}
|
||
|
||
impl SyncProgress {
|
||
/// (done, of how many), both halves together.
|
||
pub fn counts(&self) -> (usize, usize) {
|
||
use std::sync::atomic::Ordering::Relaxed;
|
||
let done = self.files.done.load(Relaxed).saturating_add(self.checked.load(Relaxed));
|
||
(done, self.files.total.load(Relaxed).saturating_add(self.to_check.load(Relaxed)))
|
||
}
|
||
|
||
/// While a scan lists the folder (before any file can be checked): the folders and files
|
||
/// listed so far. `None` once the listing is over.
|
||
pub fn listing(&self) -> Option<(usize, usize)> {
|
||
use std::sync::atomic::Ordering::{Acquire, Relaxed};
|
||
// (Acquire pairs with the Release store of `listed`: once the listing is seen to be
|
||
// over, so is what was stored before it, and the counts that follow are the final ones)
|
||
(!self.listed.load(Acquire)).then(|| (self.folders_listed.load(Relaxed), self.files_listed.load(Relaxed)))
|
||
}
|
||
}
|
||
|
||
/// What a scan of a folder needs, taken from the session so it can run on a worker thread.
|
||
pub struct SyncInput {
|
||
folder: String,
|
||
scan: ScanInput,
|
||
/// The library's photos in the folder.
|
||
ids: Vec<PhotoId>,
|
||
/// How the library names sidecars; how each photo's is named is worked out by the scan (a
|
||
/// pass over the library, which the session's thread does not wait for).
|
||
naming: SidecarNaming,
|
||
/// The library as it is now (the one the scan looks files up in): for those photos, their
|
||
/// sidecars' names and the colour label names a sidecar may use.
|
||
labels: Arc<Catalog>,
|
||
}
|
||
|
||
/// What changed in a folder since it came into the library.
|
||
#[derive(Clone, Debug, Default, Serialize)]
|
||
pub struct FolderChanges {
|
||
/// The folder, as asked for.
|
||
pub path: String,
|
||
/// Files to import.
|
||
pub new: Vec<ImportCandidate>,
|
||
/// Files whose content the library already has (under another path).
|
||
pub duplicates: usize,
|
||
/// Supported files that couldn't be read.
|
||
pub unreadable: Vec<ImportCandidate>,
|
||
/// Photos whose file is gone.
|
||
pub missing: Vec<FolderPhoto>,
|
||
/// Photos whose sidecar has news.
|
||
pub metadata: Vec<FolderPhoto>,
|
||
/// Photos whose file was renamed or moved within the folder (found by its content).
|
||
pub moved: Vec<FolderMoved>,
|
||
/// The folder itself is not there (moved or renamed whole, or on a disk that isn't
|
||
/// connected): nothing else is reported, rather than every photo as missing.
|
||
pub offline: bool,
|
||
/// The probes of the new files, kept for the import that follows.
|
||
#[serde(skip)]
|
||
probes: HashMap<String, ProbeInfo>,
|
||
/// The folder's photos as the scan saw them: the scan is current while they are the same
|
||
/// records (any change to one replaces it in the catalog).
|
||
#[serde(skip)]
|
||
seen: Vec<Arc<Photo>>,
|
||
}
|
||
|
||
/// A photo of the folder and its file.
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||
pub struct FolderPhoto {
|
||
pub id: u64,
|
||
pub path: String,
|
||
}
|
||
|
||
/// A photo whose file is now somewhere else in the folder.
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||
pub struct FolderMoved {
|
||
pub id: u64,
|
||
pub from: String,
|
||
pub to: String,
|
||
}
|
||
|
||
/// What [`synchronize`] does with the changes found.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
pub struct SyncChoice {
|
||
pub import_new: bool,
|
||
pub relink_moved: bool,
|
||
pub remove_missing: bool,
|
||
pub read_metadata: bool,
|
||
}
|
||
|
||
impl Default for SyncChoice {
|
||
/// Lightroom Classic's defaults: import the new files, leave the rest to be looked at; and
|
||
/// photos whose file moved follow it (nothing is lost, and undo puts them back).
|
||
fn default() -> Self {
|
||
SyncChoice { import_new: true, relink_moved: true, remove_missing: false, read_metadata: false }
|
||
}
|
||
}
|
||
|
||
/// What [`synchronize`] did.
|
||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||
pub struct SyncReport {
|
||
pub imported: usize,
|
||
pub relinked: usize,
|
||
pub removed: usize,
|
||
pub read: usize,
|
||
/// (path or photo, why) for what couldn't be done.
|
||
pub failed: Vec<(String, String)>,
|
||
}
|
||
|
||
fn invalid(msg: impl Into<String>) -> EngineError {
|
||
EngineError::Other(msg.into())
|
||
}
|
||
|
||
impl SyncInput {
|
||
/// The scan of folder `path`: a folder the library holds photos in, not the startup disk. A
|
||
/// whole disk or share, or a folder that holds disks (`/Volumes`, `/mnt`…), only with `disk`
|
||
/// (as for Remove Disk from Library: its missing photos can be a whole disk's).
|
||
pub fn new(s: &mut Session, path: &str, disk: bool) -> Result<SyncInput> {
|
||
let path = path.trim();
|
||
if path.is_empty() {
|
||
return Err(invalid("missing `path`"));
|
||
}
|
||
if lightcraft_catalog::folders::is_startup_disk(path) {
|
||
return Err(invalid("choose a folder, not the whole startup disk"));
|
||
}
|
||
let f = Filter { library_folder: Some(path.to_string()), ..Default::default() };
|
||
let ids = s.catalog.query(&f, &Sort::default());
|
||
if ids.is_empty() {
|
||
return Err(invalid(format!("{path}: no photo in the library was imported from it")));
|
||
}
|
||
if !disk && (lightcraft_catalog::folders::is_disk_root(path) || crate::cmd::library::covers_other_disks(s, path, &ids)) {
|
||
return Err(invalid(format!("{path} is a whole disk or holds disks: pass `disk: true` to synchronize it")));
|
||
}
|
||
let (scan, _) = ScanInput::new(s, &[path.to_string()]);
|
||
let labels = scan.library();
|
||
Ok(SyncInput { folder: path.to_string(), scan, ids, naming: s.xmp.naming, labels })
|
||
}
|
||
}
|
||
|
||
/// Find what changed in the folder (see the module docs). Reads the disk, never the session.
|
||
///
|
||
/// Both halves wait on the disk (a network share answers each file slowly), so they run at once:
|
||
/// the look for new files (which probes them on its own threads) beside the checks of the
|
||
/// library's photos, which run on several threads too. Stops early when `progress.files.cancel`
|
||
/// is set.
|
||
pub fn scan_with(input: SyncInput, progress: &SyncProgress) -> FolderChanges {
|
||
let SyncInput { folder, scan, ids, naming, labels } = input;
|
||
let seen: Vec<Arc<Photo>> = ids.iter().filter_map(|id| labels.photo(*id)).map(Arc::clone).collect();
|
||
if !cfg!(target_arch = "wasm32") && !Path::new(&folder).is_dir() {
|
||
progress.listed.store(true, std::sync::atomic::Ordering::Release);
|
||
return FolderChanges { path: folder, seen, offline: true, ..Default::default() };
|
||
}
|
||
// (how each photo's sidecar is named: a pass over the library, made here and not while the
|
||
// session waits, and not at all for a folder that is not there)
|
||
let namings = crate::sidecar::namings_in(&labels, naming, &ids);
|
||
let photos: Vec<(Arc<Photo>, SidecarNaming)> =
|
||
ids.iter().zip(namings).filter_map(|(id, naming)| labels.photo(*id).map(|p| (Arc::clone(p), naming))).collect();
|
||
// The folder is walked once, and its listing answers for the library's photos in it: on a
|
||
// network share that is one request for a folder's names where it was several for each
|
||
// photo. Then both halves wait on the disk at once: the new files are probed (on their own
|
||
// threads) beside the checks of the photos, which ask the disk only what the listing can't say.
|
||
// (how many photos there are to check is known before the walk; while the walk runs, the
|
||
// folders and files it listed tell how far it is)
|
||
progress.to_check.store(photos.len(), std::sync::atomic::Ordering::Relaxed);
|
||
let mut listing = Listing::default();
|
||
let (files, _) = crate::import::expand_noting(
|
||
std::slice::from_ref(&folder),
|
||
scan.skip(),
|
||
crate::walk::Limits::default(),
|
||
Some(&mut |dir, names, complete| {
|
||
use std::sync::atomic::Ordering::Relaxed;
|
||
progress.folders_listed.fetch_add(1, Relaxed);
|
||
let files = names.iter().filter(|(_, kind)| *kind == crate::walk::EntryKind::File).count();
|
||
progress.files_listed.fetch_add(files, Relaxed);
|
||
listing.note(dir, names, complete);
|
||
// ✕ stops the walk too (a cancelled scan's result is not used)
|
||
!progress.files.cancel.load(Relaxed)
|
||
}),
|
||
);
|
||
progress.listed.store(true, std::sync::atomic::Ordering::Release);
|
||
let listing = &listing;
|
||
// (no threads on the web: one after the other there)
|
||
#[cfg(target_arch = "wasm32")]
|
||
let (out, checks) = (crate::import::scan_files(scan, files, &progress.files), check_photos(&photos, &labels, listing, progress));
|
||
#[cfg(not(target_arch = "wasm32"))]
|
||
let (out, checks) = std::thread::scope(|sc| {
|
||
let photos = &photos;
|
||
let labels = &labels;
|
||
let checks = sc.spawn(move || check_photos(photos, labels, listing, progress));
|
||
let out = crate::import::scan_files(scan, files, &progress.files);
|
||
// (a check that panicked counts as nothing found; the scan still reports the rest)
|
||
(out, checks.join().unwrap_or_default())
|
||
});
|
||
log::debug!(
|
||
"synchronize: {} folders listed; the disk was asked {} times about the library's photos",
|
||
listing.folders.len(),
|
||
listing.asked.load(std::sync::atomic::Ordering::Relaxed)
|
||
);
|
||
let mut changes = FolderChanges { path: folder, seen, ..Default::default() };
|
||
for ((p, _), check) in photos.iter().zip(checks) {
|
||
let Some(file) = crate::cmd::missing::checked_path(p) else { continue };
|
||
let found = FolderPhoto { id: p.id.0, path: file.to_string() };
|
||
match check {
|
||
Check::Missing => changes.missing.push(found),
|
||
Check::News => changes.metadata.push(found),
|
||
Check::Nothing => {}
|
||
}
|
||
}
|
||
let moved_from = match_moved(&changes.missing, &out.candidates, &out.probes, &labels);
|
||
for c in out.candidates {
|
||
if moved_from.values().any(|to| *to == c.path) {
|
||
continue;
|
||
}
|
||
let browsed = c.existing.and_then(|id| labels.photo(PhotoId(id))).is_some_and(|p| p.local && !p.deleted);
|
||
match (c.duplicate.as_deref(), &c.error) {
|
||
// a file only looked at in Local is not in the library yet: it is new
|
||
(Some("path"), _) if browsed => changes.new.push(ImportCandidate { duplicate: None, existing: None, ..c }),
|
||
(Some("path"), _) => {}
|
||
(Some(_), _) => changes.duplicates += 1,
|
||
(None, Some(_)) => changes.unreadable.push(c),
|
||
(None, None) => changes.new.push(c),
|
||
}
|
||
}
|
||
let missing = std::mem::take(&mut changes.missing);
|
||
for m in missing {
|
||
match moved_from.get(&m.path) {
|
||
Some(to) => changes.moved.push(FolderMoved { id: m.id, from: m.path, to: to.clone() }),
|
||
None => changes.missing.push(m),
|
||
}
|
||
}
|
||
changes.probes = out.probes;
|
||
changes
|
||
}
|
||
|
||
/// Whether a photo is the same as far as synchronizing goes: its file, its place in the library
|
||
/// and what a sidecar is compared with. Face and pet regions found in the background meanwhile
|
||
/// don't make a scan stale.
|
||
fn same_for_sync(a: &Arc<Photo>, b: &Arc<Photo>) -> bool {
|
||
if Arc::ptr_eq(a, b) {
|
||
return true;
|
||
}
|
||
let meta = |p: &Photo| lightcraft_catalog::Meta { regions: Vec::new(), ..p.meta.clone() };
|
||
(&a.source, a.deleted, a.local, a.copy_of, &a.content_hash, a.file_size, a.rating, a.flag, a.label, &a.captured, &a.edited, &a.develop)
|
||
== (&b.source, b.deleted, b.local, b.copy_of, &b.content_hash, b.file_size, b.rating, b.flag, b.label, &b.captured, &b.edited, &b.develop)
|
||
&& meta(a) == meta(b)
|
||
}
|
||
|
||
/// The content a hash stands for: a Duplicate's own hash is the original's plus `:dup<id>`.
|
||
fn base_hash(h: &str) -> &str {
|
||
h.split(':').next().unwrap_or(h)
|
||
}
|
||
|
||
/// Missing photos' files found elsewhere in the folder (old path → new path): the file was
|
||
/// renamed or moved, and the photo (with its edits) follows it rather than going missing.
|
||
///
|
||
/// By content first: each missing file's hash claims one file with that content (two missing
|
||
/// files with the same content claim two). A photo with no hash on record (taken over from a
|
||
/// Lightroom catalog) is matched by file size and dimensions, and only when exactly one missing
|
||
/// file and exactly one unclaimed file of the folder agree on them.
|
||
fn match_moved(
|
||
missing: &[FolderPhoto],
|
||
candidates: &[ImportCandidate],
|
||
probes: &HashMap<String, ProbeInfo>,
|
||
labels: &Catalog,
|
||
) -> HashMap<String, String> {
|
||
// distinct missing files (a photo's virtual copies share its file), with what is known of them
|
||
let mut files: Vec<(&str, Option<&Photo>)> = Vec::new();
|
||
for m in missing {
|
||
if !files.iter().any(|(p, _)| *p == m.path) {
|
||
files.push((m.path.as_str(), labels.photo(PhotoId(m.id)).map(|p| &**p)));
|
||
}
|
||
}
|
||
let mut by_hash: HashMap<&str, std::collections::VecDeque<&str>> = HashMap::new();
|
||
for (path, p) in &files {
|
||
if let Some(h) = p.and_then(|p| p.content_hash.as_deref()) {
|
||
by_hash.entry(base_hash(h)).or_default().push_back(path);
|
||
}
|
||
}
|
||
let usable = |c: &&ImportCandidate| c.error.is_none() && c.duplicate.as_deref() != Some("path");
|
||
let mut moved: HashMap<String, String> = HashMap::new();
|
||
for c in candidates.iter().filter(usable) {
|
||
let hash = probes.get(&c.path).and_then(|i| i.content_hash.as_deref());
|
||
if let Some(from) = hash.and_then(|h| by_hash.get_mut(base_hash(h))).and_then(std::collections::VecDeque::pop_front) {
|
||
moved.insert(from.to_string(), c.path.clone());
|
||
}
|
||
}
|
||
let key = |size: u64, w: u32, h: u32| (size > 0 && w > 0 && h > 0).then_some((size, w, h));
|
||
let mut hashless: HashMap<(u64, u32, u32), Vec<&str>> = HashMap::new();
|
||
for (path, p) in &files {
|
||
if let Some(p) = p.filter(|p| p.content_hash.is_none())
|
||
&& let Some(k) = key(p.file_size, p.width, p.height)
|
||
{
|
||
hashless.entry(k).or_default().push(path);
|
||
}
|
||
}
|
||
let mut found: HashMap<(u64, u32, u32), Vec<&str>> = HashMap::new();
|
||
for c in candidates.iter().filter(usable).filter(|c| !moved.values().any(|to| *to == c.path)) {
|
||
if let Some(k) = key(c.file_size, c.width, c.height) {
|
||
found.entry(k).or_default().push(c.path.as_str());
|
||
}
|
||
}
|
||
for (k, from) in hashless {
|
||
if let ([from], Some([to])) = (from.as_slice(), found.get(&k).map(Vec::as_slice)) {
|
||
moved.insert(from.to_string(), to.to_string());
|
||
}
|
||
}
|
||
moved
|
||
}
|
||
|
||
/// What checking one photo of the folder found.
|
||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||
enum Check {
|
||
#[default]
|
||
Nothing,
|
||
Missing,
|
||
News,
|
||
}
|
||
|
||
/// The listings of the folders a scan walked: what is in each, by name. It answers "is this file
|
||
/// there" for the library's photos and their sidecars without asking the disk, which on a
|
||
/// network share is a round trip for every name (and a photo without a sidecar took five).
|
||
///
|
||
/// It answers only what holds on any file system. A name is there when the folder lists exactly
|
||
/// that name ([`asks_the_disk`] aside). It is not there when the folder was listed to its end,
|
||
/// every name in the folder is ASCII, the name asked about is plain ASCII ([`plain_name`]), and
|
||
/// no name in the folder is that name in another case. Anything else is not answered, and the
|
||
/// disk is asked as before: a name that differs in case only, a link, a folder that was not
|
||
/// walked or not to its end, and every name that is not listed in a folder that holds a name
|
||
/// outside ASCII. (File systems hold names outside ASCII equal in more ways than can be listed
|
||
/// here: by case, where the capital sharp s is `SS`; by how accents are written; and some pass
|
||
/// over marks altogether.)
|
||
///
|
||
/// Where the listing and a look-up would differ, the listing is the folder as the scan saw it:
|
||
/// a file it lists is there though it could not be opened by its path (a folder that may be
|
||
/// listed but not searched, a path longer than the system takes), and a share that drops after
|
||
/// the walk does not turn every photo the listing answers for into a missing one.
|
||
#[derive(Default)]
|
||
pub(crate) struct Listing {
|
||
folders: HashMap<PathBuf, Listed>,
|
||
/// How often the disk was asked after all (for the tests, and the log).
|
||
pub(crate) asked: std::sync::atomic::AtomicUsize,
|
||
}
|
||
|
||
/// One folder's listing.
|
||
struct Listed {
|
||
/// name → what it is.
|
||
names: HashMap<String, crate::walk::EntryKind>,
|
||
/// The names, lower-cased (only kept for a folder whose names are all ASCII): what a plain
|
||
/// name asked about could be in another case.
|
||
lower: std::collections::HashSet<String>,
|
||
/// Every entry was read, and every name is ASCII: a plain name that is not listed, in any
|
||
/// case, is not there.
|
||
sure: bool,
|
||
}
|
||
|
||
/// Whether the disk is asked about `name` whatever the listing holds: a name Windows takes for
|
||
/// a device (`NUL`, `aux.jpg`: there in every folder, or not openable where it is listed).
|
||
fn asks_the_disk(name: &str) -> bool {
|
||
let stem = name.split('.').next().unwrap_or_default().trim_end_matches(' ').to_ascii_lowercase();
|
||
// (`COM` and `LPT` with any one digit, the superscript ones Windows also takes among them)
|
||
let numbered = ["com", "lpt"].iter().any(|port| {
|
||
let mut rest = stem.strip_prefix(port).unwrap_or_default().chars();
|
||
matches!((rest.next(), rest.next()), (Some(d), None) if d.is_ascii_digit() || ['\u{b9}', '\u{b2}', '\u{b3}'].contains(&d))
|
||
});
|
||
numbered || ["con", "prn", "aux", "nul", "conin$", "conout$", "clock$"].contains(&stem.as_str())
|
||
}
|
||
|
||
/// Whether `name` can only be found under itself in some case: ASCII, and nothing in it that a
|
||
/// file system resolves another way (a short name with `~`, a stream after `:`, dots or spaces
|
||
/// at its end, which Windows drops).
|
||
fn plain_name(name: &str) -> bool {
|
||
name.is_ascii() && !name.contains(['~', ':']) && !name.ends_with(['.', ' ']) && !name.is_empty()
|
||
}
|
||
|
||
impl Listing {
|
||
/// Take the listing of `folder` from the walk ([`crate::walk::files_in_noting`]).
|
||
pub(crate) fn note(&mut self, folder: &Path, entries: Vec<(std::ffi::OsString, crate::walk::EntryKind)>, complete: bool) {
|
||
let mut sure = complete;
|
||
let mut names = HashMap::with_capacity(entries.len());
|
||
let mut lower = std::collections::HashSet::new();
|
||
for (name, kind) in entries {
|
||
match name.into_string() {
|
||
Ok(name) => {
|
||
sure &= name.is_ascii();
|
||
lower.insert(name.to_ascii_lowercase());
|
||
names.insert(name, kind);
|
||
}
|
||
// (a name that is no text is not one a photo's path can be compared with)
|
||
Err(_) => sure = false,
|
||
}
|
||
}
|
||
if !sure {
|
||
lower.clear();
|
||
}
|
||
self.folders.insert(folder.to_path_buf(), Listed { names, lower, sure });
|
||
}
|
||
|
||
/// What the listing of `path`'s folder says of it: its kind when it is listed under exactly
|
||
/// this name, `Some(None)` when it is surely not there, `None` when the listing can't say.
|
||
fn kind(&self, path: &Path) -> Option<Option<crate::walk::EntryKind>> {
|
||
let name = path.file_name()?.to_str()?;
|
||
// (the path ends in that name: no `/.` or separator after it, which the disk would resolve)
|
||
if !path.to_str()?.ends_with(name) {
|
||
return None;
|
||
}
|
||
if asks_the_disk(name) {
|
||
return None;
|
||
}
|
||
let folder = self.folders.get(path.parent()?)?;
|
||
if let Some(kind) = folder.names.get(name) {
|
||
return Some(Some(*kind));
|
||
}
|
||
(folder.sure && plain_name(name) && !folder.lower.contains(&name.to_ascii_lowercase())).then_some(None)
|
||
}
|
||
|
||
/// `Path::exists`, where the listing can say.
|
||
pub(crate) fn exists(&self, path: &Path) -> Option<bool> {
|
||
match self.kind(path)? {
|
||
Some(crate::walk::EntryKind::Other) => None,
|
||
Some(_) => Some(true),
|
||
None => Some(false),
|
||
}
|
||
}
|
||
|
||
/// `Path::is_file`, where the listing can say.
|
||
pub(crate) fn is_file(&self, path: &Path) -> Option<bool> {
|
||
match self.kind(path)? {
|
||
Some(crate::walk::EntryKind::Other) => None,
|
||
Some(kind) => Some(kind == crate::walk::EntryKind::File),
|
||
None => Some(false),
|
||
}
|
||
}
|
||
|
||
/// The listing's answer, or the disk's where it has none.
|
||
fn or_ask(&self, answer: Option<bool>, ask: impl FnOnce() -> bool) -> bool {
|
||
answer.unwrap_or_else(|| {
|
||
self.asked.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||
ask()
|
||
})
|
||
}
|
||
}
|
||
|
||
fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog, listing: &Listing) -> Check {
|
||
let Some(file) = crate::cmd::missing::checked_path(p) else { return Check::Nothing };
|
||
let file = Path::new(file);
|
||
if !listing.or_ask(listing.exists(file), || file.exists()) {
|
||
Check::Missing
|
||
} else if p.copy_of.is_none() && sidecar_has_news(labels, p, naming, listing) {
|
||
Check::News
|
||
} else {
|
||
Check::Nothing
|
||
}
|
||
}
|
||
|
||
/// [`check_photo`] for each photo, in order, on several threads (each file is a wait on the
|
||
/// disk, not work for the CPU). Photos not reached before a cancel count as nothing found.
|
||
fn check_photos(photos: &[(Arc<Photo>, SidecarNaming)], labels: &Catalog, listing: &Listing, progress: &SyncProgress) -> Vec<Check> {
|
||
use std::sync::atomic::Ordering::Relaxed;
|
||
progress.to_check.store(photos.len(), Relaxed);
|
||
let cancel = &progress.files.cancel;
|
||
let check = |p: &Photo, naming: SidecarNaming| {
|
||
let c = check_photo(p, naming, labels, listing);
|
||
progress.checked.fetch_add(1, Relaxed);
|
||
c
|
||
};
|
||
if cfg!(target_arch = "wasm32") {
|
||
return vec![Check::Nothing; photos.len()];
|
||
}
|
||
#[cfg(not(target_arch = "wasm32"))]
|
||
{
|
||
let n = crate::import::workers(photos.len());
|
||
if n > 1 {
|
||
let next = std::sync::atomic::AtomicUsize::new(0);
|
||
let found = std::sync::Mutex::new(vec![Check::Nothing; photos.len()]);
|
||
std::thread::scope(|sc| {
|
||
for _ in 0..n {
|
||
sc.spawn(|| {
|
||
while !cancel.load(Relaxed) {
|
||
let i = next.fetch_add(1, Relaxed);
|
||
let Some((p, naming)) = photos.get(i) else { break };
|
||
let c = check(p, *naming);
|
||
if let Some(slot) = found.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get_mut(i) {
|
||
*slot = c;
|
||
}
|
||
}
|
||
});
|
||
}
|
||
});
|
||
return found.into_inner().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
}
|
||
}
|
||
photos.iter().map(|(p, naming)| if cancel.load(Relaxed) { Check::Nothing } else { check(p, *naming) }).collect()
|
||
}
|
||
|
||
/// [`scan_with`] on the calling thread.
|
||
pub fn scan(s: &mut Session, path: &str, disk: bool) -> Result<FolderChanges> {
|
||
let input = SyncInput::new(s, path, disk)?;
|
||
let changes = scan_with(input, &SyncProgress::default());
|
||
if changes.offline {
|
||
return Err(not_there(&changes.path));
|
||
}
|
||
Ok(changes)
|
||
}
|
||
|
||
fn not_there(path: &str) -> EngineError {
|
||
invalid(format!("{path} is not there (moved, renamed, or on a disk that isn't connected)"))
|
||
}
|
||
|
||
impl Session {
|
||
/// The last scan ([`Session::folder_changes`]) if it is of folder `path` and still current:
|
||
/// the folder holds the same photos, none of them changed. What the person was shown is what
|
||
/// gets done. Used once. (Changes elsewhere in the library don't matter.)
|
||
pub fn take_folder_changes(&mut self, path: &str) -> Option<FolderChanges> {
|
||
let c = self.folder_changes.take()?;
|
||
if lightcraft_catalog::query::folder_key(&c.path) != lightcraft_catalog::query::folder_key(path) {
|
||
return None;
|
||
}
|
||
let f = Filter { library_folder: Some(path.to_string()), ..Default::default() };
|
||
let mut now: Vec<PhotoId> = self.catalog.query(&f, &Sort::default());
|
||
let mut then: Vec<PhotoId> = c.seen.iter().map(|p| p.id).collect();
|
||
now.sort();
|
||
then.sort();
|
||
let unchanged = now == then && c.seen.iter().all(|p| self.catalog.photo(p.id).is_some_and(|q| same_for_sync(p, q)));
|
||
unchanged.then_some(c)
|
||
}
|
||
}
|
||
|
||
/// Whether `p`'s XMP sidecar file was saved after the library last had its say about the photo
|
||
/// (imported, or edited here) and says something the library doesn't.
|
||
fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming, listing: &Listing) -> bool {
|
||
let Source::File { path } = &p.source else { return false };
|
||
// ([`crate::sidecar::find_sidecar`], each name looked up in the listing before the disk)
|
||
// (a name the listing holds is read for its date next, and one that can't be read is passed
|
||
// over for the next name, as a name that is no file was)
|
||
let found = crate::sidecar::sidecar_candidates(path, naming).find_map(|c| {
|
||
let there = listing.or_ask(listing.is_file(&c), || c.is_file());
|
||
let meta = there.then(|| std::fs::metadata(&c).ok()).flatten().filter(std::fs::Metadata::is_file)?;
|
||
Some((c, meta))
|
||
});
|
||
let Some((file, meta)) = found else { return false };
|
||
if meta.len() > MAX_SIDECAR_BYTES {
|
||
return false;
|
||
}
|
||
let saved = meta.modified().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map(|d| d.as_secs());
|
||
let Some(saved) = saved.and_then(|s| i64::try_from(s).ok()) else { return false };
|
||
let seen = [Some(p.imported.as_str()), p.edited.as_deref()].into_iter().flatten().filter_map(lightcraft_catalog::stacks::iso_seconds).max();
|
||
if seen.is_some_and(|t| saved <= t) {
|
||
return false;
|
||
}
|
||
let Ok(text) = std::fs::read_to_string(&file) else { return false };
|
||
let Ok(sc) = crate::sidecar::parse_sidecar(&text, crate::crs::Target::for_photo(p)) else { return false };
|
||
let sc = sc.resolve_label(labels);
|
||
let mut q = p.clone();
|
||
crate::sidecar::merge_into(&mut q, &sc, "");
|
||
(q.rating, q.flag, q.label, &q.meta, &q.captured, &q.develop) != (p.rating, p.flag, p.label, &p.meta, &p.captured, &p.develop)
|
||
}
|
||
|
||
/// Act on `changes` as `choice` says, as one undo step ("Synchronize Folder"). A missing photo
|
||
/// whose file is back by now is left alone. Once the import is in, a part that fails is
|
||
/// reported in `failed` and the rest is still done (and still one step).
|
||
pub fn synchronize(s: &mut Session, changes: FolderChanges, choice: SyncChoice) -> Result<SyncReport> {
|
||
let (work, mut commit) = SyncJob::start(s, changes, choice)?;
|
||
let mut steps = Vec::new();
|
||
work.run(&SyncProgress::default(), |step| {
|
||
steps.push(step);
|
||
true
|
||
});
|
||
for step in steps {
|
||
commit.apply(s, step);
|
||
}
|
||
Ok(commit.finish(s))
|
||
}
|
||
|
||
/// Synchronizing in halves, so no file is touched on the UI thread: [`SyncJob::start`] takes what
|
||
/// it needs from the session (no file-system call); the [`SyncWork`] does every file read (the
|
||
/// import's probes, whether files are still where the scan found them, the sidecars) and hands
|
||
/// over [`SyncStep`]s — on a worker thread in the app; [`SyncCommit::apply`] puts each into the
|
||
/// catalog and [`SyncCommit::finish`] makes them one undo step.
|
||
pub struct SyncJob;
|
||
|
||
/// The file-system half of a synchronize (see [`SyncJob`]).
|
||
pub struct SyncWork {
|
||
/// The import of the new files, and the files.
|
||
import: Option<(crate::import::ImportJob, Vec<String>)>,
|
||
moved: Vec<FolderMoved>,
|
||
missing: Vec<FolderPhoto>,
|
||
/// (photo, its file, how its sidecar is named)
|
||
sidecars: Vec<(PhotoId, String, SidecarNaming, crate::crs::Target)>,
|
||
}
|
||
|
||
/// The catalog half of a synchronize (see [`SyncJob`]).
|
||
pub struct SyncCommit {
|
||
opts: ImportOptions,
|
||
now: String,
|
||
/// The library the run was readied for: steps never land in another one (File › Open
|
||
/// Library while the work runs), whose photo ids name other photos.
|
||
library: Arc<()>,
|
||
/// The run's own undo steps so far, and the history's length after the last of them: they
|
||
/// are merged only while nothing else came in between (an Undo or an edit made meanwhile
|
||
/// stays its own step, and the run's parts stay theirs).
|
||
own: usize,
|
||
after_last: Option<usize>,
|
||
interleaved: bool,
|
||
report: SyncReport,
|
||
}
|
||
|
||
/// A piece of work done, for [`SyncCommit::apply`].
|
||
pub enum SyncStep {
|
||
/// A batch of new files, readied for the catalog.
|
||
Import(crate::import::Prepared),
|
||
/// Moved photos whose file is still at its new place (photo, new path).
|
||
Relink(Vec<(PhotoId, String)>),
|
||
/// Missing photos whose file is still gone.
|
||
Remove(Vec<PhotoId>),
|
||
/// Sidecars read (photo, its file, what the sidecar says or why it couldn't be read).
|
||
Sidecars(Vec<(PhotoId, String, std::result::Result<crate::sidecar::SidecarData, String>)>),
|
||
}
|
||
|
||
impl SyncJob {
|
||
/// Ready a synchronize of `changes` as `choice` says. Reads no file.
|
||
pub fn start(s: &mut Session, changes: FolderChanges, choice: SyncChoice) -> Result<(SyncWork, SyncCommit)> {
|
||
if changes.offline {
|
||
return Err(not_there(&changes.path));
|
||
}
|
||
let FolderChanges { new, missing, metadata, moved, probes, .. } = changes;
|
||
let opts = ImportOptions { mode: ImportMode::Add, ..Default::default() };
|
||
let import = if choice.import_new && !new.is_empty() {
|
||
s.import_probes = probes;
|
||
let job = crate::import::ImportJob::new(s, opts.clone())?;
|
||
Some((job, new.into_iter().map(|c| c.path).collect()))
|
||
} else {
|
||
None
|
||
};
|
||
let now = import.as_ref().map_or_else(|| (s.clock)(), |(j, _)| j.now().to_string());
|
||
let sidecars = if choice.read_metadata {
|
||
let namings = s.sidecar_namings(&metadata.iter().map(|m| PhotoId(m.id)).collect::<Vec<_>>());
|
||
metadata
|
||
.into_iter()
|
||
.zip(namings)
|
||
.filter_map(|(m, naming)| {
|
||
let p = s.catalog.photo(PhotoId(m.id))?;
|
||
Some((p.id, m.path, naming, crate::crs::Target::for_photo(p)))
|
||
})
|
||
.collect()
|
||
} else {
|
||
Vec::new()
|
||
};
|
||
let work = SyncWork {
|
||
import,
|
||
moved: if choice.relink_moved { moved } else { Vec::new() },
|
||
missing: if choice.remove_missing { missing } else { Vec::new() },
|
||
sidecars,
|
||
};
|
||
let commit = SyncCommit {
|
||
opts,
|
||
now,
|
||
library: Arc::clone(&s.library_identity),
|
||
own: 0,
|
||
after_last: None,
|
||
interleaved: false,
|
||
report: SyncReport::default(),
|
||
};
|
||
Ok((work, commit))
|
||
}
|
||
}
|
||
|
||
impl SyncWork {
|
||
/// Do the work, handing each step to `send` (`false`: nobody takes them any more — stop).
|
||
/// Stops before the next piece once `progress.files.cancel` is set; what was handed over
|
||
/// stays handed over.
|
||
pub fn run(self, progress: &SyncProgress, mut send: impl FnMut(SyncStep) -> bool) {
|
||
use std::sync::atomic::Ordering::Relaxed;
|
||
let cancel = &progress.files.cancel;
|
||
let SyncWork { import, moved, missing, sidecars } = self;
|
||
let pieces = import.as_ref().map_or(0, |(_, f)| f.len()) + moved.len() + missing.len() + sidecars.len();
|
||
progress.to_check.store(pieces, Relaxed);
|
||
progress.checked.store(0, Relaxed);
|
||
let step = |n: usize| {
|
||
progress.checked.fetch_add(n, Relaxed);
|
||
};
|
||
if let Some((mut job, files)) = import {
|
||
for chunk in files.chunks(crate::import::batch_size()) {
|
||
if cancel.load(Relaxed) {
|
||
return;
|
||
}
|
||
let prepared = job.prepare_files(chunk.to_vec(), cancel);
|
||
step(chunk.len());
|
||
// (the files are added in place: a batch nobody takes leaves nothing behind on disk)
|
||
if !send(SyncStep::Import(prepared)) {
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
if cancel.load(Relaxed) {
|
||
return;
|
||
}
|
||
// still where the scan found it, and still gone from where the photo says
|
||
let checked = moved.len();
|
||
let relink: Vec<(PhotoId, String)> =
|
||
moved.into_iter().filter(|m| Path::new(&m.to).is_file() && !Path::new(&m.from).exists()).map(|m| (PhotoId(m.id), m.to)).collect();
|
||
step(checked);
|
||
if !relink.is_empty() && !send(SyncStep::Relink(relink)) {
|
||
return;
|
||
}
|
||
if cancel.load(Relaxed) {
|
||
return;
|
||
}
|
||
let gone: Vec<PhotoId> = missing.iter().filter(|m| !Path::new(&m.path).exists()).map(|m| PhotoId(m.id)).collect();
|
||
step(missing.len());
|
||
if !gone.is_empty() && !send(SyncStep::Remove(gone)) {
|
||
return;
|
||
}
|
||
let mut read = Vec::with_capacity(sidecars.len());
|
||
for (id, path, naming, target) in sidecars {
|
||
if cancel.load(Relaxed) {
|
||
break;
|
||
}
|
||
read.push((id, path.clone(), read_sidecar(&path, naming, target)));
|
||
step(1);
|
||
}
|
||
if !read.is_empty() {
|
||
send(SyncStep::Sidecars(read));
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Read and parse photo file `path`'s XMP sidecar file.
|
||
fn read_sidecar(path: &str, naming: SidecarNaming, target: crate::crs::Target) -> std::result::Result<crate::sidecar::SidecarData, String> {
|
||
let file = crate::sidecar::find_sidecar(path, naming).ok_or_else(|| "no XMP sidecar".to_string())?;
|
||
let len = std::fs::metadata(&file).map_err(|e| format!("{}: {e}", file.display()))?.len();
|
||
if len > MAX_SIDECAR_BYTES {
|
||
return Err(format!("{}: too large for an XMP sidecar", file.display()));
|
||
}
|
||
let text = std::fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?;
|
||
crate::sidecar::parse_sidecar(&text, target).map_err(|e| format!("{}: {e}", file.display()))
|
||
}
|
||
|
||
impl SyncCommit {
|
||
/// Put a step into the catalog (no file is touched). A part that fails is reported in the
|
||
/// end, and the rest is still done.
|
||
pub fn apply(&mut self, s: &mut Session, step: SyncStep) {
|
||
if !Arc::ptr_eq(&self.library, &s.library_identity) {
|
||
self.report.failed.push((String::new(), "another library was opened: the rest of the synchronize was dropped".into()));
|
||
return;
|
||
}
|
||
let commits = s.commits();
|
||
if self.after_last.is_some_and(|n| n != s.undo.len()) {
|
||
self.interleaved = true;
|
||
}
|
||
self.apply_step(s, step);
|
||
if s.commits() != commits {
|
||
self.own += usize::try_from(s.commits().wrapping_sub(commits)).unwrap_or(0);
|
||
self.after_last = Some(s.undo.len());
|
||
}
|
||
}
|
||
|
||
fn apply_step(&mut self, s: &mut Session, step: SyncStep) {
|
||
let report = &mut self.report;
|
||
match step {
|
||
SyncStep::Import(prepared) => match crate::import::commit_prepared(s, &self.opts, &self.now, prepared) {
|
||
Ok(r) => {
|
||
report.imported += r.imported.len();
|
||
report.failed.extend(r.failed);
|
||
}
|
||
Err(e) => report.failed.push((String::new(), e.to_string())),
|
||
},
|
||
SyncStep::Relink(moved) => {
|
||
let ops: Vec<Op> =
|
||
moved.iter().filter(|(id, _)| s.catalog.photo(*id).is_some()).map(|(id, to)| crate::cmd::missing::relink_op(*id, to)).collect();
|
||
let n = ops.len();
|
||
if n > 0 {
|
||
match s.commit("Relink Moved Photos", Op::Batch { ops }) {
|
||
Ok(()) => report.relinked += n,
|
||
Err(e) => report.failed.push((String::new(), e.to_string())),
|
||
}
|
||
}
|
||
}
|
||
SyncStep::Remove(gone) => {
|
||
let ops: Vec<Op> = gone
|
||
.iter()
|
||
.filter(|id| s.catalog.photo(**id).is_some_and(|p| p.in_library()))
|
||
.map(|id| Op::SetDeleted { id: *id, deleted: true })
|
||
.collect();
|
||
let n = ops.len();
|
||
if n > 0 {
|
||
match s.commit("Remove Missing Photos", Op::Batch { ops }) {
|
||
Ok(()) => report.removed += n,
|
||
Err(e) => report.failed.push((String::new(), e.to_string())),
|
||
}
|
||
}
|
||
}
|
||
SyncStep::Sidecars(read) => {
|
||
let mut ops = Vec::new();
|
||
for (id, path, sc) in read {
|
||
match sc.map_err(EngineError::Other).and_then(|sc| s.sidecar_op(id, sc)) {
|
||
Ok(op) => ops.push(op),
|
||
Err(e) => report.failed.push((path, e.to_string())),
|
||
}
|
||
}
|
||
let n = ops.len();
|
||
if n > 0 {
|
||
match s.commit("Read Metadata from File", Op::Batch { ops }) {
|
||
Ok(()) => report.read += n,
|
||
Err(e) => report.failed.push((String::new(), e.to_string())),
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// What was done, as one undo step ("Synchronize Folder").
|
||
pub fn finish(mut self, s: &mut Session) -> SyncReport {
|
||
// a report line once, however many steps came after a library switch
|
||
self.report.failed.dedup();
|
||
let ours = Arc::ptr_eq(&self.library, &s.library_identity) && !self.interleaved && self.after_last == Some(s.undo.len());
|
||
let steps = if ours { self.own.min(s.undo.len()) } else { 0 };
|
||
s.merge_undo(steps, "Synchronize Folder");
|
||
if let Some(last) = s.undo.last_mut().filter(|_| steps == 1) {
|
||
last.label = "Synchronize Folder".into();
|
||
}
|
||
self.report
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
#[path = "sync/tests_listing.rs"]
|
||
mod tests_listing;
|