From b14cf58630008e80886924bfb56f0c1f29b2de2d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Oct 2026 02:26:44 -0400 Subject: [PATCH] Local: forget untouched browse records of folders not browsed for 30 days Local browse records can be most of a large catalog (85k photos, 90 % Local: 313.5 MB snapshot). Untouched Local records of folders whose last browse is older than library.preferences forgetLocalDays (default 30, 0 = never) are now removed from the catalog when the library opens (and by library.forgetLocal {dryRun?, days?}), as journaled, non-undoable ops, then compacted in the background. Files and XMP sidecars are never touched; browsing the folder again catalogues them afresh. Untouched = a Local file record, not deleted, not a virtual copy, whose state still hashes to the fingerprint taken when the browse catalogued it (any change keeps it); records from before fingerprints need every user-editable field empty/default. Kept too: records in albums (or as cover), stacks, sources of virtual copies, visible/selected/edited ones, ones referenced by undo/redo, ones with a smart preview. Per-folder last-browsed times live in the catalog (Op::SetBrowsed, normalised with folder_key; re-stamped at most hourly). Old catalogs load unchanged; folders without a time are stamped now, so nothing is forgotten on the first open after the upgrade. persist_bench (85k photos, 76.5k Local, every 7th folder recent, 1 in 50 rated): 63.8k forgotten, plan 162 ms, snapshot 313.5 MB -> 94.9 MB. Settings -> Performance -> Local folders. Co-Authored-By: Claude Opus 5.5 --- crates/catalog/examples/persist_bench.rs | 46 ++++ crates/catalog/src/lib.rs | 21 ++ crates/catalog/src/local.rs | 304 +++++++++++++++++++++++ crates/catalog/src/model.rs | 5 + crates/catalog/src/tests_local.rs | 267 ++++++++++++++++++++ crates/engine/src/cmd/browse.rs | 115 +++++++++ crates/engine/src/cmd/library.rs | 2 + crates/engine/src/cmd/prefs.rs | 8 +- crates/engine/src/import.rs | 4 + crates/engine/src/lib.rs | 6 + crates/engine/src/library.rs | 44 +++- crates/engine/src/tests_forget_local.rs | 218 ++++++++++++++++ crates/ui-egui/src/panels/settings.rs | 13 + docs/parity.md | 2 +- docs/whats-new.md | 3 + 15 files changed, 1054 insertions(+), 4 deletions(-) create mode 100644 crates/catalog/src/local.rs create mode 100644 crates/catalog/src/tests_local.rs create mode 100644 crates/engine/src/tests_forget_local.rs diff --git a/crates/catalog/examples/persist_bench.rs b/crates/catalog/examples/persist_bench.rs index 60ec46aa7..693e862d0 100644 --- a/crates/catalog/examples/persist_bench.rs +++ b/crates/catalog/examples/persist_bench.rs @@ -44,6 +44,11 @@ fn photo(c: &mut Catalog, i: u64) -> Photo { p.meta.iso = Some(400); p.as_shot_wb = Some((5200.0, 3.0)); p.local = !i.is_multiple_of(10); + if p.local { + // as a browse catalogues it: import defaults, a baseline fingerprint + p.develop = Arc::new(p.import_defaults()); + p.set_local_baseline(); + } if !p.local { let mut d = DevelopSettings::default(); d.light.exposure = (i % 7) as f64 * 0.1; @@ -153,6 +158,47 @@ fn main() -> std::result::Result<(), Box> { } println!(" background compaction blocking: {}", stats(blocking)); + // forget the untouched Local records (folders last browsed long ago; every 7th folder + // recently; every 50th Local record rated), then compact + let folders: std::collections::BTreeSet = cat.photos().filter(|p| p.local).filter_map(|p| folder_of(p)).collect(); + let mut ops: Vec = folders + .iter() + .enumerate() + .map(|(k, f)| Op::SetBrowsed { + folder: f.clone(), + at: Some(if k % 7 == 0 { "2026-09-30T00:00:00" } else { "2026-06-01T00:00:00" }.into()), + }) + .collect(); + ops.extend(cat.photos().filter(|p| p.local && p.id.0.is_multiple_of(50)).map(|p| Op::SetRating { id: p.id, rating: 2 }).collect::>()); + for op in &ops { + cat.apply(op.clone())?; + } + j.append(&ops)?; + j.snapshot(&cat)?; + let before = j.stats().last_snapshot.bytes; + let t = Instant::now(); + let plan = cat.forget_local_plan("2026-10-05T00:00:00", DEFAULT_FORGET_DAYS, &|_| false); + let plan_ms = ms(t); + let t = Instant::now(); + let ops = plan.ops(); + for op in &ops { + cat.apply(op.clone())?; + } + j.append(&ops)?; + let apply_ms = ms(t); + j.snapshot(&cat)?; + let after = j.stats().last_snapshot.bytes; + println!( + " forget Local: {} of {} local records forgotten (kept: {} recent, {} touched, {} in use); plan {plan_ms:.1} ms, apply+append {apply_ms:.1} ms; snapshot {:.1} MB -> {:.1} MB", + plan.evict.len(), + plan.local, + plan.kept_recent, + plan.kept_touched, + plan.kept_in_use, + before as f64 / 1e6, + after as f64 / 1e6 + ); + drop(j); if std::env::var_os("PERSIST_BENCH_NO_LOAD").is_some() { // peak memory of building + compacting only diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rs index 041c9c7d0..4b9bdcdc3 100644 --- a/crates/catalog/src/lib.rs +++ b/crates/catalog/src/lib.rs @@ -11,6 +11,7 @@ pub mod dates; pub mod journal; pub mod keywords; +pub mod local; pub mod model; pub mod query; pub mod rules; @@ -24,6 +25,7 @@ pub use dates::{DateRun, GroupBy}; pub use journal::{Journal, LoadReport, PersistStats, SnapshotPolicy, SnapshotTiming}; pub use keywords::KeywordNode; use lightcraft_develop::DevelopSettings; +pub use local::{DEFAULT_FORGET_DAYS, ForgetPlan, folder_of}; pub use model::*; pub use query::{DateGroup, Filter, RatingOp, Sort, SortKey}; pub use rules::{Match, Rule, RuleSet}; @@ -190,6 +192,12 @@ pub enum Op { label: ColorLabel, name: Option, }, + /// When a Local folder was last browsed (ISO 8601; `None` = forget the time). Not an undo + /// step: it drives forgetting untouched Local records (see [`local`]). + SetBrowsed { + folder: String, + at: Option, + }, /// Several ops as one step (undo applies the inverses in reverse). Batch { ops: Vec, @@ -209,6 +217,9 @@ pub struct Catalog { /// Custom colour label names. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] label_names: BTreeMap, + /// When each Local folder was last browsed (folder path → ISO 8601), see [`local`]. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + browsed: BTreeMap, /// Increments on every applied op. #[serde(skip)] pub revision: u64, @@ -547,6 +558,14 @@ impl Catalog { }; Op::SetLabelName { label, name: old } } + Op::SetBrowsed { folder, at } => { + let folder = crate::query::folder_key(&folder); + let old = match at { + Some(t) => self.browsed.insert(folder.clone(), t), + None => self.browsed.remove(&folder), + }; + Op::SetBrowsed { folder, at: old } + } Op::Batch { ops } => { let mut inverses = Vec::with_capacity(ops.len()); for op in ops { @@ -625,3 +644,5 @@ mod tests; mod tests_background; #[cfg(test)] mod tests_journal; +#[cfg(test)] +mod tests_local; diff --git a/crates/catalog/src/local.rs b/crates/catalog/src/local.rs new file mode 100644 index 000000000..79a4a42d7 --- /dev/null +++ b/crates/catalog/src/local.rs @@ -0,0 +1,304 @@ +//! Forgetting untouched Local browse records. +//! +//! Browsing a folder (Local) catalogues its photos as `local` records so they can be shown, +//! rated and edited in place. Most are only ever looked at, and on a large disk they can be most +//! of the catalog. A record is **forgotten** (removed from the catalog, never from the disk — +//! files and XMP sidecars are not touched) when all of these hold: +//! +//! - it is a Local record ([`Photo::local`]) of a file, not deleted, not a virtual copy; +//! - its folder was last browsed more than `days` ago ([`Catalog::last_browsed`]). A folder with +//! no recorded time (a catalog from before this existed) counts as browsed *now*: it is stamped +//! with the current time and nothing in it is forgotten this time; +//! - it is **untouched** ([`Photo::untouched_local`]): nothing about it changed since it was +//! browsed — see below; +//! - nothing refers to it: no album holds it or shows it as cover, no stack holds it, no virtual +//! copy was made of it; and the caller (the engine) says it isn't in use (visible, selected, +//! in the undo history, has a smart preview…). +//! +//! **Untouched.** A record catalogued by a browse carries a fingerprint of its whole state at +//! that moment ([`Photo::local_baseline`]). It is untouched while its state still hashes to that +//! fingerprint, i.e. *any* change since — develop settings or history, versions, rating, flag, +//! label, any metadata (title, caption, keywords, location, copyright…), capture time, crop, +//! rename or relink, culling scores, file content — keeps it. Records without a fingerprint +//! (browsed before fingerprints existed) are judged by their data alone and kept unless every +//! user-editable field is still empty / default: no rating, flag or label; develop settings equal +//! to the import defaults with no history, versions or edit time; no title, caption, alt text, +//! description, keywords, creator, copyright, rights, location or GPS; no culling scores. +//! When in doubt a record is kept. +//! +//! Browsing the folder again catalogues its files afresh (new records, new fingerprints). + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use serde::Serialize; + +use crate::{Catalog, Op, Photo, PhotoId, Source}; + +/// Default for how long a Local folder may go unbrowsed before its untouched records are +/// forgotten (days). +pub const DEFAULT_FORGET_DAYS: u32 = 30; + +/// Records removed per logged batch (keeps log lines a reasonable size). +const BATCH: usize = 2000; + +/// 64-bit FNV-1a over everything written to it (stable across builds and platforms, unlike +/// `std`'s hasher), so a photo is hashed as it serialises, without a buffer. +struct Fnv(u64); + +impl std::io::Write for Fnv { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + for b in buf { + self.0 ^= u64::from(*b); + self.0 = self.0.wrapping_mul(0x0100_0000_01b3); + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// The folder a file-backed photo is in (`None` for demo photos). +pub fn folder_of(p: &Photo) -> Option { + match &p.source { + // `/` and `\` are both separators whatever the platform (a catalog can move between them) + Source::File { path } => { + let key = crate::query::folder_key(path); + key.rsplit_once('/').map(|(dir, _)| if dir.is_empty() { "/".to_string() } else { dir.to_string() }) + } + Source::Demo { .. } => None, + } +} + +impl Photo { + /// A hash of everything about the photo except [`Photo::local_baseline`] itself. + pub fn fingerprint(&self) -> u64 { + let mut h = Fnv(0xcbf2_9ce4_8422_2325); + let r = if self.local_baseline.is_some() { + let mut p = self.clone(); + p.local_baseline = None; + serde_json::to_writer(&mut h, &p) + } else { + serde_json::to_writer(&mut h, self) + }; + // (serialising a photo can't fail; if it ever did, no baseline would match: kept) + if r.is_err() { 0 } else { h.0 } + } + + /// Record the current state as the browse baseline (done when a browse catalogues it). + pub fn set_local_baseline(&mut self) { + self.local_baseline = None; + self.local_baseline = Some(self.fingerprint()); + } + + /// A Local record nothing was changed on since it was browsed (see the module docs). Says + /// nothing about albums, stacks or virtual copies referring to it. + pub fn untouched_local(&self) -> bool { + if !self.local || self.deleted || self.copy_of.is_some() || self.copy_name.is_some() || folder_of(self).is_none() { + return false; + } + match self.local_baseline { + Some(b) => self.fingerprint() == b, + None => self.looks_unedited(), + } + } + + /// Every user-editable field is still empty / default (records without a baseline). + fn looks_unedited(&self) -> bool { + let m = &self.meta; + self.rating == 0 + && self.flag == crate::Flag::None + && self.label.is_none() + && self.edited.is_none() + && self.history.is_empty() + && self.versions.is_empty() + && *self.develop == self.import_defaults() + && self.analysis.is_none() + && m.title.is_empty() + && m.caption.is_empty() + && m.alt_text.is_empty() + && m.extended_description.is_empty() + && m.keywords.is_empty() + && m.creator.is_empty() + && m.copyright.is_empty() + && m.copyright_status.is_unknown() + && m.usage_terms.is_empty() + && m.copyright_url.is_empty() + && m.location.is_empty() + && m.city.is_empty() + && m.state.is_empty() + && m.country.is_empty() + && m.gps.is_none() + } +} + +/// What forgetting Local records would do / did. +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ForgetPlan { + /// Local records looked at. + pub local: usize, + /// Records to forget. + pub evict: Vec, + /// Kept: their folder was browsed within the period (or has no time yet). + pub kept_recent: usize, + /// Kept: changed by the user since they were browsed. + pub kept_touched: usize, + /// Kept: in an album, a stack, the source of a virtual copy, or in use. + pub kept_in_use: usize, + /// Folders with Local records but no last-browsed time: stamped now. + pub stamp: Vec, + /// Folders whose last Local record goes: their time is dropped. + pub unstamp: Vec, + /// The time stamps are given. + #[serde(skip)] + pub now: String, +} + +impl ForgetPlan { + /// The ops that carry the plan out (empty when there is nothing to do). + pub fn ops(&self) -> Vec { + let mut ops: Vec = self.stamp.iter().map(|f| Op::SetBrowsed { folder: f.clone(), at: Some(self.now.clone()) }).collect(); + ops.extend(self.evict.chunks(BATCH).map(|c| Op::Batch { ops: c.iter().map(|id| Op::RemovePhoto { id: *id }).collect() })); + ops.extend(self.unstamp.iter().map(|f| Op::SetBrowsed { folder: f.clone(), at: None })); + ops + } +} + +impl Catalog { + /// When `folder` was last browsed (ISO 8601), if known. + pub fn last_browsed(&self, folder: &str) -> Option<&str> { + self.browsed.get(&crate::query::folder_key(folder)).map(String::as_str) + } + + /// Folders with a last-browsed time. + pub fn browsed_folders(&self) -> &BTreeMap { + &self.browsed + } + + /// Plan forgetting untouched Local records whose folder wasn't browsed for `days` days (as of + /// `now`, ISO 8601). `in_use` keeps records the caller still needs. `days == 0` (never) or an + /// unreadable `now` forgets nothing. See the module docs. + pub fn forget_local_plan(&self, now: &str, days: u32, in_use: &dyn Fn(&Photo) -> bool) -> ForgetPlan { + let mut plan = ForgetPlan { now: now.to_string(), ..Default::default() }; + let now_s = crate::stacks::iso_seconds(now); + let max_age = i64::from(days) * 86_400; + let mut referenced: HashSet = HashSet::new(); + for a in self.albums.values() { + referenced.extend(a.photos.iter().copied()); + referenced.extend(a.cover); + } + for s in self.stacks.values() { + referenced.extend(s.photos.iter().copied()); + } + referenced.extend(self.photos.values().filter_map(|p| p.copy_of)); + // the fingerprints are the costly part: work them out for the candidates in parallel + let candidates: Vec<&Photo> = self + .photos + .values() + .filter(|p| p.local && !referenced.contains(&p.id)) + .filter(|p| folder_of(p).and_then(|f| self.browsed.get(&f)).is_some_and(|at| is_old(at, now_s, days, max_age))) + .map(|p| &**p) + .collect(); + let untouched: HashSet = untouched_of(&candidates); + let mut stamp = BTreeSet::new(); + let mut remaining: BTreeSet = BTreeSet::new(); + for p in self.photos.values().filter(|p| p.local) { + plan.local += 1; + let Some(folder) = folder_of(p) else { + plan.kept_in_use += 1; + continue; + }; + let recent = match (self.browsed.get(&folder), now_s) { + (None, _) => { + stamp.insert(folder.clone()); + true + } + (Some(at), _) => !is_old(at, now_s, days, max_age), + }; + let keep = if recent { + plan.kept_recent += 1; + true + } else if referenced.contains(&p.id) || in_use(p) { + plan.kept_in_use += 1; + true + } else if !untouched.contains(&p.id) { + plan.kept_touched += 1; + true + } else { + plan.evict.push(p.id); + false + }; + if keep { + remaining.insert(folder); + } + } + plan.stamp = stamp.into_iter().collect(); + plan.unstamp = self.browsed.keys().filter(|f| !remaining.contains(*f)).cloned().collect(); + plan + } +} + +/// A folder browsed at `at` is past the period (never when `days` is 0 or a time is unreadable). +fn is_old(at: &str, now_s: Option, days: u32, max_age: i64) -> bool { + days > 0 && matches!((now_s, crate::stacks::iso_seconds(at)), (Some(n), Some(a)) if n - a >= max_age) +} + +/// The untouched ones among `photos` ([`Photo::untouched_local`]), checked on several threads +/// where there are many. +fn untouched_of(photos: &[&Photo]) -> HashSet { + let check = |chunk: &[&Photo]| chunk.iter().filter(|p| p.untouched_local()).map(|p| p.id).collect::>(); + #[cfg(not(target_arch = "wasm32"))] + { + let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(8); + if threads > 1 && photos.len() >= 256 { + let size = photos.len().div_ceil(threads); + return std::thread::scope(|sc| { + let jobs: Vec<_> = photos.chunks(size).map(|c| sc.spawn(move || check(c))).collect(); + // a job that panicked (it can't) leaves its photos out: they are kept + jobs.into_iter().filter_map(|j| j.join().ok()).flatten().collect() + }); + } + } + check(photos).into_iter().collect() +} + +impl Op { + /// Every photo the op refers to. + pub fn photo_ids(&self, out: &mut impl FnMut(PhotoId)) { + match self { + Op::AddPhoto { photo } => out(photo.id), + Op::RemovePhoto { id } + | Op::SetRating { id, .. } + | Op::SetFlag { id, .. } + | Op::SetLabel { id, .. } + | Op::SetDevelop { id, .. } + | Op::SetMeta { id, .. } + | Op::SetDeleted { id, .. } + | Op::SetLocal { id, .. } + | Op::SetVersions { id, .. } + | Op::SetHistory { id, .. } + | Op::PushHistory { id, .. } + | Op::SetCaptured { id, .. } + | Op::SetAnalysis { id, .. } + | Op::SetFile { id, .. } + | Op::Relink { id, .. } + | Op::SetContent { id, .. } => out(*id), + Op::AddAlbum { album } => { + album.photos.iter().for_each(|p| out(*p)); + album.cover.into_iter().for_each(&mut *out); + } + Op::SetAlbumPhotos { photos, .. } | Op::SetStack { photos, .. } => photos.iter().for_each(|p| out(*p)), + Op::SetAlbumCover { cover, .. } => cover.iter().for_each(|p| out(*p)), + Op::AddStack { stack } => stack.photos.iter().for_each(|p| out(*p)), + Op::Batch { ops } => ops.iter().for_each(|o| o.photo_ids(out)), + Op::RemoveAlbum { .. } + | Op::RenameAlbum { .. } + | Op::MoveAlbum { .. } + | Op::SetAlbumRules { .. } + | Op::RemoveStack { .. } + | Op::SetLabelName { .. } + | Op::SetBrowsed { .. } => {} + } + } +} diff --git a/crates/catalog/src/model.rs b/crates/catalog/src/model.rs index aceadcdc2..f8e518464 100644 --- a/crates/catalog/src/model.rs +++ b/crates/catalog/src/model.rs @@ -272,6 +272,10 @@ pub struct Photo { /// picture style baked in — so it is treated as a rendered (non-raw) source. #[serde(default, skip_serializing_if = "Option::is_none")] pub preview_only: Option, + /// A Local record: the fingerprint of its state when the browse catalogued it, to tell + /// whether the user changed anything since (see [`crate::local`]). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_baseline: Option, } /// What assisted culling measured on a photo. @@ -319,6 +323,7 @@ impl Photo { import_look: None, analysis: None, preview_only: None, + local_baseline: None, } } /// A raw file developed from its sensor data: not a rendered image, and not a raw shown from diff --git a/crates/catalog/src/tests_local.rs b/crates/catalog/src/tests_local.rs new file mode 100644 index 000000000..4b76dace6 --- /dev/null +++ b/crates/catalog/src/tests_local.rs @@ -0,0 +1,267 @@ +//! Forgetting untouched Local browse records (see [`crate::local`]). + +use std::sync::Arc; + +use lightcraft_develop::DevelopSettings; + +use crate::*; + +const OLD: &str = "2026-08-01T10:00:00"; +const NOW: &str = "2026-10-05T10:00:00"; + +/// A Local record of `/pics//IMG_.JPG` as a browse catalogues it (with its baseline). +fn local(c: &mut Catalog, folder: &str, i: u32) -> PhotoId { + let id = c.alloc_photo_id(); + let mut p = Photo::new(id, Source::File { path: format!("/pics/{folder}/IMG_{i}.JPG") }, &format!("IMG_{i}.JPG"), "JPEG", 60, 40, OLD); + p.meta.camera = "Cam".into(); + p.meta.creator = "From EXIF".into(); // file metadata is part of the baseline, not a user change + p.focal_and_wb_for_test(); + p.local = true; + p.set_local_baseline(); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + id +} + +impl Photo { + fn focal_and_wb_for_test(&mut self) { + self.meta.focal_mm = Some(35.3); + self.as_shot_wb = Some((5123.4, 3.1)); + } +} + +fn browsed(c: &mut Catalog, folder: &str, at: &str) { + c.apply(Op::SetBrowsed { folder: format!("/pics/{folder}"), at: Some(at.into()) }).unwrap(); +} + +fn plan(c: &Catalog, days: u32) -> ForgetPlan { + c.forget_local_plan(NOW, days, &|_| false) +} + +fn run(c: &mut Catalog, days: u32) -> ForgetPlan { + let p = plan(c, days); + for op in p.ops() { + c.apply(op).unwrap(); + } + p +} + +#[test] +fn untouched_records_of_old_folders_are_forgotten() { + let mut c = Catalog::new(); + let old: Vec<_> = (0..5).map(|i| local(&mut c, "old", i)).collect(); + let recent: Vec<_> = (0..3).map(|i| local(&mut c, "recent", i)).collect(); + browsed(&mut c, "old", OLD); + browsed(&mut c, "recent", "2026-10-01T09:00:00"); + // a library photo in the old folder is never a candidate + let lib = c.alloc_photo_id(); + let mut p = Photo::new(lib, Source::File { path: "/pics/old/LIB.JPG".into() }, "LIB.JPG", "JPEG", 6, 4, OLD); + p.set_local_baseline(); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + + let dry = plan(&c, 30); + assert_eq!((dry.local, dry.evict.len(), dry.kept_recent), (8, 5, 3), "{dry:?}"); + assert_eq!(dry.evict, old); + assert_eq!(c.len(), 9, "a plan changes nothing"); + let done = run(&mut c, 30); + assert_eq!(done.evict.len(), 5); + assert!(old.iter().all(|id| c.photo(*id).is_none())); + assert!(recent.iter().all(|id| c.photo(*id).is_some())); + assert!(c.photo(lib).is_some()); + // the folder has no Local records left: its time goes + assert_eq!(done.unstamp, vec!["/pics/old".to_string()]); + assert_eq!(c.last_browsed("/pics/old"), None); + assert_eq!(c.last_browsed("/pics/recent/"), Some("2026-10-01T09:00:00")); + // 0 days = never + browsed(&mut c, "recent", OLD); + assert!(plan(&c, 0).evict.is_empty()); + assert_eq!(plan(&c, 30).evict.len(), 3); +} + +#[test] +fn any_user_change_keeps_a_record() { + type Change = Box; + let set = |f: fn(&mut Photo)| -> Change { + Box::new(move |c: &mut Catalog, id: PhotoId| { + let mut p = (**c.photo(id).unwrap()).clone(); + f(&mut p); + c.apply(Op::RemovePhoto { id }).unwrap(); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + }) + }; + let op = |f: fn(PhotoId) -> Op| -> Change { Box::new(move |c: &mut Catalog, id: PhotoId| drop(c.apply(f(id)).unwrap())) }; + let edited = || { + let mut d = DevelopSettings::default(); + d.light.exposure = 0.3; + Arc::new(d) + }; + let changes: Vec<(&str, Change)> = vec![ + ("rating", op(|id| Op::SetRating { id, rating: 3 })), + ("flag", op(|id| Op::SetFlag { id, flag: Flag::Reject })), + ("label", op(|id| Op::SetLabel { id, label: Some(ColorLabel::Red) })), + ("develop", Box::new(move |c, id| drop(c.apply(Op::SetDevelop { id, settings: edited(), label: "Exposure".into(), edited: None }).unwrap()))), + ("crop", set(|p| Arc::make_mut(&mut p.develop).crop.flip_h = true)), + ( + "history only", + Box::new(|c, id| { + let step = HistoryStep { label: "Reset".into(), settings: Arc::new(DevelopSettings::default()) }; + drop(c.apply(Op::PushHistory { id, step }).unwrap()) + }), + ), + ("versions", set(|p| p.versions.push(Version { name: "v".into(), created: NOW.into(), settings: p.develop.clone(), auto: false }))), + ("title", set(|p| p.meta.title = "T".into())), + ("caption", set(|p| p.meta.caption = "C".into())), + ("keywords", set(|p| p.meta.keywords.push("k".into()))), + ("location", set(|p| p.meta.city = "Oslo".into())), + ("gps", set(|p| p.meta.gps = Some((1.0, 2.0)))), + ("copyright", set(|p| p.meta.copyright = "© me".into())), + ("creator edit", set(|p| p.meta.creator = "Me".into())), + ("capture time", op(|id| Op::SetCaptured { id, captured: Some("2020-01-01T00:00:00".into()) })), + ("rename", set(|p| p.file_name = "renamed.JPG".into())), + ("culling", op(|id| Op::SetAnalysis { id, analysis: Some(Analysis::default()) })), + ("deleted", op(|id| Op::SetDeleted { id, deleted: true })), + ("pick", op(|id| Op::SetFlag { id, flag: Flag::Pick })), + ( + "album", + Box::new(|c, id| { + let a = c.alloc_album_id(); + let mut al = Album::new(a, "A"); + al.photos.push(id); + drop(c.apply(Op::AddAlbum { album: al }).unwrap()) + }), + ), + ( + "stack", + Box::new(|c, id| { + let other = local(c, "elsewhere", 99); + let s = c.alloc_stack_id(); + drop(c.apply(Op::AddStack { stack: Stack { id: s, photos: vec![id, other], collapsed: false } }).unwrap()) + }), + ), + ( + "virtual copy", + Box::new(|c, id| { + let mut v = (**c.photo(id).unwrap()).clone(); + v.id = c.alloc_photo_id(); + v.copy_of = Some(id); + v.copy_name = Some("Copy 1".into()); + v.local = false; + drop(c.apply(Op::AddPhoto { photo: Box::new(v) }).unwrap()) + }), + ), + ]; + for (what, change) in changes { + let mut c = Catalog::new(); + let id = local(&mut c, "old", 1); + let control = local(&mut c, "old", 2); + browsed(&mut c, "old", OLD); + change(&mut c, id); + let p = plan(&c, 30); + assert_eq!(p.evict, vec![control], "{what}: {p:?}"); + assert_eq!(p.kept_touched + p.kept_in_use, 1, "{what}: {p:?}"); + // the stamp stays: a record in the folder remains + assert!(p.unstamp.is_empty(), "{what}"); + } + // a change undone (back to the browse state) is untouched again + let mut c = Catalog::new(); + let id = local(&mut c, "old", 1); + browsed(&mut c, "old", OLD); + let inv = c.apply(Op::SetRating { id, rating: 2 }).unwrap(); + assert!(plan(&c, 30).evict.is_empty()); + c.apply(inv).unwrap(); + assert_eq!(plan(&c, 30).evict, vec![id]); +} + +/// Catalogs from before last-browsed times (and baselines): nothing is forgotten on the first +/// run — the folders are stamped now — and baseline-less records are judged by their data. +#[test] +fn upgrade_without_times_forgets_nothing_then_counts_from_then() { + let mut c = Catalog::new(); + let a = local(&mut c, "a", 1); + // legacy records: no baseline + let legacy = |c: &mut Catalog, i: u32, f: fn(&mut Photo)| { + let id = c.alloc_photo_id(); + let mut p = Photo::new(id, Source::File { path: format!("/pics/a/L{i}.JPG") }, "L.JPG", "JPEG", 6, 4, OLD); + p.meta.camera = "Cam".into(); + p.local = true; + f(&mut p); + c.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + id + }; + let clean = legacy(&mut c, 1, |_| {}); + let rated = legacy(&mut c, 2, |p| p.rating = 1); + let with_creator = legacy(&mut c, 3, |p| p.meta.creator = "maybe the user".into()); + let p = run(&mut c, 30); + assert!(p.evict.is_empty(), "{p:?}"); + assert_eq!(p.stamp, vec!["/pics/a".to_string()]); + assert_eq!(c.last_browsed("/pics/a"), Some(NOW)); + // 30+ days later + let later = c.forget_local_plan("2026-11-06T10:00:00", 30, &|_| false); + let mut evict = later.evict.clone(); + evict.sort(); + assert_eq!(evict, vec![a, clean]); + assert_eq!(later.kept_touched, 2, "{rated:?} {with_creator:?}"); +} + +#[test] +fn in_use_records_are_kept() { + let mut c = Catalog::new(); + let a = local(&mut c, "old", 1); + let b = local(&mut c, "old", 2); + browsed(&mut c, "old", OLD); + let p = c.forget_local_plan(NOW, 30, &|p| p.id == a); + assert_eq!((p.evict.clone(), p.kept_in_use), (vec![b], 1)); +} + +/// The ops are journaled: a crash right after forgetting reloads the same catalog, and a +/// fingerprint survives the snapshot round trip (a reloaded record is still untouched). +#[test] +fn forgetting_is_journaled_and_replays() { + let m = MemStore::new(); + let (mut j, mut c, _) = Journal::open(Box::new(m.clone())).unwrap(); + let mut ops = Vec::new(); + for i in 0..10 { + local(&mut c, if i % 2 == 0 { "old" } else { "new" }, i); + } + // log what was built so far: the ops are the records as added + for p in c.photos() { + ops.push(Op::AddPhoto { photo: Box::new((**p).clone()) }); + } + ops.push(Op::SetBrowsed { folder: "/pics/old".into(), at: Some(OLD.into()) }); + ops.push(Op::SetBrowsed { folder: "/pics/new".into(), at: Some(NOW.into()) }); + browsed(&mut c, "old", OLD); + browsed(&mut c, "new", NOW); + j.append(&ops).unwrap(); + j.snapshot(&c).unwrap(); + drop(j); + // reload from the snapshot: baselines still match + let (mut j, mut c, _) = Journal::open(Box::new(m.clone())).unwrap(); + let p = c.forget_local_plan(NOW, 30, &|_| false); + assert_eq!(p.evict.len(), 5, "{p:?}"); + let ops = p.ops(); + for op in &ops { + c.apply(op.clone()).unwrap(); + } + j.append(&ops).unwrap(); + let expect = c.to_snapshot(); + drop(j); // crash: replay the log + let (_, c2, r) = Journal::open(Box::new(m)).unwrap(); + assert_eq!(r.failed, 0); + assert_eq!(c2.to_snapshot(), expect); + assert_eq!(c2.len(), 5); + // nothing left to forget + assert!(c2.forget_local_plan(NOW, 30, &|_| false).evict.is_empty()); +} + +#[test] +fn op_photo_ids_cover_nested_ops() { + let op = Op::Batch { + ops: vec![ + Op::SetRating { id: PhotoId(1), rating: 1 }, + Op::SetAlbumPhotos { id: AlbumId(1), photos: vec![PhotoId(2), PhotoId(3)] }, + Op::Batch { ops: vec![Op::RemovePhoto { id: PhotoId(4) }] }, + ], + }; + let mut ids = Vec::new(); + op.photo_ids(&mut |id| ids.push(id.0)); + assert_eq!(ids, vec![1, 2, 3, 4]); +} diff --git a/crates/engine/src/cmd/browse.rs b/crates/engine/src/cmd/browse.rs index f93f14f18..2730822c5 100644 --- a/crates/engine/src/cmd/browse.rs +++ b/crates/engine/src/cmd/browse.rs @@ -2,6 +2,11 @@ //! The folder's photos are probed in place as `local` photos — only folder views list them — //! and edits go to the files' XMP sidecars like any photo added in place. "Add to My Photos" //! makes them ordinary library photos. +//! +//! Each browse records when the folders it listed were browsed. Untouched records of folders +//! not browsed for [`Session::forget_local_days`] are forgotten (when the library opens, or with +//! `library.forgetLocal`); see [`lightcraft_catalog::local`] for what "untouched" means. Files +//! and sidecars on disk are never touched, and browsing the folder again brings them back. use std::path::Path; @@ -35,12 +40,113 @@ fn browse(s: &mut Session, p: &Value) -> Result { v }; let report = import_with(s, &files, &ImportOptions { mode: ImportMode::Add, local: true, ..Default::default() })?; + stamp_browsed(s, &dir_s, &files); s.browse = Some(Browse { path: dir_s.clone(), subfolders }); s.source = LibrarySource::Folder; let shown = s.visible().len(); Ok(json!({"path": dir_s, "subfolders": subfolders, "photos": shown, "new": report.imported.len(), "failed": report.failed.len()})) } +/// Folders whose browse time is refreshed at most this often (keeps repeated browsing from +/// growing the log; far finer than the days the eviction counts in). +const RESTAMP_SECS: i64 = 3600; + +/// Record that `root` and the folders of `files` were browsed now (journaled, not an undo step). +fn stamp_browsed(s: &mut Session, root: &str, files: &[String]) { + let now = (s.clock)(); + let now_s = lightcraft_catalog::stacks::iso_seconds(&now); + let mut folders: std::collections::BTreeSet = + files.iter().filter_map(|f| Path::new(f).parent().map(|d| d.to_string_lossy().trim_end_matches(['/', '\\']).to_string())).collect(); + folders.insert(root.to_string()); + let ops: Vec = folders + .into_iter() + .filter(|f| { + let at = s.catalog.last_browsed(f).and_then(lightcraft_catalog::stacks::iso_seconds); + match (at, now_s) { + (Some(at), Some(n)) => n - at >= RESTAMP_SECS || n < at, + _ => true, + } + }) + .map(|folder| Op::SetBrowsed { folder, at: Some(now.clone()) }) + .collect(); + if !ops.is_empty() { + let op = Op::Batch { ops }; + if s.catalog.apply(op.clone()).is_ok() { + s.pending_log.push(op); + } + } +} + +/// Photos the session still needs, whatever their state: shown, selected, being edited, or +/// referred to by an undo / redo step. +fn in_use(s: &mut Session) -> std::collections::HashSet { + let mut keep: std::collections::HashSet = s.visible().iter().copied().collect(); + keep.extend(s.selection.ids.iter().copied()); + keep.extend(s.selection.active); + keep.extend(s.previous_active); + keep.extend(s.interaction.as_ref().map(|i| i.photo)); + keep.extend(s.before.keys().copied()); + for e in s.undo.iter().chain(s.redo.iter()) { + e.op.photo_ids(&mut |id| { + keep.insert(id); + }); + } + keep +} + +impl Session { + /// Forget untouched Local records whose folder wasn't browsed for `days` (default + /// [`Session::forget_local_days`]; 0 = never). With `dry_run`, only report what would go. + /// Journaled (durable, replayed after a crash) but not an undo step; nothing on disk changes. + pub fn forget_local(&mut self, dry_run: bool, days: Option) -> lightcraft_catalog::ForgetPlan { + let days = days.unwrap_or(self.forget_local_days); + let now = (self.clock)(); + let keep = in_use(self); + let smart = self.media.smart_dir.clone(); + let plan = self + .catalog + .forget_local_plan(&now, days, &|p| keep.contains(&p.id) || smart.as_ref().is_some_and(|d| d.join(crate::smart::file_name(p)).exists())); + if dry_run { + return plan; + } + for op in plan.ops() { + match self.catalog.apply(op.clone()) { + Ok(_) => self.pending_log.push(op), + Err(e) => log::warn!("forget Local records: {e}"), + } + } + if !plan.evict.is_empty() { + log::info!("forgot {} untouched Local record(s) of folders not browsed for {days} days", plan.evict.len()); + } + plan + } +} + +fn forget_local(s: &mut Session, p: &Value) -> Result { + let dry_run = bool_or(p, "dryRun", false); + let days = match p.get("days") { + None | Some(Value::Null) => None, + Some(v) => { + Some(v.as_u64().ok_or_else(|| bad("library.forgetLocal", "days must be a whole number (0 = never)"))?.min(u64::from(u32::MAX)) as u32) + } + }; + let used = days.unwrap_or(s.forget_local_days); + let plan = s.forget_local(dry_run, days); + if !dry_run && !plan.evict.is_empty() { + s.compact_soon(); + } + Ok(json!({ + "dryRun": dry_run, + "days": used, + "local": plan.local, + "forgotten": plan.evict.len(), + "keptRecent": plan.kept_recent, + "keptTouched": plan.kept_touched, + "keptInUse": plan.kept_in_use, + "foldersStamped": plan.stamp.len(), + })) +} + /// The browsed photos among `ids` (default: the selection) become library photos. fn add_to_library(s: &mut Session, p: &Value) -> Result { let ids: Vec = s.targets(p).into_iter().filter(|id| s.catalog.photo(*id).is_some_and(|ph| ph.local)).collect(); @@ -98,6 +204,15 @@ pub fn specs() -> Vec { always, browse ), + cmd!( + "library.forgetLocal", + "Forget Unchanged Local Photos", + [], + None, + "{dryRun?: bool, days?: n (default: the library preference, 0 = never)} — remove Local (browsed, not added) photos nobody changed, of folders not browsed for `days`, from the catalog; files and sidecars stay, browsing the folder again brings them back; not an undo step → {local, forgotten, keptRecent, keptTouched, keptInUse, foldersStamped, days, dryRun}", + always, + forget_local + ), cmd!("photo.addToLibrary", "Add to My Photos", ["Photo"], None, "{ids?} — browsed (Local) photos join the library", always, add_to_library), ] } diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index 8d1d5497e..a8123dcb1 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -867,6 +867,8 @@ pub fn specs() -> Vec { "logRecords": j.log_records(), "logBytes": j.log_bytes(), "lastError": lib.last_error, + // untouched Local records forgotten when the library opened + "forgotLocal": lib.forgot_local.as_ref().map(|p| json!({"forgotten": p.evict.len(), "local": p.local, "keptRecent": p.kept_recent, "keptTouched": p.kept_touched, "keptInUse": p.kept_in_use})), "persistence": j.stats(), "cache": cache, "load": { diff --git a/crates/engine/src/cmd/prefs.rs b/crates/engine/src/cmd/prefs.rs index ba11a38d2..78de9d850 100644 --- a/crates/engine/src/cmd/prefs.rs +++ b/crates/engine/src/cmd/prefs.rs @@ -14,6 +14,7 @@ fn current(s: &Session) -> Value { "xmp": s.xmp, "import": s.import_defaults, "cacheMb": if s.cache_mb == 0 { s.cache_bytes() >> 20 } else { u64::from(s.cache_mb) }, + "forgetLocalDays": s.forget_local_days, "persistent": s.library.is_some(), }) } @@ -86,6 +87,11 @@ fn prefs(s: &mut Session, p: &Value) -> Result { changed = true; } s.import_defaults = d; + if let Some(v) = p.get("forgetLocalDays") { + let days = v.as_u64().ok_or_else(|| bad(ID, "forgetLocalDays must be a whole number of days (0 = never)"))?; + s.forget_local_days = days.min(36_500) as u32; + changed = true; + } if let Some(mb) = p.get("cacheMb") { let mb = mb.as_u64().ok_or_else(|| bad(ID, "cacheMb must be a number of megabytes"))?; s.set_cache_mb(mb.clamp(0, 1 << 20) as u32)?; @@ -101,7 +107,7 @@ pub fn specs() -> Vec { "Library Preferences", [], None, - "{import?: {rawPreset?: presetId|\"default\", otherPreset?: presetId|\"default\", perCamera?: bool, cameras?: [{camera: \"Make Model\", preset: presetId|null}], copyright?: text, creator?: text (given to imported photos without one)}, camera?: {camera, preset?, remove?: bool}, cacheMb?: n (0 = default)} — develop defaults applied on import (raws / other images / per camera) and the thumbnail cache size, saved with the library → {xmp, import, cacheMb, persistent}", + "{import?: {rawPreset?: presetId|\"default\", otherPreset?: presetId|\"default\", perCamera?: bool, cameras?: [{camera: \"Make Model\", preset: presetId|null}], copyright?: text, creator?: text (given to imported photos without one)}, camera?: {camera, preset?, remove?: bool}, cacheMb?: n (0 = default), forgetLocalDays?: n (forget untouched Local photos of folders not browsed for n days; 0 = never)} — develop defaults applied on import (raws / other images / per camera) and the thumbnail cache size, saved with the library → {xmp, import, cacheMb, forgetLocalDays, persistent}", always, prefs )] diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 047330dc9..1109859e8 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -722,6 +722,10 @@ pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> c } report.imported.push(id.0); p.local = opts.local; + if opts.local { + // the state as browsed: a later change keeps the record from being forgotten + p.set_local_baseline(); + } ops.push(Op::AddPhoto { photo: Box::new(p) }); } let log0 = s.pending_log.len(); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d32dfe278..f53218513 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -168,6 +168,9 @@ pub struct Session { pub cache_mb: u32, /// Where smart previews are kept when not in the library folder (persisted in prefs.json). pub smart_previews_dir: Option, + /// Untouched Local records of folders not browsed for this many days are forgotten when the + /// library opens (0 = never; persisted in prefs.json). See `cmd/browse.rs`. + pub forget_local_days: u32, } impl Default for Session { @@ -228,6 +231,7 @@ impl Session { import_defaults: import::ImportDefaults::default(), cache_mb: 0, smart_previews_dir: None, + forget_local_days: lightcraft_catalog::DEFAULT_FORGET_DAYS, } } @@ -599,6 +603,8 @@ mod tests_color; #[cfg(test)] mod tests_export; #[cfg(test)] +mod tests_forget_local; +#[cfg(test)] mod tests_import; #[cfg(test)] mod tests_import_move; diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs index 24660ea51..a89cab272 100644 --- a/crates/engine/src/library.rs +++ b/crates/engine/src/library.rs @@ -52,6 +52,8 @@ pub struct Library { pub report: LoadReport, /// Last persistence error (shown by the UI; ops stay pending and are retried). pub last_error: Option, + /// What forgetting untouched Local records did when the library opened. + pub forgot_local: Option, presets_written: String, view_written: Vec, } @@ -133,6 +135,9 @@ struct PrefsFile { cache_mb: u32, /// Folder for smart previews (default: `Smart Previews` in the library). smart_previews_dir: Option, + /// Days after which untouched Local records of unbrowsed folders are forgotten (missing = + /// the default, 0 = never). + forget_local_days: Option, } fn presets_json(s: &Session) -> String { @@ -222,6 +227,7 @@ impl Session { self.recent_keywords = prefs.recent_keywords; self.import_defaults = prefs.import; self.cache_mb = prefs.cache_mb; + self.forget_local_days = prefs.forget_local_days.unwrap_or(lightcraft_catalog::DEFAULT_FORGET_DAYS); self.smart_previews_dir = prefs.smart_previews_dir.filter(|_| on_disk).map(PathBuf::from); if let Some(d) = &self.smart_previews_dir { self.media.smart_dir = Some(d.clone()); @@ -248,8 +254,23 @@ impl Session { self.media.attach_disk_cache(&dir.join("thumbs"), self.cache_bytes()); } let view_written = self.view_json(); - let library = self.library.insert(Library { dir, on_disk, journal, files, report, last_error: None, presets_written, view_written }); - Ok(&library.report) + self.library = Some(Library { dir, on_disk, journal, files, report, last_error: None, forgot_local: None, presets_written, view_written }); + // forget untouched Local records of folders not browsed for a while (journaled at once) + if self.forget_local_days > 0 { + let plan = self.forget_local(false, None); + if !plan.evict.is_empty() { + self.compact_soon(); + } else if let Err(e) = self.persist() { + log::error!("library: {e}"); + } + if let Some(lib) = self.library.as_mut() { + lib.forgot_local = Some(plan); + } + } + match &self.library { + Some(lib) => Ok(&lib.report), + None => Err(EngineError::Other("library closed while opening".into())), + } } /// Write pending ops to the log (fsynced), compact when due, and save changed presets. @@ -351,6 +372,7 @@ impl Session { import: self.import_defaults.clone(), cache_mb: self.cache_mb, smart_previews_dir: self.smart_previews_dir.as_ref().map(|d| d.to_string_lossy().to_string()), + forget_local_days: Some(self.forget_local_days), }) .unwrap_or_default(); let Some(lib) = self.library.as_mut() else { return Ok(()) }; @@ -373,6 +395,24 @@ impl Session { self.save_prefs() } + /// Save, then start compacting in the background (natively; synchronously elsewhere): after + /// a change that shrinks the catalog a lot, so the smaller snapshot replaces the old one. + /// Errors are logged and reported like other persistence errors (nothing is lost). + pub fn compact_soon(&mut self) { + if let Err(e) = self.persist() { + log::error!("library: {e}"); + return; + } + if self.interaction.is_some() { + return; + } + let Some(lib) = self.library.as_mut() else { return }; + if let Err(e) = lib.journal.snapshot_in_background(&self.catalog) { + log::error!("library: compaction: {e}"); + lib.last_error = Some(format!("compaction: {e}")); + } + } + /// Compact the log into a snapshot now. pub fn compact_library(&mut self) -> Result<()> { self.persist()?; diff --git a/crates/engine/src/tests_forget_local.rs b/crates/engine/src/tests_forget_local.rs new file mode 100644 index 000000000..14a985cd4 --- /dev/null +++ b/crates/engine/src/tests_forget_local.rs @@ -0,0 +1,218 @@ +//! Forgetting untouched Local browse records: by command and when the library opens; changed, +//! recent and visible records stay; a crash replays it; browsing again brings records back. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use serde_json::json; + +use crate::{LibrarySource, Selection, Session}; + +fn temp_dir(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("lc-forget-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn png(p: &Path, seed: u8) { + let img = lightcraft_raster::Rgba8::from_fn(16, 12, |x, y| [(x * 9) as u8, (y * 11) as u8, seed, 255]); + let b = + crate::export::encode_image(&img, &crate::export::ExportOptions { format: crate::export::ExportFormat::Png, ..Default::default() }).unwrap(); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, b).unwrap(); +} + +fn open(lib: &Path, clock: &Arc>) -> Session { + let mut s = Session::new().with_fs(); + let c = clock.clone(); + s.clock = Box::new(move || c.lock().unwrap().clone()); + s.open_library(lib, false).unwrap(); + s +} + +fn set(clock: &Arc>, t: &str) { + *clock.lock().unwrap() = t.to_string(); +} + +fn local_paths(s: &Session) -> Vec { + let mut v: Vec = s + .catalog + .photos() + .filter(|p| p.local) + .filter_map(|p| match &p.source { + lightcraft_catalog::Source::File { path } => Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()), + _ => None, + }) + .collect(); + v.sort(); + v +} + +/// Leave the folder view (nothing browsed is visible or selected any more). +fn look_elsewhere(s: &mut Session) { + s.execute("library.source", &json!({"kind": "all"})).unwrap(); + s.selection = Selection::default(); + s.previous_active = None; + s.undo.clear(); +} + +#[test] +fn untouched_records_of_unbrowsed_folders_are_forgotten_and_come_back() { + let root = temp_dir("cmd"); + let (a, b, lib) = (root.join("a"), root.join("b"), root.join("lib")); + for (i, n) in ["a1.png", "a2.png", "a3.png"].iter().enumerate() { + png(&a.join(n), i as u8); + } + png(&b.join("b1.png"), 7); + let clock = Arc::new(Mutex::new("2026-08-01T10:00:00".to_string())); + let mut s = open(&lib, &clock); + s.execute("library.browse", &json!({"path": a.to_string_lossy()})).unwrap(); + // the user rates a2 (a change: kept) + let a2 = s.catalog.photos().find(|p| p.file_name == "a2.png").unwrap().id; + s.selection = Selection::single(a2); + s.execute("photo.rate", &json!({"rating": 3})).unwrap(); + set(&clock, "2026-08-25T10:00:00"); + s.execute("library.browse", &json!({"path": b.to_string_lossy()})).unwrap(); + look_elsewhere(&mut s); + + // 40 days after a was browsed, 16 after b + set(&clock, "2026-09-10T10:00:00"); + let dry = s.execute("library.forgetLocal", &json!({"dryRun": true})).unwrap(); + assert_eq!( + (dry["local"].as_u64(), dry["forgotten"].as_u64(), dry["keptTouched"].as_u64(), dry["keptRecent"].as_u64()), + (Some(4), Some(2), Some(1), Some(1)), + "{dry}" + ); + assert_eq!(local_paths(&s).len(), 4, "a dry run changes nothing"); + let r = s.execute("library.forgetLocal", &json!({})).unwrap(); + assert_eq!(r["forgotten"], 2, "{r}"); + assert_eq!(local_paths(&s), vec!["a2.png", "b1.png"]); + // files on disk are untouched + assert!(["a1.png", "a2.png", "a3.png"].iter().all(|n| a.join(n).exists())); + // not an undo step + assert!(s.undo.is_empty()); + + // a crash now: the log replays the same catalog + let expect = s.catalog.to_snapshot(); + drop(s); + let mut s = open(&lib, &clock); + assert_eq!(s.library.as_ref().unwrap().report.failed, 0); + assert_eq!(s.catalog.to_snapshot(), expect, "nothing more to forget on open, same state after replay"); + + // browsing a again brings the forgotten files back, fresh; the rated one is the same record + let r = s.execute("library.browse", &json!({"path": a.to_string_lossy()})).unwrap(); + assert_eq!((r["photos"].as_u64(), r["new"].as_u64()), (Some(3), Some(2)), "{r}"); + assert_eq!(s.catalog.photo(a2).unwrap().rating, 3); + // while a is shown, nothing in it goes, whatever the time + set(&clock, "2027-09-10T10:00:00"); + assert_eq!(s.execute("library.forgetLocal", &json!({"dryRun": true})).unwrap()["forgotten"], 1, "only b1 (not visible)"); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn opening_the_library_forgets_per_the_preference() { + let root = temp_dir("open"); + let (a, lib) = (root.join("a"), root.join("lib")); + png(&a.join("x.png"), 1); + png(&a.join("y.png"), 2); + let clock = Arc::new(Mutex::new("2026-08-01T10:00:00".to_string())); + let mut s = open(&lib, &clock); + s.execute("library.browse", &json!({"path": a.to_string_lossy()})).unwrap(); + look_elsewhere(&mut s); + // 0 = never + let r = s.execute("library.preferences", &json!({"forgetLocalDays": 0})).unwrap(); + assert_eq!(r["forgetLocalDays"], 0); + s.close_library().unwrap(); + drop(s); + set(&clock, "2027-01-01T10:00:00"); + let mut s = open(&lib, &clock); + assert_eq!(local_paths(&s).len(), 2, "never"); + assert!(s.library.as_ref().unwrap().forgot_local.is_none()); + // back to 30 days: the next open forgets them + s.execute("library.preferences", &json!({"forgetLocalDays": 30})).unwrap(); + s.source = LibrarySource::All; + s.close_library().unwrap(); + drop(s); + let s = open(&lib, &clock); + assert!(local_paths(&s).is_empty()); + let info = { + let mut s = s; + s.execute("library.info", &json!({})).unwrap() + }; + assert_eq!(info["forgotLocal"]["forgotten"], 2, "{info}"); + assert!(a.join("x.png").exists() && a.join("y.png").exists()); + let _ = std::fs::remove_dir_all(&root); +} + +/// A file with an XMP sidecar comes in with the sidecar's rating and keywords: that is its +/// browse state, not a user change, so the record is forgotten like any other — and the sidecar +/// stays byte for byte, so browsing again reads the same values back. +#[test] +fn sidecar_values_are_not_user_changes_and_sidecars_stay() { + let root = temp_dir("xmp"); + let (a, lib) = (root.join("a"), root.join("lib")); + png(&a.join("x.png"), 1); + let x = a.join("x.png").to_string_lossy().to_string(); + let mut p = lightcraft_catalog::Photo::new( + lightcraft_catalog::PhotoId(1), + lightcraft_catalog::Source::File { path: x.clone() }, + "x.png", + "PNG", + 16, + 12, + "2026-01-01T00:00:00", + ); + p.rating = 4; + p.meta.keywords = vec!["dunes".into()]; + let side = crate::sidecar::sidecar_path(&x, Default::default()); + std::fs::write(&side, crate::sidecar::sidecar_packet(&p, &lightcraft_catalog::Catalog::new())).unwrap(); + let packet = std::fs::read(&side).unwrap(); + + let clock = Arc::new(Mutex::new("2026-08-01T10:00:00".to_string())); + let mut s = open(&lib, &clock); + s.execute("library.browse", &json!({"path": a.to_string_lossy()})).unwrap(); + let rec = s.catalog.photos().find(|p| p.local).unwrap().clone(); + assert_eq!((rec.rating, rec.meta.keywords.clone()), (4, vec!["dunes".to_string()]), "read from the sidecar"); + look_elsewhere(&mut s); + set(&clock, "2026-10-01T10:00:00"); + assert_eq!(s.execute("library.forgetLocal", &json!({})).unwrap()["forgotten"], 1); + assert_eq!(std::fs::read(&side).unwrap(), packet, "the sidecar is untouched"); + s.execute("library.browse", &json!({"path": a.to_string_lossy()})).unwrap(); + let back = s.catalog.photos().find(|p| p.local).unwrap(); + assert_ne!(back.id, rec.id, "a fresh record"); + assert_eq!((back.rating, back.meta.keywords.clone()), (4, vec!["dunes".to_string()])); + let _ = std::fs::remove_dir_all(&root); +} + +/// A catalog written before browse times existed: its Local records are kept on the first open +/// (their folders are stamped then), and forgotten only once the period has passed since. +#[test] +fn upgraded_catalogs_forget_nothing_at_first() { + let root = temp_dir("upgrade"); + let (a, lib) = (root.join("a"), root.join("lib")); + png(&a.join("x.png"), 1); + let clock = Arc::new(Mutex::new("2026-08-01T10:00:00".to_string())); + let mut s = open(&lib, &clock); + s.execute("library.browse", &json!({"path": a.to_string_lossy()})).unwrap(); + look_elsewhere(&mut s); + s.close_library().unwrap(); + drop(s); + // strip the times and baselines, as an older version would have written the snapshot + let snap = std::fs::read_to_string(lib.join("catalog.snap")).unwrap(); + let mut v: serde_json::Value = serde_json::from_str(&snap).unwrap(); + v["catalog"].as_object_mut().unwrap().remove("browsed"); + for p in v["catalog"]["photos"].as_object_mut().unwrap().values_mut() { + p.as_object_mut().unwrap().remove("local_baseline"); + } + std::fs::write(lib.join("catalog.snap"), serde_json::to_vec(&v).unwrap()).unwrap(); + set(&clock, "2027-01-01T10:00:00"); + let s = open(&lib, &clock); + assert_eq!(local_paths(&s).len(), 1, "kept on the first open after the upgrade"); + assert_eq!(s.catalog.last_browsed(&a.to_string_lossy()), Some("2027-01-01T10:00:00")); + drop(s); + set(&clock, "2027-02-15T10:00:00"); + let s = open(&lib, &clock); + assert!(local_paths(&s).is_empty(), "the untouched legacy record goes once 30 days passed"); + let _ = std::fs::remove_dir_all(&root); +} diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index 518a26e26..6c89cb1b9 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -376,6 +376,19 @@ fn performance_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) { let _ = app.run("library.clearPreviews", json!({})); } }); + heading(ui, t, "Local folders"); + row(ui, t, "Forget unchanged photos", |ui| { + let opts = [(0u32, "Never"), (7, "After 7 days"), (30, "After 30 days"), (90, "After 90 days"), (365, "After a year")]; + let mut v = app.session.forget_local_days; + if choices(ui, "settingsForgetLocal", &opts, &mut v) { + let _ = app.run("library.preferences", json!({"forgetLocalDays": v})); + } + }); + hint( + ui, + t, + "Photos seen in Local but never added or changed leave the catalog when their folder hasn't been browsed for this long (checked when the library opens). Files stay on disk; browsing the folder shows them again.", + ); #[cfg(not(target_arch = "wasm32"))] smart_previews(app, ui, t); } diff --git a/docs/parity.md b/docs/parity.md index 16f5fa095..e656d3e29 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -108,7 +108,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-IMP-RAWDEFAULT | Raw defaults | P1 | ✅ | `cmd:library.preferences`, `crates/engine/src/import.rs`, `crates/ui-egui/src/panels/settings.rs` | LightCraft default / a preset / per camera (make + model); non-raw default too; the preset look counts as unedited and Reset returns to it | | LR-IMP-MIGRATE | Migrate other catalogs | OOS | 🚫 | | | | LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported; profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported | -| LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`) | +| LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) | | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars | | LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF, WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2, ARW, NEF, RAF, RW2, PEF, ORF. Missing: CR3, compressed NEF/RAF/ORF, RW2 v4 (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode | | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic | diff --git a/docs/whats-new.md b/docs/whats-new.md index cfdd963fd..7871f6c22 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -21,6 +21,9 @@ - Export file names use the Rename Photos tokens ({title}, {seq:2}, {date:%Y-%m-%d}…), plus new {num}, {folder}, {lens}, {iso}, {rating}, {creator}. - Copyright status, rights usage terms and copyright info URL in Info, metadata presets and exports. - Auto-Tag from Tracklog: GPS locations for your photos from a GPX track log, matched by capture time. +- Smaller, faster catalogs: photos you only looked at in Local (never added, rated or edited) are forgotten once + their folder has not been browsed for 30 days — your files and sidecars stay, and browsing the folder shows them + again. Change the period (or turn it off) in Settings → Performance. ### Editing - Auto Sync: edits apply to every selected photo. Auto B&W mix. Automatic versions.