diff --git a/Cargo.lock b/Cargo.lock index 10f38d5a1..60a82a267 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2099,6 +2099,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.21", + "web-time", ] [[package]] diff --git a/apps/lightcraft-web/src/web.rs b/apps/lightcraft-web/src/web.rs index 0d50ebe82..09a195959 100644 --- a/apps/lightcraft-web/src/web.rs +++ b/apps/lightcraft-web/src/web.rs @@ -156,6 +156,7 @@ fn services(originals: Originals, backend: Option, ctx: egui::Context) })), // Preset files: browser pickers are asynchronous; not wired on the web yet. pick_preset_files: None, + pick_tracklog: None, save_preset_file: None, pick_curve_preset_files: None, save_curve_preset_file: None, diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index 7f7443301..9c8d63d16 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -174,6 +174,14 @@ fn services() -> Services { .map(|p| p.to_string_lossy().to_string()) .collect() })), + pick_tracklog: Some(Box::new(|| { + rfd::FileDialog::new() + .set_title("Auto-Tag from Tracklog") + .add_filter("GPS Track Log", &["gpx"]) + .pick_file() + .map(|p| vec![p.to_string_lossy().to_string()]) + .unwrap_or_default() + })), save_preset_file: Some(Box::new(|name: &str| { rfd::FileDialog::new() .set_title("Export Presets") diff --git a/crates/catalog/Cargo.toml b/crates/catalog/Cargo.toml index fb772703f..475083596 100644 --- a/crates/catalog/Cargo.toml +++ b/crates/catalog/Cargo.toml @@ -15,6 +15,7 @@ serde_json = { workspace = true } thiserror = { workspace = true } crc32fast = { workspace = true } log = { workspace = true } +web-time = { workspace = true } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/catalog/examples/persist_bench.rs b/crates/catalog/examples/persist_bench.rs new file mode 100644 index 000000000..60ec46aa7 --- /dev/null +++ b/crates/catalog/examples/persist_bench.rs @@ -0,0 +1,170 @@ +//! Catalog persistence costs on large synthetic catalogs (issue #37): log append, compaction +//! (snapshot serialise / write+fsync / log reset), synchronous and on the background worker, load, +//! and a catalog clone. +//! +//! ```text +//! cargo run --release -p lightcraft-catalog --example persist_bench -- [photos…] +//! ``` +//! +//! `` is a scratch directory (wiped per size). Default sizes: 1000 10000 85000. 90 % of the +//! photos are Local browse records, the rest library photos with a few edits and history steps. +//! Peak memory: run under `/usr/bin/time -l` (macOS) / `-v` (Linux) with one size; +//! `PERSIST_BENCH_NO_LOAD=1` skips the final load, so the peak is building + compacting. + +use std::sync::Arc; +use std::time::Instant; + +use lightcraft_catalog::*; +use lightcraft_develop::DevelopSettings; + +fn ms(t: Instant) -> f64 { + t.elapsed().as_secs_f64() * 1e3 +} + +fn photo(c: &mut Catalog, i: u64) -> Photo { + let id = c.alloc_photo_id(); + let mut p = Photo::new( + id, + Source::File { path: format!("/Users/someone/Pictures/2026/09/{:05}/IMG_{i:05}.CR3", i / 500) }, + &format!("IMG_{i:05}.CR3"), + "CR3", + 6000, + 4000, + "2026-09-30T12:00:00", + ); + p.kind = MediaKind::Raw; + p.file_size = 25_000_000 + i; + p.captured = Some(format!("2026-09-{:02}T10:{:02}:{:02}", 1 + i % 28, i % 60, (i / 60) % 60)); + p.content_hash = Some(format!("{:032x}", i.wrapping_mul(0x9e37_79b9_7f4a_7c15))); + p.meta.camera = "Camera Model X".into(); + p.meta.lens = "24-70mm f/2.8".into(); + p.meta.focal_mm = Some(35.0); + p.meta.aperture = Some(4.0); + p.meta.shutter = "1/250".into(); + p.meta.iso = Some(400); + p.as_shot_wb = Some((5200.0, 3.0)); + p.local = !i.is_multiple_of(10); + if !p.local { + let mut d = DevelopSettings::default(); + d.light.exposure = (i % 7) as f64 * 0.1; + let s = Arc::new(d); + p.develop = s.clone(); + p.edited = Some("2026-09-30T12:00:00".into()); + p.history = vec![HistoryStep { label: "Exposure".into(), settings: s }]; + p.rating = (i % 6) as u8; + } + p +} + +fn stats(mut v: Vec) -> String { + v.sort_by(f64::total_cmp); + let pick = |q: f64| v.get(((v.len() as f64 - 1.0) * q).round() as usize).copied().unwrap_or(0.0); + format!("median {:.2} ms, p95 {:.2} ms, max {:.2} ms", pick(0.5), pick(0.95), pick(1.0)) +} + +fn main() -> std::result::Result<(), Box> { + let mut args = std::env::args().skip(1); + let dir = std::path::PathBuf::from(args.next().ok_or("usage: persist_bench [photos…]")?); + let mut sizes: Vec = args.map(|a| a.parse()).collect::>()?; + if sizes.is_empty() { + sizes = vec![1000, 10_000, 85_000]; + } + for n in sizes { + let _ = std::fs::remove_dir_all(&dir); + let (mut j, _, _) = Journal::open(Box::new(FsStore::open(&dir)?))?; + let mut cat = Catalog::new(); + // bulk-add (as one import would) and log it, so the first compaction is an import's + for chunk in (0..n).collect::>().chunks(500) { + let ops: Vec = chunk.iter().map(|i| Op::AddPhoto { photo: Box::new(photo(&mut cat, *i)) }).collect(); + for op in &ops { + cat.apply(op.clone())?; + } + j.append(&ops)?; + } + j.snapshot(&cat)?; + let first = j.stats().last_snapshot; + println!("== {n} photos ({} local): snapshot {:.1} MB", cat.photos().filter(|p| p.local).count(), first.bytes as f64 / 1e6); + + let t = Instant::now(); + let copy = cat.clone(); + println!(" catalog clone: {:.2} ms", ms(t)); + drop(copy); + + // single-op appends (a rating change each), as commands do + let ids: Vec = cat.photos().map(|p| p.id).take(1000).collect(); + let mut appends = Vec::new(); + for k in 0..300usize { + let id = ids.get(k % ids.len().max(1)).copied().unwrap_or(PhotoId(1)); + let op = Op::SetRating { id, rating: (k % 6) as u8 }; + cat.apply(op.clone())?; + j.append(std::slice::from_ref(&op))?; + appends.push(j.stats().last_append_ms); + } + println!(" append (1 op, fsync): {}", stats(appends)); + + // compactions at the default threshold crossing + let mut totals = Vec::new(); + for _ in 0..3 { + while !j.wants_snapshot() { + let op = Op::SetRating { id: ids.first().copied().unwrap_or(PhotoId(1)), rating: (j.seq() % 6) as u8 }; + cat.apply(op.clone())?; + j.append(&[op])?; + } + j.snapshot(&cat)?; + let s = j.stats().last_snapshot; + println!( + " compaction ({} records): serialize {:.1} ms, write+sync {:.1} ms, log reset {:.1} ms, total {:.1} ms", + s.records, s.serialize_ms, s.write_sync_ms, s.reset_ms, s.total_ms + ); + totals.push(s.total_ms); + } + println!(" compaction total: {}", stats(totals)); + + // the same at the threshold, written by the worker while single-op appends go on + let mut blocking = Vec::new(); + for _ in 0..3 { + while !j.wants_snapshot() { + let op = Op::SetRating { id: ids.first().copied().unwrap_or(PhotoId(1)), rating: (j.seq() % 6) as u8 }; + cat.apply(op.clone())?; + j.append(&[op])?; + } + let t = Instant::now(); + j.snapshot_in_background(&cat)?; + let start_ms = ms(t); + let mut during = Vec::new(); + while !j.snapshot_written() { + let op = Op::SetRating { id: ids.get(1).copied().unwrap_or(PhotoId(1)), rating: (j.seq() % 6) as u8 }; + cat.apply(op.clone())?; + j.append(&[op])?; + during.push(j.stats().last_append_ms); + } + let t = Instant::now(); + j.poll()?; + let finish_ms = ms(t); + let s = j.stats().last_snapshot; + println!( + " background compaction ({} records): start {start_ms:.1} ms + finish {finish_ms:.1} ms blocking; worker {:.1} ms; {} appends during it: {}", + s.records, + s.total_ms, + during.len(), + stats(during) + ); + blocking.push(s.blocking_ms); + } + println!(" background compaction blocking: {}", stats(blocking)); + + drop(j); + if std::env::var_os("PERSIST_BENCH_NO_LOAD").is_some() { + // peak memory of building + compacting only + continue; + } + let t = Instant::now(); + let (_, loaded, _) = Journal::open(Box::new(FsStore::open(&dir)?))?; + println!(" open (load snapshot + replay): {:.1} ms", ms(t)); + if loaded.len() != cat.len() { + return Err("reloaded catalog differs".into()); + } + } + let _ = std::fs::remove_dir_all(&dir); + Ok(()) +} diff --git a/crates/catalog/proptest-regressions/tests_background.txt b/crates/catalog/proptest-regressions/tests_background.txt new file mode 100644 index 000000000..e84ff8c01 --- /dev/null +++ b/crates/catalog/proptest-regressions/tests_background.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 0227748e171237a13833c9175fe02c7fe6ff332266798961a6a43ec37b81d1b4 # shrinks to actions = [(3, 0), (0, 0), (4, 0), (0, 0)] +cc 0b0ab2531bd46fbe261b55ecffebea9d3e8949900ab7bc33fb27ca19f5cd7b8b # shrinks to actions = [(4, 443), (3, 474), (2, 27), (5, 858), (5, 466), (0, 901), (4, 907), (3, 839), (0, 630), (7, 578), (4, 13), (2, 361), (6, 136), (6, 12), (6, 223), (1, 394), (6, 78), (5, 837), (6, 765), (4, 207), (2, 185), (2, 583), (2, 56), (5, 430), (1, 702), (1, 547), (5, 552), (4, 762), (0, 443), (0, 851), (7, 240), (4, 438), (3, 599), (6, 70), (4, 694), (1, 327), (7, 859), (6, 814), (4, 336), (3, 438), (0, 131), (0, 831), (3, 18), (2, 963), (6, 10), (3, 114)] diff --git a/crates/catalog/src/journal.rs b/crates/catalog/src/journal.rs index f7089c061..c146b8b42 100644 --- a/crates/catalog/src/journal.rs +++ b/crates/catalog/src/journal.rs @@ -14,6 +14,19 @@ //! the log) are skipped; //! - a bad record **followed by good ones** is real damage: replay stops there, the log is kept //! as `catalog.log.damaged-`, and a fresh snapshot is written so the library stays usable. +//! +//! **Background compaction** ([`Journal::snapshot_in_background`], when the store offers a +//! [`Store::background_writer`]): a worker thread writes the snapshot of a copy of the catalog at +//! op `N` while appends go on (durably, as always) to the same log. Once the snapshot is durable, +//! [`Journal::poll`] atomically replaces the log with the records appended since the snapshot +//! started (`seq > N`, kept in memory). Every crash point is covered by the rules above: +//! - before the rename of `catalog.snap`: the old snapshot + the whole log (nothing was removed); +//! - between the rename and the log rewrite: the new snapshot + the whole log, whose records +//! `<= N` are skipped as stale; +//! - after the log rewrite: the new snapshot + exactly the records `> N`. +//! +//! At most one snapshot is in flight; a synchronous [`Journal::snapshot`] and dropping the journal +//! first wait for it, so an older snapshot can never replace a newer one. use crate::store::Store; use crate::{Catalog, CatalogError, Op, Result}; @@ -55,6 +68,65 @@ pub struct LoadReport { pub created: bool, } +/// Where persistence time goes (reported by `library.info` → `persistence`; printed to stderr +/// per write under `LIGHTCRAFT_PROFILE`). Times are wall-clock milliseconds on the calling +/// thread, i.e. how long the caller (the UI thread, for the app) was blocked. +#[derive(Clone, Copy, Debug, Default, PartialEq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PersistStats { + /// [`Journal::append`] calls that wrote something. + pub appends: u64, + /// Encode + write + `sync_data` of the last / slowest append. + pub last_append_ms: f64, + pub max_append_ms: f64, + /// Snapshots written (compactions, plus the ones on close / repair). + pub snapshots: u64, + /// The last snapshot, by stage. + pub last_snapshot: SnapshotTiming, + /// Total time of the slowest snapshot. + pub max_snapshot_ms: f64, + /// Longest time a snapshot blocked the caller (for background snapshots: copying the catalog + /// and starting the worker, plus rewriting the log when it is done). + pub max_blocking_ms: f64, + /// A background snapshot is being written. + pub snapshot_running: bool, + /// Background snapshots that failed (the log was kept whole; retried later). + pub failed_snapshots: u64, +} + +/// One snapshot (compaction), by stage. +#[derive(Clone, Copy, Debug, Default, PartialEq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SnapshotTiming { + /// Serialising the catalog to JSON, streamed into the temp file (buffered writes included). + pub serialize_ms: f64, + /// Flushing, `sync_all` and the atomic rename of `catalog.snap`. + pub write_sync_ms: f64, + /// Resetting `catalog.log` (an atomic rewrite to empty, or to the records appended while a + /// background snapshot ran; fsynced). + pub reset_ms: f64, + /// Start to finish (for a background snapshot: until [`Journal::poll`] saw it done). + pub total_ms: f64, + /// How long the caller was blocked (equal to `total_ms` unless `background`). + pub blocking_ms: f64, + /// Written by a worker thread. + pub background: bool, + /// Size of `catalog.snap`. + pub bytes: u64, + /// Log records the snapshot compacted. + pub records: u64, +} + +/// `LIGHTCRAFT_PROFILE` is set: print persistence timings to stderr. +fn profiling() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("LIGHTCRAFT_PROFILE").is_some()) +} + +fn ms_since(t: web_time::Instant) -> f64 { + t.elapsed().as_secs_f64() * 1e3 +} + pub struct Journal { store: Box, /// `seq` of the last durable op. @@ -63,6 +135,26 @@ pub struct Journal { log_records: u64, log_bytes: u64, pub policy: SnapshotPolicy, + stats: PersistStats, + /// The background snapshot in flight. + pending: Option, + /// After a failed background snapshot: don't retry before the log has this many records. + retry_at_records: u64, +} + +/// A background snapshot in flight. +struct Pending { + /// The snapshot holds the state after this op. + seq: u64, + /// Log bytes / records appended since it started (all `> seq`): the log once it's done. + tail: Vec, + tail_records: u64, + /// Log records it compacts. + records: u64, + started: web_time::Instant, + /// Time the caller spent starting it. + start_ms: f64, + worker: std::thread::JoinHandle>, } fn io(e: std::io::Error) -> CatalogError { @@ -90,6 +182,29 @@ pub fn decode_record(line: &str) -> Option<(u64, Op)> { serde_json::from_str(body).ok().map(|op| (seq, op)) } +/// Write `catalog.snap` for the state after op `seq`, streaming the JSON into the store (no +/// whole-file string). The bytes are exactly +/// `{"format":"lightcraft-catalog","version":1,"seq":N,"catalog":}\n`, +/// as before streaming. Fills in the serialise / write+sync times and the size. +fn write_snapshot(store: &mut dyn Store, seq: u64, catalog: &Catalog) -> std::io::Result { + let t0 = web_time::Instant::now(); + let mut serialize_ms = 0.0; + let bytes = store.write_atomic_with(SNAPSHOT, &mut |w| { + use std::io::Write; + let t = web_time::Instant::now(); + // serde_json emits many tiny writes: buffer them in a concrete writer (inlined), so only + // 1 MiB chunks go through the store's `dyn Write` + let mut b = std::io::BufWriter::with_capacity(1 << 20, w); + write!(b, "{{\"format\":\"{FORMAT}\",\"version\":{VERSION},\"seq\":{seq},\"catalog\":")?; + serde_json::to_writer(&mut b, catalog).map_err(std::io::Error::other)?; + b.write_all(b"}\n")?; + b.flush()?; + serialize_ms = ms_since(t); + Ok(()) + })?; + Ok(SnapshotTiming { serialize_ms, write_sync_ms: (ms_since(t0) - serialize_ms).max(0.0), bytes, ..Default::default() }) +} + #[derive(serde::Deserialize)] struct SnapFile { format: String, @@ -117,7 +232,17 @@ impl Journal { None => (Catalog::new(), 0), }; report.snapshot_seq = snapshot_seq; - let mut j = Journal { store, seq: snapshot_seq, snapshot_seq, log_records: 0, log_bytes: 0, policy: SnapshotPolicy::default() }; + let mut j = Journal { + store, + seq: snapshot_seq, + snapshot_seq, + log_records: 0, + log_bytes: 0, + policy: SnapshotPolicy::default(), + stats: PersistStats::default(), + pending: None, + retry_at_records: 0, + }; let log = log.unwrap_or_default(); // Scan records; `good_end` is the byte offset just past the last good record. @@ -190,6 +315,7 @@ impl Journal { if ops.is_empty() { return Ok(()); } + let t0 = web_time::Instant::now(); let mut buf = String::new(); let mut seq = self.seq; for op in ops { @@ -198,28 +324,166 @@ impl Journal { buf.push('\n'); } self.store.append(LOG, buf.as_bytes()).map_err(io)?; + if let Some(p) = self.pending.as_mut() { + p.tail.extend_from_slice(buf.as_bytes()); + p.tail_records += ops.len() as u64; + } self.seq = seq; self.log_records += ops.len() as u64; self.log_bytes += buf.len() as u64; + let ms = ms_since(t0); + self.stats.appends += 1; + self.stats.last_append_ms = ms; + self.stats.max_append_ms = self.stats.max_append_ms.max(ms); + if profiling() { + eprintln!("catalog: append {} op(s), {} B: {ms:.2} ms", ops.len(), buf.len()); + } Ok(()) } /// Write a snapshot of `catalog` (which must reflect every appended op) and reset the log. pub fn snapshot(&mut self, catalog: &Catalog) -> Result<()> { - let body = serde_json::to_string(catalog).map_err(|e| CatalogError::Invalid(e.to_string()))?; - let file = format!("{{\"format\":\"{FORMAT}\",\"version\":{VERSION},\"seq\":{},\"catalog\":{body}}}\n", self.seq); - self.store.write_atomic(SNAPSHOT, file.as_bytes()).map_err(io)?; + // never two snapshot writers; the one in flight is older, so it must land first + if let Err(e) = self.wait() { + log::warn!("catalog: background snapshot failed ({e}); writing one now"); + } + let t0 = web_time::Instant::now(); + let mut timing = write_snapshot(self.store.as_mut(), self.seq, catalog).map_err(io)?; + let t2 = web_time::Instant::now(); // A crash here leaves old records in the log; they are skipped by seq on load. self.store.write_atomic(LOG, b"").map_err(io)?; + timing.reset_ms = ms_since(t2); + timing.total_ms = ms_since(t0); + timing.blocking_ms = timing.total_ms; + timing.records = self.log_records; + self.record_snapshot(timing); self.snapshot_seq = self.seq; self.log_records = 0; self.log_bytes = 0; + self.retry_at_records = 0; Ok(()) } + /// Compact like [`Journal::snapshot`], but write the snapshot on a worker thread from a copy + /// of `catalog` (cheap: photos are shared) when the store supports it (else synchronously). + /// `catalog` must reflect every appended op. Appends go on meanwhile; [`Journal::poll`] + /// finishes the compaction. No-op while a snapshot is already in flight. + pub fn snapshot_in_background(&mut self, catalog: &Catalog) -> Result<()> { + if self.pending.is_some() { + return Ok(()); + } + let t0 = web_time::Instant::now(); + let Some(mut writer) = self.store.background_writer() else { return self.snapshot(catalog) }; + let copy = catalog.clone(); + let seq = self.seq; + let spawned = std::thread::Builder::new().name("catalog-snapshot".into()).spawn(move || write_snapshot(writer.as_mut(), seq, ©)); + let worker = match spawned { + Ok(w) => w, + Err(e) => { + log::warn!("catalog: can't start a snapshot thread ({e}); writing it now"); + return self.snapshot(catalog); + } + }; + self.pending = + Some(Pending { seq, tail: Vec::new(), tail_records: 0, records: self.log_records, started: t0, start_ms: ms_since(t0), worker }); + Ok(()) + } + + /// Finish a background snapshot if its worker is done (cheap otherwise). Returns whether one + /// finished. An error means the snapshot failed: nothing was lost (the log is whole), and + /// compaction is retried once the log has grown by another [`SnapshotPolicy::max_records`]. + pub fn poll(&mut self) -> Result { + if !self.pending.as_ref().is_some_and(|p| p.worker.is_finished()) { + return Ok(false); + } + self.finish().map(|()| true) + } + + /// Wait for the background snapshot in flight, if any, and finish it. + pub fn wait(&mut self) -> Result<()> { + if self.pending.is_none() { + return Ok(()); + } + self.finish() + } + + /// A background snapshot is in flight (until [`Journal::poll`] or [`Journal::wait`] finish it). + pub fn snapshot_running(&self) -> bool { + self.pending.is_some() + } + + /// The background worker is done (its result not yet applied by [`Journal::poll`]). + pub fn snapshot_written(&self) -> bool { + self.pending.as_ref().is_some_and(|p| p.worker.is_finished()) + } + + /// Join the worker; on success replace the log with the records appended meanwhile. + fn finish(&mut self) -> Result<()> { + let Some(p) = self.pending.take() else { return Ok(()) }; + let t0 = web_time::Instant::now(); + let written = p.worker.join().unwrap_or_else(|_| Err(std::io::Error::other("snapshot thread panicked"))); + let mut timing = match written { + Ok(t) => t, + Err(e) => { + // the log still holds every record: nothing lost, retry later + self.stats.failed_snapshots += 1; + self.retry_at_records = self.log_records + self.policy.max_records; + log::error!("catalog: background snapshot failed: {e}"); + return Err(io(e)); + } + }; + // `catalog.snap` (op `p.seq`) is durable; the log's records `<= p.seq` are now stale. + // Keep exactly the ones appended since (a crash before this rewrite skips the stale ones). + let t1 = web_time::Instant::now(); + self.snapshot_seq = p.seq; + if let Err(e) = self.store.write_atomic(LOG, &p.tail) { + // the log stays whole (stale records are skipped on load); compact again later + self.retry_at_records = self.log_records + self.policy.max_records; + log::error!("catalog: can't trim the log after a snapshot: {e}"); + return Err(io(e)); + } + timing.reset_ms = ms_since(t1); + timing.total_ms = ms_since(p.started); + timing.blocking_ms = p.start_ms + ms_since(t0); + timing.background = true; + timing.records = p.records; + self.record_snapshot(timing); + self.log_records = p.tail_records; + self.log_bytes = p.tail.len() as u64; + self.retry_at_records = 0; + Ok(()) + } + + fn record_snapshot(&mut self, t: SnapshotTiming) { + self.stats.snapshots += 1; + self.stats.last_snapshot = t; + self.stats.max_snapshot_ms = self.stats.max_snapshot_ms.max(t.total_ms); + self.stats.max_blocking_ms = self.stats.max_blocking_ms.max(t.blocking_ms); + if profiling() { + eprintln!( + "catalog: {}snapshot of {} records, {} B: serialize {:.1} ms, write+sync {:.1} ms, log reset {:.1} ms, total {:.1} ms, blocking {:.1} ms", + if t.background { "background " } else { "" }, + t.records, + t.bytes, + t.serialize_ms, + t.write_sync_ms, + t.reset_ms, + t.total_ms, + t.blocking_ms + ); + } + } + + /// Where persistence time went so far. + pub fn stats(&self) -> PersistStats { + PersistStats { snapshot_running: self.pending.is_some(), ..self.stats } + } + /// The log is long enough to be worth compacting. pub fn wants_snapshot(&self) -> bool { - self.log_records >= self.policy.max_records || self.log_bytes >= self.policy.max_bytes + self.pending.is_none() + && self.log_records >= self.retry_at_records + && (self.log_records >= self.policy.max_records || self.log_bytes >= self.policy.max_bytes) } pub fn seq(&self) -> u64 { @@ -238,3 +502,16 @@ impl Journal { self.store.describe() } } + +impl Drop for Journal { + /// Wait for a background snapshot (never leave a writer behind that could replace a newer + /// snapshot of a journal reopened on the same files). The log is left whole: the snapshot's + /// records in it are skipped as stale on the next load. + fn drop(&mut self) { + if let Some(p) = self.pending.take() + && let Ok(Err(e)) = p.worker.join() + { + log::error!("catalog: background snapshot failed: {e}"); + } + } +} diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rs index 8241372e0..041c9c7d0 100644 --- a/crates/catalog/src/lib.rs +++ b/crates/catalog/src/lib.rs @@ -21,7 +21,7 @@ use std::collections::BTreeMap; use std::sync::Arc; pub use dates::{DateRun, GroupBy}; -pub use journal::{Journal, LoadReport, SnapshotPolicy}; +pub use journal::{Journal, LoadReport, PersistStats, SnapshotPolicy, SnapshotTiming}; pub use keywords::KeywordNode; use lightcraft_develop::DevelopSettings; pub use model::*; @@ -622,4 +622,6 @@ impl Catalog { #[cfg(test)] mod tests; #[cfg(test)] +mod tests_background; +#[cfg(test)] mod tests_journal; diff --git a/crates/catalog/src/model.rs b/crates/catalog/src/model.rs index 369f40415..aceadcdc2 100644 --- a/crates/catalog/src/model.rs +++ b/crates/catalog/src/model.rs @@ -93,6 +93,65 @@ pub enum Source { Demo { scene: u32 }, } +/// Copyright status (IPTC / XMP Rights Management `xmpRights:Marked`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum CopyrightStatus { + /// Not stated (no `xmpRights:Marked`). + #[default] + Unknown, + /// `xmpRights:Marked` = True. + Copyrighted, + /// `xmpRights:Marked` = False. + PublicDomain, +} + +impl CopyrightStatus { + pub const ALL: [CopyrightStatus; 3] = [CopyrightStatus::Unknown, CopyrightStatus::Copyrighted, CopyrightStatus::PublicDomain]; + /// `unknown` / `copyrighted` / `publicDomain` (also `public domain`, `public-domain`). + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "unknown" | "" => Some(CopyrightStatus::Unknown), + "copyrighted" => Some(CopyrightStatus::Copyrighted), + "publicdomain" => Some(CopyrightStatus::PublicDomain), + _ => None, + } + } + /// The id used by commands (`photo.setMeta`'s `copyrightStatus`). + pub fn id(self) -> &'static str { + match self { + CopyrightStatus::Unknown => "unknown", + CopyrightStatus::Copyrighted => "copyrighted", + CopyrightStatus::PublicDomain => "publicDomain", + } + } + pub fn label(self) -> &'static str { + match self { + CopyrightStatus::Unknown => "Unknown", + CopyrightStatus::Copyrighted => "Copyrighted", + CopyrightStatus::PublicDomain => "Public Domain", + } + } + /// As `xmpRights:Marked`: `Some(true)` copyrighted, `Some(false)` public domain. + pub fn marked(self) -> Option { + match self { + CopyrightStatus::Unknown => None, + CopyrightStatus::Copyrighted => Some(true), + CopyrightStatus::PublicDomain => Some(false), + } + } + pub fn from_marked(m: Option) -> CopyrightStatus { + match m { + None => CopyrightStatus::Unknown, + Some(true) => CopyrightStatus::Copyrighted, + Some(false) => CopyrightStatus::PublicDomain, + } + } + pub fn is_unknown(&self) -> bool { + *self == CopyrightStatus::Unknown + } +} + /// Descriptive + capture metadata. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(default)] @@ -115,6 +174,13 @@ pub struct Meta { pub alt_text: String, pub extended_description: String, pub copyright: String, + /// Copyright status, rights usage terms and copyright info URL (IPTC Core rights fields). + #[serde(skip_serializing_if = "CopyrightStatus::is_unknown")] + pub copyright_status: CopyrightStatus, + #[serde(skip_serializing_if = "String::is_empty")] + pub usage_terms: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub copyright_url: String, pub creator: String, pub keywords: Vec, } diff --git a/crates/catalog/src/rules.rs b/crates/catalog/src/rules.rs index f4137cb61..26e3926dc 100644 --- a/crates/catalog/src/rules.rs +++ b/crates/catalog/src/rules.rs @@ -70,6 +70,7 @@ pub const FIELDS: &[(&str, &str, Kind)] = &[ ("location", "Location", Kind::Text), ("creator", "Creator", Kind::Text), ("copyright", "Copyright", Kind::Text), + ("copyrightStatus", "Copyright Status", Kind::Choice(&["copyrighted", "publicDomain", "unknown"])), ("captureDate", "Capture Date", Kind::Date), ("importDate", "Import Date", Kind::Date), ("editDate", "Edit Date", Kind::Date), @@ -311,6 +312,10 @@ impl Rule { "location" => text(&[m.location.as_str(), &m.city, &m.state, &m.country].join(" ")), "creator" => text(&m.creator), "copyright" => text(&m.copyright), + "copyrightStatus" => { + let want = crate::CopyrightStatus::parse(&want); + (want == Some(m.copyright_status)) == (op == "is") + } "captureDate" => date_op(op, p.captured.as_deref(), value), "importDate" => date_op(op, Some(&p.imported), value), "editDate" => date_op(op, p.edited.as_deref(), value), @@ -449,6 +454,13 @@ mod tests { yes(json!({"rules": [{"field": "captureDate", "op": "between", "value": ["2026-08-01", "2026-08"]}]})); no(json!({"rules": [{"field": "captureDate", "op": "after", "value": "2026-08"}]})); yes(json!({"rules": [{"field": "kind", "op": "isNot", "value": "video"}, {"field": "edited", "op": "is", "value": false}]})); + // copyright status (unknown until set) + yes(json!({"rules": [{"field": "copyrightStatus", "op": "is", "value": "unknown"}]})); + no(json!({"rules": [{"field": "copyrightStatus", "op": "is", "value": "copyrighted"}]})); + let mut pd = photo(2); + pd.meta.copyright_status = crate::CopyrightStatus::PublicDomain; + assert!(rs(json!({"rules": [{"field": "copyrightStatus", "op": "is", "value": "publicDomain"}]})).matches(&pd, &cat)); + assert!(rs(json!({"rules": [{"field": "copyrightStatus", "op": "isNot", "value": "copyrighted"}]})).matches(&pd, &cat)); // nested: rating ≥ 4 and (label is blue or keywords contain food) yes(json!({"rules": [{"field": "rating", "op": "gte", "value": 4}, {"group": {"match": "any", "rules": [ {"field": "label", "op": "is", "value": "blue"}, {"field": "keywords", "op": "contains", "value": "food"}]}}]})); diff --git a/crates/catalog/src/store.rs b/crates/catalog/src/store.rs index 6207173a0..069878c71 100644 --- a/crates/catalog/src/store.rs +++ b/crates/catalog/src/store.rs @@ -16,12 +16,30 @@ pub trait Store: Send { /// Replace a file atomically: after a crash either the old or the new content is visible, /// never a mix (native: write temp + fsync + rename + fsync directory). fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()>; + /// [`Store::write_atomic`] with streamed content: `fill` writes the new content into the + /// given writer (an error aborts and leaves the old file). Same durability: the file is + /// replaced only once all of it is written (native: and fsynced). Returns its length. + /// + /// The default buffers the content in memory; [`FsStore`] streams it into the temp file, so + /// a large snapshot is never held in memory as a whole. + fn write_atomic_with(&mut self, name: &str, fill: &mut dyn FnMut(&mut dyn io::Write) -> io::Result<()>) -> io::Result { + let mut buf = Vec::new(); + fill(&mut buf)?; + self.write_atomic(name, &buf)?; + Ok(buf.len() as u64) + } /// Append and make durable (fsync) before returning. fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()>; /// Cut a file to `len` bytes (drop a torn tail before appending again). fn truncate(&mut self, name: &str, len: u64) -> io::Result<()>; /// Human-readable location (diagnostics). fn describe(&self) -> String; + /// Another handle on the same files that a worker thread can write `catalog.snap` through + /// while this one keeps appending to the log (background compaction). `None` (the default): + /// snapshots are written synchronously. + fn background_writer(&self) -> Option> { + None + } } /// Files in a directory. @@ -77,6 +95,32 @@ impl Store for FsStore { Ok(()) } + fn write_atomic_with(&mut self, name: &str, fill: &mut dyn FnMut(&mut dyn io::Write) -> io::Result<()>) -> io::Result { + use std::io::Write; + if self.appender.as_ref().is_some_and(|(n, _)| n == name) { + self.appender = None; + } + let tmp = self.path(&format!("{name}.tmp")); + let written = (|| { + let mut w = Counter { inner: io::BufWriter::with_capacity(1 << 20, std::fs::File::create(&tmp)?), n: 0 }; + fill(&mut w)?; + w.flush()?; + let f = w.inner.into_inner().map_err(|e| e.into_error())?; + f.sync_all()?; + Ok(w.n) + })(); + let n = match written { + Ok(n) => n, + Err(e) => { + let _ = std::fs::remove_file(&tmp); + return Err(e); + } + }; + std::fs::rename(&tmp, self.path(name))?; + self.sync_dir(); + Ok(n) + } + fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> { use std::io::Write; if self.appender.as_ref().is_none_or(|(n, _)| n != name) { @@ -104,12 +148,36 @@ impl Store for FsStore { fn describe(&self) -> String { self.dir.display().to_string() } + + #[cfg(not(target_arch = "wasm32"))] + fn background_writer(&self) -> Option> { + Some(Box::new(FsStore { dir: self.dir.clone(), appender: None })) + } +} + +/// Counts the bytes written through it. +struct Counter { + inner: W, + n: u64, +} + +impl io::Write for Counter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let k = self.inner.write(buf)?; + self.n += k as u64; + Ok(k) + } + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } } /// In-memory files. Clones share the same files (so a test can "reopen" or corrupt them). #[derive(Clone, Default)] pub struct MemStore { pub files: Arc>>>, + /// Offer a [`Store::background_writer`] (off by default: snapshots stay synchronous). + pub background: bool, } impl MemStore { @@ -145,4 +213,7 @@ impl Store for MemStore { fn describe(&self) -> String { "memory".into() } + fn background_writer(&self) -> Option> { + self.background.then(|| Box::new(self.clone()) as Box) + } } diff --git a/crates/catalog/src/tests_background.rs b/crates/catalog/src/tests_background.rs new file mode 100644 index 000000000..0e06936e0 --- /dev/null +++ b/crates/catalog/src/tests_background.rs @@ -0,0 +1,338 @@ +//! Background compaction (issue #37): a snapshot written by a worker while appends go on. +//! "Crashes" are simulated by copying the files at a given moment and loading the copy: every +//! acknowledged (appended) op must be there, in order, whatever the snapshot was doing. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +use proptest::prelude::*; + +use crate::journal::{LOG, SNAPSHOT}; +use crate::*; + +/// A MemStore whose snapshot writes wait for a gate (to hold a background snapshot mid-flight) +/// and can be made to fail. +#[derive(Clone)] +struct Gated { + files: MemStore, + gate: Arc<(Mutex, Condvar)>, + fail: Arc, +} + +impl Gated { + fn new() -> Gated { + Gated { files: MemStore::new(), gate: Arc::new((Mutex::new(false), Condvar::new())), fail: Arc::new(AtomicBool::new(false)) } + } + fn set_gate(&self, open: bool) { + *self.gate.0.lock().unwrap() = open; + self.gate.1.notify_all(); + } + /// The files as a crash at this moment would leave them. + fn crash(&self) -> MemStore { + let m = MemStore::new(); + *m.files.lock().unwrap() = self.files.files.lock().unwrap().clone(); + m + } + fn snapshot_seq_on_disk(&self) -> Option { + let v: serde_json::Value = serde_json::from_slice(&self.files.get(SNAPSHOT)?).ok()?; + v["seq"].as_u64() + } +} + +impl Store for Gated { + fn read(&mut self, name: &str) -> std::io::Result>> { + self.files.read(name) + } + fn write_atomic(&mut self, name: &str, data: &[u8]) -> std::io::Result<()> { + self.files.write_atomic(name, data) + } + fn write_atomic_with(&mut self, name: &str, fill: &mut dyn FnMut(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result { + if name == SNAPSHOT { + let (lock, cv) = &*self.gate; + let (open, timeout) = cv.wait_timeout_while(lock.lock().unwrap(), Duration::from_secs(20), |open| !*open).unwrap(); + if !*open || timeout.timed_out() { + return Err(std::io::Error::other("gate never opened")); + } + } + let mut buf = Vec::new(); + fill(&mut buf)?; + if self.fail.load(Ordering::SeqCst) { + return Err(std::io::Error::other("disk full")); + } + self.files.write_atomic(name, &buf)?; + Ok(buf.len() as u64) + } + fn append(&mut self, name: &str, data: &[u8]) -> std::io::Result<()> { + self.files.append(name, data) + } + fn truncate(&mut self, name: &str, len: u64) -> std::io::Result<()> { + self.files.truncate(name, len) + } + fn describe(&self) -> String { + "gated".into() + } + fn background_writer(&self) -> Option> { + Some(Box::new(self.clone())) + } +} + +/// Apply and durably append one edit (two ops in one append every fourth time). +fn edit(j: &mut Journal, live: &mut Catalog, k: u64) { + let one = |live: &mut Catalog, k: u64| { + let op = if live.len() < 4 || k.is_multiple_of(3) { + let id = live.alloc_photo_id(); + Op::AddPhoto { photo: Box::new(Photo::new(id, Source::File { path: format!("/p/{}.jpg", id.0) }, "p.jpg", "JPEG", 4, 3, "t")) } + } else { + Op::SetRating { id: PhotoId(1 + k % live.len() as u64), rating: (k % 6) as u8 } + }; + live.apply(op.clone()).unwrap(); + op + }; + let mut ops = vec![one(live, k)]; + if k.is_multiple_of(4) { + ops.push(one(live, k / 4 + 1)); + } + j.append(&ops).unwrap(); +} + +/// Load the files a crash now would leave: they must hold exactly the live state. +fn assert_recovers(g: &Gated, live: &Catalog) -> LoadReport { + let (_, c, r) = Journal::open(Box::new(g.crash())).unwrap(); + assert_eq!(c.to_snapshot(), live.to_snapshot(), "{r:?}"); + assert_eq!((r.failed, r.torn_bytes, r.damaged.clone()), (0, 0, None)); + r +} + +fn wait_written(j: &Journal) { + for _ in 0..20_000 { + if j.snapshot_written() { + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + panic!("background snapshot never finished"); +} + +#[test] +fn background_snapshot_keeps_edits_made_while_it_runs() { + let g = Gated::new(); + let (mut j, mut live, _) = Journal::open(Box::new(g.clone())).unwrap(); + for k in 0..30 { + edit(&mut j, &mut live, k); + } + let n = j.seq(); + j.snapshot_in_background(&live).unwrap(); + assert!(j.snapshot_running() && !j.wants_snapshot() && j.stats().snapshot_running); + + // edits while the snapshot is being written are appended (durably) as usual + let mut during = 0; + for k in 30..50 { + edit(&mut j, &mut live, k); + during = j.seq() - n; + } + // crash before the snapshot landed: no snapshot yet, the whole log replays + let r = assert_recovers(&g, &live); + assert_eq!((r.snapshot_seq, r.replayed as u64), (0, j.seq())); + + // the snapshot lands (op n); crash before the log was trimmed: its records are stale + g.set_gate(true); + wait_written(&j); + assert_eq!(g.snapshot_seq_on_disk(), Some(n)); + for k in 50..55 { + edit(&mut j, &mut live, k); + during = j.seq() - n; + } + let r = assert_recovers(&g, &live); + assert_eq!((r.snapshot_seq, r.stale as u64, r.replayed as u64), (n, n, during)); + + // finishing trims the log to exactly the records after the snapshot, in order + assert!(j.poll().unwrap()); + assert!(!j.snapshot_running()); + assert_eq!((j.snapshot_seq(), j.log_records()), (n, during)); + let log = String::from_utf8(g.files.get(LOG).unwrap()).unwrap(); + let seqs: Vec = log.lines().map(|l| journal::decode_record(l).unwrap().0).collect(); + assert_eq!(seqs, ((n + 1)..=j.seq()).collect::>()); + assert_eq!(j.log_bytes(), log.len() as u64); + let r = assert_recovers(&g, &live); + assert_eq!((r.snapshot_seq, r.stale, r.replayed as u64), (n, 0, during)); + let s = j.stats(); + assert!(s.last_snapshot.background && s.last_snapshot.records == n && !s.snapshot_running, "{s:?}"); + + // and appending goes on after it + edit(&mut j, &mut live, 99); + assert_recovers(&g, &live); +} + +#[test] +fn failed_background_snapshot_loses_nothing_and_retries_later() { + let g = Gated::new(); + let (mut j, mut live, _) = Journal::open(Box::new(g.clone())).unwrap(); + j.policy = SnapshotPolicy { max_records: 10, max_bytes: u64::MAX }; + for k in 0..12 { + edit(&mut j, &mut live, k); + } + assert!(j.wants_snapshot()); + g.fail.store(true, Ordering::SeqCst); + g.set_gate(true); + j.snapshot_in_background(&live).unwrap(); + edit(&mut j, &mut live, 12); + wait_written(&j); + assert!(j.poll().is_err()); + assert_eq!((j.snapshot_seq(), j.stats().failed_snapshots), (0, 1)); + assert!(g.files.get(SNAPSHOT).is_none()); + assert_recovers(&g, &live); + // no retry storm: the next attempt waits for another `max_records` records + assert!(!j.wants_snapshot()); + g.fail.store(false, Ordering::SeqCst); + while !j.wants_snapshot() { + let k = j.seq(); + edit(&mut j, &mut live, k); + } + j.snapshot_in_background(&live).unwrap(); + j.wait().unwrap(); + assert_eq!((j.snapshot_seq(), j.log_records()), (j.seq(), 0)); + let r = assert_recovers(&g, &live); + assert_eq!((r.snapshot_seq, r.replayed), (j.seq(), 0)); +} + +/// A synchronous snapshot (close, "Optimize Library") waits for the background one, so the older +/// snapshot can't land after (and replace) the newer one. +#[test] +fn sync_snapshot_waits_for_the_background_one() { + let g = Gated::new(); + let (mut j, mut live, _) = Journal::open(Box::new(g.clone())).unwrap(); + for k in 0..10 { + edit(&mut j, &mut live, k); + } + j.snapshot_in_background(&live).unwrap(); + for k in 10..20 { + edit(&mut j, &mut live, k); + } + let opener = { + let g = g.clone(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(50)); + g.set_gate(true); + }) + }; + j.snapshot(&live).unwrap(); + opener.join().unwrap(); + assert!(!j.snapshot_running()); + assert_eq!(g.snapshot_seq_on_disk(), Some(j.seq())); + assert_eq!((j.snapshot_seq(), j.log_records()), (j.seq(), 0)); + assert!(g.files.get(LOG).unwrap().is_empty()); + assert_eq!(j.stats().snapshots, 2); + assert_recovers(&g, &live); +} + +/// Dropping the journal (e.g. reopening the library) waits for the snapshot in flight and leaves +/// the log whole: reopening finds every op. +#[test] +fn dropping_the_journal_waits_for_its_snapshot() { + let g = Gated::new(); + let (mut j, mut live, _) = Journal::open(Box::new(g.clone())).unwrap(); + for k in 0..10 { + edit(&mut j, &mut live, k); + } + let n = j.seq(); + j.snapshot_in_background(&live).unwrap(); + for k in 10..15 { + edit(&mut j, &mut live, k); + } + let opener = { + let g = g.clone(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(50)); + g.set_gate(true); + }) + }; + let total = j.seq(); + drop(j); + opener.join().unwrap(); + assert_eq!(g.snapshot_seq_on_disk(), Some(n)); + let (_, c, r) = Journal::open(Box::new(g.files.clone())).unwrap(); + assert_eq!(c.to_snapshot(), live.to_snapshot()); + assert_eq!((r.snapshot_seq, r.stale as u64, r.replayed as u64), (n, n, total - n)); +} + +/// On disk, with a real worker thread: a crash mid-write leaves only a temp file, which loading +/// ignores; edits during the snapshot survive; reopening resumes appends. +#[test] +fn fs_store_background_snapshot() { + let dir = std::env::temp_dir().join(format!("lc-journal-bg-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let (mut j, mut live, _) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap(); + for k in 0..200 { + edit(&mut j, &mut live, k); + } + // a previous crash mid-snapshot left a partial temp file + std::fs::write(dir.join(format!("{SNAPSHOT}.tmp")), b"{\"format\":\"lightcraft-catal").unwrap(); + let (mut j2, c2, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap(); + assert_eq!((c2.to_snapshot(), r.snapshot_seq), (live.to_snapshot(), 0)); + drop(j); + let n = j2.seq(); + j2.snapshot_in_background(&live).unwrap(); + for k in 200..260 { + edit(&mut j2, &mut live, k); + } + j2.wait().unwrap(); + assert_eq!((j2.snapshot_seq(), j2.log_records()), (n, j2.seq() - n)); + assert!(!dir.join(format!("{SNAPSHOT}.tmp")).exists()); + edit(&mut j2, &mut live, 999); + drop(j2); + let (_, c3, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap(); + assert_eq!(c3.to_snapshot(), live.to_snapshot()); + assert_eq!((r.snapshot_seq, r.stale), (n, 0)); + let _ = std::fs::remove_dir_all(&dir); +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(48))] + + /// Any interleaving of edits, background snapshots (started, landed, finished), synchronous + /// snapshots and crashes: a crash at any point recovers exactly the acknowledged ops. + #[test] + fn background_compaction_never_loses_acknowledged_ops(actions in proptest::collection::vec((0u8..8, 0u64..1000), 1..60)) { + let g = Gated::new(); + let (mut j, mut live, _) = Journal::open(Box::new(g.clone())).unwrap(); + for (ctl, k) in actions { + match ctl { + 0..=2 => edit(&mut j, &mut live, k), + 3 => j.snapshot_in_background(&live).unwrap(), + 4 | 5 => { + // the worker writes its snapshot; finish it now or later + if j.snapshot_running() { + g.set_gate(true); + wait_written(&j); + g.set_gate(false); + if ctl == 4 { + prop_assert!(j.poll().unwrap()); + } + } + } + 6 => { + g.set_gate(true); + j.snapshot(&live).unwrap(); + g.set_gate(false); + prop_assert_eq!(g.snapshot_seq_on_disk(), Some(j.seq())); + } + _ => { + // a crash now + let (_, c, r) = Journal::open(Box::new(g.crash())).unwrap(); + prop_assert_eq!(c.to_snapshot(), live.to_snapshot()); + prop_assert_eq!(r.failed, 0); + prop_assert!(r.damaged.is_none()); + } + } + // the snapshot on disk never covers more than what was appended + prop_assert!(g.snapshot_seq_on_disk().unwrap_or(0) <= j.seq()); + } + g.set_gate(true); + let (_, c, _) = Journal::open(Box::new(g.crash())).unwrap(); + prop_assert_eq!(c.to_snapshot(), live.to_snapshot()); + drop(j); + let (_, c, _) = Journal::open(Box::new(g.files.clone())).unwrap(); + prop_assert_eq!(c.to_snapshot(), live.to_snapshot()); + } +} diff --git a/crates/catalog/src/tests_journal.rs b/crates/catalog/src/tests_journal.rs index ec38af574..6bf7c85c2 100644 --- a/crates/catalog/src/tests_journal.rs +++ b/crates/catalog/src/tests_journal.rs @@ -274,3 +274,48 @@ fn fs_store_roundtrip() { assert_eq!(c3.to_snapshot(), c.to_snapshot()); let _ = std::fs::remove_dir_all(&dir); } + +/// The streamed snapshot is byte-for-byte the format written before streaming (issue #37), so +/// existing libraries load and older builds can read new snapshots; FsStore and MemStore agree. +#[test] +fn streamed_snapshot_keeps_the_on_disk_format() { + let (m, _, full) = sample_catalog_with_log(); + let (mut j, c, _) = open(&m); + assert_eq!(c.to_snapshot(), full.to_snapshot()); + j.snapshot(&c).unwrap(); + let legacy = format!("{{\"format\":\"lightcraft-catalog\",\"version\":1,\"seq\":{},\"catalog\":{}}}\n", j.seq(), c.to_snapshot()); + assert_eq!(String::from_utf8(m.get(SNAPSHOT).unwrap()).unwrap(), legacy); + assert_eq!(j.stats().last_snapshot.bytes, legacy.len() as u64); + + let dir = std::env::temp_dir().join(format!("lc-journal-stream-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + // an old-style snapshot (written by a build before streaming) loads, and is rewritten the same + std::fs::write(dir.join(SNAPSHOT), legacy.as_bytes()).unwrap(); + let (mut fj, fc, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap(); + assert_eq!((fc.to_snapshot(), r.snapshot_seq), (c.to_snapshot(), j.seq())); + fj.snapshot(&fc).unwrap(); + assert_eq!(std::fs::read(dir.join(SNAPSHOT)).unwrap(), legacy.as_bytes()); + drop(fj); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A snapshot that fails while streaming (serialisation or I/O error) leaves the old file in +/// place and removes its temp file. +#[test] +fn failed_streamed_write_keeps_the_old_file() { + let dir = std::env::temp_dir().join(format!("lc-journal-streamfail-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let mut s = FsStore::open(&dir).unwrap(); + s.write_atomic(SNAPSHOT, b"old").unwrap(); + let r = s.write_atomic_with(SNAPSHOT, &mut |w| { + w.write_all(b"partial new content")?; + Err(std::io::Error::other("serialisation failed")) + }); + assert!(r.is_err()); + assert_eq!(std::fs::read(dir.join(SNAPSHOT)).unwrap(), b"old"); + assert!(!dir.join(format!("{SNAPSHOT}.tmp")).exists()); + assert_eq!(s.write_atomic_with(SNAPSHOT, &mut |w| w.write_all(b"new")).unwrap(), 3); + assert_eq!(std::fs::read(dir.join(SNAPSHOT)).unwrap(), b"new"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index ebbc73c6b..8d1d5497e 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -1,7 +1,7 @@ //! Library commands: view source, filter/sort, selection, ratings/flags/labels, rotate, delete, //! metadata, albums, import. -use lightcraft_catalog::{Album, AlbumId, ColorLabel, Flag, GroupBy, Op, PhotoId, Sort, SortKey}; +use lightcraft_catalog::{Album, AlbumId, ColorLabel, CopyrightStatus, Flag, GroupBy, Op, PhotoId, Sort, SortKey}; use serde_json::{Value, json}; use super::{CommandSpec, always, bad, bool_or, cmd, has_active, has_selection, ok, str_param}; @@ -252,11 +252,12 @@ pub fn specs() -> Vec { cmd!(query "library.devices", "Cameras and Cards", [], None, "{} → [{name, path (its DCIM folder), root}] — mounted volumes with a DCIM folder", always, |_, _| { Ok(serde_json::to_value(crate::devices::devices_now()).unwrap_or_default()) }), - cmd!(query "photo.copyMetadata", "Copy Metadata", ["Photo"], None, "{} — title, caption, copyright, creator, location and keywords of the active photo", has_active, |s, _| { + cmd!(query "photo.copyMetadata", "Copy Metadata", ["Photo"], None, "{} — title, caption, copyright (notice, status, usage terms, info URL), creator, location and keywords of the active photo", has_active, |s, _| { let id = s.active().ok_or_else(|| bad("photo.copyMetadata", "no active photo"))?; let m = &s.catalog.photo(id).ok_or_else(|| bad("photo.copyMetadata", "no photo"))?.meta; let v = json!({"title": m.title, "caption": m.caption, "altText": m.alt_text, "extendedDescription": m.extended_description, - "copyright": m.copyright, "creator": m.creator, "location": m.location, "city": m.city, "state": m.state, "country": m.country, + "copyright": m.copyright, "copyrightStatus": m.copyright_status.id(), "usageTerms": m.usage_terms, "copyrightUrl": m.copyright_url, + "creator": m.creator, "location": m.location, "city": m.city, "state": m.state, "country": m.country, "keywords": m.keywords}); s.meta_clipboard = Some(v.clone()); Ok(v) @@ -266,7 +267,7 @@ pub fn specs() -> Vec { "Paste Metadata", ["Photo"], None, - "{ids?, fields?: [title|caption|copyright|creator|location|keywords] (default: all copied)}", + "{ids?, fields?: [title|caption|copyright|copyrightStatus|usageTerms|copyrightUrl|creator|location|keywords] (default: all copied)}", has_selection, |s, p| { let c = "photo.pasteMetadata"; @@ -370,7 +371,7 @@ pub fn specs() -> Vec { "Edit Info", [], None, - "{ids?, title?, caption?, altText?, extendedDescription?, copyright?, creator?, location?, city?, state?, country?, gps?: \"lat, lon\" | [lat, lon] | null, keywords?: [..], addKeywords?: [..], removeKeywords?: [..]}", + "{ids?, title?, caption?, altText?, extendedDescription?, copyright?, copyrightStatus?: unknown|copyrighted|publicDomain, usageTerms?, copyrightUrl?, creator?, location?, city?, state?, country?, gps?: \"lat, lon\" | [lat, lon] | null, keywords?: [..], addKeywords?: [..], removeKeywords?: [..]}", has_selection, |s, p| { let strs = @@ -390,6 +391,13 @@ pub fn specs() -> Vec { }, Some(_) => return Err(bad("photo.setMeta", "gps is \"lat, lon\", [lat, lon] or null")), }; + let status = match str_param(p, "copyrightStatus") { + Some(t) => Some( + CopyrightStatus::parse(t) + .ok_or_else(|| bad("photo.setMeta", format!("copyrightStatus `{t}`: unknown, copyrighted or publicDomain")))?, + ), + None => None, + }; let mut ops = Vec::new(); for id in targets { let Some(ph) = s.catalog.photo(id) else { continue }; @@ -397,10 +405,15 @@ pub fn specs() -> Vec { if let Some(g) = gps { m.gps = g; } + if let Some(st) = status { + m.copyright_status = st; + } for (k, field) in [ ("title", &mut m.title), ("caption", &mut m.caption), ("copyright", &mut m.copyright), + ("usageTerms", &mut m.usage_terms), + ("copyrightUrl", &mut m.copyright_url), ("creator", &mut m.creator), ("location", &mut m.location), ("city", &mut m.city), @@ -854,6 +867,7 @@ pub fn specs() -> Vec { "logRecords": j.log_records(), "logBytes": j.log_bytes(), "lastError": lib.last_error, + "persistence": j.stats(), "cache": cache, "load": { "created": lib.report.created, diff --git a/crates/engine/src/cmd/manage.rs b/crates/engine/src/cmd/manage.rs index 8b9ee8a24..41b43cab9 100644 --- a/crates/engine/src/cmd/manage.rs +++ b/crates/engine/src/cmd/manage.rs @@ -59,6 +59,97 @@ fn set_names_ops(s: &crate::Session, names: &[String; 5]) -> Vec { .collect() } +/// Geotag photos from a GPX track log by capture time (Lightroom Classic's Map ▸ Tracklog ▸ +/// Auto-Tag Photos). GPX times are UTC; capture times are the camera's local clock, so a photo's +/// recorded zone (Exif `OffsetTimeOriginal`) or the `offset` parameter converts them. +fn auto_tag_tracklog(s: &mut crate::Session, p: &Value) -> Result { + use lightcraft_meta::{DateTime, Match, parse_gpx}; + const C: &str = "photo.autoTagTracklog"; + let text = match (str_param(p, "gpx"), str_param(p, "path")) { + (Some(t), _) => t.to_string(), + (None, Some(path)) => std::fs::read_to_string(path).map_err(|e| bad(C, format!("can't read {path}: {e}")))?, + (None, None) => return Err(bad(C, "missing `path` (a .gpx file) or `gpx` (its text)")), + }; + let log = parse_gpx(&text).map_err(|e| bad(C, e.to_string()))?; + let (start, end) = log.span().ok_or_else(|| bad(C, "the track log has no timed track points"))?; + // minutes east of UTC; `None` = not given (zone-less capture times are then read as UTC) + let offset: Option = match p.get("offset") { + None | Some(Value::Null) => None, + Some(Value::String(o)) if o.trim().is_empty() => None, + Some(Value::Number(h)) => { + let h = h.as_f64().unwrap_or(0.0); + if h.abs() > 18.0 { + return Err(bad(C, "offset is at most ±18 hours")); + } + Some((h * 60.0).round() as i64) + } + Some(Value::String(o)) => { + Some(DateTime::parse_offset(o).ok_or_else(|| bad(C, format!("can't read offset `{o}` (e.g. +02:00 or -5)")))? as i64) + } + Some(_) => return Err(bad(C, "offset is `+HH:MM` or a number of hours")), + }; + let max_gap = f64_or(p, "maxGap", 600.0); + if max_gap.is_nan() || max_gap < 0.0 { + return Err(bad(C, "maxGap must be ≥ 0 seconds")); + } + let replace = bool_or(p, "replace", false); + let (mut no_time, mut outside, mut has_gps, mut assumed_utc, mut interpolated) = (0, 0, 0, 0, 0); + let mut ops = Vec::new(); + let mut photos = Vec::new(); + for id in s.targets(p) { + let Some(ph) = s.catalog.photo(id) else { continue }; + if ph.meta.gps.is_some() && !replace { + has_gps += 1; + continue; + } + let Some(dt) = ph.captured.as_deref().and_then(DateTime::parse_iso) else { + no_time += 1; + continue; + }; + // UTC seconds: the recorded zone wins, else the caller's offset + let utc = match dt.offset_minutes { + Some(_) => dt.unix_seconds(), + None => { + if offset.is_none() { + assumed_utc += 1; + } + dt.unix_seconds() - offset.unwrap_or(0) * 60 + } + } as f64 + + dt.millis as f64 / 1000.0; + let Some((g, m)) = log.locate(utc, max_gap) else { + outside += 1; + continue; + }; + if m == Match::Interpolated { + interpolated += 1; + } + let pos = ((g.latitude * 1e7).round() / 1e7, (g.longitude * 1e7).round() / 1e7); + photos.push(json!({"id": id.0, "gps": [pos.0, pos.1], "match": if m == Match::Interpolated { "interpolated" } else { "nearest" }})); + if ph.meta.gps != Some(pos) { + let mut meta = ph.meta.clone(); + meta.gps = Some(pos); + ops.push(Op::SetMeta { id, meta: Box::new(meta) }); + } + } + let tagged = photos.len(); + if !bool_or(p, "dryRun", false) && !ops.is_empty() { + s.commit("Auto-Tag from Tracklog", Op::Batch { ops })?; + } + let iso = |t: f64| format!("{}Z", lightcraft_catalog::dates::civil(t.floor() as i64)); + Ok(json!({ + "tagged": tagged, + "interpolated": interpolated, + "nearest": tagged - interpolated, + "skipped": {"noTime": no_time, "outside": outside, "hasGps": has_gps}, + "assumedUtc": assumed_utc, + "points": log.points.len(), + "start": iso(start), + "end": iso(end), + "photos": photos, + })) +} + pub fn specs() -> Vec { vec![ cmd!(query "photo.renameTokens", "Rename Template Tags", [], None, "{} → {tokens: [{tag, aliases, meaning, example}], dateDirectives: [{directive, meaning}], notes: [..], sample} — the file-name template tags shared by photo.rename, library.import (rename) and app.export (naming); examples are for a sample photo", always, |_, _| { @@ -130,6 +221,15 @@ pub fn specs() -> Vec { Ok(json!({"changed": n, "captured": out})) } ), + cmd!( + "photo.autoTagTracklog", + "Auto-Tag Photos from Tracklog", + [], + None, + "{path?: GPX file | gpx?: GPX text, ids?, offset?: camera clock's UTC offset for photos whose capture time has no zone (`+02:00`, or hours; default UTC), maxGap?: seconds (600), replace?: bool (also photos that already have GPS), dryRun?: bool} → {tagged, interpolated, nearest, skipped: {noTime, outside, hasGps}, assumedUtc, points, start, end, photos: [{id, gps, match}]} — positions from the track log by capture time; one undo step", + has_selection, + auto_tag_tracklog + ), cmd!(query "label.names", "Color Label Names", [], None, "{} → [{label, name, custom}]", always, |s, _| { Ok(json!(ColorLabel::ALL .iter() diff --git a/crates/engine/src/cmd/metadata.rs b/crates/engine/src/cmd/metadata.rs index ca306f299..ca52f6813 100644 --- a/crates/engine/src/cmd/metadata.rs +++ b/crates/engine/src/cmd/metadata.rs @@ -1,7 +1,7 @@ //! Metadata presets: named sets of descriptive fields (copyright, creator, place, keywords…) //! applied to photos in one step or to every import (Settings → Import). Saved with the library. -use lightcraft_catalog::Meta; +use lightcraft_catalog::{CopyrightStatus, Meta}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value, json}; @@ -9,7 +9,20 @@ use super::{CommandSpec, always, bad, cmd, str_param}; use crate::{Result, Session}; /// The fields a metadata preset can hold (photo.setMeta names). -pub const FIELDS: &[&str] = &["title", "caption", "copyright", "creator", "location", "city", "state", "country", "keywords"]; +pub const FIELDS: &[&str] = &[ + "title", + "caption", + "copyright", + "copyrightStatus", + "usageTerms", + "copyrightUrl", + "creator", + "location", + "city", + "state", + "country", + "keywords", +]; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct MetadataPreset { @@ -23,6 +36,8 @@ fn text_field<'a>(m: &'a mut Meta, k: &str) -> Option<&'a mut String> { "title" => &mut m.title, "caption" => &mut m.caption, "copyright" => &mut m.copyright, + "usageTerms" => &mut m.usage_terms, + "copyrightUrl" => &mut m.copyright_url, "creator" => &mut m.creator, "location" => &mut m.location, "city" => &mut m.city, @@ -35,7 +50,11 @@ fn text_field<'a>(m: &'a mut Meta, k: &str) -> Option<&'a mut String> { /// Apply preset `fields` to `m`: text fields replace, keywords are added. pub fn apply_to(m: &mut Meta, fields: &Value) { for (k, v) in fields.as_object().into_iter().flatten() { - if k == "keywords" { + if k == "copyrightStatus" { + if let Some(st) = v.as_str().and_then(CopyrightStatus::parse) { + m.copyright_status = st; + } + } else if k == "keywords" { for kw in v.as_array().into_iter().flatten().filter_map(Value::as_str) { if !m.keywords.iter().any(|x| x.eq_ignore_ascii_case(kw)) { m.keywords.push(kw.to_string()); @@ -59,6 +78,9 @@ fn save(s: &mut Session, p: &Value) -> Result { if let Some(k) = o.keys().find(|k| !FIELDS.contains(&k.as_str())) { return Err(bad(C, format!("unknown field `{k}` ({})", FIELDS.join(", ")))); } + if let Some(v) = o.get("copyrightStatus").filter(|v| v.as_str().and_then(CopyrightStatus::parse).is_none()) { + return Err(bad(C, format!("copyrightStatus {v}: unknown, copyrighted or publicDomain"))); + } Value::Object(o.clone()) } Some(_) => return Err(bad(C, "`fields` must be an object")), @@ -66,17 +88,24 @@ fn save(s: &mut Session, p: &Value) -> Result { None => { let id = s.active().ok_or_else(|| bad(C, "no `fields` and no active photo"))?; let mut m = s.catalog.photo(id).ok_or_else(|| bad(C, "no photo"))?.meta.clone(); - let only: Vec = p - .get("only") - .and_then(Value::as_array) - .map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()) - .unwrap_or_else(|| ["copyright", "creator", "location", "city", "state", "country"].map(String::from).to_vec()); + let only: Vec = + p.get("only").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()).unwrap_or_else( + || { + ["copyright", "copyrightStatus", "usageTerms", "copyrightUrl", "creator", "location", "city", "state", "country"] + .map(String::from) + .to_vec() + }, + ); let mut o = Map::new(); for k in only.iter().filter(|k| FIELDS.contains(&k.as_str())) { if k == "keywords" { if !m.keywords.is_empty() { o.insert(k.clone(), json!(m.keywords)); } + } else if k == "copyrightStatus" { + if !m.copyright_status.is_unknown() { + o.insert(k.clone(), json!(m.copyright_status.id())); + } } else if let Some(v) = text_field(&mut m, k).filter(|v| !v.trim().is_empty()) { o.insert(k.clone(), json!(v.clone())); } @@ -136,7 +165,7 @@ pub fn specs() -> Vec { "Save Metadata Preset", [], None, - "{name, fields?: {copyright?, creator?, title?, caption?, location?, city?, state?, country?, keywords?: [..]}, only?: [field] (from the active photo when `fields` is absent; default copyright, creator, place)} — adds or replaces", + "{name, fields?: {copyright?, copyrightStatus?: unknown|copyrighted|publicDomain, usageTerms?, copyrightUrl?, creator?, title?, caption?, location?, city?, state?, country?, keywords?: [..]}, only?: [field] (from the active photo when `fields` is absent; default the copyright fields, creator, place)} — adds or replaces", always, save ), diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rs index ed9490db1..70f89a7ad 100644 --- a/crates/engine/src/export.rs +++ b/crates/engine/src/export.rs @@ -853,7 +853,16 @@ fn parse_shutter(s: &str) -> Option { pub fn export_metadata(photo: &lightcraft_catalog::Photo, o: &ExportOptions) -> Option { let m = &photo.meta; let text = |s: &str| (!s.trim().is_empty()).then(|| s.to_string()); - let mut out = Metadata { copyright: text(&m.copyright), artist: text(&m.creator), software: Some("LightCraft".into()), ..Default::default() }; + // copyright info (also under "copyright only"): notice, creator, status, usage terms, info URL + let mut out = Metadata { + copyright: text(&m.copyright), + copyright_marked: m.copyright_status.marked(), + usage_terms: text(&m.usage_terms), + copyright_url: text(&m.copyright_url), + artist: text(&m.creator), + software: Some("LightCraft".into()), + ..Default::default() + }; match o.metadata { MetadataPolicy::None => return None, MetadataPolicy::Copyright => return Some(out), @@ -1169,6 +1178,9 @@ mod tests { let mut p = Photo::new(PhotoId(1), Source::Demo { scene: 0 }, "a.jpg", "jpeg", 10, 10, "2026-09-30T00:00:00"); p.meta.camera = "Synthetic X2".into(); p.meta.copyright = "(c) Me".into(); + p.meta.copyright_status = lightcraft_catalog::CopyrightStatus::Copyrighted; + p.meta.usage_terms = "Editorial use only".into(); + p.meta.copyright_url = "https://example.com/rights".into(); p.meta.shutter = "1/250".into(); p.meta.gps = Some((43.0, -110.0)); let all = export_metadata(&p, &ExportOptions::default()).unwrap(); @@ -1180,12 +1192,19 @@ mod tests { assert!(m.model.is_none() && m.gps.is_none() && m.copyright.is_some()); let c = export_metadata(&p, &ExportOptions { metadata: MetadataPolicy::Copyright, ..Default::default() }).unwrap(); assert!(c.model.is_none() && c.gps.is_none() && c.copyright.as_deref() == Some("(c) Me")); + // "copyright only" keeps all the copyright info: status, usage terms, info URL + assert_eq!( + (c.copyright_marked, c.usage_terms.as_deref(), c.copyright_url.as_deref()), + (Some(true), Some("Editorial use only"), Some("https://example.com/rights")) + ); assert!(export_metadata(&p, &ExportOptions { metadata: MetadataPolicy::None, ..Default::default() }).is_none()); // embedded and readable back from the JPEG let jpg = encode_with_metadata(&test_image(), &ExportOptions::default(), Some(&all)).unwrap(); let back = lightcraft_meta::extract(&jpg); assert_eq!(back.model.as_deref(), Some("Synthetic X2")); assert_eq!(back.copyright.as_deref(), Some("(c) Me")); + assert_eq!((back.copyright_marked, back.usage_terms.as_deref()), (Some(true), Some("Editorial use only"))); + assert_eq!(back.copyright_url.as_deref(), Some("https://example.com/rights")); assert!(back.gps.is_some()); } diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index 94a6ffc4f..32296c5af 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -33,6 +33,9 @@ fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option) { title: m.title.clone().unwrap_or_default(), caption: m.caption.clone().unwrap_or_default(), copyright: m.copyright.clone().unwrap_or_default(), + copyright_status: lightcraft_catalog::CopyrightStatus::from_marked(m.copyright_marked), + usage_terms: m.usage_terms.clone().unwrap_or_default(), + copyright_url: m.copyright_url.clone().unwrap_or_default(), creator: m.artist.clone().unwrap_or_default(), keywords: m.keywords.clone(), }; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 2b1973d00..d32dfe278 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -73,6 +73,9 @@ pub struct Interaction { pub original: Arc, } +/// Source of [`Session::visible_shared`] generations (process-wide, so two sessions never share one). +static VISIBLE_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + pub struct Session { /// Auto Sync: edits to the active photo also change the other selected photos (the settings /// that changed, nothing else). @@ -83,8 +86,12 @@ pub struct Session { pub sort: Sort, pub selection: Selection, /// The grid order for the current source/filter/sort (cached by catalog revision). - visible: Vec, + /// Shared, so frontends can hold it across frames without copying ([`Session::visible_shared`]). + visible: Arc<[PhotoId]>, visible_key: Option<(u64, String)>, + /// Identifies the current `visible` list: a new value (unique in the process) every time it + /// is recomputed, so frontends can key their per-view caches on it instead of hashing the ids. + visible_gen: u64, /// Photos in the current source with no filter on (cached like `visible`). total: Option<((u64, String), usize)>, pub undo: Vec, @@ -178,8 +185,9 @@ impl Session { filter: Filter::default(), sort: Sort::default(), selection: Selection::default(), - visible: Vec::new(), + visible: Vec::new().into(), visible_key: None, + visible_gen: 0, total: None, undo: Vec::new(), redo: Vec::new(), @@ -487,7 +495,7 @@ impl Session { f.folder = Some(b.path); f.subfolders = b.subfolders; } - self.visible = self.catalog.query(&f, &self.sort); + let mut visible = self.catalog.query(&f, &self.sort); if matches!(self.source, LibrarySource::Album(_)) && self.sort.key == lightcraft_catalog::SortKey::CaptureDate && let LibrarySource::Album(a) = self.source @@ -496,15 +504,15 @@ impl Session { && self.filter == Filter::default() { let order = al.photos.clone(); - self.visible.sort_by_key(|id| order.iter().position(|x| x == id).unwrap_or(usize::MAX)); + visible.sort_by_key(|id| order.iter().position(|x| x == id).unwrap_or(usize::MAX)); if !self.sort.ascending { - self.visible.reverse(); + visible.reverse(); } } if self.source == LibrarySource::RecentlyAdded { // newest import first, whatever the sort (the grid groups by import day) let cat = &self.catalog; - self.visible.sort_by(|a, b| { + visible.sort_by(|a, b| { let key = |id: &PhotoId| cat.photo(*id).map(|p| p.imported.clone()).unwrap_or_default(); key(b).cmp(&key(a)).then(a.cmp(b)) }); @@ -512,11 +520,13 @@ impl Session { if self.source == LibrarySource::Missing { // only the photos the query kept (library photos, not Local browse records) are checked let cat = &self.catalog; - self.visible.retain(|id| cmd::missing::is_missing(cat, *id)); + visible.retain(|id| cmd::missing::is_missing(cat, *id)); } if self.source != LibrarySource::RecentlyDeleted { - self.visible = self.catalog.arrange_stacks(&self.visible); + visible = self.catalog.arrange_stacks(&visible); } + self.visible = visible.into(); + self.visible_gen = VISIBLE_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed); self.visible_key = Some(key); } &self.visible @@ -546,6 +556,14 @@ impl Session { self.visible().to_vec() } + /// The visible photos without copying them, with their generation: equal generations mean + /// the very same list (same catalog revision, source, filter, sort), so a frontend can cache + /// whatever it derives from the list under that number. + pub fn visible_shared(&mut self) -> (u64, Arc<[PhotoId]>) { + self.visible(); + (self.visible_gen, self.visible.clone()) + } + /// Targets of photo commands: explicit `ids`/`id` param, else the selection. pub fn targets(&self, p: &Value) -> Vec { if let Some(a) = p.get("ids").and_then(Value::as_array) { diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs index 6644efa35..24660ea51 100644 --- a/crates/engine/src/library.rs +++ b/crates/engine/src/library.rs @@ -15,7 +15,8 @@ //! //! Every top-level [`Session::execute`] persists the ops it produced (fsynced) before returning, //! so a crash loses at most the command in flight. The log is compacted into a snapshot when it -//! grows (see [`lightcraft_catalog::SnapshotPolicy`]) and on [`Session::close_library`]. +//! grows (see [`lightcraft_catalog::SnapshotPolicy`]; written by a worker thread on native, see +//! [`lightcraft_catalog::journal`]) and on [`Session::close_library`]. use std::path::{Path, PathBuf}; @@ -91,6 +92,10 @@ impl Library { pub fn journal(&self) -> &Journal { &self.journal } + #[cfg(test)] + pub(crate) fn journal_mut(&mut self) -> &mut Journal { + &mut self.journal + } pub fn thumbs_dir(&self) -> PathBuf { self.dir.join("thumbs") } @@ -141,6 +146,15 @@ fn presets_json(s: &Session) -> String { serde_json::to_string_pretty(&f).unwrap_or_default() } +/// Finish a background compaction that is done (cheap otherwise). A failed one lost nothing (the +/// log is kept whole) and is retried later; it is reported like other persistence errors. +fn poll_compaction(lib: &mut Library) { + if let Err(e) = lib.journal.poll() { + log::error!("library: compaction: {e}"); + lib.last_error = Some(format!("compaction: {e}")); + } +} + fn read_json(store: &mut dyn Store, name: &str) -> Option { store.read(name).ok().flatten().and_then(|b| serde_json::from_slice(&b).ok()) } @@ -160,6 +174,12 @@ impl Session { pub fn open_library_in(&mut self, stores: LibraryStores, seed_demo: bool) -> Result<&LoadReport> { let LibraryStores { dir, catalog, mut files, on_disk } = stores; self.media.smart_dir = on_disk.then(|| crate::smart::dir(&dir)); + // the current library may be these same files: let its background snapshot land first + if let Some(old) = self.library.as_mut() + && let Err(e) = old.journal.wait() + { + log::error!("library: {e}"); + } let (mut journal, catalog, report) = Journal::open(catalog)?; self.catalog = catalog; self.undo.clear(); @@ -245,9 +265,11 @@ impl Session { self.pending_log.clear(); lib.last_error = None; } + poll_compaction(lib); // Never snapshot mid-interaction: the catalog then holds an uncommitted preview value. + // The snapshot is written by a worker thread (natively); appends go on meanwhile. if lib.journal.wants_snapshot() && self.interaction.is_none() { - lib.journal.snapshot(&self.catalog)?; + lib.journal.snapshot_in_background(&self.catalog)?; } let presets = presets_json(self); let Some(lib) = self.library.as_mut() else { return Ok(()) }; @@ -266,6 +288,8 @@ impl Session { pub fn persist_if_dirty(&mut self) { if self.library.is_some() && !self.pending_log.is_empty() { let _ = self.persist(); + } else if let Some(lib) = self.library.as_mut() { + poll_compaction(lib); } } diff --git a/crates/engine/src/sidecar.rs b/crates/engine/src/sidecar.rs index 5deedd3fb..a49e5feef 100644 --- a/crates/engine/src/sidecar.rs +++ b/crates/engine/src/sidecar.rs @@ -74,6 +74,10 @@ pub struct SidecarData { pub title: Option, pub caption: Option, pub copyright: Option, + /// `xmpRights:Marked`: `Some(true)` copyrighted, `Some(false)` public domain. + pub copyright_marked: Option, + pub usage_terms: Option, + pub copyright_url: Option, pub creator: Option, pub location: Option, pub city: Option, @@ -105,6 +109,9 @@ pub fn parse_sidecar(xmp: &str, raw: bool) -> std::result::Result bool { (&sc.title, &mut m.title), (&sc.caption, &mut m.caption), (&sc.copyright, &mut m.copyright), + (&sc.usage_terms, &mut m.usage_terms), + (&sc.copyright_url, &mut m.copyright_url), (&sc.creator, &mut m.creator), (&sc.location, &mut m.location), (&sc.city, &mut m.city), @@ -174,6 +183,9 @@ pub fn merge_into(p: &mut Photo, sc: &SidecarData, now: &str) -> bool { *dst = v.clone(); } } + if let Some(marked) = sc.copyright_marked { + m.copyright_status = lightcraft_catalog::CopyrightStatus::from_marked(Some(marked)); + } if let Some(k) = &sc.keywords { m.keywords = k.clone(); } @@ -222,6 +234,9 @@ pub fn sidecar_packet(p: &Photo, cat: &lightcraft_catalog::Catalog) -> String { state: nz(&p.meta.state), country: nz(&p.meta.country), copyright: nz(&p.meta.copyright), + copyright_marked: p.meta.copyright_status.marked(), + usage_terms: nz(&p.meta.usage_terms), + copyright_url: nz(&p.meta.copyright_url), artist: nz(&p.meta.creator), keywords: p.meta.keywords.clone(), rating: Some(p.rating.min(5) as i8), diff --git a/crates/engine/src/tests_libops.rs b/crates/engine/src/tests_libops.rs index 5ac8e19d3..909414984 100644 --- a/crates/engine/src/tests_libops.rs +++ b/crates/engine/src/tests_libops.rs @@ -401,3 +401,68 @@ fn keyword_sets_and_recent_keywords() { s.execute("keyword.deleteSet", &json!({"name": "Wedding"})).unwrap(); assert_eq!(s.execute("keyword.sets", &json!({})).unwrap()["current"], "Recent Keywords"); } + +/// Auto-Tag from Tracklog: GPS positions from a GPX file by capture time — interpolated within a +/// segment, the camera's zone from the photo or `offset`; photos with GPS kept unless `replace`; +/// one undo step, replayed from the op log. +#[test] +fn tracklog_auto_tag() { + use lightcraft_catalog::{Op, PhotoId}; + const GPX: &str = r#" + + + + +"#; + let dir = temp_dir("tracklog"); + let gpx_path = dir.join("walk.gpx"); + std::fs::write(&gpx_path, GPX).unwrap(); + let mut s = Session::new(); + s.open_library(dir.join("lib"), true).unwrap(); + let ids: Vec = s.visible_cloned().iter().take(4).map(|p| p.0).collect(); + let set = |s: &mut Session, id: u64, t: &str| { + s.commit("t", Op::SetCaptured { id: PhotoId(id), captured: Some(t.into()) }).unwrap(); + }; + set(&mut s, ids[0], "2026-05-01T12:00:30"); // camera clock on +02:00, no zone recorded + set(&mut s, ids[1], "2026-05-01T12:01:00+02:00"); // zone recorded: exactly on the 2nd point + set(&mut s, ids[2], "2026-05-01T15:00:00"); // hours after the log ends + set(&mut s, ids[3], "2026-05-01T12:02:00"); + s.execute("photo.setMeta", &json!({"ids": [ids[3]], "gps": [10.0, 20.0]})).unwrap(); + let gps = |s: &Session, id: u64| s.catalog.photo(PhotoId(id)).unwrap().meta.gps; + let before: Vec<_> = ids.iter().map(|i| gps(&s, *i)).collect(); + let path = gpx_path.to_string_lossy().to_string(); + // a dry run reports without changing anything + let r = s.execute("photo.autoTagTracklog", &json!({"path": path, "ids": ids, "offset": "+02:00", "dryRun": true})).unwrap(); + assert_eq!((r["tagged"].as_u64(), r["interpolated"].as_u64()), (Some(2), Some(2)), "{r}"); + assert_eq!(r["skipped"], json!({"noTime": 0, "outside": 1, "hasGps": 1})); + assert_eq!((r["points"].as_u64(), r["start"].as_str()), (Some(3), Some("2026-05-01T10:00:00Z"))); + assert_eq!(ids.iter().map(|i| gps(&s, *i)).collect::>(), before); + // tag: one undo step + let undo_depth = s.undo.len(); + s.execute("photo.autoTagTracklog", &json!({"path": path, "ids": ids, "offset": 2})).unwrap(); + assert_eq!(s.undo.len(), undo_depth + 1); + assert_eq!(gps(&s, ids[0]), Some((46.0005, 7.001)), "interpolated half-way between the first two points"); + assert_eq!(gps(&s, ids[1]), Some((46.001, 7.002))); + assert_eq!(gps(&s, ids[2]), None); + assert_eq!(gps(&s, ids[3]), Some((10.0, 20.0)), "existing GPS kept without `replace`"); + // without an offset, zone-less capture times count as UTC (and are reported) + let r = s.execute("photo.autoTagTracklog", &json!({"gpx": GPX, "ids": [ids[0]], "replace": true, "dryRun": true})).unwrap(); + assert_eq!((r["tagged"].as_u64(), r["assumedUtc"].as_u64()), (Some(0), Some(1)), "{r}"); + // replace: the photo that had GPS gets the track's position (its last point, 12:02 local) + s.execute("photo.autoTagTracklog", &json!({"gpx": GPX, "ids": [ids[3]], "offset": "+02:00", "replace": true})).unwrap(); + assert_eq!(gps(&s, ids[3]), Some((46.002, 7.004))); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(gps(&s, ids[3]), Some((10.0, 20.0))); + // the op log replays the tags + let expect = s.catalog.to_snapshot(); + drop(s); + let mut s = Session::new(); + s.open_library(dir.join("lib"), false).unwrap(); + assert_eq!(s.catalog.to_snapshot(), expect); + // errors: no file, not GPX, no timed points, bad offset + assert!(s.execute("photo.autoTagTracklog", &json!({"ids": [ids[0]]})).is_err()); + assert!(s.execute("photo.autoTagTracklog", &json!({"gpx": "", "ids": [ids[0]]})).is_err()); + assert!(s.execute("photo.autoTagTracklog", &json!({"gpx": "", "ids": [ids[0]]})).is_err()); + assert!(s.execute("photo.autoTagTracklog", &json!({"gpx": GPX, "ids": [ids[0]], "offset": "noon"})).is_err()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/engine/src/tests_library.rs b/crates/engine/src/tests_library.rs index 6b8ee8d92..fd09fdc82 100644 --- a/crates/engine/src/tests_library.rs +++ b/crates/engine/src/tests_library.rs @@ -52,6 +52,57 @@ fn edits_survive_restart_without_close() { let _ = std::fs::remove_dir_all(&dir); } +/// Compaction at the threshold runs on a worker thread (issue #37): commands go on (and stay +/// durable) meanwhile, the frame loop's `persist_if_dirty` finishes it, and a crash at any time +/// keeps every command. +#[test] +fn threshold_compaction_runs_in_the_background() { + let dir = temp_dir("bg-compact"); + let mut s = open(&dir, true); + s.library.as_mut().unwrap().journal_mut().policy = lightcraft_catalog::SnapshotPolicy { max_records: 8, max_bytes: u64::MAX }; + let ids: Vec<_> = s.catalog.photos().map(|p| p.id).collect(); + let mut background = 0; + for k in 0..60usize { + s.selection = crate::Selection::single(ids[k % ids.len()]); + s.execute("photo.rate", &json!({"rating": k % 6})).unwrap(); + if s.execute("library.info", &json!({})).unwrap()["persistence"]["snapshotRunning"].as_bool().unwrap() { + background += 1; + } + if k == 30 { + // a crash while a compaction may be in flight + let expect = s.catalog.to_snapshot(); + let copy = temp_dir("bg-compact-crash"); + std::fs::create_dir_all(©).unwrap(); + for f in ["catalog.snap", "catalog.log"] { + if dir.join(f).exists() { + std::fs::copy(dir.join(f), copy.join(f)).unwrap(); + } + } + let s2 = open(©, true); + assert_eq!(s2.catalog.to_snapshot(), expect); + let _ = std::fs::remove_dir_all(©); + } + } + assert!(background > 0, "compactions ran in the background"); + for _ in 0..10_000 { + s.persist_if_dirty(); + if !s.library.as_ref().unwrap().journal().snapshot_running() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + let info = s.execute("library.info", &json!({})).unwrap(); + let p = &info["persistence"]; + assert!(p["snapshots"].as_u64() >= Some(2) && p["lastSnapshot"]["background"] == true, "{p}"); + // the log holds exactly the records after the snapshot + assert_eq!(info["logRecords"].as_u64().unwrap(), info["seq"].as_u64().unwrap() - info["snapshotSeq"].as_u64().unwrap(), "{info}"); + let expect = s.catalog.to_snapshot(); + drop(s); // no close: like a crash + let s2 = open(&dir, true); + assert_eq!(s2.catalog.to_snapshot(), expect); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn close_writes_snapshot_and_view_state() { let dir = temp_dir("close"); @@ -121,6 +172,12 @@ fn new_library_without_seed_is_empty_and_compacts() { let info = s.execute("library.info", &json!({})).unwrap(); assert_eq!(info["logRecords"], 0); assert_eq!(info["snapshotSeq"], info["seq"]); + // persistence timings (issue #37): the append and the compaction were measured + let p = &info["persistence"]; + assert!(p["appends"].as_u64() >= Some(1), "{p}"); + assert!(p["snapshots"].as_u64() >= Some(1), "{p}"); + assert!(p["lastSnapshot"]["bytes"].as_u64() > Some(0), "{p}"); + assert!(p["lastSnapshot"]["totalMs"].as_f64() >= p["lastSnapshot"]["serializeMs"].as_f64(), "{p}"); let s2 = open(&dir, true); assert_eq!(s2.catalog.len(), 0, "an existing library is never seeded"); assert_eq!(s2.catalog.albums().count(), 1); diff --git a/crates/engine/src/tests_xmp.rs b/crates/engine/src/tests_xmp.rs index c0331ea72..00b8e3b08 100644 --- a/crates/engine/src/tests_xmp.rs +++ b/crates/engine/src/tests_xmp.rs @@ -497,3 +497,82 @@ fn sidecar_capture_time_fills_in_when_the_file_has_none() { let _ = std::fs::remove_dir_all(&src); let _ = std::fs::remove_dir_all(&lib); } + +/// Copyright status, rights usage terms and copyright info URL: edited with `photo.setMeta` (one +/// undo step), copied / pasted, kept in metadata presets, written to the sidecar as XMP Rights +/// Management fields and read back on a fresh import. +#[test] +fn copyright_status_usage_terms_and_url_round_trip() { + use lightcraft_catalog::CopyrightStatus; + let src = temp_dir("rights-src"); + let lib = temp_dir("rights-lib"); + write_png(&src.join("a.png"), 1); + write_png(&src.join("b.png"), 2); + let mut s = Session::new().with_fs(); + s.open_library(&lib, false).unwrap(); + s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap(); + let id = |s: &Session, name: &str| s.catalog.photos().find(|p| p.file_name == name).unwrap().id; + let (a, b) = (id(&s, "a.png"), id(&s, "b.png")); + let meta = |s: &Session, id: PhotoId| s.catalog.photo(id).unwrap().meta.clone(); + assert_eq!(meta(&s, a).copyright_status, CopyrightStatus::Unknown); + s.execute("library.select", &json!({"ids": [a.0]})).unwrap(); + s.execute( + "photo.setMeta", + &json!({"copyright": "© 2026 A. Person", "copyrightStatus": "copyrighted", "usageTerms": "Editorial use only", "copyrightUrl": "https://example.com/rights"}), + ) + .unwrap(); + let m = meta(&s, a); + assert_eq!( + (m.copyright_status, m.usage_terms.as_str(), m.copyright_url.as_str()), + (CopyrightStatus::Copyrighted, "Editorial use only", "https://example.com/rights") + ); + assert!(s.execute("photo.setMeta", &json!({"copyrightStatus": "maybe"})).is_err()); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(meta(&s, a).copyright_status, CopyrightStatus::Unknown, "one undo step"); + s.execute("edit.redo", &json!({})).unwrap(); + // copy → paste onto b + let clip = s.execute("photo.copyMetadata", &json!({})).unwrap(); + assert_eq!(clip["copyrightStatus"], "copyrighted"); + s.execute("photo.pasteMetadata", &json!({"ids": [b.0], "fields": ["copyrightStatus", "usageTerms"]})).unwrap(); + let mb = meta(&s, b); + assert_eq!((mb.copyright_status, mb.usage_terms.as_str(), mb.copyright_url.as_str()), (CopyrightStatus::Copyrighted, "Editorial use only", "")); + // a metadata preset from the active photo carries the copyright fields; applying it sets them + s.execute("metadata.savePreset", &json!({"name": "Rights"})).unwrap(); + let presets = s.execute("metadata.presets", &json!({})).unwrap(); + assert_eq!(presets[0]["fields"]["copyrightStatus"], "copyrighted", "{presets}"); + assert_eq!(presets[0]["fields"]["copyrightUrl"], "https://example.com/rights"); + s.execute("photo.setMeta", &json!({"ids": [b.0], "copyrightStatus": "publicDomain", "usageTerms": ""})).unwrap(); + s.execute("metadata.applyPreset", &json!({"name": "Rights", "ids": [b.0]})).unwrap(); + let mb = meta(&s, b); + assert_eq!((mb.copyright_status, mb.copyright_url.as_str()), (CopyrightStatus::Copyrighted, "https://example.com/rights")); + assert!(s.execute("metadata.savePreset", &json!({"name": "Bad", "fields": {"copyrightStatus": "sort of"}})).is_err()); + // b becomes public domain; smart-album rule on the status + s.execute("photo.setMeta", &json!({"ids": [b.0], "copyrightStatus": "public domain"})).unwrap(); + let r = s + .execute( + "album.createSmart", + &json!({"name": "PD", "rules": {"ruleSet": {"rules": [{"field": "copyrightStatus", "op": "is", "value": "publicDomain"}]}}}), + ) + .unwrap(); + assert_eq!(r["count"], 1, "{r}"); + // sidecars: XMP Rights Management fields, read back by a fresh library + s.execute("photo.saveMetadataToFile", &json!({"ids": [a.0, b.0]})).unwrap(); + let xmp = std::fs::read_to_string(src.join("a.xmp")).unwrap(); + assert!(xmp.contains("True"), "{xmp}"); + assert!(xmp.contains("https://example.com/rights"), "{xmp}"); + assert!(std::fs::read_to_string(src.join("b.xmp")).unwrap().contains("False")); + let expect_a = meta(&s, a); + drop(s); + std::fs::remove_dir_all(&lib).unwrap(); + let mut s = Session::new().with_fs(); + s.open_library(&lib, false).unwrap(); + s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap(); + let ma = meta(&s, id(&s, "a.png")); + assert_eq!( + (ma.copyright.as_str(), ma.copyright_status, ma.usage_terms.as_str(), ma.copyright_url.as_str()), + (expect_a.copyright.as_str(), CopyrightStatus::Copyrighted, "Editorial use only", "https://example.com/rights") + ); + assert_eq!(meta(&s, id(&s, "b.png")).copyright_status, CopyrightStatus::PublicDomain); + let _ = std::fs::remove_dir_all(&src); + let _ = std::fs::remove_dir_all(&lib); +} diff --git a/crates/meta/src/gpx.rs b/crates/meta/src/gpx.rs new file mode 100644 index 000000000..a8e8e9071 --- /dev/null +++ b/crates/meta/src/gpx.rs @@ -0,0 +1,374 @@ +//! GPS track logs in GPX 1.0 / 1.1 (the open GPS Exchange Format, ): +//! the timed track points of every ``, and the position at any moment for geotagging photos. +//! +//! What we use from the format: `` → `` → `` with optional `` +//! (metres) and `