M1.3: import (recursive folders, content-hash duplicates, add vs copy)

- engine::import: expand files/folders recursively (sorted; hidden entries
  and the library folder skipped), skip known paths, probe + hash in
  parallel (std::thread::scope), skip duplicates by content hash (against
  the library and within the batch), add in place or copy into
  <library>/Originals/YYYY/YYYY-MM-DD/ with unique names; one undoable op.
- library.import {paths, mode: add|copy, album?} returns
  {imported, duplicates[{path, existing, reason}], failed, scanned}.
- probe_bytes records the content hash (SipHash-1-3 128) on every photo.
- system clock for import/edit times on native (Session::with_system_clock).
- app: command-line folders are expanded by the engine (re-importing the
  same folder is a no-op); example gen_photos writes procedural test JPEGs
  (never committed).
- Measured (200 procedural 2000 px JPEGs, M-series): import 2.25 s; reopen:
  library load 1.7 ms, control ready 0.12 s, first thumbnail 0.17 s,
  visible grid drawn 0.48 s (thumbnails from the disk cache).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
2026-10-01 00:04:07 -04:00
co-authored by Claude Opus 5.5
parent bcecab7d84
commit 759d3c7f8a
9 changed files with 469 additions and 85 deletions
+45
View File
@@ -0,0 +1,45 @@
//! Generate a folder of procedural test photos (JPEG) for import/startup benchmarks.
//!
//! `cargo run --release -p lightcraft --example gen_photos -- OUT_DIR [COUNT=200] [LONG_EDGE=2000]`
//!
//! Each photo is a `lightcraft-scenes` demo scene rendered through the pipeline with a varied
//! look (exposure, temperature, vibrance), so every file has distinct bytes. The output is our own
//! procedural content; keep it out of the repository (write to a scratch or gitignored folder).
use lightcraft_engine::Session;
use serde_json::json;
fn main() {
let mut args = std::env::args().skip(1);
let out = std::path::PathBuf::from(args.next().expect("usage: gen_photos OUT_DIR [COUNT] [LONG_EDGE]"));
let count: usize = args.next().and_then(|c| c.parse().ok()).unwrap_or(200);
let edge: usize = args.next().and_then(|c| c.parse().ok()).unwrap_or(2000);
std::fs::create_dir_all(&out).expect("create output folder");
let mut s = Session::with_demo();
let ids = s.visible_cloned();
let t0 = std::time::Instant::now();
for i in 0..count {
let id = ids[i % ids.len()];
let round = i / ids.len();
s.execute("library.select", &json!({"ids": [id.0]})).expect("select");
s.execute("develop.reset", &json!({})).expect("reset");
let v = json!({
"light.exposure": (round as f64 * 0.37).sin() * 0.8,
"wb.temp": 6500.0 + (round as f64 * 1.3).cos() * 1500.0,
"color.vibrance": (round * 13 % 50) as f64,
});
let _ = s.execute("develop.set", &json!({"values": v}));
let img = s.render_now(id, edge, edge).expect("render");
let bytes = lightcraft_codecs::encode_jpeg(
&lightcraft_codecs::EncodeImage::rgba8(&img.image),
88,
lightcraft_codecs::ChromaSubsampling::S420,
&lightcraft_codecs::EncodeMeta::default(),
)
.expect("encode");
let dir = out.join(format!("roll-{:02}", i / 50 + 1));
std::fs::create_dir_all(&dir).expect("mkdir");
std::fs::write(dir.join(format!("LCG{:05}.jpg", i + 1)), bytes).expect("write");
}
eprintln!("wrote {count} photos ({edge} px) to {} in {:.1} s", out.display(), t0.elapsed().as_secs_f64());
}
+3 -23
View File
@@ -92,29 +92,9 @@ fn services() -> Services {
}
}
/// Expand folders (recursively) into photo files.
fn collect(path: &std::path::Path, out: &mut Vec<String>) {
if path.is_dir() {
if let Ok(rd) = std::fs::read_dir(path) {
let mut v: Vec<_> = rd.flatten().map(|e| e.path()).collect();
v.sort();
for p in v {
collect(&p, out);
}
}
} else if path.extension().is_some_and(|e| {
matches!(
e.to_string_lossy().to_lowercase().as_str(),
"jpg" | "jpeg" | "png" | "tif" | "tiff" | "webp" | "dng" | "cr2" | "nef" | "arw" | "psd" | "jxl" | "gif" | "bmp"
)
}) {
out.push(path.to_string_lossy().to_string());
}
}
/// The persistent library session (or an in-memory one with `--memory` / if the library can't open).
fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> Session {
let fallback = || if seed_demo { Session::with_demo() } else { Session::new() }.with_fs();
let fallback = || if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock();
if in_memory {
return fallback();
}
@@ -123,7 +103,7 @@ fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: boo
return fallback();
};
let t0 = std::time::Instant::now();
let mut s = Session::new().with_fs();
let mut s = Session::new().with_fs().with_system_clock();
match s.open_library(&dir, seed_demo) {
Ok(r) => {
let (replayed, torn) = (r.replayed, r.torn_bytes);
@@ -160,7 +140,7 @@ fn main() -> eframe::Result {
println!("lightcraft {}", env!("CARGO_PKG_VERSION"));
return Ok(());
}
_ => collect(std::path::Path::new(&a), &mut files),
_ => files.push(a),
}
}
let options = eframe::NativeOptions {