Merge slideshow-playback into slideshow-composer (origin/main: batches 32–33)

This commit is contained in:
Jason Baldwin
2026-10-13 16:55:11 +02:00
192 changed files with 9753 additions and 1231 deletions
+405 -17
View File
@@ -10,7 +10,11 @@ use lightcraft_raster::{
Rgb32f,
resample::{Filter, fit, resize},
};
use lightcraft_raw::{RawFormat, RawImage, color::CameraTransform, profile::HsvTable};
use lightcraft_raw::{
ColorData, RawFormat, RawImage,
color::CameraTransform,
profile::{HsvTable, ProfileLook, ToneCurve},
};
/// The version of the camera-look fit: how a raw's colour matrix, hue/saturation table and tone
/// and chroma curves are chosen. Smart previews record it in their header (`look_version`), because
@@ -25,7 +29,8 @@ use lightcraft_raw::{RawFormat, RawImage, color::CameraTransform, profile::HsvTa
/// and the ILCE-7CR profile (#528, #583, #568, #616); 3: the camera JPEG read in the colour space the raw's EXIF
/// states, for every make (#778).
/// 4: a tone curve from black-and-white camera JPEGs (#577).
pub const LOOK_VERSION: u32 = 4;
/// 5: dark JPEG luminance pairs for the camera toe (#1307).
pub const LOOK_VERSION: u32 = 5;
#[derive(Clone, Debug)]
pub(crate) struct CameraLook {
@@ -107,6 +112,69 @@ fn starting_colour_with(
(xy, t, look)
}
/// The colour a raw starts from ([`starting_colour`]) as DNG colour tags, for a DNG of the raw's camera RGB (Photo
/// Merge, issue #965): without them a merge of CR2s, whose colour comes from a look fitted to their JPEG, rendered
/// green and dark. `None` when the file has colour matrices of its own (a DNG keeps them).
pub(crate) fn starting_colour_tags(raw: &mut RawImage, bytes: &[u8]) -> Option<ColorData> {
if lightcraft_raw::color::has_matrix(&raw.color) {
return None;
}
let (_, t, look) = starting_colour(raw, bytes);
dng_colour(&raw.color, &t, look.as_ref())
}
/// DNG tags that render white-balanced camera RGB at its as-shot white as `t` with `look` on top does: one forward
/// matrix (with a colour matrix and as-shot neutral that agree with it at that white), the look's hue/saturation
/// table as `ProfileHueSatMap` and its tone curve as `ProfileToneCurve`. Not quite the same render: a DNG table
/// doesn't keep each pixel's luminance as the fitted one does ([`HueSat`]), and the chroma curve has no tag; on two
/// CC0 CR2s (EOS 550D, 40D) an HDR merge of three copies renders within mean CIE76 ΔE 1.07 and 0.47 of the photo
/// (1.47 and 0.66 without the table; 26 and 23 with no colour tags at all).
/// `None` when the white balance or the matrix can't give tags.
fn dng_colour(tags: &ColorData, t: &CameraTransform, look: Option<&CameraLook>) -> Option<ColorData> {
let m = look.map_or(t.matrix, |l| l.matrix.mul(&t.matrix));
// the DNG model renders the camera's white at unit brightness: what a look does to white's brightness goes
// into the baseline exposure
let white = m.apply([1.0; 3]);
let k = (white[0] + white[1] + white[2]) / 3.0;
if !(k.is_finite() && k > 1e-6) || !t.wb.iter().all(|w| w.is_finite() && *w > 0.0) {
return None;
}
let m = Mat3(m.0.map(|r| r.map(|v| v / k)));
let forward = bradford(D65, D50).mul(&REC2020.to_xyz()).mul(&m);
let wb = t.wb.map(f64::from);
let neutral = wb.map(|w| wb.iter().cloned().fold(f64::MAX, f64::min) / w);
let camera_to_xyz = bradford(D50, t.white_xy).mul(&forward).mul(&Mat3::diag(1.0 / neutral[0], 1.0 / neutral[1], 1.0 / neutral[2]));
Some(ColorData {
illuminant: [21, 0],
color_matrix: [Some(camera_to_xyz.inverse()?), None],
forward_matrix: [Some(forward), None],
camera_calibration: [None, None],
analog_balance: None,
as_shot_neutral: Some(neutral),
as_shot_white_xy: None,
baseline_exposure: t.baseline_exposure + k.log2(),
baseline_sharpness: tags.baseline_sharpness,
kelvin_presets: None,
profile: ProfileLook {
hue_sat_map: [look.and_then(|l| l.hue_sat.clone()), None],
tone_curve: look.and_then(|l| dng_tone_curve(&l.tone)),
..Default::default()
},
})
}
/// A camera tone curve as a DNG `ProfileToneCurve` (linear in, linear out, 1 = white): sampled densely over the
/// 12 stops below white, the range the curve is read back over (`files::dng_tone_curve`).
fn dng_tone_curve(tone: &CameraTone) -> Option<ToneCurve> {
const N: usize = 256;
let samples = (0..=N).map(|i| {
let x = 2f32.powf(-12.0 + 12.0 * i as f32 / N as f32);
[x, tone.apply(x).min(1.0)]
});
let flat: Vec<f64> = std::iter::once([0.0, 0.0]).chain(samples).flatten().map(f64::from).collect();
ToneCurve::from_tag(&flat)
}
/// The look fitted to the file's camera JPEG, and whether that JPEG is black and white ([`monochrome`]).
pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransform) -> (Option<CameraLook>, bool) {
if !transform.matrix_is_fallback || !file_local_look(raw.format) {
@@ -265,12 +333,19 @@ fn at_most(a: &CameraTone, b: &CameraTone) -> bool {
})
}
/// Pixel cap for the embedded JPEG decoded for the fit. The codec checks it before decoding and
/// decodes at full size, so a 66.5 MP full-frame ILCE-7RM6 JPEG transiently needs ~200 MB (3 B/px);
/// 128 MP also admits 100 MP medium-format previews.
const FIT_PREVIEW_MAX_PIXELS: u64 = 128_000_000;
/// Same-size proxies of the sensor (white-balanced, baseline exposure, through `transform`'s
/// matrix: the generic camera ≈ sRGB model) and of the file's embedded camera JPEG (linear Rec.2020),
/// and per proxy pixel whether any sensor sample it covers is clipped.
fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usize) -> Option<(Rgb32f, Rgb32f, Vec<bool>)> {
let edge = (2 * size).max(384) as u32;
let decoded = crate::files::decode_raw_preview(bytes, lightcraft_codecs::DecodeOptions { max_size: Some((edge, edge)), max_pixels: 64_000_000 })?;
let decoded = crate::files::decode_raw_preview(
bytes,
lightcraft_codecs::DecodeOptions { max_size: Some((edge, edge)), max_pixels: FIT_PREVIEW_MAX_PIXELS },
)?;
let mut reference = decoded.to_working();
let (a, crop) = (raw.active_area, raw.crop.clipped(raw.active_area.width, raw.active_area.height));
if crop.width == 0 || crop.height == 0 || reference.width == 0 || reference.height == 0 {
@@ -569,7 +644,7 @@ fn collect_pairs(sensor: &Rgb32f, reference: &Rgb32f, min_chroma: f32, edge_limi
if (0.015..=1.0).contains(&y) {
bright.push((input.map(f64::from), output.map(f64::from)));
}
if !output.iter().all(|v| *v > 0.004 && *v < 0.98) || !(0.015..0.85).contains(&y) {
if !colour_pair(*input, *output) {
continue;
}
let min = output.iter().copied().fold(f32::INFINITY, f32::min);
@@ -580,6 +655,77 @@ fn collect_pairs(sensor: &Rgb32f, reference: &Rgb32f, min_chroma: f32, edge_limi
(pairs.len() >= 256 && colour >= pairs.len() / 20).then_some((pairs, bright))
}
/// The original colour-pair eligibility, also used to preserve its held-out split in tone fitting.
fn colour_pair(input: [f32; 3], output: [f32; 3]) -> bool {
input.iter().all(|v| v.is_finite() && *v > 0.001 && *v < 1.5)
&& output.iter().all(|v| *v > 0.004 && *v < 0.98)
&& (0.015..0.85).contains(&luminance_2020(output))
}
/// Fit tone independently of colour eligibility when enough dark JPEG signal is available.
/// The original pairs and their split still own the colour fit and its acceptance gates.
/// Luminance needs no channel ratios, so the binned proxy can use a lower sensor floor and
/// keep JPEG pixels with a channel near black. Quantile medians limit the effect of noise.
fn fit_tone_with_shadows(
fit: ToneFit,
pairs: &[([f64; 3], [f64; 3])],
sensor: &Rgb32f,
reference: &Rgb32f,
edge_limit: Option<f32>,
colour: impl Fn([f64; 3]) -> [f64; 3],
) -> Option<CameraTone> {
let luminances = |x: [f64; 3], y: [f64; 3]| (luma(colour(x).map(|v| v.max(0.0))), luma(y));
let original = fit.fit(pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| luminances(*x, *y)).collect());
let mut eligible = Vec::new();
let (mut colour_index, mut extra_index) = (0, 0);
for (i, (input, output)) in sensor.data.iter().zip(&reference.data).enumerate() {
if edge_limit.is_some_and(|limit| local_contrast(sensor, i) > limit || local_contrast(reference, i) > limit) {
continue;
}
// About one 8-bit sRGB code above black: don't learn a flat black segment from
// JPEG compression. The sensor proxy averages many samples before this test.
if !input.iter().all(|v| v.is_finite() && *v > 0.0001 && *v < 1.5)
|| !output.iter().all(|v| v.is_finite() && *v >= 0.0)
|| !(0.0003..0.85).contains(&luminance_2020(*output))
{
continue;
}
// A colour holdout must never enter tone training just because the wider sample set
// changes its index. Split the extra samples separately, keeping every colour index.
let index = if colour_pair(*input, *output) { &mut colour_index } else { &mut extra_index };
let held = *index % 3 == 0;
*index += 1;
let (x, y) = luminances(input.map(f64::from), output.map(f64::from));
if x.is_finite() && x > 0.0 {
eligible.push((x, y, held));
}
}
let dark = eligible.iter().filter(|(_, y, _)| *y < 0.015).count();
let held: Pairs = eligible.iter().filter(|(_, y, held)| *held && *y < 0.015).map(|(x, y, _)| ([*x; 3], [*y; 3])).collect();
// Sparse or uncorrelated shadow noise cannot teach a toe. Tied JPEG levels share a rank,
// so a single quantized grey also keeps the original curve.
if dark < 96 || held.len() < 32 || rank_correlation_ties(&held) < PARTIAL_MIN_RANK_CORRELATION {
return original;
}
let training = eligible.iter().filter(|(_, _, held)| !held).map(|(x, y, _)| (*x, *y)).collect();
let Some(candidate) = fit.fit(training) else { return original };
let error = |tone: &CameraTone| -> f64 {
held.iter().map(|(x, y)| (f64::from(tone.apply(x[0] as f32)).powf(1.0 / 2.2) - y[0].powf(1.0 / 2.2)).powi(2)).sum()
};
// A wider histogram must also help independent dark pixels, not just redistribute the
// knots. The usual held-out gates still check the completed look on the colour pairs.
if let Some(old) = original {
let (before, after) = (error(&old), error(&candidate));
if lightcraft_pipeline::profiling() {
eprintln!("[profile] camera toe: {dark} dark pairs, held-out error {before:.5} -> {after:.5}");
}
if !after.is_finite() || after >= before * 0.99 {
return original;
}
}
Some(candidate)
}
/// Ridge-regularised 3×3 chromaticity matrix (luminance-normalised RGB) on the training pairs.
fn fit_matrix(pairs: &[([f64; 3], [f64; 3])]) -> Option<Mat3> {
let mut gram = [[0.0; 3]; 3];
@@ -705,9 +851,7 @@ fn fit_pairs_on(
let p = matrix.apply(x);
correction.as_ref().map_or(p, |c| c.apply(p.map(|v| v as f32)).map(f64::from))
};
let tone_pairs: Vec<_> =
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(colour(*x).map(|v| v.max(0.0))), luma(*y))).collect();
let Some(curve) = tone_fit.fit(tone_pairs) else { continue };
let Some(curve) = fit_tone_with_shadows(tone_fit, &pairs, sensor, reference, edge_limit, colour) else { continue };
let mut look = CameraLook { matrix, tone: curve, hue_sat: hue_sat.clone() };
if let Some(tone) = fit_chroma(&bright, &look) {
look.tone = tone;
@@ -849,9 +993,8 @@ fn fit_partial_on(sensor: &Rgb32f, reference: &Rgb32f, edge_limit: Option<f32>)
})),
None => Mat3::IDENTITY,
};
let tone_pairs: Vec<_> =
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(matrix.apply(*x).map(|v| v.max(0.0))), luma(*y))).collect();
let mut look = CameraLook { matrix, tone: fit_tone(tone_pairs)?, hue_sat: None };
let tone = fit_tone_with_shadows(ToneFit::Quantile, &pairs, sensor, reference, edge_limit, |x| matrix.apply(x))?;
let mut look = CameraLook { matrix, tone, hue_sat: None };
if let Some(tone) = fit_chroma(&bright, &look) {
look.tone = tone;
}
@@ -957,9 +1100,8 @@ fn fit_tone_only_on(sensor: &Rgb32f, reference: &Rgb32f, colour: &Option<(Mat3,
let p = matrix.apply(x);
correction.as_ref().map_or(p, |c| c.apply(p.map(|v| v as f32)).map(f64::from))
};
let tone_pairs: Vec<_> =
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(coloured(*x).map(|v| v.max(0.0))), luma(*y))).collect();
let look = CameraLook { matrix, tone: fit_tone(tone_pairs)?, hue_sat: hue_sat.clone() };
let tone = fit_tone_with_shadows(ToneFit::Quantile, &pairs, sensor, reference, edge_limit, coloured)?;
let look = CameraLook { matrix, tone, hue_sat: hue_sat.clone() };
let (tone, neutral) = (ToneMap::camera(&look.tone, 0.0, 0.0, 0.0), ToneMap::new(0.0, 0.0, 0.0));
let encoded = |v: f64| v.max(0.0).powf(1.0 / 2.2);
let (mut before, mut after, mut before_encoded, mut after_encoded, mut samples) = (0.0, 0.0, 0.0, 0.0, 0);
@@ -1317,6 +1459,167 @@ fn fit_tone_conditional(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
mod tests {
use super::*;
fn steep_toe_scene() -> (Rgb32f, Rgb32f) {
let mut sensor = Rgb32f::new(96, 64);
let mut reference = sensor.clone();
for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() {
// A broad scene with a steep camera toe below the old JPEG floor of 0.015.
// Permute brightness so the held-out pixels span the same range independently.
let y = 0.002 + ((i * 7919) % 6144) as f32 / 6144.0 * 0.25;
let rgb = [0.8 + (i % 11) as f32 * 0.04, 1.0, 0.8 + (i % 17) as f32 * 0.025];
*src = rgb.map(|v| v * y / luminance_2020(rgb));
let display = 3.0 * y * (y / (y + 0.02)).powi(2);
*dst = src.map(|v| v * display / y);
}
(sensor, reference)
}
#[test]
fn camera_tone_follows_the_jpegs_deep_shadows() {
let (sensor, reference) = steep_toe_scene();
let look = fit_pairs_with(&sensor, &reference, Some((Mat3::IDENTITY, None))).unwrap();
let dark: Vec<_> = sensor.data.iter().zip(&reference.data).filter(|(_, p)| luminance_2020(**p) < 0.015).collect();
assert!(dark.len() > 256);
let error = dark.iter().map(|(x, y)| (look.tone.apply(luminance_2020(**x)) - luminance_2020(**y)).abs()).sum::<f32>() / dark.len() as f32;
assert!(error < 0.001, "deep-shadow luminance error {error}");
assert_eq!(look.matrix, Mat3::IDENTITY);
}
#[test]
fn colour_holdouts_stay_out_of_the_wider_tone_training() {
let (sensor, reference) = steep_toe_scene();
let fit = |reference: &Rgb32f| {
let (pairs, _) = collect_pairs(&sensor, reference, 0.005, None).unwrap();
fit_tone_with_shadows(ToneFit::Quantile, &pairs, &sensor, reference, None, |x| x).unwrap()
};
let expected = fit(&reference);
let mut changed = reference.clone();
let mut index = 0;
for (x, y) in sensor.data.iter().zip(&mut changed.data) {
if colour_pair(*x, *y) {
if index % 3 == 0 {
*y = y.map(|v| v * 1.1);
assert!(colour_pair(*x, *y));
}
index += 1;
}
}
assert_eq!(fit(&changed), expected);
}
#[test]
fn camera_toe_is_robust_to_shadow_noise() {
let (sensor, clean) = steep_toe_scene();
let mut noisy = clean.clone();
for (i, p) in noisy.data.iter_mut().enumerate() {
if luminance_2020(*p) < 0.015 {
// Quantized JPEG levels with bounded noise and occasional compression outliers.
let noise = ((i * 17 % 11) as f32 - 5.0) * 0.00004;
*p = p.map(|v| ((v + noise).max(0.0) / 0.0003).round() * 0.0003);
if i % 23 == 0 {
*p = [0.014; 3];
}
}
}
let look = fit_pairs_with(&sensor, &noisy, Some((Mat3::IDENTITY, None))).unwrap();
let dark: Vec<_> = sensor.data.iter().zip(&clean.data).filter(|(_, p)| luminance_2020(**p) < 0.015).collect();
let error = dark.iter().map(|(x, y)| (look.tone.apply(luminance_2020(**x)) - luminance_2020(**y)).abs()).sum::<f32>() / dark.len() as f32;
assert!(error < 0.001, "noisy shadow luminance error {error}");
assert_eq!(look.tone.apply(0.0), 0.0);
assert_eq!(look.matrix, Mat3::IDENTITY);
for i in 1..1000 {
assert!(look.tone.apply(i as f32 / 1000.0) >= look.tone.apply((i - 1) as f32 / 1000.0));
}
}
#[test]
fn camera_toe_requires_lower_independent_dark_error() {
for holdout_matches_original in [true, false] {
let mut sensor = Rgb32f::new(96, 64);
let mut reference = sensor.clone();
for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() {
let y = if i < 600 { 0.0015 + (i % 200) as f32 * 0.00001 } else { 0.02 + ((i * 7919) % 6144) as f32 / 6144.0 * 0.23 };
*src = [y; 3];
// Midtones teach the original slope of 3. Dark training teaches 0.3,
// but the independent dark holdout can agree with either slope.
let slope = if i < 600 && !(holdout_matches_original && i % 3 == 0) { 0.3 } else { 3.0 };
*dst = [y * slope; 3];
assert_eq!(colour_pair(*src, *dst), i >= 600);
}
let (pairs, _) = collect_pairs(&sensor, &reference, f32::NEG_INFINITY, None).unwrap();
let original = fit_tone(pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(*x), luma(*y))).collect()).unwrap();
// Both sample sets start at a multiple of three, so this split matches
// the helper's separate colour and extra-sample indices.
let candidate = fit_tone(
sensor
.data
.iter()
.zip(&reference.data)
.enumerate()
.filter(|(i, _)| i % 3 != 0)
.map(|(_, (x, y))| (luma(x.map(f64::from)), luma(y.map(f64::from))))
.collect(),
)
.unwrap();
let held: Pairs =
sensor.data.iter().zip(&reference.data).take(600).step_by(3).map(|(x, y)| (x.map(f64::from), y.map(f64::from))).collect();
assert_eq!(held.len(), 200);
assert!(held.iter().all(|(_, y)| (0.0003..0.015).contains(&luma(*y))));
assert!(rank_correlation_ties(&held) > 0.99);
let error = |tone: &CameraTone| {
held.iter().map(|(x, y)| (f64::from(tone.apply(luma(*x) as f32)).powf(1.0 / 2.2) - luma(*y).powf(1.0 / 2.2)).powi(2)).sum::<f64>()
};
assert_ne!(candidate, original);
let selected = fit_tone_with_shadows(ToneFit::Quantile, &pairs, &sensor, &reference, None, |x| x).unwrap();
if holdout_matches_original {
assert!(error(&candidate) > error(&original) + 0.1);
assert_eq!(selected, original, "a correlated but misleading training set must keep the original curve");
} else {
assert!(error(&candidate) < error(&original) * 0.99);
assert_eq!(selected, candidate, "matching dark training and holdout must accept the candidate");
}
let mut previous = 0.0;
assert_eq!(selected.apply(0.0), 0.0);
for i in 0..10_000 {
let y = selected.apply(10f32.powf(-8.0 + i as f32 / 1000.0));
assert!(y.is_finite() && y >= previous && y <= 1.0);
previous = y;
}
assert_eq!(selected.apply(f32::NAN), 0.0);
assert_eq!(selected.apply(f32::INFINITY), 0.0);
}
}
#[test]
fn unreliable_dark_pairs_keep_the_original_tone_and_colour() {
let (sensor, reference) = steep_toe_scene();
let dark: Vec<_> = reference.data.iter().enumerate().filter(|(_, p)| luminance_2020(**p) < 0.015).map(|(i, _)| i).collect();
let mut excluded = sensor.clone();
for &i in &dark {
excluded.data[i] = [f32::NAN; 3];
}
let colour = Some((Mat3::IDENTITY, None));
let original = fit_pairs_with(&excluded, &reference, colour.clone()).unwrap();
for case in 0..6 {
let (mut x, mut y) = (sensor.clone(), reference.clone());
for (j, &i) in dark.iter().enumerate() {
match case {
0 => y.data[i] = [0.0; 3], // JPEG black gives no measured shadow signal
1 => x.data[i] = [0.00001; 3], // below the sensor noise floor
2 if j >= 95 => y.data[i] = [0.0; 3], // too few dark pairs
3 => x.data[i] = [0.003; 3], // no scene luminance spread
4 => y.data[i] = [0.003; 3], // one quantized grey, not a toe
5 => y.data[i] = [0.0004 + ((j * 7919) % 1000) as f32 * 0.000014; 3], // unrelated noise
_ => {}
}
}
let look = fit_pairs_with(&x, &y, colour.clone()).unwrap();
assert_eq!(look.tone, original.tone, "case {case}");
assert_eq!(look.matrix, original.matrix, "case {case}");
assert!(look.hue_sat.is_none());
}
}
#[test]
fn camera_jpeg_matches_the_sensor_proxy_size_exactly() {
// An ILCE-7RM4 ARW (9504×6336 crop) embeds a 1616×1080 preview; for the camera-profile proxy
@@ -1834,17 +2137,19 @@ mod tests {
let mut raw = lightcraft_raw::decode(&bytes).unwrap();
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
let own = lightcraft_raw::color::has_matrix(&raw.color);
let covered = raw.metadata.model.as_deref().and_then(|m| lightcraft_raw::spectral::find(raw.metadata.make.as_deref(), m)).is_some();
let (make, model) = (raw.metadata.make.as_deref(), raw.metadata.model.as_deref().unwrap_or_default());
let measured = lightcraft_raw::spectral::find(make, model).is_some();
let covered = measured || lightcraft_raw::kelvin_presets::covers(make, model);
let before = raw.color.clone();
let (_, t, fitted) = starting_colour(&mut raw, &bytes);
let spectral = !own && fitted.is_none() && covered;
let spectral = !own && fitted.is_none() && measured;
eprintln!("{name}: own matrices {own}, JPEG fit {}, spectral {spectral}", fitted.is_some());
assert_eq!(raw.color != before, spectral, "{name}");
assert_eq!(t.matrix_is_fallback, !own && !spectral, "{name}");
let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap();
assert_eq!(info.camera_tone.is_some(), fitted.is_some(), "{name}");
// white balance: relative only without matrices anywhere; a covered camera edits in Kelvin, through the
// spectral matrices, whichever look it got (issue #730)
// white balance: relative only without matrices anywhere (or presets); a covered camera edits in Kelvin,
// through the spectral matrices or its presets, whichever look it got (issue #730)
assert_eq!(info.relative_wb, !own && !covered, "{name}");
assert_eq!(info.legacy_relative_wb, !own && covered, "{name}");
if covered && !own {
@@ -2062,6 +2367,43 @@ mod tests {
assert!(info.camera_tone.is_some(), "no camera look fitted");
}
/// A merged DNG of a raw's camera RGB (issue #965) carries the colour the raw opens with: its tags give the same
/// white balance, colour and brightness as the fitted look at the as-shot white, the look's hue/saturation
/// table and its tone curve; without a look, those of the camera transform.
#[test]
fn dng_colour_renders_like_the_raws_starting_colour() {
use lightcraft_raw::color::{as_shot_white_of, camera_transform_of};
let t = camera_transform_of(&ColorData::default(), Xy::new(0.36, 0.37));
assert!(t.wb.iter().any(|w| (w - 1.0).abs() > 0.1), "a white balance to keep: {:?}", t.wb);
let look = dro_test_look(&std::array::from_fn(|i| 0.005 * 1.2f32.powi(i as i32)), &|x| 0.95 * x / (x + 0.1));
let samples = [[0.2, 0.3, 0.1], [0.05, 0.02, 0.4], [0.6, 0.6, 0.6], [0.01, 0.5, 0.03]];
let render = |m: &Mat3, wb: [f32; 3], ev: f64, c: [f64; 3]| m.apply(std::array::from_fn(|i| c[i] * wb[i] as f64)).map(|v| v * ev.exp2());
for look in [Some(&look), None] {
let dng = dng_colour(&ColorData::default(), &t, look).unwrap();
let back = camera_transform_of(&dng, as_shot_white_of(&dng, None));
for i in 0..3 {
assert!((back.wb[i] - t.wb[i]).abs() < 1e-4 * t.wb[i], "{:?} vs {:?}", back.wb, t.wb);
}
let m = look.map_or(t.matrix, |l| l.matrix.mul(&t.matrix));
for c in samples {
let (want, got) = (render(&m, t.wb, t.baseline_exposure, c), render(&back.matrix, back.wb, dng.baseline_exposure, c));
assert!(want.iter().zip(got).all(|(a, b)| (a - b).abs() < 1e-5), "{c:?}: {want:?} vs {got:?}");
}
assert_eq!(dng.profile.hue_sat_for(1.0), look.and_then(|l| l.hue_sat.clone()));
let tone = dng.profile.tone_curve.as_ref().and_then(crate::files::dng_tone_curve);
match look {
Some(look) => {
let tone = tone.expect("the look's tone curve");
let worst =
(0..=120).map(|i| 2f32.powf(-10.0 + i as f32 / 12.0)).map(|x| (tone.apply(x) - look.tone.apply(x)).abs()).fold(0.0, f32::max);
// (the reader resamples a `ProfileToneCurve` at 32 knots: 0.0031 of white at worst here)
assert!(worst < 4e-3, "{worst}");
}
None => assert!(tone.is_none()),
}
}
}
/// A look with tone curve `f` at knots `xs` and a colour model and chroma curve to keep.
fn dro_test_look(xs: &[f32; 32], f: &dyn Fn(f32) -> f32) -> CameraLook {
CameraLook {
@@ -2317,6 +2659,36 @@ mod tests {
}
}
/// Sony ARWs without measured matrices read Kelvin from their white-balance presets
/// (`lightcraft_raw::kelvin_presets`): the ILCE-3500 sample was shot at 5600 K on the camera's Kelvin setting and
/// the ILCE-7S sample on its Shade preset (7500 K on Sony's scale); the SLT-A33's Auto white is within 20 K of its
/// Daylight preset. Header and decode agree. Skips cleanly without these CC0 corpus files.
#[test]
fn corpus_arw_white_balance_in_kelvin_through_the_presets() {
let dir = std::env::var_os("LIGHTCRAFT_CORPUS")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))
.join("raw");
for (name, kelvin) in [("arw-sony-a3500-5600k.arw", 5600.0), ("arw-sony-a7s-shade.arw", 7500.0), ("arw-sony-a33.arw", 5311.0)] {
let Ok(bytes) = std::fs::read(dir.join(name)) else {
eprintln!("skip: {name} absent");
continue;
};
let header = crate::files::probe_bytes(name, &bytes).unwrap();
let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap();
assert_eq!(header.as_shot_wb, Some((info.as_shot_temp, info.as_shot_tint)), "{name}");
assert!(info.raw && !info.relative_wb && info.legacy_relative_wb, "{name}: {info:?}");
assert!(
(info.as_shot_temp - kelvin).abs() <= 20.0 && info.as_shot_tint.abs() <= 2.0,
"{name}: {} {}",
info.as_shot_temp,
info.as_shot_tint
);
let camera = format!("SONY {}", lightcraft_meta::extract(&bytes).model.unwrap_or_default());
assert!(!lightcraft_catalog::relative_wb_camera("ARW", &camera), "{camera}");
}
}
#[test]
fn rejects_monochrome_invalid_and_unrelated_previews() {
let mut sensor = Rgb32f::new(32, 32);
@@ -2810,4 +3182,20 @@ mod tests {
reference.data.iter_mut().for_each(|p| *p = [0.4; 3]);
assert!(fit_tone_only(&sorted, &reference, None).is_none(), "a flat JPEG");
}
/// ILCE-7RM6 full-frame: the 66.5 MP camera JPEG exceeded the old 64 MP cap, so no look was fitted.
#[test]
fn corpus_arw6_full_frame_gets_a_camera_look() {
const { assert!(FIT_PREVIEW_MAX_PIXELS >= 100_000_000) };
let path = std::env::var_os("LIGHTCRAFT_CORPUS")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))
.join("raw/arw6-sony-ilce7rm6-ff-land-c.arw");
let Ok(bytes) = std::fs::read(&path) else {
eprintln!("skip: {} absent", path.display());
return;
};
let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap();
assert!(info.camera_tone.is_some(), "no camera look fitted");
}
}
+14 -3
View File
@@ -303,9 +303,20 @@ fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
if !src.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) {
return Err(bad(C, "choose an .onnx file with denoise-model.json beside it"));
}
let bytes = read_capped(&src.with_file_name("denoise-model.json"), MAX_MANIFEST_BYTES)
.ok_or_else(|| bad(C, "denoise-model.json is missing or too large"))?;
let m = manifest::parse(&bytes).map_err(|e| bad(C, e.to_string()))?;
let sibling = src.with_file_name("denoise-model.json");
let Some(bytes) = read_capped(&sibling, MAX_MANIFEST_BYTES) else {
let why = if sibling.is_file() {
"denoise-model.json could not be read or is too large"
} else if src.with_file_name("model.json").is_file() {
// other apps ship their models with a model.json of their own (#1381)
"No denoise-model.json beside this .onnx file. The model.json there is another app's description, which LightCraft \
does not read: it needs its own denoise-model.json (see docs/denoise.md)"
} else {
"No denoise-model.json beside this .onnx file: LightCraft needs one that says how to use the model (see docs/denoise.md)"
};
return Err(bad(C, why));
};
let m = manifest::parse(&bytes).map_err(|e| bad(C, format!("denoise-model.json: {e} (see docs/denoise.md)")))?;
let dir = models_dir(s, C)?;
Ok(json!({"kind": "model", "domain": "denoise", "path": path, "fileName": src.file_name().map(|n| n.to_string_lossy()), "model": m,
"sizeBytes": std::fs::metadata(src).ok().map(|m| m.len()), "alreadyInstalled": dir.join(&m.id).join("model.onnx").is_file()}))
+68 -7
View File
@@ -1079,15 +1079,47 @@ pub fn specs() -> Vec<CommandSpec> {
let id = active(s, "history.clear")?;
let d = s.develop_of(id).unwrap_or_default();
let history = vec![lightcraft_catalog::HistoryStep { label: "Cleared History".into(), settings: d }];
s.commit("Clear History", Op::SetHistory { id, history })?;
s.commit("Clear History", Op::Batch { ops: vec![Op::SetHistory { id, history }, Op::SetHistoryCursor { id, cursor: None }] })?;
ok()
}),
cmd!("history.restore", "Go to History Step", [], None, "{index}", has_active, |s, p| {
let id = active(s, "history.restore")?;
let i = f64_req(p, "index", "history.restore")? as usize;
let st = s.catalog.photo(id).and_then(|ph| ph.history.get(i).cloned()).ok_or_else(|| bad("history.restore", "no such step"))?;
s.set_develop(id, (*st.settings).clone(), &format!("History: {}", st.label))?;
ok()
cmd!(
"history.restore",
"Go to History Step",
[],
None,
"{index} — loads that step's settings and moves the History cursor there; later steps stay listed until a new edit",
has_active,
|s, p| {
let c = "history.restore";
let id = active(s, c)?;
let i = f64_req(p, "index", c)?;
if !(i.is_finite() && i >= 0.0) {
return Err(bad(c, "no such step"));
}
go_to_history_step(s, c, id, i as usize)
}
),
cmd!("history.previous", "Previous History Step", [], None, "{} — steps the History cursor back one step", has_history_before, |s, _| {
let c = "history.previous";
let id = active(s, c)?;
let i = s
.catalog
.photo(id)
.and_then(|ph| ph.history_index())
.and_then(|i| i.checked_sub(1))
.ok_or_else(|| bad(c, "at the first History step"))?;
go_to_history_step(s, c, id, i)
}),
cmd!("history.next", "Next History Step", [], None, "{} — steps the History cursor forward one step", has_history_after, |s, _| {
let c = "history.next";
let id = active(s, c)?;
let ph = s.catalog.photo(id).ok_or_else(|| bad(c, "no photo"))?;
let i = ph
.history_index()
.map(|i| i.saturating_add(1))
.filter(|i| *i < ph.history.len())
.ok_or_else(|| bad(c, "at the latest History step"))?;
go_to_history_step(s, c, id, i)
}),
// ---- interactions (slider drags, brush strokes)
cmd!("develop.beginInteraction", "Begin Interaction", [], None, "{label}", has_active, |s, p| {
@@ -1104,3 +1136,32 @@ pub fn specs() -> Vec<CommandSpec> {
}),
]
}
/// Load History step `i` of photo `id` and put the cursor there, without adding a step: the later
/// steps stay listed (one undo step).
fn go_to_history_step(s: &mut Session, c: &str, id: PhotoId, i: usize) -> Result<Value> {
let ph = s.catalog.photo(id).ok_or_else(|| bad(c, "no photo"))?;
let st = ph.history.get(i).cloned().ok_or_else(|| bad(c, "no such step"))?;
let cursor = (i.saturating_add(1) < ph.history.len()).then_some(i);
let label = format!("History: {}", st.label);
let edited = Some((s.clock)());
s.commit(
&label,
Op::Batch { ops: vec![Op::SetDevelop { id, settings: st.settings, label: label.clone(), edited }, Op::SetHistoryCursor { id, cursor }] },
)?;
ok()
}
fn has_history_before(s: &Session) -> std::result::Result<(), String> {
let at = s.active().and_then(|id| s.catalog.photo(id)).and_then(|p| p.history_index());
if at.is_some_and(|i| i > 0) { Ok(()) } else { Err("at the first History step".into()) }
}
fn has_history_after(s: &Session) -> std::result::Result<(), String> {
let photo = s.active().and_then(|id| s.catalog.photo(id));
if photo.is_some_and(|p| p.history_index().is_some_and(|i| i.saturating_add(1) < p.history.len())) {
Ok(())
} else {
Err("at the latest History step".into())
}
}
+24
View File
@@ -219,6 +219,12 @@ fn step(s: &mut Session, d: isize) -> Result<Value> {
Ok(json!({"active": vis[next].0}))
}
/// `library.up` / `library.down`: how many photos to move (at least 1, capped so hostile input
/// can't overflow).
fn row_step(p: &Value) -> isize {
p.get("rows").and_then(Value::as_u64).unwrap_or(1).clamp(1, 1 << 20) as isize
}
fn rotate(s: &mut Session, p: &Value, cw: bool) -> Result<Value> {
let targets = s.targets(p);
let mut ops = Vec::new();
@@ -641,6 +647,24 @@ pub fn specs() -> Vec<CommandSpec> {
}),
cmd!("library.next", "Next Photo", [], Some("Right"), "{}", always, |s, _| step(s, 1)),
cmd!("library.previous", "Previous Photo", [], Some("Left"), "{}", always, |s, _| step(s, -1)),
cmd!(
"library.down",
"Photo Below",
[],
Some("Down"),
"{rows?: n (photos to move forward; default 1, the grid passes the distance to the cell below)}",
always,
|s, p| { step(s, row_step(p)) }
),
cmd!(
"library.up",
"Photo Above",
[],
Some("Up"),
"{rows?: n (photos to move back; default 1, the grid passes the distance to the cell above)}",
always,
|s, p| { step(s, -row_step(p)) }
),
// ---- rating / flags / labels
cmd!("photo.rate", "Set Rating", ["Photo", "Set Rating"], None, "{rating: 0..5, ids?, advance?: bool}", has_selection, |s, p| {
let r = super::f64_req(p, "rating", "photo.rate")? as i64;
+9 -8
View File
@@ -195,20 +195,21 @@ pub fn specs() -> Vec<CommandSpec> {
cmd!(query "presets.list", "List Presets", [], None, "{}", always, |s, _| {
Ok(Value::Array(s.presets.iter().map(|p| json!({"id": p.id, "name": p.name, "group": p.group, "favorite": p.favorite, "builtin": p.builtin})).collect()))
}),
cmd!(query "profiles.list", "List Profiles", [], None, "{} — every profile with its group and favourite flag", always, |s, _| {
Ok(Value::Array(
crate::presets::PROFILES
.iter()
.map(|p| json!({"id": p.id, "name": p.name, "group": p.group, "favorite": s.profile_favorites.iter().any(|f| f == p.id)}))
.collect(),
))
cmd!(query "profiles.list", "List Profiles", [], None, "{} — every profile (built-in, then imported LUTs marked `imported`) with its group and favourite flag", always, |s, _| {
let favorite = |id: &str| s.profile_favorites.iter().any(|f| f == id);
let builtin = crate::presets::PROFILES.iter().map(|p| json!({"id": p.id, "name": p.name, "group": p.group, "favorite": favorite(p.id)}));
// imported LUT profiles too (#618), marked as in `profiles.menu`
let imported =
s.lut_profiles.iter().map(|p| json!({"id": p.id, "name": p.name, "group": p.group, "favorite": favorite(&p.id), "imported": true}));
Ok(Value::Array(builtin.chain(imported).collect()))
}),
cmd!(query "profiles.menu", "Profile Menu", [], None, "{} — favourites, recent (newest first) and groups", always, |s, _| Ok(s.profile_menu())),
cmd!(query "history.list", "List History", [], None, "{id?}", has_active, |s, p| {
cmd!(query "history.list", "List History", [], None, "{id?} — returns {history: labels oldest first, cursor: index of the step the settings are at (later steps can be returned to), versions}", has_active, |s, p| {
let id = photo_arg(s, p, "history.list")?;
let ph = s.catalog.photo(id).ok_or_else(|| bad("history.list", "no such photo"))?;
Ok(json!({
"history": ph.history.iter().map(|h| h.label.clone()).collect::<Vec<_>>(),
"cursor": ph.history_index(),
"versions": ph.versions.iter().map(|v| json!({"name": v.name, "created": v.created})).collect::<Vec<_>>(),
}))
}),
+5 -1
View File
@@ -49,6 +49,10 @@ fn read(s: &mut Session, p: &Value) -> Result<Value> {
}
if !ops.is_empty() {
s.commit("Read Metadata from File", Op::Batch { ops })?;
// Don't auto-write the sidecars just read: that would add our `lc:settings` to one that had
// only another app's `crs:` settings, and later reads prefer `lc:settings`, so that app's
// next edits would be ignored (issue #934). The next edit, or Save Metadata, writes them.
s.skip_auto_write = true;
}
let mut result = json!({"read": read, "failed": failed});
if !warnings.is_empty() {
@@ -90,7 +94,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Read Metadata from File",
["Photo"],
None,
"{ids?} — reads each photo's XMP sidecar (or a raw/DNG's embedded XMP): metadata and develop settings, one undo step → {read, failed, warnings?: [{path, code, profile?, mode?, fields?}]}",
"{ids?} — reads each photo's XMP sidecar (or a raw/DNG's embedded XMP): metadata and develop settings, one undo step; never writes the sidecar, even with auto-write on → {read, failed, warnings?: [{path, code, profile?, mode?, fields?}]}",
has_selection,
read
),
+110 -3
View File
@@ -28,6 +28,8 @@ pub enum ExportFormat {
Original,
/// A DNG of the raw data (raw photos only), the edits embedded as XMP.
Dng,
/// Render to DNG: a linear DNG (16-bit float) of the rendered photo, the edit baked in.
RenderedDng,
}
impl ExportFormat {
@@ -42,6 +44,7 @@ impl ExportFormat {
"jxl" | "jpegxl" | "jpeg-xl" | "jpeg_xl" => Self::Jxl,
"original" => Self::Original,
"dng" => Self::Dng,
"rendereddng" | "renderdng" | "render-dng" | "lineardng" => Self::RenderedDng,
_ => return None,
})
}
@@ -56,7 +59,7 @@ impl ExportFormat {
Self::Jxl => "jxl",
// the original keeps its own extension (see `ExportOptions::file_name_for`)
Self::Original => "",
Self::Dng => "dng",
Self::Dng | Self::RenderedDng => "dng",
}
}
@@ -623,6 +626,32 @@ pub enum MetadataPolicy {
None,
}
/// Transfer curve of an HDR AVIF export (`hdrTransfer`): PQ (absolute, SDR white at 203 cd/m²) or
/// HLG (relative to the display; BT.2100 with SDR white at signal 0.75).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum HdrCurve {
#[default]
Pq,
Hlg,
}
impl HdrCurve {
pub fn parse(s: &str) -> Option<Self> {
match s.to_ascii_lowercase().as_str() {
"pq" => Some(Self::Pq),
"hlg" => Some(Self::Hlg),
_ => None,
}
}
fn codec(self) -> lightcraft_codecs::HdrTransfer {
match self {
Self::Pq => lightcraft_codecs::HdrTransfer::Pq,
Self::Hlg => lightcraft_codecs::HdrTransfer::Hlg,
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct ExportOptions {
@@ -681,6 +710,8 @@ pub struct ExportOptions {
pub add_to_photos: bool,
/// The Photos album they go into, made when missing (empty = none).
pub photos_album: String,
/// The curve of an HDR AVIF: PQ (default) or HLG.
pub hdr_transfer: HdrCurve,
}
impl Default for ExportOptions {
@@ -710,6 +741,7 @@ impl Default for ExportOptions {
jxl_progressive: false,
add_to_photos: false,
photos_album: String::new(),
hdr_transfer: HdrCurve::Pq,
}
}
}
@@ -751,6 +783,7 @@ pub const OPTION_PARAMS: &[&str] = &[
"jxlProgressive",
"addToPhotos",
"photosAlbum",
"hdrTransfer",
];
/// Why an export can't add its files to Apple Photos on this platform (`None`: it can, when the
@@ -923,6 +956,7 @@ impl ExportOptions {
}
"sharpen" => one_of(k, v, &["none", "screen", "matte", "glossy"])?,
"sharpenAmount" => one_of(k, v, &["low", "standard", "high"])?,
"hdrTransfer" => one_of(k, v, &["pq", "hlg"])?,
"conflict" => one_of(k, v, &["unique", "overwrite", "skip"])?,
"tiffCompression" => one_of(k, v, &["none", "lzw", "zip", "deflate"])?,
"jxlEffort" => one_of(k, v, &["fast", "normal"])?,
@@ -1061,6 +1095,7 @@ impl ExportOptions {
jxl_progressive: p.get("jxlProgressive").and_then(Value::as_bool).unwrap_or(d.jxl_progressive),
add_to_photos: p.get("addToPhotos").and_then(Value::as_bool).unwrap_or(d.add_to_photos),
photos_album: s("photosAlbum").map_or(d.photos_album, |a| a.trim().to_string()),
hdr_transfer: s("hdrTransfer").and_then(HdrCurve::parse).unwrap_or(d.hdr_transfer),
}
}
@@ -1133,6 +1168,8 @@ impl ExportOptions {
(ExportFormat::Tiff, Some(32)) if self.hdr => OutputDepth::F32Hdr,
(ExportFormat::Tiff, Some(32)) => OutputDepth::F32Linear,
(ExportFormat::Tiff, _) => OutputDepth::U16,
(ExportFormat::RenderedDng, _) if self.hdr => OutputDepth::F32Hdr,
(ExportFormat::RenderedDng, _) => OutputDepth::F32Linear,
(ExportFormat::Psd, Some(8)) => OutputDepth::U8,
(ExportFormat::Psd, _) => OutputDepth::U16,
(ExportFormat::Avif, _) if self.hdr => OutputDepth::F32Hdr,
@@ -1151,6 +1188,7 @@ impl ExportOptions {
ExportFormat::Png => &[(8, "8-bit"), (16, "16-bit")],
ExportFormat::Tiff => &[(16, "16-bit"), (8, "8-bit"), (32, "32-bit float")],
ExportFormat::Psd => &[(16, "16-bit"), (8, "8-bit")],
ExportFormat::RenderedDng => &[(16, "16-bit float")],
ExportFormat::Avif => &[(8, "8-bit"), (10, "10-bit")],
ExportFormat::Jxl => &[(8, "8-bit"), (16, "16-bit")],
}
@@ -1341,6 +1379,7 @@ pub fn encode_with_metadata(img: &Rgba8, o: &ExportOptions, meta: Option<&Metada
ExportFormat::Avif => encode::encode_avif(&e, o.quality, 8, &meta),
ExportFormat::Jxl => lightcraft_codecs::encode_jxl(&e, &o.jxl(space, false), &meta),
f @ (ExportFormat::Original | ExportFormat::Dng) => return Err(format!("{f:?} export does not encode pixels")),
ExportFormat::RenderedDng => return Err("Render to DNG needs a float render".into()),
};
r.map_err(|e| e.to_string())
}
@@ -1431,6 +1470,7 @@ pub fn encode_rendered(r: &lightcraft_pipeline::Rendered, o: &ExportOptions, met
/// Encode a high-bit-depth image (see [`encode_rendered`]).
pub fn encode_deep(img: &DeepImage, o: &ExportOptions, meta: Option<&Metadata>) -> Result<Vec<u8>, String> {
let meta_src = meta;
let img = finish_deep(img, o);
let profile = match img.samples {
DeepSamples::F32(_) => icc::write_matrix_trc(&img.space.rgb_space(), &lightcraft_codecs::Trc::Linear),
@@ -1449,8 +1489,12 @@ pub fn encode_deep(img: &DeepImage, o: &ExportOptions, meta: Option<&Metadata>)
(ExportFormat::Png, _) => encode::encode_png(&e, &meta),
(ExportFormat::Tiff, _) => encode::encode_tiff(&e, o.tiff_compression, &meta),
(ExportFormat::Psd, _) => encode::encode_psd(&e, &meta),
(ExportFormat::RenderedDng, DeepSamples::F32(v)) => {
let m = meta_src.cloned().unwrap_or_default();
return lightcraft_raw::write_rendered_dng(img.width, img.height, v, &img.space.rgb_space(), &m, xmp.clone()).map_err(|e| e.to_string());
}
(ExportFormat::Avif, DeepSamples::F32(v)) if img.space == OutputSpace::Rec2020 => {
lightcraft_codecs::encode_avif_pq(w, h, v, o.quality, 8, &meta)
lightcraft_codecs::encode_avif_hdr(w, h, v, o.quality, 8, &meta, o.hdr_transfer.codec())
}
(ExportFormat::Avif, _) => encode::encode_avif(&e, o.quality, 8, &meta),
(ExportFormat::Jxl, DeepSamples::F32(_)) if img.space == OutputSpace::Rec2020 => {
@@ -1536,9 +1580,11 @@ pub struct Exported {
/// Output size of photo `p` under `o` (its cropped full size when `o.resize` is `None`).
pub fn output_size(p: &lightcraft_catalog::Photo, o: &ExportOptions) -> (usize, usize) {
let (w, h) = lightcraft_pipeline::native_output_size(p.width.max(1) as usize, p.height.max(1) as usize, &p.develop);
// whole pixels first: a crop's float noise (5999.999… for 6000) must not decide a half-pixel tie
let (w, h) = (w.round().max(1.0), h.round().max(1.0));
match &o.resize {
Some(r) => r.apply(w, h),
None => ((w.round() as usize).max(1), (h.round() as usize).max(1)),
None => (w as usize, h as usize),
}
}
@@ -2696,6 +2742,47 @@ mod tests {
assert_eq!(size(json!({"width": 1}), 6000.0, 40.0).map(|s| s.1), Some(1));
}
#[test]
fn resize_size_survives_the_render() {
// the render fits the photo into the size `apply` gives; that must not cost a pixel (#1312)
let cropped = |r: Resize, w: u32, h: u32, x1: f64| {
let mut p = named("IMG.ARW");
(p.width, p.height) = (w, h);
std::sync::Arc::make_mut(&mut p.develop).crop.geometry.rect = lightcraft_geom::Rect::new(0.0, 0.0, x1, 1.0);
let (bw, bh) = output_size(&p, &ExportOptions { resize: Some(r), ..Default::default() });
lightcraft_pipeline::output_size(w as usize, h as usize, &p.develop, &lightcraft_pipeline::RenderRequest::fit(bw, bh))
};
let rendered = |r: Resize, w: u32, h: u32| cropped(r, w, h, 1.0);
let mode = |mode: ResizeMode, value: f32, height: u32| Resize { mode, value, height, dont_enlarge: true };
// a frame a hair narrower than 3:2 (a raw's default crop in floats) puts 1333 × 1.5 just under 1999.5
let hair = 1.0 - 1e-9;
assert_eq!(cropped(Resize::long_edge(2000), 6000, 4000, hair), (2000, 1333));
assert_eq!(cropped(Resize::long_edge(2048), 6000, 4000, hair), (2048, 1365));
assert_eq!(cropped(mode(ResizeMode::Width, 2000.0, 0), 6000, 4000, hair), (2000, 1333));
assert_eq!(cropped(mode(ResizeMode::ShortEdge, 1333.0, 0), 6000, 4000, hair), (2000, 1333));
assert_eq!(cropped(mode(ResizeMode::Height, 1333.0, 0), 6000, 4000, hair), (2000, 1333));
assert_eq!(rendered(Resize::long_edge(2000), 4000, 6000), (1333, 2000));
// 3:2, 4:3, 16:9, 1:1 and their portraits, every mode: the render keeps what `apply` says
for (w, h) in [(6000u32, 4000u32), (6240, 4160), (4032, 3024), (5184, 3888), (3840, 2160), (6000, 3375), (4000, 4000)] {
for (w, h) in [(w, h), (h, w)] {
for value in [1000.0, 1080.0, 1333.0, 2000.0, 2048.0, 2500.0, 3000.0] {
for r in [
Resize::long_edge(value as u32),
mode(ResizeMode::ShortEdge, value, 0),
mode(ResizeMode::Width, value, 0),
mode(ResizeMode::Height, value, 0),
mode(ResizeMode::Dimensions, value, 1365),
mode(ResizeMode::Megapixels, value / 1000.0, 0),
mode(ResizeMode::Percent, value / 100.0, 0),
] {
let want = r.apply(w.into(), h.into());
assert_eq!(rendered(r, w, h), want, "{r:?} on {w}×{h}");
}
}
}
}
}
/// A flat field of a saturated green inside Display P3 but outside sRGB, rendered into `space`.
fn p3_green(space: OutputSpace) -> Rgba8 {
use lightcraft_color::{DISPLAY_P3, REC2020};
@@ -3109,6 +3196,26 @@ mod tests {
}
// Issue #496: photos rendered side by side give the same files, names, order and progress as
// Render to DNG: the rendered photo as a linear float DNG that reads back as the same picture
#[test]
fn rendered_dng_matches_the_rendered_tiff() {
let mut s = crate::Session::with_demo();
let id = s.visible()[0];
let dng = export_photo(&mut s, id, &ExportOptions::from_json(&json!({"format": "rendereddng", "width": 64})), 1).unwrap();
assert!(dng.file_name.ends_with(".dng"), "{}", dng.file_name);
let png = export_photo(&mut s, id, &ExportOptions::from_json(&json!({"format": "png", "width": 64})), 1).unwrap();
let raw = lightcraft_raw::decode(&dng.bytes).unwrap();
assert_eq!((raw.width, raw.height, raw.cpp), (dng.width, dng.height, 3));
assert!(raw.color.profile.tone_curve.is_some(), "identity tone curve keeps converters from adding one");
let back = raw.develop(lightcraft_raw::Method::Bilinear).unwrap();
let reference = lightcraft_codecs::decode_thumbnail(&png.bytes, 4096).unwrap().image;
assert_eq!((reference.width, reference.height), (back.width, back.height));
// mean brightness: the PNG holds 8-bit sRGB, the DNG linear sRGB
let mean_dng: f32 = back.data.iter().map(|p| lightcraft_color::transfer::linear_to_srgb(p[1].min(1.0))).sum::<f32>() / back.data.len() as f32;
let mean_tif: f32 = reference.data.iter().map(|p| p[1] as f32 / 255.0).sum::<f32>() / reference.data.len() as f32;
assert!((mean_dng - mean_tif).abs() < 0.02, "{mean_dng} vs {mean_tif}");
}
// one at a time, and a cancel stops at the same photo
#[test]
fn side_by_side_batches_match_one_at_a_time() {
+401 -24
View File
@@ -176,6 +176,7 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result<ProbeInfo, String> {
kind: MediaKind::Raw,
file_size: bytes.len() as u64,
captured,
file_modified: None,
meta,
as_shot_wb,
content_hash,
@@ -207,6 +208,7 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result<ProbeInfo, String> {
kind: MediaKind::Image,
file_size: bytes.len() as u64,
captured,
file_modified: None,
meta,
as_shot_wb: None,
content_hash,
@@ -508,15 +510,16 @@ fn load_bytes_now(
///
/// A file with colour matrices of its own (DNG) has an absolute scale from its tags. A file without
/// (`camera_preview::file_local_look`) has one when its camera's matrices are measured
/// (`lightcraft_raw::spectral`): they turn its as-shot multipliers into Kelvin and tint, nothing else — the look
/// stays the file's fit. Any other raw is edited relative to its as-shot look (6500 K / 0 means as shot).
/// (`lightcraft_raw::spectral`), else when its camera's white-balance presets give one
/// (`lightcraft_raw::kelvin_presets`): either turns its as-shot multipliers into Kelvin and tint, nothing else — the
/// look stays the file's fit. Any other raw is edited relative to its as-shot look (6500 K / 0 means as shot).
struct WbScaleOf {
own_matrix: bool,
relative: bool,
/// Kelvin through the camera's measured matrices: edited on the relative scale before those were used.
/// Kelvin through the camera's measured matrices or presets: edited on the relative scale before those were used.
legacy_relative: bool,
/// The as-shot white through the measured matrices, when that is where the scale comes from.
spectral_white: Option<lightcraft_color::Xy>,
/// The as-shot white through the measured matrices or the presets, when that is where the scale comes from.
scale_white: Option<lightcraft_color::Xy>,
}
impl WbScaleOf {
@@ -528,18 +531,26 @@ impl WbScaleOf {
) -> Self {
let own_matrix = lightcraft_raw::color::has_matrix(color);
let fitted = crate::camera_preview::file_local_look(format) && !own_matrix;
let camera = fitted.then(|| meta.model.as_deref().and_then(|model| lightcraft_raw::spectral::find(meta.make.as_deref(), model))).flatten();
let spectral_white = camera.and_then(|c| lightcraft_raw::spectral::as_shot_white_through(c, color, wb_multipliers));
WbScaleOf { own_matrix, relative: fitted && camera.is_none(), legacy_relative: fitted && camera.is_some(), spectral_white }
let model = meta.model.as_deref().filter(|_| fitted);
let camera = model.and_then(|model| lightcraft_raw::spectral::find(meta.make.as_deref(), model));
let presets = camera.is_none() && model.is_some_and(|model| lightcraft_raw::kelvin_presets::covers(meta.make.as_deref(), model));
let scale_white = match camera {
Some(c) => lightcraft_raw::spectral::as_shot_white_through(c, color, wb_multipliers),
None if presets => lightcraft_raw::kelvin_presets::as_shot_white(color, wb_multipliers),
None => None,
};
let covered = camera.is_some() || presets;
WbScaleOf { own_matrix, relative: fitted && !covered, legacy_relative: fitted && covered, scale_white }
}
/// The as-shot temperature and tint on the file's scale: the relative scale's reference, the white through
/// the measured matrices, else `own` (the white the file's own tags give). Whole numbers, as the catalog
/// keeps them. A covered camera whose file names no white (no multipliers) falls back to `own`, D65.
/// the measured matrices or the presets, else `own` (the white the file's own tags give). Whole numbers, as the
/// catalog keeps them. A covered camera whose file names no white (no multipliers, or no presets) falls back to
/// `own`, D65.
fn as_shot(&self, own: lightcraft_color::Xy) -> (f64, f64) {
if self.relative {
return (6500.0, 0.0);
}
let (t, tint) = xy_to_temp_tint(self.spectral_white.unwrap_or(own));
let (t, tint) = xy_to_temp_tint(self.scale_white.unwrap_or(own));
(t.round(), tint.round())
}
}
@@ -643,9 +654,16 @@ pub(crate) fn decode_raw_preview(bytes: &[u8], opts: lightcraft_codecs::DecodeOp
/// Oriented size of the embedded preview of a raw file.
fn embedded_preview_size(bytes: &[u8]) -> Option<(u32, u32)> {
let jpeg = stand_in_image(bytes)?;
let d = lightcraft_codecs::decode(&jpeg, lightcraft_codecs::DecodeOptions::fit(64, 64)).ok()?;
let (mut w, mut h) = (d.source_width, d.source_height);
if preview_orientation(bytes, d.orientation).swaps_axes() {
// headers only, as for other images (issue #367): decoding a full-size preview just for its
// size took about a second per file in an import; a decode remains the fallback
let (mut w, mut h, orientation) = match lightcraft_codecs::read_header(&jpeg) {
Ok(header) => (header.width, header.height, header.orientation),
Err(_) => {
let d = lightcraft_codecs::decode(&jpeg, lightcraft_codecs::DecodeOptions::fit(64, 64)).ok()?;
(d.source_width, d.source_height, d.orientation)
}
};
if preview_orientation(bytes, orientation).swaps_axes() {
std::mem::swap(&mut w, &mut h);
}
Some((w, h))
@@ -681,9 +699,10 @@ fn fit_box(max_edge: usize) -> lightcraft_codecs::DecodeOptions {
/// Filesystem-backed embedded-preview hook (native).
///
/// The quick path reads the file's header and its embedded preview and nothing else (see
/// [`preview_window`]); a format whose preview the header cannot point at falls back to the
/// whole-file path, which is what every format used to cost. The memory gate still counts the whole
/// file, which is what the decode of one can hold.
/// [`preview_window`]); a container the header cannot point into is planned from its tags
/// (`lightcraft_raw::sparse::preview_plan`, the same module the import path probes with), and only a
/// file the planner cannot describe falls back to the whole-file path. The memory gate still counts
/// the whole file, which is what the decode of one can hold.
pub fn fs_preview_loader() -> PreviewLoader {
Arc::new(|path: &str, max_edge: usize| {
let len = std::fs::metadata(path).map(|m| usize::try_from(m.len()).unwrap_or(usize::MAX)).unwrap_or(0);
@@ -706,6 +725,10 @@ pub struct PreviewRead {
/// Whether the preview came from a bounded read (`false`: the whole file was read, which is what
/// every preview cost before the header path existed).
pub header_path: bool,
/// Whether the sparse planner named the ranges, rather than a format's own header fields: the
/// path a Canon CR3 and the TIFF raws whose preview tag sits past the prefix take (`false` on
/// the header path, and on the whole-file fallback).
pub planned: bool,
}
impl PreviewRead {
@@ -745,7 +768,13 @@ pub fn preview_window(path: &str) -> Result<Option<(Vec<u8>, Vec<u8>, PreviewRea
{
prefix = read_prefix_at(&mut file, path, extent.min(total))?;
}
let Some(range) = lightcraft_raw::bounded_preview(&prefix) else { return Ok(None) };
let Some(range) = lightcraft_raw::bounded_preview(&prefix) else {
// The header carries no pointer this path reads — a Canon CR3's boxes, a TIFF raw whose
// preview tag sits past the prefix, the containers whose preview only a scan finds: ask the
// sparse planner which ranges `embedded_preview` reads before handing the caller the
// whole-file fallback.
return planned_preview(&mut file, path, total);
};
// A preview at the file's end may have been padded past it: clamp rather than fail.
let end = range.offset.saturating_add(range.window_len).min(total);
let len = usize::try_from(end.saturating_sub(range.offset)).map_err(|_| format!("{path}: preview too large"))?;
@@ -767,7 +796,7 @@ pub fn preview_window(path: &str) -> Result<Option<(Vec<u8>, Vec<u8>, PreviewRea
if lightcraft_raw::embedded_preview_from_range(&window, range).is_none() {
return Ok(None);
}
let read = PreviewRead { read: bytes_read, total, header_path: true };
let read = PreviewRead { read: bytes_read, total, header_path: true, planned: false };
// On a network share the bytes are the wait, so say what this preview actually cost. A window
// taken from the prefix cost nothing beyond the prefix itself.
if lightcraft_pipeline::profiling() {
@@ -776,6 +805,107 @@ pub fn preview_window(path: &str) -> Result<Option<(Vec<u8>, Vec<u8>, PreviewRea
Ok(Some((prefix, window, read)))
}
/// The most times the sparse planner may name ranges before the preview gives up: the plan is
/// monotone (it never asks for a range twice), so a real file needs a handful.
const MAX_PREVIEW_PLAN_ROUNDS: usize = 8;
/// Tenths of a file the planner may have read before the preview gives up and the whole-file path
/// takes over. The import path bounds its own loop the same way; a "preview" that wants most of the
/// file has stopped being one.
const MAX_PREVIEW_PLAN_SHARE: u64 = 6;
/// The largest file the planner path builds its buffer for. The raws it serves are far below this;
/// a file past it is read whole as before rather than allocating its length up front.
const MAX_PREVIEW_PLAN_BYTES: u64 = 1 << 30;
/// The header the quick path is handed on the planner route. The header route grows its prefix to
/// the preview's declared extent, so `quick_image_srgb` can read the file's own EXIF orientation out
/// of it when the preview JPEG carries none; this route reads a bounded prefix instead, big enough
/// for IFD0 (which is where that tag lives) and for a CR3's metadata boxes.
const PLAN_PREFIX: u64 = 64 * 1024;
/// The preview of a container the header-only path cannot answer for, from the byte ranges
/// [`lightcraft_raw::sparse::preview_plan`] says [`lightcraft_raw::embedded_preview`] reads.
///
/// The planner is asked about a buffer of the file's whole length in which only the ranges named so
/// far hold bytes, and only those ranges are read: the sensor data in the bulk ranges is not. The
/// buffer is the file's length, which the loader's memory gate already counts, and its untouched
/// pages stay zero and are never written — though on macOS a freed buffer of that size may stay in
/// the allocator's cache rather than going back to the OS (measured on a 38k-photo import, #832), so
/// this path's cost is the bytes read, not a promise about RSS.
///
/// `None` — the caller's cue to read the file whole, exactly as when the header declines — when the
/// planner says it cannot describe the file (NEF, ORF and PEF probe from samples; a structure it
/// does not trust), when the ranges it names do not fit the buffer or want most of the file, or when
/// the quick path could not reproduce what the whole-file path shows (the two container facts
/// below). The planner's contract is the one its corpus test asserts over real files (what it loaded
/// gives what the whole file gives), so a window that still fails to decode ends in the same
/// whole-file fallback in [`fs_preview_loader`], never in a wrong picture.
fn planned_preview(file: &mut std::fs::File, path: &str, total: u64) -> Result<Option<(Vec<u8>, Vec<u8>, PreviewRead)>, String> {
use std::io::{Read, Seek, SeekFrom};
if total > MAX_PREVIEW_PLAN_BYTES {
return Ok(None);
}
let Ok(len) = usize::try_from(total) else { return Ok(None) };
let mut buf = vec![0u8; len];
let header = read_prefix_at(file, path, PLAN_PREFIX.min(total))?;
let Some(head) = buf.get_mut(..header.len()) else { return Ok(None) };
head.copy_from_slice(&header);
let mut loaded = lightcraft_raw::sparse::Ranges::new();
loaded.insert(0, header.len() as u64);
for _ in 0..MAX_PREVIEW_PLAN_ROUNDS {
match lightcraft_raw::sparse::preview_plan(&buf, &loaded) {
lightcraft_raw::sparse::HeaderPlan::Need(ranges) => {
let wanted = loaded.total().saturating_add(ranges.total());
if wanted.saturating_mul(10) > total.saturating_mul(MAX_PREVIEW_PLAN_SHARE) {
return Ok(None);
}
for &(start, end) in ranges.spans() {
let (Ok(a), Ok(b)) = (usize::try_from(start), usize::try_from(end)) else { return Ok(None) };
let Some(dst) = buf.get_mut(a..b) else { return Ok(None) };
file.seek(SeekFrom::Start(start)).map_err(|e| format!("{path}: {e}"))?;
file.read_exact(dst).map_err(|e| format!("{path}: {e}"))?;
loaded.insert(start, end);
}
}
lightcraft_raw::sparse::HeaderPlan::Ready => {
let Some(window) = lightcraft_raw::embedded_preview(&buf) else { return Ok(None) };
// The quick path hands the JPEG to the decoder alone, so what the *container* says
// about the preview has to be something the bytes it is handed can show. Two container
// facts change the pixels and are not in the JPEG: the preview's colour space (which
// the whole-file path applies through `decode_raw_preview`) and, when the JPEG carries
// no orientation of its own, the file's EXIF orientation. Where either is present and
// the header cannot answer for it, the whole-file path keeps the preview — exactly as
// it already does for a container-declared colour space. A container that says sRGB
// agrees with the JPEG-only decode (its own declaration, else sRGB), so only the other
// space declines.
if lightcraft_raw::embedded_preview_color_space(&buf) == Some(lightcraft_raw::PreviewColorSpace::AdobeRgb) {
return Ok(None);
}
// `extract` copies a TIFF's bytes to hand on its Exif block, so ask the cheap side
// first: a JPEG that carries its own orientation needs nothing from the container.
if lightcraft_meta::extract(&window).orientation.is_none()
&& lightcraft_meta::extract(&header).orientation.is_none()
&& lightcraft_meta::extract(&buf).orientation.is_some()
{
return Ok(None);
}
let read = PreviewRead { read: loaded.total(), total, header_path: true, planned: true };
// A plan-driven preview is the one that used to cost the whole file, so it is worth
// saying what it cost now too.
if lightcraft_pipeline::profiling() {
eprintln!("[profile] preview {path}: read {} of {} bytes (plan, window {})", read.read, total, window.len());
}
// The planner's buffer is the file's length, so give its pages back the way a decode
// does (see `fs_pair_loader`).
drop(buf);
crate::memory::release();
return Ok(Some((header, window, read)));
}
lightcraft_raw::sparse::HeaderPlan::Whole(_) => return Ok(None),
}
}
Ok(None)
}
/// The file's first [`lightcraft_raw::PREVIEW_PREFIX`] bytes (or the whole file when it is shorter),
/// which is what every format [`lightcraft_raw::bounded_preview`] covers keeps its pointer in.
fn read_prefix(file: &mut std::fs::File, path: &str, total: u64) -> Result<Vec<u8>, String> {
@@ -856,11 +986,21 @@ pub fn fs_hooks() -> (FileLoader, FileProbe) {
let len = std::fs::metadata(path).map(|m| usize::try_from(m.len()).unwrap_or(usize::MAX)).unwrap_or(0);
let _permit = crate::memory::work_gate().acquire(len);
let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;
probe_bytes(path, &bytes)
let mut info = probe_bytes(path, &bytes)?;
info.file_modified = file_modified(path);
Ok(info)
});
(loader, probe)
}
/// `path`'s modification time as ISO 8601 (see [`ProbeInfo::file_modified`]); `None` when the
/// filesystem doesn't say or the time is before 1970.
fn file_modified(path: &str) -> Option<String> {
let t = std::fs::metadata(path).ok()?.modified().ok()?;
let secs = i64::try_from(t.duration_since(std::time::UNIX_EPOCH).ok()?.as_secs()).ok()?;
Some(crate::import::civil(secs))
}
impl crate::Session {
/// Install the filesystem file hooks (desktop, CLI, MCP headless), and on macOS the Apple Photos
/// runner ([`crate::apple_photos::osascript`]; installed, not run).
@@ -908,28 +1048,49 @@ mod tests {
let d65 = lightcraft_color::D65;
// a covered camera: Kelvin through the spectral matrices, near the daylight these multipliers describe
let a7m3 = WbScaleOf::new(RawFormat::Arw, &ColorData::default(), mul, &meta("SONY", "ILCE-7M3"));
assert!(!a7m3.own_matrix && !a7m3.relative && a7m3.legacy_relative && a7m3.spectral_white.is_some());
assert!(!a7m3.own_matrix && !a7m3.relative && a7m3.legacy_relative && a7m3.scale_white.is_some());
let (t, tint) = a7m3.as_shot(d65);
assert!((3500.0..7000.0).contains(&t) && t != 6500.0 && tint.abs() < 30.0 && t.fract() == 0.0 && tint.fract() == 0.0, "{t} {tint}");
assert!(!lightcraft_catalog::relative_wb_camera("ARW", "SONY ILCE-7M3"));
// the same camera without multipliers: the scale is Kelvin still, the white falls back to the tags' (D65)
let bare = WbScaleOf::new(RawFormat::Arw, &ColorData::default(), None, &meta("SONY", "ILCE-7M3"));
assert!(bare.legacy_relative && bare.spectral_white.is_none());
assert!(bare.legacy_relative && bare.scale_white.is_none());
assert_eq!(bare.as_shot(d65), {
let (t, tint) = xy_to_temp_tint(d65);
(t.round(), tint.round())
});
// a camera without measured matrices: relative, whatever the multipliers
let a7m2 = WbScaleOf::new(RawFormat::Arw, &ColorData::default(), mul, &meta("SONY", "ILCE-7M2"));
assert!(a7m2.relative && !a7m2.legacy_relative && a7m2.spectral_white.is_none());
assert!(a7m2.relative && !a7m2.legacy_relative && a7m2.scale_white.is_none());
assert_eq!(a7m2.as_shot(d65), (6500.0, 0.0));
assert!(lightcraft_catalog::relative_wb_camera("ARW", "SONY ILCE-7M2"));
// a camera without matrices whose presets give the scale: Kelvin through them, Sony's daylight preset as
// 5500 K or so; without presets in the file the scale is Kelvin still, the white the tags' (D65)
let g = |r: f32, b: f32| [r / 1024.0, 1.0, b / 1024.0];
let presets = lightcraft_raw::kelvin_presets::KelvinPresets::new([
(2500.0, g(1336.0, 3672.0)),
(3200.0, g(1720.0, 2620.0)),
(4500.0, g(2256.0, 1880.0)),
(6000.0, g(2680.0, 1544.0)),
(8500.0, g(3136.0, 1300.0)),
]);
let with = ColorData { kelvin_presets: presets, ..Default::default() };
let a99m2 = WbScaleOf::new(RawFormat::Arw, &with, Some(g(2500.0, 1668.0)), &meta("SONY", "ILCA-99M2"));
assert!(!a99m2.own_matrix && !a99m2.relative && a99m2.legacy_relative && a99m2.scale_white.is_some());
let (t, tint) = a99m2.as_shot(d65);
assert!((5000.0..6000.0).contains(&t) && tint.abs() < 10.0 && t.fract() == 0.0, "{t} {tint}");
assert!(!lightcraft_catalog::relative_wb_camera("ARW", "SONY ILCA-99M2"));
let bare = WbScaleOf::new(RawFormat::Arw, &ColorData::default(), mul, &meta("SONY", "ILCA-99M2"));
assert!(bare.legacy_relative && bare.scale_white.is_none());
// presets in a file of a camera not listed: relative, as the catalog decides from the camera alone
let other = WbScaleOf::new(RawFormat::Arw, &with, mul, &meta("SONY", "ILCE-1"));
assert!(other.relative && other.scale_white.is_none());
let unknown = WbScaleOf::new(RawFormat::Nef, &ColorData::default(), mul, &lightcraft_meta::Metadata::default());
assert!(unknown.relative && unknown.as_shot(d65) == (6500.0, 0.0));
// a file with matrices of its own (DNG): absolute from them, no matter whose camera
let own = ColorData { illuminant: [21, 0], color_matrix: [Some(lightcraft_color::Mat3::IDENTITY), None], ..Default::default() };
let dng = WbScaleOf::new(RawFormat::Dng, &own, mul, &meta("SONY", "ILCE-7M3"));
assert!(dng.own_matrix && !dng.relative && !dng.legacy_relative && dng.spectral_white.is_none());
assert!(dng.own_matrix && !dng.relative && !dng.legacy_relative && dng.scale_white.is_none());
let xy = lightcraft_color::cct::temp_tint_to_xy(4200.0, 12.0);
assert_eq!(dng.as_shot(xy), (4200.0, 12.0));
assert!(!lightcraft_catalog::relative_wb_camera("DNG", "SONY ILCE-7M3"));
@@ -967,6 +1128,7 @@ mod tests {
let (header, window, read) = preview_window(&path).expect("the file reads").expect("it has a preview");
assert_eq!(header.len(), lightcraft_raw::PREVIEW_PREFIX, "the header prefix is bounded");
assert!(read.header_path, "the header path is what answered");
assert!(!read.planned, "a RAF pointer in the header needs no planner");
assert_eq!(read.total, file.len() as u64);
assert!(read.fraction() < 0.5, "the read is a fraction of the file: {read:?}");
assert!(window.len() < file.len() / 2, "the raw tail must not be read: {} of {} bytes", window.len(), file.len());
@@ -985,6 +1147,179 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
/// What a fixture's IFD0 says about itself: the tag that makes `probe` call the file a raw, the
/// file's own orientation, and the EXIF colour space the container declares for its preview.
#[derive(Clone, Copy, Default)]
struct Fixture {
dng: bool,
sony: bool,
orientation: Option<u16>,
/// EXIF `ColorSpace` (0xA001): 1 = sRGB, 0xFFFF with an "R03" InteropIndex = Adobe RGB.
color_space: Option<u16>,
}
/// A TIFF-family raw no header field points into: IFD0 is the JPEG-compressed preview strip (the
/// shape a DNG's `PreviewImage`, a CR2's or an ARW's preview takes), IFD1 a CFA "sensor" strip of
/// `tail` bytes. The writer puts each IFD's table after its own chunk, so a `strip` bigger than
/// the plan's bounded header puts IFD0 (and any orientation tag in it) out of that header's reach.
fn tiff_with_preview_strip(strip: &[u8], tail: usize, what: Fixture) -> Vec<u8> {
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value, tags as t};
let mut preview = IfdBuilder::new();
if what.dng {
preview.set(t::DNG_VERSION, Value::Byte(vec![1, 4, 0, 0]));
}
if what.sony {
preview.set(t::MAKE, Value::Ascii("SONY".into()));
}
if let Some(o) = what.orientation {
preview.set(t::ORIENTATION, Value::Short(vec![o]));
}
if let Some(space) = what.color_space {
let mut exif = IfdBuilder::new();
exif.set(0xA001, Value::Short(vec![space]));
if space == 0xFFFF {
let mut interop = IfdBuilder::new();
interop.set(0x0001, Value::Undefined(b"R03".to_vec()));
exif.set_child(t::INTEROP_IFD, interop);
}
preview.set_child(t::EXIF_IFD, exif);
}
preview.set(t::COMPRESSION, Value::Short(vec![7]));
preview.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::RGB]));
preview.set(t::IMAGE_WIDTH, Value::Long(vec![48]));
preview.set(t::IMAGE_LENGTH, Value::Long(vec![32]));
preview.set_image(ImageData::Strips { rows_per_strip: 32, strips: vec![strip.to_vec()] });
let mut raw = IfdBuilder::new();
raw.set(t::COMPRESSION, Value::Short(vec![1]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA]));
raw.set(t::IMAGE_WIDTH, Value::Long(vec![6000]));
raw.set(t::IMAGE_LENGTH, Value::Long(vec![4000]));
raw.set_image(ImageData::Strips { rows_per_strip: 4000, strips: vec![vec![0x33u8; tail]] });
TiffWriter::default().write(&[preview, raw]).expect("the fixture writes")
}
/// A raw whose preview only its tags locate — a Canon CR3's JPEG track, a TIFF raw whose preview
/// tag sits past the prefix — is read in part too: `sparse::preview_plan` names the ranges
/// `embedded_preview` reads, so the preview strip is read and the sensor strip is not. The pixels
/// must equal the whole-file path's, exactly; a file the planner cannot describe must decline
/// rather than guess, so the caller keeps reading it whole.
#[test]
fn preview_loading_plans_the_ranges_it_needs() {
let (w, h) = (48u32, 32u32);
let rgb: Vec<u8> = (0..w * h * 3).map(|i| (i * 29 % 241) as u8).collect();
let img = EncodeImage::new(w, h, 3, Samples::U8(&rgb));
let jpeg = encode_jpeg(&img, 90, ChromaSubsampling::S420, &EncodeMeta::default()).unwrap();
let file = tiff_with_preview_strip(&jpeg, 400_000, Fixture { dng: true, ..Default::default() });
let dir = crate::tests_xmp::temp_dir("preview-plan");
let path = dir.join("preview.dng");
std::fs::write(&path, &file).unwrap();
let path = path.to_string_lossy().into_owned();
let max_edge = 512;
let (header, window, read) = preview_window(&path).expect("the file reads").expect("the plan locates its preview");
assert_eq!(header.len(), PLAN_PREFIX as usize, "the plan's header is bounded");
assert!(read.planned, "the planner is what answered");
assert!(read.header_path, "and it is still a bounded read");
assert_eq!(read.total, file.len() as u64);
assert!(read.fraction() < 0.5, "the read is a fraction of the file: {read:?} of {} bytes", file.len());
assert!(window.len() < file.len() / 2, "the sensor strip must not be read: {} of {} bytes", window.len(), file.len());
// The plan path and the whole-file path agree byte for byte.
let quick = quick_image_srgb(&header, &window, max_edge).expect("the window holds a decodable preview");
let whole = embedded_preview_srgb(&file, max_edge).expect("the whole file decodes too");
assert_eq!((quick.width, quick.height), (whole.width, whole.height), "same preview size");
assert_eq!(quick.data, whole.data, "byte for byte");
// The same structure without the tag that makes it a raw: the planner declines, and the
// caller's whole-file read is what answers (no guessing at what a file might hold).
let not_a_raw = tiff_with_preview_strip(&jpeg, 400_000, Fixture::default());
std::fs::write(&path, &not_a_raw).unwrap();
assert!(preview_window(&path).expect("the file reads").is_none(), "an unknown container falls back");
let _ = std::fs::remove_dir_all(&dir);
}
/// A file whose preview the quick path could not show the way the whole-file path does is left to
/// the whole file. Here the orientation tag sits in an IFD0 past the bounded header (its chunk is
/// bigger than that header) and the preview JPEG carries none of its own, so a window read would
/// come out unrotated where the whole-file path rotates it — the plan must decline instead of
/// handing on a wrong picture.
#[test]
fn preview_loading_declines_an_orientation_only_the_whole_file_shows() {
let (w, h) = (48u32, 32u32);
let rgb: Vec<u8> = (0..w * h * 3).map(|i| (i * 17 % 239) as u8).collect();
let img = EncodeImage::new(w, h, 3, Samples::U8(&rgb));
let jpeg = encode_jpeg(&img, 90, ChromaSubsampling::S420, &EncodeMeta::default()).unwrap();
// the strip (what the IFD's chunk covers) is padded past the plan's header, which is what puts
// IFD0 out of its reach; `embedded_preview` trims back to the JPEG's end-of-image
let mut strip = jpeg.clone();
strip.extend(std::iter::repeat_n(0u8, PLAN_PREFIX as usize));
let file = tiff_with_preview_strip(&strip, 400_000, Fixture { dng: true, orientation: Some(6), ..Default::default() });
let dir = crate::tests_xmp::temp_dir("preview-plan-orientation");
let path = dir.join("rotated.dng");
std::fs::write(&path, &file).unwrap();
let path = path.to_string_lossy().into_owned();
assert_eq!(lightcraft_meta::extract(&file).orientation, Some(Orientation::from_exif(6)), "the fixture states an orientation");
assert!(lightcraft_meta::extract(&file[..PLAN_PREFIX as usize]).orientation.is_none(), "which the bounded header cannot see");
assert!(lightcraft_meta::extract(&jpeg).orientation.is_none(), "and the preview JPEG carries none of its own");
assert!(preview_window(&path).expect("the file reads").is_none(), "the whole-file path keeps it");
let _ = std::fs::remove_dir_all(&dir);
}
/// A container can declare what its preview JPEG is in, and the whole-file path applies that
/// declaration (`decode_raw_preview`). A JPEG-only decode has the JPEG's own space, else sRGB, so
/// a container that declares Adobe RGB must keep the whole-file path — while one that declares
/// sRGB (what most bodies write) must **not** take the fast path away: that half is a regression
/// test, since a guard that declined every declared space did exactly that to a real Sony ARW.
#[test]
fn preview_loading_declines_only_a_container_colour_space_it_cannot_apply() {
use lightcraft_raw::PreviewColorSpace;
let (w, h) = (48u32, 32u32);
let rgb: Vec<u8> = (0..w * h * 3).map(|i| (i * 23 % 233) as u8).collect();
let img = EncodeImage::new(w, h, 3, Samples::U8(&rgb));
let jpeg = encode_jpeg(&img, 90, ChromaSubsampling::S420, &EncodeMeta::default()).unwrap();
let dir = crate::tests_xmp::temp_dir("preview-plan-space");
let path = dir.join("space.arw");
let path = path.to_string_lossy().into_owned();
let adobe = tiff_with_preview_strip(&jpeg, 400_000, Fixture { sony: true, color_space: Some(0xFFFF), ..Default::default() });
assert_eq!(lightcraft_raw::embedded_preview_color_space(&adobe), Some(PreviewColorSpace::AdobeRgb), "the fixture declares Adobe RGB");
std::fs::write(&path, &adobe).unwrap();
assert!(preview_window(&path).expect("the file reads").is_none(), "the whole-file path applies the space");
let srgb = tiff_with_preview_strip(&jpeg, 400_000, Fixture { sony: true, color_space: Some(1), ..Default::default() });
assert_eq!(lightcraft_raw::embedded_preview_color_space(&srgb), Some(PreviewColorSpace::Srgb), "the fixture declares sRGB");
std::fs::write(&path, &srgb).unwrap();
let (_, window, read) = preview_window(&path).expect("the file reads").expect("sRGB is what the JPEG-only decode assumes");
assert!(read.planned, "the plan still answers");
assert!(window.starts_with(&[0xff, 0xd8]), "and hands on the JPEG");
}
/// A JPEG with no date in its metadata (an export saved without it) takes the file's date, as
/// in Lightroom, instead of grouping under "Unknown Date"; a dated one keeps its capture time.
#[test]
fn undated_files_take_the_file_date() {
let dir = std::env::temp_dir().join(format!("lightcraft-undated-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let rgb = vec![128u8; 16 * 16 * 3];
let img = EncodeImage::new(16, 16, 3, Samples::U8(&rgb));
let jpeg = encode_jpeg(&img, 90, ChromaSubsampling::S420, &EncodeMeta::default()).unwrap();
let path = dir.join("export.jpg");
std::fs::write(&path, &jpeg).unwrap();
let path = path.to_string_lossy().to_string();
let mtime = std::fs::metadata(&path).unwrap().modified().unwrap().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs();
let want = crate::import::civil(mtime as i64);
let (_, probe) = fs_hooks();
let info = probe(&path).unwrap();
assert_eq!(info.captured, None, "the file itself has no date");
assert_eq!(info.file_modified.as_deref(), Some(want.as_str()));
let mut s = crate::Session::new().with_fs();
let r = crate::import::import(&mut s, &[path], crate::import::ImportMode::Add).unwrap();
let id = lightcraft_catalog::PhotoId(r.imported[0]);
assert_eq!(s.catalog.photo(id).unwrap().captured.as_deref(), Some(want.as_str()));
let _ = std::fs::remove_dir_all(&dir);
}
/// Issue #367: probes read headers, not pixels, and report what the decode-based probe did:
/// oriented dimensions (EXIF orientation 6 swaps them), metadata, the error for a truncated
/// file or one that isn't an image.
@@ -1016,6 +1351,17 @@ mod tests {
assert!(probe_bytes("x.jpg", b"not an image").is_err());
}
/// Issue #1212: a transparent PNG develops on white (what renders, previews, exports and contact sheets
/// start from), not on the black its transparent pixels store.
#[test]
fn transparent_png_develops_on_white() {
let rgba = [200, 40, 40, 255, 0, 0, 0, 0];
let png = lightcraft_codecs::encode_png(&EncodeImage::new(2, 1, 4, Samples::U8(&rgba)), &EncodeMeta::default()).unwrap();
let (img, _) = load_bytes(&png, 64).unwrap();
assert!(img.get(0, 0)[0] > 0.3 && img.get(0, 0)[1] < 0.1, "{:?}", img.get(0, 0));
assert!(img.get(1, 0).iter().all(|&c| (c - 1.0).abs() < 1e-4), "{:?}", img.get(1, 0));
}
/// Issue #548: a small preview bins the mosaic by 8; specular points that clip a single green sample
/// of a block on a grey surface must not become green speckles (the block's green used to take the
/// clip level, which highlight reconstruction then kept).
@@ -1846,6 +2192,37 @@ mod tests {
assert!(e.contains("without an embedded preview"), "{e}");
}
/// Issue #1275: an uncompressed Leaf MOS (IFD0 is the 16-bit mosaic, no Make tag; Leaf's data block `0x8606`
/// holds the preview) imports as a raw shown from its preview, not as a greyscale image of the mosaic.
#[test]
fn uncompressed_leaf_mos_imports_as_a_preview_only_raw() {
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value, tags as t};
let px: Vec<u8> = (0..40 * 30).flat_map(|i| [(i % 251) as u8, 128, 200]).collect();
let jpeg = encode_jpeg(&EncodeImage::new(40, 30, 3, Samples::U8(&px)), 90, ChromaSubsampling::S444, &EncodeMeta::default()).unwrap();
let mos = |leaf_data: bool| {
let mut ifd0 = IfdBuilder::new();
ifd0.set(t::IMAGE_WIDTH, Value::Long(vec![16]));
ifd0.set(t::IMAGE_LENGTH, Value::Long(vec![12]));
ifd0.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
ifd0.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
ifd0.set(t::PHOTOMETRIC, Value::Short(vec![1]));
ifd0.set(t::COMPRESSION, Value::Short(vec![1]));
ifd0.set_image(ImageData::Tiles { tile_width: 16, tile_height: 12, tiles: vec![vec![0x30; 16 * 12 * 2]] });
if leaf_data {
ifd0.set(0x8606, Value::Undefined([&b"PKTS\0\0\0\x01"[..], &jpeg].concat()));
}
TiffWriter::default().write(&[ifd0]).unwrap()
};
let p = probe_bytes("aptus22.mos", &mos(true)).unwrap();
assert_eq!((p.kind, p.format.as_str(), p.width, p.height), (MediaKind::Raw, "MOS", 40, 30));
assert!(p.preview_only.is_some_and(|why| !why.is_empty()));
let (img, src) = load_bytes(&mos(true), 20).unwrap();
assert_eq!((img.width, img.height, src.raw), (20, 15, false));
// the same grey image without Leaf's block is an ordinary 16-bit TIFF
let plain = probe_bytes("grey.tif", &mos(false)).unwrap();
assert_eq!((plain.kind, plain.preview_only, plain.width, plain.height), (MediaKind::Image, None, 16, 12));
}
/// A GoPro GPR (DNG with VC-5 coded data, no preview anywhere) still imports, described from its
/// headers; loading it says what is wrong instead of naming a bare TIFF compression number.
#[test]
+78 -9
View File
@@ -466,10 +466,18 @@ pub fn launch_path(arg: &str, cwd: Option<&Path>, home: Option<&Path>, dcim: imp
/// Probe files (headers + content hash) as an import would.
pub(crate) fn probe_paths(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
probe_all(s.media.file_probe.as_ref(), paths, &ScanProgress::default())
let progress = ScanProgress::default();
probe_all(s.media.file_probe.as_ref(), paths, &progress.done, &progress.cancel)
}
fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress: &ScanProgress) -> Vec<Result<ProbeInfo, String>> {
/// Probe `paths` (several at once), counting each in `done`. Stops before the next file once
/// `cancel` is set: the files not reached come back as errors.
fn probe_all(
probe: Option<&crate::media::FileProbe>,
paths: &[String],
done: &std::sync::atomic::AtomicUsize,
cancel: &std::sync::atomic::AtomicBool,
) -> Vec<Result<ProbeInfo, String>> {
use std::sync::atomic::Ordering::Relaxed;
let Some(probe) = probe.cloned() else {
return paths
@@ -497,11 +505,11 @@ fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress
if i >= paths.len() {
break;
}
if progress.cancel.load(Relaxed) {
if cancel.load(Relaxed) {
break;
}
let r = probe(&paths[i]);
progress.done.fetch_add(1, Relaxed);
done.fetch_add(1, Relaxed);
out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r);
}
});
@@ -513,11 +521,11 @@ fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress
paths
.iter()
.map(|p| {
if progress.cancel.load(Relaxed) {
if cancel.load(Relaxed) {
return Err("cancelled".into());
}
let r = probe(p);
progress.done.fetch_add(1, Relaxed);
done.fetch_add(1, Relaxed);
r
})
.collect()
@@ -620,7 +628,7 @@ pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress
.collect();
let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
progress.done.store(todo.len() - missing.len(), Relaxed);
let mut fresh = probe_all(input.probe.as_ref(), &missing, progress).into_iter();
let mut fresh = probe_all(input.probe.as_ref(), &missing, &progress.done, &progress.cancel).into_iter();
let probed: Vec<Result<ProbeInfo, String>> =
cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
crate::memory::release();
@@ -641,6 +649,8 @@ pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress
Some(Ok(info)) => {
(c.format, c.kind, c.width, c.height, c.file_size, c.captured) =
(info.format.clone(), info.kind, info.width, info.height, info.file_size, info.captured.clone());
// as the import will date it (a sidecar's date, when there is one, still wins there)
c.captured = c.captured.or_else(|| info.file_modified.clone());
c.preview_only = info.preview_only.clone();
if let Some(h) = &info.content_hash {
match seen_hash.get(h) {
@@ -705,6 +715,10 @@ pub struct ImportJob {
/// Files found so far (folders expanded) and handled so far, for progress.
pub total: std::sync::Arc<std::sync::atomic::AtomicUsize>,
pub done: std::sync::Arc<std::sync::atomic::AtomicUsize>,
/// Of those, the files that have to be read first (not probed by a preceding scan) and the
/// ones read so far: the step before they are handled.
pub read_total: std::sync::Arc<std::sync::atomic::AtomicUsize>,
pub read_done: std::sync::Arc<std::sync::atomic::AtomicUsize>,
}
/// The outcome of [`ImportJob::prepare`] for one batch, in file order.
@@ -857,6 +871,8 @@ impl ImportJob {
now: (s.clock)(),
total: Default::default(),
done: Default::default(),
read_total: Default::default(),
read_done: Default::default(),
})
}
@@ -917,8 +933,10 @@ impl ImportJob {
})
.collect();
let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
let progress = ScanProgress::default();
let mut fresh = probe_all(self.probe.as_ref(), &missing, &progress).into_iter();
// Reading the files is the slow part of adding them where they are: it is counted on its
// own, and `cancel` stops it too (a large library takes long enough to want both).
self.read_total.fetch_add(missing.len(), Relaxed);
let mut fresh = probe_all(self.probe.as_ref(), &missing, &self.read_done, cancel).into_iter();
let probed: Vec<Result<ProbeInfo, String>> =
cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
crate::memory::release();
@@ -974,6 +992,10 @@ impl ImportJob {
if info.captured.is_none() {
info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone());
}
// no date in the file or its sidecar: the file's own date, as Lightroom does
if info.captured.is_none() {
info.captured = info.file_modified.clone();
}
let stored = match (mode, &self.copy_root) {
(ImportMode::Copy | ImportMode::Move, Some(root))
if !crate::import_move::inside(Path::new(&path), root)
@@ -1499,4 +1521,51 @@ mod prepared_tests {
assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. })));
assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_))));
}
fn unread_paths(n: usize) -> Vec<String> {
(0..n).map(|i| std::env::temp_dir().join(format!("lightcraft-probe-progress-{i}.jpg")).to_string_lossy().into()).collect()
}
/// Reading the files of a large import (a Lightroom catalog's originals) reported nothing:
/// the progress stood still at "14 of 24554" for as long as it took.
#[test]
fn prepare_counts_the_files_it_reads() {
use std::sync::atomic::Ordering::Relaxed;
let mut s = Session::new();
let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let (seen_in_probe, done_in_probe) = (seen.clone(), std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)));
let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap();
let (done, last) = (job.read_done.clone(), done_in_probe.clone());
job.probe = Some(std::sync::Arc::new(move |_: &str| {
seen_in_probe.fetch_add(1, Relaxed);
last.fetch_max(done.load(Relaxed), Relaxed);
Ok(ProbeInfo { format: "JPEG".into(), ..ProbeInfo::default() })
}));
let prepared = job.prepare(&unread_paths(40), &std::sync::atomic::AtomicBool::new(false));
assert_eq!(prepared.items.len(), 40);
assert_eq!(seen.load(Relaxed), 40);
assert!(done_in_probe.load(Relaxed) > 0, "progress moves while files are read");
assert_eq!((job.read_done.load(Relaxed), job.read_total.load(Relaxed)), (40, 40), "every file read is counted");
assert_eq!((job.done.load(Relaxed), job.total.load(Relaxed)), (40, 40), "handling them is counted on its own");
}
/// Cancel had no effect until every file had been read.
#[test]
fn cancel_stops_reading_files() {
use std::sync::atomic::Ordering::Relaxed;
let mut s = Session::new();
let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut job = ImportJob::new(&mut s, ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }).unwrap();
let (seen_in_probe, cancel_in_probe) = (seen.clone(), cancel.clone());
job.probe = Some(std::sync::Arc::new(move |_: &str| {
seen_in_probe.fetch_add(1, Relaxed);
cancel_in_probe.store(true, Relaxed);
Ok(ProbeInfo { format: "JPEG".into(), ..ProbeInfo::default() })
}));
let prepared = job.prepare(&unread_paths(400), &cancel);
// at most the files the (up to 8) readers had already started
assert!(seen.load(Relaxed) <= 8, "read {} files after cancel", seen.load(Relaxed));
assert!(prepared.items.is_empty());
}
}
+54 -3
View File
@@ -41,14 +41,25 @@ pub fn lookup(db: &LensDb, m: &lightcraft_meta::Metadata, w: f64, h: f64, orient
let focal = m.focal_length.filter(|f| f.is_finite() && *f > 0.0)?;
let camera = db.find_camera(m.make.as_deref().unwrap_or_default(), m.model.as_deref().unwrap_or_default());
let lens = db.find_lens(m.lens_make.as_deref(), m.lens_model.as_deref()?, camera)?;
// the crop factor: the camera's, else what the 35 mm equivalent focal length implies
let crop =
camera.map(|c| c.crop_factor).or_else(|| m.focal_length_35mm.map(|f35| f35 / focal).filter(|c| c.is_finite() && (0.1..=20.0).contains(c)));
let crop = frame_crop_factor(camera.map(|c| c.crop_factor), focal, m.focal_length_35mm);
let shot = Shot { focal, aperture: m.f_number, distance: None, crop_factor: crop, width: w, height: h };
let corr = lens.correction(&shot)?;
Some(lightcraft_pipeline::optics::reorient_lens(&corr, orientation, w, h))
}
/// The crop factor of the recorded frame, which scales the profile's radii: the body's (`camera`), else what the
/// 35 mm equivalent focal length implies. A body shooting a crop of its sensor (a full-frame camera in APS-C /
/// Super 35 mode, or switched there by an APS-C lens) records a larger 35 mm equivalent than its own crop factor
/// gives; the frame's corners are then nearer the optical axis, so that crop wins. Within 10 % of the body's, the
/// difference is the tag's whole-millimetre rounding and the body's value stands.
fn frame_crop_factor(camera: Option<f64>, focal: f64, focal_35mm: Option<f64>) -> Option<f64> {
let frame = focal_35mm.map(|f35| f35 / focal).filter(|c| c.is_finite() && (0.1..=20.0).contains(c));
match (camera, frame) {
(Some(body), Some(frame)) if frame > body * 1.1 => Some(frame),
(body, frame) => body.or(frame),
}
}
/// [`crate::files::embedded_lens`], else the database profile ([`lookup`]) for a raw with metadata `meta`.
pub fn for_raw(raw: &lightcraft_raw::RawInfo, meta: impl FnOnce() -> lightcraft_meta::Metadata) -> Option<EmbeddedLens> {
crate::files::embedded_lens(raw).or_else(|| {
@@ -98,6 +109,46 @@ mod tests {
assert!(lookup(&db, &other, 6000.0, 4000.0, Orientation::Normal).is_none());
}
/// A full-frame body in APS-C / Super 35 mode (an ILCE-1M2 at 70 mm records 105 mm equivalent) frames only the
/// middle of the image circle: the profile is applied at the frame's crop factor, not the body's, so the crop's
/// corners don't get the full frame's corner correction.
#[test]
fn in_camera_crop_mode_uses_the_frames_crop_factor() {
let db = LensDb::parse(
r#"<lensdatabase>
<camera><maker>Sony</maker><model>ILCE-1M2</model><mount>Sony E</mount><cropfactor>1</cropfactor></camera>
<lens><maker>Sigma</maker><model>24-70mm F2.8 DG DN | Art 019</model><mount>Sony E</mount><cropfactor>1.0</cropfactor>
<calibration>
<distortion model="ptlens" focal="70" a="0.0019947" b="0.0035263" c="0.0005016" />
<vignetting model="pa" focal="70" aperture="2.8" distance="1000" k1="-0.4" k2="0" k3="0" />
</calibration>
</lens></lensdatabase>"#,
)
.unwrap();
let m = |f35: Option<f64>| lightcraft_meta::Metadata {
make: Some("SONY".into()),
model: Some("ILCE-1M2".into()),
lens_model: Some("24-70mm F2.8 DG DN | Art 019".into()),
focal_length: Some(70.0),
focal_length_35mm: f35,
f_number: Some(2.8),
..Default::default()
};
let at = |crop: f64, w: f64, h: f64| {
let shot = Shot { focal: 70.0, aperture: Some(2.8), distance: None, crop_factor: Some(crop), width: w, height: h };
db.lenses[0].correction(&shot).unwrap()
};
let aps_c = lookup(&db, &m(Some(105.0)), 5616.0, 3744.0, Orientation::Normal).unwrap();
assert_eq!(aps_c, at(1.5, 5616.0, 3744.0));
assert_ne!(aps_c, at(1.0, 5616.0, 3744.0), "the crop is not corrected as a full frame");
// full frame, a 35 mm equivalent rounded off by a millimetre, or none at all: the body's crop factor
for f35 in [Some(70.0), Some(71.0), None] {
assert_eq!(lookup(&db, &m(f35), 8640.0, 5760.0, Orientation::Normal).unwrap(), at(1.0, 8640.0, 5760.0), "{f35:?}");
}
assert_eq!(frame_crop_factor(None, 70.0, Some(105.0)), Some(1.5), "no body in the database: the 35 mm equivalent's");
assert_eq!(frame_crop_factor(Some(1.0), 0.0, Some(105.0)), Some(1.0), "a hostile focal length is ignored");
}
/// A synthetic grid recorded through the profile's barrel distortion comes out of the pipeline's geometry
/// stage with straight lines.
#[test]
+17 -5
View File
@@ -132,7 +132,8 @@ fn single_photo(op: &Op, depth: usize) -> Option<PhotoId> {
| Op::SetMeta { id, .. }
| Op::SetVersions { id, .. }
| Op::SetHistory { id, .. }
| Op::PushHistory { id, .. } => Some(*id),
| Op::PushHistory { id, .. }
| Op::SetHistoryCursor { id, .. } => Some(*id),
Op::Batch { ops } if depth < 8 => {
let mut ids = ops.iter().map(|o| single_photo(o, depth + 1));
let first = ids.next()??;
@@ -223,7 +224,8 @@ pub struct Session {
pub previous_active: Option<PhotoId>,
/// Groups last used for Copy (Lightroom remembers them).
pub copy_groups: Vec<lightcraft_develop::SettingsGroup>,
pub presets: Vec<lightcraft_develop::Preset>,
/// Built-in then user presets; its generation tells savers and caches when it changed.
pub presets: presets::PresetList,
/// Favourite profile ids (persisted with the library, like preset favourites).
pub profile_favorites: Vec<String>,
/// Recently applied profile ids, newest first (at most [`presets::RECENT_PROFILES`]).
@@ -363,7 +365,7 @@ impl Session {
library_folder: None,
previous_active: None,
copy_groups: lightcraft_develop::SettingsGroup::default_copy(),
presets: presets::builtin(),
presets: presets::PresetList::new(presets::builtin()),
profile_favorites: Vec::new(),
profile_recent: Vec::new(),
journal: Vec::new(),
@@ -893,11 +895,17 @@ impl Session {
}
}
if self.source == LibrarySource::RecentlyAdded {
// newest import first, whatever the sort (the grid groups by import day)
// newest import first, whatever the sort direction (the grid groups by import day);
// a batch stays in file order (name, then id)
let cat = &self.catalog;
visible.sort_by(|a, b| {
let key = |id: &PhotoId| cat.photo(*id).map(|p| p.imported.clone()).unwrap_or_default();
key(b).cmp(&key(a)).then(a.cmp(b))
let name = |id: &PhotoId| cat.photo(*id).map(|p| p.file_name.as_str());
let by_name = match (name(a), name(b)) {
(Some(x), Some(y)) => lightcraft_catalog::query::cmp_name(x, y),
_ => std::cmp::Ordering::Equal,
};
key(b).cmp(&key(a)).then(by_name).then(a.cmp(b))
});
}
if self.source == LibrarySource::Missing {
@@ -1011,6 +1019,8 @@ mod tests_folders;
#[cfg(test)]
mod tests_forget_local;
#[cfg(test)]
mod tests_history_cursor;
#[cfg(test)]
mod tests_import;
#[cfg(test)]
mod tests_import_move;
@@ -1031,6 +1041,8 @@ mod tests_persist;
#[cfg(test)]
mod tests_prefs;
#[cfg(test)]
mod tests_presets_scale;
#[cfg(test)]
mod tests_process;
#[cfg(test)]
mod tests_segment;
+69 -10
View File
@@ -88,7 +88,8 @@ pub struct Library {
retry_at: Option<web_time::Instant>,
/// What forgetting untouched Local records did when the library opened.
pub forgot_local: Option<lightcraft_catalog::ForgetPlan>,
presets_written: String,
/// What presets.json holds (see [`PresetsSaved`]).
presets_saved: PresetsSaved,
view_written: Vec<u8>,
/// Settings files that were unreadable or damaged when the library opened (`library.info` →
/// `settingsWarnings`; shown by the UI once, see [`Session::take_library_warnings`]).
@@ -123,6 +124,53 @@ struct PresetsFile {
profile_recent: Vec<String>,
}
/// The state presets.json was last written (or read) from. Saving checks the preset list's
/// generation and the small profile lists before serializing anything: with a 10 000-preset
/// collection presets.json is ~40 MB, and serializing it after every command made each slider
/// tick take seconds.
#[derive(Default)]
struct PresetsSaved {
generation: u64,
profile_favorites: Vec<String>,
profile_recent: Vec<String>,
/// Hash of the bytes written: a list borrowed mutably but left as it was isn't rewritten.
hash: u64,
}
impl PresetsSaved {
fn state_of(s: &Session) -> PresetsSaved {
PresetsSaved {
generation: s.presets.generation(),
profile_favorites: s.profile_favorites.clone(),
profile_recent: s.profile_recent.clone(),
hash: 0,
}
}
/// `s` changed its presets or profile lists since this state.
fn differs(&self, s: &Session) -> bool {
self.generation != s.presets.generation() || self.profile_favorites != s.profile_favorites || self.profile_recent != s.profile_recent
}
}
fn bytes_hash(b: &[u8]) -> u64 {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
b.hash(&mut h);
h.finish()
}
#[cfg(test)]
thread_local! {
static PRESETS_SERIALIZED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
/// How many times this thread serialized the preset list (tests).
#[cfg(test)]
pub(crate) fn presets_serialized() -> usize {
PRESETS_SERIALIZED.with(|c| c.get())
}
#[derive(Default, Serialize, Deserialize)]
#[serde(default)]
struct ViewFile {
@@ -191,6 +239,8 @@ struct PrefsFile {
}
fn presets_json(s: &Session) -> String {
#[cfg(test)]
PRESETS_SERIALIZED.with(|c| c.set(c.get() + 1));
let presets = &s.presets;
let f = PresetsFile {
user: presets.iter().filter(|p| !p.builtin).cloned().collect(),
@@ -420,8 +470,9 @@ impl Session {
for p in &mut self.presets {
p.favorite = p.builtin && f.favorites.contains(&p.id) || (!p.builtin && p.favorite);
}
let mut ids: std::collections::HashSet<String> = self.presets.iter().map(|p| p.id.clone()).collect();
for u in f.user {
if !self.presets.iter().any(|p| p.id == u.id) {
if ids.insert(u.id.clone()) {
self.presets.push(u);
}
}
@@ -453,7 +504,8 @@ impl Session {
{
self.selection = Selection::single(*first);
}
let presets_written = presets_json(self);
// presets.json is as read: saved again only after the list changes (hash 0 = unknown bytes)
let presets_saved = PresetsSaved::state_of(self);
// photo ids are per library: drop decoded sources of the previous one
self.media.clear_sources();
if on_disk {
@@ -470,7 +522,7 @@ impl Session {
unsaved_error: None,
retry_at: None,
forgot_local: None,
presets_written,
presets_saved,
view_written,
settings_warnings: unlocked_warning(lock.as_ref()).into_iter().chain(settings.warnings).collect(),
warnings_reported: 0,
@@ -532,14 +584,21 @@ impl Session {
log::error!("library: compaction: {e}");
lib.last_error = Some(format!("compaction: {e}"));
}
let due = self.library.as_ref().is_some_and(|l| l.presets_saved.differs(self) && !l.blocked.contains(&"presets.json"));
if !due {
return Ok(());
}
let presets = presets_json(self);
let mut state = PresetsSaved::state_of(self);
state.hash = bytes_hash(presets.as_bytes());
let Some(lib) = self.library.as_mut() else { return Ok(()) };
if presets != lib.presets_written && !lib.blocked.contains(&"presets.json") {
if let Err(e) = lib.files.write_atomic("presets.json", presets.as_bytes()) {
log::error!("library: presets: {e}");
} else {
lib.presets_written = presets;
}
if state.hash == lib.presets_saved.hash {
lib.presets_saved = state;
} else if let Err(e) = lib.files.write_atomic("presets.json", presets.as_bytes()) {
// the state stays stale: the next save tries again
log::error!("library: presets: {e}");
} else {
lib.presets_saved = state;
}
Ok(())
}
+118 -23
View File
@@ -129,6 +129,65 @@ pub struct LightroomCompletion {
/// Alias used by hosts that call the result a commit.
pub type LightroomCommit = LightroomCompletion;
/// The step of preparing a Lightroom import that is under way ([`LightroomProgress::step`]).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LightroomStep {
/// Reading the catalog's tables.
Catalog,
/// Reading the photos the catalog names (headers, content hash): nearly all of the time.
ReadPhotos,
/// Readying the photos that were read for the catalog.
AddPhotos,
}
/// Progress of [`LightroomJob::prepare`] step by step, shared with the thread that runs it.
#[derive(Clone, Default)]
pub struct LightroomProgress {
/// Catalog tables to read and read so far ([`lightroom_catalog::read_with_progress`]).
pub catalog_total: Arc<AtomicUsize>,
pub catalog_done: Arc<AtomicUsize>,
read_total: Arc<AtomicUsize>,
read_done: Arc<AtomicUsize>,
add_total: Arc<AtomicUsize>,
add_done: Arc<AtomicUsize>,
/// The catalog is read: the photos are next.
photos: Arc<AtomicBool>,
}
impl LightroomProgress {
/// Reading the photos is nearly all of an import's time; the catalog and readying the photos
/// take the rest of [`LightroomProgress::fraction`].
const CATALOG_SHARE: f32 = 0.05;
const READ_SHARE: f32 = 0.85;
/// The step under way, with how much of it is done and its total (0 while not yet known).
pub fn step(&self) -> (LightroomStep, usize, usize) {
let load = |n: &AtomicUsize| n.load(Ordering::Relaxed);
if !self.photos.load(Ordering::Relaxed) {
return (LightroomStep::Catalog, load(&self.catalog_done), load(&self.catalog_total));
}
let (read_done, read_total) = (load(&self.read_done), load(&self.read_total));
let (add_done, add_total) = (load(&self.add_done), load(&self.add_total));
// nothing to read (every photo is already in the library) goes straight to adding
if read_done < read_total || (read_total == 0 && add_done == 0) {
(LightroomStep::ReadPhotos, read_done, read_total)
} else {
(LightroomStep::AddPhotos, add_done, add_total)
}
}
/// The whole preparation, 0..=1.
pub fn fraction(&self) -> f32 {
let (step, done, total) = self.step();
let part = if total == 0 { 0.0 } else { (done as f32 / total as f32).clamp(0.0, 1.0) };
match step {
LightroomStep::Catalog => Self::CATALOG_SHARE * part,
LightroomStep::ReadPhotos => Self::CATALOG_SHARE + Self::READ_SHARE * part,
LightroomStep::AddPhotos => Self::CATALOG_SHARE + Self::READ_SHARE + (1.0 - Self::CATALOG_SHARE - Self::READ_SHARE) * part,
}
}
}
/// A Lightroom import detached from the owner thread.
pub struct LightroomJob {
path: PathBuf,
@@ -136,6 +195,7 @@ pub struct LightroomJob {
import: ImportJob,
token: LightroomLibraryToken,
archive_dir: Option<PathBuf>,
progress: LightroomProgress,
}
impl LightroomJob {
@@ -145,33 +205,25 @@ impl LightroomJob {
let opts = ImportOptions { mode: ImportMode::Add, on_deleted: crate::import::OnDeleted::Skip, ..ImportOptions::default() };
let import = ImportJob::new(s, opts)?;
let archive_dir = s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.join("Interop"));
Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir })
let progress = LightroomProgress {
read_total: Arc::clone(&import.read_total),
read_done: Arc::clone(&import.read_done),
add_total: Arc::clone(&import.total),
add_done: Arc::clone(&import.done),
..LightroomProgress::default()
};
Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir, progress })
}
pub fn total(&self) -> usize {
self.import.total.load(Ordering::Relaxed)
}
pub fn done(&self) -> usize {
self.import.done.load(Ordering::Relaxed)
}
pub fn progress(&self) -> (usize, usize) {
(self.done(), self.total())
/// The job's progress, to watch from another thread while [`LightroomJob::prepare`] runs.
pub fn progress(&self) -> LightroomProgress {
self.progress.clone()
}
pub fn token(&self) -> &LightroomLibraryToken {
&self.token
}
pub fn total_atomic(&self) -> std::sync::Arc<AtomicUsize> {
std::sync::Arc::clone(&self.import.total)
}
pub fn done_atomic(&self) -> std::sync::Arc<AtomicUsize> {
std::sync::Arc::clone(&self.import.done)
}
/// Finish an owner commit's deferred archive-index write on a worker.
pub fn finish_archive(finalization: LightroomArchiveFinalization) -> crate::Result<()> {
finalization.finish()
@@ -182,14 +234,12 @@ impl LightroomJob {
if cancel.load(Ordering::Relaxed) {
return Err(error("Lightroom import cancelled"));
}
self.import.total.store(1, Ordering::Relaxed);
self.import.done.store(0, Ordering::Relaxed);
let mut data = lightroom_catalog::read_with_progress(&self.path, cancel, &self.import.total, &self.import.done).map_err(error)?;
let mut data =
lightroom_catalog::read_with_progress(&self.path, cancel, &self.progress.catalog_total, &self.progress.catalog_done).map_err(error)?;
lightroom_catalog::validate_for_job(&data).map_err(error)?;
if cancel.load(Ordering::Relaxed) {
return Err(error("Lightroom import cancelled"));
}
self.import.done.fetch_add(1, Ordering::Relaxed);
let paths: Vec<String> = data
.photos
.iter()
@@ -203,6 +253,9 @@ impl LightroomJob {
.filter(|photo| !Path::new(&photo.path).is_file())
.map(|photo| normalize(&photo.path))
.collect();
// the catalog step's last unit (validation, and looking for each original), then the photos
self.progress.catalog_done.fetch_add(1, Ordering::Relaxed);
self.progress.photos.store(true, Ordering::Relaxed);
let normal = self.import.prepare(&paths, cancel);
if cancel.load(Ordering::Relaxed) {
normal.rollback();
@@ -335,6 +388,48 @@ mod tests {
}
}
/// One count for the whole import read "14 of 24554" while the photos were read, then twice
/// the number of photos: each step has its own count, and the whole only moves forward.
#[test]
fn progress_names_each_step_with_its_own_count() {
let p = LightroomProgress::default();
let set = |n: &AtomicUsize, v: usize| n.store(v, Ordering::Relaxed);
assert_eq!(p.step(), (LightroomStep::Catalog, 0, 0));
assert_eq!(p.fraction(), 0.0);
set(&p.catalog_total, 14);
set(&p.catalog_done, 7);
assert_eq!(p.step(), (LightroomStep::Catalog, 7, 14));
let mut last = p.fraction();
assert!(last > 0.0);
let mut forward = |p: &LightroomProgress| {
let f = p.fraction();
assert!(f >= last && f <= 1.0, "{f} after {last}");
last = f;
};
set(&p.catalog_done, 14);
p.photos.store(true, Ordering::Relaxed);
assert_eq!(p.step(), (LightroomStep::ReadPhotos, 0, 0), "the photos are not counted yet");
forward(&p);
set(&p.add_total, 100);
set(&p.read_total, 100);
set(&p.read_done, 40);
assert_eq!(p.step(), (LightroomStep::ReadPhotos, 40, 100));
forward(&p);
set(&p.read_done, 100);
set(&p.add_done, 25);
assert_eq!(p.step(), (LightroomStep::AddPhotos, 25, 100));
forward(&p);
set(&p.add_done, 100);
forward(&p);
assert!(p.fraction() > 0.999);
// every photo already in the library: nothing to read
let known = LightroomProgress::default();
known.photos.store(true, Ordering::Relaxed);
set(&known.add_total, 5);
set(&known.add_done, 5);
assert_eq!(known.step(), (LightroomStep::AddPhotos, 5, 5));
}
#[test]
fn cancellation_before_commit_leaves_owner_catalog_unchanged() {
let mut session = crate::Session::new();
+176 -18
View File
@@ -62,7 +62,7 @@ impl SettingsHashes {
}
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
pub const RENDER_CACHE_VERSION: u64 = 61;
pub const RENDER_CACHE_VERSION: u64 = 62;
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
@@ -223,6 +223,13 @@ impl DecodedSource {
#[derive(Clone)]
pub enum SourceRef {
Loaded(Box<DecodedSource>),
/// A larger decoded source of the photo, fitted to `max_edge` by the worker: the thumbnail of
/// the photo open in the loupe is cut from its preview-level source instead of decoding the
/// file again (a second or more for a 64 MP raw).
Fit {
from: Box<DecodedSource>,
max_edge: usize,
},
Demo {
scene: Box<lightcraft_scenes::Scene>,
max_edge: usize,
@@ -251,6 +258,13 @@ impl SourceRef {
pub fn load_source(&self) -> Result<DecodedSource, String> {
let image = match self {
SourceRef::Loaded(a) => return Ok((**a).clone()),
SourceRef::Fit { from, max_edge } => {
if from.image.width.max(from.image.height) <= *max_edge {
return Ok((**from).clone());
}
let image = lightcraft_raster::resample::fit(&from.image, *max_edge, *max_edge, lightcraft_raster::resample::Filter::Box);
return Ok(DecodedSource { image: Arc::new(image), denoised: None, ..(**from).clone() });
}
SourceRef::Demo { scene, max_edge } => crate::demo::scene_pixels(scene, *max_edge)?,
SourceRef::File { path, max_edge, loader, fallback, denoise } => {
let plain = || match loader {
@@ -314,6 +328,8 @@ pub struct MediaCache {
full: Vec<(PhotoId, DecodedSource, u64)>,
/// How many full-resolution originals to keep (LRU, at least one).
full_capacity: usize,
/// The photo the loupe shows ([`Self::set_on_screen`]): its original is not evicted for the budget.
on_screen: Option<PhotoId>,
/// How many previews to keep (LRU).
pub preview_capacity: usize,
/// Bytes all decoded sources and rendered previews in memory may take together (the cache
@@ -391,6 +407,7 @@ impl Default for MediaCache {
previews: Vec::new(),
full: Vec::new(),
full_capacity: 1,
on_screen: None,
preview_capacity: 0,
budget,
file_loader: None,
@@ -447,6 +464,16 @@ impl MediaCache {
self.trim_full();
}
/// The photo the loupe shows (the frontend sets it every frame; `None` in other views). Its original stays
/// even over budget: a 64 MP original (797 MB as linear float) alone exceeds the default share, and evicting it
/// made every pan of a zoomed view decode the raw again. Showing another photo lets it go at once.
pub fn set_on_screen(&mut self, id: Option<PhotoId>) {
if self.on_screen != id {
self.on_screen = id;
self.enforce_budget();
}
}
fn trim_full(&mut self) {
while self.full.len() > self.full_capacity {
let oldest = self.full.iter().enumerate().min_by_key(|(_, e)| e.2).map(|(i, _)| i).unwrap_or(0);
@@ -475,15 +502,20 @@ impl MediaCache {
/// Evict least recently used entries across the source caches and the rendered previews
/// until they fit the budget. The newest preview-level source (the photo on screen, or the
/// one just decoded) is never evicted here.
/// one just decoded) is never evicted here, nor is the original of the photo on screen
/// ([`Self::set_on_screen`]).
pub fn enforce_budget(&mut self) {
let mut held = self.held();
// The on-screen original is never evicted here, so it does not count either: counted, a 64 MP original (760 MiB)
// nearly fills the default share (768 MiB) alone and everything else went instead — the photo's preview and
// thumbnail sources and the neighbours' were decoded again from the raw on every edit, zoom-out and step.
let pinned: usize = self.full.iter().filter(|e| Some(e.0) == self.on_screen).map(|e| e.1.bytes()).sum();
let mut held = self.held().saturating_sub(pinned);
while held > self.budget {
let newest_preview = self.previews.iter().map(|e| e.2).max();
let candidates = [
self.thumbs.oldest_tick().map(|t| (t, 0)),
self.previews.iter().filter(|e| Some(e.2) != newest_preview).map(|e| e.2).min().map(|t| (t, 1)),
self.full.iter().map(|e| e.2).min().map(|t| (t, 2)),
self.full.iter().filter(|e| Some(e.0) != self.on_screen).map(|e| e.2).min().map(|t| (t, 2)),
self.rendered.oldest_tick().map(|t| (t, 3)),
];
let Some((tick, which)) = candidates.into_iter().flatten().min() else { break };
@@ -625,6 +657,15 @@ impl MediaCache {
if cached && let Some(a) = self.get_source(p.id, level) {
return SourceRef::Loaded(Box::new(a));
}
// a thumbnail of a photo whose preview is decoded (the one in the loupe) is cut from it; not
// with a denoised picture, which the thumbnail would lose
if cached
&& level == SourceLevel::Thumb
&& self.denoise.spec(p.id).is_none()
&& let Some(big) = self.get_source(p.id, SourceLevel::Preview)
{
return SourceRef::Fit { from: Box::new(big), max_edge: level.max_edge() };
}
// Procedural scenes have a nominal size: "full" is that size, not unbounded.
let max_edge = level.max_edge().min(match (&p.source, level) {
(Source::Demo { .. }, SourceLevel::Full) => p.width.max(p.height).max(1) as usize,
@@ -806,6 +847,30 @@ impl RenderJob {
self
}
/// Render. A panic inside (a decoder or pipeline bug) is this job's failure: the view waiting
/// for it is told, instead of waiting forever on a worker that died (the pool doesn't replace
/// it, so a few such panics used to stop every render).
pub fn run(self) -> RenderResult {
let failed = self.failed(String::new());
crate::guard::catch("the render", || self.run_unguarded()).unwrap_or_else(|e| RenderResult { rendered: Err(e), ..failed })
}
/// This job's result when it fails with `e`.
fn failed(&self, e: String) -> RenderResult {
RenderResult {
request_id: self.request_id,
source_key: self.source_key,
photo: self.photo,
level: self.level,
key: self.key,
rendered: Err(e),
gpu_fallback: None,
loaded: None,
quick: None,
display: None,
}
}
/// Compose the render into a slideshow slide (layout, options, backdrop): the result is the
/// slide, not the photo. Gets its own result key; the photo's cached view preview stays the
/// photo. The slide stays sRGB ([`Self::with_display`] leaves it alone).
@@ -817,8 +882,8 @@ impl RenderJob {
self
}
/// [`Self::run_photo`], composed into its slide when this is a slide job.
pub fn run(self) -> RenderResult {
/// [`Self::run_photo`], composed into its slide when this is a slide job (inside [`Self::run`]'s guard).
fn run_unguarded(self) -> RenderResult {
let slide = self.slide.clone();
let mut r = self.run_photo();
if let (Some(slide), Ok(rendered)) = (slide, r.rendered.as_mut()) {
@@ -903,18 +968,7 @@ impl RenderJob {
display,
}
}
Err(e) => RenderResult {
request_id: self.request_id,
source_key: self.source_key,
photo: self.photo,
level: self.level,
key: self.key,
rendered: Err(e),
gpu_fallback: None,
loaded: None,
quick: None,
display: None,
},
Err(e) => self.failed(e),
}
}
}
@@ -1002,7 +1056,25 @@ pub struct QuickJob {
}
impl QuickJob {
/// The stand-in, from the first source that has one. A panic inside is this job's failure
/// (see [`RenderJob::run`]).
pub fn run(self) -> RenderResult {
let (request_id, photo, key) = (self.request_id, self.photo, self.key);
crate::guard::catch("the render", || self.run_unguarded()).unwrap_or_else(|e| RenderResult {
request_id,
source_key: None,
photo,
level: SourceLevel::Thumb,
key,
rendered: Err(e),
gpu_fallback: None,
loaded: None,
quick: None,
display: None,
})
}
fn run_unguarded(self) -> RenderResult {
let (photo, key) = (self.photo, self.key);
let request_id = self.request_id;
let done = |rendered: Result<Rendered, String>, quick| RenderResult {
@@ -1508,6 +1580,10 @@ pub struct ProbeInfo {
pub kind: MediaKind,
pub file_size: u64,
pub captured: Option<String>,
/// The file's modification time (ISO 8601, the clock of [`crate::import::system_clock`]): the
/// capture date of a file whose metadata has none, as Lightroom does (exports saved without
/// metadata would otherwise all land under "Unknown Date").
pub file_modified: Option<String>,
pub meta: lightcraft_catalog::Meta,
pub as_shot_wb: Option<(f64, f64)>,
/// Hash of the file's bytes (hex), for duplicate detection. When it is a 32-digit
@@ -1617,6 +1693,31 @@ mod thumbnail_hash_tests {
s.accept(&old);
assert_eq!(s.media.source_usage().0, 0);
}
/// A panic while rendering (here in the decoder) used to end the pool's worker thread without
/// an answer, so the view that asked showed "Rendering…" forever. It is the job's failure now,
/// for renders and for stand-ins, answering the request it came from.
#[test]
fn a_render_that_panics_fails_instead_of_never_answering() {
let mut s = session();
s.media.file_loader = Some(Arc::new(|_, _| panic!("decoder bug")));
let mut job = s.thumb_job(PhotoId(1), 256).unwrap();
job.request_id = 42;
let (key, level, source_key) = (job.key, job.level, job.source_key);
let r = job.run();
assert_eq!((r.request_id, r.photo, r.key, r.level, r.source_key), (42, PhotoId(1), key, level, source_key));
assert_eq!(r.rendered.err().as_deref(), Some("the render failed unexpectedly: decoder bug"));
let quick = QuickJob {
request_id: 7,
photo: PhotoId(1),
key: 9,
cached: Vec::new(),
embedded: Some(("x.arw".into(), Arc::new(|_: &str, _: usize| -> Option<Rgba8> { panic!("preview reader bug") }), 256)),
small: None,
};
let r = quick.run();
assert_eq!((r.request_id, r.key), (7, 9));
assert_eq!(r.rendered.err().as_deref(), Some("the render failed unexpectedly: preview reader bug"));
}
}
#[cfg(test)]
@@ -1822,6 +1923,45 @@ mod tests {
assert!(m.has_source(PhotoId(3), SourceLevel::Full));
}
// Given the on-screen original alone over the budget, the photo's other sources and its neighbour's stay
// (counted, it evicted them all: each edit decoded the raw again for the thumbnail and the whole-frame render)
#[test]
fn the_on_screen_original_does_not_evict_the_other_sources() {
let img = |w: usize| Arc::new(Rgb32f::new(w, w));
let mut m = MediaCache::default();
let mb = 1 << 20;
m.set_budget(20 * mb);
m.set_on_screen(Some(PhotoId(1)));
m.insert(PhotoId(1), SourceLevel::Preview, img(500));
m.insert(PhotoId(1), SourceLevel::Thumb, img(200));
m.insert(PhotoId(1), SourceLevel::Full, img(1500)); // 27 MB
m.insert(PhotoId(2), SourceLevel::Preview, img(500)); // a prefetched neighbour
for (id, level) in [(1, SourceLevel::Preview), (1, SourceLevel::Thumb), (1, SourceLevel::Full), (2, SourceLevel::Preview)] {
assert!(m.has_source(PhotoId(id), level), "{id} {level:?}");
}
}
// A 64 MP original (797 MB as linear float) is bigger than the default cache share (768 MiB):
// evicted on insert, every pan of a zoomed view decoded the raw again. The original of the photo
// on screen stays, whatever is decoded after it (prefetched neighbours), until another is shown.
#[test]
fn the_original_on_screen_stays_even_over_budget() {
let img = |w: usize| Arc::new(Rgb32f::new(w, w));
let mut m = MediaCache::default();
let mb = 1 << 20;
m.set_budget(20 * mb);
m.set_on_screen(Some(PhotoId(1)));
m.insert(PhotoId(1), SourceLevel::Preview, img(500));
m.insert(PhotoId(1), SourceLevel::Full, img(1500)); // 27 MB
assert!(m.has_source(PhotoId(1), SourceLevel::Full));
m.insert(PhotoId(2), SourceLevel::Preview, img(500)); // a prefetched neighbour
m.insert(PhotoId(3), SourceLevel::Thumb, img(100));
assert!(m.has_source(PhotoId(1), SourceLevel::Full));
m.set_on_screen(Some(PhotoId(2)));
assert!(!m.has_source(PhotoId(1), SourceLevel::Full));
assert!(m.held() <= 20 * mb, "{}", m.held());
}
#[test]
fn work_gate_waits_for_room() {
use crate::memory::WorkGate;
@@ -1960,6 +2100,24 @@ mod tests {
assert!(mean(&a) > mean(&b) + 10.0, "two stops brighter after: {} vs {}", mean(&a), mean(&b));
}
/// The thumbnail of a photo whose preview-level source is decoded is cut from it: the file (here
/// one nothing can decode) is not read again.
#[test]
fn a_thumbnail_is_cut_from_the_decoded_preview() {
let mut s = crate::Session::new();
let p = Photo::new(PhotoId(1), Source::File { path: "/nowhere/a.ARW".into() }, "a.ARW", "ARW", 2400, 1600, "");
s.catalog.apply(lightcraft_catalog::Op::AddPhoto { photo: Box::new(p.clone()) }).unwrap();
s.media.insert(p.id, SourceLevel::Preview, Arc::new(Rgb32f::from_fn(2400, 1600, |x, _| [x as f32 / 2400.0, 0.5, 0.25])));
let job = s.thumb_job(p.id, 512).unwrap();
assert!(matches!(job.source, SourceRef::Fit { .. }));
let r = job.run();
let thumb = r.rendered.as_ref().unwrap();
assert_eq!((thumb.image.width, thumb.image.height), (512, 341));
s.accept(&r);
let cached = s.media.get(p.id, SourceLevel::Thumb).unwrap();
assert_eq!((cached.width, cached.height), (512, 341), "a thumbnail-sized source is kept, not the preview");
}
#[test]
fn has_source_follows_what_the_session_accepted() {
let mut s = crate::Session::with_demo();
+3 -1
View File
@@ -110,7 +110,9 @@ impl MergeJob {
.par_iter()
.map(|(_, path)| {
let bytes = (self.read)(path)?;
let f = lightcraft_merge::load_frame(&bytes, self.preview, orient).map_err(|e| format!("{path}: {e}"));
// a raw without colour matrices of its own starts from the colour it has when opened alone (#965)
let f = lightcraft_merge::load_frame_with(&bytes, self.preview, orient, &crate::camera_preview::starting_colour_tags)
.map_err(|e| format!("{path}: {e}"));
let k = done.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
progress(span.0 + (span.1 - span.0) * k as f32 / n as f32, "Reading photos");
f
+81 -3
View File
@@ -23,6 +23,73 @@ pub struct PresetFile {
pub presets: Vec<Preset>,
}
/// The session's presets (built-in, then the user's) with a generation that changes on every
/// mutable access, so saving and the UI's caches skip their O(presets) work while the list is
/// untouched: a Lightroom collection imported whole is 10 000+ presets and ~40 MB of JSON.
///
/// Reads go through `Deref` (`iter`, `len`, `find`…) and keep the generation; `push`, `iter_mut`,
/// `remove`… go through `DerefMut` and bump it, whether or not anything is then changed.
/// Generations are unique across all lists in the process.
#[derive(Clone, Debug)]
pub struct PresetList {
items: Vec<Preset>,
generation: u64,
}
static PRESET_GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
fn next_generation() -> u64 {
PRESET_GENERATION.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
}
impl PresetList {
pub fn new(items: Vec<Preset>) -> PresetList {
PresetList { items, generation: next_generation() }
}
/// Changes whenever the list may have changed.
pub fn generation(&self) -> u64 {
self.generation
}
}
impl Default for PresetList {
fn default() -> PresetList {
PresetList::new(Vec::new())
}
}
impl std::ops::Deref for PresetList {
type Target = Vec<Preset>;
fn deref(&self) -> &Vec<Preset> {
&self.items
}
}
impl std::ops::DerefMut for PresetList {
fn deref_mut(&mut self) -> &mut Vec<Preset> {
self.generation = next_generation();
&mut self.items
}
}
impl<'a> IntoIterator for &'a PresetList {
type Item = &'a Preset;
type IntoIter = std::slice::Iter<'a, Preset>;
fn into_iter(self) -> Self::IntoIter {
self.items.iter()
}
}
impl<'a> IntoIterator for &'a mut PresetList {
type Item = &'a mut Preset;
type IntoIter = std::slice::IterMut<'a, Preset>;
fn into_iter(self) -> Self::IntoIter {
self.generation = next_generation();
self.items.iter_mut()
}
}
/// Lower-case ASCII slug for ids (`"Warm & Soft"` → `warm-soft`).
pub fn slug(s: &str) -> String {
let mut out = String::new();
@@ -107,22 +174,33 @@ impl Session {
/// Add imported presets. Identical presets (same name, group and settings) already present are
/// skipped; id clashes get a fresh id. Returns the ids added.
pub fn add_presets(&mut self, presets: Vec<Preset>) -> Vec<String> {
use std::collections::{HashMap, HashSet};
// indexed: a collection of 10 000+ presets was quadratic (minutes) to import
let mut ids: HashSet<String> = self.presets.iter().map(|q| q.id.clone()).collect();
let mut by_name: HashMap<(String, String), Vec<usize>> = HashMap::new();
for (i, q) in self.presets.iter().enumerate() {
by_name.entry((q.name.clone(), q.group.clone())).or_default().push(i);
}
let mut added = Vec::new();
for mut p in presets {
if self.presets.iter().any(|q| q.name == p.name && q.group == p.group && q.settings == p.settings) {
let key = (p.name.clone(), p.group.clone());
let same = by_name.get(&key).is_some_and(|ix| ix.iter().any(|&i| self.presets.get(i).is_some_and(|q| q.settings == p.settings)));
if same {
continue;
}
if p.id.trim().is_empty() || p.id.starts_with("lc.") || self.presets.iter().any(|q| q.id == p.id) {
if p.id.trim().is_empty() || p.id.starts_with("lc.") || ids.contains(&p.id) {
let base = format!("user.{}", slug(&format!("{}-{}", p.group, p.name)));
let mut id = base.clone();
let mut n = 2;
while self.presets.iter().any(|q| q.id == id) {
while ids.contains(&id) {
id = format!("{base}-{n}");
n += 1;
}
p.id = id;
}
p.builtin = false;
ids.insert(p.id.clone());
by_name.entry(key).or_default().push(self.presets.len());
added.push(p.id.clone());
self.presets.push(p);
}
+88 -17
View File
@@ -319,6 +319,17 @@ pub(crate) trait MoveFs {
/// Another photo file beside the stem-named sidecar `sidecar` that shares its stem (e.g.
/// `IMG_1.JPG` for `IMG_1.xmp`), which still uses that sidecar.
fn stem_sibling(&self, sidecar: &Path) -> Option<String>;
/// The folder lists an entry spelled exactly like `p` (letter case included). On a
/// case-insensitive volume `a.XMP` "exists" when only `a.xmp` is there; this tells the two
/// apart. When the folder can't be listed: `true`, so nothing is skipped.
fn listed(&self, p: &Path) -> bool {
let (Some(dir), Some(name)) = (p.parent(), p.file_name()) else { return true };
let dir = if dir.as_os_str().is_empty() { Path::new(".") } else { dir };
match std::fs::read_dir(dir) {
Ok(entries) => entries.flatten().any(|e| e.file_name() == name),
Err(_) => true,
}
}
/// Content observed in a packet, independent of its pathname or inode. Model volumes
/// override this; disk implementations stream a bounded buffer without retaining the XML.
fn packet_version(&self, path: &Path) -> std::io::Result<PacketVersion> {
@@ -366,15 +377,23 @@ impl MoveFs for RealFs {
}
#[cfg(not(unix))]
{
// no stable file id in std here: `b` is `a` under another spelling when they look
// alike and the folder has no entry spelled exactly like `b` (on a case-sensitive
// folder holding both `IMG.JPG` and `img.jpg`, both entries are listed)
let (Some(dir), Some(name)) = (b.parent(), b.file_name()) else { return false };
if a.parent() != Some(dir) || ma.len() != mb.len() || ma.modified().ok() != mb.modified().ok() {
// no stable file id in std here: `a` and `b` are one file under two spellings when
// they look alike and the folder doesn't list both (on a case-sensitive folder
// holding both `IMG.JPG` and `img.jpg`, both entries are listed). Either may be the
// listed one: the answer must not depend on the argument order.
let (Some(dir), Some(name_a), Some(name_b)) = (b.parent(), a.file_name(), b.file_name()) else { return false };
let respelled = name_a.to_string_lossy().to_lowercase() == name_b.to_string_lossy().to_lowercase();
if !respelled || a.parent() != Some(dir) || ma.len() != mb.len() || ma.modified().ok() != mb.modified().ok() {
return false;
}
let Ok(entries) = std::fs::read_dir(dir) else { return false };
!entries.flatten().any(|e| e.file_name() == name)
let (mut listed_a, mut listed_b) = (false, false);
for e in entries.flatten() {
let name = e.file_name();
listed_a |= name == name_a;
listed_b |= name == name_b;
}
!(listed_a && listed_b)
}
}
@@ -699,15 +718,14 @@ pub(crate) fn move_file_with(fs: &dyn MoveFs, from: &str, to: &str) -> std::resu
if fs.exists(t) && !is_respelling(fs, f, t) {
return Err(fail(format!("{to} already exists")));
}
let mut packets: Vec<RetainedPacket> = sidecars(from)
.into_iter()
.map(|(p, _, _)| p)
.filter(|p| fs.is_file(p))
.map(|path| RetainedPacket { version: fs.packet_version(&path).ok(), path })
.collect();
// An upper-case `.XMP` spelling that only answers through a case-insensitive volume is the
// lower-case sidecar: track and move it under its own name, so `a.xmp` doesn't become `b.XMP`.
let present = |(sc, _, upper): &(PathBuf, SidecarNaming, bool)| fs.is_file(sc) && (!*upper || fs.listed(sc));
let mut packets: Vec<RetainedPacket> =
sidecars(from).into_iter().filter(present).map(|(path, _, _)| RetainedPacket { version: fs.packet_version(&path).ok(), path }).collect();
move_one(fs, f, t).map_err(|e| fail(format!("rename {from}: {e}")))?;
let mut done: Vec<SidecarMove> = Vec::new();
let mut remaining = sidecars(from).into_iter().filter(|(sc, _, _)| fs.is_file(sc)).map(|(sc, naming, upper)| {
let mut remaining = sidecars(from).into_iter().filter(present).map(|(sc, naming, upper)| {
let mut dst = crate::sidecar::sidecar_path(to, naming);
if upper {
dst = dst.with_extension("XMP");
@@ -1124,6 +1142,24 @@ pub(crate) mod tests {
use super::*;
use std::cell::RefCell;
/// Windows has no file id in std: two spellings of one file are the same file whichever is
/// asked about first (a save after a partial rename saw `Trip.XMP` as a collision with the
/// known `Trip.xmp`), and only the spelling on disk is listed.
#[cfg(windows)]
#[test]
fn same_file_spellings_are_symmetric_on_windows() {
let dir = std::env::temp_dir().join(format!("lc-same-file-spelling-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let (real, alias, other) = (dir.join("Trip.xmp"), dir.join("Trip.XMP"), dir.join("other.xmp"));
std::fs::write(&real, b"packet").unwrap();
std::fs::write(&other, b"packet").unwrap();
assert!(RealFs.same_file(&real, &alias) && RealFs.same_file(&alias, &real));
assert!(!RealFs.same_file(&real, &other) && !RealFs.same_file(&other, &alias));
assert!(RealFs.listed(&real) && !RealFs.listed(&alias));
std::fs::remove_dir_all(&dir).unwrap();
}
/// The cross-volume move's check compares files chunk by chunk: equal files across several
/// chunks, a difference in the last chunk, different lengths, a missing file.
#[test]
@@ -1171,7 +1207,10 @@ pub(crate) mod tests {
}
fn idx(&self, p: &Path) -> Option<usize> {
let p = Self::norm(p);
self.files.borrow().iter().position(|(f, _)| if self.ci { folded_path(f) == folded_path(&p) } else { *f == p })
self.files.borrow().iter().position(|(f, _)| {
let f = f.replace('\\', "/");
if self.ci { folded_path(&f) == folded_path(&p) } else { f == p }
})
}
/// The listing: (exact path, contents), sorted.
pub fn listing(&self) -> Vec<(String, String)> {
@@ -1196,7 +1235,7 @@ pub(crate) mod tests {
self.idx(a).is_some() && self.idx(a) == self.idx(b)
}
fn rename_no_replace(&self, a: &Path, b: &Path) -> std::io::Result<()> {
if self.fail.borrow().iter().any(|f| Self::norm(a).contains(f.as_str())) {
if self.fail.borrow().iter().any(|f| Self::norm(a).contains(&f.replace('\\', "/"))) {
return Err(std::io::Error::other("injected failure"));
}
if self.exists(b) {
@@ -1207,7 +1246,7 @@ pub(crate) mod tests {
Ok(())
}
fn copy_no_replace(&self, a: &Path, b: &Path) -> std::io::Result<()> {
if self.fail.borrow().iter().any(|f| Self::norm(a).contains(f.as_str())) {
if self.fail.borrow().iter().any(|f| Self::norm(a).contains(&f.replace('\\', "/"))) {
return Err(std::io::Error::other("injected failure"));
}
if self.exists(b) {
@@ -1223,6 +1262,10 @@ pub(crate) mod tests {
self.files.borrow_mut().remove(i);
Ok(())
}
fn listed(&self, p: &Path) -> bool {
let p = Self::norm(p);
self.files.borrow().iter().any(|(f, _)| f.replace('\\', "/") == p)
}
fn stem_sibling(&self, sidecar: &Path) -> Option<String> {
let stem = Self::norm(&sidecar.with_extension(""));
self.files.borrow().iter().map(|(f, _)| f.clone()).find(|f| {
@@ -1293,7 +1336,7 @@ pub(crate) mod tests {
} else {
serde_json::json!({"path": from, "into": root.join("Archive")})
};
let to = if command == "folder.rename" { root.join("Holiday") } else { root.join("Archive/Trip") };
let to = if command == "folder.rename" { root.join("Holiday") } else { root.join("Archive").join("Trip") };
let before = s.catalog.to_snapshot();
let history = (s.undo.len(), s.redo.len());
assert!(s.execute(command, &params).is_err());
@@ -1421,6 +1464,34 @@ pub(crate) mod tests {
assert!(move_file_with(&fs, "/p/A.JPG", "/p/B.JPG").is_err());
}
/// On a case-insensitive volume (Windows, macOS) `a.XMP` answers for `a.xmp`: a renamed
/// photo's sidecars keep the spelling they have on disk, and a failure on the real spelling
/// is the one reported and queued.
#[test]
fn case_insensitive_renames_keep_each_sidecars_spelling() {
let fs = FakeFs::new(true, &[("/p/a.jpg", "A"), ("/p/a.xmp", "stem"), ("/p/a.jpg.XMP", "full"), ("/p/b.jpg", "B"), ("/p/b.Xmp", "mixed")]);
move_file_with(&fs, "/p/a.jpg", "/p/c.jpg").unwrap();
move_file_with(&fs, "/p/b.jpg", "/p/d.jpg").unwrap();
assert_eq!(
fs.listing(),
vec![
("/p/c.jpg".into(), "A".into()),
("/p/c.jpg.XMP".into(), "full".into()),
("/p/c.xmp".into(), "stem".into()),
("/p/d.jpg".into(), "B".into()),
("/p/d.xmp".into(), "mixed".into())
]
);
// the move of the sidecar's real spelling is the one attempted (and here fails): the photo
// goes back and nothing changed
let before = fs.listing();
*fs.fail.borrow_mut() = vec!["/p/c.xmp".into()];
let e = move_file_with(&fs, "/p/c.jpg", "/p/a.jpg").unwrap_err();
assert!(e.message.replace('\\', "/").contains("rename /p/c.xmp: injected failure"), "{e}");
assert!(!e.moved && e.pending_sidecars.is_empty());
assert_eq!(fs.listing(), before);
}
/// Issue #105: a rollback that fails is reported, and the library follows the files that kept
/// their new names (an undoable partial rename), so nothing goes missing.
#[test]
+6 -1
View File
@@ -566,7 +566,12 @@ impl Session {
.map_err(|e| EngineError::Other(format!("{}: unfinished XMP repair: {e}", preferred.display())))?;
// Keep an existing packet's spelling/style so foreign properties survive this save and
// subsequent Undo/Redo writes. Shared stems owned by another photo were excluded above.
let path = candidates.into_iter().find(|path| path.is_file()).unwrap_or(preferred);
// On a case-insensitive volume `a.xmp` also answers for `a.XMP`: the spelling the folder
// lists comes first, so the atomic write doesn't rename the packet.
use crate::rename::MoveFs;
let existing: Vec<PathBuf> = candidates.into_iter().filter(|path| path.is_file()).collect();
let listed = existing.iter().find(|path| crate::rename::RealFs.listed(path)).cloned();
let path = listed.or_else(|| existing.into_iter().next()).unwrap_or(preferred);
self.validate_sidecar_packets_for_paths(std::slice::from_ref(&path), &crate::rename::RealFs)?;
let p = self.catalog.photo(id).ok_or(lightcraft_catalog::CatalogError::NoPhoto(id))?;
let err = |e: std::io::Error| EngineError::Other(format!("{}: {e}", path.display()));
@@ -1,6 +1,7 @@
//! Real Session partial moves, sidecar serialization/merging, AutoWrite and history.
use super::*;
use crate::rename::{MoveFs, RealFs};
use crate::tests_xmp::{same_path_for_volume, same_path_or_file};
use lightcraft_catalog::{Op, Photo, Source};
use serde_json::json;
@@ -16,7 +17,7 @@ impl MoveFs for Faults {
RealFs.same_file(a, b)
}
fn rename_no_replace(&self, a: &Path, b: &Path) -> std::io::Result<()> {
if self.0.iter().any(|p| p == a) {
if self.0.iter().any(|p| same_path_or_file(p, a)) {
Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "injected partial move"))
} else {
RealFs.rename_no_replace(a, b)
@@ -261,8 +262,8 @@ fn queued_stem_retry_preserves_shared_packet_after_batch_rollback() {
assert!(!second_packet.exists());
let pending = queues(&s);
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].from, shared);
assert_eq!(pending[0].to, second_packet);
assert!(same_path_for_volume(&pending[0].from, &shared), "{:?}", pending[0].from);
assert!(same_path_for_volume(&pending[0].to, &second_packet), "{:?}", pending[0].to);
let catalog_before = s.catalog.to_snapshot();
let history_before = (s.undo.len(), s.redo.len(), s.commits());
@@ -1,6 +1,7 @@
//! Native files and genuine Session entry points: packet ownership survives partial renames.
use super::*;
use crate::rename::{MoveFs, RealFs};
use crate::tests_xmp::same_path_or_file;
use lightcraft_catalog::{Op, Photo, Source};
use serde_json::json;
@@ -16,7 +17,7 @@ impl MoveFs for Faults {
RealFs.same_file(a, b)
}
fn rename_no_replace(&self, a: &Path, b: &Path) -> std::io::Result<()> {
if self.0.iter().any(|p| p == a) {
if self.0.iter().any(|p| same_path_or_file(p, a)) {
Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "injected provenance partial move"))
} else {
RealFs.rename_no_replace(a, b)
@@ -1,6 +1,7 @@
//! Real Session/serializer regressions for saves that overlap retained physical repairs.
use super::*;
use crate::rename::{MoveFs, RealFs};
use crate::tests_xmp::same_path_or_file;
use lightcraft_catalog::{Op, Photo, Source};
use serde_json::json;
@@ -19,14 +20,14 @@ impl MoveFs for PartialRename {
RealFs.same_file(a, b)
}
fn rename_no_replace(&self, a: &Path, b: &Path) -> std::io::Result<()> {
if a == self.sidecar || a == self.photo {
if same_path_or_file(a, &self.sidecar) || same_path_or_file(a, &self.photo) {
Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "transient rename fault"))
} else {
RealFs.rename_no_replace(a, b)
}
}
fn copy_no_replace(&self, a: &Path, b: &Path) -> std::io::Result<()> {
if a == self.sidecar {
if same_path_or_file(a, &self.sidecar) {
Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "transient copy fault"))
} else {
RealFs.copy_no_replace(a, b)
@@ -151,7 +152,9 @@ fn autowrite_repairs_partial_rename_before_merging_foreign_xmp_across_library_sw
assert_eq!(parsed.title.as_deref(), Some("Final title"));
assert!(parsed.keywords.as_ref().unwrap().contains(&"café".to_string()));
if upper {
assert!(!preferred.exists(), "merge uses the repaired uppercase packet");
// by listing: on a case-insensitive volume `preferred` answers for the upper-case packet
let listed = std::fs::read_dir(&root).unwrap().any(|e| Some(e.unwrap().file_name().as_os_str()) == preferred.file_name());
assert!(!listed, "merge uses the repaired uppercase packet");
}
for (step, title) in [("edit.undo", "Fresh title"), ("edit.redo", "Final title")] {
s.execute(step, &json!({})).unwrap();
+241 -15
View File
@@ -55,6 +55,28 @@ const HEADER_MAX: u64 = 4096;
/// wrote the proxy's pixels (colour matrix and profile are baked into them) and stored tone curve.
const LOOK_FIELD: &str = "look_version";
/// The JPEG quality a smart preview's pixels are stored at. The original 92 left the 8×8 DCT's
/// quantised DC — each block's own mean, luma and chroma — visible as block steps across the frame
/// and as a small constant colour shift; that is the largest part of what a proxy gets wrong
/// against the original, and the only part a parameter can reach. 97 shrinks those quantisers to
/// roughly 1/2.6 of their steps, which is what removes the steps, at about twice the file size
/// (~1 MB per preview at 2560 px). What is left is the 8-bit sRGB quantisation itself, which only
/// a lossless proxy format can close. (ICED: the measured change is printed by the fidelity test
/// below; drop this to 95 for ~1.4× the size and roughly half the gain.)
const JPEG_QUALITY: u8 = 97;
/// The header field that records which version of this module's own encoding wrote a proxy's pixels
/// (the source mapping above and [`JPEG_QUALITY`]). Not to be confused with [`LOOK_FIELD`], which
/// tracks the camera-look fit that is baked into those pixels.
const ENC_FIELD: &str = "enc";
/// The value of [`ENC_FIELD`] this build writes. Bump it whenever [`encode`]'s mapping or JPEG
/// settings change, so proxies an older build wrote are rebuilt: a missing field reads as 0, which
/// makes every proxy from before this version stale. Version 2 is the first to record the field at
/// all (a version 1 proxy is stored at JPEG quality 92, which this build raises to [`JPEG_QUALITY`]).
/// Crate-visible so the look-marker test pins the marker format to it instead of to a literal.
pub(crate) const ENCODE_VERSION: u64 = 2;
/// Encode a source image (and the decoder's camera tone curve, if any) as a smart preview.
pub fn encode(img: &Rgb32f, tone: Option<&CameraTone>) -> Result<Vec<u8>, String> {
// scale so all but the brightest 0.05 % fit into 0..1
@@ -78,13 +100,19 @@ pub fn encode(img: &Rgb32f, tone: Option<&CameraTone>) -> Result<Vec<u8>, String
let rgba = Rgba8 { width: img.width, height: img.height, data };
let jpg = lightcraft_codecs::encode_jpeg(
&lightcraft_codecs::EncodeImage::rgba8(&rgba),
92,
JPEG_QUALITY,
lightcraft_codecs::ChromaSubsampling::S444,
&Default::default(),
)
.map_err(|e| e.to_string())?;
let mut out = MAGIC.to_vec();
let mut head = serde_json::json!({"w": img.width, "h": img.height, "scale": scale, LOOK_FIELD: crate::camera_preview::LOOK_VERSION});
let mut head = serde_json::json!({
"w": img.width,
"h": img.height,
"scale": scale,
LOOK_FIELD: crate::camera_preview::LOOK_VERSION,
ENC_FIELD: ENCODE_VERSION,
});
if let Some(t) = tone {
head["tone"] = serde_json::to_value(t).map_err(|e| e.to_string())?;
}
@@ -265,10 +293,17 @@ fn header_look_version(head: &serde_json::Value) -> u64 {
head.get(LOOK_FIELD).and_then(serde_json::Value::as_u64).unwrap_or(0)
}
/// Whether the complete proxy at `path` was written by an older camera-look fit than this build's
/// (so its stored curve, and the colour baked into its pixels, are out of date). A proxy from a
/// newer build is left alone (rebuilding it here would only undo that build's work), as is one
/// that can't be read: those are [`is_valid`]'s business.
/// The encode version a header records: a missing, negative, fractional or otherwise malformed
/// value counts as 0 (older than any real version), like [`header_look_version`].
fn header_encode_version(head: &serde_json::Value) -> u64 {
head.get(ENC_FIELD).and_then(serde_json::Value::as_u64).unwrap_or(0)
}
/// Whether the complete proxy at `path` is out of date for this build: written by an older
/// camera-look fit than this build's (so its stored curve, and the colour baked into its pixels, are
/// out of date), or by an older encoding of its own pixels ([`ENCODE_VERSION`], i.e. a lower JPEG
/// quality). A proxy from a newer build is left alone (rebuilding it here would only undo that
/// build's work), as is one that can't be read: those are [`is_valid`]'s business.
pub fn is_stale(path: &Path) -> bool {
use std::io::Read;
if !is_valid(path) {
@@ -283,15 +318,18 @@ pub fn is_stale(path: &Path) -> bool {
let Some(nl) = rest.iter().position(|b| *b == b'\n') else { return false };
rest.get(..nl)
.and_then(|h| serde_json::from_slice::<serde_json::Value>(h).ok())
.is_some_and(|h| header_look_version(&h) < u64::from(crate::camera_preview::LOOK_VERSION))
.is_some_and(|h| header_look_version(&h) < u64::from(crate::camera_preview::LOOK_VERSION) || header_encode_version(&h) < ENCODE_VERSION)
}
/// The marker file in a smart previews folder: the `LOOK_VERSION` its proxies were last checked
/// at ([`look_checked`]).
/// The marker file in a smart previews folder: the camera-look fit version and encode version its
/// proxies were last checked at ([`look_checked`]).
const LOOK_MARKER: &str = "look-version";
/// Whether the smart previews folder `dir` has been checked for stale proxies at this build's
/// camera-look fit version (or a newer one). A missing or unreadable marker counts as not.
/// Whether the smart previews folder `dir` has been checked for out-of-date proxies at this build's
/// camera-look fit version and encode version (or newer ones). A missing or unreadable marker counts
/// as not, and so does one written before the encode version was recorded — it only says the look
/// was checked, so this build scans the folder once more and rebuilds the proxies [`is_stale`]
/// finds.
pub fn look_checked(dir: &Path) -> bool {
use std::io::Read;
let Ok(f) = std::fs::File::open(dir.join(LOOK_MARKER)) else { return false };
@@ -299,14 +337,20 @@ pub fn look_checked(dir: &Path) -> bool {
if f.take(64).read_to_string(&mut text).is_err() {
return false;
}
text.trim().parse::<u64>().is_ok_and(|v| v >= u64::from(crate::camera_preview::LOOK_VERSION))
let mut versions = text.split_whitespace();
let (Some(look), Some(enc)) = (versions.next(), versions.next()) else { return false };
match (look.parse::<u64>(), enc.parse::<u64>()) {
(Ok(look), Ok(enc)) => look >= u64::from(crate::camera_preview::LOOK_VERSION) && enc >= ENCODE_VERSION,
_ => false,
}
}
/// Record that the proxies in `dir` were checked at this build's camera-look fit version.
/// Record that the proxies in `dir` were checked at this build's camera-look fit version and encode
/// version.
pub fn mark_look_checked(dir: &Path) -> Result<(), String> {
let path = dir.join(LOOK_MARKER);
lightcraft_catalog::safe_file::write_atomic(&path, format!("{}\n", crate::camera_preview::LOOK_VERSION).as_bytes())
.map_err(|e| format!("{}: {e}", path.display()))
let text = format!("{} {}\n", crate::camera_preview::LOOK_VERSION, ENCODE_VERSION);
lightcraft_catalog::safe_file::write_atomic(&path, text.as_bytes()).map_err(|e| format!("{}: {e}", path.display()))
}
/// Load the proxy at `path`.
@@ -451,4 +495,186 @@ mod tests {
assert!(mean < 0.02, "mean encoded error {mean}");
assert!(decode(b"nope").is_err());
}
/// A smooth, deterministic stand-in for a photo: a bright, near-neutral "sky" crossing over into
/// a saturated colour field, every value inside 0..1 so the proxy's 0.05 % highlight clip never
/// enters what is measured. Built from a rendered demo scene so the dimensions and the flat
/// row-major layout are the real ones.
fn smooth_scene(w: usize, h: usize) -> Rgb32f {
let mut img = lightcraft_scenes::demo_library()[0].render(w, h);
let (w, h) = (img.width.max(1), img.height.max(1));
for (i, px) in img.data.iter_mut().enumerate() {
let u = (i % w) as f32 / (w - 1).max(1) as f32;
let v = (i / w) as f32 / (h - 1).max(1) as f32;
let t = (v * v * (3.0 - 2.0 * v)).clamp(0.0, 1.0); // smoothstep: a fade, not a step
let sky = [0.10 + 0.70 * v, 0.14 + 0.72 * v, 0.30 + 0.62 * v];
let field = [0.86 - 0.26 * u, 0.18 + 0.62 * u, 0.12 + 0.30 * (1.0 - u)];
*px = std::array::from_fn(|k| (sky[k] * (1.0 - t) + field[k] * t).clamp(0.0, 1.0));
}
img
}
/// CIE L*a*b* (D65) of a scene-linear colour: the perceptual yardstick a proxy is held to. The
/// input is clamped into 0..1, the only range a proxy ever stores.
fn lab(c: [f32; 3]) -> [f64; 3] {
let (r, g, b) = (c[0].clamp(0.0, 1.0) as f64, c[1].clamp(0.0, 1.0) as f64, c[2].clamp(0.0, 1.0) as f64);
let (x, y, z) = (
(0.4124 * r + 0.3576 * g + 0.1805 * b) / 0.95047,
0.2126 * r + 0.7152 * g + 0.0722 * b,
(0.0193 * r + 0.1192 * g + 0.9505 * b) / 1.08883,
);
let f = |t: f64| if t > 216.0 / 24389.0 { t.cbrt() } else { (841.0 / 108.0) * t + 4.0 / 29.0 };
let (fx, fy, fz) = (f(x), f(y), f(z));
[116.0 * fy - 16.0, 500.0 * (fx - fy), 200.0 * (fy - fz)]
}
/// The mean CIE76 ΔE between two same-sized scene-linear images: monotone in CIEDE2000 for a
/// low-contrast, near-neutral difference like a proxy's, and enough to rank two encoders.
fn mean_delta_e(a: &Rgb32f, b: &Rgb32f) -> f64 {
let n = a.data.len().min(b.data.len());
if n == 0 {
return 0.0;
}
(0..n)
.map(|i| {
let (p, q) = (lab(a.data[i]), lab(b.data[i]));
((p[0] - q[0]).powi(2) + (p[1] - q[1]).powi(2) + (p[2] - q[2]).powi(2)).sqrt()
})
.sum::<f64>()
/ n as f64
}
/// A deliberately frozen copy of the pixel mapping and container this module used before the
/// quality raise: the 99.95 % highlight scale, the sRGB 8-bit mapping, JPEG quality 92, and the
/// version-1 header (width, height, scale and camera-look fit — no encode version). A copy on
/// purpose: it must not follow `encode` as that changes, so the fidelity test measures what the
/// quality raise bought instead of trusting a written-down number.
fn encode_at_q92(img: &Rgb32f) -> Vec<u8> {
let mut lum: Vec<f32> = img.data.iter().map(|c| c[0].max(c[1]).max(c[2])).filter(|v| v.is_finite()).collect();
let scale = if lum.is_empty() {
1.0
} else {
let k = ((lum.len() as f64) * 0.9995) as usize;
let k = k.min(lum.len() - 1);
let (_, v, _) = lum.select_nth_unstable_by(k, f32::total_cmp);
v.max(1.0)
};
let data: Vec<[u8; 4]> = img
.data
.iter()
.map(|c| {
let e = |v: f32| (linear_to_srgb((v / scale).clamp(0.0, 1.0)) * 255.0).round() as u8;
[e(c[0]), e(c[1]), e(c[2]), 255]
})
.collect();
let rgba = Rgba8 { width: img.width, height: img.height, data };
let jpg = lightcraft_codecs::encode_jpeg(
&lightcraft_codecs::EncodeImage::rgba8(&rgba),
92,
lightcraft_codecs::ChromaSubsampling::S444,
&Default::default(),
)
.unwrap();
// the version-1 container the decoder needs: magic, header, JPEG — and no `enc` field
let mut out = MAGIC.to_vec();
let head = serde_json::json!({"w": img.width, "h": img.height, "scale": scale, LOOK_FIELD: crate::camera_preview::LOOK_VERSION});
out.extend_from_slice(head.to_string().as_bytes());
out.push(b'\n');
out.extend_from_slice(&jpg);
out
}
/// The mean CIE76 ΔE between two same-sized scene-linear images, each reduced first to one
/// sample per 8×8 block — the block's mean colour. JPEG stores a block's DC coefficient, its
/// mean, and rebuilds the whole block from it, so a proxy differs from the original by a
/// constant per block: the "8×8 block pattern" a preview shows. Averaging a block's 64 pixels
/// keeps that constant while shrinking the 8-bit sRGB rounding the two encoders share
/// eightfold, which is what a per-pixel ΔE drowns in.
fn mean_block_delta_e(a: &Rgb32f, b: &Rgb32f) -> f64 {
let w = a.width.min(b.width).max(1);
let h = a.height.min(b.height).max(1);
let (mut sum, mut n) = (0.0f64, 0usize);
for by in (0..h).step_by(8) {
for bx in (0..w).step_by(8) {
let (mut sa, mut sb, mut count) = ([0.0f64; 3], [0.0f64; 3], 0u32);
for y in by..(by + 8).min(h) {
for x in bx..(bx + 8).min(w) {
if let (Some(ca), Some(cb)) = (a.data.get(y * a.width + x), b.data.get(y * b.width + x)) {
for k in 0..3 {
sa[k] += f64::from(ca[k]);
sb[k] += f64::from(cb[k]);
}
count += 1;
}
}
}
let d = f64::from(count.max(1));
let (pa, pb) = (
lab([(sa[0] / d) as f32, (sa[1] / d) as f32, (sa[2] / d) as f32]),
lab([(sb[0] / d) as f32, (sb[1] / d) as f32, (sb[2] / d) as f32]),
);
sum += ((pa[0] - pb[0]).powi(2) + (pa[1] - pb[1]).powi(2) + (pa[2] - pb[2]).powi(2)).sqrt();
n += 1;
}
}
sum / n.max(1) as f64
}
/// for [`a_smooth_proxy_keeps_the_scene_closer_than_the_old_quality`] to pass. The shared 8-bit
/// sRGB quantization floor both encoders share caps the achievable ratio at 0.632 on this scene,
/// a no-op encoder scores 1.000, and this is their geometric midpoint: it fails the moment the
/// quality raise is reverted and never trips on the honest result (measured 0.683).
const PROXY_BLOCK_GAIN: f64 = 0.8;
#[test]
fn a_smooth_proxy_keeps_the_scene_closer_than_the_old_quality() {
let img = smooth_scene(256, 160);
let ours = decode(&encode(&img, None).unwrap()).unwrap().0;
let before = decode(&encode_at_q92(&img)).unwrap().0;
let block = (mean_block_delta_e(&img, &ours), mean_block_delta_e(&img, &before));
let pixel = (mean_delta_e(&img, &ours), mean_delta_e(&img, &before));
println!(
"smooth-scene proxy round-trip: block-mean ΔE76 {:.3} at q{JPEG_QUALITY} vs {:.3} at q92 \
(per-pixel {:.3} vs {:.3})",
block.0, block.1, pixel.0, pixel.1
);
assert!(block.1 > 0.0, "the old encoder must measure something to compare against");
assert!(
block.0 <= PROXY_BLOCK_GAIN * block.1,
"q{JPEG_QUALITY} leaves a block-mean ΔE of {:.3}, not the ≤{:.3} the quality raise should buy over q92's {:.3} \
(the 8-bit sRGB floor caps the achievable ratio at ~0.63 on this scene)",
block.0,
PROXY_BLOCK_GAIN * block.1,
block.1
);
}
#[test]
fn a_proxy_from_an_older_encode_is_stale() {
let dir = temp("encode-version");
let img = lightcraft_scenes::demo_library()[0].render(64, 40);
let bytes = encode(&img, None).unwrap();
let path = dir.join("p.lcsp");
std::fs::write(&path, &bytes).unwrap();
assert!(!is_stale(&path), "this build's own proxy is up to date");
// one written before the encode version was recorded is not
let nl = MAGIC.len() + bytes[MAGIC.len()..].iter().position(|b| *b == b'\n').unwrap();
let head: serde_json::Value = serde_json::from_slice(&bytes[MAGIC.len()..nl]).unwrap();
assert_eq!(head[ENC_FIELD].as_u64(), Some(ENCODE_VERSION));
let mut older = head.as_object().cloned().unwrap_or_default();
older.remove(ENC_FIELD);
let mut old = MAGIC.to_vec();
old.extend_from_slice(serde_json::Value::Object(older).to_string().as_bytes());
old.extend_from_slice(&bytes[nl..]);
std::fs::write(&path, &old).unwrap();
assert!(is_valid(&path), "losing the field keeps the file readable");
assert!(is_stale(&path), "an older encoding is rebuilt");
// the marker records both versions, and one from before the encode version forces a scan
assert!(!look_checked(&dir));
mark_look_checked(&dir).unwrap();
assert!(look_checked(&dir));
std::fs::write(dir.join(LOOK_MARKER), "99\n").unwrap();
assert!(!look_checked(&dir), "a look-only marker is not enough to skip the rebuild");
let _ = std::fs::remove_dir_all(&dir);
}
}
+20 -2
View File
@@ -81,11 +81,11 @@ fn quick_develop_reset_and_batch_paths_fit_too() {
let got = try_cmd(&mut s, "develop.set", json!({"control": "crop.angle", "value": 12.0, "ids": [id.0]}));
assert!(close(got, want), "develop.set ids {got:?} vs {want:?}");
// back to 0°: the crop that fitted at 12° fits at 0° too and is kept
// back to 0°: the crop shrunk to fit at 12° grows back to the full photo (issue #1388)
s.execute("develop.set", &json!({"control": "crop.angle", "value": 12.0})).unwrap();
s.execute("develop.resetControl", &json!({"control": "crop.angle"})).unwrap();
let g = geometry(&s);
assert_eq!((g.rect, g.angle), (want.rect, 0.0));
assert_eq!((g.rect, g.angle), (Rect::UNIT, 0.0));
assert!(g.is_within_image(w, h));
}
@@ -106,3 +106,21 @@ fn a_straighten_slider_drag_fits_live_and_is_one_undo_step() {
s.execute("edit.undo", &json!({})).unwrap();
assert!(close(geometry(&s), start));
}
#[test]
fn a_crop_shrunk_by_straightening_grows_back_towards_0() {
// issue #1388: slide the Straighten slider out and back, in one drag or in several
let mut s = Session::with_demo();
let (w, h) = dims(&s);
s.execute("develop.beginInteraction", &json!({"label": "Straighten"})).unwrap();
for angle in [5.0, 12.0, 20.0, 8.0, -20.0, -3.0, 0.0] {
s.execute("crop.straighten", &json!({"angle": angle})).unwrap();
let g = geometry(&s);
assert!(close(g, lightcraft_geom::crop_fit_angle(w, h, angle, None)), "the largest crop at {angle}°, got {g:?}");
}
s.execute("develop.endInteraction", &json!({})).unwrap();
s.execute("crop.straighten", &json!({"angle": 20.0})).unwrap();
s.execute("crop.straighten", &json!({"angle": 0.0})).unwrap();
let g = geometry(&s);
assert!(close(g, CropGeometry::default()), "two separate drags: back to the full photo at 0°, got {g:?}");
}
+23
View File
@@ -277,6 +277,29 @@ fn a_manifest_that_does_not_validate_or_name_the_file_is_refused() {
let _ = std::fs::remove_dir_all(&x.dir);
}
/// Another app's model folder (#1381): an `.onnx` with no manifest, then with that app's `model.json` beside it, then
/// with the `model.json` renamed. Each refusal says what is wrong and where LightCraft's manifest is described.
#[test]
fn inspecting_another_apps_model_folder_says_what_is_missing() {
let dir = temp_dir("denoise-foreign");
let mut s = Session::new().with_fs();
s.set_denoise_models_dir(Some(dir.join("denoise-models")));
let src = dir.join("rawdenoise");
std::fs::create_dir_all(&src).unwrap();
let onnx = src.join("rawdenoise.onnx");
std::fs::write(&onnx, b"not an onnx file").unwrap();
let mut inspect = || s.execute("denoise.models.inspect", &json!({"path": onnx.to_string_lossy()})).unwrap_err().to_string();
let e = inspect();
assert!(e.contains("No denoise-model.json beside") && e.contains("docs/denoise.md") && !e.contains("model.json there"), "{e}");
std::fs::write(src.join("model.json"), json!({"name": "rawdenoise", "input": {"channels": 4}, "scale": 2}).to_string()).unwrap();
let e = inspect();
assert!(e.contains("No denoise-model.json beside") && e.contains("The model.json there") && e.contains("docs/denoise.md"), "{e}");
std::fs::rename(src.join("model.json"), src.join("denoise-model.json")).unwrap();
let e = inspect();
assert!(e.contains("not a denoise-model manifest") && e.contains("docs/denoise.md"), "{e}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_photo_is_denoised_in_the_background_and_the_amount_mixes_it_in() {
let mut x = setup("background", true);
+15
View File
@@ -455,6 +455,21 @@ fn hdr_avif_export_is_pq_rec2020() {
assert!(nclx(&sdr.bytes).is_none_or(|c| c[3] != 16));
}
#[test]
fn hdr_avif_export_hlg() {
let mut s = Session::with_demo();
let id = sunset(&mut s);
let o = ExportOptions::from_json(&json!({"longEdge": 160, "format": "avif", "hdr": true, "hdrTransfer": "hlg", "quality": 70}));
assert_eq!(o.hdr_transfer, crate::export::HdrCurve::Hlg);
assert_eq!(o.to_json()["hdrTransfer"], "hlg");
s.execute("develop.hdr", &json!({"enabled": true, "maxEv": 2})).unwrap();
let hdr = export_photo(&mut s, id, &o, 1).unwrap();
// nclx: BT.2020 primaries (9), HLG (18), BT.2020 NCL (9), full range; no content light level box
let i = hdr.bytes.windows(4).position(|w| w == b"nclx").expect("colr nclx box");
assert_eq!(hdr.bytes[i + 4..i + 11].to_vec(), vec![0, 9, 0, 18, 0, 9, 0x80]);
assert!(!hdr.bytes.windows(4).any(|w| w == b"clli"));
}
/// "Remove location info" holds for the file-copying formats too: the sidecar and the DNG's
/// packet carry no location, and an original that embeds GPS is refused rather than copied.
#[test]
+22 -11
View File
@@ -31,11 +31,21 @@ fn file_history_step(s: &mut Session, fs: &dyn crate::rename::MoveFs, redo: bool
})
}
/// The model volume spells paths with `/`; the code under test joins them with the host's separator.
fn slash(path: &str) -> String {
path.replace('\\', "/")
}
fn file_source(s: &Session, id: u64) -> String {
let Source::File { path } = &s.catalog.photo(PhotoId(id)).unwrap().source else { panic!("synthetic file fixture") };
slash(path)
}
fn assert_file_catalog_and_journal(s: &Session, fs: &crate::rename::tests::FakeFs, store: &lightcraft_catalog::MemStore) {
let files = fs.listing();
for photo in s.catalog.photos() {
let Source::File { path } = &photo.source else { panic!("synthetic file fixture") };
assert!(files.iter().any(|(file, _)| file == path), "photo {} points to missing {path}: {files:?}", photo.id.0);
assert!(files.iter().any(|(file, _)| *file == slash(path)), "photo {} points to missing {path}: {files:?}", photo.id.0);
assert_eq!(std::path::Path::new(path).file_name().unwrap().to_string_lossy(), photo.file_name.as_str());
}
let replayed = memory_library(store);
@@ -102,7 +112,8 @@ fn merged_renames_coalesce_physical_moves_and_keep_retry_inverses_and_journal()
let renamed = fs.listing();
s.merge_undo(2, "Rename Twice");
assert_eq!(s.undo.len(), 1);
assert_eq!(s.file_moves(&s.undo.last().unwrap().op), vec![("/p/final.jpg".into(), "/p/a.jpg".into())]);
let moves: Vec<_> = s.file_moves(&s.undo.last().unwrap().op).iter().map(|(from, to)| (slash(from), slash(to))).collect();
assert_eq!(moves, vec![("/p/final.jpg".to_string(), "/p/a.jpg".to_string())]);
// First exercise the ordinary merged step, before any corrective paths exist.
for redo in [false, true] {
file_history_step(&mut s, &fs, redo).unwrap();
@@ -274,7 +285,7 @@ fn history_retries_both_sidecar_styles_after_following_a_photo_with_failed_rollb
file_history_step(&mut s, &fs, redo).unwrap();
let mut after = before
.iter()
.map(|(path, bytes)| (path.replace(from.trim_end_matches(".jpg"), to.trim_end_matches(".jpg")), bytes.clone()))
.map(|(path, bytes)| (path.replace(slash(&from).trim_end_matches(".jpg"), slash(&to).trim_end_matches(".jpg")), bytes.clone()))
.collect::<Vec<_>>();
after.sort();
assert_eq!(fs.listing(), after);
@@ -343,7 +354,7 @@ fn initial_partial_rename_keeps_sidecar_repairs_for_noop_retry_and_history() {
let plans = s.plan_rename_with(&fs, &[PhotoId(1)], "Trip", 1);
*fs.fail.borrow_mut() = vec!["/p/a.xmp".into(), "/p/Trip.jpg".into()];
assert!(s.execute_fn("photo.rename", |s| Ok(json!({"renamed": s.apply_rename_with(&fs, &plans)?}))).is_err());
assert_eq!(s.catalog.photo(PhotoId(1)).unwrap().source, Source::File { path: "/p/Trip.jpg".into() });
assert_eq!(file_source(&s, 1), "/p/Trip.jpg");
assert_eq!(s.undo.len(), 1);
assert_eq!(s.undo.last().unwrap().retry_sidecars.len(), 2);
assert_file_catalog_and_journal(&s, &fs, &store);
@@ -513,10 +524,11 @@ fn merged_rename_failure_keeps_rolled_back_photo_identity() {
);
s.merge_undo(3, "Rename Three Times");
assert_eq!((s.undo.len(), s.redo.len()), (1, 0));
assert_eq!(s.file_moves(&s.undo.last().unwrap().op), vec![("/p/d.jpg".into(), "/p/b.jpg".into()), ("/p/c.jpg".into(), "/p/a.jpg".into())]);
let moves: Vec<_> = s.file_moves(&s.undo.last().unwrap().op).iter().map(|(from, to)| (slash(from), slash(to))).collect();
assert_eq!(moves, vec![("/p/d.jpg".to_string(), "/p/b.jpg".to_string()), ("/p/c.jpg".to_string(), "/p/a.jpg".to_string())]);
// The second photo move sticks; the preceding d -> b move must roll back to d.
*fs.fail.borrow_mut() = vec!["/p/c.xmp".into(), "/p/a.jpg".into()];
let error = file_history_step(&mut s, &fs, false).unwrap_err().to_string();
let error = slash(&file_history_step(&mut s, &fs, false).unwrap_err().to_string());
assert!(error.contains("rename /p/c.xmp: injected failure"), "{error}");
assert!(error.contains("/p/c.jpg could not be moved back"), "{error}");
let files = fs.listing();
@@ -529,9 +541,8 @@ fn merged_rename_failure_keeps_rolled_back_photo_identity() {
]
);
assert_eq!((s.undo.len(), s.redo.len()), (1, 0));
assert_eq!(s.catalog.photo(PhotoId(1)).unwrap().source, Source::File { path: "/p/a.jpg".into() });
let photo2 = s.catalog.photo(PhotoId(2)).unwrap();
assert_eq!(photo2.source, Source::File { path: "/p/d.jpg".into() }, "photo 2 rolled back physically and must still identify d.jpg");
let Source::File { path } = &photo2.source else { panic!("file fixture") };
assert_eq!(files.iter().find(|(file, _)| file == path).map(|(_, bytes)| bytes.as_str()), Some("Photo 2"));
assert_eq!(file_source(&s, 1), "/p/a.jpg");
let path = file_source(&s, 2);
assert_eq!(path, "/p/d.jpg", "photo 2 rolled back physically and must still identify d.jpg");
assert_eq!(files.iter().find(|(file, _)| *file == path).map(|(_, bytes)| bytes.as_str()), Some("Photo 2"));
}
+2 -1
View File
@@ -40,7 +40,8 @@ impl Scratch {
Scratch(dir)
}
fn path(&self, rel: &str) -> String {
self.0.join(rel).to_string_lossy().to_string()
// the host's separator throughout, as the code under test spells joined paths
self.0.join(rel).components().collect::<std::path::PathBuf>().to_string_lossy().to_string()
}
}
+100
View File
@@ -0,0 +1,100 @@
//! Feature: the History panel has a cursor. Going to a step loads its settings without adding a
//! step; later steps stay listed until a new edit drops them; undo/redo stay coherent.
use serde_json::json;
use crate::Session;
fn exposure(s: &Session) -> f64 {
s.develop_of(s.active().unwrap()).unwrap().light.exposure
}
fn list(s: &mut Session) -> (usize, Option<usize>) {
let h = s.execute("history.list", &json!({})).unwrap();
(h["history"].as_array().unwrap().len(), h["cursor"].as_u64().map(|c| c as usize))
}
/// A demo session with three exposure edits (1, 2, 3) as its latest History steps.
fn three_edits() -> (Session, usize) {
let mut s = Session::with_demo();
for v in [1.0, 2.0, 3.0] {
s.execute("develop.set", &json!({"control": "light.exposure", "value": v})).unwrap();
}
let (n, cursor) = list(&mut s);
assert_eq!(cursor, Some(n - 1), "the cursor starts at the latest step");
(s, n)
}
#[test]
fn going_back_adds_no_step_and_forward_returns() {
let (mut s, n) = three_edits();
s.execute("history.restore", &json!({"index": n - 3})).unwrap();
assert_eq!(exposure(&s), 1.0);
assert_eq!(list(&mut s), (n, Some(n - 3)), "no new step; later steps kept");
s.execute("history.next", &json!({})).unwrap();
assert_eq!(exposure(&s), 2.0);
s.execute("history.restore", &json!({"index": n - 1})).unwrap();
assert_eq!(exposure(&s), 3.0);
assert_eq!(list(&mut s), (n, Some(n - 1)));
assert!(s.execute("history.next", &json!({})).is_err(), "nothing after the latest step");
s.execute("history.previous", &json!({})).unwrap();
assert_eq!((exposure(&s), list(&mut s)), (2.0, (n, Some(n - 2))));
assert!(s.execute("history.restore", &json!({"index": n})).is_err());
assert!(s.execute("history.restore", &json!({"index": -1})).is_err());
}
#[test]
fn a_new_edit_behind_the_end_drops_the_later_steps() {
let (mut s, n) = three_edits();
s.execute("history.restore", &json!({"index": n - 3})).unwrap();
s.execute("develop.set", &json!({"control": "light.exposure", "value": -1.0})).unwrap();
assert_eq!(list(&mut s), (n - 1, Some(n - 2)), "steps 2 and 3 dropped, the new one appended");
assert_eq!(exposure(&s), -1.0);
// undo brings the dropped steps and the cursor back
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!((exposure(&s), list(&mut s)), (1.0, (n, Some(n - 3))));
s.execute("edit.redo", &json!({})).unwrap();
assert_eq!((exposure(&s), list(&mut s)), (-1.0, (n - 1, Some(n - 2))));
}
#[test]
fn undo_and_redo_move_the_cursor_with_the_settings() {
let (mut s, n) = three_edits();
s.execute("history.restore", &json!({"index": n - 3})).unwrap();
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!((exposure(&s), list(&mut s)), (3.0, (n, Some(n - 1))), "undoing a step move");
s.execute("edit.redo", &json!({})).unwrap();
assert_eq!((exposure(&s), list(&mut s)), (1.0, (n, Some(n - 3))));
}
#[test]
fn clearing_history_resets_the_cursor() {
let (mut s, n) = three_edits();
s.execute("history.restore", &json!({"index": n - 2})).unwrap();
s.execute("history.clear", &json!({})).unwrap();
assert_eq!((exposure(&s), list(&mut s)), (2.0, (1, Some(0))));
assert_eq!(s.catalog.photo(s.active().unwrap()).unwrap().history_cursor, None);
}
#[test]
fn the_cursor_survives_the_op_log_and_old_catalogs_load_at_the_end() {
let mut s = Session::with_demo();
let snap = s.catalog.to_snapshot();
let _ = s.drain_log();
for v in [1.0, 2.0] {
s.execute("develop.set", &json!({"control": "light.exposure", "value": v})).unwrap();
}
let (n, _) = list(&mut s);
s.execute("history.restore", &json!({"index": n - 2})).unwrap();
let log: String = s.drain_log().iter().map(lightcraft_catalog::Catalog::op_to_log_line).collect();
let mut c = lightcraft_catalog::Catalog::from_snapshot(&snap).unwrap();
c.replay(&log).unwrap();
assert_eq!(c.to_snapshot(), s.catalog.to_snapshot());
let id = s.active().unwrap();
assert_eq!(c.photo(id).unwrap().history_cursor, Some(n - 2));
// a photo written before the cursor existed has no `history_cursor` key: at the latest step
let mut v = serde_json::to_value(c.photo(id).unwrap()).unwrap();
v.as_object_mut().unwrap().remove("history_cursor");
let old: lightcraft_catalog::Photo = serde_json::from_value(v).unwrap();
assert_eq!((old.history_cursor, old.history_index()), (None, Some(n - 1)));
}
+42 -2
View File
@@ -21,6 +21,9 @@ fn write_png(path: &Path, seed: u8) {
let bytes = lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap();
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, bytes).unwrap();
// undated files take the file's date: the tests' clock time (2026-01-14T05:58:48)
let t = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_768_370_328);
std::fs::File::options().write(true).open(path).unwrap().set_modified(t).unwrap();
}
fn ids(v: &Value, key: &str) -> usize {
@@ -309,6 +312,27 @@ fn recently_added_covers_recent_imports_newest_first() {
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn recently_added_keeps_an_import_batch_in_file_order() {
use lightcraft_catalog::{Op, Photo, PhotoId, Source};
let mut s = Session::new();
// one import batch: one `imported` string, ids running opposite to the names
let mut by_name = Vec::new();
for (n, name) in ["IMG_0001.png", "IMG_0002.png", "IMG_0003.png", "IMG_0004.png"].iter().enumerate() {
let id = PhotoId(5000 - n as u64);
let p = Photo::new(id, Source::File { path: name.to_string() }, name, "PNG", 16, 12, "2026-09-30T09:00:00");
s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
by_name.push(name.to_string());
}
s.clock = Box::new(|| "2026-10-01T09:00:00".to_string());
for ascending in [true, false] {
s.execute("library.sort", &serde_json::json!({"key": "captureDate", "ascending": ascending})).unwrap();
s.execute("library.source", &serde_json::json!({"kind": "recentlyAdded"})).unwrap();
let names: Vec<String> = s.visible_cloned().iter().map(|id| s.catalog.photo(*id).unwrap().file_name.clone()).collect();
assert_eq!(names, by_name, "ascending: {ascending}");
}
}
/// Copy imports: a destination folder, flat / by-month folders, renamed copies (numbered in
/// import order), and a metadata preset on every photo.
#[test]
@@ -560,6 +584,14 @@ fn cube_luts_become_profiles() {
assert_eq!(s.profile_info(&id), Some(("Warm Test", "Film Looks")));
s.execute("profile.favorite", &json!({"id": id})).unwrap();
assert!(s.profile_favorites.contains(&id));
// `profiles.list` lists it next to the built-in profiles (#618)
let listed = |s: &mut Session| {
let list = s.execute("profiles.list", &json!({})).unwrap();
let list = list.as_array().unwrap();
assert!(list.iter().any(|p| p["id"] == "lc.color"), "built-in profiles stay listed");
list.iter().find(|p| p["id"] == id.as_str()).cloned()
};
assert_eq!(listed(&mut s), Some(json!({"id": id, "name": "Warm Test", "group": "Film Looks", "favorite": true, "imported": true})));
// the library keeps it
drop(s);
lightcraft_pipeline::lut::unregister(&id);
@@ -569,8 +601,10 @@ fn cube_luts_become_profiles() {
assert!(s.profile_favorites.contains(&id), "imported LUT favorite survives reopen");
assert_eq!(s.profile_info(&id), Some(("Warm Test", "Film Looks")));
assert!(lightcraft_pipeline::lut::get(&id).is_some(), "registered again on open");
assert!(listed(&mut s).is_some_and(|p| p["favorite"] == true), "listed after reopen");
s.execute("profile.deleteImported", &json!({"id": id})).unwrap();
assert!(lightcraft_pipeline::lut::get(&id).is_none());
assert_eq!(listed(&mut s), None, "a deleted profile is no longer listed");
let _ = std::fs::remove_dir_all(&src);
let _ = std::fs::remove_dir_all(&lib);
}
@@ -1404,14 +1438,20 @@ fn smart_preview_look_marker_is_read_safely() {
let dir = temp_dir("smartlook-marker");
assert!(!crate::smart::look_checked(&dir), "no marker");
let cur = crate::camera_preview::LOOK_VERSION;
let enc = crate::smart::ENCODE_VERSION;
for (text, checked) in [
(String::new(), false),
("banana".into(), false),
("-1".into(), false),
(format!("{}", cur.saturating_sub(1)), false),
("9".repeat(200), false),
(format!("{cur}\n"), true),
(format!(" {}\n", cur + 1), true),
// Written before the encode version was recorded: only says the look was checked, so the
// folder is scanned once more and the old-encoding proxies are rebuilt.
(format!("{cur}\n"), false),
(format!("{cur} {}\n", enc.saturating_sub(1)), false),
(format!("{cur} banana\n"), false),
(format!("{cur} {enc}\n"), true),
(format!(" {} {}\n", cur + 1, enc + 1), true),
] {
std::fs::write(dir.join("look-version"), &text).unwrap();
assert_eq!(crate::smart::look_checked(&dir), checked, "{text:?}");
+3
View File
@@ -22,6 +22,9 @@ fn write_png(path: &Path, seed: u8) {
let bytes = lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap();
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, bytes).unwrap();
// undated files take the file's date: the tests' clock time (2026-01-14T05:58:48)
let t = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_768_370_328);
std::fs::File::options().write(true).open(path).unwrap().set_modified(t).unwrap();
}
fn xmp(rating: u8) -> String {
+15 -6
View File
@@ -155,13 +155,22 @@ fn odd_folders_never_fail_the_check() {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
let bad = root.join(std::ffi::OsStr::from_bytes(b"caf\xe9 Previews.lrdata"));
std::fs::create_dir_all(&bad).unwrap();
std::fs::write(root.join("Cat.lrcat"), b"x").unwrap();
assert_eq!(lightroom_catalog_at(&bad), Some(LightroomPlace { catalog: Some(root.join("Cat.lrcat")) }), "non-UTF-8 name");
let lr = root.join(std::ffi::OsStr::from_bytes(b"\xff\xfe"));
// APFS (macOS) refuses names that are not valid UTF-8 (EILSEQ): there the non-UTF-8 cases are skipped and
// the symlink cases below still run on a plain-named catalog folder
let probe = root.join(std::ffi::OsStr::from_bytes(b"\xff"));
let utf8_only = std::fs::create_dir(&probe).is_err();
let _ = std::fs::remove_dir(&probe);
if !utf8_only {
let bad = root.join(std::ffi::OsStr::from_bytes(b"caf\xe9 Previews.lrdata"));
std::fs::create_dir_all(&bad).unwrap();
std::fs::write(root.join("Cat.lrcat"), b"x").unwrap();
assert_eq!(lightroom_catalog_at(&bad), Some(LightroomPlace { catalog: Some(root.join("Cat.lrcat")) }), "non-UTF-8 name");
}
let lr = root.join(if utf8_only { std::ffi::OsStr::new("lr") } else { std::ffi::OsStr::from_bytes(b"\xff\xfe") });
std::fs::create_dir_all(&lr).unwrap();
std::fs::write(lr.join(std::ffi::OsStr::from_bytes(b"\xff.lrcat")), b"x").unwrap();
if !utf8_only {
std::fs::write(lr.join(std::ffi::OsStr::from_bytes(b"\xff.lrcat")), b"x").unwrap();
}
std::fs::write(lr.join("b.lrcat"), b"x").unwrap();
assert_eq!(lightroom_catalog_at(&lr), Some(LightroomPlace { catalog: Some(lr.join("b.lrcat")) }), "non-UTF-8 entries skipped");
// a symlink to a catalog folder is one too; a dangling one is not
+25
View File
@@ -67,6 +67,31 @@ fn hdr_merge_command_creates_and_imports_a_dng() {
let _ = std::fs::remove_dir_all(&dir);
}
/// Raws without colour matrices of their own (CR2, NEF, ARW…) merge into a DNG with the colour they open with,
/// not with no colour at all (issue #965: an HDR merge of CR2s rendered green and dark).
#[test]
fn a_merge_of_raws_without_colour_matrices_keeps_their_colour() {
let dir = temp_dir("hdr-colour");
let neutral = [0.5, 1.0, 0.7];
let mut paths = Vec::new();
for (i, b) in lightcraft_merge::synth::bracket_dngs(240, 160, &[-1.0, 0.0, 1.0]).unwrap().into_iter().enumerate() {
// the same pixels with a white balance but no colour matrices
let mut raw = lightcraft_raw::decode(&b).unwrap();
raw.color = lightcraft_raw::ColorData { as_shot_neutral: Some(neutral), ..Default::default() };
let p = dir.join(format!("IMG_{i}.dng"));
std::fs::write(&p, lightcraft_raw::write_dng(&raw, &Default::default()).unwrap()).unwrap();
paths.push(p.to_string_lossy().to_string());
}
let mut s = Session::new().with_fs();
let ids = import(&mut s, paths);
let r = s.execute("merge.hdr", &json!({"ids": ids, "autoSettings": false})).unwrap();
let merged = lightcraft_raw::decode(&std::fs::read(r["path"].as_str().unwrap()).unwrap()).unwrap();
assert!(lightcraft_raw::color::has_matrix(&merged.color), "{:?}", merged.color);
let got = merged.color.as_shot_neutral.expect("the sources' white balance");
assert!(got.iter().zip(neutral).all(|(a, b)| (a - b).abs() < 1e-3), "{got:?}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn panorama_command_with_auto_crop() {
let dir = temp_dir("pano");
+126
View File
@@ -0,0 +1,126 @@
//! A library with thousands of user presets (a Lightroom preset collection imported whole) stays
//! responsive: commands that don't touch presets never re-serialize them, and opening or importing
//! doesn't compare every preset with every other.
use std::path::PathBuf;
use std::time::Instant;
use lightcraft_develop::Preset;
use serde_json::json;
use crate::Session;
fn temp_dir(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-presets-scale-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn many(n: usize) -> Vec<Preset> {
(0..n)
.map(|i| Preset {
id: format!("user.coll-{}.look-{i}", i / 100),
name: format!("Look {i}"),
group: format!("Collection {}", i / 100),
settings: json!({"light": {"exposure": (i % 40) as f64 / 10.0 - 2.0, "contrast": (i % 50) as f64}, "color": {"vibrance": (i % 30) as f64}}),
favorite: false,
builtin: false,
})
.collect()
}
#[test]
fn develop_commands_do_not_reserialize_unchanged_presets() {
let lib = temp_dir("no-reserialize");
let mut s = Session::new();
s.open_library(&lib, true).unwrap();
let added = s.add_presets(many(2000));
assert_eq!(added.len(), 2000);
s.persist().unwrap();
assert!(std::fs::metadata(lib.join("presets.json")).unwrap().len() > 100_000, "presets saved");
let before = crate::library::presets_serialized();
for i in 0..20 {
s.execute("develop.set", &json!({"control": "light.exposure", "value": i as f64 / 10.0})).unwrap();
}
assert_eq!(crate::library::presets_serialized(), before, "slider commands left the preset list alone");
// a preset change is still saved
let pid = added[7].clone();
s.execute("preset.favorite", &json!({"id": pid})).unwrap();
assert!(crate::library::presets_serialized() > before);
s.close_library().unwrap();
drop(s);
let mut t = Session::new();
t.open_library(&lib, false).unwrap();
assert!(t.presets.iter().any(|p| p.id == pid && p.favorite), "favourite persisted");
assert_eq!(t.presets.iter().filter(|p| !p.builtin).count(), 2000);
let _ = std::fs::remove_dir_all(&lib);
}
#[test]
fn preset_list_generation_follows_mutable_access_only() {
let mut s = Session::new();
let g0 = s.presets.generation();
let _ = s.presets.iter().filter(|p| p.builtin).count();
let _ = s.presets.len();
assert_eq!(s.presets.generation(), g0, "reads keep the generation");
s.presets.push(many(1).remove(0));
let g1 = s.presets.generation();
assert_ne!(g1, g0);
for p in &mut s.presets {
p.favorite = false;
}
assert_ne!(s.presets.generation(), g1);
// generations are unique across sessions (UI caches key on them alone)
let other = Session::new();
assert_ne!(other.presets.generation(), s.presets.generation());
}
#[test]
fn importing_thousands_of_presets_is_not_quadratic() {
let mut s = Session::new();
let t = Instant::now();
let added = s.add_presets(many(20_000));
assert_eq!(added.len(), 20_000);
// the same presets again are all skipped as duplicates
assert!(s.add_presets(many(20_000)).is_empty());
// a clashing id with different settings gets a fresh, unique one
let mut clash = many(1).remove(0);
clash.settings = json!({"light": {"exposure": 3.0}});
let id = s.add_presets(vec![clash]).remove(0);
assert_ne!(id, "user.coll-0.look-0");
assert_eq!(s.presets.iter().filter(|p| p.id == id).count(), 1);
// quadratic was ~200 M comparisons here (tens of seconds in debug builds)
assert!(t.elapsed().as_secs() < 20, "took {:?}", t.elapsed());
}
/// Timing with a real preset collection: `LIGHTCRAFT_BENCH_PRESETS=path/to/presets.json cargo test
/// -p lightcraft-engine --release presets_scale_bench -- --ignored --nocapture`. The file is copied
/// into a temporary library; the original is only read.
#[test]
#[ignore]
fn presets_scale_bench() {
let Ok(src) = std::env::var("LIGHTCRAFT_BENCH_PRESETS") else {
eprintln!("set LIGHTCRAFT_BENCH_PRESETS");
return;
};
let lib = temp_dir("bench");
std::fs::copy(&src, lib.join("presets.json")).unwrap();
let mut s = Session::new();
let t = Instant::now();
s.open_library(&lib, true).unwrap();
eprintln!("open library: {:?} ({} presets)", t.elapsed(), s.presets.len());
let t = Instant::now();
for i in 0..20 {
s.execute("develop.set", &json!({"control": "light.exposure", "value": i as f64 / 10.0})).unwrap();
}
eprintln!("develop.set: {:?} per command", t.elapsed() / 20);
let pid = s.presets.iter().rev().find(|p| !p.builtin).map(|p| p.id.clone()).unwrap();
let t = Instant::now();
s.execute("preset.apply", &json!({"id": pid, "amount": 100})).unwrap();
eprintln!("preset.apply: {:?}", t.elapsed());
let t = Instant::now();
s.execute("preset.favorite", &json!({"id": pid})).unwrap();
eprintln!("preset.favorite (saves presets.json): {:?}", t.elapsed());
let _ = std::fs::remove_dir_all(&lib);
}
+17
View File
@@ -4,6 +4,23 @@ use serde_json::json;
use crate::Session;
/// `library.up` / `library.down` (#1171): move by the given number of photos, clamped at the ends.
#[test]
fn up_and_down_move_by_rows_and_stop_at_the_ends() {
let mut s = Session::with_demo();
let vis = s.visible_cloned();
assert!(vis.len() >= 6);
s.selection = crate::Selection::single(vis[0]);
s.execute("library.down", &json!({"rows": 4})).unwrap();
assert_eq!(s.selection.active, Some(vis[4]));
s.execute("library.up", &json!({})).unwrap();
assert_eq!(s.selection.active, Some(vis[3]), "default is one photo (the filmstrip)");
s.execute("library.up", &json!({"rows": 99})).unwrap();
assert_eq!(s.selection.active, Some(vis[0]), "clamped at the first photo");
s.execute("library.down", &json!({"rows": u64::MAX})).unwrap();
assert_eq!(s.selection.active, vis.last().copied(), "hostile counts clamp, never overflow");
}
fn spots(s: &Session) -> Vec<lightcraft_develop::Spot> {
s.develop_of(s.active().unwrap()).unwrap().spots.clone()
}
+85
View File
@@ -7,6 +7,26 @@ use lightcraft_catalog::{ColorLabel, Flag, PhotoId};
use serde_json::json;
use crate::Session;
use crate::rename::{MoveFs, RealFs};
/// Whether `dir` is on a case-insensitive volume (the default on macOS and Windows): its path in upper case still resolves.
pub(crate) fn case_insensitive(dir: &Path) -> bool {
Path::new(&dir.to_string_lossy().to_uppercase()).exists()
}
/// The same path, or the same file under another spelling (`a.XMP` and `a.xmp` on a case-insensitive volume). Fault
/// injectors match this way, so an injected failure fires whichever spelling the code under test uses for a file; on a
/// case-sensitive volume only identical paths match.
pub(crate) fn same_path_or_file(a: &Path, b: &Path) -> bool {
a == b || RealFs.same_file(a, b)
}
/// The same path, or on a case-insensitive volume a path that differs only in ASCII case: the code under test may spell
/// a sidecar `.XMP` for a file stored as `.xmp` there, while on a case-sensitive volume it offers only the spelling that
/// exists.
pub(crate) fn same_path_for_volume(a: &Path, b: &Path) -> bool {
a == b || (a.parent().is_some_and(case_insensitive) && a.to_string_lossy().eq_ignore_ascii_case(&b.to_string_lossy()))
}
pub(crate) fn temp_dir(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-xmp-{tag}-{}", std::process::id()));
@@ -175,6 +195,71 @@ fn foreign_crs_sidecar_on_import_and_read_from_file() {
let _ = std::fs::remove_dir_all(&src);
}
/// A sidecar with only `crs:` develop settings, as LRTimelapse writes them (issue #934).
fn crs_exposure_sidecar(exposure: &str) -> String {
format!(
r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description rdf:about="" xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/" crs:Version="15.0" crs:Exposure2012="{exposure}"/></rdf:RDF></x:xmpmeta>"#
)
}
/// Issue #934: with auto-write on, Read Metadata from File wrote what it had just read straight back
/// into the sidecar, adding our `lc:settings`. Reading leaves the file byte for byte; an edit made
/// after it is still written.
#[test]
fn reading_metadata_leaves_the_sidecar_alone_with_auto_write_on() {
let src = temp_dir("read-no-autowrite");
write_png(&src.join("frame.png"), 6);
let sidecar = src.join("frame.xmp");
std::fs::write(&sidecar, crs_exposure_sidecar("0.5")).unwrap();
let mut s = Session::new().with_fs();
s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
let id = only(&s);
assert_eq!(s.catalog.photo(id).unwrap().develop.light.exposure, 0.5);
// something for the read to change back (auto-write is still off, so the file stays as it is)
s.execute("develop.reset", &json!({"ids": [id.0]})).unwrap();
s.execute("library.xmpPreferences", &json!({"autoWrite": true})).unwrap();
let before = std::fs::read(&sidecar).unwrap();
let r = s.execute("photo.readMetadataFromFile", &json!({"ids": [id.0]})).unwrap();
assert_eq!(r["read"].as_array().unwrap().len(), 1, "{r}");
assert_eq!(s.catalog.photo(id).unwrap().develop.light.exposure, 0.5, "the sidecar was read");
let now = std::fs::read(&sidecar).unwrap();
assert!(now == before, "reading leaves the sidecar byte for byte, but it is now:\n{}", String::from_utf8_lossy(&now));
// a develop edit afterwards is still written (only the read is exempt)
s.execute("develop.set", &json!({"ids": [id.0], "control": "light.exposure", "value": 0.9})).unwrap();
let after = std::fs::read_to_string(&sidecar).unwrap();
match crate::sidecar::parse_sidecar(&after, crate::crs::Target::Rendered).unwrap().develop {
Some(crate::sidecar::DevelopPatch::Full(d)) => assert_eq!(d.light.exposure, 0.9),
other => panic!("the edit wasn't auto-written: {other:?}\n{after}"),
}
assert!(after.contains(r#"crs:Exposure2012="0.5""#), "the other app's settings are kept: {after}");
let _ = std::fs::remove_dir_all(&src);
}
/// Issue #934, the round trip the write-back broke: after a read with auto-write on, another app
/// changes `crs:Exposure2012` from 0.5 to 1.2 (keeping the rest of the file), and the next read
/// takes 1.2.
#[test]
fn a_crs_change_after_a_read_is_taken_with_auto_write_on() {
let src = temp_dir("read-then-crs-change");
write_png(&src.join("frame.png"), 7);
let sidecar = src.join("frame.xmp");
std::fs::write(&sidecar, crs_exposure_sidecar("0.5")).unwrap();
let mut s = Session::new().with_fs();
s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
let id = only(&s);
s.execute("library.xmpPreferences", &json!({"autoWrite": true})).unwrap();
s.execute("photo.readMetadataFromFile", &json!({"ids": [id.0]})).unwrap();
assert_eq!(s.catalog.photo(id).unwrap().develop.light.exposure, 0.5);
// the other app edits the file as it is now
let text = std::fs::read_to_string(&sidecar).unwrap();
let changed = text.replace(r#"crs:Exposure2012="0.5""#, r#"crs:Exposure2012="1.2""#);
assert_ne!(changed, text);
std::fs::write(&sidecar, changed).unwrap();
s.execute("photo.readMetadataFromFile", &json!({"ids": [id.0]})).unwrap();
assert_eq!(s.catalog.photo(id).unwrap().develop.light.exposure, 1.2, "the second read takes the other app's exposure");
let _ = std::fs::remove_dir_all(&src);
}
/// Lightroom's default profiles (`Adobe Standard`, `Adobe Color`) are named by nearly every sidecar
/// and our own starting look stands in for them: importing one gives no camera-profile warning.
#[test]
+28 -1
View File
@@ -57,6 +57,8 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
if let Ok(c) = std::fs::canonicalize(root) {
linked.insert(c);
}
// Include aliases to the library in the exclusion, not just its lexical path.
let skip_canonical = skip.and_then(|p| std::fs::canonicalize(p).ok());
// (path, depth of the folder it is in, is a folder); popped in path order
let mut stack: Vec<(PathBuf, usize, bool)> = vec![(root.to_path_buf(), 0, true)];
while let Some((path, depth, is_dir)) = stack.pop() {
@@ -66,7 +68,7 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
}
continue;
}
if skip.is_some_and(|s| path == s) {
if skip.is_some_and(|s| path == s) || skip_canonical.as_ref().is_some_and(|s| std::fs::canonicalize(&path).is_ok_and(|p| p.starts_with(s))) {
continue;
}
if depth >= limits.max_depth {
@@ -165,6 +167,31 @@ mod tests {
let _ = std::fs::remove_dir_all(&d);
}
/// Folder imports must not revisit the user's own library through symlinks
/// or through an alias to the library's parent.
#[cfg(unix)]
#[test]
fn skipped_library_stays_skipped_through_symlinks_and_ancestors() {
let d = tmp("skip-alias");
let root = d.join("incoming");
let library = d.join("library");
std::fs::create_dir_all(&root).unwrap();
std::fs::create_dir_all(library.join("Originals")).unwrap();
std::fs::write(root.join("ordinary.jpg"), b"photo").unwrap();
std::fs::write(library.join("Originals").join("library-only.jpg"), b"original").unwrap();
std::os::unix::fs::symlink(&library, root.join("library-link")).unwrap();
std::os::unix::fs::symlink(&d, root.join("parent-link")).unwrap();
let photos = files_in(&root, Some(&library), Limits::default(), |p| p.extension().is_some_and(|e| e == "jpg"));
assert!(photos.files.contains(&root.join("ordinary.jpg")));
assert!(!photos.files.iter().any(|p| p.ends_with("library-only.jpg")), "{photos:?}");
// Scanning a subfolder already inside the excluded library is also forbidden.
let inside = files_in(&library.join("Originals"), Some(&library), Limits::default(), |_| true);
assert!(inside.files.is_empty(), "{inside:?}");
let _ = std::fs::remove_dir_all(&d);
}
/// A link back up the tree (a loop) is followed at most once: the walk ends.
#[cfg(unix)]
#[test]