From d5a6d6c509df1d9c4d6ce1ce936e95554c08306d Mon Sep 17 00:00:00 2001 From: Alex Chambers Date: Sat, 10 Oct 2026 10:58:38 +0200 Subject: [PATCH] LR-PROF-CAMERACOLOR: fit the tone curve of low-contrast scenes (range limit 1.5 to 1.2) Both tone fits refused a curve when the brightest 1/32 of the training pairs was less than 1.5x the darkest 1/32, a hard cliff before any acceptance gate: a hazy ILCE-7RM4 burst (issue #631) opened with the neutral fallback on frames just under it (DSC01630: range 1.489, dE76 vs the camera JPEG 23.8) and with a camera look on the frame just over it. MIN_TONE_RANGE = 1.2 still refuses nearly uniform pairs. A curve under the old 1.5 must also keep the bright areas (camera JPEG luminance 0.85 and up), which the fit and the held-out gates leave out, no further from the camera JPEG than the neutral fallback (highlights_hold, full and partial looks): fitted to a narrow range it extrapolates them, and reviews found synthetic frames whose accepted curve rendered a 0.93 sky at 0.28 to 0.61. The bright pixels come from the proxies before any training filter, and for DRO photos the curve is checked lowered as it will be used, inside the search. The tone curve of a black-and-white camera JPEG (#577) comes from the same fit, so it gets the same check, by luminance as its own gates compare: a synthetic frame's 0.93 sky rendered at 0.28 without it. DSC01630 (DRO Auto): look accepted, dE76 1.94. Every other file of a 309-file sweep (135 public Sony) renders pixel-identical, also with the camera JPEGs made grey. RENDER_CACHE_VERSION 63, LOOK_VERSION 6 (smart previews refit). Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/camera_preview.rs | 408 +++++++++++++++++++++++++--- crates/engine/src/media.rs | 2 +- docs/camera-preview-colour.md | 3 +- 3 files changed, 368 insertions(+), 45 deletions(-) diff --git a/crates/engine/src/camera_preview.rs b/crates/engine/src/camera_preview.rs index 614c58c60..3eeba8c8e 100644 --- a/crates/engine/src/camera_preview.rs +++ b/crates/engine/src/camera_preview.rs @@ -34,7 +34,8 @@ use lightcraft_raw::{ /// 7: the sensor proxy follows the file's lens warp when the camera JPEG is lens-corrected. /// 8: the bundled NIKON Z 6_2 profile. /// 9: the bundled NIKON Z 8 and NIKON D5300 profiles. -pub const LOOK_VERSION: u32 = 9; +/// 12: tone curves of low-contrast scenes (range limit 1.2, with the bright-area check for curves under 1.5, #699). +pub const LOOK_VERSION: u32 = 12; #[derive(Clone, Debug)] pub(crate) struct CameraLook { @@ -270,20 +271,21 @@ pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransf // try its colour first and fit tone/chroma per photo (picture styles vary). let profile = raw.metadata.model.as_deref().and_then(crate::camera_profiles::get); let colour = profile.as_ref().and_then(|p| Some((p.matrix().mul(&transform.matrix.inverse()?), p.hue_sat.clone()))); - let look = match fit_look(&sensor, &reference, &clipped, colour.clone()) { + // Sony's Dynamic Range Optimizer (on by default) brightens the camera JPEG's darker regions, + // not the raw: keep the colour fitted to the JPEG, take the tone curve without DRO. + let dro = lightcraft_raw::embedded_preview_dynamic_range_optimized(bytes) == Some(true); + let look = match fit_look_with(&sensor, &reference, &clipped, colour.clone(), dro) { Some(look) => look, // A black-and-white JPEG has no colour to fit: the profile's colour and the JPEG's tone curve // (`monochrome_look` does it without a profile). None if colour.is_some() && black_and_white => { let unclipped = without_clipped(&sensor, &clipped); - let Some(look) = fit_tone_only(unclipped.as_ref().unwrap_or(&sensor), &reference, colour) else { return (None, true) }; + let bright = HighlightCheck { pairs: highlights(&sensor, &reference), dro, grey: true }; + let Some(look) = fit_tone_only_with(unclipped.as_ref().unwrap_or(&sensor), &reference, colour, &bright) else { return (None, true) }; look } None => return (None, black_and_white), }; - // Sony's Dynamic Range Optimizer (on by default) brightens the camera JPEG's darker regions, - // not the raw: keep the colour fitted to the JPEG, take the tone curve without DRO. - let dro = lightcraft_raw::embedded_preview_dynamic_range_optimized(bytes) == Some(true); let look = if dro { with_dro_off_tone(look) } else { look }; if lightcraft_pipeline::profiling() { eprintln!( @@ -302,6 +304,12 @@ pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransf /// the render's highlight reconstruction. const SENSOR_CLIP: f32 = 0.99; +/// [`fit_look_with`] for a photo without Sony's DRO. +#[cfg(test)] +fn fit_look(sensor: &Rgb32f, reference: &Rgb32f, clipped: &[bool], colour: Option<(Mat3, Option)>) -> Option { + fit_look_with(sensor, reference, clipped, colour, false) +} + /// The photo's look, refitted without the proxy pixels whose sensor values are clipped. /// /// A clipped raw pixel no longer records the scene's colour: with one channel held at the clip @@ -320,17 +328,19 @@ const SENSOR_CLIP: f32 = 0.99; /// replace a look fitted away from edges). A photo that had no look gets one when the search /// without the clipped pixels finds it, and otherwise the [`fit_partial`] look when that is clearly /// closer to the camera JPEG than the neutral fallback. -fn fit_look(sensor: &Rgb32f, reference: &Rgb32f, clipped: &[bool], colour: Option<(Mat3, Option)>) -> Option { +fn fit_look_with(sensor: &Rgb32f, reference: &Rgb32f, clipped: &[bool], colour: Option<(Mat3, Option)>, dro: bool) -> Option { let unclipped = without_clipped(sensor, clipped); - let Some((look, attempt)) = search_ordered(sensor, reference, colour.clone(), TONE_FITS) else { + // from the proxies as they are: the retries' filters (clipped pixels, edges) don't remove them + let bright = HighlightCheck { pairs: highlights(sensor, reference), dro, grey: false }; + let Some((look, attempt)) = search_ordered(sensor, reference, colour.clone(), TONE_FITS, &bright) else { return unclipped .as_ref() - .and_then(|u| search_ordered(u, reference, colour, TONE_FITS)) + .and_then(|u| search_ordered(u, reference, colour, TONE_FITS, &bright)) .map(|(look, _)| look) - .or_else(|| fit_partial(unclipped.as_ref().unwrap_or(sensor), reference)); + .or_else(|| fit_partial(unclipped.as_ref().unwrap_or(sensor), reference, &bright)); }; let Some(unclipped) = unclipped else { return Some(look) }; - let refit = fit_attempt(&unclipped, reference, &colour, attempt); + let refit = fit_attempt(&unclipped, reference, &colour, attempt, &bright); if refit.is_none() && lightcraft_pipeline::profiling() { eprintln!("[profile] camera look: no fit without the clipped pixels on the accepted attempt, keeping the fit on all pixels"); } @@ -389,8 +399,14 @@ fn sony_dro_off_tone() -> Option { /// curve is brighter than the photo's own (the 1″ compacts map raw values darker), the photo keeps /// its own. Its colour (matrix, hue/saturation table and chroma curve) stays as fitted. fn with_dro_off_tone(look: CameraLook) -> CameraLook { - let Some(dro_off) = sony_dro_off_tone() else { return look }; - let own = look.tone.knots(); + CameraLook { tone: dro_off(&look.tone), ..look } +} + +/// The curve of [`with_dro_off_tone`]: `own` lowered to Sony's without DRO wherever that is darker, +/// or `own` unchanged when it can't be. +fn dro_off(own_tone: &CameraTone) -> CameraTone { + let Some(dro_off) = sony_dro_off_tone() else { return *own_tone }; + let own = own_tone.knots(); let mut knots = own.map(|[x, y]| [x, y.min(dro_off.apply(x))]); // Past its last knot a curve goes on as a shoulder whose rate comes from its last two knots // (`CameraTone::apply`): lowering the second-last knot more than the last steepens it, and the @@ -400,11 +416,11 @@ fn with_dro_off_tone(look: CameraLook) -> CameraLook { let rate = ((b[1] - a[1]) / (b[0] - a[0])).clamp(0.1, 16.0) / (1.0 - b[1]).max(0.01); let floor = last[1] - rate * (1.0 - last[1]).max(0.01) * (b[0] - a[0]); knots[30][1] = knots[30][1].max(floor).min(last[1]); - let Some(tone) = CameraTone::new(knots).and_then(|tone| tone.with_chroma(*look.tone.chroma())) else { return look }; - if !at_most(&tone, &look.tone) { - return look; + let Some(tone) = CameraTone::new(knots).and_then(|tone| tone.with_chroma(*own_tone.chroma())) else { return *own_tone }; + if !at_most(&tone, own_tone) { + return *own_tone; } - CameraLook { tone, ..look } + tone } /// Whether curve `a` is nowhere brighter than `b` over the scene luminances the finish stage's tone @@ -452,10 +468,11 @@ fn fit_dng_tone(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, lens: let unclipped = without_clipped(&sensor, &clipped); let fit_on = unclipped.as_ref().unwrap_or(&sensor); let colour = Some((Mat3::IDENTITY, None)); + let bright = HighlightCheck { pairs: highlights(&sensor, &reference), dro: false, grey: false }; let look = TONE_FITS.iter().find_map(|&tone| { - [false, true] - .into_iter() - .find_map(|away_from_edges| fit_attempt(fit_on, &reference, &colour, Attempt { tone, profile: true, away_from_edges, dark: true })) + [false, true].into_iter().find_map(|away_from_edges| { + fit_attempt(fit_on, &reference, &colour, Attempt { tone, profile: true, away_from_edges, dark: true }, &bright) + }) }); let Some(look) = look else { return refuse("fit") }; let plain = match raw.color.profile.tone_curve.as_ref().and_then(crate::files::dng_tone_curve) { @@ -984,7 +1001,8 @@ fn local_contrast(image: &Rgb32f, i: usize) -> f32 { /// `edge_limit`, pixels whose 3×3 neighbourhood in either proxy spans a larger luminance ratio /// are left out: there a small geometric mismatch between the raw and its JPEG pairs unrelated /// colours (see [`EDGE_CONTRAST`]). -type Pairs = Vec<([f64; 3], [f64; 3])>; +type Pair = ([f64; 3], [f64; 3]); +type Pairs = Vec; fn collect_pairs(sensor: &Rgb32f, reference: &Rgb32f, min_chroma: f32, edge_limit: Option, dark: bool) -> Option<(Pairs, Pairs)> { if (sensor.width, sensor.height) != (reference.width, reference.height) || sensor.data.len() != reference.data.len() { return None; @@ -1146,7 +1164,8 @@ fn fit_pairs_with(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Opt /// The look from [`search_ordered`] without its attempt. #[cfg(test)] fn fit_pairs_ordered(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option)>, order: &[ToneFit]) -> Option { - search_ordered(sensor, reference, colour, order).map(|(look, _)| look) + search_ordered(sensor, reference, colour, order, &HighlightCheck { pairs: highlights(sensor, reference), dro: false, grey: false }) + .map(|(look, _)| look) } /// The tone fits the search tries, in order: the first whose look passes the acceptance gates is @@ -1176,21 +1195,33 @@ struct Attempt { /// valid when the camera JPEG's geometry differs slightly from the raw's (issue #232). The whole /// search runs with one tone fit before the next is tried, so the conditional-median fallback /// accepts exactly what the previous releases accepted. -fn search_ordered(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option)>, order: &[ToneFit]) -> Option<(CameraLook, Attempt)> { +fn search_ordered( + sensor: &Rgb32f, + reference: &Rgb32f, + colour: Option<(Mat3, Option)>, + order: &[ToneFit], + bright: &HighlightCheck, +) -> Option<(CameraLook, Attempt)> { let profiles: &[bool] = if colour.is_some() { &[true, false] } else { &[false] }; order.iter().find_map(|&tone| { profiles.iter().find_map(|&profile| { [false, true].into_iter().find_map(|away_from_edges| { let attempt = Attempt { tone, profile, away_from_edges, dark: false }; - fit_attempt(sensor, reference, &colour, attempt).map(|look| (look, attempt)) + fit_attempt(sensor, reference, &colour, attempt, bright).map(|look| (look, attempt)) }) }) }) } -fn fit_attempt(sensor: &Rgb32f, reference: &Rgb32f, colour: &Option<(Mat3, Option)>, attempt: Attempt) -> Option { +fn fit_attempt( + sensor: &Rgb32f, + reference: &Rgb32f, + colour: &Option<(Mat3, Option)>, + attempt: Attempt, + bright: &HighlightCheck, +) -> Option { let colour = if attempt.profile { colour.clone() } else { None }; - fit_pairs_on(sensor, reference, colour, attempt.away_from_edges.then_some(EDGE_CONTRAST), attempt.tone, attempt.dark) + fit_pairs_on(sensor, reference, colour, attempt.away_from_edges.then_some(EDGE_CONTRAST), attempt.tone, attempt.dark, bright) } fn fit_pairs_on( @@ -1200,6 +1231,7 @@ fn fit_pairs_on( edge_limit: Option, tone_fit: ToneFit, dark: bool, + highlights: &HighlightCheck, ) -> Option { // A known colour model needs enough signal for tone fitting, not a scene rich enough to // learn a new colour matrix. Still reject monochrome references. @@ -1227,6 +1259,9 @@ fn fit_pairs_on( look.tone = tone; } let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0); + if !highlights_hold(&look.tone, highlights, colour) { + continue; + } let (mut linear, mut perceptual, mut samples) = (0.0, 0.0, 0); for (x, target) in pairs.iter().step_by(3) { let corrected = displayed(colour(*x), &tone); @@ -1341,11 +1376,11 @@ fn rank_correlation_ties(pairs: &[([f64; 3], [f64; 3])]) -> f64 { /// and in gamma-encoded display values, and when the JPEG shows the same picture /// ([`PARTIAL_MIN_RANK_CORRELATION`]). There is no limit on its own error: it only has to be clearly /// closer to the camera JPEG than what the photo would get otherwise. -fn fit_partial(sensor: &Rgb32f, reference: &Rgb32f) -> Option { - [None, Some(EDGE_CONTRAST)].into_iter().find_map(|edge_limit| fit_partial_on(sensor, reference, edge_limit)) +fn fit_partial(sensor: &Rgb32f, reference: &Rgb32f, highlights: &HighlightCheck) -> Option { + [None, Some(EDGE_CONTRAST)].into_iter().find_map(|edge_limit| fit_partial_on(sensor, reference, edge_limit, highlights)) } -fn fit_partial_on(sensor: &Rgb32f, reference: &Rgb32f, edge_limit: Option) -> Option { +fn fit_partial_on(sensor: &Rgb32f, reference: &Rgb32f, edge_limit: Option, highlights: &HighlightCheck) -> Option { // as for a known colour model: enough signal for the tone, but no monochrome reference let (pairs, bright) = collect_pairs(sensor, reference, 0.005, edge_limit, false)?; let related = rank_correlation(&pairs); @@ -1369,6 +1404,9 @@ fn fit_partial_on(sensor: &Rgb32f, reference: &Rgb32f, edge_limit: Option) look.tone = tone; } let (tone, neutral) = (ToneMap::camera(&look.tone, 0.0, 0.0, 0.0), ToneMap::new(0.0, 0.0, 0.0)); + if !highlights_hold(&look.tone, highlights, |x| matrix.apply(x)) { + return None; + } let encoded = |v: f64| v.max(0.0).powf(1.0 / 2.2); let (mut before, mut after, mut before_encoded, mut after_encoded, mut samples) = (0.0, 0.0, 0.0, 0.0, 0); for (x, target) in pairs.iter().step_by(3) { @@ -1419,7 +1457,7 @@ fn monochrome(reference: &Rgb32f) -> bool { /// would start from the spectral matrices or the neutral fallback with the default tone curve, about 1.3 EV darker /// and flatter than the same body's photos shot in colour. But the JPEG still shows the camera's tone curve. /// `transform` is the colour the photo gets otherwise (the spectral matrices or the fallback's); the look keeps it and -/// adds the tone curve alone ([`fit_tone_only`]), as [`fit_look`] tries the photo's own colour after a profile's. The +/// adds the tone curve alone ([`fit_tone_only_with`]), as [`fit_look_with`] tries the photo's own colour after a profile's. The /// chroma curve stays neutral: the JPEG has no colourfulness to learn. fn monochrome_look(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, lens: Option<&LensGeometry>) -> Option { if !file_local_look(raw.format) { @@ -1429,10 +1467,11 @@ fn monochrome_look(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, le if !monochrome(&reference) { return None; } - let unclipped = without_clipped(&sensor, &clipped); - let look = fit_tone_only(unclipped.as_ref().unwrap_or(&sensor), &reference, None)?; // Sony's Dynamic Range Optimizer brightens a B/W JPEG as much as a colour one let dro = lightcraft_raw::embedded_preview_dynamic_range_optimized(bytes) == Some(true); + let unclipped = without_clipped(&sensor, &clipped); + let bright = HighlightCheck { pairs: highlights(&sensor, &reference), dro, grey: true }; + let look = fit_tone_only_with(unclipped.as_ref().unwrap_or(&sensor), &reference, None, &bright)?; let look = if dro { with_dro_off_tone(look) } else { look }; if lightcraft_pipeline::profiling() { eprintln!( @@ -1449,12 +1488,30 @@ fn monochrome_look(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, le /// space) or, without one, of the sensor proxy's own colour. Fitted like [`fit_partial`]'s, by quantile matching of /// the luminances, on all pixels and then away from edges, and used only when the JPEG shows the same picture /// ([`PARTIAL_MIN_RANK_CORRELATION`]) and the look cuts the held-out squared luminance error to below -/// [`MIN_IMPROVEMENT`] of the default curve's, in linear and in gamma-encoded display values. -fn fit_tone_only(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option)>) -> Option { - [None, Some(EDGE_CONTRAST)].into_iter().find_map(|edge_limit| fit_tone_only_on(sensor, reference, &colour, edge_limit)) +/// [`MIN_IMPROVEMENT`] of the default curve's, in linear and in gamma-encoded display values. A curve narrower than +/// [`NARROW_TONE_RANGE`] must also keep the JPEG's bright areas ([`highlights_hold`], by luminance). +fn fit_tone_only_with( + sensor: &Rgb32f, + reference: &Rgb32f, + colour: Option<(Mat3, Option)>, + highlights: &HighlightCheck, +) -> Option { + [None, Some(EDGE_CONTRAST)].into_iter().find_map(|edge_limit| fit_tone_only_on(sensor, reference, &colour, edge_limit, highlights)) } -fn fit_tone_only_on(sensor: &Rgb32f, reference: &Rgb32f, colour: &Option<(Mat3, Option)>, edge_limit: Option) -> Option { +/// [`fit_tone_only_with`] for a photo without Sony's DRO, checked against its own bright areas. +#[cfg(test)] +fn fit_tone_only(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option)>) -> Option { + fit_tone_only_with(sensor, reference, colour, &HighlightCheck { pairs: highlights(sensor, reference), dro: false, grey: true }) +} + +fn fit_tone_only_on( + sensor: &Rgb32f, + reference: &Rgb32f, + colour: &Option<(Mat3, Option)>, + edge_limit: Option, + highlights: &HighlightCheck, +) -> Option { // every pair counts: the JPEG has no colour to require let (pairs, _) = collect_pairs(sensor, reference, f32::NEG_INFINITY, edge_limit, false)?; let related = rank_correlation_ties(&pairs); @@ -1472,6 +1529,9 @@ fn fit_tone_only_on(sensor: &Rgb32f, reference: &Rgb32f, colour: &Option<(Mat3, }; let tone = fit_tone_with_shadows(ToneFit::Quantile, &pairs, sensor, reference, edge_limit, coloured)?; let look = CameraLook { matrix, tone, hue_sat: hue_sat.clone() }; + if !highlights_hold(&look.tone, highlights, coloured) { + return None; + } let (tone, neutral) = (ToneMap::camera(&look.tone, 0.0, 0.0, 0.0), ToneMap::new(0.0, 0.0, 0.0)); let encoded = |v: f64| v.max(0.0).powf(1.0 / 2.2); let (mut before, mut after, mut before_encoded, mut after_encoded, mut samples) = (0.0, 0.0, 0.0, 0.0, 0); @@ -1779,6 +1839,89 @@ impl ToneFit { } } +/// The bright areas a narrow tone curve is checked against ([`highlights_hold`]): every proxy pixel +/// whose camera JPEG luminance is 0.85 or more, with its sensor value as developed (for a clipped +/// pixel a lower bound of the scene). Taken from the proxies before any training filter (clipped +/// pixels, edges, the sensor value range of [`collect_pairs`]), so no retry of the search can +/// leave them out. +fn highlights(sensor: &Rgb32f, reference: &Rgb32f) -> Pairs { + if sensor.data.len() != reference.data.len() { + return Vec::new(); + } + sensor + .data + .iter() + .zip(&reference.data) + .filter(|(x, y)| x.iter().chain(y.iter()).all(|v| v.is_finite() && *v >= 0.0) && luminance_2020(**y) >= 0.85) + .map(|(x, y)| (x.map(f64::from), y.map(f64::from))) + .collect() +} + +/// Smallest ratio between the scene luminance of the brightest and of the darkest 1/32 of the +/// pairs a tone curve is fitted to (its last and first knot). Below it the training pairs are +/// nearly uniform (under 0.26 EV from their darker to their brighter parts): the curve would be +/// fitted to a sliver of the tone range and the rest of it extrapolated. Above it the acceptance +/// gates judge the curve. This was 1.5, which rejected low-contrast scenes the curve fits very +/// well: in a hazy ILCE-7RM4 burst the frames at 1.46–1.49 opened with the neutral fallback (no full or partial +/// look, 18 L* darker than the camera JPEG) while the frame at 1.51 fitted at a held-out RMS of +/// 0.013, as the others do once allowed (issue #631). Across 135 public Sony files the lowest is 4.0. +/// A curve under [`NARROW_TONE_RANGE`] must also pass [`highlights_hold`]. +const MIN_TONE_RANGE: f32 = 1.2; + +/// [`MIN_TONE_RANGE`] before issue #631. A curve fitted to a narrower range than this must also +/// render the photo's bright areas no further from the camera JPEG than the neutral fallback does +/// ([`highlights_hold`]). Neither the fit nor the held-out gates see those areas (camera JPEG +/// luminance 0.85 and up, left out of the training pairs by [`collect_pairs`]), and a curve fitted +/// to a sliver of the tone range extrapolates them: in review, a synthetic frame whose curve passed +/// the gates at a held-out RMS of 0.0004 rendered its bright sky at 0.29 where the camera JPEG has +/// 0.93. +const NARROW_TONE_RANGE: f32 = 1.5; + +/// What a narrow tone curve is checked against ([`highlights_hold`]). +struct HighlightCheck { + /// The frame's [`highlights`]. + pairs: Pairs, + /// The photo's curve will be lowered to Sony's without DRO ([`with_dro_off_tone`]) after the + /// fit, so the check looks at it lowered. + dro: bool, + /// The camera JPEG is black and white ([`fit_tone_only_with`]): the check compares luminance, as that + /// fit's gates do, since the JPEG has no colour to match. + grey: bool, +} + +/// Whether a look with this `tone` and `colour` (sensor value to the finish stage's input) keeps +/// the frame's [`highlights`] (camera JPEG luminance 0.85 and up, left out of the fit) no further +/// from the camera JPEG than the neutral fallback does, with the tone curve as the photo will get +/// it (lowered for DRO), in luminance alone for a black-and-white JPEG. Only asked of curves narrower than +/// [`NARROW_TONE_RANGE`]: true for wider ones, and for frames without such pixels. +fn highlights_hold(tone: &CameraTone, check: &HighlightCheck, colour: impl Fn([f64; 3]) -> [f64; 3]) -> bool { + let knots = tone.knots(); + if knots[31][0] >= knots[0][0] * NARROW_TONE_RANGE { + return true; + } + // the curve as the photo will get it + let used = if check.dro { dro_off(tone) } else { *tone }; + let (map, neutral) = (ToneMap::camera(&used, 0.0, 0.0, 0.0), ToneMap::new(0.0, 0.0, 0.0)); + let (mut look, mut fallback) = (0.0, 0.0); + for (x, target) in &check.pairs { + let (rendered, base) = (displayed(colour(*x), &map), displayed(*x, &neutral)); + if check.grey { + look += (luma(rendered) - luma(*target)).powi(2); + fallback += (luma(base) - luma(*target)).powi(2); + continue; + } + for c in 0..3 { + look += (rendered[c] - target[c]).powi(2); + fallback += (base[c] - target[c]).powi(2); + } + } + let hold = look.is_finite() && look <= fallback; + if !hold && lightcraft_pipeline::profiling() { + eprintln!("[profile] camera look: narrow tone curve renders the bright areas worse than the fallback ({look:.3} > {fallback:.3})"); + } + hold +} + /// The camera's tone curve from scene/JPEG luminance pairs, by quantile matching: knot `i` pairs /// the median of the `i`-th 1/32 of the sorted scene luminances with the median of the `i`-th /// 1/32 of the sorted JPEG luminances. A tone curve is monotone, so it maps each quantile of the @@ -1806,7 +1949,7 @@ fn fit_tone(pairs: Vec<(f64, f64)>) -> Option { let mid = a + (b - a) / 2; *knot = [*xs.get(mid)? as f32, *ys.get(mid)? as f32]; } - if knots[31][0] < knots[0][0] * 1.5 { + if knots[31][0] < knots[0][0] * MIN_TONE_RANGE { return None; } CameraTone::new(knots) @@ -1834,7 +1977,7 @@ fn fit_tone_conditional(mut pairs: Vec<(f64, f64)>) -> Option { let mut ys: Vec<_> = bin.iter().map(|p| p.1).collect(); *knot = [median(&mut xs)? as f32, median(&mut ys)? as f32]; } - if knots[31][0] < knots[0][0] * 1.5 { + if knots[31][0] < knots[0][0] * MIN_TONE_RANGE { return None; } // Pool adjacent violating bins (isotonic regression): no reversals or arbitrary polynomial. @@ -3503,7 +3646,8 @@ mod tests { let none = fit_look(&sensor, &reference, &vec![false; sensor.data.len()], None).unwrap(); assert_eq!((none.matrix.0, none.tone), (all.matrix.0, all.tone)); // without clipped pixels the refit keeps the accepted attempt (here: all pixels, own colour, quantile tone) - let (_, attempt) = search_ordered(&sensor, &reference, None, TONE_FITS).unwrap(); + let check = HighlightCheck { pairs: highlights(&sensor, &reference), dro: false, grey: false }; + let (_, attempt) = search_ordered(&sensor, &reference, None, TONE_FITS, &check).unwrap(); assert_eq!(attempt, Attempt { tone: ToneFit::Quantile, profile: false, away_from_edges: false, dark: false }); assert!(without_clipped(&sensor, &[true; 3]).is_none(), "a mask of another size is ignored"); } @@ -3635,7 +3779,8 @@ mod tests { let neutral = ToneMap::new(0.0, 0.0, 0.0); let mut reference = sensor.clone(); reference.map_in_place(|p| displayed(p.map(f64::from), &neutral).map(|v| v as f32)); - assert!(fit_partial(&sensor, &reference).is_none(), "no clear gain over the fallback"); + let check = HighlightCheck { pairs: highlights(&sensor, &reference), dro: false, grey: false }; + assert!(fit_partial(&sensor, &reference, &check).is_none(), "no clear gain over the fallback"); // an unrelated picture for (i, p) in reference.data.iter_mut().enumerate() { *p = [0.05 + (i % 7) as f32 * 0.07, 0.05 + (i % 19) as f32 * 0.02, 0.05 + (i % 29) as f32 * 0.01]; @@ -3894,8 +4039,10 @@ mod tests { } } let colour = Some((Mat3::IDENTITY, None)); - let look = fit_attempt(&sensor, &reference, &colour, Attempt { tone: ToneFit::Quantile, profile: true, away_from_edges: false, dark: true }) - .expect("tone fitted"); + let check = HighlightCheck { pairs: highlights(&sensor, &reference), dro: false, grey: false }; + let look = + fit_attempt(&sensor, &reference, &colour, Attempt { tone: ToneFit::Quantile, profile: true, away_from_edges: false, dark: true }, &check) + .expect("tone fitted"); assert_eq!(look.matrix, Mat3::IDENTITY); assert!(look.hue_sat.is_none()); let (fitted, default) = (ToneMap::camera(&look.tone, 0.0, 0.0, 0.0), ToneMap::new(0.0, 0.0, 0.0)); @@ -3983,4 +4130,181 @@ mod tests { looked.profile.look_table = HsvTable::from_tags(&[1, 2, 1], &[0.0, 1.0, 1.0, 0.0, 1.0, 1.0], 0); assert!(looked.profile.look_table.is_some() && !eligible(RawFormat::Dng, looked), "a look table is the file's own rendering"); } + + /// Pairs whose scene luminance spans `ratio` from the darkest to the brightest pixel, mapped + /// to the JPEG through a steep camera curve (a hazy scene the camera stretches). + fn low_contrast_pairs(ratio: f64) -> Vec<(f64, f64)> { + (0..4096).map(|i| 0.08 * ratio.powf(i as f64 / 4095.0)).map(|x| (x, (0.2 + (x - 0.08) * 3.0).min(0.95))).collect() + } + + /// The ratio the tone fits test: scene luminance at the middle of the brightest 1/32 over that + /// of the darkest 1/32 (their last and first knots). + fn knot_range(mut scene: Vec) -> f64 { + scene.sort_by(f64::total_cmp); + let n = scene.len(); + scene[31 * n / 32 + n / 64] / scene[n / 64] + } + + /// Issue #631: a low-contrast scene (its darker and brighter parts under 1.5× apart) still gets + /// a tone curve; only a nearly uniform one doesn't. Both tone fits use the same limit. + #[test] + fn tone_fits_accept_low_contrast_scenes() { + for fit in [fit_tone, fit_tone_conditional] { + let pairs = low_contrast_pairs(1.47); + let range = knot_range(pairs.iter().map(|p| p.0).collect()); + assert!((1.4..1.5).contains(&range), "the scene sits just below the old limit of 1.5: {range}"); + let curve = fit(pairs).expect("a low-contrast scene, as in the burst of issue #631"); + assert!((curve.apply(0.1) - 0.26).abs() < 0.01, "follows the camera curve: {}", curve.apply(0.1)); + assert!(fit(low_contrast_pairs(1.1)).is_none(), "a nearly uniform frame"); + } + } + + /// The same scene end to end: colour varies, luminance spans under 1.5×, and the camera + /// stretches it; the look is accepted and lands on the JPEG instead of the neutral fallback. + #[test] + fn a_low_contrast_scene_gets_a_camera_look() { + let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]); + let mut sensor = Rgb32f::new(96, 64); + let mut reference = sensor.clone(); + for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() { + let ev = 0.06 + (i % 31) as f32 * 0.0008; + *src = [ev * (0.8 + (i % 11) as f32 * 0.025), ev, ev * (0.8 + (i % 17) as f32 * 0.014)]; + let p = known.apply_f32(*src); + let y = luminance_2020(p); + *dst = p.map(|v| v * (0.2 + (y - 0.07) * 3.0) / y); + } + let look = fit_pairs(&sensor, &reference).expect("a camera look, not the neutral fallback"); + let range = knot_range(sensor.data.iter().map(|x| luma(look.matrix.apply(x.map(f64::from)))).collect()); + assert!(range < 1.5, "the scene sits below the old limit of 1.5: {range}"); + let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0); + let error: f64 = sensor + .data + .iter() + .zip(&reference.data) + .map(|(x, y)| { + let p = displayed(look.matrix.apply(x.map(f64::from)), &tone); + (0..3).map(|c| (p[c] - f64::from(y[c])).powi(2)).sum::() / 3.0 + }) + .sum::() + / sensor.data.len() as f64; + assert!(error.sqrt() < 0.02, "{}", error.sqrt()); + } + + /// Codex review of #631: a low-contrast scene's curve is fitted to its midtones only, and the + /// camera JPEG's bright areas (luminance 0.85 and up) are in neither the fit nor the held-out + /// gates. With 1.2 alone, this frame's curve passed the gates and rendered its bright sky + /// (camera JPEG 0.93) at about 0.28. The second review found the same through the retries' + /// filters: a clipped sky (left out by the refit without clipped pixels), scattered bright + /// pixels (left out away from edges) and sensor values over 1.5 (out of the pairs' range). + #[test] + fn a_narrow_curve_that_darkens_the_bright_areas_is_refused() { + let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]); + for (case, sky_value, is_sky) in [ + ("bright sky", 0.9, &(|i: usize| i < 5600) as &dyn Fn(usize) -> bool), + ("clipped sky", 1.0, &|i: usize| i < 5600), + ("scattered bright pixels", 0.9, &|i: usize| i.is_multiple_of(8)), + ("sky over the pairs' range", 1.6, &|i: usize| i < 5600), + ] { + let mut sensor = Rgb32f::new(96, 64); + let mut reference = sensor.clone(); + let mut clipped = vec![false; sensor.data.len()]; + for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() { + if is_sky(i) { + // bright, nearly white in the camera JPEG + *src = [sky_value; 3]; + *dst = [0.93; 3]; + clipped[i] = case == "clipped sky"; + continue; + } + let ev = 0.06 + (i % 31) as f32 * 0.0008; + *src = [ev * (0.8 + (i % 11) as f32 * 0.025), ev, ev * (0.8 + (i % 17) as f32 * 0.014)]; + let p = known.apply_f32(*src); + let y = luminance_2020(p); + // nearly flat midtones: the fitted curve's extrapolation can't reach the sky + *dst = p.map(|v| v * (0.2 + (y - 0.07) * 0.05) / y); + } + let Some(look) = fit_look(&sensor, &reference, &clipped, None) else { continue }; + let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0); + let correction = look.hue_sat.as_ref().and_then(HueSat::new); + let p = look.matrix.apply([f64::from(sky_value); 3]); + let p = correction.as_ref().map_or(p, |c| c.apply(p.map(|v| v as f32)).map(f64::from)); + let sky = luma(displayed(p, &tone)); + assert!(sky > 0.8, "{case}: an accepted look must not darken what the camera renders at 0.93: {sky}"); + } + } + + /// Third Codex review of #631: for a Sony photo shot with DRO, the fitted curve is lowered to + /// Sony's without DRO after the fit, so it must be checked as lowered. On this frame the + /// fitted curve keeps the sky (0.81 against the fallback's 0.79), and lowered it rendered the + /// sky at 0.61 where the camera JPEG has 0.93. + #[test] + fn a_narrow_curve_is_checked_as_lowered_for_dro() { + let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]); + let mut sensor = Rgb32f::new(96, 64); + let mut reference = sensor.clone(); + for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() { + if i < 5600 { + *src = [0.9; 3]; + *dst = [0.93; 3]; + continue; + } + let ev = 0.06 + (i % 31) as f32 * 0.0008; + *src = [ev * (0.8 + (i % 11) as f32 * 0.025), ev, ev * (0.8 + (i % 17) as f32 * 0.014)]; + let p = known.apply_f32(*src); + let y = luminance_2020(p); + *dst = p.map(|v| v * (0.6 + (y - 0.07) * 0.25) / y); + } + let neutral = luma(displayed([0.9; 3], &ToneMap::new(0.0, 0.0, 0.0))); + if let Some(look) = fit_look_with(&sensor, &reference, &vec![false; sensor.data.len()], None, true) { + let look = with_dro_off_tone(look); + let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0); + let correction = look.hue_sat.as_ref().and_then(HueSat::new); + let p = look.matrix.apply([0.9; 3]); + let p = correction.as_ref().map_or(p, |c| c.apply(p.map(|v| v as f32)).map(f64::from)); + let sky = luma(displayed(p, &tone)); + assert!(sky > neutral - 0.02, "the DRO-lowered curve must not darken the sky past the fallback ({neutral}): {sky}"); + } + } + + /// A black-and-white JPEG of a low-contrast scene (issues #577 and #631): grey pixels, scene luminance spanning + /// under 1.5×, which the old range limit refused before any gate. Its tone curve is fitted. + #[test] + fn a_low_contrast_black_and_white_jpeg_gives_its_tone_curve() { + let mut sensor = Rgb32f::new(96, 64); + let mut reference = sensor.clone(); + for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() { + let ev = 0.06 + (i % 31) as f32 * 0.0008; + *src = [ev * (0.8 + (i % 11) as f32 * 0.025), ev, ev * (0.8 + (i % 17) as f32 * 0.014)]; + *dst = [0.2 + (luminance_2020(*src) - 0.06) * 3.0; 3]; + } + let range = knot_range(sensor.data.iter().map(|x| luma(x.map(f64::from))).collect()); + assert!(range < 1.5, "the scene sits below the old limit of 1.5: {range}"); + let look = fit_tone_only(&sensor, &reference, None).expect("the JPEG's tone curve"); + let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0); + let error = sensor.data.iter().zip(&reference.data).map(|(x, y)| (luma(displayed(x.map(f64::from), &tone)) - f64::from(y[0])).powi(2)); + let rms = (error.sum::() / sensor.data.len() as f64).sqrt(); + assert!(rms < 0.02, "{rms}"); + } + + /// [`a_narrow_curve_that_darkens_the_bright_areas_is_refused`] for a black-and-white JPEG: its tone curve is + /// fitted to nearly flat midtones under 1.5×, and the bright sky (camera JPEG 0.93) is left out of the fit. + #[test] + fn a_narrow_black_and_white_curve_that_darkens_the_bright_areas_is_refused() { + let mut sensor = Rgb32f::new(96, 64); + let mut reference = sensor.clone(); + for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() { + if i < 5600 { + *src = [0.9; 3]; + *dst = [0.93; 3]; + continue; + } + let ev = 0.06 + (i % 31) as f32 * 0.0008; + *src = [ev * (0.8 + (i % 11) as f32 * 0.025), ev, ev * (0.8 + (i % 17) as f32 * 0.014)]; + *dst = [0.2 + (luminance_2020(*src) - 0.06) * 0.05; 3]; + } + if let Some(look) = fit_tone_only(&sensor, &reference, None) { + let sky = luma(displayed(look.matrix.apply([0.9; 3]), &ToneMap::camera(&look.tone, 0.0, 0.0, 0.0))); + assert!(sky > 0.8, "an accepted curve must not darken what the camera renders at 0.93: {sky}"); + } + } } diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 22d76528d..524446e7d 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 64; +pub const RENDER_CACHE_VERSION: u64 = 69; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/docs/camera-preview-colour.md b/docs/camera-preview-colour.md index 9a259e9ed..dc888d459 100644 --- a/docs/camera-preview-colour.md +++ b/docs/camera-preview-colour.md @@ -4,8 +4,7 @@ ARW, NEF, RW2, RAF, CR3, CR2, PEF, SRW, ORF and ERF decoding supply sensor data ## Colour and tone are separate -A fixed sensor proxy of at most 96×96 is developed with the file's WB, black/white levels and active crop. The embedded JPEG is decoded with DCT scaling and a 64-million-pixel input limit, converted through the codec's colour management, and reduced to the same size. Aspect ratios must agree within 2%. For colour fitting, near-black, clipped and nonfinite samples are excluded. Learning a colour matrix requires at least 256 paired pixels, 5% coloured samples (linear channel spread >0.05) and a nonsingular input covariance with bounded condition. An existing camera profile supplies the colour model: fitting only tone still requires 256 pairs and 5% non-monochrome samples, but permits channel spread >0.005 so dark or dull scenes can use the known colours. -A fixed sensor proxy of at most 96×96 is developed with the file's WB, black/white levels and active crop. The embedded JPEG is decoded with DCT scaling and a 128-million-pixel input limit (61 MP bodies' JPEGs, e.g. the ILCE-7RM6 full frame at 66.5 MP, exceed 64 MP; the codec checks the cap before decoding at full size, about 200 MB transient), converted through the codec's colour management, and reduced to the same size. Aspect ratios must agree within 2%. Near-black, clipped and nonfinite samples are excluded. Learning a colour matrix requires at least 256 paired pixels, 5% coloured samples (linear channel spread >0.05) and a nonsingular input covariance with bounded condition. An existing camera profile supplies the colour model: fitting only tone still requires 256 pairs and 5% non-monochrome samples, but permits channel spread >0.005 so dark or dull scenes can use the known colours. +A fixed sensor proxy of at most 96×96 is developed with the file's WB, black/white levels and active crop. The embedded JPEG is decoded with DCT scaling and a 128-million-pixel input limit (61 MP bodies' JPEGs, e.g. the ILCE-7RM6 full frame at 66.5 MP, exceed 64 MP; the codec checks the cap before decoding at full size, about 200 MB transient), converted through the codec's colour management, and reduced to the same size. Aspect ratios must agree within 2%. For colour fitting, near-black, clipped and nonfinite samples are excluded. Learning a colour matrix requires at least 256 paired pixels, 5% coloured samples (linear channel spread >0.05) and a nonsingular input covariance with bounded condition. An existing camera profile supplies the colour model: fitting only tone still requires 256 pairs and 5% non-monochrome samples, but permits channel spread >0.005 so dark or dull scenes can use the known colours. Any tone curve (full, partial or black-and-white look) also needs some contrast: the scene luminance of the brightest 1/32 of the pairs must be at least 1.2× that of the darkest 1/32 (`MIN_TONE_RANGE`); below that, the range check refuses the curve, which leaves only nearly uniform frames. A curve under 1.5× (the limit until issue #631) must also render the frame's bright areas, which the fit and the held-out gates leave out (camera JPEG luminance 0.85 and up), no further from the camera JPEG than the neutral fallback does (in luminance for a black-and-white JPEG): fitted to a narrow range, it extrapolates them. A hazy ILCE-7RM4 burst at 1.46 to 1.49 opened with the neutral fallback, 18 L\* darker than its camera JPEG, while the frame at 1.51 got a look at a held-out RMS of 0.013; with 1.2 they all do. Across 135 public Sony files the lowest range is 4.0, so none of them changes. Two thirds of the usable pixels train a ridge-regularised 3×3 **chromaticity** correction (luminance-normalised RGB). This avoids trying to encode the camera's nonlinear tone curve in a matrix, which left skin grey in the earlier matrix-only estimate. Coefficients are bounded by 8. A matrix can't follow the camera's hue-dependent rendering, though: on a photo of a lime shirt among grey rocks the best matrix rendered the shirt olive (16° hue error), because the few saturated pixels have little weight against the many dull ones, and a matrix fitted to the shirt alone breaks everything else. So a **hue/saturation table** (5 values × 72 hues × 5 saturations, the DNG `ProfileHueSatMap` layout with an sRGB-encoded value axis, applied in linear ProPhoto RGB after the baseline exposure) is fitted to the residual hue shifts and saturation ratios after the matrix: Gaussian kernels around each node (2.5° hue, 0.15 saturation, 0.12 encoded value), weighted by saturation, shrunk toward identity where the photo has few samples, saturation 0 left as identity so neutrals stay neutral. The value axis matters: on a sunny waterfall scene dark yellows had to turn ~12° toward orange while brighter yellow-greens turned ~9° toward green, and one shift per hue made both worse. The table restores each pixel's luminance (the tone curve owns luminance; without that, saturation boosts darkened colours and the table lost on held-out pixels).