diff --git a/crates/engine/src/camera_preview.rs b/crates/engine/src/camera_preview.rs index f62d1ee9e..ed9aa98ef 100644 --- a/crates/engine/src/camera_preview.rs +++ b/crates/engine/src/camera_preview.rs @@ -80,7 +80,8 @@ use tone::{HighlightCheck, ToneFit, fit_tone, fit_tone_conditional, highlights}; /// lens-corrected proxy when the file carries lens opcodes; validated A7CR distortion, including the lens data /// stored with each smart preview. /// 13: the sensor proxy of every per-file fit gets the embedded vignetting correction the render applies (Fujifilm -/// RAF; other makes with a lens database), so the render no longer applies it twice. +/// RAF; other makes with a lens database), so the render no longer applies it twice, at the position a lens warp samples; +/// the warped candidate's alignment leaves out the pixels outside the sensor instead of reading them as black. pub const LOOK_VERSION: u32 = 13; #[derive(Clone, Debug)] diff --git a/crates/engine/src/camera_preview/look_fit.rs b/crates/engine/src/camera_preview/look_fit.rs index 202fdd632..d93cbffe9 100644 --- a/crates/engine/src/camera_preview/look_fit.rs +++ b/crates/engine/src/camera_preview/look_fit.rs @@ -214,7 +214,9 @@ pub(super) fn fit_dng_tone( } None }; - let Some((mut sensor, reference, clipped)) = proxies(raw, bytes, transform, PROXY, MIN_PREVIEW_FRACTION, None) else { return refuse("preview") }; + let Some((mut sensor, reference, clipped)) = proxies(raw, bytes, transform, PROXY, MIN_PREVIEW_FRACTION, None, None) else { + return refuse("preview"); + }; // As the loader renders it: the profile's tables around the baseline exposure. let xy = lightcraft_raw::color::as_shot_white_xy(raw); if let Some(tables) = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy)) { @@ -265,7 +267,7 @@ pub(super) fn clips_more(sensor: &Rgb32f, reference: &Rgb32f, fitted: &ToneMap, } /// The proxies a photo's look is fitted on ([`proxies`] at [`PROXY`]), the sensor's with the embedded vignetting -/// correction the render applies ([`correct_vignetting`]). Without it the tone curve learns the camera JPEG's corrected +/// correction the render applies. Without it the tone curve learns the camera JPEG's corrected /// brightness from the uncorrected sensor, and the render then corrects it again: 21 X-S10 RAFs rendered up to 0.3 EV /// brighter in their midtones and 0.6 EV in their shadows than their camera JPEGs, about 0.05 EV once corrected. pub(super) fn look_proxies( @@ -275,9 +277,7 @@ pub(super) fn look_proxies( lens: Option<&LensGeometry>, embedded: Option<&lightcraft_develop::EmbeddedLens>, ) -> Option<(Rgb32f, Rgb32f, Vec)> { - let (mut sensor, reference, clipped) = proxies(raw, bytes, transform, PROXY, 0.0, lens)?; - correct_vignetting(&mut sensor, embedded, raw.orientation); - Some((sensor, reference, clipped)) + proxies(raw, bytes, transform, PROXY, 0.0, lens, embedded) } /// Multiply the sensor proxy (sensor orientation, default crop) by the embedded vignetting correction, the gain the diff --git a/crates/engine/src/camera_preview/proxy.rs b/crates/engine/src/camera_preview/proxy.rs index b3db0a160..e5a9cf06c 100644 --- a/crates/engine/src/camera_preview/proxy.rs +++ b/crates/engine/src/camera_preview/proxy.rs @@ -1,6 +1,6 @@ use super::colour::{PROFILE_PROXY, PROXY, file_local_look}; use super::hue_sat::fit_hue_sat; -use super::look_fit::{FIT_PREVIEW_MAX_PIXELS, SENSOR_CLIP, clip_mask, without_clipped}; +use super::look_fit::{FIT_PREVIEW_MAX_PIXELS, SENSOR_CLIP, clip_mask, correct_vignetting, without_clipped}; use super::pairs::{collect_pairs, fit_matrix}; use super::{CameraTransform, Filter, HsvTable, Mat3, RawFormat, RawImage, Rgb32f, fit, luminance_2020, resize}; @@ -82,7 +82,8 @@ pub(super) fn structure_agrees(sensor: &Rgb32f, reference: &Rgb32f) -> bool { /// Same-size proxies of the sensor (white-balanced, baseline exposure, through `transform`'s /// matrix: the generic camera ≈ sRGB model) and of the file's embedded camera JPEG (linear Rec.2020), -/// and per proxy pixel whether any sensor sample it covers is clipped. +/// and per proxy pixel whether any sensor sample it covers is clipped. `embedded`'s vignetting correction is applied to +/// the sensor ([`correct_vignetting`]) before any `lens` warp, at the source position, as the render applies it. pub(super) fn proxies( raw: &RawImage, bytes: &[u8], @@ -90,6 +91,7 @@ pub(super) fn proxies( size: usize, min_fraction: f64, lens: Option<&LensGeometry>, + embedded: Option<&lightcraft_develop::EmbeddedLens>, ) -> Option<(Rgb32f, Rgb32f, Vec)> { let edge = (2 * size).max(384) as u32; let decoded = crate::files::decode_raw_preview( @@ -125,6 +127,7 @@ pub(super) fn proxies( let k = (crop.width.max(crop.height).div_ceil(edge as usize).max(2)).div_ceil(2) * 2; let mut sensor = sensor_proxy(raw, k, edge as usize)?; let mut clipped = clip_mask(&sensor); + correct_vignetting(&mut sensor, embedded, raw.orientation); // The camera may have corrected its JPEG with the lens warp the file carries (Nikon Z lenses always, at up to 9 % // zoom; other cameras when their correction setting is on): the sensor moved into that geometry is a candidate, // kept below when it lines up with the JPEG better than the uncorrected one. @@ -319,7 +322,7 @@ pub(crate) fn profile_pairs(raw: &RawImage, bytes: &[u8], lens: Option<&LensGeom return None; } let transform = lightcraft_raw::color::camera_transform(raw, lightcraft_raw::color::as_shot_white_xy(raw)); - let (sensor, reference, clipped) = proxies(raw, bytes, &transform, PROFILE_PROXY, 0.0, lens)?; + let (sensor, reference, clipped) = proxies(raw, bytes, &transform, PROFILE_PROXY, 0.0, lens, None)?; let to_camera = transform.matrix.inverse()?; // clipped sensor pixels don't record the scene's colour (see `fit_look`) let sensor = without_clipped(&sensor, &clipped).unwrap_or(sensor); @@ -369,7 +372,10 @@ pub(super) fn alignment(a: &Rgb32f, b: &Rgb32f) -> f64 { if (b.width, b.height) != (w, h) || w < 3 || h < 3 { return -1.0; } - let lum = |img: &Rgb32f| img.data.iter().map(|p| (luminance_2020(*p).max(1e-4) as f64).ln()).collect::>(); + // NaN (outside a lens-warped proxy) stays NaN: `max` would turn it into a black border and a false edge + let lum = |img: &Rgb32f| { + img.data.iter().map(|p| luminance_2020(*p)).map(|y| if y.is_nan() { f64::NAN } else { (y.max(1e-4) as f64).ln() }).collect::>() + }; let high = |l: &[f64]| -> Vec { (0..w * h) .map(|i| { diff --git a/crates/engine/src/camera_preview/tests.rs b/crates/engine/src/camera_preview/tests.rs index edc923fe4..84a1d30a5 100644 --- a/crates/engine/src/camera_preview/tests.rs +++ b/crates/engine/src/camera_preview/tests.rs @@ -661,7 +661,7 @@ fn corpus_lens_corrected_jpegs_pair_with_the_warped_sensor() { raw.opcodes.list3.retain(|op| !op.is_lens_correction()); let t = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw)); let (Some((plain, reference, _)), Some((chosen, chosen_reference, _))) = - (proxies(&raw, &bytes, &t, PROXY, 0.0, None), proxies(&raw, &bytes, &t, PROXY, 0.0, Some(&lens))) + (proxies(&raw, &bytes, &t, PROXY, 0.0, None, None), proxies(&raw, &bytes, &t, PROXY, 0.0, Some(&lens), None)) else { continue; }; @@ -1348,7 +1348,7 @@ fn corpus_arw_with_dro_is_lowered_to_the_dro_off_tone() { let transform = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw)); let look = fit_preview(&raw, &bytes, &transform, None, None).0.expect("a camera look"); // the look fitted to the JPEG, as before this change - let (sensor, reference, clipped) = proxies(&raw, &bytes, &transform, PROXY, 0.0, None).unwrap(); + let (sensor, reference, clipped) = proxies(&raw, &bytes, &transform, PROXY, 0.0, None, None).unwrap(); let profile = raw.metadata.model.as_deref().and_then(crate::camera_profiles::get); let colour = profile.as_ref().and_then(|p| Some((p.matrix().mul(&transform.matrix.inverse()?), p.hue_sat.clone()))); let fitted = fit_look(&sensor, &reference, &clipped, colour).unwrap(); @@ -2076,6 +2076,81 @@ fn the_look_is_fitted_on_the_vignetting_corrected_sensor() { assert!(uncorrected > camera + 0.03, "without the correction the curve brightens: {uncorrected} vs {camera}"); } +/// The colour fit gets the embedded vignetting correction too ([`starting_colour`] forwards it to every fit): the +/// tinted camera JPEG of the vignetted sensor takes the colour path, not the black-and-white one. +#[test] +fn the_colour_look_is_fitted_on_the_vignetting_corrected_sensor() { + use lightcraft_develop::{EmbeddedLens, EmbeddedVignette}; + let (w, h) = (192usize, 128usize); + let vignette = EmbeddedVignette { k: [1.0, 0.0, 0.0, 0.0, 0.0], center: lightcraft_geom::Point::new(0.5, 0.5), radius: 0.6 }; + let gain = |x: usize, y: usize| { + let point = lightcraft_geom::Point::new(x as f64 + 0.5, y as f64 + 0.5); + lightcraft_pipeline::optics::embedded_vignette_gain(&vignette, point, w as f64, h as f64) as f32 + }; + let scene = |x: usize, y: usize| 0.01 + 0.5 * (x as f32 / w as f32) * (0.3 + 0.7 * y as f32 / h as f32); + let (raw, bytes) = grey_raw_with_jpeg((w, h), |x, y| scene(x, y) / gain(x, y), scene, [1.0, 0.9, 0.8]); + let lens = EmbeddedLens { vignette: Some(vignette), ..Default::default() }; + let at_quarter = |embedded: Option<&EmbeddedLens>| { + let (_, _, look) = starting_colour(&mut raw.clone(), &bytes, None, embedded); + let look = look.expect("a colour look"); + assert!(look.matrix != Mat3::IDENTITY || look.hue_sat.is_some(), "the colour fit, not the black-and-white one"); + ToneMap::camera(&look.tone, 0.0, 0.0, 0.0).apply(0.25) + }; + let (corrected, uncorrected) = (at_quarter(Some(&lens)), at_quarter(None)); + assert!(uncorrected > corrected + 0.03, "without the correction the curve brightens: {uncorrected} vs {corrected}"); +} + +/// A lens-warped sensor proxy gets the vignetting gain of the position it samples, as the render applies it: the +/// vignetted sensor, corrected, gives the proxy of the same sensor without vignetting. +#[test] +fn the_vignetting_correction_is_applied_before_the_lens_warp() { + use lightcraft_develop::{EmbeddedLens, EmbeddedVignette}; + // twice the 384 px proxy edge: the demosaiced proxy of this linear raw is then the size a warp needs (bins of 2) + let (w, h) = (768usize, 512usize); + let vignette = EmbeddedVignette { k: [1.0, 0.0, 0.0, 0.0, 0.0], center: lightcraft_geom::Point::new(0.5, 0.5), radius: 0.6 }; + let gain = |x: usize, y: usize| { + let point = lightcraft_geom::Point::new(x as f64 + 0.5, y as f64 + 0.5); + lightcraft_pipeline::optics::embedded_vignette_gain(&vignette, point, w as f64, h as f64) as f32 + }; + // a texture for the warped candidate to line up with: the JPEG shows the sensor's centre zoomed by 1 / 0.8 + let texture = |x: f64, y: f64| 0.2 * (1.0 + 0.3 * ((x / 48.0).floor() + (y / 48.0).floor()).rem_euclid(2.0)) as f32; + let warp = LensGeometry { k: [0.8, 0.0, 0.0, 0.0], center: [0.5, 0.5] }; + let (cx, cy) = ((w - 1) as f64 / 2.0, (h - 1) as f64 / 2.0); + let jpeg = |x: usize, y: usize| texture(cx + (x as f64 - cx) * 0.8, cy + (y as f64 - cy) * 0.8); + let sensor_proxy = |vignetted: bool, embedded: Option<&EmbeddedLens>| { + let sensor = |x: usize, y: usize| texture(x as f64, y as f64) / if vignetted { gain(x, y) } else { 1.0 }; + let (raw, bytes) = grey_raw_with_jpeg((w, h), sensor, jpeg, [1.0; 3]); + let t = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw)); + let (plain, _, _) = proxies(&raw, &bytes, &t, PROXY, 0.0, None, embedded).unwrap(); + let (sensor, _, _) = proxies(&raw, &bytes, &t, PROXY, 0.0, Some(&warp), embedded).unwrap(); + assert!(sensor.data.iter().zip(&plain.data).any(|(a, b)| a != b), "the warped candidate is chosen"); + sensor + }; + let lens = EmbeddedLens { vignette: Some(vignette), ..Default::default() }; + let (corrected, expected) = (sensor_proxy(true, Some(&lens)), sensor_proxy(false, None)); + let worst = corrected.data.iter().zip(&expected.data).map(|(a, b)| (a[1] / b[1] - 1.0).abs()).filter(|d| d.is_finite()).fold(0.0, f32::max); + assert!(worst < 0.02, "largest deviation {worst}"); +} + +/// [`alignment`] leaves out the pixels a proxy lacks (NaN outside a lens-warped one): read as black, they made a +/// false edge around the border, and the warped candidate lost against a worse unwarped one. +#[test] +fn alignment_leaves_out_missing_pixels() { + let (w, h) = (48usize, 32usize); + let mut a = Rgb32f::new(w, h); + for (i, p) in a.data.iter_mut().enumerate() { + *p = [0.1 + 0.05 * ((i * 7919) % 13) as f32; 3]; + } + let mut b = a.clone(); + for (i, p) in b.data.iter_mut().enumerate() { + let (x, y) = (i % w, i / w); + if x < 4 || y < 4 || x >= w - 4 || y >= h - 4 { + *p = [f32::NAN; 3]; + } + } + assert!(alignment(&a, &b) > 0.999, "{}", alignment(&a, &b)); +} + /// The black-and-white fit has the partial look's checks: a JPEG of another picture, or one the default curve /// already renders, gives no look. #[test] diff --git a/docs/camera-preview-colour.md b/docs/camera-preview-colour.md index 7d1468baf..a17184b64 100644 --- a/docs/camera-preview-colour.md +++ b/docs/camera-preview-colour.md @@ -135,6 +135,11 @@ it (Enable Profile Corrections). The camera applied the same correction to its J `look_proxies`, which multiplies it by that gain as `fit_dng_tone` already did (`correct_vignetting`). Before `LOOK_VERSION` 13 the RAF fit saw the uncorrected sensor: the tone curve learned the JPEG's corrected brightness, and the render added the correction again. Lens-database corrections (`LIGHTCRAFT_LENSFUN_DIR`) reach the fit the same way. +`proxies` applies the gain before it moves the sensor into the JPEG's lens geometry, so each pixel gets the gain of the +position it samples, as in the render. The choice between the warped and the plain sensor (`alignment`) leaves out +the pixels the warp finds no sensor for. It used to read them as black, a false edge along the border that made the +warped candidate lose on some wide-angle shots with barrel distortion; neither change moved ΔE00 by more than 0.04 on +the photos below. On 21 X-S10 photos the render was up to 0.3 EV brighter than the camera JPEG in the midtones (a wide-open blue-hour shot) and up to 0.6 EV in the darkest 3 % of the pixels (ISO 12800 indoors). On 60 other X-S10 photos, picked at