mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-24 10:09:01 -06:00
Merge PR #499
This commit is contained in:
Brandon Thomas
@@ -402,19 +402,38 @@ const EDGE_CONTRAST: f32 = 3.0;
|
||||
/// it is tried once more on the pixels away from edges (same gates): there colour pairs stay
|
||||
/// valid when the camera JPEG's geometry differs slightly from the raw's.
|
||||
fn fit_pairs_with(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {
|
||||
fit_pairs_ordered(sensor, reference, colour, &[ToneFit::Quantile, ToneFit::ConditionalMedian])
|
||||
}
|
||||
|
||||
/// [`fit_pairs_with`] with the tone fits to try, in order: the first whose look passes the
|
||||
/// acceptance gates is used. Quantile matching comes first because it cannot produce flat steps;
|
||||
/// the conditional-median fit follows because on a few photos (compacts, early Micro Four Thirds)
|
||||
/// its slightly lower held-out error is what clears the gates, and a photo must not lose its look
|
||||
/// to the change of fit. The whole search (all pixels, then away from edges, with and without the
|
||||
/// camera profile's colour) runs with one tone fit before the next is tried, so the fallback
|
||||
/// accepts exactly what the previous releases accepted.
|
||||
fn fit_pairs_ordered(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, order: &[ToneFit]) -> Option<CameraLook> {
|
||||
let has_profile = colour.is_some();
|
||||
// A different picture style can make a camera profile fail the gates. Preserve the
|
||||
// photo's own colour fit before resorting to the neutral fallback.
|
||||
fit_candidate(sensor, reference, colour).or_else(|| has_profile.then(|| fit_candidate(sensor, reference, None)).flatten())
|
||||
order.iter().find_map(|&tone| {
|
||||
// A different picture style can make a camera profile fail the gates. Preserve the
|
||||
// photo's own colour fit before resorting to the neutral fallback.
|
||||
fit_candidate(sensor, reference, colour.clone(), tone).or_else(|| has_profile.then(|| fit_candidate(sensor, reference, None, tone)).flatten())
|
||||
})
|
||||
}
|
||||
|
||||
/// One candidate colour model, fitted on all pixels and, when that fails the gates, once more on
|
||||
/// the pixels away from edges (issue #232).
|
||||
fn fit_candidate(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {
|
||||
fit_pairs_on(sensor, reference, colour.clone(), None).or_else(|| fit_pairs_on(sensor, reference, colour, Some(EDGE_CONTRAST)))
|
||||
fn fit_candidate(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, tone: ToneFit) -> Option<CameraLook> {
|
||||
fit_pairs_on(sensor, reference, colour.clone(), None, tone).or_else(|| fit_pairs_on(sensor, reference, colour, Some(EDGE_CONTRAST), tone))
|
||||
}
|
||||
|
||||
fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, edge_limit: Option<f32>) -> Option<CameraLook> {
|
||||
fn fit_pairs_on(
|
||||
sensor: &Rgb32f,
|
||||
reference: &Rgb32f,
|
||||
colour: Option<(Mat3, Option<HsvTable>)>,
|
||||
edge_limit: Option<f32>,
|
||||
tone_fit: ToneFit,
|
||||
) -> Option<CameraLook> {
|
||||
// A known colour model needs enough signal for tone fitting, not a scene rich enough to
|
||||
// learn a new colour matrix. Still reject monochrome references.
|
||||
let (pairs, bright) = collect_pairs(sensor, reference, if colour.is_some() { 0.005 } else { 0.05 }, edge_limit)?;
|
||||
@@ -437,7 +456,7 @@ fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Optio
|
||||
};
|
||||
let tone_pairs: Vec<_> =
|
||||
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(colour(*x).map(|v| v.max(0.0))), luma(*y))).collect();
|
||||
let Some(curve) = fit_tone(tone_pairs) else { continue };
|
||||
let Some(curve) = tone_fit.fit(tone_pairs) else { continue };
|
||||
let mut look = CameraLook { matrix, tone: curve, hue_sat: hue_sat.clone() };
|
||||
if let Some(tone) = fit_chroma(&bright, &look) {
|
||||
look.tone = tone;
|
||||
@@ -468,10 +487,11 @@ fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Optio
|
||||
.sum();
|
||||
if lightcraft_pipeline::profiling() {
|
||||
eprintln!(
|
||||
"[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels{})",
|
||||
"[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels{}{})",
|
||||
(before / samples as f64).sqrt(),
|
||||
(after / samples as f64).sqrt(),
|
||||
if edge_limit.is_some() { ", away from edges" } else { "" }
|
||||
if edge_limit.is_some() { ", away from edges" } else { "" },
|
||||
if tone_fit == ToneFit::ConditionalMedian { ", conditional-median tone" } else { "" }
|
||||
);
|
||||
}
|
||||
if samples == 0 || after >= before * MIN_IMPROVEMENT || after / samples as f64 > MAX_HOLDOUT_RMS.powi(2) {
|
||||
@@ -681,12 +701,68 @@ fn fit_chroma(pairs: &[([f64; 3], [f64; 3])], look: &CameraLook) -> Option<Camer
|
||||
(after.is_finite() && after < before).then_some(fitted)
|
||||
}
|
||||
|
||||
/// How the per-photo tone curve is fitted ([`fit_pairs_ordered`] tries them in order).
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
enum ToneFit {
|
||||
/// [`fit_tone`]: no flat steps, no pixel-exact registration needed.
|
||||
Quantile,
|
||||
/// [`fit_tone_conditional`]: the earlier fit, kept as the fallback.
|
||||
ConditionalMedian,
|
||||
}
|
||||
|
||||
impl ToneFit {
|
||||
fn fit(self, pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
match self {
|
||||
ToneFit::Quantile => fit_tone(pairs),
|
||||
ToneFit::ConditionalMedian => fit_tone_conditional(pairs),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The camera's tone curve from scene/JPEG luminance pairs, by quantile matching: knot `i` pairs
|
||||
/// the median of the `i`-th 1/32 of the sorted scene luminances with the median of the `i`-th
|
||||
/// 1/32 of the sorted JPEG luminances. A tone curve is monotone, so it maps each quantile of the
|
||||
/// scene to the same quantile of the JPEG; matching the two distributions needs no pixel-exact
|
||||
/// registration and is increasing by construction.
|
||||
///
|
||||
/// Binning the pairs by scene luminance and taking the JPEG's median per bin instead (followed by
|
||||
/// isotonic pooling) fails on real files: camera JPEGs are lens-corrected and sharpened, so on a
|
||||
/// 96 px proxy of a busy scene neighbouring pairs disagree by more than the gap between bins.
|
||||
/// The conditional medians then zig-zag, pooling turns them into flat steps (13 of 31 segments
|
||||
/// on a Z 6II forest, issue #475) that render as posterised patches, and regression toward the
|
||||
/// mean flattens both ends of the curve (lifted blacks, dull highlights).
|
||||
fn fit_tone(pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
|
||||
return None;
|
||||
}
|
||||
let mut xs: Vec<f64> = pairs.iter().map(|p| p.0).collect();
|
||||
let mut ys: Vec<f64> = pairs.iter().map(|p| p.1).collect();
|
||||
xs.sort_by(f64::total_cmp);
|
||||
ys.sort_by(f64::total_cmp);
|
||||
let n = xs.len();
|
||||
let mut knots = [[0.0; 2]; 32];
|
||||
for (i, knot) in knots.iter_mut().enumerate() {
|
||||
let (a, b) = (i * n / 32, (i + 1) * n / 32);
|
||||
let mid = a + (b - a) / 2;
|
||||
*knot = [*xs.get(mid)? as f32, *ys.get(mid)? as f32];
|
||||
}
|
||||
if knots[31][0] < knots[0][0] * 1.5 {
|
||||
return None;
|
||||
}
|
||||
CameraTone::new(knots)
|
||||
}
|
||||
|
||||
fn median(values: &mut [f64]) -> Option<f64> {
|
||||
values.sort_by(f64::total_cmp);
|
||||
values.get(values.len() / 2).copied()
|
||||
}
|
||||
|
||||
fn fit_tone(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
/// The earlier tone fit, kept as the fallback of [`ToneFit`]: the JPEG's median per equal-population
|
||||
/// bin of scene luminance, then pooled-adjacent-violators isotonic regression. It follows the
|
||||
/// per-pixel pairs closely when the two images are registered, which is why it can clear the
|
||||
/// acceptance gates where [`fit_tone`] just misses them, but on lens-corrected JPEGs of busy scenes
|
||||
/// the pooling leaves flat steps (issue #475).
|
||||
fn fit_tone_conditional(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
|
||||
return None;
|
||||
}
|
||||
@@ -1228,4 +1304,77 @@ mod tests {
|
||||
reference.data.truncate(8);
|
||||
assert!(fit_pairs(&sensor, &reference).is_none());
|
||||
}
|
||||
|
||||
/// A busy scene whose JPEG is the camera curve of the scene two proxy pixels to the right (a
|
||||
/// lens-corrected preview): the fitted curve follows the camera's and has no flat steps.
|
||||
#[test]
|
||||
fn tone_fit_has_no_steps_on_a_misregistered_jpeg() {
|
||||
let camera = |x: f64| 0.6 * x.powf(0.55) / (1.0 + 0.2 * x);
|
||||
let (w, h) = (96usize, 64usize);
|
||||
let mut seed = 0x2545_f491_u32;
|
||||
let mut scene = Vec::with_capacity(w * h);
|
||||
for _ in 0..w * h {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
scene.push(0.01 * (5.0 * f64::from(seed >> 8) / f64::from(1u32 << 24)).exp());
|
||||
}
|
||||
let pairs: Vec<(f64, f64)> =
|
||||
(0..h).flat_map(|y| (0..w - 2).map(move |x| (y * w + x, y * w + x + 2))).map(|(i, j)| (scene[i], camera(scene[j]))).collect();
|
||||
let curve = fit_tone(pairs).unwrap();
|
||||
let mut sorted = scene.clone();
|
||||
sorted.sort_by(f64::total_cmp);
|
||||
let (lo, hi) = (sorted[sorted.len() / 50], sorted[sorted.len() * 49 / 50]);
|
||||
let mut x = lo;
|
||||
while x < hi {
|
||||
let (a, b) = (curve.apply(x as f32), curve.apply((x * 1.02) as f32));
|
||||
assert!(b > a, "flat step at {x}: {a} -> {b}");
|
||||
assert!((f64::from(a) / camera(x) - 1.0).abs() < 0.05, "at {x}: {a} vs {}", camera(x));
|
||||
x *= 1.02;
|
||||
}
|
||||
}
|
||||
|
||||
/// A synthetic photo whose JPEG is the camera look of the scene scaled per pixel by noise of
|
||||
/// `spread` (a pseudo-random, registration-free stand-in for local tone mapping): the per-bin
|
||||
/// median recovers the curve, while the JPEG's broadened distribution stretches the quantile fit.
|
||||
fn noisy_look(spread: f32) -> (Rgb32f, Rgb32f) {
|
||||
let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]);
|
||||
let (w, h) = (96usize, 64usize);
|
||||
let mut sensor = Rgb32f::new(w, h);
|
||||
let mut reference = sensor.clone();
|
||||
let mut seed = 0x9e37_79b9_u32;
|
||||
let mut next = move || {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
f32::from((seed >> 16) as u16) / 32768.0 - 1.0
|
||||
};
|
||||
for (src, dst) in sensor.data.iter_mut().zip(&mut reference.data) {
|
||||
let ev = 0.05 + (next() + 1.0) * 0.2;
|
||||
*src = [ev * (0.8 + (next() + 1.0) * 0.1), ev, ev * (0.8 + (next() + 1.0) * 0.1)];
|
||||
let p = known.apply_f32(*src);
|
||||
let y = luminance_2020(p);
|
||||
let scale = (1.0 - (-2.5 * y).exp()) / y * (1.0 + spread * next());
|
||||
*dst = p.map(|v| v * scale);
|
||||
}
|
||||
(sensor, reference)
|
||||
}
|
||||
|
||||
/// Quantile matching is tried first; the conditional-median fit is the fallback and is used
|
||||
/// only where the quantile look misses the acceptance gates, so no photo loses its look.
|
||||
#[test]
|
||||
fn tone_fit_order_quantile_first_conditional_median_as_fallback() {
|
||||
// Clean data: both fits pass, the default order uses the quantile curve (increasing).
|
||||
let (sensor, reference) = noisy_look(0.0);
|
||||
let default = fit_pairs(&sensor, &reference).expect("clean look");
|
||||
let quantile = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::Quantile]).expect("quantile passes");
|
||||
assert_eq!(default.tone, quantile.tone);
|
||||
// Noisy data: the quantile fit misses the gates, the conditional median still passes.
|
||||
let (sensor, reference) = noisy_look(0.33);
|
||||
assert!(fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::Quantile]).is_none(), "quantile alone misses the gates");
|
||||
let conditional = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian]).expect("conditional median passes");
|
||||
let default = fit_pairs(&sensor, &reference).expect("the fallback keeps the look");
|
||||
assert_eq!(default.tone, conditional.tone);
|
||||
// The order matters only through the first pass; with the fits swapped the conditional one wins even on clean data.
|
||||
let (sensor, reference) = noisy_look(0.0);
|
||||
let swapped = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian, ToneFit::Quantile]).unwrap();
|
||||
let conditional = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian]).unwrap();
|
||||
assert_eq!(swapped.tone, conditional.tone);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ Two thirds of the usable pixels train a ridge-regularised 3×3 **chromaticity**
|
||||
|
||||
The JPEG's colourfulness also depends on brightness, as a camera's per-channel tone curve makes it: relative to a luminance-only curve, shadows were up to 1.4× more colourful and highlights above 0.85 luminance bleached to 0.1–0.3×, so warm sunlit rock rendered cream instead of white (+4 b* in the brightest fifth of the picture). A **chroma curve** (8 nodes over display luminance, `CameraTone::chroma`) is fitted to that ratio, including the bright pixels the colour and tone fits leave out, and applied in the finish stage right after the tone map (CPU and GPU), scaling chroma toward luminance so luminance is kept. It is kept only when it lowers the held-out error; smart previews carry it in their header, and older headers load without one.
|
||||
|
||||
Matrix-only and matrix + table looks are each completed (tone curve, then chroma curve) and compared on the held-out pixels in gamma-encoded display values: in linear values a slightly missed bright rock outweighed a clearly wrong dark shirt. The acceptance gates below still use linear values. Luminance is fitted separately: 32 equal-population bins with robust median knots and pooled-adjacent-violators isotonic regression. The curve preserves black, cannot reverse, and extends beyond observed highlights with a bounded shoulder.
|
||||
Matrix-only and matrix + table looks are each completed (tone curve, then chroma curve) and compared on the held-out pixels in gamma-encoded display values: in linear values a slightly missed bright rock outweighed a clearly wrong dark shirt. The acceptance gates below still use linear values. Luminance is fitted separately, by quantile matching: knot *i* pairs the median of the *i*-th 1/32 of the sorted scene luminances with the median of the *i*-th 1/32 of the sorted JPEG luminances. A tone curve is monotone, so it maps each scene quantile to the same JPEG quantile; this needs no pixel-exact registration and is increasing by construction. The earlier fit (the JPEG's median per scene-luminance bin, then pooled-adjacent-violators isotonic regression) broke on lens-corrected, sharpened JPEGs of busy scenes: neighbouring bins disagreed by more than the gap between them, pooling turned them into flat steps (13 of 31 segments on a CC0 Z 6II forest shot) that rendered as posterised patches with hard edges, and regression toward the mean flattened both ends of the curve (issue #475, also #211, #454). On the 1,781 files of the CC0 archive that render, 384 of the 1,023 accepted fits had at least one flat segment (1,073 in total, up to 14 of 31); with quantile matching 1,017 of the 1,023 have none. The quantile fit is slightly worse by least squares than the conditional medians (median held-out RMS 0.0331 against 0.0330, worst +0.013), and on 7 files (compacts, bridge cameras, early Micro Four Thirds) the look cleared the acceptance gates only with the old curve. The search therefore runs once with the quantile fit (all pixels, away from edges, with and without the camera profile's colour) and, only when no look passes, once more with the earlier conditional-median fit, kept as `fit_tone_conditional`; the gates are the same, so no file loses its look. Those 7 files render exactly as before, and 6 of them still have flat segments (35 in total). The curve preserves black, cannot reverse, and extends beyond observed highlights with a bounded shoulder.
|
||||
|
||||
The matrix is applied to scene-linear sensor pixels. The curve is carried in `SourceInfo` and applied in the pipeline finish stage, after exposure/WB/local edits. It is never baked into decoded luminance or clipped to JPEG precision. CPU and GPU use the same tone LUT. Contrast, whites and blacks adjust the starting look through the display-domain controls. JPEG neutral rendering and the existing DNG colour model are unchanged.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user