pipeline: apply 8x8 ordered Bayer dithering to prevent display banding (#3509)

* pipeline: apply 8x8 ordered Bayer dithering to prevent display banding

Quantizing float linear/display pixel values to 8-bit integers using
round-to-nearest produces noticeable Mach bands and contour stepping on
smooth gradients (skies, shallow depth-of-field backgrounds, skin tones).

This applies an 8x8 ordered Bayer dithering pattern centered around round-to-nearest
with sub-LSB amplitude in both CPU finish (crates/pipeline) and GPU compute
shader (crates/gpu/src/wgsl/finish.wgsl). Exact integer levels, pure black,
and white remain noise-free, while fractional gradients smoothly dissolve into
continuous gradations. Dithering coordinates account for window/tile view offsets
to maintain phase alignment across zoom tiles.

* pipeline, gpu: address review feedback on Bayer dithering

- Always pack VIG_VIEW in GPU params so windowed renders without a vignette
  maintain the correct absolute view origin for Bayer dithering coordinates.
- Restore the full Bayer threshold range (0..1) without deadzones, preserving
  float white via direct half-LSB rounding to prevent edge quantization drop.
- Return the true sampled bounds midpoint in geometry sub-pixel edge detector
  fallbacks.
- Add docstrings to satisfy coverage checks.

* pipeline, gpu: narrow highlight white guard to allow dithering on 254.5

* tests(pipeline): assert edge contrast in manual keystone tracking windows

* docs(pipeline): add docstrings to touched test functions

* Export tests: count distinct levels however the dither orders them

* Mask geometry tests ignore a level of 8-bit dither

---------

Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com>

crates/engine/src/export/tests.rs

@@ -912,6 +912,8 @@ fn ramp(depth: OutputDepth, space: OutputSpace) -> lightcraft_pipeline::Rendered
 fn distinct_levels(bytes: &[u8]) -> (usize, lightcraft_codecs::Decoded) {
     let d = lightcraft_codecs::decode(bytes, Default::default()).expect("decodes");
     let mut v: Vec<u32> = (0..d.image.width).map(|x| d.image.get(x, 1)[1].to_bits()).collect();
+    // sorted first: 8-bit output is dithered, so equal levels aren't all side by side
+    v.sort_unstable();
     v.dedup();
     (v.len(), d)
 }

crates/gpu/src/params.rs

@@ -257,6 +257,7 @@ pub fn finish_block(fp: &FinishParams, masks: &[[f32; MASK_TERMS]], present: &Pr
     }
     p.f("SKIN", ops.skin);
     p.fs("BANDH", lightcraft_pipeline::colorops::band_hues());
+    p.fs("VIG_VIEW", &fp.view);
 
     if let Some(v) = &fp.vig {
         p.b("VIG", true);
@@ -266,7 +267,6 @@ pub fn finish_block(fp: &FinishParams, masks: &[[f32; MASK_TERMS]], present: &Pr
         p.f("VIG_ASPECT_MIX", v.aspect_mix);
         p.f("VIG_POWER", v.power);
         p.f("VIG_HL", v.highlights);
-        p.fs("VIG_VIEW", &fp.view);
         p.u(
             "VIG_STYLE",
             match v.style {

crates/gpu/src/wgsl/finish.wgsl

@@ -326,8 +326,23 @@ fn lut_apply(c: vec3<f32>) -> vec3<f32> {
     return mix(mix(c00, c10, fr.y), mix(c01, c11, fr.y), fr.z);
 }
 
-fn enc8(v: f32) -> u32 {
-    return u32(clamp(v, 0.0, 1.0) * 255.0 + 0.5);
+const BAYER_8X8 = array<f32, 64>(
+     0.0, 32.0,  8.0, 40.0,  2.0, 34.0, 10.0, 42.0,
+    48.0, 16.0, 56.0, 24.0, 50.0, 18.0, 58.0, 26.0,
+    12.0, 44.0,  4.0, 36.0, 14.0, 46.0,  6.0, 38.0,
+    60.0, 28.0, 52.0, 20.0, 62.0, 30.0, 54.0, 22.0,
+     3.0, 35.0, 11.0, 43.0,  1.0, 33.0,  9.0, 41.0,
+    51.0, 19.0, 59.0, 27.0, 49.0, 17.0, 57.0, 25.0,
+    15.0, 47.0,  7.0, 39.0, 13.0, 45.0,  5.0, 37.0,
+    63.0, 31.0, 55.0, 23.0, 61.0, 29.0, 53.0, 21.0,
+);
+
+fn enc8(v: f32, d: f32) -> u32 {
+    let y = clamp(v, 0.0, 1.0) * 255.0;
+    if (y >= 254.9) {
+        return 255u;
+    }
+    return u32(clamp(y + d, 0.0, 255.0));
 }
 
 // sRGB-curve-encoded value → the output space's own curve (`OutputTrc`: 0 sRGB, 1 gamma, 2 Rec.709).
@@ -662,5 +677,8 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
         e = vec3<f32>(PROOF_DISP_R, PROOF_DISP_G, PROOF_DISP_B);
     }
     e = vec3<f32>(out_encode(e.x), out_encode(e.y), out_encode(e.z));
-    out[i] = enc8(e.x) | (enc8(e.y) << 8u) | (enc8(e.z) << 16u) | (255u << 24u);
+    let abs_x = u32(pf(F_VIG_VIEW)) + x;
+    let abs_y = u32(pf(F_VIG_VIEW + 1u)) + y;
+    let dither = (BAYER_8X8[(abs_y & 7u) * 8u + (abs_x & 7u)] + 0.5) / 64.0;
+    out[i] = enc8(e.x, dither) | (enc8(e.y, dither) << 8u) | (enc8(e.z, dither) << 16u) | (255u << 24u);
 }

crates/pipeline/src/finish.rs

@@ -350,6 +350,7 @@ pub fn base_tone(s: &DevelopSettings, info: &SourceInfo, peak: f32) -> ToneMap {
     }
 }
 
+/// Finish the prepared pipeline stages into an 8-bit RGBA image in `space` (or `display`).
 pub(crate) fn finish(
     p: &Prepared,
     s: &DevelopSettings,
@@ -367,11 +368,14 @@ pub(crate) fn finish(
         fp.for_display(d, proof);
     }
     let trc = fp.out_trc;
-    let data = finish_with(p, &fp, false, |e| match trc {
-        OutputTrc::Srgb => [enc(e[0]), enc(e[1]), enc(e[2]), 255],
-        t => {
-            let x = e.map(|v| enc(t.encode(lightcraft_color::transfer::srgb_to_linear(v.clamp(0.0, 1.0)))));
-            [x[0], x[1], x[2], 255]
+    let data = finish_with(p, &fp, false, |x, y, e| {
+        let d = bayer_dither_offset(x, y);
+        match trc {
+            OutputTrc::Srgb => [enc(e[0], d), enc(e[1], d), enc(e[2], d), 255],
+            t => {
+                let c = e.map(|v| enc(t.encode(lightcraft_color::transfer::srgb_to_linear(v.clamp(0.0, 1.0))), d));
+                [c[0], c[1], c[2], 255]
+            }
         }
     });
     Rgba8 { width: w, height: h, data }
@@ -395,18 +399,18 @@ pub(crate) fn finish_deep(
     let trc = fp.out_trc;
     let samples = match depth {
         OutputDepth::F32Linear => {
-            let v = finish_with(p, &fp, true, |e| e.map(|v| srgb_to_linear(v.clamp(0.0, 1.0))));
+            let v = finish_with(p, &fp, true, |_x, _y, e| e.map(|v| srgb_to_linear(v.clamp(0.0, 1.0))));
             DeepSamples::F32(v.into_flattened())
         }
         OutputDepth::F32Hdr => {
             // the sRGB curve extended past 1 (it is unbounded), so values up to `peak` survive
             let peak = fp.peak;
-            let v = finish_with(p, &fp, true, |e| e.map(|v| srgb_to_linear(v.max(0.0)).clamp(0.0, peak)));
+            let v = finish_with(p, &fp, true, |_x, _y, e| e.map(|v| srgb_to_linear(v.max(0.0)).clamp(0.0, peak)));
             DeepSamples::F32(v.into_flattened())
         }
         _ => {
             let q = |v: f32| (v.clamp(0.0, 1.0) * 65535.0 + 0.5) as u16;
-            let v = finish_with(p, &fp, true, |e| match trc {
+            let v = finish_with(p, &fp, true, |_x, _y, e| match trc {
                 OutputTrc::Srgb => e.map(q),
                 t => e.map(|v| q(t.encode(srgb_to_linear(v.clamp(0.0, 1.0))))),
             });
@@ -423,7 +427,7 @@ pub(crate) fn finish_with<T: Copy + Default + Send>(
     p: &Prepared,
     fp: &FinishParams,
     exact: bool,
-    store: impl Fn([f32; 3]) -> T + Sync + Send,
+    store: impl Fn(usize, usize, [f32; 3]) -> T + Sync + Send,
 ) -> Vec<T> {
     let (w, h) = (p.img.width, p.img.height);
     let p_lut = fp.lut.clone();
@@ -432,6 +436,7 @@ pub(crate) fn finish_with<T: Copy + Default + Send>(
     let out_to_norm = fp.out_to_norm;
     let long = fp.ow.max(fp.oh);
     let [vx, vy, vw, vh] = fp.view;
+    let (vx_u, vy_u) = (vx.round() as usize, vy.round() as usize);
     // the vignette belongs to the whole output frame, not to the window being drawn
     let aspect = vw / vh;
 
@@ -603,7 +608,7 @@ pub(crate) fn finish_with<T: Copy + Default + Send>(
                 }
                 None => e,
             };
-            *px = store(warn.unwrap_or(e));
+            *px = store(vx_u + x, vy_u + y, warn.unwrap_or(e));
         }
     });
     out
@@ -857,10 +862,35 @@ pub(crate) fn encode_srgb(lut: &[f32; SRGB_LUT_N + 1], v: f32) -> f32 {
     lut[i] + (lut[i + 1] - lut[i]) * t
 }
 
+#[rustfmt::skip]
+const BAYER_8X8: [u8; 64] = [
+     0, 32,  8, 40,  2, 34, 10, 42,
+    48, 16, 56, 24, 50, 18, 58, 26,
+    12, 44,  4, 36, 14, 46,  6, 38,
+    60, 28, 52, 20, 62, 30, 54, 22,
+     3, 35, 11, 43,  1, 33,  9, 41,
+    51, 19, 59, 27, 49, 17, 57, 25,
+    15, 47,  7, 39, 13, 45,  5, 37,
+    63, 31, 55, 23, 61, 29, 53, 21,
+];
+
+/// Returns the ordered Bayer 8×8 dither threshold offset in `0..1` for pixel `(x, y)`.
 #[inline]
-fn enc(v: f32) -> u8 {
-    // `v` is already sRGB-encoded; round to 8 bits.
-    (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8
+fn bayer_dither_offset(x: usize, y: usize) -> f32 {
+    let idx = (y & 7) * 8 + (x & 7);
+    (BAYER_8X8[idx] as f32 + 0.5) / 64.0
+}
+
+/// Quantizes display-curve-encoded value `v` (0..1) to 8-bit integer with 8×8 ordered Bayer dithering.
+#[inline]
+fn enc(v: f32, dither: f32) -> u8 {
+    let y = v.clamp(0.0, 1.0) * 255.0;
+    // Guard against floating point rounding error (e.g. sRGB <-> Rec.2020 matrix conversion)
+    // where numerical white (1.0) evaluates to ~254.95.
+    if y >= 254.9 {
+        return 255;
+    }
+    (y + dither).clamp(0.0, 255.0) as u8
 }
 
 #[cfg(test)]
@@ -902,4 +932,62 @@ mod tests {
         assert_eq!(encode_srgb(lut, -1.0), 0.0);
         assert!((encode_srgb(lut, 2.0) - 1.0).abs() < 1e-6);
     }
+
+    /// Verifies 8x8 Bayer dithering preserves endpoints, exact 8-bit quantization steps, and fractional levels.
+    #[test]
+    fn bayer_dithering_preserves_extremes_and_dithers_smoothly() {
+        for y in 0..8 {
+            for x in 0..8 {
+                let d = bayer_dither_offset(x, y);
+                // Zero is strictly 0 across all dither offsets
+                assert_eq!(enc(0.0, d), 0);
+                // One is strictly 255 across all dither offsets
+                assert_eq!(enc(1.0, d), 255);
+                // Exact 8-bit quantization steps have zero spatial noise
+                for k in 0..=255 {
+                    assert_eq!(enc(k as f32 / 255.0, d), k);
+                }
+            }
+        }
+
+        // Fractional level: 128.5 / 255.0 should dither equally between 128 and 129
+        let v = 128.5 / 255.0;
+        let mut sum = 0u32;
+        let mut count_128 = 0;
+        let mut count_129 = 0;
+        for y in 0..8 {
+            for x in 0..8 {
+                let val = enc(v, bayer_dither_offset(x, y));
+                sum += val as u32;
+                if val == 128 {
+                    count_128 += 1;
+                } else if val == 129 {
+                    count_129 += 1;
+                }
+            }
+        }
+        assert_eq!(count_128, 32);
+        assert_eq!(count_129, 32);
+        assert_eq!(sum, 32 * 128 + 32 * 129);
+
+        // Fractional highlight level: 254.5 / 255.0 should dither equally between 254 and 255
+        let v = 254.5 / 255.0;
+        let mut sum_hl = 0u32;
+        let mut count_254 = 0;
+        let mut count_255 = 0;
+        for y in 0..8 {
+            for x in 0..8 {
+                let val = enc(v, bayer_dither_offset(x, y));
+                sum_hl += val as u32;
+                if val == 254 {
+                    count_254 += 1;
+                } else if val == 255 {
+                    count_255 += 1;
+                }
+            }
+        }
+        assert_eq!(count_254, 32);
+        assert_eq!(count_255, 32);
+        assert_eq!(sum_hl, 32 * 254 + 32 * 255);
+    }
 }

crates/pipeline/src/tests_geometry.rs

@@ -27,6 +27,10 @@ pub(crate) fn line_x(img: &Rgba8, y: usize, x0: f64, win: f64) -> f64 {
     let (a, b) = ((x0 - win).max(0.0) as usize, ((x0 + win) as usize).min(img.width - 1));
     let row: Vec<f64> = (a..=b).map(|x| img.get(x, y)[1] as f64).collect();
     let max = row.iter().cloned().fold(0.0, f64::max);
+    let min = row.iter().cloned().fold(f64::MAX, f64::min);
+    if max - min <= 1.0 {
+        return (a as f64 + b as f64) / 2.0 + 0.5;
+    }
     let (mut s, mut sw) = (0.0, 0.0);
     for (i, v) in row.iter().enumerate() {
         let wgt = (max - v).max(0.0).powi(2);
@@ -41,6 +45,10 @@ pub(crate) fn line_y(img: &Rgba8, x: usize, y0: f64, win: f64) -> f64 {
     let (a, b) = ((y0 - win).max(0.0) as usize, ((y0 + win) as usize).min(img.height - 1));
     let col: Vec<f64> = (a..=b).map(|y| img.get(x, y)[1] as f64).collect();
     let max = col.iter().cloned().fold(0.0, f64::max);
+    let min = col.iter().cloned().fold(f64::MAX, f64::min);
+    if max - min <= 1.0 {
+        return (a as f64 + b as f64) / 2.0 + 0.5;
+    }
     let (mut s, mut sw) = (0.0, 0.0);
     for (i, v) in col.iter().enumerate() {
         let wgt = (max - v).max(0.0).powi(2);
@@ -54,6 +62,20 @@ fn spread(v: &[f64]) -> f64 {
     v.iter().cloned().fold(f64::MIN, f64::max) - v.iter().cloned().fold(f64::MAX, f64::min)
 }
 
+/// Asserts that a vertical edge with contrast > 1 LSB exists in row `y` around `x0 ± win`.
+fn assert_vertical_edge(img: &Rgba8, y: usize, x0: f64, win: f64) {
+    let (a, b) = ((x0 - win).max(0.0) as usize, ((x0 + win) as usize).min(img.width - 1));
+    let values: Vec<f64> = (a..=b).map(|x| img.get(x, y)[1] as f64).collect();
+    assert!(spread(&values) > 1.0, "no vertical edge near x={x0} on row {y}");
+}
+
+/// Asserts that a horizontal edge with contrast > 1 LSB exists in column `x` around `y0 ± win`.
+fn assert_horizontal_edge(img: &Rgba8, x: usize, y0: f64, win: f64) {
+    let (a, b) = ((y0 - win).max(0.0) as usize, ((y0 + win) as usize).min(img.height - 1));
+    let values: Vec<f64> = (a..=b).map(|y| img.get(x, y)[1] as f64).collect();
+    assert!(spread(&values) > 1.0, "no horizontal edge near y={y0} on column {x}");
+}
+
 /// Barrel distortion with coefficient `k` (< 0): distorted radius = r·(1 + k·r²), r in half-diagonal units.
 fn barrel_undistort(w: f64, h: f64, k: f64) -> impl Fn(Point) -> Point + Sync {
     let (cx, cy, hd) = (w / 2.0, h / 2.0, w.hypot(h) / 2.0);
@@ -213,6 +235,7 @@ fn manual_vertical_keystone_widens_the_top() {
     assert!(lb.y - lt.y > rb.y - rt.y, "left side enlarged");
 }
 
+/// Verifies manual geometry corrections straighten a keystoned grid test fixture.
 #[test]
 fn manual_transforms_straighten_known_keystone() {
     let (w, h) = (480usize, 320usize);
@@ -226,6 +249,12 @@ fn manual_transforms_straighten_known_keystone() {
         let src = keystoned_grid(w, h, fwd, 40.0);
         let s = DevelopSettings { geometry: g.clone(), ..Default::default() };
         let img = render(&src, &SourceInfo::default(), &s, &RenderRequest::fit(w, h)).image;
+        for &y in &[60usize, 150, 260] {
+            assert_vertical_edge(&img, y, 200.0, 15.0);
+        }
+        for &x in &[60usize, 220, 420] {
+            assert_horizontal_edge(&img, x, 200.0, 15.0);
+        }
         let xs: Vec<f64> = [60usize, 150, 260].iter().map(|&y| line_x(&img, y, 200.0, 15.0)).collect();
         let ys: Vec<f64> = [60usize, 220, 420].iter().map(|&x| line_y(&img, x, 200.0, 15.0)).collect();
         assert!(spread(&xs) < 1.0 && (xs[0] - 200.0).abs() < 1.0, "{g:?}: vertical line at {xs:?}");

crates/pipeline/src/tests_mask_geometry.rs

@@ -71,7 +71,12 @@ fn mark_at(s: &DevelopSettings) -> (f64, f64) {
     let req = RenderRequest::fit(W, H);
     let info = SourceInfo::default();
     let (a, b) = (render(&marked(true), &info, s, &req).image, render(&marked(false), &info, s, &req).image);
-    centroid(a.width, (0..a.data.len()).map(|i| (luma(&a, i) - luma(&b, i)).max(0.0)))
+    centroid(a.width, (0..a.data.len()).map(|i| above_dither(luma(&a, i) - luma(&b, i))))
+}
+
+/// A difference between two renders, without the one level of 8-bit dither any pixel can differ by.
+fn above_dither(d: f64) -> f64 {
+    (d - 1.5).max(0.0)
 }
 
 fn centroid(w: usize, weights: impl Iterator<Item = f64>) -> (f64, f64) {
@@ -87,7 +92,7 @@ fn centroid(w: usize, weights: impl Iterator<Item = f64>) -> (f64, f64) {
 
 /// Where the mask brightened the render (`plain` is the same render without the mask).
 fn mask_at(masked: &Rgba8, plain: &Rgba8) -> (f64, f64) {
-    centroid(masked.width, (0..masked.data.len()).map(|i| (luma(masked, i) - luma(plain, i)).max(0.0)))
+    centroid(masked.width, (0..masked.data.len()).map(|i| above_dither(luma(masked, i) - luma(plain, i))))
 }
 
 /// The geometry changes a user makes after drawing a mask.