Issue #150: Raw: NEF colour from the camera JPEG; 12-bit NEF black level

LightCraft has no colour matrix for Nikon cameras. Because of this, NEF
files showed colours that were too gray and too green.

- Colour: LightCraft now calculates a colour matrix and a tone curve from
  the JPEG preview in each NEF file. Sony ARW files use the same
  procedure and the same quality checks. White balance is relative to
  the as-shot look, as for ARW files.
- One function, camera_preview::file_local_look, selects the formats for
  the fit and for the relative white balance. Photo::relative_wb in the
  catalog uses the same formats.
- Black level: Nikon stores the black level (maker note 0x003d) in
  14-bit units, also in 12-bit files. The decoder read it as a 12-bit
  value. Because of this, 12-bit files from the D750, D780, D850, D7500
  and Z 50 were too dark. Now the decoder divides the value by 4 for
  12-bit files.
- The corpus gets the D7500 12-bit and 14-bit lossless samples. A test
  compares their levels. The render cache version increases to 9.

Results on 13 NEF files that LightCraft can decode: LightCraft accepted
the fit for all 13 files. The mean colour difference from the camera
JPEG (CIE76 dE) decreased from 13-46 to 2.9-7.8. One file with mixed
light improved by a small quantity only (12.7 to 12.0).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/engine/src/camera_preview.rs

@@ -1,12 +1,13 @@
-//! Estimate an ARW starting look from its own JPEG. Colour and luminance are fitted separately;
-//! the JPEG supplies correspondences only, never output pixels or a replacement for RAW editing.
+//! Estimate the starting look of a raw without a camera colour matrix (Sony ARW, Nikon NEF) from its
+//! own JPEG. Colour and luminance are fitted separately; the JPEG supplies correspondences only,
+//! never output pixels or a replacement for RAW editing.
 use lightcraft_color::{Mat3, luminance_2020};
 use lightcraft_pipeline::tone::{CameraTone, ToneMap};
 use lightcraft_raster::{
     Rgb32f,
     resample::{Filter, fit},
 };
-use lightcraft_raw::{RawImage, color::CameraTransform};
+use lightcraft_raw::{RawFormat, RawImage, color::CameraTransform};
 
 #[derive(Clone, Copy, Debug)]
 pub(crate) struct CameraLook {
@@ -14,8 +15,16 @@ pub(crate) struct CameraLook {
     pub tone: CameraTone,
 }
 
+/// Raw formats whose decoder supplies vendor white-balance multipliers but no camera colour matrix:
+/// their starting look is fitted to the file's own JPEG, and white balance is relative to the
+/// as-shot look (`docs/camera-preview-colour.md`). The catalog's `Photo::relative_wb` matches the
+/// same formats by file extension.
+pub(crate) fn file_local_look(format: RawFormat) -> bool {
+    matches!(format, RawFormat::Arw | RawFormat::Nef | RawFormat::Nrw)
+}
+
 pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransform) -> Option<CameraLook> {
-    if !transform.matrix_is_fallback || raw.format != lightcraft_raw::RawFormat::Arw {
+    if !transform.matrix_is_fallback || !file_local_look(raw.format) {
         return None;
     }
     let jpeg = lightcraft_raw::embedded_preview(bytes)?;
@@ -38,7 +47,7 @@ pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransf
     sensor.map_in_place(|p| transform.matrix.apply_f32(std::array::from_fn(|i| p[i] * transform.wb[i] * gain)));
     let look = fit_pairs(&sensor, &reference)?;
     if lightcraft_pipeline::profiling() {
-        eprintln!("[profile] ARW camera look: {:?}, {:?}", look.matrix.0, look.tone);
+        eprintln!("[profile] {:?} camera look: {:?}, {:?}", raw.format, look.matrix.0, look.tone);
     }
     Some(look)
 }
@@ -137,7 +146,11 @@ fn fit_pairs(sensor: &Rgb32f, reference: &Rgb32f) -> Option<CameraLook> {
         }
     }
     if lightcraft_pipeline::profiling() {
-        eprintln!("[profile] ARW holdout RMS {:.5} -> {:.5} ({samples} channels)", (before / samples as f64).sqrt(), (after / samples as f64).sqrt());
+        eprintln!(
+            "[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels)",
+            (before / samples as f64).sqrt(),
+            (after / samples as f64).sqrt()
+        );
     }
     if !after.is_finite() || samples == 0 || after >= before * MIN_IMPROVEMENT || after / samples as f64 > MAX_HOLDOUT_RMS.powi(2) {
         return None;
@@ -248,6 +261,29 @@ mod tests {
         assert!(fit_pairs(&sensor, &reference).is_some());
     }
 
+    #[test]
+    fn sony_and_nikon_raws_get_a_file_local_look() {
+        assert!([RawFormat::Arw, RawFormat::Nef, RawFormat::Nrw].into_iter().all(file_local_look));
+        assert!(![RawFormat::Dng, RawFormat::Cr2, RawFormat::Raf].into_iter().any(file_local_look));
+    }
+
+    /// A public D7500 NEF (skipped without the corpus): its look is fitted to its own JPEG and white
+    /// balance is relative to the as-shot look.
+    #[test]
+    fn corpus_nef_gets_a_camera_look() {
+        let path = std::env::var_os("LIGHTCRAFT_CORPUS")
+            .map(std::path::PathBuf::from)
+            .unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))
+            .join("raw/nef-nikon-d7500-lossless14.nef");
+        let Ok(bytes) = std::fs::read(&path) else {
+            eprintln!("skip: {} absent", path.display());
+            return;
+        };
+        let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap();
+        assert!(info.camera_tone.is_some(), "no camera look fitted");
+        assert!(info.relative_wb && info.as_shot_temp == 6500.0 && info.as_shot_tint == 0.0);
+    }
+
     #[test]
     fn rejects_monochrome_invalid_and_unrelated_previews() {
         let mut sensor = Rgb32f::new(32, 32);

crates/engine/src/files.rs

@@ -132,7 +132,7 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result<ProbeInfo, String> {
         }
         let (t, tint) = xy_to_temp_tint(lightcraft_raw::color::as_shot_white_xy_of(&raw));
         // Vendor RGB multipliers do not identify an absolute illuminant without camera calibration.
-        let relative = raw.format == lightcraft_raw::RawFormat::Arw && !lightcraft_raw::color::has_matrix(&raw.color);
+        let relative = crate::camera_preview::file_local_look(raw.format) && !lightcraft_raw::color::has_matrix(&raw.color);
         let as_shot_wb = Some(if relative { (6500.0, 0.0) } else { (t.round(), tint.round()) });
         let embedded_lens = embedded_lens(&raw);
         return Ok(ProbeInfo {
@@ -285,7 +285,7 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
             eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", "));
         }
         let (temp, tint) = xy_to_temp_tint(xy);
-        let relative = raw.format == lightcraft_raw::RawFormat::Arw && t.matrix_is_fallback;
+        let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback;
         let camera_tone = camera_look.map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve));
         let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) };
         return Ok((img, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone }));

crates/engine/src/media.rs

@@ -26,7 +26,7 @@ use lightcraft_raster::{Histogram, Rgb32f, Rgba8};
 use serde::{Deserialize, Serialize};
 
 /// Bump when the pipeline's output changes, to invalidate cached thumbnails.
-pub const RENDER_CACHE_VERSION: u64 = 8;
+pub const RENDER_CACHE_VERSION: u64 = 9;
 
 /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
 pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];