aleleba/lightcraft
1
0
Fork 0
mirror of https://github.com/storytold/lightcraft.git synced 2026-10-24 10:09:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge PR #1087

Browse Source 
# Conflicts:
#	ROADMAP.md
#	docs/gaps.md
#	docs/parity-checklist.md
#	docs/raw-parity.md
#	docs/raw/coverage.tsv
#	docs/raw/evidence.tsv
#	xtask/src/main.rs
This commit is contained in:
Brandon Thomas
2026-10-14 21:59:31 -04:00
parent d3edca8000 50cb797872
commit aba04aaf5c
20 changed files with 421 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ROADMAP.md
+3
View File
@@ -117,6 +117,8 @@ Ranked toward beta; detail in [docs/roadmap.md](docs/roadmap.md), gaps in [docs/
- 2026-10-14: 97 Canon CR2, Pentax and Ricoh models verified, one CC0 raw.pixls.us file each pinned and
checked against the camera JPEG (`crates/raw/tests/canon_pentax_corpus.rs`). 334 models verified, ~50% of
photographers on a verified body; headline numbers unchanged.
- 2026-10-12: Epson ERF (R-D1, R-D1s, R-D1x) decodes from sensor data, white balance measured from each
file's preview; 3 bodies verified on CC0 files. 240 models verified; headline numbers unchanged.
- 2026-10-11: Leica Q3 verified by a recorded check of 38 private DNGs (all decode and develop at full
size; mean ΔE to the camera JPEG 6.3–13.9). 236 models verified; headline numbers unchanged.
- 2026-10-12: 22 more Canon CR3 bodies verified (EOS R, RP, R3, R5, R6, R6 Mark II / III, R7, R10, R50,
@@ -181,6 +183,7 @@ Ranked toward beta; detail in [docs/roadmap.md](docs/roadmap.md), gaps in [docs/
| 2026-10-14 | minor | 21 Sony and Leica models verified (258 models, 18%); headline numbers unchanged |
| 2026-10-12 | minor | UI/UX remaining 34–58 h → 32–54 h (canvas tools by keyboard); headline numbers unchanged |
| 2026-10-14 | minor | 17 uncompressed Olympus and Samsung models verified on one CC0 file each (254 models verified, 18%); headline numbers unchanged |
| 2026-10-12 | minor | Epson ERF decoded: 3 more verified models (240); headline numbers unchanged |
| 2026-10-12 | minor | UI/UX remaining 40–70 h → 34–58 h (accessibility audit landed); headline numbers unchanged |
| 2026-10-12 | minor | 22 more CR3 bodies verified, EOS R6 Mark III white balance: RAW decoding 77% → 83%, RAW ~56% → ~60%; mainstream 49% → 51%, essentials 64% → 65%; full unchanged at ~61% |
| 2026-10-12 | minor | 117 Nikon and Fujifilm bodies verified (354 models, 24%); headline numbers unchanged |
crates/catalog/src/model.rs
+2 -1
View File
@@ -324,7 +324,7 @@ pub struct Analysis {
/// file's white balance is edited relative to its as-shot look or in Kelvin depends on the camera:
/// [`relative_wb_camera`].
pub fn relative_wb_format(format: &str) -> bool {
["ARW", "NEF", "NRW", "RW2", "RWL", "RAW", "RAF", "CR3", "CR2", "PEF", "SRW", "ORF"].iter().any(|f| format.eq_ignore_ascii_case(f))
["ARW", "NEF", "NRW", "RW2", "RWL", "RAW", "RAF", "CR3", "CR2", "PEF", "SRW", "ORF", "ERF"].iter().any(|f| format.eq_ignore_ascii_case(f))
}
/// Whether a raw of `format` shot on `camera` (Exif make and model, as [`Meta::camera`] keeps them) is developed
@@ -604,6 +604,7 @@ mod edited_tests {
("a.cr2", "CR2", true),
("a.pef", "pef", true),
("a.orf", "ORF", true),
("a.erf", "ERF", true),
("a.dng", "DNG", false),
] {
let mut p = Photo::new(PhotoId(1), Source::Demo { scene: 0 }, name, format, 10, 10, "2026-10-01T00:00:00");
crates/engine/src/camera_preview.rs
+2
View File
@@ -64,6 +64,7 @@ pub(crate) fn file_local_look(format: RawFormat) -> bool {
| RawFormat::Pef
| RawFormat::Srw
| RawFormat::Orf
| RawFormat::Erf
)
}
@@ -2192,6 +2193,7 @@ mod tests {
RawFormat::Pef,
RawFormat::Srw,
RawFormat::Orf,
RawFormat::Erf,
];
assert!(local.into_iter().all(file_local_look));
// DNG carries its own colour model
crates/raw/src/lib.rs
+8
View File
@@ -47,6 +47,7 @@ pub mod spectral;
mod tiffraw;
mod unpack;
mod vendor;
mod wb_estimate;
pub use demosaic::{Method, demosaic};
pub use dngwrite::{DngCompression, DngWriteOptions, write_dng, write_rendered_dng};
@@ -111,6 +112,8 @@ pub enum RawFormat {
CfaTiff,
/// Kodak DCS520C TIFF: a two-component lossless JPEG CFA sub-IFD with a response curve (`vendor::kodak`).
KodakDcs,
/// Epson ERF: a CFA sub-IFD of 12-bit samples in 16-byte blocks (`vendor::erf`).
Erf,
/// Another TIFF-based raw (3FR, IIQ, ERF, KDC, DCR, MOS, …).
OtherTiff,
}
@@ -132,6 +135,7 @@ impl RawFormat {
| RawFormat::CfaTiff
| RawFormat::Srw
| RawFormat::KodakDcs
| RawFormat::Erf
)
}
}
@@ -206,6 +210,9 @@ pub fn probe(bytes: &[u8]) -> Option<RawFormat> {
if make.starts_with("SAMSUNG") && samsung_raw {
return Some(RawFormat::Srw);
}
if vendor::erf::raw_ifd(&t, bytes.len()).is_some() {
return Some(RawFormat::Erf);
}
if vendor::kodak::raw_ifd(&t, bytes).is_some() {
return Some(RawFormat::KodakDcs);
}
@@ -385,6 +392,7 @@ fn decode_with(bytes: &[u8], mode: Mode) -> Result<RawImage> {
RawFormat::CfaTiff => dng::decode_as(bytes, mode, RawFormat::CfaTiff).map(lift_clipped),
RawFormat::Srw => vendor::srw::decode(bytes, mode),
RawFormat::KodakDcs => vendor::kodak::decode(bytes, mode),
RawFormat::Erf => vendor::erf::decode(bytes, mode),
RawFormat::OtherTiff => Err(RawError::Unsupported(other_tiff_reason(bytes))),
other => Err(RawError::Unsupported(format!("{other:?} files are not decoded yet"))),
}
crates/raw/src/preview.rs
+2 -2
View File
@@ -218,8 +218,8 @@ fn preview_covering(bytes: &[u8], min_edge: Option<u32>) -> Option<Vec<u8>> {
.and_then(|(min, big)| smallest_covering(&found, big, min))
.or(largest)
.map(|s| if is_jxl(s) { s.to_vec() } else { trim_eoi(s).to_vec() });
// a TIFF raw we can't decode whose preview no tag points to (Leaf MOS, Epson ERF)
best.or_else(|| format.filter(|f| !f.is_supported()).and_then(|_| scan_for_jpeg(bytes)))
// a TIFF raw whose preview no tag points to: one we can't decode (Leaf MOS), or Epson ERF (in its maker note)
best.or_else(|| format.filter(|f| !f.is_supported() || *f == crate::RawFormat::Erf).and_then(|_| scan_for_jpeg(bytes)))
}
/// The end of the DCT JPEG whose SOI is at `soi` (exclusive index after its EOI) and its component count,
crates/raw/src/sparse.rs
+4 -2
View File
@@ -227,7 +227,7 @@ fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
return plan;
}
// (the Make is usually in the first kilobytes: a NEF, PEF or ORF is known before the rest is read)
if matches!(crate::probe(bytes), Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef)) {
if matches!(crate::probe(bytes), Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef | RawFormat::Erf)) {
return HeaderPlan::Whole("its header probe reads samples");
}
let Ok(tiff) = Tiff::parse(bytes) else { return HeaderPlan::Whole("TIFF structure") };
@@ -262,7 +262,9 @@ fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
holes.insert(off.saturating_add(HEAD), len);
}
finish(len, loaded, &holes, &meta, || match crate::probe(bytes) {
Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef) => HeaderPlan::Whole("its header probe reads samples"),
Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef | RawFormat::Erf) => {
HeaderPlan::Whole("its header probe reads samples")
}
Some(f) if f.is_supported() => HeaderPlan::Ready,
_ => HeaderPlan::Whole("not a supported raw"),
})
crates/raw/src/tiffraw.rs
+1 -1
View File
@@ -217,7 +217,7 @@ fn jxl_chunk_check(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp:
}
/// Decode one baseline JPEG chunk to 8-bit samples: (samples, width, height, channels).
fn lossy_jpeg(src: &[u8], cpp: usize) -> Result<(Vec<u8>, usize, usize, usize)> {
pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize) -> Result<(Vec<u8>, usize, usize, usize)> {
use zune_core::bytestream::ZCursor;
use zune_core::colorspace::ColorSpace;
use zune_core::options::DecoderOptions;
crates/raw/src/vendor/erf.rs
Vendored
+197
View File
@@ -0,0 +1,197 @@
//! Epson ERF (R-D1, R-D1s, R-D1x): a big-endian TIFF whose `SubIFDs` child is a 12-bit CFA image in 16-byte blocks.
//!
//! Sources: TIFF 6.0 (container, `CFARepeatPatternDim` / `CFAPattern`) and our own black-box analysis of the three
//! CC0 ERF samples on raw.pixls.us (one per body); every number below was measured on them.
//!
//! - IFD0 is a 160 x 120 RGB thumbnail; its `SubIFDs` child is the raw: 3040 x 2024, 12 bits, `Compression` 32769,
//! photometric CFA, one strip of exactly `width * 8 / 5` bytes per row. Byte 15 of every 16 is zero in all rows:
//! a row is 16-byte blocks, each 15 bytes of ten 12-bit samples, most significant bits first (`b0 << 4 | b1 >> 4`,
//! `(b1 & 15) << 8 | b2`), then a pad byte. Read the other way (least significant first, as the Olympus blocks are)
//! the frame is noise: the mean step between same-colour neighbours is 150..199 codes against 11..18.
//! - `CFAPattern` `00 01 01 02`: RGGB.
//! - The maker note (`EPSON\0\x01\0`, then a big-endian IFD with offsets from the file start) holds four black levels
//! as `u32` in tag `0x0401` (61..65; in Bayer-cell order), above which the darkest 0.01% of samples sit; `0x020b` x
//! `0x020c` is the picture size, 3008 x 2000. Registered against the 640 x 424 preview JPEG (correlation of the
//! green channel over every even offset), that picture starts at column 20, row 12 in all three files.
//! - White: the saturation plateau (4095 on the clipped R-D1x frame), else 4095.
//! - No maker-note tag tracks the as-shot white balance across the three files, so it is measured from the
//! preview's neutral pixels ([`crate::wb_estimate`]); on the R-D1s sample (54,000 neutral preview pixels) that
//! gives red 2.68 and blue 1.31.
//! - The preview is the maker note's tag `0x0280`, a JPEG whose first byte is stored as `ee` (found by the
//! structural scan in `preview::scan_for_jpeg`).
use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
use lightcraft_geom::Orientation;
use lightcraft_tiff::image::chunk_bytes;
use lightcraft_tiff::{Ifd, Tiff, tags as t};
/// Epson's private compression code for the 16-byte block layout.
const BLOCKS16: u16 = 32769;
/// Maker-note tags.
const MN_BLACK: u16 = 0x0401;
const MN_WIDTH: u16 = 0x020b;
const MN_HEIGHT: u16 = 0x020c;
/// Where the picture starts in the sensor frame (measured, see the module notes).
const CROP_ORIGIN: (usize, usize) = (20, 12);
/// The raw IFD of an Epson ERF: CFA, 12 bits, compression 32769, one strip of 16-byte blocks of ten samples.
pub(crate) fn raw_ifd(tiff: &Tiff, len: usize) -> Option<&Ifd> {
let make = tiff.find(t::MAKE).and_then(|e| e.value.as_str())?;
if !make.to_ascii_uppercase().contains("EPSON") {
return None;
}
tiff.all_ifds().into_iter().find(|i| {
let Ok(info) = i.image() else { return false };
let (w, h) = (info.width as u64, info.height as u64);
i.u16(t::PHOTOMETRIC) == Some(t::photometric::CFA)
&& i.u16(t::COMPRESSION) == Some(BLOCKS16)
&& i.u16(t::BITS_PER_SAMPLE) == Some(12)
&& w > 0
&& w % 10 == 0
&& matches!(info.chunks(len as u64).as_slice(), [c] if Some(c.len) == (w * 8 / 5).checked_mul(h))
})
}
/// The maker-note IFD (`EPSON\0\x01\0` then an IFD in the file's byte order, offsets from the file start).
fn maker_note(bytes: &[u8], tiff: &Tiff) -> Option<Ifd> {
let e = tiff.exif()?.get(t::MAKER_NOTE)?;
let start = usize::try_from(e.offset).ok()?;
if !bytes.get(start..)?.starts_with(b"EPSON\0") {
return None;
}
let opts = lightcraft_tiff::ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() };
lightcraft_tiff::parse_ifd_at(bytes, e.offset.checked_add(8)?, tiff.order, 0, false, &opts).ok().map(|(i, _)| i)
}
/// Unpack a strip of 16-byte blocks (15 bytes of ten MSB-first 12-bit samples, one pad byte).
fn unpack(src: &[u8], out: &mut [u16]) {
let (blocks, _) = src.as_chunks::<16>();
let (pixels, _) = out.as_chunks_mut::<10>();
for (b, px) in blocks.iter().zip(pixels) {
crate::unpack::unpack_msb(&b[..15], 12, px);
}
}
pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
let tiff = Tiff::parse(bytes)?;
let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("Erf TIFF without IFDs".into()))?;
let raw = raw_ifd(&tiff, bytes.len()).ok_or_else(|| RawError::Unsupported("Erf raw image not found".into()))?;
let info = raw.image()?;
let (w, h) = (info.width as usize, info.height as usize);
let n = w.checked_mul(h).filter(|n| *n > 0 && *n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?;
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() },
_ => return Err(RawError::Unsupported("Erf raw without a 2x2 CFAPattern".into())),
};
let mn = maker_note(bytes, &tiff);
let black = match mn.as_ref().and_then(|m| m.u64s(MN_BLACK)).as_deref() {
Some(v @ [_, _, _, _]) if v.iter().all(|&b| b < 4095) => {
BlackLevel { repeat_rows: 2, repeat_cols: 2, values: v.iter().map(|&b| b as f32).collect(), delta_h: vec![], delta_v: vec![] }
}
_ => BlackLevel::uniform(0.0),
};
let size = mn.as_ref().and_then(|m| Some((m.u32(MN_WIDTH)? as usize, m.u32(MN_HEIGHT)? as usize)));
let crop = match size {
Some((cw, ch)) if cw > 0 && ch > 0 && CROP_ORIGIN.0 + cw <= w && CROP_ORIGIN.1 + ch <= h => Rect::new(CROP_ORIGIN.0, CROP_ORIGIN.1, cw, ch),
_ => Rect::new(0, 0, w, h),
};
// the white balance is measured from the samples, so the header probe reads them too
let chunks = info.chunks(bytes.len() as u64);
let src = chunks.first().and_then(|c| chunk_bytes(bytes, c)).ok_or_else(|| RawError::Corrupt("Erf strip outside file".into()))?;
let mut d = vec![0u16; n];
unpack(src, &mut d);
let white = super::white_from_data(&d, 12);
let wb = crate::wb_estimate::as_shot(bytes, &crate::wb_estimate::Frame { data: &d, width: w, crop, cfa: &cfa, black: &black, white });
let data = if mode == Mode::Full { RawData::U16(d) } else { RawData::U16(Vec::new()) };
let mut metadata = lightcraft_meta::from_tiff(&tiff);
metadata.width = Some(crop.width as u32);
metadata.height = Some(crop.height as u32);
let img = RawImage {
format: RawFormat::Erf,
width: w,
height: h,
cpp: 1,
data,
cfa: Some(cfa),
bits: 12,
black,
white: vec![white],
active_area: Rect::new(0, 0, w, h),
crop,
orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
opcodes: OpcodeLists::default(),
metadata,
};
img.validate_for(mode)?;
Ok(img)
}
#[cfg(test)]
mod tests {
use super::*;
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
/// Pack ten-sample groups into 16-byte blocks, MSB first, zero pad.
fn pack(px: &[u16]) -> Vec<u8> {
let mut out = Vec::new();
for g in px.chunks(10) {
for p in g.chunks(2) {
let (a, b) = (p[0], p[1]);
out.extend_from_slice(&[(a >> 4) as u8, ((a & 15) << 4) as u8 | (b >> 8) as u8, b as u8]);
}
out.push(0);
}
out
}
fn sample(w: u32, h: u32, px: &[u16]) -> Vec<u8> {
let mut raw = IfdBuilder::new();
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
raw.set(t::IMAGE_WIDTH, Value::Long(vec![w]));
raw.set(t::IMAGE_LENGTH, Value::Long(vec![h]));
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![12]));
raw.set(t::COMPRESSION, Value::Short(vec![BLOCKS16]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA]));
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2]));
raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![pack(px)] });
let mut ifd0 = IfdBuilder::new();
ifd0.set(t::MAKE, Value::Ascii("SEIKO EPSON CORP.".into()));
ifd0.set(t::MODEL, Value::Ascii("R-D1".into()));
ifd0.set(t::ORIENTATION, Value::Short(vec![1]));
ifd0.add_sub_ifd(raw);
TiffWriter::new(lightcraft_tiff::ByteOrder::Big, false).write(&[ifd0]).unwrap()
}
#[test]
fn decodes_the_block_layout() {
let (w, h) = (20u32, 6u32);
let px: Vec<u16> = (0..w * h).map(|i| (i * 37 % 4096) as u16).collect();
let f = sample(w, h, &px);
assert_eq!(crate::probe(&f), Some(RawFormat::Erf));
let r = crate::decode(&f).unwrap();
assert_eq!(r.data, RawData::U16(px));
assert_eq!(r.cfa.as_ref().map(Cfa::name).as_deref(), Some("RGGB"));
assert!(r.wb_multipliers.is_some());
assert_eq!(crate::probe_info(&f).unwrap(), r.info());
}
#[test]
fn malformed_files_never_panic() {
let px: Vec<u16> = (0..200).map(|i| (i * 13 % 4096) as u16).collect();
let f = sample(20, 10, &px);
for n in (0..f.len()).step_by(3) {
let _ = crate::decode(&f[..n]);
}
for i in 0..f.len() {
for v in [0u8, 0xff] {
let mut g = f.clone();
g[i] = v;
let _ = crate::decode(&g);
let _ = crate::probe_info(&g);
}
}
}
}
crates/raw/src/vendor/mod.rs
Vendored
+1
View File
@@ -11,6 +11,7 @@ pub mod cr2;
pub mod cr3;
pub mod crx;
pub(crate) mod crx_wavelet;
pub mod erf;
pub mod kodak;
pub mod nef;
mod nef_lens;
crates/raw/src/wb_estimate.rs
+146
View File
@@ -0,0 +1,146 @@
//! As-shot white balance for raws that record none we can read: measured from the file's own embedded preview.
//!
//! Sources: own design. The camera rendered its preview with the as-shot white balance, so pixels the preview shows
//! as neutral grey were neutral for that white balance. The mosaic is averaged per colour over the preview's pixel
//! grid (inside the default crop); over the preview pixels whose channels lie within a few codes of each other
//! (not near black, not near clipping, no clipped sensor sample under them) the median green-over-red and
//! green-over-blue ratios are the gains. A colour matrix maps neutral to neutral, so its absence doesn't bias the
//! result. Without enough neutral pixels the mean of the unclipped mosaic is used instead (grey world).
use crate::{BlackLevel, Cfa, Rect};
/// Preview pixels whose largest and smallest channel differ by less than this (8-bit codes) count as neutral.
const NEUTRAL_SPREAD: u8 = 12;
/// At least this many neutral preview pixels are needed.
const MIN_NEUTRAL: usize = 64;
/// Previews larger than this (pixels) are not decoded for the estimate.
const MAX_PREVIEW_PIXELS: usize = 1 << 24;
/// The raw frame the estimate looks at.
pub(crate) struct Frame<'a> {
pub data: &'a [u16],
pub width: usize,
pub crop: Rect,
pub cfa: &'a Cfa,
pub black: &'a BlackLevel,
pub white: f32,
}
/// Gains `[R/G, 1, B/G]`: from the neutral pixels of the embedded preview, else grey world. `None` for an empty or
/// degenerate frame.
pub(crate) fn as_shot(bytes: &[u8], f: &Frame) -> Option<[f32; 3]> {
from_preview(bytes, f).or_else(|| grey_world(f))
}
/// Per preview-grid cell means of R, G, B (black subtracted) and whether a clipped sample lies in the cell.
fn cell_means(f: &Frame, pw: usize, ph: usize) -> Option<Vec<([f64; 3], bool)>> {
let c = f.crop;
if pw == 0 || ph == 0 || c.width < pw || c.height < ph || f.width == 0 {
return None;
}
let mut cells = vec![([0f64; 3], [0u32; 3], false); pw.checked_mul(ph)?];
for y in c.y..c.y.saturating_add(c.height) {
let row = f.data.get(y.checked_mul(f.width)?..y.checked_mul(f.width)?.checked_add(f.width)?)?;
let cy = (y - c.y) * ph / c.height;
for x in c.x..c.x.saturating_add(c.width).min(f.width) {
let v = f32::from(*row.get(x)?);
let cx = (x - c.x) * pw / c.width;
let col = usize::from(f.cfa.color_at(x, y)).min(2);
let cell = cells.get_mut(cy * pw + cx)?;
if v >= f.white * 0.98 {
cell.2 = true;
}
cell.0[col] += f64::from(v - f.black.at(x, y, 0, 1));
cell.1[col] += 1;
}
}
Some(cells.into_iter().map(|(s, n, clip)| ([0, 1, 2].map(|k| if n[k] > 0 { s[k] / f64::from(n[k]) } else { 0.0 }), clip)).collect())
}
fn median(v: &mut [f64]) -> Option<f64> {
if v.is_empty() {
return None;
}
v.sort_by(f64::total_cmp);
v.get(v.len() / 2).copied()
}
fn from_preview(bytes: &[u8], f: &Frame) -> Option<[f32; 3]> {
let jpeg = crate::preview::embedded_preview(bytes)?;
let (px, pw, ph, ch) = crate::tiffraw::lossy_jpeg(&jpeg, 3).ok()?;
if ch != 3 || pw.checked_mul(ph)? > MAX_PREVIEW_PIXELS {
return None;
}
// the preview must show the crop: aspect ratios within 3%
let (ra, pa) = (f.crop.width as f64 / f.crop.height.max(1) as f64, pw as f64 / ph as f64);
if (ra / pa - 1.0).abs() > 0.03 {
return None;
}
let cells = cell_means(f, pw, ph)?;
let (mut gr, mut gb) = (Vec::new(), Vec::new());
for (i, (m, clip)) in cells.iter().enumerate() {
let p = px.get(i * 3..i * 3 + 3)?;
let (lo, hi) = (p.iter().min().copied().unwrap_or(0), p.iter().max().copied().unwrap_or(255));
if *clip || lo < 30 || hi > 240 || hi - lo >= NEUTRAL_SPREAD || m.iter().any(|v| *v <= 1.0) {
continue;
}
gr.push(m[1] / m[0]);
gb.push(m[1] / m[2]);
}
if gr.len() < MIN_NEUTRAL {
return None;
}
gains(median(&mut gr)?, median(&mut gb)?)
}
fn gains(gr: f64, gb: f64) -> Option<[f32; 3]> {
let ok = |g: f64| g.is_finite() && (0.1..=10.0).contains(&g);
(ok(gr) && ok(gb)).then_some([gr as f32, 1.0, gb as f32])
}
fn grey_world(f: &Frame) -> Option<[f32; 3]> {
let c = f.crop;
let (mut s, mut n) = ([0f64; 3], [0u64; 3]);
for y in c.y..c.y.saturating_add(c.height) {
for x in c.x..c.x.saturating_add(c.width).min(f.width) {
let Some(&v) = f.data.get(y.checked_mul(f.width)?.checked_add(x)?) else { continue };
let v = f32::from(v);
if v >= f.white * 0.98 {
continue;
}
let k = usize::from(f.cfa.color_at(x, y)).min(2);
s[k] += f64::from((v - f.black.at(x, y, 0, 1)).max(0.0));
n[k] += 1;
}
}
if n.contains(&0) {
return None;
}
let m = [0, 1, 2].map(|k| s[k] / n[k] as f64);
if m.iter().any(|v| *v <= 0.0) {
return None;
}
gains(m[1] / m[0], m[1] / m[2])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn grey_world_balances_a_tinted_flat_frame() {
// RGGB, R at 400, G at 800, B at 200 above a black of 100
let (w, h) = (8, 8);
let d: Vec<u16> = (0..w * h).map(|i| [500, 900, 900, 300][(i / w % 2) * 2 + i % 2]).collect();
let cfa = Cfa::bayer_static("RGGB");
let black = BlackLevel::uniform(100.0);
let f = Frame { data: &d, width: w, crop: Rect::new(0, 0, w, h), cfa: &cfa, black: &black, white: 4095.0 };
assert_eq!(as_shot(b"not a raw", &f), Some([2.0, 1.0, 4.0]));
// degenerate: everything clipped
let d = vec![4095u16; w * h];
let f = Frame { data: &d, ..f };
assert_eq!(as_shot(b"", &f), None);
// a crop outside the data never panics
let f = Frame { crop: Rect::new(4, 4, 100, 100), ..f };
let _ = as_shot(b"", &f);
}
}
crates/raw/tests/corpus.rs
+33
View File
@@ -1249,3 +1249,36 @@ fn fuji_embedded_lens_tables_follow_the_validated_models() {
}
eprintln!("Fuji lens tables: {checked} RAFs, {warped} warps, {vignetted} vignette gains");
}
/// Epson ERF (`crates/raw/src/vendor/erf.rs`): the 16-byte block strip decodes to a smooth 3040 x 2024 RGGB
/// mosaic with the maker note's black levels, the 3008 x 2000 crop and a white balance measured from the
/// preview's neutral pixels. Skips files that are absent.
#[test]
fn corpus_epson_erf() {
let root = corpus_root().join("raw");
let mut checked = 0;
for name in ["erf-epson-rd1.erf", "erf-epson-rd1s.erf", "erf-epson-rd1x.erf"] {
let Ok(bytes) = std::fs::read(root.join(name)) else {
eprintln!("skip: {name} absent");
continue;
};
assert_eq!(probe(&bytes), Some(RawFormat::Erf), "{name}");
let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}"));
assert_eq!((img.width, img.height, img.bits), (3040, 2024, 12), "{name}");
assert_eq!((img.crop.x, img.crop.y, img.crop.width, img.crop.height), (20, 12, 3008, 2000), "{name}");
assert_eq!(img.cfa.as_ref().map(|c| c.name()).as_deref(), Some("RGGB"), "{name}");
assert!(img.black.values.len() == 4 && img.black.values.iter().all(|b| (55.0..=70.0).contains(b)), "{name}: {:?}", img.black.values);
let wb = img.wb_multipliers.unwrap_or_else(|| panic!("{name}: no white balance"));
eprintln!("{name}: wb {wb:?}");
if name == "erf-epson-rd1s.erf" {
assert!((wb[0] - 2.68).abs() < 0.1 && (wb[2] - 1.31).abs() < 0.1, "{name}: {wb:?}");
}
// same-colour neighbours differ little: the bit order is right
let lightcraft_raw::RawData::U16(d) = &img.data else { panic!("float data") };
let step: f64 = d.chunks(3040).take(400).flat_map(|r| r.windows(3).map(|w| f64::from(w[0].abs_diff(w[2])))).sum::<f64>() / (400.0 * 3038.0);
assert!(step < 40.0, "{name}: rough {step}");
assert!(embedded_preview(&bytes).is_some(), "{name}: no preview");
checked += 1;
}
eprintln!("ERF: {checked} corpus files checked");
}
docs/camera-preview-colour.md
+2 -2
View File
@@ -1,6 +1,6 @@
# Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2, Pentax PEF, Samsung SRW and Olympus ORF starting look
# Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2, Pentax PEF, Samsung SRW, Olympus ORF and Epson ERF starting look
ARW, NEF, RW2, RAF, CR3, CR2, PEF, SRW and ORF decoding supply sensor data and camera white-balance multipliers, but currently no measured camera colour matrix. Treating camera RGB as linear sRGB produces a dark, muted default rendering. The loader now estimates a file-local starting look from that file's embedded camera JPEG, only when a usable camera matrix is absent. All output pixels still come from the RAW sensor data; there is no JPEG replacement or uniform saturation boost.
ARW, NEF, RW2, RAF, CR3, CR2, PEF, SRW, ORF and ERF decoding supply sensor data and camera white-balance multipliers, but currently no measured camera colour matrix. Treating camera RGB as linear sRGB produces a dark, muted default rendering. The loader now estimates a file-local starting look from that file's embedded camera JPEG, only when a usable camera matrix is absent. All output pixels still come from the RAW sensor data; there is no JPEG replacement or uniform saturation boost.
## Colour and tone are separate
docs/gaps.md
+1
View File
@@ -78,6 +78,7 @@ Beta-blocking gaps (B) total **~230–430 h**; with the stability and platform w
| 2026-10-13 | patch | Gaps 10, 22 and 23 each had two rows (two branches edited them; the merges kept both): the older copies are removed, and `cargo xtask parity` (in `ci`) now fails when a gap number is listed twice |
| 2026-10-13 | minor | Closed recently: Sony ILCA-99M2 profile |
| 2026-10-14 | minor | Gap 10: 21 Sony and Leica models verified on one CC0 file each (258 models) |
| 2026-10-12 | minor | Gap 20 shrinks: Epson ERF decodes (`vendor/erf.rs`), 3 bodies verified; gap 10 count 237 → 240 |
| 2026-10-11 | minor | Gap 10 shrinks: Leica Q3 verified by a recorded check of 38 private DNGs (236 models verified; Leica 14); gap not closed |
| 2026-10-12 | minor | Camera-look deep shadows follow the JPEG toe (#1307); camera calibration remains partial |
| 2026-10-12 | minor | Gap 23 (accessibility): crop handles, spots, gradient handles and mask pins by keyboard; 4–8 h → 2–4 h; gap not closed |
docs/parity-checklist.md
+1
View File
@@ -71,6 +71,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts).
| Id | Feature | Tier | Status | Evidence | Missing / notes |
|---|---|---|---|---|---|
| 2026-10-12 | minor | LR-IMP-FORMATS: Epson ERF decodes from sensor data (`vendor/erf.rs`), no longer preview-only |
| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog (Import Photos): the scanned source (a scanned folder is not added to Local), candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked; with several file types, a toggle per type checks or unchecks all of its files, e.g. the raws without the JPEGs beside them, issue #344) filling a resizable window (issue #337), transfer mode (add in place / copy / move, each explained), album (existing/new), preset, keywords; the folder scan runs on a background thread, shown in the activity stack with ✕ to cancel (a NAS folder no longer freezes the window); the import itself (probing, sidecars, copying / moving files) runs on a worker thread and its batches join the library between frames, shown in the activity stack with ✕ to cancel; one undo step. Drag-and-drop goes through the same background import. File dialogs list the importable extensions (`lightcraft_engine::import::EXTENSIONS`) in any letter case on Linux, where GTK and the desktop portal match globs case-sensitively (`apps/lightcraft/src/dialog_filter.rs`, issue #342; 0.4.0 hid `.JPG` / `.NEF`, issue #705). The folder scan, Browse Folder, drag-and-drop and the sidecar lookup match extensions in any case (`crates/engine/src/tests_import.rs`) |
| LR-IMP-DRAGDROP | Drop files/folders to import | P0 | ✅ | `crates/ui-egui/src/lib.rs` (dropped files → `cmd:library.import`) | dropping onto a specific album not supported |
| LR-IMP-DUPES | Skip duplicates by content | P1 | ✅ | `crates/engine/src/import.rs`, `crates/engine/src/tests_import.rs` | |
docs/raw-parity.md
+5 -3
View File
@@ -42,7 +42,7 @@ committed table is stale.
`lightcraft_engine::import::EXTENSIONS`, and `KNOWN_UNSUPPORTED` in `crates/raw/tests/corpus.rs`. Doc
claims were not used.
- **Verified** means a real file from that body decoded from sensor data in: the corpus test over the
`RAW_SAMPLES` manifest (`xtask/src/main.rs`, 101 CC0 files; bodies whose sample is in
`RAW_SAMPLES` manifest (`xtask/src/main.rs`, 104 CC0 files; bodies whose sample is in
`KNOWN_UNSUPPORTED` excluded), `crates/raw/tests/cr3_corpus.rs` (3 bodies exact against an external decoder;
the EOS R5 Mark II pinned, with a QP-map check of 33,665 private files; 22 more bodies pinned on one CC0
file each and checked against the camera JPEG, [cr3.md](cr3.md#further-bodies-one-cc0-file-each)), the RAF reference
@@ -113,6 +113,7 @@ checked, and Canon, the largest bucket, is the weakest of the big four.
| Format | Variant | Status | Evidence |
|---|---|---|---|
| 2026-10-12 | minor | Epson ERF decoded (`vendor/erf.rs`): R-D1, R-D1s, R-D1x verified; 240 verified, 229 preview (merged after the Leica Q3 and Hasselblad Lunar; duplicated summary rows removed) |
| CR2 | lossless JPEG slices | ✅ | 8 corpus bodies; CFA phase from `CR2CFAPattern` (#85, 35 bodies); masked-column black level; white level from the active area only (#1272: masked rows at 16383 hid the image plateau on the EOS 50D, 60D, 450D (Rebel XSi), 500D (Rebel T1i), 550D (Rebel T2i), 1200D (Rebel T5) and 100D (Rebel SL1), clipped highlights at 81–98%) |
| CR2 | sRAW / mRAW (YCbCr) | ✅ | `vendor/cr2/sraw.rs`, 31 samples from 16 models |
| CR3 | CRX lossless, version 0x100 | ✅ | M50, R100, R8 exact against a reference; R5 Mark II (odd-height planes) pinned |
@@ -148,7 +149,8 @@ checked, and Canon, the largest bucket, is the weakest of the big four.
| DNG | GoPro GPR (VC-5) | ⬜ | refused |
| 3FR / FFF | DNG-like layouts | 🟡 | decoded when uncompressed or LJ92 |
| 3FR / FFF | Hasselblad compression | ⬜ | preview |
| IIQ, MOS, MRW, ERF, KDC, X3F | all | ⬜ | preview; an uncompressed MOS, whose IFD0 is the mosaic itself with no Make tag, is recognised by Leaf's data block `0x8606` (#1275) |
| ERF | 12-bit in 16-byte blocks (R-D1, R-D1s, R-D1x) | ✅ | 3 CC0 corpus files; crop registered and WB measured against the embedded preview (no WB tag found); `vendor/erf.rs` |
| IIQ, MOS, MRW, KDC, X3F | all | ⬜ | preview; an uncompressed MOS, whose IFD0 is the mosaic itself with no Make tag, is recognised by Leaf's data block `0x8606` (#1275) |
| DCR, MEF, MFW, DXO, GPR, TIFF raws | all | ⬜ | not imported |
### Compressed ORF: a clean-room starting point
@@ -257,7 +259,7 @@ works, from [raw/usage-weights.tsv](raw/usage-weights.tsv).
| 7 | Sony A100 and DSC-F828 regression (#702) | +0.1 | 1–2 | ~0.07 | samples on raw.pixls.us |
| 8 | Canon CRW (2000–2004 bodies) | +1.2 by the uniform tail weight, far less by real use | 6–12 | ≤ 0.1 | samples |
| 9 | Nikon HE / HE★ NEF (Z 8, Z 9, Z 6III, Z f, Z50II, Z5II set to HE) | +1.1 | 20–40 | ~0.04 | a pure-Rust JPEG XS decoder |
| 10 | Compressed SRW, Hasselblad 3FR, IIQ, X3F, MRW, KDC, ERF, DCR, MEF, GPR | +0.6 together | 45–85 | ~0.01 | samples; low use |
| 10 | Compressed SRW, Hasselblad 3FR, IIQ, X3F, MRW, KDC, DCR, MEF, GPR (ERF done) | +0.6 together | 45–85 | ~0.01 | samples; low use |
Beyond decoding:
docs/raw/coverage.py
+3 -1
View File
@@ -83,7 +83,9 @@ def classify(maker, camera, ext_field):
return "preview", "SRW", "compressed SRW is not decoded"
return "unverified", "SRW", "uncompressed / packed SRW"
e = es[0] if es else ""
if e in ("CRW", "3FR", "FFF", "IIQ", "MOS", "ERF", "KDC", "MRW", "X3F", "RAW", "ORI"):
if e == "ERF":
return "unverified", "ERF", "ERF decoder (12-bit 16-byte blocks)"
if e in ("CRW", "3FR", "FFF", "IIQ", "MOS", "KDC", "MRW", "X3F", "RAW", "ORI"):
return "preview", e, f"{e} is opened from its embedded JPEG only"
return "unsupported", e, f"{e or 'unknown'} is not imported"
docs/raw/coverage.tsv
+3 -3
View File
@@ -296,9 +296,9 @@ DJI DJI Mavic 4 Pro (DJI FC9284) DNG 2025 unverified DNG decoder (uncompressed,
DJI DJI Mavic 4 Pro (DJI FC9287) DNG 2025 unverified DNG decoder (uncompressed, LJ92, lossy, Deflate, JPEG XL, LinearRaw)
DJI DJI Mavic 4 Pro (Hasselblad L3D-100c) DNG 2025 unverified DNG decoder (uncompressed, LJ92, lossy, Deflate, JPEG XL, LinearRaw)
DxO DxO ONE DNG 2015 unverified DNG decoder (uncompressed, LJ92, lossy, Deflate, JPEG XL, LinearRaw)
Epson R-D1 ERF 2003 preview ERF is opened from its embedded JPEG only
Epson R-D1s ERF 2005 preview ERF is opened from its embedded JPEG only
Epson R-D1x ERF 2008 preview ERF is opened from its embedded JPEG only
Epson R-D1 ERF 2003 verified corpus: crates/raw/tests/corpus.rs (xtask RAW_SAMPLES)
Epson R-D1s ERF 2005 verified corpus: crates/raw/tests/corpus.rs (xtask RAW_SAMPLES)
Epson R-D1x ERF 2008 verified corpus: crates/raw/tests/corpus.rs (xtask RAW_SAMPLES)
Fujifilm GFX 50R RAF 2017 verified corpus: crates/raw/tests/body_corpus.rs (pinned sensor hash, camera-JPEG check)
Fujifilm GFX 50S RAF 2015 verified corpus: crates/raw/tests/corpus.rs (xtask RAW_SAMPLES)
Fujifilm GFX 50S II RAF 2020 verified corpus: crates/raw/tests/body_corpus.rs (pinned sensor hash, camera-JPEG check)
Can't render this file because it contains an unexpected character in line 86 and column 19.
docs/raw/evidence.tsv
+3
View File
@@ -529,3 +529,6 @@ Samsung NX1000 verified corpus: crates/raw/tests/olympus_samsung_corpus.rs (pinn
Samsung NX1100 verified corpus: crates/raw/tests/olympus_samsung_corpus.rs (pinned sensor hash, camera-JPEG check)
Sony A100 (DSLR-A100) unsupported issue #702: regression, refused as not a readable photo
Sony DSC-F828 unsupported issue #702 (SRF refused); SRF not in the import list
Epson R-D1 verified corpus: crates/raw/tests/corpus.rs (xtask RAW_SAMPLES)
Epson R-D1s verified corpus: crates/raw/tests/corpus.rs (xtask RAW_SAMPLES)
Epson R-D1x verified corpus: crates/raw/tests/corpus.rs (xtask RAW_SAMPLES)
Can't render this file because it contains an unexpected character in line 2 and column 3.
docs/target-app-parity.md
+1
View File
@@ -90,6 +90,7 @@ search embeddings; generative remove runs in Adobe's cloud.
| Feature | LightCraft | Status |
|---|---|---|
| 2026-10-12 | minor | Epson ERF decoded (raw-parity.md): 240 models verified, 229 preview-only; shares and rounded numbers unchanged |
| Subject / Sky / Background masks | classical heuristics; the file's own DNG semantic mattes | 🟡 |
| People (faces, body, hair, skin…) and Landscape masks | — | ⬜ |
| Object / Describe masks | SAM 3 in pure Rust (Metal / CPU); opt-in 3.4 GB download, no mirror configured | 🟡 |
xtask/src/main.rs
+3
View File
@@ -2343,6 +2343,9 @@ const RAW_SAMPLES: &[(&str, &str, &str)] = &[
"https://raw.pixls.us/getfile.php/1791/nice/Canon%20-%20EOS%2030D%20-%20RAW%20%283:2%29.CR2",
"fb216f8df8d7abca4eb9d9f8dc594c1d2b69d0a6369fb8b4107743343c37144f",
),
("erf-epson-rd1.erf", "https://raw.pixls.us/data/Epson/R-D1/_EPS0672.ERF", "09d8e533d93116294a9f3e161ed868e929454c7b946e083b644bd6bb75bcb5e4"),
("erf-epson-rd1s.erf", "https://raw.pixls.us/data/Epson/R-D1s/EPSN3575.ERF", "45bb0081baf1c813c7cd9e05b3e6ae734b90aba408dd1195118dc94dc6abbbb9"),
("erf-epson-rd1x.erf", "https://raw.pixls.us/data/Epson/R-D1x/_EPS1919.ERF", "fa1bff3e0b9b9a14f83b8db9fae022c2aca8d455896bf4f25f2c1a0034e295e6"),
];
fn cmd_corpus(download: bool) -> Result<(), String> {