This commit is contained in:
Brandon Thomas
2026-10-08 22:43:10 -04:00
5 changed files with 186 additions and 30 deletions
+1 -1
View File
@@ -62,7 +62,7 @@ impl SettingsHashes {
}
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
pub const RENDER_CACHE_VERSION: u64 = 15;
pub const RENDER_CACHE_VERSION: u64 = 16;
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
+5
View File
@@ -16,6 +16,9 @@ pub enum Packing {
Lsb,
/// Each sample stored in 16 bits in file byte order, regardless of `bits`.
Word16,
/// MSB-first bit stream inside 32-bit words stored in file byte order ([`unpack_words32_msb`]); rows start on a
/// word boundary.
Words32Msb,
}
enum ChunkPx {
@@ -263,6 +266,7 @@ fn unpack_chunk(
(16, _) | (_, Packing::Word16) => (row_n * 2, 1),
(_, Packing::Msb) => ((row_n * bits as usize).div_ceil(8), 2),
(_, Packing::Lsb) => ((row_n * bits as usize).div_ceil(8), 3),
(_, Packing::Words32Msb) => ((row_n * bits as usize).div_ceil(32) * 4, 4),
};
for y in 0..ch {
let s = src.get(y * row_bytes..).unwrap_or(&[]);
@@ -272,6 +276,7 @@ fn unpack_chunk(
0 => row.iter_mut().zip(s).for_each(|(o, &b)| *o = b as u16),
1 => read_u16s(s, order, row),
2 => unpack_msb(s, bits, row),
4 => unpack_words32_msb(s, order, bits, row),
_ => unpack_lsb(s, bits, row),
}
if factor > 0 {
+37
View File
@@ -40,6 +40,19 @@ pub fn unpack_lsb(src: &[u8], bits: u32, out: &mut [u16]) {
}
}
/// Unpack MSB-first packed samples of `bits` from a stream of 32-bit words stored in `order` (a word is read as a
/// number first, then its bits are taken from the top): the layout of some cameras' uncompressed 12- and 14-bit
/// data, which is not the byte-wise MSB stream of [`unpack_msb`] unless `order` is big-endian. A short last word
/// is padded with zeros.
pub fn unpack_words32_msb(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16]) {
let words = src.chunks(4).flat_map(|c| {
let mut w = [0u8; 4];
w[..c.len()].copy_from_slice(c);
order.u32(w).to_be_bytes()
});
unpack_msb(&words.collect::<Vec<u8>>(), bits, out);
}
pub fn read_u16s(src: &[u8], order: ByteOrder, out: &mut [u16]) {
for (o, c) in out.iter_mut().zip(src.as_chunks::<2>().0) {
*o = order.u16([c[0], c[1]]);
@@ -128,6 +141,30 @@ pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride:
mod tests {
use super::*;
#[test]
fn words32_are_read_as_numbers_then_msb_first() {
// eight 12-bit samples occupy 96 bits = three 32-bit words, filled from the top
let samples: [u16; 8] = [0x123, 0x456, 0x789, 0xabc, 0xdef, 0x012, 0x345, 0x678];
let stream = samples.iter().fold(0u128, |acc, &v| (acc << 12) | v as u128);
let words = [(stream >> 64) as u32, (stream >> 32) as u32, stream as u32];
let le: Vec<u8> = words.iter().flat_map(|w| w.to_le_bytes()).collect();
let be: Vec<u8> = words.iter().flat_map(|w| w.to_be_bytes()).collect();
let mut o = [0u16; 8];
unpack_words32_msb(&le, ByteOrder::Little, 12, &mut o);
assert_eq!(o, samples);
o = [0; 8];
unpack_words32_msb(&be, ByteOrder::Big, 12, &mut o);
assert_eq!(o, samples);
// read as a plain byte stream the little-endian words give different samples
o = [0; 8];
unpack_msb(&le, 12, &mut o);
assert_ne!(o, samples);
// a short last word is zero padded
let mut o = [9u16; 2];
unpack_words32_msb(&[0x34, 0x12], ByteOrder::Little, 12, &mut o);
assert_eq!(o, [0x000, 0x012]); // the word 0x00001234, top bits first
}
#[test]
fn packing() {
// 12-bit MSB: 0xABC, 0xDEF -> AB CD EF
+93 -8
View File
@@ -18,6 +18,25 @@ const WB_RB_LEVELS: u16 = 0x000c;
const BLACK_LEVEL: u16 = 0x003d;
const LINEARIZATION_TABLE: u16 = 0x0096;
const CROP_AREA: u16 = 0x0045;
/// Maker note `0x0014`, the "NRW" data block of Coolpix raws: `"NRW "` + a 4-character version, then fields. In
/// version `0104` the `u32` at byte `0x20` is the black level in sample units (200 in the four 12-bit files here
/// that have 200 as their darkest samples, 0 in the one whose samples reach 0).
const NRW_DATA: u16 = 0x0014;
const NRW_BLACK_AT: usize = 0x20;
/// The NRW data block, when the maker note has one.
fn nrw_data(mn: Option<&makernote::MakerNote>) -> Option<(&[u8], ByteOrder)> {
let m = mn?;
m.ifd.bytes(NRW_DATA).filter(|b| b.starts_with(b"NRW ")).map(|b| (b, m.order))
}
/// Black level from the NRW data block, bounded to a quarter of the sample range so an unknown version's field
/// can't turn into a large offset.
fn nrw_black(mn: Option<&makernote::MakerNote>, bits: u32) -> Option<f32> {
let (b, order) = nrw_data(mn)?;
let v = order.u32(b.get(NRW_BLACK_AT..NRW_BLACK_AT + 4)?.try_into().ok()?);
(u64::from(v) < (1u64 << bits.min(16)) / 4).then_some(v as f32)
}
/// Nikon CropArea is `[left, top, width, height]` in sensor pixels (maker-note tag documentation).
/// Keep the active area/CFA origin unchanged: the default crop is applied after demosaicing.
@@ -65,8 +84,11 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
let make = ifd0.string(t::MAKE).unwrap_or_default();
let mn =
tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make));
let data =
if info.compression == t::compression::NIKON { compressed(bytes, &info, mn.as_ref())? } else { uncompressed(bytes, &info, tiff.order)? };
let data = if info.compression == t::compression::NIKON {
compressed(bytes, &info, mn.as_ref())?
} else {
uncompressed(bytes, &info, tiff.order, mn.as_ref())?
};
let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float NEF".into())) };
// Maker note 0x003d is in 14-bit units whatever the sample depth: 12-bit files from the D750, D780, D850,
@@ -80,7 +102,7 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
..Default::default()
},
_ => BlackLevel::uniform(0.0),
_ => BlackLevel::uniform(nrw_black(mn.as_ref(), bits).unwrap_or(0.0)),
};
let wb = mn
.as_ref()
@@ -120,8 +142,10 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
Ok(img)
}
/// Uncompressed strips: 16-bit words or 12-bit MSB-packed, told apart by the strip size.
fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder) -> Result<RawData> {
/// Uncompressed strips: 16-bit words or 12-bit packed, told apart by the strip size. Packed 12-bit data is an MSB-first
/// byte stream, except in files with an NRW data block (Coolpix), where it is MSB-first inside little-endian 32-bit
/// words (the rows are whole words).
fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder, mn: Option<&makernote::MakerNote>) -> Result<RawData> {
let (w, h) = (info.width as usize, info.height as usize);
let bits = info.bits() as u32;
let row_samples = w * info.samples_per_pixel as usize;
@@ -131,7 +155,7 @@ fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder) -> Result<RawD
let packing = if bytes_per_row >= row_samples * 2 {
Packing::Word16
} else if bits == 12 && bytes_per_row * 8 >= row_samples * 12 && bytes_per_row * 8 < row_samples * 13 {
Packing::Msb
if nrw_data(mn).is_some() && bytes_per_row.is_multiple_of(4) { Packing::Words32Msb } else { Packing::Msb }
} else if info.compression == 1 && bits != 8 && bits != 16 {
return Err(RawError::Unsupported(format!("NEF uncompressed packing ({bytes_per_row} bytes per {row_samples}-sample row)")));
} else {
@@ -171,6 +195,12 @@ mod tests {
/// [`nef`] with an optional maker note (`Nikon\0` v2 header + embedded big-endian TIFF holding `note`).
fn nef_with_note(compression: u16, bits: u16, strips: Vec<Vec<u8>>, w: u32, h: u32, rps: u32, note: Option<IfdBuilder>) -> Vec<u8> {
nef_in(ByteOrder::Big, compression, bits, strips, w, h, rps, note)
}
/// [`nef_with_note`] in either byte order (file and maker note).
#[allow(clippy::too_many_arguments)]
fn nef_in(order: ByteOrder, compression: u16, bits: u16, strips: Vec<Vec<u8>>, w: u32, h: u32, rps: u32, note: Option<IfdBuilder>) -> Vec<u8> {
let mut raw = IfdBuilder::new();
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
raw.set(t::IMAGE_WIDTH, Value::Long(vec![w]));
@@ -186,13 +216,13 @@ mod tests {
ifd0.set(t::MODEL, Value::Ascii("NIKON TEST".into()));
if let Some(mn) = note {
let mut bytes = b"Nikon\0\x02\x10\0\0".to_vec();
bytes.extend(TiffWriter::new(ByteOrder::Big, false).write(&[mn]).unwrap());
bytes.extend(TiffWriter::new(order, false).write(&[mn]).unwrap());
let mut exif = IfdBuilder::new();
exif.set(t::MAKER_NOTE, Value::Undefined(bytes));
ifd0.set_child(t::EXIF_IFD, exif);
}
ifd0.add_sub_ifd(raw);
TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap()
TiffWriter::new(order, false).write(&[ifd0]).unwrap()
}
#[test]
@@ -266,6 +296,61 @@ mod tests {
assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px));
}
/// An NRW data block (`"NRW 0104"`, fields from byte 8) with `black` at byte `0x20`, as a maker note.
fn nrw_note(black: u32) -> IfdBuilder {
let mut block = b"NRW 0104".to_vec();
block.resize(0x20, 0);
block.extend_from_slice(&black.to_le_bytes());
block.resize(0x40, 0);
let mut mn = IfdBuilder::new();
mn.set(NRW_DATA, Value::Undefined(block));
mn
}
/// 12-bit samples as an MSB-first bit stream inside 32-bit words in `order` (rows are whole words).
fn words32(px: &[u16], w: usize, order: ByteOrder) -> Vec<u8> {
let mut out = Vec::new();
for row in px.chunks(w) {
let stream = row.iter().fold(Vec::new(), |mut bits: Vec<bool>, v| {
bits.extend((0..12).rev().map(|b| v >> b & 1 == 1));
bits
});
for word in stream.chunks(32) {
let v = word.iter().enumerate().fold(0u32, |a, (i, b)| a | (*b as u32) << (31 - i));
out.extend_from_slice(&match order {
ByteOrder::Little => v.to_le_bytes(),
ByteOrder::Big => v.to_be_bytes(),
});
}
}
out
}
#[test]
fn nrw_block_selects_32_bit_word_packing_and_supplies_the_black_level() {
let (w, h) = (16usize, 6usize);
let px: Vec<u16> = (0..w * h).map(|i| (200 + i * 131 % 3800) as u16).collect();
let packed = words32(&px, w, ByteOrder::Little);
assert_eq!(packed.len(), w * h * 3 / 2);
let decode = |note: Option<IfdBuilder>| {
crate::decode(&nef_in(ByteOrder::Little, 1, 12, vec![packed.clone()], w as u32, h as u32, h as u32, note)).unwrap()
};
let r = decode(Some(nrw_note(200)));
assert_eq!(r.data, RawData::U16(px.clone()));
assert_eq!(r.black.values, vec![200.0]);
assert_eq!(decode(Some(nrw_note(0))).black.values, vec![0.0]);
// an implausible field is not a black level
assert_eq!(decode(Some(nrw_note(60000))).black.values, vec![0.0]);
// without the block the same bytes are an MSB-first byte stream
let plain = decode(None);
assert_ne!(plain.data, RawData::U16(px));
assert_eq!(plain.black.values, vec![0.0]);
// the maker-note black level (14-bit units) wins when present
let mut both = nrw_note(200);
both.set(BLACK_LEVEL, Value::Short(vec![400, 404, 408, 412]));
assert_eq!(decode(Some(both)).black.values, vec![100.0, 101.0, 102.0, 103.0]);
}
#[test]
fn compressed_without_table_is_unsupported() {
let bytes = nef(34713, 14, vec![vec![0; 64]], 8, 8, 8);
+50 -21
View File
@@ -12,6 +12,8 @@
//! IFD; offsets in it are relative to the raw block.
//! - Sample packing, found by testing candidate bit orders for the smoothest image: 16-bit little-endian words;
//! 12-bit LSB-first bit stream; 14-bit (and other depths) stored as little-endian 32-bit words read MSB-first.
//! Raw-IFD flag `0xf006` = 1 selects the 32-bit-word packing at any depth: 12-bit files with the flag set
//! (FinePix S1, X-A10, SL1000) are word-packed like the 14-bit ones, the 12-bit files without it are LSB-first.
//! `0xf005`, when non-zero, is the row stride in 32-bit words.
//! - CFA: the X-Trans layout record (36 bytes, 0 = R, 1 = G, 2 = B) is stored in reverse order and, reversed,
//! is anchored at raw pixel (0, 0) — verified on X-Trans I and III samples by minimising the difference
@@ -20,7 +22,7 @@
//! - Striped predictive compression is decoded by `rafc`; see `docs/raf-compression.md`.
use super::white_from_data;
use crate::unpack::{unpack_lsb, unpack_msb};
use crate::unpack::{unpack_lsb, unpack_words32_msb};
use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
use lightcraft_geom::Orientation;
use lightcraft_tiff::{ByteOrder, Ifd, Tiff};
@@ -29,6 +31,9 @@ use rayon::prelude::*;
const CROP_TOP_LEFT: u16 = 0x0110;
const CROPPED_SIZE: u16 = 0x0111;
const XTRANS_LAYOUT: u16 = 0x0131;
/// Raw-IFD flag `0xf006`: 1 in every 12- and 14-bit file here whose samples are packed into 32-bit words (the 12-bit
/// ones are otherwise an LSB-first stream), 0 where they are an LSB-first stream or 16-bit words.
const PACKED_WORDS: u16 = 0xf006;
/// The fixed-position header pointers.
pub(crate) struct Header<'a> {
@@ -95,22 +100,14 @@ fn cfa(h: &Header) -> Cfa {
}
}
/// Unpack one row of `bits`-bit samples in Fujifilm's packing.
pub(crate) fn unpack_row(src: &[u8], bits: u32, out: &mut [u16]) {
/// Unpack one row of `bits`-bit samples in Fujifilm's packing. `words32`: the raw IFD's `0xf006` flag is set, the
/// samples are MSB-first in little-endian 32-bit words whatever the depth; otherwise 16-bit words, a 12-bit
/// LSB-first bit stream, or (other depths) the same 32-bit words.
pub(crate) fn unpack_row(src: &[u8], bits: u32, words32: bool, out: &mut [u16]) {
match bits {
16 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])),
12 => unpack_lsb(src, 12, out),
_ => {
let swapped: Vec<u8> = src
.chunks(4)
.flat_map(|c| {
let mut w = [0u8; 4];
w[..c.len()].copy_from_slice(c);
[w[3], w[2], w[1], w[0]]
})
.collect();
unpack_msb(&swapped, bits, out)
}
12 if !words32 => unpack_lsb(src, 12, out),
_ => unpack_words32_msb(src, ByteOrder::Little, bits, out),
}
}
@@ -147,6 +144,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
}
data = super::rafc::decode(src, w, hgt, bits, cfa.width == 6, mode)?;
} else {
let words32 = ifd.u64(PACKED_WORDS).is_some_and(|v| v != 0);
// Samples may use 16-bit words (e.g. 14-bit X-T20); otherwise they are packed at `bits`.
let store = if len as u64 >= (n as u64) * 2 { 16 } else { bits };
let packed_row = w.checked_mul(store as usize).map(|n| n.div_ceil(8)).ok_or(RawError::Limit("RAF row too large"))?;
@@ -163,7 +161,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
data.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
// `required <= src.len()` proves these row offsets fit, including the final row.
let s = src.get(y * stride..y * stride + packed_row).unwrap_or(&[]);
unpack_row(s, store, row);
unpack_row(s, store, words32, row);
});
}
}
@@ -220,19 +218,25 @@ pub(super) mod tests {
/// Build a minimal RAF: header, records, raw block (LE TIFF whose IFD0 points to a 0xf000 raw IFD).
pub(crate) fn raf(w: u32, h: u32, bits: u32, strip: Vec<u8>, layout: Option<[u8; 36]>, jpeg: &[u8]) -> Vec<u8> {
raf_flagged(w, h, bits, strip, layout, jpeg, 0)
}
/// [`raf`] with the raw IFD's `0xf006` flag set to `words32`.
pub(crate) fn raf_flagged(w: u32, h: u32, bits: u32, strip: Vec<u8>, layout: Option<[u8; 36]>, jpeg: &[u8], words32: u32) -> Vec<u8> {
let mut block = b"II*\0\x08\0\0\0".to_vec();
let sub_off: u32 = 8 + 2 + 12 + 4;
block.extend_from_slice(&1u16.to_le_bytes());
block.extend_from_slice(&[0x00, 0xf0, 13, 0, 1, 0, 0, 0]);
block.extend_from_slice(&sub_off.to_le_bytes());
block.extend_from_slice(&0u32.to_le_bytes());
let n = 7u32;
let n = 8u32;
let wb_off = sub_off + 2 + 12 * n + 4;
let strip_off = wb_off + 12;
let entries: [(u16, u32, u32); 7] = [
let entries: [(u16, u32, u32); 8] = [
(0xf001, 1, w),
(0xf002, 1, h),
(0xf003, 1, bits),
(0xf006, 1, words32),
(0xf007, 1, strip_off),
(0xf008, 1, strip.len() as u32),
(0xf00a, 1, 64),
@@ -345,14 +349,39 @@ pub(super) mod tests {
assert!(matches!(crate::decode(&bytes), Err(RawError::Unsupported(_))));
}
/// Raw IFD flag `0xf006` selects the 32-bit-word packing at 12 bits; without it 12 bits are an LSB-first stream.
#[test]
fn flag_selects_word_packed_12_bit() {
let (w, h) = (16u32, 6u32);
let px: Vec<u16> = (0..w * h).map(|i| ((i * 977) % 4096) as u16).collect();
// the same 12-bit samples as little-endian 32-bit words (what the 14-bit branch of `pack` writes)
let mut stream: Vec<u8> = Vec::new();
let (mut acc, mut n) = (0u64, 0u32);
for &v in &px {
acc = (acc << 12) | v as u64;
n += 12;
while n >= 8 {
stream.push((acc >> (n - 8)) as u8);
n -= 8;
}
}
let le: Vec<u8> = stream.chunks(4).flat_map(|c| [c[3], c[2], c[1], c[0]]).collect();
let decode = |bytes: Vec<u8>| crate::decode(&bytes).unwrap().data;
assert_eq!(decode(raf_flagged(w, h, 12, le.clone(), None, b"", 1)), RawData::U16(px.clone()));
// flag clear: the same bytes are read as the LSB-first stream, which gives other samples
assert_ne!(decode(raf_flagged(w, h, 12, le, None, b"", 0)), RawData::U16(px.clone()));
// flag clear: the LSB-first stream still decodes
assert_eq!(decode(raf_flagged(w, h, 12, pack(&px, 12), None, b"", 0)), RawData::U16(px));
}
#[test]
fn overflowing_row_stride_is_an_error() {
let mut bytes = raf(16, 6, 16, vec![0; 192], None, &[]);
let off = be32(&bytes, 100).unwrap();
// Replace WB with a LONG8 stride, pointing to the existing out-of-line WB storage.
bytes[off + 100..off + 104].copy_from_slice(&[0x05, 0xf0, 16, 0]);
bytes[off + 104..off + 108].copy_from_slice(&1u32.to_le_bytes());
bytes[off + 116..off + 124].copy_from_slice(&u64::MAX.to_le_bytes());
bytes[off + 112..off + 116].copy_from_slice(&[0x05, 0xf0, 16, 0]);
bytes[off + 116..off + 120].copy_from_slice(&1u32.to_le_bytes());
bytes[off + 128..off + 136].copy_from_slice(&u64::MAX.to_le_bytes());
assert!(matches!(crate::decode(&bytes), Err(RawError::Limit(_))));
}
}