diff --git a/crates/raw/proptest-regressions/vendor/rafc.txt b/crates/raw/proptest-regressions/vendor/rafc.txt new file mode 100644 index 000000000..7633abe0f --- /dev/null +++ b/crates/raw/proptest-regressions/vendor/rafc.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 8cb6b5999a8279e655a148a18b220d75b02c519f644583f4593166286297a3f4 # shrinks to data = [0, 0, 0, 8], reads = [(false, 13, 0), (true, 0, 15)] diff --git a/crates/raw/src/vendor/rafc.rs b/crates/raw/src/vendor/rafc.rs index f3b925b51..fd08f9869 100644 --- a/crates/raw/src/vendor/rafc.rs +++ b/crates/raw/src/vendor/rafc.rs @@ -114,26 +114,57 @@ impl<'a> Bits<'a> { Self { src, pos: 0, acc: 0, available: 0 } } + /// Tops the accumulator up to at least 57 bits, or to every remaining byte near the end of the stream. #[inline] - fn get(&mut self, n: u32) -> Result { - while self.available < n { - let b = self.src.get(self.pos).copied().ok_or_else(|| corrupt("truncated entropy stream"))?; + fn refill(&mut self) { + while self.available <= 56 { + let Some(&b) = self.src.get(self.pos) else { return }; self.pos += 1; self.acc = (self.acc << 8) | u64::from(b); self.available += 8; } + } + + #[inline] + fn get(&mut self, n: u32) -> Result { + if self.available < n { + self.refill(); + if self.available < n { + return Err(corrupt("truncated entropy stream")); + } + } self.available -= n; Ok(((self.acc >> self.available) & ((1u64 << n) - 1)) as u32) } - fn residual(&mut self, k: u32, bits: u32, range: u32, limit: u32) -> Result { + /// Reads the unary prefix: the zeros before the next one bit, which is consumed. Errors exactly where reading + /// bit by bit would: more than `limit` zeros, or a stream that ends first. + #[inline] + fn zeros(&mut self, limit: u32) -> Result { let mut zeros = 0; - while self.get(1)? == 0 { - zeros += 1; - if zeros > limit { + loop { + if self.available == 0 { + self.refill(); + if self.available == 0 { + return Err(corrupt("truncated entropy stream")); + } + } + // the unread bits, left-aligned; the zeros shifted in below them are cut off by `min` + let run = (self.acc << (64 - self.available)).leading_zeros().min(self.available); + if zeros + run > limit { return Err(corrupt("invalid residual escape")); } + zeros += run; + if run < self.available { + self.available -= run + 1; + return Ok(zeros); + } + self.available = 0; } + } + + fn residual(&mut self, k: u32, bits: u32, range: u32, limit: u32) -> Result { + let zeros = self.zeros(limit)?; let code = if zeros == limit { self.get(bits)? + 1 } else { (zeros << k) | self.get(k)? }; if code > range { return Err(corrupt(&format!( @@ -501,5 +532,48 @@ mod tests { src.extend_from_slice(&data); let _ = decode(&src, 780, 6, bits, xtrans, Mode::Full); } + + /// The word-at-a-time reader returns what reading bit by bit returns, and fails where it fails: long zero + /// runs, the escape limit and the end of the stream included. + #[test] + fn bit_reader_matches_reading_bit_by_bit( + data in prop::collection::vec(prop_oneof![3 => Just(0u8), 1 => any::()], 0..64), + reads in prop::collection::vec((any::(), 0u32..=24, 0u32..=63), 1..200), + ) { + let bit = |i: usize| data.get(i / 8).map(|b| (b >> (7 - i % 8)) & 1 == 1); + let mut at = 0usize; + let mut reader = Bits::new(&data); + for (prefix, n, limit) in reads { + let expected = if prefix { + let mut zeros = 0; + loop { + let Some(b) = bit(at) else { break Err("truncated") }; + at += 1; + if b { + break Ok(zeros); + } + zeros += 1; + if zeros > limit { + break Err("escape"); + } + } + } else if at + n as usize > data.len() * 8 { + Err("truncated") + } else { + let value = (0..n as usize).fold(0, |v, i| (v << 1) | u32::from(bit(at + i) == Some(true))); + at += n as usize; + Ok(value) + }; + let actual = if prefix { reader.zeros(limit) } else { reader.get(n) }; + match (expected, actual) { + (Ok(want), Ok(got)) => prop_assert_eq!(want, got), + (Err(want), Err(got)) => { + prop_assert!(got.to_string().contains(want), "{got}"); + break; + } + (want, got) => prop_assert!(false, "expected {want:?}, got {got:?}"), + } + } + } } } diff --git a/docs/raf-compression.md b/docs/raf-compression.md index 0141d893e..29056502e 100644 --- a/docs/raf-compression.md +++ b/docs/raf-compression.md @@ -17,7 +17,7 @@ Fujifilm now uses a guarded colour/tone estimate from each file’s own embedded - Signed prediction errors use adaptive Rice coding: a unary prefix followed by a context-dependent number of bits, with a bounded escape for large errors. Three pair phases have separate even/odd statistics. The 81 signed gradient contexts fold into 41 states. - Lossy quantizers change the error range and reconstruction step. Low-gradient sites retain finer precision (0, 1 or 2), with five folded contexts per precision and statistics that survive main-quantizer changes. The main context statistics reset when its quantizer changes. Their initial sum is `max(2, (range + 32) / 64)`; using 16 as the minimum breaks more heavily quantized files. -Stripes decode in parallel using the existing Rayon dependency. Offsets, sizes, dimensions and entropy reads are checked; truncated data, impossible codes and inconsistent headers return errors. Unknown container compression modes, disagreements between container and stream modes, and compressed X-Trans layouts other than the supported CFA return errors. Header-only probing validates the tables and stripe ranges without decompressing the image. +Stripes decode in parallel using the existing Rayon dependency. The entropy reader keeps up to 64 bits buffered and counts a unary prefix's zeros in one step instead of bit by bit; a property test checks it against reading bit by bit, errors included. Offsets, sizes, dimensions and entropy reads are checked; truncated data, impossible codes and inconsistent headers return errors. Unknown container compression modes, disagreements between container and stream modes, and compressed X-Trans layouts other than the supported CFA return errors. Header-only probing validates the tables and stripe ranges without decompressing the image. ## Verification @@ -36,7 +36,7 @@ The 17 CC0 files live in the gitignored corpus. Their published SHA-256 identiti Unconditional procedural tests cover Bayer/X-Trans, 12/14/16 bits, partial final stripes, quantizer changes, residual signs and escapes, malformed headers and truncated/random entropy streams. Regressions also check overflowing uncompressed row strides, unsupported modes and CFA layouts, and container/stream mode mismatches. -Local release decode measurements: about 0.16–0.24 seconds for 24–40 MP X-Trans files, 0.29 seconds for 50 MP Bayer, and 0.5–0.6 seconds for 100 MP Bayer. These measure sensor decoding, not the full develop/export pipeline. +Local release decode measurements: about 0.16–0.24 seconds for 24–40 MP X-Trans files, 0.29 seconds for 50 MP Bayer, and 0.5–0.6 seconds for 100 MP Bayer. These measure sensor decoding, not the full develop/export pipeline. The buffered prefix count took 13–17 % off the decode time on an Apple M2 (8 cores, minimum of 5 runs, old and new build alternated): X-T2 178 → 152 ms, X-S10 lossless 265 → 226 ms, X-E5 442 → 385 ms, X-H2 448 → 378 ms, GFX 50S 522 → 435 ms, GFX 100 762 → 644 ms, with sensor arrays unchanged bit for bit. App verification imported all four supplied compressed files, confirmed `previewOnly: null`, applied +2 EV exposure edits and inspected headless UI screenshots. CLI JPEG exports at a 1600-pixel long edge succeeded for all four files (0.58–0.76 seconds each, including process startup).