diff --git a/crates/raw/src/llvc.rs b/crates/raw/src/llvc.rs index 0e2cc39cc..ad386157c 100644 --- a/crates/raw/src/llvc.rs +++ b/crates/raw/src/llvc.rs @@ -14,6 +14,7 @@ fn corrupt(why: &str) -> RawError { } /// One bit, failing when the reader has run past the `source_bits` of its half. +#[inline(always)] fn bit(bits: &mut Bits<'_>, source_bits: usize) -> Result { let b = bits.get(1); if bits.consumed_bits() > source_bits { @@ -24,12 +25,14 @@ fn bit(bits: &mut Bits<'_>, source_bits: usize) -> Result { /// Count zero bits up to and including the terminating 1 (the `0^n 1` codes), bounded by `MAX_DPT`. fn zeros_then_one(bits: &mut Bits<'_>, source_bits: usize) -> Result { - let mut n = 0; - while bit(bits, source_bits)? == 0 { - n += 1; - if n > MAX_DPT { - return Err(corrupt("DPT code exceeds MAX_DPT")); - } + // the next MAX_DPT + 1 bits at once (more zeros than that is corrupt either way) + let n = bits.peek(MAX_DPT + 1).leading_zeros() - (32 - (MAX_DPT + 1)); + bits.skip((n + 1).min(MAX_DPT + 1)); + if bits.consumed_bits() > source_bits { + return Err(corrupt("past the end of its half")); + } + if n > MAX_DPT { + return Err(corrupt("DPT code exceeds MAX_DPT")); } Ok(n) } @@ -42,7 +45,7 @@ pub(crate) fn vld_decode_line(bits: &mut Bits<'_>, source_bits: usize, width: us let sets = width.div_ceil(4); let (mut abs, mut dpt, mut cnt, mut dpts) = ([0u32; 4], 0u32, sets, 0u8); let (mut abs_v, mut abs_e, mut dpt_v, mut dpt_e, mut zr) = (false, false, true, false, 0u64); - let mut out: Vec = Vec::new(); + let mut out: Vec = Vec::with_capacity(sets * 4); loop { // DPT value decoding (Figure 6.6). if cnt == 0 { @@ -180,43 +183,56 @@ pub(crate) fn band_rows(n: usize, phase: u8) -> (usize, usize) { (low, n - low) } -/// `v[i]` with `i` clamped into range (the whole-sample symmetric extension of the band); 0 for an empty band. -fn at(v: &[i32], i: isize) -> i32 { - let last = v.len() as isize - 1; - if last < 0 { - return 0; - } - v.get(i.clamp(0, last) as usize).copied().unwrap_or(0) -} - /// Saturating narrowing: hostile coefficients clamp instead of overflowing. fn sat(v: i64) -> i32 { v.clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32 } -/// 1-D inverse 5/3 lifting, RDD 34 §6.5, with X[-1] := X[1], X[n] := X[n-2] (see `band_rows` for the sizes). -fn inverse_1d(l: &[i32], h: &[i32], phase: u8) -> Result> { - let (n, p) = (l.len() + h.len(), isize::from(phase)); - if band_rows(n, phase) != (l.len(), h.len()) || phase > 1 { - return Err(RawError::Corrupt("ARW6 wavelet: band sizes do not match".into())); +/// Sample `i` (`w` values wide) of a band of `n` samples, clamped into range (the whole-sample symmetric extension); +/// `zeros` for an empty band. +#[inline] +fn band_sample<'a>(v: &'a [i32], w: usize, n: usize, i: isize, zeros: &'a [i32]) -> &'a [i32] { + if n == 0 { + return zeros; } - let even: Vec = l - .iter() - .enumerate() - .map(|(j, &lv)| sat(i64::from(lv) - ((i64::from(at(h, j as isize - 1 + p)) + i64::from(at(h, j as isize + p)) + 2) >> 2))) - .collect(); - let mut x = vec![0; n]; - let (lo, hi) = (usize::from(phase), 1 - usize::from(phase)); - for (xv, &e) in x.iter_mut().skip(lo).step_by(2).zip(&even) { - *xv = e; - } - for (k, (xv, &hv)) in x.iter_mut().skip(hi).step_by(2).zip(h).enumerate() { - *xv = sat(i64::from(hv) + ((i64::from(at(&even, k as isize - p)) + i64::from(at(&even, k as isize + 1 - p))) >> 1)); - } - Ok(x) + let i = i.clamp(0, n as isize - 1) as usize; + v.get(i * w..(i + 1) * w).unwrap_or(zeros) } -/// Gathers the lines of the `low`/`high` pair along one axis (columns when `vertical`), transforms them, and scatters back. +/// Inverse 5/3 lifting, RDD 34 §6.5, with X[-1] := X[1], X[n] := X[n-2] (see `band_rows` for the sizes), of a +/// signal whose samples are `w` values wide: `w = 1` for one line, a band's width for all its columns at once (row +/// by row, as each column alone would give). +fn inverse_lift(l: &[i32], h: &[i32], w: usize, phase: u8) -> Result> { + let w = w.max(1); + let (nl, nh) = (l.len() / w, h.len() / w); + let (n, p) = (nl + nh, isize::from(phase)); + if band_rows(n, phase) != (nl, nh) || phase > 1 { + return Err(RawError::Corrupt("ARW6 wavelet: band sizes do not match".into())); + } + let zeros = vec![0; w]; + let mut even = vec![0; nl * w]; + for (j, (e, lv)) in even.chunks_exact_mut(w).zip(l.chunks_exact(w)).enumerate() { + let (a, b) = (band_sample(h, w, nh, j as isize - 1 + p, &zeros), band_sample(h, w, nh, j as isize + p, &zeros)); + for (((e, &lv), &a), &b) in e.iter_mut().zip(lv).zip(a).zip(b) { + *e = sat(i64::from(lv) - ((i64::from(a) + i64::from(b) + 2) >> 2)); + } + } + let mut out = vec![0; n * w]; + let (lo, hi) = (usize::from(phase), 1 - usize::from(phase)); + for (o, e) in out.chunks_exact_mut(w).skip(lo).step_by(2).zip(even.chunks_exact(w)) { + o.copy_from_slice(e); + } + for (k, (o, hv)) in out.chunks_exact_mut(w).skip(hi).step_by(2).zip(h.chunks_exact(w)).enumerate() { + let (a, b) = (band_sample(&even, w, nl, k as isize - p, &zeros), band_sample(&even, w, nl, k as isize + 1 - p, &zeros)); + for (((o, &hv), &a), &b) in o.iter_mut().zip(hv).zip(a).zip(b) { + *o = sat(i64::from(hv) + ((i64::from(a) + i64::from(b)) >> 1)); + } + } + Ok(out) +} + +/// Inverse transform of the `low`/`high` pair along one axis: down the columns when `vertical` (all at once, row by +/// row), else along each row. fn lines(low: &Plane, high: &Plane, phase: u8, vertical: bool) -> Result { if !low.ok() || !high.ok() { return Err(RawError::Corrupt("ARW6 wavelet: plane size mismatch".into())); @@ -228,17 +244,16 @@ fn lines(low: &Plane, high: &Plane, phase: u8, vertical: bool) -> Result let (nl, nh) = if vertical { (low.height, high.height) } else { (low.width, high.width) }; let n = nl + nh; let (w, h) = if vertical { (across_l, n) } else { (n, across_l) }; + if across_l == 0 { + return Ok(Plane::zeros(w, h)); + } + if vertical { + return Ok(Plane { width: w, height: h, data: inverse_lift(&low.data, &high.data, w, phase)? }); + } let mut out = Plane::zeros(w, h); - let get = |p: &Plane, i: usize| -> Vec { - if vertical { p.data.iter().skip(i).step_by(p.width.max(1)).copied().collect() } else { p.row(i).map(<[i32]>::to_vec).unwrap_or_default() } - }; for i in 0..across_l { - let x = inverse_1d(&get(low, i), &get(high, i), phase)?; - if vertical { - for (o, v) in out.data.iter_mut().skip(i).step_by(w.max(1)).zip(x) { - *o = v; - } - } else if let Some(r) = out.data.get_mut(i * w..(i + 1) * w) { + let x = inverse_lift(low.row(i).unwrap_or_default(), high.row(i).unwrap_or_default(), 1, phase)?; + if let Some(r) = out.data.get_mut(i * w..(i + 1) * w) { r.copy_from_slice(&x); } } diff --git a/crates/raw/src/vendor/arw6.rs b/crates/raw/src/vendor/arw6.rs index f6f535bd6..cb903a6f3 100644 --- a/crates/raw/src/vendor/arw6.rs +++ b/crates/raw/src/vendor/arw6.rs @@ -468,36 +468,36 @@ pub(crate) fn colour(p: &TilePlanes) -> Result> { } let mw = w2.checked_mul(2).ok_or_else(|| corrupt("tile size"))?; let total = len.checked_mul(4).ok_or_else(|| corrupt("tile size"))?; - let at = |q: &Plane, j: usize, x: usize| -> i64 { q.data.get(j * w2 + x).copied().unwrap_or(0) as i64 }; + // (rows as slices: `at` is 0 past a row's end, as never happens once the sizes above match) + let at = |r: &[i32], x: usize| i64::from(r.get(x).copied().unwrap_or(0)); let mut g1 = vec![0i64; len]; - for j in 0..h2 { - let jp = j.saturating_sub(1); - for x in 0..w2 { + g1.par_chunks_mut(w2.max(1)).enumerate().for_each(|(j, row)| { + let (Some(m), Some(rp), Some(rj)) = (p.m.row(j), p.res.row(j.saturating_sub(1)), p.res.row(j)) else { return }; + for (x, g) in row.iter_mut().enumerate() { let xn = (x + 1).min(w2 - 1); - let r = at(&p.res, jp, x) + at(&p.res, jp, xn) + at(&p.res, j, x) + at(&p.res, j, xn); - if let Some(g) = g1.get_mut(j * w2 + x) { - *g = at(&p.m, j, x) - ((r + 4) >> 3); - } + let r = at(rp, x) + at(rp, xn) + at(rj, x) + at(rj, xn); + *g = at(m, x) - ((r + 4) >> 3); } - } - let g = |j: usize, x: usize| g1.get(j * w2 + x).copied().unwrap_or(0); + }); + let g1_row = |j: usize| g1.get(j * w2..(j + 1) * w2).unwrap_or(&[]); + let g = |r: &[i64], x: usize| r.get(x).copied().unwrap_or(0); let mut out = vec![0u16; total]; - for j in 0..h2 { - let jn = (j + 1).min(h2 - 1); - for x in 0..w2 { + // each plane row gives two mosaic rows + out.par_chunks_mut((2 * mw).max(1)).enumerate().for_each(|(j, rows)| { + let (gj, gn) = (g1_row(j), g1_row((j + 1).min(h2 - 1))); + let (Some(res), Some(c1), Some(c2), Some((top, bottom))) = (p.res.row(j), p.c1.row(j), p.c2.row(j), rows.split_at_mut_checked(mw)) else { + return; + }; + for (x, (t, b)) in top.as_chunks_mut::<2>().0.iter_mut().zip(bottom.as_chunks_mut::<2>().0.iter_mut()).enumerate() { let xp = x.saturating_sub(1); - let g2 = ((g(j, xp) + g(j, x) + g(jn, xp) + g(jn, x)) >> 2) + at(&p.res, j, x); - let mean = (g(j, x).clamp(0, 4095) + g2.clamp(0, 4095)) >> 1; - let r = 2 * at(&p.c1, j, x) + mean; - let b = 2 * at(&p.c2, j, x) + mean; - let cells = [(0, r), (1, g(j, x)), (mw, g2), (mw + 1, b)]; - for (off, v) in cells { - if let Some(o) = out.get_mut(2 * j * mw + 2 * x + off) { - *o = curve(v.clamp(0, 4095) as i32); - } + let g2 = ((g(gj, xp) + g(gj, x) + g(gn, xp) + g(gn, x)) >> 2) + at(res, x); + let mean = (g(gj, x).clamp(0, 4095) + g2.clamp(0, 4095)) >> 1; + let cells = [2 * at(c1, x) + mean, g(gj, x), g2, 2 * at(c2, x) + mean]; + for (o, v) in t.iter_mut().chain(b.iter_mut()).zip(cells) { + *o = curve(v.clamp(0, 4095) as i32); } } - } + }); Ok(out) } diff --git a/docs/arw6-compression.md b/docs/arw6-compression.md index 3b65c1d37..a45f8365f 100644 --- a/docs/arw6-compression.md +++ b/docs/arw6-compression.md @@ -171,4 +171,13 @@ Raw decode only, 4 CPUs, release build, best of 3 wall-clock (2026-10-10). Metho | ILCE-7RM6 APS-C, Compressed (HQ) | 29.7 MB | 6592x4372 | 0.35 s | | ILCE-7RM6 APS-C, Compressed | 30.9 MB | 6592x4372 | 0.34 s | -The 1.0 s budget for the full-frame file is met. Parallelised: rayon over the tiles, over the 13 streams of each tile and over its 3 components. +The 1.0 s budget for the full-frame file is met. Parallelised: rayon over the tiles, over the 13 streams of each tile and over its 3 components, and the colour reconstruction over rows. + +2026-10-12 performance pass (output unchanged, bit for bit): the vertical wavelet pass lifts whole rows instead of gathering each column, `0^n 1` codes are counted with one `leading_zeros`, and the colour reconstruction reads rows as slices, in parallel. Same method, best of 5, the previous and the new decoder run alternately on the same machine (4 CPUs, which was busier than on 2026-10-10), best of two rounds: + +| File | Before | After | +|---|---|---| +| ILCE-7RM6 FF, Compressed (HQ) | 0.56 s | 0.46 s | +| ILCE-7RM6 FF, Compressed | 0.54 s | 0.38 s | +| ILCE-7RM6 APS-C, Compressed (HQ) | 0.34 s | 0.23 s | +| ILCE-7RM6 APS-C, Compressed | 0.35 s | 0.24 s |