refactor(av1): split inverse transform into stages and passes (S3776)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/av1/src/transform.rs

@@ -165,6 +165,15 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..len {
         t[i] = copy[brev(n, i)];
     }
+    inverse_dct_stage1(t, n, r);
+    inverse_dct_stage2(t, n, r);
+    inverse_dct_stage3(t, n, r);
+    inverse_dct_stage4(t, n, r);
+    inverse_dct_stage5(t, n, r);
+    inverse_dct_stage6(t, n, r);
+}
+
+fn inverse_dct_stage1<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n == 6 {
         for i in 0..16 {
             bf(t, 32 + i, 63 - i, 63 - 4 * brev(4, i) as i32, false);
@@ -190,6 +199,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             hd(t, 16 + 2 * i, 17 + 2 * i, i & 1 == 1, r);
         }
     }
+}
+
+fn inverse_dct_stage2<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n == 6 {
         for i in 0..4 {
             for j in 0..2 {
@@ -221,6 +233,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
+}
+
+fn inverse_dct_stage3<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..2 {
         bf(t, 2 * i, 2 * i + 1, 32 + 16 * i as i32, i == 0);
     }
@@ -248,6 +263,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
+}
+
+fn inverse_dct_stage4<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..2 {
         hd(t, i, 3 - i, false, r);
     }
@@ -273,6 +291,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
+}
+
+fn inverse_dct_stage5<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n >= 3 {
         for i in 0..4 {
             hd(t, i, 7 - i, false, r);
@@ -300,6 +321,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             hd(t, i, 15 - i, false, r);
         }
     }
+}
+
+fn inverse_dct_stage6<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n >= 5 {
         for i in 0..4 {
             bf(t, 27 - i, 20 + i, 32, true);
@@ -531,10 +555,9 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
     let col_clamp = (bit_depth + 6).max(16);
     let rk = row_kind(tx_type);
     let ck = col_kind(tx_type);
-    let mut t = [0i32; 64];
     let rect2 = log2w.abs_diff(log2h) == 1;
+    let p = Tx2d { w, h, log2w, log2h, row_shift, col_shift, row_clamp, col_clamp, rk, ck, rect2 };
     if !lossless && bit_depth <= 10 {
-        let p = Tx2d { w, h, log2w, log2h, row_shift, col_shift, row_clamp, col_clamp, rk, ck, rect2 };
         if w >= 8 && h >= 8 {
             transform_lanes::<8>(dequant, residual, &p);
         } else {
@@ -542,6 +565,13 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
         }
         return;
     }
+    scalar_row_pass(dequant, residual, &p, lossless);
+    scalar_col_pass(residual, &p, lossless);
+}
+
+fn scalar_row_pass(dequant: &[i32], residual: &mut [i32], p: &Tx2d, lossless: bool) {
+    let (w, h, log2w, row_shift, row_clamp, col_clamp, rk, rect2) = (p.w, p.h, p.log2w, p.row_shift, p.row_clamp, p.col_clamp, p.rk, p.rect2);
+    let mut t = [0i32; 64];
     let in_w = w.min(32);
     for i in 0..h {
         // Every 1D transform maps an all-zero row to zeros (each step is linear with
@@ -572,6 +602,11 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
             residual[i * w + j] = round2(t[j], row_shift).clamp(lo, hi);
         }
     }
+}
+
+fn scalar_col_pass(residual: &mut [i32], p: &Tx2d, lossless: bool) {
+    let (w, h, log2h, col_shift, col_clamp, ck) = (p.w, p.h, p.log2h, p.col_shift, p.col_clamp, p.ck);
+    let mut t = [0i32; 64];
     for j in 0..w {
         for i in 0..h {
             t[i] = residual[i * w + j];
@@ -617,11 +652,16 @@ fn transform_1d<T: Lane>(t: &mut [T], kind: Kind1d, n: u32, r: u32) {
 /// The 2D inverse transform with `L` rows (row pass) or columns (column pass) at a time;
 /// bit-identical to the one-at-a-time process for bit depths up to 10.
 fn transform_lanes<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx2d) {
+    let mut t = [[0i32; L]; 64];
+    lanes_row_pass(dequant, residual, p, &mut t);
+    lanes_col_pass(residual, p, &mut t);
+}
+
+fn lanes_row_pass<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx2d, t: &mut [[i32; L]; 64]) {
     let (w, h) = (p.w, p.h);
     let in_w = w.min(32);
     let lo = -(1i32 << (p.col_clamp - 1));
     let hi = (1i32 << (p.col_clamp - 1)) - 1;
-    let mut t = [[0i32; L]; 64];
     let mut i0 = 0;
     while i0 < h {
         // Rows without coefficients (and rows 32..63) transform to zeros.
@@ -649,6 +689,10 @@ fn transform_lanes<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx
         }
         i0 += L;
     }
+}
+
+fn lanes_col_pass<const L: usize>(residual: &mut [i32], p: &Tx2d, t: &mut [[i32; L]; 64]) {
+    let (w, h) = (p.w, p.h);
     let mut j0 = 0;
     while j0 < w {
         for i in 0..h {

crates/h264/src/slicedec.rs

@@ -665,38 +665,33 @@ impl<'a> SliceDecoder<'a> {
             return Ok(());
         }
         for p in 0..4 {
-            self.derive_sub_mb_motion(p, n)?;
-        }
-        Ok(())
-    }
-
-    /// Motion of one 8x8 sub-macroblock `p` of a P8x8/B8x8 MB.
-    fn derive_sub_mb_motion(&mut self, p: usize, n: [usize; 2]) -> Result<()> {
-        let sub = self.cur.sub[p];
-        let (x8, y8) = ((p & 1) * 2, (p >> 1) * 2);
-        if sub.direct {
-            return self.derive_direct(1 << p);
-        }
-        let mut refs = [-1i8; 2];
-        for (l, r) in refs.iter_mut().enumerate() {
-            if sub.pred & (1 << l) != 0 {
-                let v = self.cur.ref_idx[l][p];
-                ensure!((v as usize) < n[l], "ref_idx {v} out of range (list {l}, size {})", n[l]);
-                *r = v;
+            let sub = self.cur.sub[p];
+            let (x8, y8) = ((p & 1) * 2, (p >> 1) * 2);
+            if sub.direct {
+                self.derive_direct(1 << p)?;
+                continue;
             }
-        }
-        for j in 0..sub.shape.num_parts() {
-            let (sx, sy, w, h) = sub.shape.rect(j);
-            let (x, y) = (x8 + sx, y8 + sy);
-            let mut mvs = [[0i16; 2]; 2];
-            for l in 0..2 {
-                if refs[l] >= 0 {
-                    let mvp = self.mvp(l, x, y, w, refs[l], 0, 0);
-                    let d = self.cur.mvd[l][p * 4 + j];
-                    mvs[l] = [mvp[0].wrapping_add(d[0]), mvp[1].wrapping_add(d[1])];
+            let mut refs = [-1i8; 2];
+            for (l, r) in refs.iter_mut().enumerate() {
+                if sub.pred & (1 << l) != 0 {
+                    let v = self.cur.ref_idx[l][p];
+                    ensure!((v as usize) < n[l], "ref_idx {v} out of range (list {l}, size {})", n[l]);
+                    *r = v;
                 }
             }
-            self.set_motion(x, y, w, h, refs, mvs);
+            for j in 0..sub.shape.num_parts() {
+                let (sx, sy, w, h) = sub.shape.rect(j);
+                let (x, y) = (x8 + sx, y8 + sy);
+                let mut mvs = [[0i16; 2]; 2];
+                for l in 0..2 {
+                    if refs[l] >= 0 {
+                        let mvp = self.mvp(l, x, y, w, refs[l], 0, 0);
+                        let d = self.cur.mvd[l][p * 4 + j];
+                        mvs[l] = [mvp[0].wrapping_add(d[0]), mvp[1].wrapping_add(d[1])];
+                    }
+                }
+                self.set_motion(x, y, w, h, refs, mvs);
+            }
         }
         Ok(())
     }
@@ -1385,11 +1380,6 @@ impl<'a> SliceDecoder<'a> {
             cavlc::residual_block(r, &mut lv, 0, 15, 16, nc)?;
             self.put_luma_dc(&lv);
         }
-        self.residual_cavlc_luma(r, cbp, kind, t8)?;
-        self.residual_cavlc_chroma(r, cbp)
-    }
-
-    fn residual_cavlc_luma(&mut self, r: &mut BitReader, cbp: u8, kind: MbKind, t8: bool) -> Result<()> {
         for b8 in 0..4 {
             let (bx8, by8) = ((b8 & 1) * 2, (b8 >> 1) * 2);
             if cbp & (1 << b8) == 0 {
@@ -1423,10 +1413,6 @@ impl<'a> SliceDecoder<'a> {
                 }
             }
         }
-        Ok(())
-    }
-
-    fn residual_cavlc_chroma(&mut self, r: &mut BitReader, cbp: u8) -> Result<()> {
         let cbp_c = cbp >> 4;
         if cbp_c & 3 != 0 {
             for c in 0..2 {