refactor(h264): split derive_inter_motion and residual_cavlc (S3776)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/av1/src/transform.rs

@@ -165,15 +165,6 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..len {
         t[i] = copy[brev(n, i)];
     }
-    inverse_dct_stage1(t, n, r);
-    inverse_dct_stage2(t, n, r);
-    inverse_dct_stage3(t, n, r);
-    inverse_dct_stage4(t, n, r);
-    inverse_dct_stage5(t, n, r);
-    inverse_dct_stage6(t, n, r);
-}
-
-fn inverse_dct_stage1<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n == 6 {
         for i in 0..16 {
             bf(t, 32 + i, 63 - i, 63 - 4 * brev(4, i) as i32, false);
@@ -199,9 +190,6 @@ fn inverse_dct_stage1<T: Lane>(t: &mut [T], n: u32, r: u32) {
             hd(t, 16 + 2 * i, 17 + 2 * i, i & 1 == 1, r);
         }
     }
-}
-
-fn inverse_dct_stage2<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n == 6 {
         for i in 0..4 {
             for j in 0..2 {
@@ -233,9 +221,6 @@ fn inverse_dct_stage2<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
-}
-
-fn inverse_dct_stage3<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..2 {
         bf(t, 2 * i, 2 * i + 1, 32 + 16 * i as i32, i == 0);
     }
@@ -263,9 +248,6 @@ fn inverse_dct_stage3<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
-}
-
-fn inverse_dct_stage4<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..2 {
         hd(t, i, 3 - i, false, r);
     }
@@ -291,9 +273,6 @@ fn inverse_dct_stage4<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
-}
-
-fn inverse_dct_stage5<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n >= 3 {
         for i in 0..4 {
             hd(t, i, 7 - i, false, r);
@@ -321,9 +300,6 @@ fn inverse_dct_stage5<T: Lane>(t: &mut [T], n: u32, r: u32) {
             hd(t, i, 15 - i, false, r);
         }
     }
-}
-
-fn inverse_dct_stage6<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n >= 5 {
         for i in 0..4 {
             bf(t, 27 - i, 20 + i, 32, true);
@@ -555,9 +531,10 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
     let col_clamp = (bit_depth + 6).max(16);
     let rk = row_kind(tx_type);
     let ck = col_kind(tx_type);
+    let mut t = [0i32; 64];
     let rect2 = log2w.abs_diff(log2h) == 1;
-    let p = Tx2d { w, h, log2w, log2h, row_shift, col_shift, row_clamp, col_clamp, rk, ck, rect2 };
     if !lossless && bit_depth <= 10 {
+        let p = Tx2d { w, h, log2w, log2h, row_shift, col_shift, row_clamp, col_clamp, rk, ck, rect2 };
         if w >= 8 && h >= 8 {
             transform_lanes::<8>(dequant, residual, &p);
         } else {
@@ -565,13 +542,6 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
         }
         return;
     }
-    scalar_row_pass(dequant, residual, &p, lossless);
-    scalar_col_pass(residual, &p, lossless);
-}
-
-fn scalar_row_pass(dequant: &[i32], residual: &mut [i32], p: &Tx2d, lossless: bool) {
-    let (w, h, log2w, row_shift, row_clamp, col_clamp, rk, rect2) = (p.w, p.h, p.log2w, p.row_shift, p.row_clamp, p.col_clamp, p.rk, p.rect2);
-    let mut t = [0i32; 64];
     let in_w = w.min(32);
     for i in 0..h {
         // Every 1D transform maps an all-zero row to zeros (each step is linear with
@@ -602,11 +572,6 @@ fn scalar_row_pass(dequant: &[i32], residual: &mut [i32], p: &Tx2d, lossless: bo
             residual[i * w + j] = round2(t[j], row_shift).clamp(lo, hi);
         }
     }
-}
-
-fn scalar_col_pass(residual: &mut [i32], p: &Tx2d, lossless: bool) {
-    let (w, h, log2h, col_shift, col_clamp, ck) = (p.w, p.h, p.log2h, p.col_shift, p.col_clamp, p.ck);
-    let mut t = [0i32; 64];
     for j in 0..w {
         for i in 0..h {
             t[i] = residual[i * w + j];
@@ -652,16 +617,11 @@ fn transform_1d<T: Lane>(t: &mut [T], kind: Kind1d, n: u32, r: u32) {
 /// The 2D inverse transform with `L` rows (row pass) or columns (column pass) at a time;
 /// bit-identical to the one-at-a-time process for bit depths up to 10.
 fn transform_lanes<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx2d) {
-    let mut t = [[0i32; L]; 64];
-    lanes_row_pass(dequant, residual, p, &mut t);
-    lanes_col_pass(residual, p, &mut t);
-}
-
-fn lanes_row_pass<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx2d, t: &mut [[i32; L]; 64]) {
     let (w, h) = (p.w, p.h);
     let in_w = w.min(32);
     let lo = -(1i32 << (p.col_clamp - 1));
     let hi = (1i32 << (p.col_clamp - 1)) - 1;
+    let mut t = [[0i32; L]; 64];
     let mut i0 = 0;
     while i0 < h {
         // Rows without coefficients (and rows 32..63) transform to zeros.
@@ -689,10 +649,6 @@ fn lanes_row_pass<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx2
         }
         i0 += L;
     }
-}
-
-fn lanes_col_pass<const L: usize>(residual: &mut [i32], p: &Tx2d, t: &mut [[i32; L]; 64]) {
-    let (w, h) = (p.w, p.h);
     let mut j0 = 0;
     while j0 < w {
         for i in 0..h {

crates/h264/src/slicedec.rs

@@ -665,33 +665,38 @@ impl<'a> SliceDecoder<'a> {
             return Ok(());
         }
         for p in 0..4 {
-            let sub = self.cur.sub[p];
-            let (x8, y8) = ((p & 1) * 2, (p >> 1) * 2);
-            if sub.direct {
-                self.derive_direct(1 << p)?;
-                continue;
+            self.derive_sub_mb_motion(p, n)?;
+        }
+        Ok(())
+    }
+
+    /// Motion of one 8x8 sub-macroblock `p` of a P8x8/B8x8 MB.
+    fn derive_sub_mb_motion(&mut self, p: usize, n: [usize; 2]) -> Result<()> {
+        let sub = self.cur.sub[p];
+        let (x8, y8) = ((p & 1) * 2, (p >> 1) * 2);
+        if sub.direct {
+            return self.derive_direct(1 << p);
+        }
+        let mut refs = [-1i8; 2];
+        for (l, r) in refs.iter_mut().enumerate() {
+            if sub.pred & (1 << l) != 0 {
+                let v = self.cur.ref_idx[l][p];
+                ensure!((v as usize) < n[l], "ref_idx {v} out of range (list {l}, size {})", n[l]);
+                *r = v;
             }
-            let mut refs = [-1i8; 2];
-            for (l, r) in refs.iter_mut().enumerate() {
-                if sub.pred & (1 << l) != 0 {
-                    let v = self.cur.ref_idx[l][p];
-                    ensure!((v as usize) < n[l], "ref_idx {v} out of range (list {l}, size {})", n[l]);
-                    *r = v;
+        }
+        for j in 0..sub.shape.num_parts() {
+            let (sx, sy, w, h) = sub.shape.rect(j);
+            let (x, y) = (x8 + sx, y8 + sy);
+            let mut mvs = [[0i16; 2]; 2];
+            for l in 0..2 {
+                if refs[l] >= 0 {
+                    let mvp = self.mvp(l, x, y, w, refs[l], 0, 0);
+                    let d = self.cur.mvd[l][p * 4 + j];
+                    mvs[l] = [mvp[0].wrapping_add(d[0]), mvp[1].wrapping_add(d[1])];
                 }
             }
-            for j in 0..sub.shape.num_parts() {
-                let (sx, sy, w, h) = sub.shape.rect(j);
-                let (x, y) = (x8 + sx, y8 + sy);
-                let mut mvs = [[0i16; 2]; 2];
-                for l in 0..2 {
-                    if refs[l] >= 0 {
-                        let mvp = self.mvp(l, x, y, w, refs[l], 0, 0);
-                        let d = self.cur.mvd[l][p * 4 + j];
-                        mvs[l] = [mvp[0].wrapping_add(d[0]), mvp[1].wrapping_add(d[1])];
-                    }
-                }
-                self.set_motion(x, y, w, h, refs, mvs);
-            }
+            self.set_motion(x, y, w, h, refs, mvs);
         }
         Ok(())
     }
@@ -1380,6 +1385,11 @@ impl<'a> SliceDecoder<'a> {
             cavlc::residual_block(r, &mut lv, 0, 15, 16, nc)?;
             self.put_luma_dc(&lv);
         }
+        self.residual_cavlc_luma(r, cbp, kind, t8)?;
+        self.residual_cavlc_chroma(r, cbp)
+    }
+
+    fn residual_cavlc_luma(&mut self, r: &mut BitReader, cbp: u8, kind: MbKind, t8: bool) -> Result<()> {
         for b8 in 0..4 {
             let (bx8, by8) = ((b8 & 1) * 2, (b8 >> 1) * 2);
             if cbp & (1 << b8) == 0 {
@@ -1413,6 +1423,10 @@ impl<'a> SliceDecoder<'a> {
                 }
             }
         }
+        Ok(())
+    }
+
+    fn residual_cavlc_chroma(&mut self, r: &mut BitReader, cbp: u8) -> Result<()> {
         let cbp_c = cbp >> 4;
         if cbp_c & 3 != 0 {
             for c in 0..2 {