refactor(av1): split inverse transform into stages and passes (S3776)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/av1/src/transform.rs

@@ -165,6 +165,15 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..len {
         t[i] = copy[brev(n, i)];
     }
+    inverse_dct_stage1(t, n, r);
+    inverse_dct_stage2(t, n, r);
+    inverse_dct_stage3(t, n, r);
+    inverse_dct_stage4(t, n, r);
+    inverse_dct_stage5(t, n, r);
+    inverse_dct_stage6(t, n, r);
+}
+
+fn inverse_dct_stage1<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n == 6 {
         for i in 0..16 {
             bf(t, 32 + i, 63 - i, 63 - 4 * brev(4, i) as i32, false);
@@ -190,6 +199,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             hd(t, 16 + 2 * i, 17 + 2 * i, i & 1 == 1, r);
         }
     }
+}
+
+fn inverse_dct_stage2<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n == 6 {
         for i in 0..4 {
             for j in 0..2 {
@@ -221,6 +233,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
+}
+
+fn inverse_dct_stage3<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..2 {
         bf(t, 2 * i, 2 * i + 1, 32 + 16 * i as i32, i == 0);
     }
@@ -248,6 +263,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
+}
+
+fn inverse_dct_stage4<T: Lane>(t: &mut [T], n: u32, r: u32) {
     for i in 0..2 {
         hd(t, i, 3 - i, false, r);
     }
@@ -273,6 +291,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             }
         }
     }
+}
+
+fn inverse_dct_stage5<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n >= 3 {
         for i in 0..4 {
             hd(t, i, 7 - i, false, r);
@@ -300,6 +321,9 @@ pub(crate) fn inverse_dct<T: Lane>(t: &mut [T], n: u32, r: u32) {
             hd(t, i, 15 - i, false, r);
         }
     }
+}
+
+fn inverse_dct_stage6<T: Lane>(t: &mut [T], n: u32, r: u32) {
     if n >= 5 {
         for i in 0..4 {
             bf(t, 27 - i, 20 + i, 32, true);
@@ -531,10 +555,9 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
     let col_clamp = (bit_depth + 6).max(16);
     let rk = row_kind(tx_type);
     let ck = col_kind(tx_type);
-    let mut t = [0i32; 64];
     let rect2 = log2w.abs_diff(log2h) == 1;
+    let p = Tx2d { w, h, log2w, log2h, row_shift, col_shift, row_clamp, col_clamp, rk, ck, rect2 };
     if !lossless && bit_depth <= 10 {
-        let p = Tx2d { w, h, log2w, log2h, row_shift, col_shift, row_clamp, col_clamp, rk, ck, rect2 };
         if w >= 8 && h >= 8 {
             transform_lanes::<8>(dequant, residual, &p);
         } else {
@@ -542,6 +565,13 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
         }
         return;
     }
+    scalar_row_pass(dequant, residual, &p, lossless);
+    scalar_col_pass(residual, &p, lossless);
+}
+
+fn scalar_row_pass(dequant: &[i32], residual: &mut [i32], p: &Tx2d, lossless: bool) {
+    let (w, h, log2w, row_shift, row_clamp, col_clamp, rk, rect2) = (p.w, p.h, p.log2w, p.row_shift, p.row_clamp, p.col_clamp, p.rk, p.rect2);
+    let mut t = [0i32; 64];
     let in_w = w.min(32);
     for i in 0..h {
         // Every 1D transform maps an all-zero row to zeros (each step is linear with
@@ -572,6 +602,11 @@ pub(crate) fn inverse_transform_2d(dequant: &[i32], residual: &mut [i32], tx_sz:
             residual[i * w + j] = round2(t[j], row_shift).clamp(lo, hi);
         }
     }
+}
+
+fn scalar_col_pass(residual: &mut [i32], p: &Tx2d, lossless: bool) {
+    let (w, h, log2h, col_shift, col_clamp, ck) = (p.w, p.h, p.log2h, p.col_shift, p.col_clamp, p.ck);
+    let mut t = [0i32; 64];
     for j in 0..w {
         for i in 0..h {
             t[i] = residual[i * w + j];
@@ -617,11 +652,16 @@ fn transform_1d<T: Lane>(t: &mut [T], kind: Kind1d, n: u32, r: u32) {
 /// The 2D inverse transform with `L` rows (row pass) or columns (column pass) at a time;
 /// bit-identical to the one-at-a-time process for bit depths up to 10.
 fn transform_lanes<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx2d) {
+    let mut t = [[0i32; L]; 64];
+    lanes_row_pass(dequant, residual, p, &mut t);
+    lanes_col_pass(residual, p, &mut t);
+}
+
+fn lanes_row_pass<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx2d, t: &mut [[i32; L]; 64]) {
     let (w, h) = (p.w, p.h);
     let in_w = w.min(32);
     let lo = -(1i32 << (p.col_clamp - 1));
     let hi = (1i32 << (p.col_clamp - 1)) - 1;
-    let mut t = [[0i32; L]; 64];
     let mut i0 = 0;
     while i0 < h {
         // Rows without coefficients (and rows 32..63) transform to zeros.
@@ -649,6 +689,10 @@ fn transform_lanes<const L: usize>(dequant: &[i32], residual: &mut [i32], p: &Tx
         }
         i0 += L;
     }
+}
+
+fn lanes_col_pass<const L: usize>(residual: &mut [i32], p: &Tx2d, t: &mut [[i32; L]; 64]) {
+    let (w, h) = (p.w, p.h);
     let mut j0 = 0;
     while j0 < w {
         for i in 0..h {