From 5881842a2e36de1c70c219d7077bd8e858f3fa09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 30 Sep 2026 06:45:35 -0400 Subject: [PATCH] M4: LUT-based linear conversion and linear-light box decimation for reduced-resolution playback Co-Authored-By: Claude Opus 5.5 --- crates/frame/src/lib.rs | 150 ++++++++++++++++++++++++++++----------- crates/render/src/lib.rs | 19 ++++- 2 files changed, 125 insertions(+), 44 deletions(-) diff --git a/crates/frame/src/lib.rs b/crates/frame/src/lib.rs index 771a1fd8d..6f4232dd9 100644 --- a/crates/frame/src/lib.rs +++ b/crates/frame/src/lib.rs @@ -100,71 +100,139 @@ impl VideoFrame { /// Convert to premultiplied linear RGBA f32 (the compositor's working format). pub fn to_linear_f32(&self) -> Vec { + self.to_linear_f32_decimated(1).2 + } + + /// Convert to premultiplied linear RGBA f32, box-filtering `n`×`n` blocks (n = 1, 2, 4, 8…) + /// in linear light. Reduced-resolution playback uses this so it never builds full-size float + /// buffers. Returns (width, height, pixels). + pub fn to_linear_f32_decimated(&self, n: usize) -> (usize, usize, Vec) { + let n = n.max(1); let (w, h) = (self.width as usize, self.height as usize); - let mut out = vec![0f32; w * h * 4]; + let (ow, oh) = ((w / n).max(1), (h / n).max(1)); + let mut out = vec![0f32; ow * oh * 4]; + if let PixelData::RgbaF32(d) = &self.data + && n == 1 + { + out.copy_from_slice(d); + return (ow, oh, out); + } + // Encoded (0..1, quantised to 12 bits) → linear lookup for this frame's transfer. + let info = self.color; + let lin: Vec = (0..4096).map(|i| to_linear(i as f32 / 4095.0, info.transfer)).collect(); + let q = |v: f32| lin[(v.clamp(0.0, 1.0) * 4095.0 + 0.5) as usize]; + let inv = 1.0 / (n * n) as f32; match &self.data { - PixelData::RgbaF32(d) => out.copy_from_slice(d), + PixelData::RgbaF32(d) => { + out.par_chunks_mut(ow * 4).enumerate().for_each(|(oy, row)| { + for ox in 0..ow { + let mut acc = [0f32; 4]; + for dy in 0..n { + let y = (oy * n + dy).min(h - 1); + for dx in 0..n { + let x = (ox * n + dx).min(w - 1); + let i = (y * w + x) * 4; + for k in 0..4 { + acc[k] += d[i + k]; + } + } + } + for k in 0..4 { + row[ox * 4 + k] = acc[k] * inv; + } + } + }); + } PixelData::Rgba8(d) => { let lut = srgb_u8_to_linear_table(); - out.par_chunks_mut(w * 4).zip(d.par_chunks(w * 4)).for_each(|(o, s)| { - for (o, s) in o.chunks_exact_mut(4).zip(s.chunks_exact(4)) { - let a = s[3] as f32 / 255.0; - o[0] = lut[s[0] as usize] * a; - o[1] = lut[s[1] as usize] * a; - o[2] = lut[s[2] as usize] * a; - o[3] = a; + out.par_chunks_mut(ow * 4).enumerate().for_each(|(oy, row)| { + for ox in 0..ow { + let mut acc = [0f32; 4]; + for dy in 0..n { + let y = (oy * n + dy).min(h - 1); + for dx in 0..n { + let x = (ox * n + dx).min(w - 1); + let s = &d[(y * w + x) * 4..(y * w + x) * 4 + 4]; + let a = s[3] as f32 / 255.0; + acc[0] += lut[s[0] as usize] * a; + acc[1] += lut[s[1] as usize] * a; + acc[2] += lut[s[2] as usize] * a; + acc[3] += a; + } + } + for k in 0..4 { + row[ox * 4 + k] = acc[k] * inv; + } } }); } PixelData::Yuv8 { planes, chroma, alpha } => { let (sx, sy) = chroma.shifts(); let cw = w.div_ceil(1 << sx); - let info = self.color; - // Precompute per-code tables for Y and C. let ytab: Vec = (0..256).map(|v| normalize_y(v, 8, info.range)).collect(); let ctab: Vec = (0..256).map(|v| normalize_c(v, 8, info.range)).collect(); - out.par_chunks_mut(w * 4).enumerate().for_each(|(y, row)| { - let yrow = &planes[0][y * w..y * w + w]; - let cy = y >> sy; - let urow = &planes[1][cy * cw..cy * cw + cw]; - let vrow = &planes[2][cy * cw..cy * cw + cw]; - for x in 0..w { - let cx = x >> sx; - let rgb = ycbcr_to_rgb(ytab[yrow[x] as usize], ctab[urow[cx] as usize], ctab[vrow[cx] as usize], info.matrix); - let a = alpha.as_ref().map_or(1.0, |al| al[y * w + x] as f32 / 255.0); - let o = &mut row[x * 4..x * 4 + 4]; - o[0] = to_linear(rgb[0].clamp(0.0, 1.0), info.transfer) * a; - o[1] = to_linear(rgb[1].clamp(0.0, 1.0), info.transfer) * a; - o[2] = to_linear(rgb[2].clamp(0.0, 1.0), info.transfer) * a; - o[3] = a; + let (kr, kb) = info.matrix.kr_kb(); + let kg = 1.0 - kr - kb; + let (cr_r, cb_b) = (2.0 * (1.0 - kr), 2.0 * (1.0 - kb)); + let (cr_g, cb_g) = (cr_r * kr / kg, cb_b * kb / kg); + out.par_chunks_mut(ow * 4).enumerate().for_each(|(oy, row)| { + for ox in 0..ow { + let mut acc = [0f32; 4]; + for dy in 0..n { + let y = (oy * n + dy).min(h - 1); + let cy = y >> sy; + for dx in 0..n { + let x = (ox * n + dx).min(w - 1); + let cx = x >> sx; + let yy = ytab[planes[0][y * w + x] as usize]; + let u = ctab[planes[1][cy * cw + cx] as usize]; + let v = ctab[planes[2][cy * cw + cx] as usize]; + let a = alpha.as_ref().map_or(1.0, |al| al[y * w + x] as f32 / 255.0); + acc[0] += q(yy + cr_r * v) * a; + acc[1] += q(yy - cr_g * v - cb_g * u) * a; + acc[2] += q(yy + cb_b * u) * a; + acc[3] += a; + } + } + for k in 0..4 { + row[ox * 4 + k] = acc[k] * inv; + } } }); } PixelData::Yuv16 { planes, chroma, bits, alpha } => { let (sx, sy) = chroma.shifts(); let cw = w.div_ceil(1 << sx); - let info = self.color; let bits = *bits; let amax = ((1u32 << bits) - 1) as f32; - out.par_chunks_mut(w * 4).enumerate().for_each(|(y, row)| { - let cy = y >> sy; - for x in 0..w { - let cx = x >> sx; - let yy = normalize_y(planes[0][y * w + x] as u32, bits, info.range); - let u = normalize_c(planes[1][cy * cw + cx] as u32, bits, info.range); - let v = normalize_c(planes[2][cy * cw + cx] as u32, bits, info.range); - let rgb = ycbcr_to_rgb(yy, u, v, info.matrix); - let a = alpha.as_ref().map_or(1.0, |al| al[y * w + x] as f32 / amax); - let o = &mut row[x * 4..x * 4 + 4]; - o[0] = to_linear(rgb[0].clamp(0.0, 1.0), info.transfer) * a; - o[1] = to_linear(rgb[1].clamp(0.0, 1.0), info.transfer) * a; - o[2] = to_linear(rgb[2].clamp(0.0, 1.0), info.transfer) * a; - o[3] = a; + out.par_chunks_mut(ow * 4).enumerate().for_each(|(oy, row)| { + for ox in 0..ow { + let mut acc = [0f32; 4]; + for dy in 0..n { + let y = (oy * n + dy).min(h - 1); + let cy = y >> sy; + for dx in 0..n { + let x = (ox * n + dx).min(w - 1); + let cx = x >> sx; + let yy = normalize_y(planes[0][y * w + x] as u32, bits, info.range); + let u = normalize_c(planes[1][cy * cw + cx] as u32, bits, info.range); + let v = normalize_c(planes[2][cy * cw + cx] as u32, bits, info.range); + let rgb = ycbcr_to_rgb(yy, u, v, info.matrix); + let a = alpha.as_ref().map_or(1.0, |al| al[y * w + x] as f32 / amax); + acc[0] += q(rgb[0]) * a; + acc[1] += q(rgb[1]) * a; + acc[2] += q(rgb[2]) * a; + acc[3] += a; + } + } + for k in 0..4 { + row[ox * 4 + k] = acc[k] * inv; + } } }); } } - out + (ow, oh, out) } /// Convert to straight-alpha sRGB RGBA8 for display (fast paths for 8-bit sources). diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index e5368f83b..531678c03 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -207,8 +207,9 @@ fn item_layer( ItemKind::Media(_) | ItemKind::Subclip { .. } => { let src = sources.source(item.item)?; let frame = src.video_frame(FrameRequest { time: mt, scale: want }).ok()?; - let data = frame.to_linear_f32(); - Image { w: frame.width as usize, h: frame.height as usize, px: data } + let n = decimation(frame.width as f32, src_size.0 as f32 * want); + let (w, h, px) = frame.to_linear_f32_decimated(n); + Image { w, h, px } } ItemKind::Sequence(nested) => { let sub = RenderOptions { scale: want, effects: opts.effects, depth: opts.depth + 1 }; @@ -242,6 +243,15 @@ fn item_layer( Some((placed, op, bl)) } +/// Largest power-of-two box decimation that keeps at least `target_w` pixels of width. +fn decimation(have_w: f32, target_w: f32) -> usize { + let mut n = 1usize; + while n < 16 && have_w / (n as f32 * 2.0) >= target_w.max(1.0) { + n *= 2; + } + n +} + /// Render a single project item (e.g. for the Source monitor) at media time `t`. pub fn render_item(project: &Project, item: ItemId, t: Tick, scale: f32, sources: &dyn SourceProvider) -> Option { let pi = project.item(item)?; @@ -250,7 +260,10 @@ pub fn render_item(project: &Project, item: ItemId, t: Tick, scale: f32, sources _ => { let src = sources.source(item)?; let f = src.video_frame(FrameRequest { time: t, scale }).ok()?; - Some(Image { w: f.width as usize, h: f.height as usize, px: f.to_linear_f32() }) + let full_w = src.info().video.as_ref().map_or(f.width, |v| v.width) as f32; + let n = decimation(f.width as f32, full_w * scale); + let (w, h, px) = f.to_linear_f32_decimated(n); + Some(Image { w, h, px }) } } }