diff --git a/crates/codecs/src/gop.rs b/crates/codecs/src/gop.rs index 3db8d389b..8b49a97fc 100644 --- a/crates/codecs/src/gop.rs +++ b/crates/codecs/src/gop.rs @@ -163,6 +163,7 @@ impl State { self.bytes = self.bytes.saturating_sub(f.byte_size()); pool.bytes.fetch_sub(f.byte_size(), Ordering::Relaxed); EVICTED.fetch_add(1, Ordering::Relaxed); + filmcraft_frame::pool::recycle(f); } } @@ -410,6 +411,7 @@ impl GopCache { st.bytes += f.byte_size(); if let Some(old) = st.frames.insert(pts, Arc::new(f)) { st.bytes -= old.byte_size(); + filmcraft_frame::pool::recycle(old); } // evict frames far from the most recent (keep a window around the working position) while st.bytes > budget && st.frames.len() > 2 { @@ -419,6 +421,8 @@ impl GopCache { st.drafts.remove(&victim); st.bytes -= v.byte_size(); EVICTED.fetch_add(1, Ordering::Relaxed); + // its planes serve the next decoded pictures instead of going back to the allocator + filmcraft_frame::pool::recycle(v); } } let pool = &self.shared.pool; @@ -984,4 +988,68 @@ mod tests { assert_eq!(index_of(&ca.frame(&a, 0).expect("cached")), 0); assert_eq!(a.decodes.load(Ordering::Relaxed), decodes); } + + /// Pictures big enough for the plane pool (its own size, so that parallel tests cannot take + /// the buffers), no reordering, one long GOP. + struct Big(usize); + const BIG: (u32, u32) = (200, 101); + + impl VideoDecoder for Big { + fn decode(&mut self, _sample: &[u8], pts: i64) -> crate::Result> { + let mut px = filmcraft_frame::pool::take_u8((BIG.0 * BIG.1 * 4) as usize); + px.resize((BIG.0 * BIG.1 * 4) as usize, (pts / 1000) as u8); + Ok(vec![DecodedFrame { pts, frame: VideoFrame::rgba8(BIG.0, BIG.1, px), draft: false }]) + } + fn flush(&mut self) -> Vec { + Vec::new() + } + fn reset(&mut self) {} + fn name(&self) -> &str { + "big" + } + } + + impl VideoSamples for Big { + fn count(&self) -> usize { + self.0 + } + fn pts(&self, i: usize) -> i64 { + i as i64 * 1000 + } + fn sync_before(&self, _i: usize) -> usize { + 0 + } + fn sample_at(&self, t: i64) -> Option { + let i = (t / 1000) as usize; + (i < self.0).then_some(i) + } + fn read(&self, i: usize) -> crate::Result> { + Ok(vec![i as u8]) + } + fn make_decoder(&self) -> crate::Result> { + Ok(Box::new(Big(self.0))) + } + } + + #[test] + fn evicted_frames_are_decoded_into_again() { + let s = Big(200); + // room for MIN_FRAMES pictures only + let mut c = GopCache::new(None); + c.budget = 0; + let reused = || filmcraft_frame::pool::stats().reused; + let before = reused(); + // the cache holds MIN_FRAMES pictures: none is evicted yet, every plane is new + for i in 0..MIN_FRAMES as i64 { + assert_eq!(c.frame(&s, i * 1000).expect("frame").width, BIG.0); + } + assert_eq!(reused(), before); + // from here on each picture evicts one, whose buffer the next picture is decoded into + let more = 40; + for i in MIN_FRAMES as i64..MIN_FRAMES as i64 + more { + let f = c.frame(&s, i * 1000).expect("frame"); + assert!(matches!(&f.data, filmcraft_frame::PixelData::Rgba8(d) if d.iter().all(|&b| b == i as u8)), "frame {i} has its own pixels"); + } + assert!(reused() - before >= more as u64 - 2, "{} of {more} planes recycled", reused() - before); + } } diff --git a/crates/codecs/src/video.rs b/crates/codecs/src/video.rs index 001585a0b..1fd52b4b4 100644 --- a/crates/codecs/src/video.rs +++ b/crates/codecs/src/video.rs @@ -204,12 +204,14 @@ pub struct H264Decoder { impl H264Decoder { pub fn new(avcc: Vec) -> Result { + recycle_h264_planes(); let dec = filmcraft_h264::Decoder::from_avcc(&avcc).map_err(|e| CodecError::Decode(e.to_string()))?; let length_size = avcc_length_size(&avcc); Ok(Self { avcc, dec, length_size, draft: false }) } /// A decoder for Annex B byte-stream samples (start codes, in-band parameter sets: MXF, TS). pub fn annexb() -> Self { + recycle_h264_planes(); Self { avcc: Vec::new(), dec: filmcraft_h264::Decoder::new(), length_size: 0, draft: false } } fn convert(p: filmcraft_h264::Picture) -> DecodedFrame { @@ -269,6 +271,12 @@ impl VideoDecoder for H264Decoder { } } +/// The H.264 decoder's output planes come from the frames the caches evicted +/// (`filmcraft_frame::pool`) instead of the allocator. +fn recycle_h264_planes() { + filmcraft_h264::set_plane_allocator(filmcraft_frame::pool::take_u8); +} + /// Worker threads each H.264 decoder uses (frame threading; 1 on wasm). pub fn h264_threads() -> usize { filmcraft_h264::default_threads() diff --git a/crates/engine/src/perf.rs b/crates/engine/src/perf.rs index 98339a92b..f1b420fa0 100644 --- a/crates/engine/src/perf.rs +++ b/crates/engine/src/perf.rs @@ -26,6 +26,10 @@ pub fn decode_json() -> Value { // decoded frames all sources hold, and the budget idle sources are trimmed to "cacheMB": filmcraft_codecs::cached_bytes() as f64 / 1e6, "cacheBudgetMB": filmcraft_codecs::FRAME_BUDGET as f64 / 1e6, + // plane buffers of evicted frames waiting for the next decoded pictures, and how many + // planes were decoded into a recycled buffer + "planePoolMB": filmcraft_frame::pool::stats().idle_bytes as f64 / 1e6, + "planesReused": filmcraft_frame::pool::stats().reused, "decodeMs": g.decode_ns as f64 / 1e6, "decodeMsPerSample": g.decode_ms_per_sample(), "framesDecoded": g.frames, @@ -79,6 +83,8 @@ mod tests { "liveDecoders", "cacheMB", "cacheBudgetMB", + "planePoolMB", + "planesReused", ] { assert!(v["decode"][k].is_number(), "decode.{k} in {v}"); } diff --git a/crates/frame/src/lib.rs b/crates/frame/src/lib.rs index a5eb78c3b..5501dc182 100644 --- a/crates/frame/src/lib.rs +++ b/crates/frame/src/lib.rs @@ -7,6 +7,8 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] +pub mod pool; + use std::sync::Arc; use filmcraft_color::{ColorInfo, DecodeTable, Matrix, Range, linear_to_srgb_u8, normalize_c, normalize_y, srgb_u8_to_linear_table, to_linear, ycbcr_to_rgb}; diff --git a/crates/frame/src/pool.rs b/crates/frame/src/pool.rs new file mode 100644 index 000000000..53049d80c --- /dev/null +++ b/crates/frame/src/pool.rs @@ -0,0 +1,177 @@ +//! Recycled plane buffers. +//! +//! A decoder allocates three planes per picture and the caches free them a few hundred frames +//! later. At megabytes per plane that traffic leaves the system allocator holding gigabytes of +//! freed pages (measured: 2.5 GB retained next to 2.5 GB live after twelve clips). Decoders take +//! their planes here instead ([`take_u8`], [`take_u16`]) and caches hand evicted frames back +//! ([`recycle`]), so steady-state decoding allocates nothing. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + +use crate::{PixelData, VideoFrame}; + +/// Bytes of idle buffers kept per sample type; a buffer that does not fit is freed. +const MAX_BYTES: usize = 192 << 20; + +/// Smaller buffers are not worth keeping (the allocator recycles those well). +const MIN_BYTES: usize = 64 << 10; + +struct Shelf { + bufs: Vec>, + bytes: usize, +} + +impl Shelf { + const fn new() -> Self { + Self { bufs: Vec::new(), bytes: 0 } + } + + fn size(b: &Vec) -> usize { + b.capacity().saturating_mul(std::mem::size_of::()) + } + + /// The smallest idle buffer that holds `len` samples without wasting over a quarter of it. + fn take(&mut self, len: usize) -> Option> { + let k = self + .bufs + .iter() + .enumerate() + .filter(|(_, b)| b.capacity() >= len && b.capacity() - len <= len / 4) + .min_by_key(|(_, b)| b.capacity()) + .map(|(k, _)| k)?; + let b = self.bufs.swap_remove(k); + self.bytes = self.bytes.saturating_sub(Self::size(&b)); + Some(b) + } + + fn give(&mut self, mut b: Vec, max_bytes: usize) { + let n = Self::size(&b); + if n < MIN_BYTES || self.bytes.saturating_add(n) > max_bytes { + return; + } + b.clear(); + self.bytes += n; + self.bufs.push(b); + } +} + +static U8: Mutex> = Mutex::new(Shelf::new()); +static U16: Mutex> = Mutex::new(Shelf::new()); +static REUSED: AtomicU64 = AtomicU64::new(0); + +fn take(shelf: &Mutex>, len: usize) -> Vec { + let found = shelf.lock().unwrap_or_else(PoisonError::into_inner).take(len); + match found { + Some(b) => { + REUSED.fetch_add(1, Ordering::Relaxed); + b + } + None => Vec::with_capacity(len), + } +} + +fn give(shelf: &Mutex>, plane: Arc>) { + // a plane something else still shows (a monitor, a GPU upload in flight) is theirs + if let Ok(b) = Arc::try_unwrap(plane) { + shelf.lock().unwrap_or_else(PoisonError::into_inner).give(b, MAX_BYTES); + } +} + +/// An empty buffer with room for `len` samples: a recycled one when one fits. +pub fn take_u8(len: usize) -> Vec { + take(&U8, len) +} + +/// [`take_u8`] for 16-bit planes. +pub fn take_u16(len: usize) -> Vec { + take(&U16, len) +} + +/// Hand back a frame nobody needs any more: its planes are kept for the next [`take_u8`] / +/// [`take_u16`] unless something else still holds the frame or a plane. +pub fn recycle(frame: Arc) { + let Ok(f) = Arc::try_unwrap(frame) else { return }; + match f.data { + PixelData::Rgba8(d) => give(&U8, d), + PixelData::RgbaF32(_) => {} + PixelData::Yuv8 { planes, alpha, .. } => planes.into_iter().chain(alpha).for_each(|p| give(&U8, p)), + PixelData::Yuv16 { planes, alpha, .. } => planes.into_iter().chain(alpha).for_each(|p| give(&U16, p)), + } +} + +/// Idle buffers held and buffers handed out again so far (`perf.stats`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PoolStats { + pub idle_bytes: usize, + pub reused: u64, +} + +pub fn stats() -> PoolStats { + let idle = |n: usize, m: usize| n.saturating_add(m); + let a = U8.lock().unwrap_or_else(PoisonError::into_inner).bytes; + let b = U16.lock().unwrap_or_else(PoisonError::into_inner).bytes; + PoolStats { idle_bytes: idle(a, b), reused: REUSED.load(Ordering::Relaxed) } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Chroma; + + #[test] + fn shelf_hands_back_a_fitting_buffer_and_stays_within_its_budget() { + let mut s = Shelf::::new(); + s.give(Vec::with_capacity(1 << 20), 3 << 20); + s.give(Vec::with_capacity(2 << 20), 3 << 20); + assert_eq!((s.bufs.len(), s.bytes), (2, 3 << 20)); + // over the budget and too small to keep: freed + s.give(Vec::with_capacity(1 << 20), 3 << 20); + s.give(Vec::with_capacity(1024), 3 << 20); + assert_eq!(s.bufs.len(), 2); + // nothing fits a far smaller or a larger request + assert!(s.take(100 << 10).is_none()); + assert!(s.take(3 << 20).is_none()); + // the smallest buffer that fits, with its contents gone + let b = s.take((1 << 20) - 4096).expect("fits"); + assert_eq!((b.len(), b.capacity()), (0, 1 << 20)); + assert_eq!((s.bufs.len(), s.bytes), (1, 2 << 20)); + assert!(s.take(1 << 20).is_none(), "2 MiB wastes too much for 1 MiB"); + } + + #[test] + fn recycled_frames_feed_the_next_take_unless_still_shared() { + // sizes nothing else in this test binary asks for + let (n, c) = (777_001usize, 333_003usize); + let yuv = |fill: u8| { + Arc::new(VideoFrame { + width: 1, + height: 1, + data: PixelData::Yuv8 { + planes: [Arc::new(vec![fill; n]), Arc::new(vec![fill; c]), Arc::new(vec![fill; c])], + chroma: Chroma::C420, + alpha: None, + }, + color: filmcraft_color::ColorInfo::SRGB_FULL, + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + }) + }; + let before = stats().reused; + // a frame someone else still holds is left alone + let shared = yuv(1); + let held = shared.clone(); + recycle(shared); + assert!(take_u8(n).capacity() >= n); + assert_eq!(stats().reused, before); + assert_eq!(held.byte_size(), n + 2 * c); + // an unshared frame's planes come back, empty + recycle(yuv(2)); + let (y, u, v) = (take_u8(n), take_u8(c), take_u8(c)); + assert!(y.is_empty() && y.capacity() >= n && u.capacity() >= c && v.capacity() >= c); + assert_eq!(stats().reused, before + 3); + // and are not handed out twice + take_u8(n); + assert_eq!(stats().reused, before + 3); + } +} diff --git a/crates/h264/src/lib.rs b/crates/h264/src/lib.rs index 2d030d158..6cc05a0cf 100644 --- a/crates/h264/src/lib.rs +++ b/crates/h264/src/lib.rs @@ -35,6 +35,7 @@ mod intra; mod mbtypes; pub mod params; mod picture; +pub use picture::set_plane_allocator; pub mod slice; mod slicedec; #[cfg(test)] diff --git a/crates/h264/src/picture.rs b/crates/h264/src/picture.rs index bd7ca50e9..65a34d650 100644 --- a/crates/h264/src/picture.rs +++ b/crates/h264/src/picture.rs @@ -255,14 +255,15 @@ impl Frame { /// Crop and copy the frame into planar buffers (waits for completion). pub fn copy_cropped(&self, crop: (usize, usize, usize, usize)) -> (Vec, Vec, Vec) { let (cx, cy, cw, ch) = crop; - let mut y = Vec::with_capacity(cw * ch); + let alloc = plane_allocator(); + let mut y = alloc(cw * ch); for r in cy..cy + ch { let row = self.row(r >> 4); y.extend_from_slice(&row.y[(r & 15) * self.width + cx..(r & 15) * self.width + cx + cw]); } let (ccx, ccy, ccw, cch) = (cx / 2, cy / 2, cw.div_ceil(2), ch.div_ceil(2)); - let mut u = Vec::with_capacity(ccw * cch); - let mut v = Vec::with_capacity(ccw * cch); + let mut u = alloc(ccw * cch); + let mut v = alloc(ccw * cch); for r in ccy..ccy + cch { let row = self.row(r >> 3); let o = (r & 7) * self.cwidth + ccx; @@ -273,6 +274,18 @@ impl Frame { } } +static PLANE_ALLOCATOR: std::sync::OnceLock Vec> = std::sync::OnceLock::new(); + +/// Where output planes come from: an empty buffer with room for the given number of samples. +/// The host sets it once to recycle the planes of pictures it is done with; the first call wins. +pub fn set_plane_allocator(alloc: fn(usize) -> Vec) { + let _ = PLANE_ALLOCATOR.set(alloc); +} + +fn plane_allocator() -> fn(usize) -> Vec { + PLANE_ALLOCATOR.get().copied().unwrap_or(Vec::with_capacity) +} + /// Copy `w` samples of `line` starting at x0 into `out`; when possible a fixed-size block of `N` /// bytes is copied instead (cheaper than a variable-length copy; `out` must then have room for `N`). #[inline(always)] diff --git a/crates/platform/src/videotoolbox.rs b/crates/platform/src/videotoolbox.rs index 880ca250e..2ac6723ca 100644 --- a/crates/platform/src/videotoolbox.rs +++ b/crates/platform/src/videotoolbox.rs @@ -22,7 +22,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use filmcraft_codecs::hw::{NalCodec, NalStreamInfo}; use filmcraft_codecs::{CodecError, DecodedFrame, Result, VideoDecoder}; -use filmcraft_frame::{Chroma, PixelData, VideoFrame}; +use filmcraft_frame::{Chroma, PixelData, VideoFrame, pool}; use objc2_core_foundation::{CFBoolean, CFDictionary, CFNumber, CFRetained, CFString, CFType}; use objc2_core_media::{ CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMSampleTimingInfo, CMTime, CMTimeFlags, CMVideoFormatDescriptionCreateFromH264ParameterSets, @@ -216,11 +216,11 @@ fn copy_out(pb: &CVImageBuffer, l: &Layout) -> std::result::Result().0 { u.push(pair[0]); @@ -230,11 +230,11 @@ fn copy_out(pb: &CVImageBuffer, l: &Layout) -> std::result::Result().0.iter().map(|b| u16::from_ne_bytes([b[0], b[1]]) >> shift)); } - let (mut u, mut v) = (Vec::with_capacity(cw * ch), Vec::with_capacity(cw * ch)); + let (mut u, mut v) = (pool::take_u16(cw * ch), pool::take_u16(cw * ch)); for y in 0..ch { for q in cs.row(coy + y, cox * 2 * bps, cw * 2 * bps)?.as_chunks::<4>().0 { u.push(u16::from_ne_bytes([q[0], q[1]]) >> shift); diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 55954846d..4e32413a1 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -29,7 +29,7 @@ Methods (handlers in `crates/ui-egui/src/control.rs`): | `ui.playback` | `{action: play|stop|toggle, speed?}` | | | `ui.screenshot` | `{path?, panel?}` | PNG of the window or one panel; fails after 10 s when no frame is presented (window hidden, display asleep) | | `ui.resize`, `ui.focus`, `app.quit` | | `ui.focus` is the only request that activates the app and takes keyboard focus | -| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, `draftFrames` (decoded in draft mode), `h264Threads` (frame-threading workers per H.264 decoder), `liveDecoders` (sources holding a decoder now), `cacheMB` / `cacheBudgetMB` (decoded frames all sources hold, and the shared budget), decoder ms (total and per sample), `framesDecoded`, `hardware` (Settings ▸ Playback ▸ Hardware decoding: `enabled`, pictures from hardware decoders `frames` vs `softwareFrames`, `sessions`, `declined` streams, mid-stream `fallbacks`); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution, `draftDecode` (Settings ▸ Playback ▸ Draft decoding); `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | +| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, `draftFrames` (decoded in draft mode), `h264Threads` (frame-threading workers per H.264 decoder), `liveDecoders` (sources holding a decoder now), `cacheMB` / `cacheBudgetMB` (decoded frames all sources hold, and the shared budget), `planePoolMB` / `planesReused` (idle plane buffers of evicted frames, and planes decoded into a recycled one), decoder ms (total and per sample), `framesDecoded`, `hardware` (Settings ▸ Playback ▸ Hardware decoding: `enabled`, pictures from hardware decoders `frames` vs `softwareFrames`, `sessions`, `declined` streams, mid-stream `fallbacks`); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution, `draftDecode` (Settings ▸ Playback ▸ Draft decoding); `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | **Focus.** Driving the app never steals the user's keyboard. Started with `--control`, the app opens without activating itself. UI requests and screenshots that need a rendered frame bring the window diff --git a/docs/performance.md b/docs/performance.md index b4fc202df..f284313b6 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -43,6 +43,18 @@ to a 12 GB footprint, 10.8 GB of it live heap in three piles. memory appeared in no counter (`frames.cache` covers the frame server's rendered images and GPU plans only). +4. **Freed planes the allocator keeps (2.5 GB), MEM4.** With MEM1–MEM3 the same twelve clips took + 5.1 GB, only 2.5 GB of it live: every decoded picture allocated three planes and every + eviction freed them, and the system allocator kept the freed pages (`footprint`: 1.7 GB + reclaimable). `filmcraft_frame::pool` now keeps the planes of evicted frames (up to 192 MiB per + sample type; a frame something else still holds is left alone) and the H.264 decoder and the + VideoToolbox path decode into them, so steady-state decoding allocates no planes. + `perf.stats` reports `decode.planePoolMB` and `decode.planesReused`. + +The same session showed that the desktop app never used hardware decoding: `register()` was an +argument of a `log::info!` that no logger evaluated (fixed under HW1; `decode.hardware.sessions` +was 0 with every frame decoded in software). + What remains per live decoder is unchanged: the H.264 decoder publishes each macroblock row of a picture as its own allocations (`Frame::make_row`, five per row), which is where the 290 MB per decoder comes from.