diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index e1b953d7b..d351ca411 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -406,7 +406,8 @@ impl EffectcraftApp { } let cur = comp.frame_rate.frame_at(self.session.time()).clamp(wa, wb); let fit = self.frames_that_fit(comp, series.scale as f64 / 1000.0).min(n); - let slots = (self.frames_parallelism() / 2).max(1); + // One thread each (`Frames::request_background`), so every frame thread takes one. + let slots = self.frames_parallelism().max(1); let mut queued = self.frames.inflight(); let scale = series.scale as f64 / 1000.0; let mut pending = false; @@ -419,7 +420,7 @@ impl EffectcraftApp { if queued >= slots { break; } - self.request_frame(ItemId(series.comp), f, scale); + self.request_frame_background(ItemId(series.comp), f, scale); queued += 1; } if pending { diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rs index 3c8632cea..c878fafa4 100644 --- a/crates/ui-egui/src/app/viewer_frames.rs +++ b/crates/ui-egui/src/app/viewer_frames.rs @@ -126,12 +126,18 @@ impl EffectcraftApp { /// Request a prefetch frame render (no-op if cached/in flight). pub fn request_frame(&self, comp: ItemId, frame: i64, scale: f64) { - self.request_frame_with(comp, frame, scale, false); + self.request_frame_with(comp, frame, scale, frames::Frames::request); } /// Request the frame on screen: rendered before any queued prefetch. pub fn request_frame_urgent(&self, comp: ItemId, frame: i64, scale: f64) { - self.request_frame_with(comp, frame, scale, true); + self.request_frame_with(comp, frame, scale, frames::Frames::request_urgent); + } + + /// Request a frame nobody waits for (Cache Frames When Idle): rendered on one thread + /// (`Frames::request_background`). + pub fn request_frame_background(&self, comp: ItemId, frame: i64, scale: f64) { + self.request_frame_with(comp, frame, scale, frames::Frames::request_background); } /// Render options of viewer frames of `comp` at `scale`. @@ -152,15 +158,17 @@ impl EffectcraftApp { } } - fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { + fn request_frame_with( + &self, + comp: ItemId, + frame: i64, + scale: f64, + request: fn(&frames::Frames, &frames::RenderSource, FrameKey, ItemId, Tick, RenderOpts), + ) { let Some(c) = self.session.project.comp(comp) else { return }; let key = self.frame_key(comp, frame, scale); let t = c.frame_rate.tick_of(frame); let opts = self.frame_opts(comp, scale); - if urgent { - self.frames.request_urgent(&self.render_source(), key, comp, t, opts); - } else { - self.frames.request(&self.render_source(), key, comp, t, opts); - } + request(&self.frames, &self.render_source(), key, comp, t, opts); } } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index b524b5959..883cbf172 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -1,7 +1,8 @@ //! Background frame rendering and the RAM preview cache. //! //! Frames render on a dedicated thread pool from immutable `Arc` snapshots, so the UI -//! never waits for the compositor. Results land in a memory-budgeted cache keyed by the comp's +//! never waits for the compositor (a playing preview's frames and idle caching on one thread +//! each, so the frames started first are done first). Results land in a memory-budgeted cache keyed by the comp's //! *content* ([`comp_content`]: the comp and everything it uses), frame, scale, view and render //! options; the timeline draws the cached range as the green cache bar. An edit keeps the frames //! of every comp it doesn't touch, and undo finds the frames of the state it returns to. @@ -456,6 +457,10 @@ pub struct Frames { #[cfg(not(target_arch = "wasm32"))] /// None if the OS refused the threads: jobs then go to rayon's global pool. pool: Option, + #[cfg(not(target_arch = "wasm32"))] + /// One thread each, for a playing preview's frames and idle caching (see [`Queue::idle_lanes`]). Empty if + /// the OS refused the threads: those frames then render on `pool` too. + lanes: Vec, ctx: Option, /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, @@ -484,10 +489,25 @@ impl Default for Frames { fn default() -> Self { #[cfg(not(target_arch = "wasm32"))] let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(2, 16); + #[cfg(not(target_arch = "wasm32"))] + let lanes: Vec = (0..threads) + .map_while(|i| { + rayon::ThreadPoolBuilder::new() + .num_threads(1) + .thread_name(move |_| format!("ec-preview-{i}")) + .panic_handler(|_| log::error!("a frame render panicked; that frame was skipped")) + .build() + .ok() + }) + .collect(); + #[cfg(not(target_arch = "wasm32"))] + let queue = Queue { idle_lanes: (0..lanes.len()).collect(), lanes: lanes.len(), ..Default::default() }; + #[cfg(target_arch = "wasm32")] + let queue = Queue::default(); Frames { cache: Arc::new(Mutex::new(Cache { map: HashMap::new(), order: VecDeque::new(), bytes: 0, gpu_bytes: 0, budget: 3 << 30, gpu_budget: GPU_BUDGET, playhead: None, scale: None })), inflight: Arc::new(Mutex::new(HashSet::new())), - queue: Arc::new(Mutex::new(Queue::default())), + queue: Arc::new(Mutex::new(queue)), #[cfg(not(target_arch = "wasm32"))] // A panicking job would abort the whole process (rayon's default for `spawn`): log it // and lose only that frame (the panic hook has logged the message). @@ -497,6 +517,8 @@ impl Default for Frames { .panic_handler(|_| log::error!("a frame render panicked; that frame was skipped")) .build() .ok(), + #[cfg(not(target_arch = "wasm32"))] + lanes, ctx: None, last_ms: Arc::new(Mutex::new(0.0)), content_keys: Arc::default(), @@ -911,26 +933,77 @@ impl Frames { c.playhead = playhead; } let Ok(mut q) = self.queue.lock() else { return }; + // A preview starting or stopping moves the queued prefetch frames between the lanes and + // the pool (see [`Queue::idle_lanes`]). + #[cfg(not(target_arch = "wasm32"))] + let moved = q.playhead.is_some() != playhead.is_some(); q.playhead = playhead; - let Some(p) = playhead else { return }; - let Ok(mut inf) = self.inflight.lock() else { return }; - q.jobs.retain(|j| { - let gone = !j.urgent && j.key.same_series(&p.series) && p.ahead(&j.key).is_none(); - if gone { - inf.remove(&j.key); + if let Some(p) = playhead + && let Ok(mut inf) = self.inflight.lock() + { + q.jobs.retain(|j| { + let gone = !j.urgent && j.key.same_series(&p.series) && p.ahead(&j.key).is_none(); + if gone { + inf.remove(&j.key); + } + !gone + }); + } + #[cfg(not(target_arch = "wasm32"))] + if moved { + self.start_lanes(&mut q, usize::MAX); + let pool_jobs = q.jobs.iter().filter(|j| !q.on_lane(j)).count().min(self.parallelism()); + drop(q); + for _ in 0..pool_jobs { + self.start_pool(); } - !gone - }); + } + } + + /// Start up to `n` idle lanes on the queued jobs (see [`Queue::idle_lanes`]). + #[cfg(not(target_arch = "wasm32"))] + fn start_lanes(&self, q: &mut Queue, n: usize) { + if self.remote.is_some() { + return; + } + let jobs = q.jobs.iter().filter(|j| q.on_lane(j)).count(); + for _ in 0..n.min(jobs) { + let Some(lane) = q.idle_lanes.pop() else { break }; + let Some(pool) = self.lanes.get(lane) else { break }; + let w = self.worker(); + pool.spawn(move || w.drain(Some(lane))); + } + } + + /// Have a pool thread render the queued jobs that are the pool's. + #[cfg(not(target_arch = "wasm32"))] + fn start_pool(&self) { + if self.remote.is_some() { + return; + } + let w = self.worker(); + let job = move || w.drain(None); + match &self.pool { + Some(pool) => pool.spawn(job), + None => rayon::spawn(job), + } } /// Queue a prefetch frame (no-op if cached or already queued/rendering). pub fn request(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts) { - self.request_with(src, key, comp, t, opts, false); + self.request_with(src, key, comp, t, opts, Want::Prefetch); + } + + /// Queue a frame nobody waits for (Cache Frames When Idle): as [`Frames::request`], but it + /// renders on a lane, one thread, and leaves the pool to the viewer's frame (see + /// [`Queue::idle_lanes`]). + pub fn request_background(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts) { + self.request_with(src, key, comp, t, opts, Want::Background); } /// Queue the frame the viewer is showing: it jumps ahead of every prefetch job. pub fn request_urgent(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts) { - self.request_with(src, key, comp, t, opts, true); + self.request_with(src, key, comp, t, opts, Want::Viewer); } /// An urgent (viewer) frame is queued or rendering. @@ -938,7 +1011,8 @@ impl Frames { self.queue.lock().map(|q| q.urgent_running > 0 || q.jobs.iter().any(|j| j.urgent)).unwrap_or(false) } - fn request_with(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts, urgent: bool) { + fn request_with(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts, want: Want) { + let urgent = want == Want::Viewer; // The frame on screen sets the scale evictions keep: a prefetch or a passive viewer at // another zoom must not evict the active viewer's frames. let cached = self @@ -990,22 +1064,32 @@ impl Frames { } } if !inf.insert(key) { + drop(inf); + drop(q); + // Queued for a lane and now the viewer's frame: it is the pool's. + #[cfg(not(target_arch = "wasm32"))] + if urgent { + self.start_pool(); + } return; } q.seq += 1; let seq = q.seq; - q.jobs.push(Job { key, src: src.clone(), comp, t, opts, urgent, seq, cancel: Arc::default() }); + let job = Job { key, src: src.clone(), comp, t, opts, urgent, background: want == Want::Background, seq, cancel: Arc::default() }; + // A lane's frame: a busy lane takes the next job when it is done, an idle one starts now. + #[cfg(not(target_arch = "wasm32"))] + let on_lane = q.on_lane(&job); + q.jobs.push(job); + #[cfg(not(target_arch = "wasm32"))] + if on_lane { + self.start_lanes(&mut q, 1); + } drop(inf); drop(q); + // One spawn per job; each spawn renders whichever queued jobs matter most right now. #[cfg(not(target_arch = "wasm32"))] - if self.remote.is_none() { - let w = self.worker(); - // One spawn per job; each spawn renders whichever queued jobs matter most right now. - let job = move || w.drain(); - match &self.pool { - Some(pool) => pool.spawn(job), - None => rayon::spawn(job), - } + if !on_lane { + self.start_pool(); } #[cfg(target_arch = "wasm32")] if let Some(ctx) = &self.ctx { @@ -1147,7 +1231,7 @@ impl Frames { } let t0 = web_time::Instant::now(); let w = self.worker(); - while w.run_next() { + while w.run_next(None) { if t0.elapsed() >= budget { break; } @@ -1195,22 +1279,29 @@ impl Worker { /// every UI frame, so renders piled up on every thread until memory ran out (#647). A spawn /// that lands inside a render returns at once instead; the render below it takes the queued /// jobs when it is done, so each thread renders one frame at a time. + /// + /// `lane`: the lane this runs on (see [`Queue::idle_lanes`]), `None` on the pool. #[cfg(not(target_arch = "wasm32"))] - fn drain(&self) { + fn drain(&self, lane: Option) { if DRAINING.get() { return; } DRAINING.set(true); let _reset = Draining; - while self.run_next() {} + while self.run_next(lane) {} } - /// Render the queued job that matters most right now. `false` when the queue was empty. - fn run_next(&self) -> bool { + /// Render the queued job that matters most right now, on lane `lane` (`None`: on + /// the pool, or the UI thread). `false` when the queue had no job for it; the lane is then + /// idle again. + fn run_next(&self, lane: Option) -> bool { let Worker { queue, content_keys, .. } = self; let job = { let Ok(mut q) = queue.lock() else { return false }; - let Some(i) = q.best() else { return false }; + let Some(i) = q.next(lane.is_some()) else { + q.idle_lanes.extend(lane); + return false; + }; let job = q.jobs.swap_remove(i); q.running.insert(job.key); if job.urgent { @@ -1405,6 +1496,14 @@ impl Worker { } } +/// Who asked for a frame (see [`Frames::request_with`]). +#[derive(Clone, Copy, PartialEq, Eq)] +enum Want { + Viewer, + Prefetch, + Background, +} + struct Job { key: FrameKey, src: RenderSource, @@ -1412,6 +1511,8 @@ struct Job { t: Tick, opts: RenderOpts, urgent: bool, + /// Nobody waits for it (see [`Frames::request_background`]). + background: bool, seq: u64, /// Set when the viewer no longer waits for this frame (see [`Frames::request_urgent`]). cancel: Arc, @@ -1434,18 +1535,45 @@ struct Queue { seq: u64, /// The playing preview, if any: its prefetch renders nearest first. playhead: Option, + /// How many lanes there are, and the ones not rendering. A lane is one thread that renders + /// one frame at a time. The frames of a playing preview and the frames nobody waits for + /// (Cache Frames When Idle) render on the lanes; the pool keeps the viewer's frame and the + /// few frames prefetched around it while paused. On the pool a frame's parallel work is + /// shared: a thread that took a piece of one frame and waited inside it started rendering + /// the queued frames on top, and went back to its piece only once the queue was empty. The + /// frames a preview started first, the ones right ahead of the playhead, were done last: + /// the preview stood still on a gap while the frames further ahead turned green. Idle + /// caching left the same gaps, and held up the viewer's frame when it shared a piece. + lanes: usize, + idle_lanes: Vec, } impl Queue { + /// Whether `j` renders on a lane (else on the pool). + fn on_lane(&self, j: &Job) -> bool { + self.lanes > 0 && !j.urgent && (j.background || self.playhead.is_some()) + } + + /// The job a lane (`lane`) or the pool renders next: the [`Queue::best`] of its own. + fn next(&self, lane: bool) -> Option { + self.best_of(|j| self.on_lane(j) == lane) + } + /// The job that matters most: urgent first (newest), then, while a preview plays, the /// prefetch frame it reaches first, then prefetch in request order. In request order alone, /// frames asked for again after an eviction (the ones right ahead of the playhead) waited /// behind every frame further ahead (#595). fn best(&self) -> Option { + self.best_of(|_| true) + } + + /// [`Queue::best`] among the jobs `of` accepts. + fn best_of(&self, of: impl Fn(&Job) -> bool) -> Option { let p = self.playhead.as_ref(); self.jobs .iter() .enumerate() + .filter(|(_, j)| of(j)) .max_by_key(|(_, j)| { let near = if j.urgent { None } else { p.and_then(|p| p.ahead(&j.key)) }; (j.urgent, near.is_some(), near.map_or(0, |d| -d), if j.urgent { j.seq as i64 } else { -(j.seq as i64) }) diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index a2603dfec..49f77a6cf 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -437,7 +437,9 @@ fn a_newer_viewer_frame_drops_the_ones_still_waiting() { gpu_display: false, disk: None, }; - let request = |k: FrameKey, urgent: bool| f.request_with(&src, k, ItemId(k.comp), Tick::ZERO, RenderOpts::default(), urgent); + let request = |k: FrameKey, urgent: bool| { + f.request_with(&src, k, ItemId(k.comp), Tick::ZERO, RenderOpts::default(), if urgent { Want::Viewer } else { Want::Prefetch }) + }; let other = FrameKey { comp: 2, ..key(1, 0, 0) }; request(other, true); request(key(1, 20, 0), false); @@ -558,6 +560,131 @@ fn prefetch_renders_the_frames_the_playhead_reaches_first() { assert_eq!(f.inflight(), 6, "frame 5, behind the playhead, is no longer in flight"); } +/// A decoder that works in parallel, as the renderer does, in a comp that shows it. +#[derive(Default)] +struct ParallelClip { + started: std::sync::atomic::AtomicUsize, + /// Frames started by the time frame 1 was done. + started_at_first: std::sync::atomic::AtomicUsize, +} + +impl FootageSource for ParallelClip { + fn frame(&self, _: ItemId, _: &effectcraft_engine::project::Footage, t: Tick) -> Option> { + self.started.fetch_add(1, Ordering::SeqCst); + (0..16).into_par_iter().for_each(|_| (0..4).into_par_iter().for_each(|_| std::thread::sleep(std::time::Duration::from_millis(1)))); + if effectcraft_engine::time::FrameRate::FPS_30.frame_at(t) == 1 { + self.started_at_first.store(self.started.load(Ordering::SeqCst), Ordering::SeqCst); + } + Some(Arc::new(effectcraft_engine::render::Image::new(4, 4))) + } +} + +/// A 10 s comp of a [`ParallelClip`]: the frame cache, the clip, the render source, the comp +/// and the key of its frame 0. +fn parallel_clip() -> (Frames, Arc, RenderSource, ItemId, FrameKey) { + use effectcraft_engine::Session; + use effectcraft_engine::project::{Footage, FootageKind}; + use effectcraft_engine::time::FrameRate; + use serde_json::json; + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 10})).unwrap()["comp"].as_u64().unwrap()); + let clip = Footage { + kind: FootageKind::Video, + width: 4, + height: 4, + has_video: true, + frame_rate: FrameRate::FPS_30, + duration: Tick::from_seconds_f64(10.0), + ..Default::default() + }; + let item = Arc::make_mut(&mut s.project).add_item("clip.mov", Default::default(), None, ItemKind::Footage(clip)); + s.execute("layer.addItem", json!({"item": item.0})).unwrap(); + let footage = Arc::new(ParallelClip::default()); + let src = RenderSource { + project: s.project.clone(), + footage: footage.clone(), + expr: None, + layer_cache: Arc::default(), + gpu: None, + gpu_display: false, + disk: None, + }; + let f = Frames::default(); + let series = FrameKey { revision: s.revision, content: f.content_of(&s.project, s.revision, cid), comp: cid.0, frame: 0, scale: 1000, view: 0, opts: 0 }; + (f, footage, src, cid, series) +} + +fn wait_for_renders(f: &Frames) { + for _ in 0..8000 { + if f.inflight() == 0 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!(f.inflight(), 0, "every queued frame rendered"); +} + +/// A playing preview's frames render side by side, and each works in parallel: a render +/// waiting for its own parallel work took over pieces of the frames started after it, and +/// its frame was done only once theirs were. The frames right ahead of the playhead, started +/// first, came last: the preview stood still while the frames further ahead turned green. +/// Cache Frames When Idle left the same gaps. +#[test] +fn frames_started_first_are_done_first() { + use effectcraft_engine::time::FrameRate; + for playing in [true, false] { + let (f, footage, src, cid, series) = parallel_clip(); + let frames = 60; + if playing { + f.set_playhead(Some(Playhead { series, cur: 0, start: 0, end: 299, step: 1, looping: false })); + } + for frame in 1..=frames { + let (key, t) = (FrameKey { frame, ..series }, FrameRate::FPS_30.tick_of(frame)); + if playing { + f.request(&src, key, cid, t, RenderOpts::default()); + } else { + f.request_background(&src, key, cid, t, RenderOpts::default()); + } + if frame == 1 { + // Its parallel work has started (and idle threads have taken pieces of it). + std::thread::sleep(std::time::Duration::from_millis(3)); + } + } + wait_for_renders(&f); + assert_eq!(f.cached_frames(&series).len(), frames as usize); + let before = footage.started_at_first.load(Ordering::SeqCst); + assert!(before <= 2 * f.parallelism(), "{before} frames were started before the first one was done (playing: {playing})"); + } +} + +/// Prefetch frames queued when a preview starts or stops, or that become the viewer's +/// frame, still render: they change hands between the lanes and the pool. +#[test] +fn queued_frames_render_when_a_preview_starts_or_stops() { + use effectcraft_engine::time::FrameRate; + let (f, _, src, cid, series) = parallel_clip(); + let playhead = Playhead { series, cur: 0, start: 0, end: 299, step: 1, looping: false }; + let request = |frames: std::ops::Range| { + for frame in frames { + f.request(&src, FrameKey { frame, ..series }, cid, FrameRate::FPS_30.tick_of(frame), RenderOpts::default()); + } + }; + // Queued for the pool, then a preview starts. + request(1..40); + f.set_playhead(Some(playhead)); + // Queued for the lanes, then the preview stops. + request(40..80); + f.set_playhead(None); + wait_for_renders(&f); + // Queued for the lanes, then the viewer shows one of them. + for frame in 80..120 { + f.request_background(&src, FrameKey { frame, ..series }, cid, FrameRate::FPS_30.tick_of(frame), RenderOpts::default()); + } + f.request_urgent(&src, FrameKey { frame: 119, ..series }, cid, FrameRate::FPS_30.tick_of(119), RenderOpts::default()); + wait_for_renders(&f); + assert_eq!(f.cached_frames(&series).len(), 119); +} + /// The cache is full. A zoom (or preview-resolution) change left the frames of the old /// size to the plain least-recently-shown order, so the frames of the zoom the viewer is back /// at went before the ones of the abandoned zoom. The other size's frames go first, then the