diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 746d917bc..fb26641a3 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -217,6 +217,12 @@ pub struct Playback { /// fits in the RAM preview) is cached, so it never plays over skipped frames (#103), or /// sooner once the silent preview keeps real time with the frames ahead cached (#208). pub audio_held: bool, + /// Starting a preview with sound (#947): the playhead stays on the first frame while the + /// frames ahead render (the prefetch window, at most what fits in the RAM preview), then + /// sound and picture start together from it. Without this the picture played on silently + /// while they rendered and the sound started at the later frame, so the opening was lost. + /// A stall later on still falls back to [`Playback::audio_held`]. + pub starting: bool, /// When the last frames were shown (seconds, the last second's worth): the achieved rate. pub shown_at: std::collections::VecDeque, } @@ -1004,11 +1010,14 @@ impl EffectcraftApp { restore_max, audio_frame: (!plan.video).then_some(plan.first), audio_held: false, + starting: false, shown_at: Default::default(), }; if !self.playback.caching { if plan.video { - self.playback.audio_held = self.audio_plays(); + let audible = self.audio_plays(); + self.playback.audio_held = audible; + self.playback.starting = audible; } else { self.start_audio(fr.tick_of(plan.first)); } @@ -1068,6 +1077,7 @@ impl EffectcraftApp { let was = std::mem::replace(&mut self.playback.playing, false); self.audio = None; self.playback.caching = false; + self.playback.starting = false; if !was { return; } @@ -1262,9 +1272,28 @@ impl EffectcraftApp { return; } let snap = |f: i64| wa + (f - wa).div_euclid(step) * step; - // Held sound starts once the frames ahead are cached (see `Playback::audio_held`): the - // prefetch window when the silent preview keeps real time, else the rest of the preview. let fit = self.frames_that_fit(&c, scale); + // Starting with sound: hold the first frame until the frames ahead are cached, then start + // the sound from that same frame (see `Playback::starting`). Without an output device the + // picture plays on its own clock from it. + if self.playback.starting + && let Some(p) = plan + { + self.playback.start_wall = now; + self.playback.start_frame = cur; + if !self.cached_ahead(&p, &series, cur, ahead.min(fit)) { + self.playback.waiting = true; + ctx.request_repaint(); + return; + } + self.playback.starting = false; + self.playback.audio_held = false; + self.playback.waiting = false; + self.start_audio(fr.tick_of(cur)); + } + // Held sound starts once the frames ahead are cached again after a stall (see + // `Playback::audio_held`): the prefetch window when the silent preview keeps real time, + // else the rest of the preview. if self.playback.audio_held && let Some(p) = plan && ((self.playback.keeps_up(now) && self.cached_ahead(&p, &series, cur, ahead.min(fit))) || self.cached_ahead(&p, &series, cur, fit)) diff --git a/crates/ui-egui/tests/ui_audio_scrub.rs b/crates/ui-egui/tests/ui_audio_scrub.rs index 02fefb14f..2d256475d 100644 --- a/crates/ui-egui/tests/ui_audio_scrub.rs +++ b/crates/ui-egui/tests/ui_audio_scrub.rs @@ -113,47 +113,55 @@ fn cmd_dragging_the_current_time_scrubs_and_an_idle_scrub_closes() { assert!(h.state().scrub.is_none() && fake.0.lock().unwrap().1, "closed and stopped"); } -/// A preview with audio waits for frames like a silent one (#103): while the frames ahead aren't -/// cached, every frame shows as it renders, silently; the sound starts once the rest of the -/// preview is cached, and a frame that isn't cached stops it instead of being skipped. -#[test] -fn preview_with_audio_shows_every_frame_and_sounds_once_cached() { - let fake = Fake::default(); - let (mut app, cid) = setup(true, &fake); - // Playback requests at most twice as many frames ahead per step as there are render threads. - // Four times that many frames take at least four steps to cache, so frames are shown silently - // first however fast the renderer is (with a fixed 2 s comp, a machine with many cores cached - // all of it before the first step and the sound started at once). - let ahead = (app.frames.parallelism() * 2).max(4); - app.session.execute("comp.settings", json!({"comp": cid.0, "duration": (4 * ahead) as f64 / 30.0})).unwrap(); - // A little under one frame of input time per step. - let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 31.0).build_eframe(|_| app); - h.run_steps(2); - let frame = |h: &Harness<'_, EffectcraftApp>| { - let app = h.state(); - app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time()) - }; - let now = h.ctx.input(|i| i.time); - h.state_mut().play(now); - assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet"); - let mut last = frame(&h); - let mut shown = 0; - // Wait on the clock, not a step count: under a loaded machine (parallel test binaries) the - // background cache can take far longer than 2000 short steps to get ahead. +/// The comp's frame at the current time. +fn frame(h: &Harness<'_, EffectcraftApp>, cid: ItemId) -> i64 { + let app = h.state(); + app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time()) +} + +/// The frame under the preview audio's first sample (its clock before the device pulls any). +fn audio_frame(h: &Harness<'_, EffectcraftApp>, cid: ItemId) -> Option { + let app = h.state(); + let a = app.audio.as_ref()?; + Some(effectcraft_ui_egui::audio::frame_of_sample(a.clock(), a.rate, app.session.project.comp(cid).unwrap().frame_rate)) +} + +/// Step until the preview's sound starts (or 60 s pass, for a loaded machine), checking the +/// playhead holds `first` meanwhile. Returns whether the sound started. +fn until_sound(h: &mut Harness<'_, EffectcraftApp>, cid: ItemId, first: i64) -> bool { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); while std::time::Instant::now() < deadline { h.step(); if h.state().audio.is_some() { - break; + return true; } - let f = frame(&h); - assert!(f == last || f == last + 1 || f < last, "{last} → {f}: a frame was skipped"); - shown += usize::from(f != last); - last = f; + assert_eq!(frame(h, cid), first, "the playhead holds the first frame until the sound can start"); + assert!(h.state().playback.starting && h.state().playback.playing); std::thread::sleep(std::time::Duration::from_millis(1)); } - assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started once the frames ahead were cached"); - assert!(shown > 0, "frames played silently first"); + false +} + +/// Starting a preview with sound before its frames are cached (#947): the playhead holds the +/// first frame while the frames ahead render, then sound and picture start together from it (the +/// sound used to start at a later frame, losing the opening). A frame that isn't cached later on +/// still stops the sound rather than being skipped (#103). +#[test] +fn preview_with_audio_holds_the_first_frame_and_sounds_from_it() { + let fake = Fake::default(); + let (mut app, cid) = setup(true, &fake); + // Several steps' worth of frames to cache before the sound can start (see the prefetch). + let ahead = (app.frames.parallelism() * 2).max(4); + app.session.execute("comp.settings", json!({"comp": cid.0, "duration": (4 * ahead) as f64 / 30.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 31.0).build_eframe(|_| app); + h.run_steps(2); + let now = h.ctx.input(|i| i.time); + h.state_mut().play(now); + assert!(h.state().playback.starting && h.state().audio.is_none(), "nothing is cached: no sound yet"); + assert!(until_sound(&mut h, cid, 0), "the sound started once the frames ahead were cached"); + assert!(fake.0.lock().unwrap().0.is_some()); + assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame, not a later one"); + assert!(!h.state().playback.starting); // A frame that isn't cached stops the sound rather than being skipped. (Frames render fast // enough here to be cached again before the check: keep them out of the cache.) h.state().frames.set_budget(0); @@ -162,33 +170,106 @@ fn preview_with_audio_shows_every_frame_and_sounds_once_cached() { assert!(h.state().audio.is_none() && h.state().playback.audio_held && h.state().playback.playing); } -/// A preview whose frames render in real time sounds without waiting for the whole work area to -/// be cached (#208): once the silent preview has kept up for a moment with the frames ahead -/// cached, the sound starts, still without skipping a frame before it. +/// A long (30 s) preview sounds once the frames ahead of its first frame are cached, not the +/// whole work area (#208), and from that first frame (#947). #[test] -fn preview_with_audio_sounds_once_rendering_keeps_up() { +fn preview_with_audio_sounds_without_caching_the_work_area() { let fake = Fake::default(); let (app, cid) = setup_long(true, &fake, 30.0); let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app); h.run_steps(2); - let frame = |h: &Harness<'_, EffectcraftApp>| { - let app = h.state(); - app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time()) - }; let now = h.ctx.input(|i| i.time); h.state_mut().play(now); - assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet"); - let mut last = frame(&h); - // Two seconds of playback, each step giving the frame workers time to keep up. - for _ in 0..60 { + assert!(until_sound(&mut h, cid, 0), "the sound started"); + assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame"); +} + +/// Pull `frames` stereo frames from the output, waiting for the feeder thread to mix them. +fn pull(feed: &AudioFeed, frames: usize) -> Vec { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while feed.queued_frames() < frames && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + let mut out = vec![0.0f32; frames * 2]; + feed.pull(&mut out); + out +} + +/// The opening of the preview is heard (#947): a sound that lasts only the first 100 ms after the +/// start, from the beginning of the comp or from the middle, reaches the output device. +#[test] +fn a_sound_in_the_first_100_ms_is_heard() { + for start in [0.0, 1.0] { + let fake = Fake::default(); + let (mut app, cid) = setup_long(false, &fake, 4.0); + // A tone only from `start` to `start` + 0.1 s. + let l = app.session.execute("layer.newSolid", json!({"color": "#808080"})).unwrap()["layer"].as_u64().unwrap(); + app.session.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap(); + app.session.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap(); + app.session.execute("layer.timing", json!({"layers": [l], "in": start, "out": start + 0.1})).unwrap(); + app.session.execute("time.set", json!({"time": start})).unwrap(); + let first = (start * 30.0) as i64; + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app); + h.run_steps(2); + let now = h.ctx.input(|i| i.time); + h.state_mut().play(now); + assert!(until_sound(&mut h, cid, first), "{start}: the sound started"); + assert_eq!(audio_frame(&h, cid), Some(first), "{start}: from the first frame"); + let feed = fake.0.lock().unwrap().0.clone().expect("the output opened"); + // The first 100 ms at 48 kHz. + let out = pull(&feed, 4800); + assert!(out.iter().any(|v| v.abs() > 0.01), "{start}: the opening tone reached the output"); + } +} + +/// A preview without video starts its sound at once, from the first frame (unchanged), and so +/// does one whose frames are already cached. +#[test] +fn audio_only_previews_start_at_once() { + let fake = Fake::default(); + let (app, cid) = setup(true, &fake); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app); + h.run_steps(2); + let now = h.ctx.input(|i| i.time); + h.state_mut().play_with(now, effectcraft_engine::preview::PreviewShortcut::AltNumpad0); + assert!(h.state().audio.is_some() && !h.state().playback.starting, "Video off: the sound starts at once"); + assert_eq!(audio_frame(&h, cid), Some(0)); +} + +/// Without an output device the preview still plays, from the first frame, once the frames ahead +/// are cached; Stop while it prepares leaves nothing playing. +#[test] +fn no_device_plays_the_picture_and_stop_ends_the_wait() { + let fake = Fake::default(); + let (mut app, cid) = setup_long(true, &fake, 10.0); + app.hooks.audio_device = Some(Box::new(|_| None)); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app); + h.run_steps(2); + let now = h.ctx.input(|i| i.time); + h.state_mut().play(now); + assert!(h.state().playback.starting); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while h.state().playback.starting && std::time::Instant::now() < deadline { + h.step(); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!(!h.state().playback.starting && h.state().audio.is_none() && h.state().playback.playing, "no device: the picture plays alone"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while frame(&h, cid) == 0 && std::time::Instant::now() < deadline { h.step(); - if h.state().audio.is_some() { - break; - } - let f = frame(&h); - assert!(f == last || f == last + 1, "{last} → {f}: a frame was skipped"); - last = f; - std::thread::sleep(std::time::Duration::from_millis(20)); - } - assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started within 2 s, at frame {last} of 900"); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(frame(&h, cid) > 0, "from the first frame on"); + // Stop while preparing. + h.state_mut().stop(); + let now = h.ctx.input(|i| i.time); + h.state_mut().hooks.audio_device = Some(Box::new(move |_| Some(Box::new(fake.clone()) as Box))); + h.state().frames.clear(); + h.state_mut().play(now); + assert!(h.state().playback.starting); + h.state_mut().stop(); + for _ in 0..10 { + h.step(); + } + assert!(!h.state().playback.playing && !h.state().playback.starting && h.state().audio.is_none(), "Stop while preparing: nothing plays"); }