M11.2: audio playback + waveforms + audio effects

- Audio effects (Backwards, Bass & Treble, Delay, Flange & Chorus,
  High-Low Pass, Modulator, Parametric EQ, Reverb, Stereo Mixer, Tone)
  applied per layer in the comp mixdown, so export and preview share them.
  RBJ-cookbook biquads, Schroeder/Moorer reverb, modulated delay lines;
  stateless per block via pre-roll. Tone makes any layer audible.
- Real-time audio preview: ui-egui AudioFeed/AudioPlayback (feeder thread
  mixing ahead) behind an AudioDevice hook; the desktop app implements it
  with cpal (L6). Audio clock drives playback (consumed frames minus device
  latency, looped through the work area). Preview panel Include Audio /
  playback.muteAudio.
- Audio panel VU meters with peak hold and clip indicators; selected
  layer's Audio Levels.
- Timeline Audio > Waveform for footage with audio (L reveals Audio
  Levels, LL the Waveform), drawn from a cached per-item peak summary.
- ui.playback reports audio state and meter levels.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
2026-10-02 00:16:34 -04:00
co-authored by Claude Opus 5.5
parent 1c0da40173
commit ddb33cdfd0
19 changed files with 1483 additions and 77 deletions
+345
View File
@@ -0,0 +1,345 @@
//! Real-time audio preview.
//!
//! The frontend owns no audio API: the native app supplies an [`AudioDevice`] (cpal) through
//! [`crate::Hooks::audio_device`]. During preview playback an [`AudioPlayback`] runs a feeder
//! thread that mixes the composition ahead of the device with the same mixdown export uses
//! (`effectcraft_render::audio::mix_comp`: Audio switch, solo, Audio Levels, audio effects) and
//! pushes it into an [`AudioFeed`]; the device callback pulls from the feed.
//!
//! **A/V sync:** while audio plays, the audio clock drives playback. The clock is the number of
//! sample frames the device has actually consumed (underruns do not advance it) minus the
//! device's output latency, mapped through the work-area loop ([`clock_sample`]); the viewer
//! shows the comp frame at that sample ([`frame_of_sample`]), dropping video frames rather
//! than drifting.
//!
//! The feed also tracks per-channel peaks for the Audio panel's VU meters ([`Meter`]).
use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use effectcraft_engine::project::ItemId;
use effectcraft_engine::time::{FrameRate, Tick};
use crate::frames::RenderSource;
/// An audio output (implemented by the native app with cpal).
pub trait AudioDevice: Send {
/// Device sample rate in Hz.
fn sample_rate(&self) -> u32;
/// Start pulling interleaved stereo frames from `feed` (see [`AudioFeed::pull`]).
fn start(&mut self, feed: Arc<AudioFeed>) -> Result<(), String>;
fn stop(&mut self);
/// Output latency in sample frames (device buffering), for A/V sync.
fn latency_frames(&self) -> u64 {
0
}
}
/// Opens the default output device (`None` when there is none).
pub type AudioDeviceFactory = Box<dyn Fn() -> Option<Box<dyn AudioDevice>>>;
/// Lock-protected sample queue between the feeder thread and the device callback.
#[derive(Default)]
pub struct AudioFeed {
queue: Mutex<VecDeque<f32>>,
/// Sample frames handed to the device (real audio only: underruns do not count).
consumed: AtomicU64,
/// Peak |sample| per channel since the last [`AudioFeed::take_peaks`] (f32 bits).
peak: [AtomicU32; 2],
/// The feeder reached the end of a non-looping span.
finished: AtomicBool,
}
impl AudioFeed {
/// Fill `out` (interleaved stereo) from the queue; silence where the queue runs dry.
pub fn pull(&self, out: &mut [f32]) {
let mut real = 0usize;
let mut pk = [0.0f32; 2];
if let Ok(mut q) = self.queue.lock() {
let n = q.len().min(out.len()) & !1;
for (i, (o, s)) in out.iter_mut().zip(q.drain(..n)).enumerate() {
*o = s;
pk[i & 1] = pk[i & 1].max(s.abs());
}
real = n;
}
out[real..].iter_mut().for_each(|o| *o = 0.0);
self.consumed.fetch_add((real / 2) as u64, Ordering::Relaxed);
for (c, v) in pk.into_iter().enumerate() {
let _ = self.peak[c].fetch_update(Ordering::Relaxed, Ordering::Relaxed, |old| (v > f32::from_bits(old)).then_some(v.to_bits()));
}
}
pub fn push(&self, samples: &[f32]) {
if let Ok(mut q) = self.queue.lock() {
q.extend(samples.iter().copied());
}
}
/// Sample frames waiting in the queue.
pub fn queued_frames(&self) -> usize {
self.queue.lock().map(|q| q.len() / 2).unwrap_or(0)
}
pub fn consumed(&self) -> u64 {
self.consumed.load(Ordering::Relaxed)
}
/// Peak |sample| per channel since the last call (resets them).
pub fn take_peaks(&self) -> [f32; 2] {
[f32::from_bits(self.peak[0].swap(0, Ordering::Relaxed)), f32::from_bits(self.peak[1].swap(0, Ordering::Relaxed))]
}
pub fn finished(&self) -> bool {
self.finished.load(Ordering::Relaxed) && self.queued_frames() == 0
}
}
/// What a preview plays, in comp sample indices at the device rate.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PlaySpan {
pub start: i64,
/// Work area [wa, wb).
pub wa: i64,
pub wb: i64,
pub looping: bool,
}
impl PlaySpan {
/// Span for comp times: start, work area [wa, wb).
pub fn new(start: Tick, wa: Tick, wb: Tick, looping: bool, rate: u32) -> PlaySpan {
let r = rate as i64;
PlaySpan { start: start.to_units_floor(r), wa: wa.to_units_floor(r), wb: wb.to_units_floor(r).max(wa.to_units_floor(r) + 1), looping }
}
}
/// The comp sample index audible now: `consumed` frames played by the device minus its
/// `latency`, from the span start, wrapped into the work area when looping (clamped at its end
/// otherwise).
pub fn clock_sample(span: &PlaySpan, consumed: u64, latency: u64) -> i64 {
let s = span.start + consumed.saturating_sub(latency) as i64;
if s < span.wb {
return s;
}
if span.looping { span.wa + (s - span.wb).rem_euclid((span.wb - span.wa).max(1)) } else { span.wb - 1 }
}
/// Comp time of sample index `s` at `rate` Hz.
pub fn sample_time(s: i64, rate: u32) -> Tick {
Tick::from_units(s, rate.max(1) as i64)
}
/// The comp frame showing sample `s` (the frame whose interval contains it).
pub fn frame_of_sample(s: i64, rate: u32, fps: FrameRate) -> i64 {
fps.frame_at(sample_time(s, rate))
}
/// Block size the feeder mixes at a time.
const BLOCK: usize = 1024;
/// A running audio preview: device + feeder thread.
pub struct AudioPlayback {
pub feed: Arc<AudioFeed>,
device: Box<dyn AudioDevice>,
stop: Arc<AtomicBool>,
thread: Option<JoinHandle<()>>,
pub rate: u32,
pub span: PlaySpan,
}
impl AudioPlayback {
/// Start mixing `comp` over `span` into `device`.
pub fn start(
mut device: Box<dyn AudioDevice>,
src: RenderSource,
comp: ItemId,
start: Tick,
wa: Tick,
wb: Tick,
looping: bool,
) -> Result<AudioPlayback, String> {
let rate = device.sample_rate().max(8000);
let span = PlaySpan::new(start, wa, wb, looping, rate);
let feed = Arc::new(AudioFeed::default());
let stop = Arc::new(AtomicBool::new(false));
// Prime ~50 ms so the device does not start on an empty queue.
let mut cursor = span.start;
let prime = (rate as usize / 20).div_ceil(BLOCK);
for _ in 0..prime {
cursor = feed_block(&src, comp, &feed, &span, cursor, rate);
}
let thread = {
let (feed, stop) = (feed.clone(), stop.clone());
std::thread::Builder::new()
.name("ec-audio-feed".into())
.spawn(move || {
// Keep ~200 ms queued ahead of the device.
let target = rate as usize / 5;
while !stop.load(Ordering::Relaxed) {
if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) {
cursor = feed_block(&src, comp, &feed, &span, cursor, rate);
} else {
std::thread::sleep(std::time::Duration::from_millis(4));
}
}
})
.map_err(|e| e.to_string())?
};
device.start(feed.clone())?;
Ok(AudioPlayback { feed, device, stop, thread: Some(thread), rate, span })
}
/// The comp sample index audible now.
pub fn clock(&self) -> i64 {
clock_sample(&self.span, self.feed.consumed(), self.device.latency_frames())
}
pub fn finished(&self) -> bool {
self.feed.finished()
}
pub fn stop(&mut self) {
self.device.stop();
self.stop.store(true, Ordering::Relaxed);
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
impl Drop for AudioPlayback {
fn drop(&mut self) {
self.stop();
}
}
/// Mix one block at `cursor` into the feed; returns the next cursor (wrapping or finishing at
/// the work-area end).
fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32) -> i64 {
let mut cursor = cursor;
if cursor >= span.wb {
if !span.looping {
feed.finished.store(true, Ordering::Relaxed);
return cursor;
}
cursor = span.wa;
}
let n = ((span.wb - cursor) as usize).min(BLOCK);
let buf = effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, sample_time(cursor, rate), n, rate);
feed.push(&buf);
cursor + n as i64
}
/// VU meter state for the Audio panel: smoothed level with a falling decay and peak hold, in
/// dBFS per channel.
#[derive(Clone, Copy, Debug)]
pub struct Meter {
pub level_db: [f32; 2],
pub peak_db: [f32; 2],
peak_at: [f64; 2],
/// Clip indicators (latched until clicked / playback restarts).
pub clipped: [bool; 2],
last: f64,
}
impl Default for Meter {
fn default() -> Self {
Meter { level_db: [METER_FLOOR; 2], peak_db: [METER_FLOOR; 2], peak_at: [0.0; 2], clipped: [false; 2], last: 0.0 }
}
}
/// Bottom of the meter scale (dB).
pub const METER_FLOOR: f32 = -48.0;
/// How long the peak marker holds (s) and how fast levels fall (dB/s).
const PEAK_HOLD: f64 = 1.5;
const FALL_DB_PER_S: f32 = 30.0;
pub fn to_db(v: f32) -> f32 {
if v <= 1e-6 { METER_FLOOR } else { (20.0 * v.log10()).max(METER_FLOOR) }
}
impl Meter {
/// Feed the peaks measured since the last update at wall time `now` (seconds).
pub fn update(&mut self, peaks: [f32; 2], now: f64) {
let dt = if self.last > 0.0 { (now - self.last).clamp(0.0, 0.5) as f32 } else { 0.0 };
self.last = now;
for c in 0..2 {
let db = to_db(peaks[c]);
self.level_db[c] = db.max(self.level_db[c] - FALL_DB_PER_S * dt);
if db >= self.peak_db[c] || now - self.peak_at[c] > PEAK_HOLD {
self.peak_db[c] = db.max(self.level_db[c]);
self.peak_at[c] = now;
}
if peaks[c] >= 1.0 {
self.clipped[c] = true;
}
}
}
/// Anything to animate (levels still falling)?
pub fn active(&self) -> bool {
self.level_db.iter().chain(self.peak_db.iter()).any(|d| *d > METER_FLOOR)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn clock_follows_consumed_frames_and_loops() {
let span = PlaySpan { start: 4800, wa: 0, wb: 48000, looping: true };
assert_eq!(clock_sample(&span, 0, 0), 4800);
// Latency holds the clock back until the device has played through its buffer.
assert_eq!(clock_sample(&span, 512, 1024), 4800);
assert_eq!(clock_sample(&span, 2048, 1024), 4800 + 1024);
// Wrap at the work-area end.
assert_eq!(clock_sample(&span, 48000 - 4800 + 100, 0), 100);
let once = PlaySpan { looping: false, ..span };
assert_eq!(clock_sample(&once, 100_000, 0), 47999);
}
#[test]
fn frame_mapping_matches_comp_frames() {
let fps = FrameRate::new(30, 1);
// 48 kHz at 30 fps: 1600 samples per frame.
assert_eq!(frame_of_sample(0, 48000, fps), 0);
assert_eq!(frame_of_sample(1599, 48000, fps), 0);
assert_eq!(frame_of_sample(1600, 48000, fps), 1);
assert_eq!(frame_of_sample(48000 * 2, 48000, fps), 60);
let span = PlaySpan::new(Tick::from_seconds_f64(1.0), Tick::ZERO, Tick::from_seconds_f64(2.0), true, 48000);
assert_eq!((span.start, span.wa, span.wb), (48000, 0, 96000));
// 29.97: frame boundaries are fractional in samples.
let ntsc = FrameRate::new(30000, 1001);
assert_eq!(frame_of_sample(1601, 48000, ntsc), 0);
assert_eq!(frame_of_sample(1602, 48000, ntsc), 1);
}
#[test]
fn feed_counts_real_frames_and_peaks() {
let f = AudioFeed::default();
f.push(&[0.5, -0.25, 0.1, 0.9]);
let mut out = [1.0f32; 8];
f.pull(&mut out);
assert_eq!(out, [0.5, -0.25, 0.1, 0.9, 0.0, 0.0, 0.0, 0.0]);
assert_eq!(f.consumed(), 2);
assert_eq!(f.take_peaks(), [0.5, 0.9]);
assert_eq!(f.take_peaks(), [0.0, 0.0]);
}
#[test]
fn meter_decays_and_holds_peaks() {
let mut m = Meter::default();
m.update([1.0, 0.5], 1.0);
assert!((m.level_db[0] - 0.0).abs() < 1e-4 && (m.level_db[1] + 6.0206).abs() < 1e-3);
assert!(m.clipped[0] && !m.clipped[1]);
m.update([0.0, 0.0], 1.1);
assert!((m.level_db[0] + 3.0).abs() < 1e-3, "{}", m.level_db[0]);
assert_eq!(m.peak_db[0], 0.0, "peak holds");
// After the hold time the peak marker follows the falling level.
m.update([0.0, 0.0], 3.0);
assert!(m.peak_db[0] < -10.0, "{}", m.peak_db[0]);
}
}
+4 -1
View File
@@ -304,9 +304,12 @@ pub fn handle(app: &mut EffectcraftApp, ctx: &egui::Context, req: &ControlReques
match s("action").unwrap_or("toggle") {
"play" => app.play(now),
"stop" => app.stop(),
"status" => {}
_ => app.toggle_play(now),
}
ok(json!({"playing": app.playback.playing, "time": app.session.time().seconds()}))
ok(
json!({"playing": app.playback.playing, "time": app.session.time().seconds(), "audio": app.audio.is_some(), "levelsDb": app.meter.level_db, "peaksDb": app.meter.peak_db}),
)
}
"ui.screenshot" => {
let crop = s("panel").and_then(PanelKind::from_name).and_then(|pk| app.auto.find(&format!("panel.{}", pk.id())).map(|e| e.rect));
+60
View File
@@ -5,6 +5,7 @@
//! Everything goes through `effectcraft_engine::Session::execute` (or `menus::invoke` for
//! frontend-only commands) so every gesture is also available to agents.
pub mod audio;
pub mod automation;
pub mod control;
pub mod dock;
@@ -54,6 +55,8 @@ pub struct Hooks {
pub pick_files: Option<Box<dyn Fn(&[&str]) -> Vec<String>>>,
pub pick_save: Option<Box<dyn Fn(&str) -> Option<String>>>,
pub pick_open_project: Option<Box<dyn Fn() -> Option<String>>>,
/// Opens the audio output for preview playback (the desktop app uses cpal).
pub audio_device: Option<audio::AudioDeviceFactory>,
}
#[derive(Default)]
@@ -101,6 +104,14 @@ pub struct EffectcraftApp {
pub(crate) pointer_comp: Option<[f32; 2]>,
pub(crate) dialog_state: panels::dialogs::DialogState,
pub integrated_titlebar: bool,
/// Audio preview while playing (audio clock drives playback).
pub audio: Option<audio::AudioPlayback>,
/// Audio panel VU meters.
pub meter: audio::Meter,
/// Waveform peak summaries per footage item (see `panels::waveform`).
pub(crate) waveforms: panels::waveform::Cache,
/// Last reveal shortcut and when (double-press shortcuts such as LL).
pub(crate) last_reveal: Option<(String, f64)>,
}
impl EffectcraftApp {
@@ -133,6 +144,10 @@ impl EffectcraftApp {
pointer_comp: None,
dialog_state: Default::default(),
integrated_titlebar: false,
audio: None,
meter: Default::default(),
waveforms: Default::default(),
last_reveal: None,
}
}
@@ -244,10 +259,32 @@ impl EffectcraftApp {
waiting: false,
fps: c.frame_rate.as_f64(),
};
self.start_audio(t);
}
/// Start audio preview from comp time `t` when the Preview panel includes audio, the comp
/// has something audible and an output device opens.
fn start_audio(&mut self, t: Tick) {
self.audio = None;
self.meter.clipped = [false; 2];
if !self.ui.preview_audio {
return;
}
let Some(cid) = self.session.active_comp_id() else { return };
let Some(c) = self.session.project.comp(cid).cloned() else { return };
if !effectcraft_engine::render::audio::comp_has_audio(&self.session.project, cid) {
return;
}
let Some(dev) = self.hooks.audio_device.as_ref().and_then(|f| f()) else { return };
match audio::AudioPlayback::start(dev, self.render_source(), cid, t, c.work_area.0, c.work_area.1, self.ui.preview_loop) {
Ok(a) => self.audio = Some(a),
Err(e) => log::warn!("audio preview: {e}"),
}
}
pub fn stop(&mut self) {
self.playback.playing = false;
self.audio = None;
}
pub fn toggle_play(&mut self, now: f64) {
@@ -291,9 +328,32 @@ impl EffectcraftApp {
queued += 1;
}
}
if let Some(a) = &self.audio {
self.meter.update(a.feed.take_peaks(), now);
} else if self.meter.active() {
self.meter.update([0.0; 2], now);
ctx.request_repaint();
}
if !self.playback.playing {
return;
}
// Audio clock drives playback: show the frame under the audible sample (frames that
// are not rendered yet are dropped, never delayed).
if let Some(a) = &self.audio {
if a.finished() {
self.session.set_time(fr.tick_of(wb));
self.stop();
return;
}
let target = audio::frame_of_sample(a.clock(), a.rate, fr).clamp(wa, wb);
self.playback.waiting = !self.frames.is_cached(&self.frame_key(cid, target, scale));
if target != cur {
self.session.set_time(fr.tick_of(target));
self.playback.shown += 1;
}
ctx.request_repaint();
return;
}
let elapsed = now - self.playback.start_wall;
let mut target = self.playback.start_frame + (elapsed * fr.as_f64()).floor() as i64;
if target > wb {
+18 -2
View File
@@ -64,6 +64,9 @@ pub const UI_COMMANDS: &[UiCommand] = &[
uic!("timeline.reveal.effects", "Reveal Effects", [], Some("E")),
uic!("timeline.reveal.masks", "Reveal Masks", [], Some("M")),
uic!("timeline.reveal.feather", "Reveal Mask Feather", [], Some("F")),
uic!("timeline.reveal.levels", "Reveal Audio Levels (press twice: Waveform)", [], Some("L")),
uic!("timeline.reveal.waveform", "Reveal Audio Waveform", [], None),
uic!("playback.muteAudio", "Mute Audio", ["Composition", "Preview"], None),
uic!("timeline.collapseAll", "Collapse All", [], Some("Cmd+`")),
uic!("window.workspace.default", "Default", ["Window", "Workspace"], Some("Shift+F10")),
uic!("window.workspace.standard", "Standard", ["Window", "Workspace"], Some("Shift+F11")),
@@ -102,7 +105,13 @@ pub fn panel_command_id(p: PanelKind) -> String {
format!("window.panel.{}", p.id())
}
fn reveal(app: &mut EffectcraftApp, kind: &str) {
fn reveal(app: &mut EffectcraftApp, kind: &str, now: f64) {
// AE's double-press shortcuts: L then L again quickly reveals the Waveform (LL).
let kind = match app.last_reveal.take() {
Some((k, t)) if k == "levels" && kind == "levels" && now - t < 0.6 => "waveform",
_ => kind,
};
app.last_reveal = Some((kind.to_string(), now));
let add = app.ui.timeline.reveal.contains(&kind.to_string());
if add && app.ui.timeline.reveal.len() == 1 {
app.ui.timeline.reveal.clear();
@@ -169,7 +178,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V
return Ok(json!({"tool": tool}));
}
if let Some(k) = id.strip_prefix("timeline.reveal.") {
reveal(app, k);
reveal(app, k, now);
return Ok(Value::Null);
}
if let Some(r) = id.strip_prefix("view.res.") {
@@ -191,6 +200,13 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V
app.stop();
return Ok(Value::Null);
}
"playback.muteAudio" => {
app.ui.preview_audio = !app.ui.preview_audio;
if !app.ui.preview_audio {
app.audio = None;
}
return Ok(json!({"muted": !app.ui.preview_audio}));
}
"view.zoomIn" | "view.zoomOut" => {
let cur = app.ui.viewer.zoom.unwrap_or_else(|| crate::panels::viewer::last_fit(ctx));
const STEPS: [f32; 16] = [0.015, 0.03, 0.0625, 0.125, 0.25, 0.333, 0.5, 0.66, 1.0, 1.5, 2.0, 3.0, 4.0, 8.0, 16.0, 32.0];
+89 -10
View File
@@ -1,6 +1,6 @@
//! Smaller panels: Preview, Audio, History, Markers, Wiggler, the Home screen and placeholders.
use egui::{Align2, Color32, Rect, Sense, pos2, vec2};
use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2};
use serde_json::json;
use crate::dock::PanelKind;
@@ -51,6 +51,12 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
}
app.auto.add("preview.loop", lr, "Loop");
p.text(pos2(lr.max.x + 6.0, yy), Align2::LEFT_CENTER, "Loop", Tokens::ui(12.0), t.text);
let ar = Rect::from_min_size(pos2(rect.min.x + 90.0, yy - 8.0), vec2(16.0, 16.0));
if widgets::checkbox(ui, ar, app.ui.preview_audio, &t, egui::Id::new("pv-audio")).clicked() {
let _ = crate::menus::invoke(app, &ctx, "playback.muteAudio", json!({}));
}
app.auto.add("preview.audio", ar, "Include Audio");
p.text(pos2(ar.max.x + 6.0, yy), Align2::LEFT_CENTER, "Include Audio", Tokens::ui(12.0), t.text);
yy += 24.0;
row(&p, yy, "Range", "Work Area Extended By Current Time");
yy += 22.0;
@@ -74,19 +80,92 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
}
}
/// Audio panel: L/R VU meters (dBFS, 0 to -48) with peak hold and clip indicators, fed by the
/// audio preview; the selected layer's Audio Levels on the right.
pub fn audio(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
use crate::audio::METER_FLOOR;
let t = app.tokens;
let p = ui.painter().with_clip_rect(rect);
// Level meters (silent until audio playback lands).
for (i, x) in [rect.min.x + 20.0, rect.min.x + 34.0].into_iter().enumerate() {
let r = Rect::from_min_max(pos2(x, rect.min.y + 14.0), pos2(x + 10.0, rect.max.y - 20.0));
p.rect_filled(r, 1.0, t.field_bg);
let _ = i;
let top = rect.min.y + 26.0;
let bottom = rect.max.y - 22.0;
if bottom - top < 40.0 {
return;
}
for (k, db) in [0, -6, -12, -18, -24, -36, -48].iter().enumerate() {
let y = rect.min.y + 14.0 + (rect.height() - 34.0) * (-*db as f32 / 48.0);
p.text(pos2(rect.min.x + 52.0, y), Align2::LEFT_CENTER, format!("{db} dB"), Tokens::ui(10.0), t.text_faint);
let _ = k;
let y_of = |db: f32| top + (bottom - top) * (db.clamp(METER_FLOOR, 0.0) / METER_FLOOR);
let m = app.meter;
let green = Color32::from_rgb(0x3c, 0xc8, 0x5a);
let yellow = Color32::from_rgb(0xe6, 0xc8, 0x3c);
let red = Color32::from_rgb(0xe6, 0x46, 0x3c);
let seg_col = |db: f32| {
if db > -3.0 {
red
} else if db > -12.0 {
yellow
} else {
green
}
};
for c in 0..2 {
let x = rect.min.x + 18.0 + c as f32 * 16.0;
let bar = Rect::from_min_max(pos2(x, top), pos2(x + 12.0, bottom));
p.rect_filled(bar, 1.0, t.field_bg);
// Lit segments, 1.5 dB each, coloured by their level.
let lvl = m.level_db[c];
let mut db = METER_FLOOR;
while db < lvl.min(0.0) {
let hi = (db + 1.5).min(lvl);
p.rect_filled(Rect::from_min_max(pos2(bar.min.x + 1.0, y_of(hi)), pos2(bar.max.x - 1.0, y_of(db) - 0.5)), 0.0, seg_col(db));
db += 1.5;
}
if m.peak_db[c] > METER_FLOOR {
let py = y_of(m.peak_db[c]);
p.line_segment([pos2(bar.min.x, py), pos2(bar.max.x, py)], Stroke::new(2.0, seg_col(m.peak_db[c])));
}
// Clip indicator (click to reset).
let clip = Rect::from_min_max(pos2(bar.min.x, top - 14.0), pos2(bar.max.x, top - 4.0));
p.rect_filled(clip, 1.0, if m.clipped[c] { red } else { t.field_bg });
let id = if c == 0 { "audio.clipLeft" } else { "audio.clipRight" };
if ui.interact(clip, egui::Id::new(id), Sense::click()).clicked() {
app.meter.clipped[c] = false;
}
app.auto.add(id, clip, if c == 0 { "Left clip indicator" } else { "Right clip indicator" });
app.auto.add(if c == 0 { "audio.meterLeft" } else { "audio.meterRight" }, bar, &format!("{:.1} dB", m.level_db[c]));
p.text(pos2(bar.center().x, bottom + 9.0), Align2::CENTER_CENTER, if c == 0 { "L" } else { "R" }, Tokens::ui(10.0), t.text_dim);
}
let sx = rect.min.x + 54.0;
for db in [0, -6, -12, -18, -24, -30, -36, -42, -48] {
let y = y_of(db as f32);
p.line_segment([pos2(sx - 4.0, y), pos2(sx - 1.0, y)], Stroke::new(1.0, t.text_faint));
p.text(pos2(sx + 2.0, y), Align2::LEFT_CENTER, format!("{db:.1}"), Tokens::ui(10.0), t.text_faint);
}
p.text(pos2(sx + 30.0, top - 9.0), Align2::LEFT_CENTER, "dB", Tokens::ui(10.0), t.text_dim);
// Selected layer's Audio Levels.
let lx = rect.min.x + 120.0;
if rect.max.x - lx < 80.0 {
return;
}
let sel = app.session.state.selected_layers.first().copied();
let levels = sel.and_then(|id| {
let c = app.session.active_comp()?;
let l = c.layer(id)?;
let pr = l.props.sub("audio")?.get("levels")?;
Some((l.name.clone(), pr.value.as_vec2()))
});
match levels {
Some((name, [a, b])) => {
p.text(pos2(lx, top - 9.0), Align2::LEFT_CENTER, &name, Tokens::ui(11.0), t.text_dim);
for (i, (k, v)) in [("Left", a), ("Right", b)].into_iter().enumerate() {
let y = top + 12.0 + i as f32 * 20.0;
p.text(pos2(lx, y), Align2::LEFT_CENTER, k, Tokens::ui(12.0), t.text_dim);
p.text(pos2(lx + 44.0, y), Align2::LEFT_CENTER, format!("{v:+.1} dB"), Tokens::ui(12.0), t.hot_text);
}
}
None => {
p.text(pos2(lx, top + 12.0), Align2::LEFT_CENTER, "No audio layer selected", Tokens::ui(11.0), t.text_faint);
}
}
if app.meter.active() || app.audio.is_some() {
ui.ctx().request_repaint();
}
}
+1
View File
@@ -15,6 +15,7 @@ pub mod render_queue;
pub mod text_panels;
pub mod timeline;
pub mod viewer;
pub mod waveform;
use effectcraft_engine::Session;
use effectcraft_engine::project::{Comp, TimeDisplayStyle};
+50 -1
View File
@@ -36,12 +36,20 @@ enum RowKind {
uid: u64,
lines: usize,
},
/// Audio > Waveform of a footage layer with audio (drawn from the item's peak summary).
Waveform {
item: u64,
},
}
/// Synthetic group uid for a layer's Audio > Waveform twirl (never a real property uid).
const WAVE_BIT: u64 = 1 << 60;
/// Row height (expression editors grow with their text).
fn row_height(row: &Row, rh: f32) -> f32 {
match row.kind {
RowKind::Expr { lines, .. } => rh * lines.clamp(1, 8) as f32 + 6.0,
RowKind::Waveform { .. } => rh * 3.0,
_ => rh,
}
}
@@ -227,9 +235,15 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec<Row> {
"opacity" => ("transform", vec!["opacity"]),
"anchor" => ("transform", vec!["anchor"]),
"feather" => ("masks", vec!["feather"]),
"levels" => ("audio", vec!["levels"]),
_ => ("", vec![]),
};
match kind.as_str() {
"waveform" => {
if let Some(item) = super::waveform::audio_item(&app.session.project, l) {
rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Waveform { item: item.0 } });
}
}
"effects" => {
if let Some(fx) = l.effects() {
for g in fx.groups() {
@@ -294,6 +308,21 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec<Row> {
});
if open {
push_group(&mut rows, l, g, 2, &tl.open_groups);
// Audio > Waveform > Waveform (footage with audio).
if g.match_id == "audio"
&& let Some(item) = super::waveform::audio_item(&app.session.project, l)
{
let wuid = WAVE_BIT | g.uid;
let wopen = tl.open_groups.contains(&wuid);
rows.push(Row {
layer: l.id,
depth: 2,
kind: RowKind::Group { uid: wuid, name: "Waveform".into(), open: wopen, has_children: true, fx: None },
});
if wopen {
rows.push(Row { layer: l.id, depth: 3, kind: RowKind::Waveform { item: item.0 } });
}
}
}
}
Node::Prop(p) if prop_visible(p, l) => rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid: p.uid } }),
@@ -952,13 +981,33 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
lp.text(pos2(ir.max.x + 4.0, cy), Align2::LEFT_CENTER, "Inverted", Tokens::ui(11.0), t.text_dim);
}
let gr = ui.interact(Rect::from_min_max(pos2(indent + 8.0, r.min.y), pos2(cw.switches, r.max.y)), egui::Id::new(("grow", uid)), Sense::click());
if gr.clicked() {
if gr.clicked() && uid & WAVE_BIT == 0 {
actions.push(("prop.select".into(), json!({"layer": layer.id.0, "prop": uid, "selectKeys": false})));
}
if gr.double_clicked() {
ui_actions.push(UiAct::ToggleGroup(*uid));
}
}
RowKind::Waveform { item } => {
lp.rect_filled(left, 0.0, if ri % 2 == 0 { t.row } else { t.row_alt });
let indent = cw.name + 6.0 + 14.0 * row.depth as f32;
lp.text(pos2(indent + 10.0, cy), Align2::LEFT_CENTER, "Waveform", Tokens::ui(12.0), t.text);
let wr = Rect::from_min_max(pos2(graph_x0, r.min.y), r.max);
gp.rect_filled(wr, 0.0, t.tl_bg);
app.auto.add(&format!("timeline.layer.{}.waveform", layer.id.0), wr, "Waveform");
match super::waveform::summary(app, &ctx, effectcraft_engine::project::ItemId(*item)) {
Some(s) => super::waveform::draw(&gp, wr.shrink2(vec2(0.0, 2.0)), &tm, layer, &s, Color32::from_rgb(0x5f, 0xc8, 0x8a)),
None => {
gp.text(
pos2(tm.x(layer.in_point.seconds()).max(wr.min.x) + 6.0, wr.center().y),
Align2::LEFT_CENTER,
"Building waveform…",
Tokens::ui(11.0),
t.text_faint,
);
}
}
}
RowKind::Expr { uid, lines } => {
let Some(prop) = layer.props.find(*uid) else { continue };
let Some(ex) = prop.expr.clone() else { continue };
+95
View File
@@ -0,0 +1,95 @@
//! Audio waveforms for the Timeline's Audio > Waveform property (revealed with `LL`).
//!
//! Each footage item's audio is summarised once into min/max peaks
//! (`effectcraft_render::audio::footage_peaks`, [`BINS_PER_SEC`] bins per second) on a
//! background thread; the timeline draws from that summary at any zoom.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use effectcraft_engine::project::{ItemId, ItemKind, Layer, LayerSource, Project};
use effectcraft_engine::time::Tick;
use egui::{Color32, Painter, Rect, Stroke, pos2};
use super::timeline::TMap;
use crate::EffectcraftApp;
/// Peak-summary resolution.
pub const BINS_PER_SEC: u32 = 200;
/// A footage item's peak summary: `[min L, max L, min R, max R]` per bin.
pub struct Summary {
pub bins_per_sec: u32,
pub peaks: Vec<[f32; 4]>,
}
/// Summaries by footage item (`None` while computing).
#[derive(Default)]
pub struct Cache {
map: Arc<Mutex<HashMap<u64, Option<Arc<Summary>>>>>,
}
/// The footage item whose audio a layer plays, when it has audio.
pub fn audio_item(project: &Project, l: &Layer) -> Option<ItemId> {
let LayerSource::Footage { item } = &l.source else { return None };
match &project.item(*item)?.kind {
ItemKind::Footage(f) if f.has_audio && !f.missing => Some(*item),
_ => None,
}
}
/// The summary for `item`, starting its computation on first request.
pub fn summary(app: &EffectcraftApp, ctx: &egui::Context, item: ItemId) -> Option<Arc<Summary>> {
let mut m = app.waveforms.map.lock().ok()?;
if let Some(s) = m.get(&item.0) {
return s.clone();
}
let ItemKind::Footage(f) = &app.session.project.item(item)?.kind else { return None };
m.insert(item.0, None);
let (f, footage, map, ctx) = (f.clone(), app.session.footage.clone(), app.waveforms.map.clone(), ctx.clone());
std::thread::Builder::new()
.name("ec-waveform".into())
.spawn(move || {
let peaks = effectcraft_engine::render::audio::footage_peaks(footage.as_ref(), item, &f, f.duration, BINS_PER_SEC);
if let Ok(mut m) = map.lock() {
m.insert(item.0, Some(Arc::new(Summary { bins_per_sec: BINS_PER_SEC, peaks })));
}
ctx.request_repaint();
})
.ok()?;
None
}
/// Draw `layer`'s waveform in `rect` (left channel above right), columns mapped through the
/// timeline's time map and the layer's time mapping, limited to its [in, out) span.
pub(crate) fn draw(p: &Painter, rect: Rect, tm: &TMap, layer: &Layer, s: &Summary, color: Color32) {
let mid = [rect.min.y + rect.height() * 0.25, rect.min.y + rect.height() * 0.75];
let half = rect.height() * 0.25 - 1.0;
let x0 = tm.x(layer.in_point.seconds()).max(rect.min.x).floor() as i32;
let x1 = tm.x(layer.out_point.seconds()).min(rect.max.x).ceil() as i32;
let n = s.peaks.len() as i64;
for c in 0..2 {
p.line_segment([pos2(x0 as f32, mid[c]), pos2(x1 as f32, mid[c])], Stroke::new(1.0, color.gamma_multiply(0.5)));
}
for x in x0..x1 {
let (ta, tb) = (tm.t(x as f32), tm.t(x as f32 + 1.0));
let src = |t: f64| layer.layer_time(Tick::from_seconds_f64(t)).seconds() * s.bins_per_sec as f64;
let (ba, bb) = (src(ta).floor() as i64, src(tb).ceil() as i64);
let (ba, bb) = (ba.min(bb).clamp(0, n), ba.max(bb).max(ba + 1).clamp(0, n));
if ba >= bb {
continue;
}
let mut agg = [0.0f32; 4];
for b in &s.peaks[ba as usize..bb as usize] {
agg[0] = agg[0].min(b[0]);
agg[1] = agg[1].max(b[1]);
agg[2] = agg[2].min(b[2]);
agg[3] = agg[3].max(b[3]);
}
for c in 0..2 {
let (lo, hi) = (agg[c * 2].clamp(-1.0, 1.0), agg[c * 2 + 1].clamp(-1.0, 1.0));
let (ya, yb) = (mid[c] - hi * half, mid[c] - lo * half);
p.line_segment([pos2(x as f32 + 0.5, ya), pos2(x as f32 + 0.5, yb.max(ya + 1.0))], Stroke::new(1.0, color));
}
}
}
+4
View File
@@ -338,6 +338,9 @@ pub struct UiState {
/// Preview panel options.
pub preview_loop: bool,
pub preview_cache_first: bool,
/// Preview panel "Include Audio" (Mute Audio off).
#[serde(default = "yes")]
pub preview_audio: bool,
pub start_screen: bool,
}
@@ -369,6 +372,7 @@ impl Default for UiState {
tool_creates_shape: true,
preview_loop: true,
preview_cache_first: false,
preview_audio: true,
start_screen: false,
}
}