diff --git a/AGENTS.md b/AGENTS.md index a1125cef..87dd6814 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ People trust SoundCraft with their recordings; a crash loses takes. **This outra - No panics in non-test code: no `unwrap()`, `expect()`, `panic!`, `unreachable!`, `todo!`, `unimplemented!`; no `unsafe` (`unsafe_code = "forbid"`). - The one named exception is `soundcraft-clap-host` (`crates/clap-host`), the isolated unsafe helper crate for CLAP plugin FFI: `unsafe_code = "deny"` crate-wide, allowed only in `src/ffi.rs` with a `// SAFETY:` comment on every block, safe `Result` API. - The same exception, same rules, for `soundcraft-vst3-host` (`crates/vst3-host`), the isolated unsafe helper crate for VST3 plugin FFI (COM vtables via the `vst3` bindings, module loading, host-side COM objects). +- And for `soundcraft-au-host` (`crates/au-host`), the isolated unsafe helper crate for Audio Unit FFI (hand-written AudioToolbox/CoreFoundation declarations, macOS only; a safe stub elsewhere). - Errors are `Result` + `?`. An unfinished feature returns an "unsupported" error. - Input-derived numbers are hostile (files, commands, MCP/control params): `get()` not `[i]`, checked/saturating arithmetic, no NaN casts, cap allocations. - The audio callback never blocks, allocates in steady state, or panics. diff --git a/Cargo.lock b/Cargo.lock index bc14aaa3..a3ca54c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1687,6 +1687,8 @@ dependencies = [ "num-traits", "png", "tiff", + "zune-core", + "zune-jpeg", ] [[package]] @@ -3372,6 +3374,16 @@ dependencies = [ "winresource", ] +[[package]] +name = "soundcraft-au-host" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "soundcraft-dsp", + "thiserror 2.0.21", +] + [[package]] name = "soundcraft-audio-io" version = "0.1.0" @@ -3443,6 +3455,7 @@ dependencies = [ "soundcraft-mix", "soundcraft-model", "soundcraft-time", + "soundcraft-video", "soundcraft-vst3-host", "thiserror 2.0.21", ] @@ -3463,6 +3476,7 @@ version = "0.1.0" dependencies = [ "log", "rayon", + "soundcraft-au-host", "soundcraft-audio-io", "soundcraft-clap-host", "soundcraft-dsp", @@ -3518,6 +3532,7 @@ dependencies = [ "log", "serde", "serde_json", + "soundcraft-au-host", "soundcraft-audio-io", "soundcraft-clap-host", "soundcraft-dsp", @@ -3527,15 +3542,30 @@ dependencies = [ "soundcraft-model", "soundcraft-playback", "soundcraft-time", + "soundcraft-video", "soundcraft-vst3-host", ] +[[package]] +name = "soundcraft-video" +version = "0.1.0" +dependencies = [ + "image", + "proptest", + "rayon", + "serde", + "thiserror 2.0.21", +] + [[package]] name = "soundcraft-vst3-host" version = "0.1.0" dependencies = [ "libloading", "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", "serde", "soundcraft-dsp", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index c9e7a516..35520baf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,9 +15,11 @@ repository = "https://github.com/storytold/soundcraft" soundcraft-time = { path = "crates/time" } soundcraft-audio-io = { path = "crates/audio-io" } soundcraft-midi = { path = "crates/midi" } +soundcraft-video = { path = "crates/video" } soundcraft-dsp = { path = "crates/dsp" } soundcraft-clap-host = { path = "crates/clap-host" } soundcraft-vst3-host = { path = "crates/vst3-host" } +soundcraft-au-host = { path = "crates/au-host" } soundcraft-model = { path = "crates/model" } soundcraft-mix = { path = "crates/mix" } soundcraft-engine = { path = "crates/engine" } diff --git a/crates/au-host/Cargo.toml b/crates/au-host/Cargo.toml new file mode 100644 index 00000000..af793163 --- /dev/null +++ b/crates/au-host/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "soundcraft-au-host" +description = "SoundCraft Audio Unit (AUv2, macOS) plugin host: lists, loads and runs system-registered Audio Units behind the safe soundcraft-dsp Plugin trait (an isolated unsafe FFI crate)" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[dependencies] +soundcraft-dsp.workspace = true +serde.workspace = true +thiserror.workspace = true +log.workspace = true + +# An isolated unsafe helper crate (craftrules never-crash), like soundcraft-vst3-host: this crate +# opts out of the workspace `unsafe_code = "forbid"` for itself only. It denies unsafe everywhere +# and allows it solely on the FFI module (`src/ffi.rs`). The rest mirrors `[workspace.lints]`. +[lints.rust] +unsafe_code = "deny" + +[lints.clippy] +dbg_macro = "warn" +todo = "warn" +too_many_arguments = "allow" +type_complexity = "allow" diff --git a/crates/au-host/src/ffi.rs b/crates/au-host/src/ffi.rs new file mode 100644 index 00000000..a1152449 --- /dev/null +++ b/crates/au-host/src/ffi.rs @@ -0,0 +1,769 @@ +//! The only module with `unsafe`: hand-written AudioToolbox / CoreFoundation declarations and a +//! thin safe wrapper over one Audio Unit instance. +//! +//! Every pointer handed to the Audio Unit points into memory this module owns for at least as +//! long as the unit can use it: the input state the render callback reads is a leaked `Box` +//! freed only after the unit is disposed, and output buffers are re-pointed before each render. + +use std::ffi::{CStr, c_char, c_void}; +use std::ptr; + +type OSStatus = i32; +type AudioComponent = *mut c_void; +type AudioUnit = *mut c_void; +type CFTypeRef = *const c_void; +type CFIndex = isize; + +/// Most channels on one Audio Unit bus that we set up. +pub const MAX_AU_CHANNELS: usize = 8; +/// Largest render slice we configure. +pub const MAX_SLICE: usize = 8192; +/// Most parameters read from one unit. +const MAX_PARAMS: usize = 4096; +/// Most components listed for one type. +const MAX_COMPONENTS: usize = 4096; +/// Most value strings read for one indexed parameter. +const MAX_VALUE_STRINGS: CFIndex = 256; +/// Largest state blob accepted or produced. +const MAX_STATE: usize = 64 << 20; +/// Longest CFString converted (bytes). +const MAX_STRING: CFIndex = 4096; + +const K_AUDIO_COMPONENT_FLAG_REQUIRES_ASYNC: u32 = 8; + +const PROP_CLASS_INFO: u32 = 0; +const PROP_PARAMETER_LIST: u32 = 3; +const PROP_PARAMETER_INFO: u32 = 4; +const PROP_STREAM_FORMAT: u32 = 8; +const PROP_LATENCY: u32 = 12; +const PROP_MAX_FRAMES: u32 = 14; +const PROP_PARAMETER_VALUE_STRINGS: u32 = 16; +const PROP_TAIL_TIME: u32 = 20; +const PROP_SET_RENDER_CALLBACK: u32 = 23; + +const SCOPE_GLOBAL: u32 = 0; +const SCOPE_INPUT: u32 = 1; +const SCOPE_OUTPUT: u32 = 2; + +const FORMAT_LINEAR_PCM: u32 = u32::from_be_bytes(*b"lpcm"); +const FORMAT_FLAG_FLOAT: u32 = 1; +const FORMAT_FLAG_PACKED: u32 = 1 << 3; +const FORMAT_FLAG_NON_INTERLEAVED: u32 = 1 << 5; + +const TIMESTAMP_SAMPLE_TIME_VALID: u32 = 1; + +const CF_STRING_ENCODING_UTF8: u32 = 0x0800_0100; +const CF_PROPERTY_LIST_BINARY_FORMAT: CFIndex = 200; + +pub const PARAM_FLAG_CF_NAME_RELEASE: u32 = 1 << 4; +pub const PARAM_FLAG_HAS_CF_NAME: u32 = 1 << 27; +pub const PARAM_FLAG_DISPLAY_LOG: u32 = 1 << 22; +#[cfg(test)] +pub const PARAM_FLAG_READABLE: u32 = 1 << 30; +pub const PARAM_FLAG_WRITABLE: u32 = 1 << 31; +pub const PARAM_FLAG_METER_READ_ONLY: u32 = 1 << 15; + +pub const UNIT_INDEXED: u32 = 1; +pub const UNIT_CUSTOM: u32 = 26; + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct AudioComponentDescription { + component_type: u32, + component_sub_type: u32, + component_manufacturer: u32, + component_flags: u32, + component_flags_mask: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct AudioStreamBasicDescription { + sample_rate: f64, + format_id: u32, + format_flags: u32, + bytes_per_packet: u32, + frames_per_packet: u32, + bytes_per_frame: u32, + channels_per_frame: u32, + bits_per_channel: u32, + reserved: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct SmpteTime { + subframes: i16, + subframe_divisor: i16, + counter: u32, + smpte_type: u32, + flags: u32, + hours: i16, + minutes: i16, + seconds: i16, + frames: i16, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct AudioTimeStamp { + sample_time: f64, + host_time: u64, + rate_scalar: f64, + word_clock_time: u64, + smpte_time: SmpteTime, + flags: u32, + reserved: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct AudioBuffer { + number_channels: u32, + data_byte_size: u32, + data: *mut c_void, +} + +/// `AudioBufferList` with room for [`MAX_AU_CHANNELS`] buffers (the unit reads only +/// `number_buffers` of them). +#[repr(C)] +struct AudioBufferList { + number_buffers: u32, + buffers: [AudioBuffer; MAX_AU_CHANNELS], +} + +/// The variable-length list the unit hands the render callback (we only see its header and +/// index into the buffers it says it has). +#[repr(C)] +struct AudioBufferListHeader { + number_buffers: u32, + first: AudioBuffer, +} + +type AuRenderCallback = unsafe extern "C" fn( + ref_con: *mut c_void, + action_flags: *mut u32, + time_stamp: *const AudioTimeStamp, + bus: u32, + frames: u32, + data: *mut AudioBufferListHeader, +) -> OSStatus; + +#[repr(C)] +struct AuRenderCallbackStruct { + input_proc: Option, + input_proc_ref_con: *mut c_void, +} + +#[repr(C)] +struct AudioUnitParameterInfo { + name: [c_char; 52], + unit_name: CFTypeRef, + clump_id: u32, + cf_name_string: CFTypeRef, + unit: u32, + min_value: f32, + max_value: f32, + default_value: f32, + flags: u32, +} + +#[link(name = "AudioToolbox", kind = "framework")] +unsafe extern "C" { + fn AudioComponentFindNext(component: AudioComponent, desc: *const AudioComponentDescription) -> AudioComponent; + fn AudioComponentCopyName(component: AudioComponent, name: *mut CFTypeRef) -> OSStatus; + fn AudioComponentGetDescription(component: AudioComponent, desc: *mut AudioComponentDescription) -> OSStatus; + fn AudioComponentGetVersion(component: AudioComponent, version: *mut u32) -> OSStatus; + fn AudioComponentInstanceNew(component: AudioComponent, instance: *mut AudioUnit) -> OSStatus; + fn AudioComponentInstanceDispose(instance: AudioUnit) -> OSStatus; + fn AudioUnitInitialize(unit: AudioUnit) -> OSStatus; + fn AudioUnitUninitialize(unit: AudioUnit) -> OSStatus; + fn AudioUnitGetPropertyInfo(unit: AudioUnit, id: u32, scope: u32, element: u32, size: *mut u32, writable: *mut u8) -> OSStatus; + fn AudioUnitGetProperty(unit: AudioUnit, id: u32, scope: u32, element: u32, data: *mut c_void, size: *mut u32) -> OSStatus; + fn AudioUnitSetProperty(unit: AudioUnit, id: u32, scope: u32, element: u32, data: *const c_void, size: u32) -> OSStatus; + fn AudioUnitGetParameter(unit: AudioUnit, id: u32, scope: u32, element: u32, value: *mut f32) -> OSStatus; + fn AudioUnitSetParameter(unit: AudioUnit, id: u32, scope: u32, element: u32, value: f32, offset: u32) -> OSStatus; + fn AudioUnitRender( + unit: AudioUnit, + action_flags: *mut u32, + time_stamp: *const AudioTimeStamp, + bus: u32, + frames: u32, + data: *mut AudioBufferList, + ) -> OSStatus; + fn AudioUnitReset(unit: AudioUnit, scope: u32, element: u32) -> OSStatus; + fn MusicDeviceMIDIEvent(unit: AudioUnit, status: u32, data1: u32, data2: u32, offset: u32) -> OSStatus; +} + +#[link(name = "CoreFoundation", kind = "framework")] +unsafe extern "C" { + fn CFRelease(cf: CFTypeRef); + fn CFGetTypeID(cf: CFTypeRef) -> usize; + fn CFStringGetTypeID() -> usize; + fn CFDictionaryGetTypeID() -> usize; + fn CFStringGetLength(s: CFTypeRef) -> CFIndex; + fn CFStringGetMaximumSizeForEncoding(length: CFIndex, encoding: u32) -> CFIndex; + fn CFStringGetCString(s: CFTypeRef, buffer: *mut c_char, size: CFIndex, encoding: u32) -> u8; + fn CFArrayGetCount(a: CFTypeRef) -> CFIndex; + fn CFArrayGetValueAtIndex(a: CFTypeRef, i: CFIndex) -> CFTypeRef; + fn CFDataCreate(alloc: CFTypeRef, bytes: *const u8, length: CFIndex) -> CFTypeRef; + fn CFDataGetLength(d: CFTypeRef) -> CFIndex; + fn CFDataGetBytePtr(d: CFTypeRef) -> *const u8; + fn CFPropertyListCreateData(alloc: CFTypeRef, plist: CFTypeRef, format: CFIndex, options: usize, error: *mut CFTypeRef) -> CFTypeRef; + fn CFPropertyListCreateWithData(alloc: CFTypeRef, data: CFTypeRef, options: usize, format: *mut CFIndex, error: *mut CFTypeRef) -> CFTypeRef; +} + +/// Releases a CF object we own (null is ignored). +fn release(cf: CFTypeRef) { + if !cf.is_null() { + // SAFETY: `cf` is a non-null CF object the caller owns a +1 reference to. + unsafe { CFRelease(cf) } + } +} + +/// Converts a borrowed CFString to a Rust string (empty on any failure or a non-string). +fn cf_string(s: CFTypeRef) -> String { + if s.is_null() { + return String::new(); + } + // SAFETY: `s` is a live, non-null CF object (borrowed from the caller); the type check + // guards the CFString calls, and the buffer is sized by CoreFoundation's own maximum. + unsafe { + if CFGetTypeID(s) != CFStringGetTypeID() { + return String::new(); + } + let len = CFStringGetLength(s).clamp(0, MAX_STRING); + let cap = CFStringGetMaximumSizeForEncoding(len, CF_STRING_ENCODING_UTF8); + if cap <= 0 { + return String::new(); + } + let cap = cap.saturating_add(1).min(MAX_STRING * 4 + 1); + let mut buf = vec![0 as c_char; cap as usize]; + if CFStringGetCString(s, buf.as_mut_ptr(), cap, CF_STRING_ENCODING_UTF8) == 0 { + return String::new(); + } + CStr::from_ptr(buf.as_ptr()).to_string_lossy().into_owned() + } +} + +/// One registered component. +#[derive(Debug, Clone)] +pub struct RawComponent { + pub ty: u32, + pub subtype: u32, + pub manufacturer: u32, + /// `"Vendor: Name"` as registered. + pub name: String, + pub version: u32, +} + +/// Every component of type `ty` that can be instantiated synchronously. +pub fn components(ty: u32) -> Vec { + let desc = AudioComponentDescription { component_type: ty, ..Default::default() }; + let mut out = Vec::new(); + let mut comp: AudioComponent = ptr::null_mut(); + for _ in 0..MAX_COMPONENTS { + // SAFETY: `desc` is a valid description; `comp` is null or a component returned by the + // previous call (components live for the process). + comp = unsafe { AudioComponentFindNext(comp, &desc) }; + if comp.is_null() { + break; + } + let mut d = AudioComponentDescription::default(); + let mut name: CFTypeRef = ptr::null(); + let mut version = 0u32; + // SAFETY: `comp` is a live component; the out pointers point at locals of the right types. + let (ds, ns) = unsafe { + let ds = AudioComponentGetDescription(comp, &mut d); + let ns = AudioComponentCopyName(comp, &mut name); + let _ = AudioComponentGetVersion(comp, &mut version); + (ds, ns) + }; + let full = if ns == 0 { cf_string(name) } else { String::new() }; + release(name); + if ds != 0 || d.component_type != ty || d.component_flags & K_AUDIO_COMPONENT_FLAG_REQUIRES_ASYNC != 0 { + continue; + } + out.push(RawComponent { ty, subtype: d.component_sub_type, manufacturer: d.component_manufacturer, name: full, version }); + } + out +} + +/// One global-scope parameter as the unit describes it. +#[derive(Debug, Clone)] +pub struct RawParam { + pub id: u32, + pub name: String, + pub unit: u32, + pub custom_unit: String, + pub min: f32, + pub max: f32, + pub default: f32, + pub flags: u32, + /// Names of the values of an indexed parameter (when it has them). + pub value_strings: Vec, +} + +/// The state the input render callback reads: one buffer per channel, filled by the host before +/// each render. +struct InputState { + bufs: Vec>, + frames: usize, +} + +/// Supplies the effect's input: copies (or lends) our input buffers into the unit's list. +unsafe extern "C" fn input_callback( + ref_con: *mut c_void, + _action_flags: *mut u32, + _time_stamp: *const AudioTimeStamp, + _bus: u32, + frames: u32, + data: *mut AudioBufferListHeader, +) -> OSStatus { + if ref_con.is_null() || data.is_null() { + return -50; // paramErr + } + // SAFETY: `ref_con` is the `InputState` leaked by `Instance::new`, alive until after the unit + // is disposed, and only read here while the host is inside `AudioUnitRender` (it does not + // touch it concurrently). `data` is the unit's buffer list with `number_buffers` buffers laid + // out contiguously from `first`. + unsafe { + let input = &mut *(ref_con as *mut InputState); + let n = (*data).number_buffers as usize; + let first: *mut AudioBuffer = ptr::addr_of_mut!((*data).first); + let frames = (frames as usize).min(MAX_SLICE); + let avail = input.frames.min(frames); + for i in 0..n.min(64) { + let b = &mut *first.add(i); + let last = input.bufs.len().saturating_sub(1); + let Some(src) = input.bufs.get_mut(i.min(last)) else { continue }; + if b.data.is_null() { + // Lend our buffer (the unit may process in place; it is ours to scribble on). + if src.len() >= frames { + b.data = src.as_mut_ptr().cast(); + b.data_byte_size = (frames * 4) as u32; + b.number_channels = 1; + } + continue; + } + let room = (b.data_byte_size as usize / 4).min(frames); + let dst = std::slice::from_raw_parts_mut(b.data as *mut f32, room); + for (k, d) in dst.iter_mut().enumerate() { + *d = if k < avail { src.get(k).copied().unwrap_or(0.0) } else { 0.0 }; + } + } + } + 0 +} + +/// A live Audio Unit instance. +pub struct Instance { + unit: AudioUnit, + /// Leaked `Box` read by `input_callback`; freed in `Drop` after disposal. + input: *mut InputState, + outputs: Vec>, + abl: AudioBufferList, + has_input: bool, + initialized: bool, + channels: usize, + max_frames: usize, + sample_rate: f64, + sample_time: f64, +} + +// SAFETY: an Audio Unit instance may be used from any thread as long as calls are not +// concurrent; `Instance` is only reachable through `&mut`/`&` of its owner, which the `Plugin` +// contract keeps on one thread at a time. The input state is owned exclusively by it. +unsafe impl Send for Instance {} + +impl Instance { + /// Creates (but does not initialize) the component `(ty, subtype, manufacturer)`. + pub fn new(ty: u32, subtype: u32, manufacturer: u32, has_input: bool) -> Result { + let desc = + AudioComponentDescription { component_type: ty, component_sub_type: subtype, component_manufacturer: manufacturer, ..Default::default() }; + // SAFETY: `desc` is a valid description. + let comp = unsafe { AudioComponentFindNext(ptr::null_mut(), &desc) }; + if comp.is_null() { + return Err("component not registered".into()); + } + let mut d = AudioComponentDescription::default(); + // SAFETY: `comp` is a live component; `d` is a local of the right type. + if unsafe { AudioComponentGetDescription(comp, &mut d) } == 0 && d.component_flags & K_AUDIO_COMPONENT_FLAG_REQUIRES_ASYNC != 0 { + return Err("requires asynchronous instantiation (not supported)".into()); + } + let mut unit: AudioUnit = ptr::null_mut(); + // SAFETY: `comp` is a live component; `unit` receives the new instance. + let st = unsafe { AudioComponentInstanceNew(comp, &mut unit) }; + if st != 0 || unit.is_null() { + return Err(format!("AudioComponentInstanceNew failed ({st})")); + } + let input = Box::into_raw(Box::new(InputState { bufs: Vec::new(), frames: 0 })); + let empty = AudioBuffer { number_channels: 1, data_byte_size: 0, data: ptr::null_mut() }; + Ok(Instance { + unit, + input, + outputs: Vec::new(), + abl: AudioBufferList { number_buffers: 0, buffers: [empty; MAX_AU_CHANNELS] }, + has_input, + initialized: false, + channels: 0, + max_frames: 0, + sample_rate: 48_000.0, + sample_time: 0.0, + }) + } + + fn set_property(&self, id: u32, scope: u32, element: u32, value: &T) -> OSStatus { + // SAFETY: `value` points at a live `T` of `size_of::()` bytes, matching what the + // property expects (callers pass the documented type). + unsafe { AudioUnitSetProperty(self.unit, id, scope, element, (value as *const T).cast(), std::mem::size_of::() as u32) } + } + + fn get_f64(&self, id: u32) -> Option { + let mut v = 0f64; + let mut size = std::mem::size_of::() as u32; + // SAFETY: the property is a `Float64`; `v`/`size` are locals of the right types. + let st = unsafe { AudioUnitGetProperty(self.unit, id, SCOPE_GLOBAL, 0, (&mut v as *mut f64).cast(), &mut size) }; + (st == 0 && v.is_finite()).then_some(v) + } + + fn format(&self, scope: u32, sample_rate: f64, channels: usize) -> OSStatus { + let asbd = AudioStreamBasicDescription { + sample_rate, + format_id: FORMAT_LINEAR_PCM, + format_flags: FORMAT_FLAG_FLOAT | FORMAT_FLAG_PACKED | FORMAT_FLAG_NON_INTERLEAVED, + bytes_per_packet: 4, + frames_per_packet: 1, + bytes_per_frame: 4, + channels_per_frame: channels as u32, + bits_per_channel: 32, + reserved: 0, + }; + self.set_property(PROP_STREAM_FORMAT, scope, 0, &asbd) + } + + /// (Re)configures and initializes the unit for `sample_rate`, `max_frames`-frame slices and + /// (preferably) `channels` channels on its main input and output buses. Returns the channel + /// count it accepted. Not realtime-safe (allocates the buffers). + pub fn configure(&mut self, sample_rate: f64, max_frames: usize, channels: usize) -> Result { + if self.initialized { + // SAFETY: the unit is live and initialized. + unsafe { AudioUnitUninitialize(self.unit) }; + self.initialized = false; + } + self.max_frames = 0; + let mf = max_frames.clamp(1, MAX_SLICE); + let _ = self.set_property(PROP_MAX_FRAMES, SCOPE_GLOBAL, 0, &(mf as u32)); + if self.has_input { + let cb = AuRenderCallbackStruct { input_proc: Some(input_callback), input_proc_ref_con: self.input.cast() }; + let st = self.set_property(PROP_SET_RENDER_CALLBACK, SCOPE_INPUT, 0, &cb); + if st != 0 { + return Err(format!("cannot set the input callback ({st})")); + } + } + let want = channels.clamp(1, MAX_AU_CHANNELS); + let mut tried = [0usize; 3]; + let mut ok = None; + let mut last = 0; + for (k, ch) in [want, 2, 1].into_iter().enumerate() { + if tried.contains(&ch) { + continue; + } + if let Some(t) = tried.get_mut(k) { + *t = ch; + } + let o = self.format(SCOPE_OUTPUT, sample_rate, ch); + let i = if self.has_input { self.format(SCOPE_INPUT, sample_rate, ch) } else { 0 }; + if o != 0 || i != 0 { + last = if o != 0 { o } else { i }; + continue; + } + // SAFETY: the unit is live and uninitialized. + let st = unsafe { AudioUnitInitialize(self.unit) }; + if st == 0 { + ok = Some(ch); + break; + } + last = st; + } + let ch = ok.ok_or_else(|| format!("the unit accepts no 1/2/{want}-channel float format ({last})"))?; + self.initialized = true; + self.channels = ch; + self.sample_rate = sample_rate; + self.outputs = (0..ch).map(|_| vec![0.0; mf]).collect(); + // SAFETY: the callback only runs inside `AudioUnitRender`, which is not running now + // (we hold `&mut self`), so this exclusive access does not alias. + let input = unsafe { &mut *self.input }; + input.bufs = if self.has_input { (0..ch).map(|_| vec![0.0; mf]).collect() } else { Vec::new() }; + input.frames = 0; + self.max_frames = mf; + Ok(ch) + } + + pub fn is_ready(&self) -> bool { + self.initialized && self.max_frames > 0 + } + + pub fn max_frames(&self) -> usize { + self.max_frames + } + + /// Input buffer `ch` (for effects), `max_frames` long. + pub fn input_mut(&mut self, ch: usize) -> Option<&mut [f32]> { + // SAFETY: see `configure`: no render is running while we hold `&mut self`. + let input = unsafe { &mut *self.input }; + input.bufs.get_mut(ch).map(Vec::as_mut_slice) + } + + /// Renders `frames` (≤ `max_frames`) into the output buffers. Never allocates. + pub fn render(&mut self, frames: usize) -> Result<(), OSStatus> { + if !self.is_ready() || frames == 0 || frames > self.max_frames { + return Err(-50); + } + { + // SAFETY: as in `input_mut`. + let input = unsafe { &mut *self.input }; + input.frames = frames; + } + let n = self.outputs.len().min(MAX_AU_CHANNELS); + for (b, out) in self.abl.buffers.iter_mut().zip(self.outputs.iter_mut()) { + *b = AudioBuffer { number_channels: 1, data_byte_size: (frames * 4) as u32, data: out.as_mut_ptr().cast() }; + } + self.abl.number_buffers = n as u32; + let ts = AudioTimeStamp { sample_time: self.sample_time, flags: TIMESTAMP_SAMPLE_TIME_VALID, ..Default::default() }; + let mut flags = 0u32; + // SAFETY: the unit is initialized; `ts` and `flags` are locals; `abl` holds `n` buffers + // each pointing at `frames` floats we own; the input callback's state is alive. + let st = unsafe { AudioUnitRender(self.unit, &mut flags, &ts, 0, frames as u32, &mut self.abl) }; + self.sample_time += frames as f64; + if st == 0 { Ok(()) } else { Err(st) } + } + + /// Output channel `ch` of the last render (`frames` long), wherever the unit left it. + pub fn output(&self, ch: usize, frames: usize) -> Option<&[f32]> { + if ch >= self.abl.number_buffers as usize { + return None; + } + let b = self.abl.buffers.get(ch)?; + if b.data.is_null() { + return None; + } + let len = (b.data_byte_size as usize / 4).min(frames); + // SAFETY: after a successful render each buffer points at `data_byte_size` bytes of + // floats, either ours or the unit's own (valid until its next render). + Some(unsafe { std::slice::from_raw_parts(b.data as *const f32, len) }) + } + + pub fn channels(&self) -> usize { + self.channels + } + + pub fn reset(&mut self) { + // SAFETY: the unit is live. + unsafe { AudioUnitReset(self.unit, SCOPE_GLOBAL, 0) }; + } + + /// Latency in samples at the configured rate. + pub fn latency(&self) -> usize { + self.seconds_to_samples(self.get_f64(PROP_LATENCY), 10.0) + } + + /// Tail in samples at the configured rate (capped at a minute). + pub fn tail(&self) -> usize { + self.seconds_to_samples(self.get_f64(PROP_TAIL_TIME), 60.0) + } + + fn seconds_to_samples(&self, s: Option, cap: f64) -> usize { + match s { + Some(s) if s > 0.0 => (s.min(cap) * self.sample_rate).round() as usize, + _ => 0, + } + } + + /// A MIDI channel message at `offset` frames into the next render. + pub fn midi(&self, status: u8, data1: u8, data2: u8, offset: u32) -> bool { + // SAFETY: the unit is live; the call only takes integers. + unsafe { MusicDeviceMIDIEvent(self.unit, u32::from(status), u32::from(data1), u32::from(data2), offset) == 0 } + } + + pub fn get_param(&self, id: u32) -> Option { + let mut v = 0f32; + // SAFETY: the unit is live; `v` is a local `f32`. + let st = unsafe { AudioUnitGetParameter(self.unit, id, SCOPE_GLOBAL, 0, &mut v) }; + (st == 0 && v.is_finite()).then_some(v) + } + + pub fn set_param(&self, id: u32, value: f32) -> bool { + // SAFETY: the unit is live; the call only takes plain values. + unsafe { AudioUnitSetParameter(self.unit, id, SCOPE_GLOBAL, 0, value, 0) == 0 } + } + + /// The global-scope parameters. + pub fn params(&self) -> Vec { + let mut size = 0u32; + let mut writable = 0u8; + // SAFETY: the unit is live; out pointers are locals. + let st = unsafe { AudioUnitGetPropertyInfo(self.unit, PROP_PARAMETER_LIST, SCOPE_GLOBAL, 0, &mut size, &mut writable) }; + if st != 0 || size == 0 { + return Vec::new(); + } + let n = (size as usize / 4).min(MAX_PARAMS); + let mut ids = vec![0u32; n]; + let mut size = (n * 4) as u32; + // SAFETY: `ids` has room for `size` bytes. + let st = unsafe { AudioUnitGetProperty(self.unit, PROP_PARAMETER_LIST, SCOPE_GLOBAL, 0, ids.as_mut_ptr().cast(), &mut size) }; + if st != 0 { + return Vec::new(); + } + ids.truncate(size as usize / 4); + ids.into_iter().filter_map(|id| self.param_info(id)).collect() + } + + fn param_info(&self, id: u32) -> Option { + // SAFETY: an all-zero `AudioUnitParameterInfo` is valid (null pointers, zero numbers). + let mut info: AudioUnitParameterInfo = unsafe { std::mem::zeroed() }; + let mut size = std::mem::size_of::() as u32; + // SAFETY: `info` is a local of the documented type and size. + let st = unsafe { + AudioUnitGetProperty(self.unit, PROP_PARAMETER_INFO, SCOPE_GLOBAL, id, (&mut info as *mut AudioUnitParameterInfo).cast(), &mut size) + }; + if st != 0 { + return None; + } + let owned = info.flags & PARAM_FLAG_CF_NAME_RELEASE != 0; + let mut name = if info.flags & PARAM_FLAG_HAS_CF_NAME != 0 { cf_string(info.cf_name_string) } else { String::new() }; + if name.is_empty() { + let bytes: Vec = info.name.iter().take_while(|&&c| c != 0).map(|&c| c as u8).collect(); + name = String::from_utf8_lossy(&bytes).into_owned(); + } + let custom_unit = if info.unit == UNIT_CUSTOM { cf_string(info.unit_name) } else { String::new() }; + if owned { + if info.flags & PARAM_FLAG_HAS_CF_NAME != 0 { + release(info.cf_name_string); + } + if info.unit == UNIT_CUSTOM { + release(info.unit_name); + } + } + let value_strings = if info.unit == UNIT_INDEXED { self.value_strings(id) } else { Vec::new() }; + Some(RawParam { + id, + name, + unit: info.unit, + custom_unit, + min: info.min_value, + max: info.max_value, + default: info.default_value, + flags: info.flags, + value_strings, + }) + } + + fn value_strings(&self, id: u32) -> Vec { + let mut arr: CFTypeRef = ptr::null(); + let mut size = std::mem::size_of::() as u32; + // SAFETY: the property is a `CFArrayRef` (returned +1); `arr` is a local pointer. + let st = unsafe { + AudioUnitGetProperty(self.unit, PROP_PARAMETER_VALUE_STRINGS, SCOPE_GLOBAL, id, (&mut arr as *mut CFTypeRef).cast(), &mut size) + }; + if st != 0 || arr.is_null() { + return Vec::new(); + } + // SAFETY: `arr` is a live CFArray we own; indices are below its count. + let out = unsafe { + let n = CFArrayGetCount(arr).clamp(0, MAX_VALUE_STRINGS); + (0..n).map(|i| cf_string(CFArrayGetValueAtIndex(arr, i))).collect() + }; + release(arr); + out + } + + /// The unit's `ClassInfo` as a binary property list. + pub fn save_state(&self) -> Option> { + let mut plist: CFTypeRef = ptr::null(); + let mut size = std::mem::size_of::() as u32; + // SAFETY: the property is a `CFPropertyListRef` returned +1; `plist` is a local pointer. + let st = unsafe { AudioUnitGetProperty(self.unit, PROP_CLASS_INFO, SCOPE_GLOBAL, 0, (&mut plist as *mut CFTypeRef).cast(), &mut size) }; + if st != 0 || plist.is_null() { + release(plist); + return None; + } + let mut err: CFTypeRef = ptr::null(); + // SAFETY: `plist` is a live property list; `err` is a local out pointer. + let data = unsafe { CFPropertyListCreateData(ptr::null(), plist, CF_PROPERTY_LIST_BINARY_FORMAT, 0, &mut err) }; + release(err); + release(plist); + if data.is_null() { + return None; + } + // SAFETY: `data` is a live CFData we own; its byte pointer is valid for its length. + let out = unsafe { + let len = CFDataGetLength(data); + let p = CFDataGetBytePtr(data); + if len <= 0 || len as usize > MAX_STATE || p.is_null() { None } else { Some(std::slice::from_raw_parts(p, len as usize).to_vec()) } + }; + release(data); + out + } + + /// Restores a blob from [`Instance::save_state`] (hostile input: parsed by CoreFoundation, + /// must be a dictionary, and the unit validates it). + pub fn load_state(&self, bytes: &[u8]) -> bool { + if bytes.is_empty() || bytes.len() > MAX_STATE { + return false; + } + // SAFETY: `bytes` is a live slice of `len` bytes (copied by CFDataCreate). + let data = unsafe { CFDataCreate(ptr::null(), bytes.as_ptr(), bytes.len() as CFIndex) }; + if data.is_null() { + return false; + } + let mut err: CFTypeRef = ptr::null(); + // SAFETY: `data` is a live CFData; the format out pointer may be null; `err` is local. + let plist = unsafe { CFPropertyListCreateWithData(ptr::null(), data, 0, ptr::null_mut(), &mut err) }; + release(err); + release(data); + if plist.is_null() { + return false; + } + // SAFETY: `plist` is a live CF object we own. + let is_dict = unsafe { CFGetTypeID(plist) == CFDictionaryGetTypeID() }; + let st = if is_dict { self.set_property(PROP_CLASS_INFO, SCOPE_GLOBAL, 0, &plist) } else { -50 }; + release(plist); + st == 0 + } +} + +impl Drop for Instance { + fn drop(&mut self) { + // SAFETY: the unit is live (disposed only here); after disposal nothing can call the + // input callback, so the leaked input state can be freed. + unsafe { + if self.initialized { + AudioUnitUninitialize(self.unit); + } + AudioComponentInstanceDispose(self.unit); + drop(Box::from_raw(self.input)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn struct_layouts_match_the_c_headers() { + assert_eq!(std::mem::size_of::(), 20); + assert_eq!(std::mem::size_of::(), 40); + assert_eq!(std::mem::size_of::(), 24); + assert_eq!(std::mem::size_of::(), 64); + assert_eq!(std::mem::size_of::(), 16); + assert_eq!(std::mem::offset_of!(AudioBufferList, buffers), 8); + assert_eq!(std::mem::offset_of!(AudioBufferListHeader, first), 8); + assert_eq!(std::mem::size_of::(), 16); + assert_eq!(std::mem::offset_of!(AudioUnitParameterInfo, unit_name), 56); + assert_eq!(std::mem::offset_of!(AudioUnitParameterInfo, cf_name_string), 72); + assert_eq!(std::mem::offset_of!(AudioUnitParameterInfo, flags), 96); + assert_eq!(std::mem::size_of::(), 104); + } +} diff --git a/crates/au-host/src/lib.rs b/crates/au-host/src/lib.rs new file mode 100644 index 00000000..7d07e15c --- /dev/null +++ b/crates/au-host/src/lib.rs @@ -0,0 +1,354 @@ +//! SoundCraft's Audio Unit (AUv2, macOS) plugin host. +//! +//! Lists the effect (`aufx`), music effect (`aumf`) and instrument (`aumu`) Audio Units +//! registered with the system's component manager and exposes each one as a +//! [`soundcraft_dsp::Plugin`], so the mix engine treats them exactly like the built-in plugins. +//! Plugin ids are `au:::` with the three four-char codes written out +//! (`au:aufx:dely:appl`); a code with bytes outside printable ASCII is written `#` + 8 upper-case +//! hex digits instead. Parameter ids are global-scope `AudioUnitParameterID`s as decimal strings; +//! values are the Audio Unit's own (plain) values, ranges and units, so no mapping is needed. +//! Boolean parameters become toggles, indexed ones starting at 0 become choices. +//! +//! Like `soundcraft-vst3-host`, this is an isolated `unsafe` crate (craftrules never-crash): +//! `unsafe` is denied crate-wide and allowed only in the private `ffi` module (hand-written +//! AudioToolbox / CoreFoundation declarations). The public API is safe, returns +//! `Result`/`Option` and never panics. Limits of in-process hosting: an Audio Unit that itself +//! crashes takes the process down with it. Components that require asynchronous instantiation +//! (out-of-process AUv3 extensions) are not listed. +//! +//! `PluginInfo` must be `&'static`: the registry leaks exactly one `PluginInfo` (with its param +//! table) per plugin *type* the first time that type is instantiated, and caches it. +//! +//! Plugin state: [`Plugin::save_state`] returns the unit's `ClassInfo` property list +//! (`kAudioUnitProperty_ClassInfo`) as a binary plist; [`Plugin::load_state`] parses it back and +//! sets it, then re-reads the parameter values. +//! +//! Not hosted yet: Audio Unit editor views (Cocoa `AUCocoaUIBase`), sidechain/extra buses, +//! parameters outside the global scope, and host callbacks (tempo, transport). +//! +//! Off macOS (and on `wasm32`) the crate compiles to stubs: scans are empty and nothing can be +//! created. + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +#[cfg(target_os = "macos")] +#[allow(unsafe_code)] +mod ffi; +#[cfg(target_os = "macos")] +mod plugin; +#[cfg(target_os = "macos")] +mod registry; + +#[cfg(target_os = "macos")] +pub use plugin::AuPlugin; + +use soundcraft_dsp::{Category, Plugin, PluginInfo}; + +/// Prefix of every hosted-Audio-Unit plugin id. +pub const ID_PREFIX: &str = "au:"; + +/// `aufx`: an audio effect. +pub const TYPE_EFFECT: u32 = u32::from_be_bytes(*b"aufx"); +/// `aumf`: a music effect (an effect that also takes MIDI). +pub const TYPE_MUSIC_EFFECT: u32 = u32::from_be_bytes(*b"aumf"); +/// `aumu`: a music device (instrument). +pub const TYPE_INSTRUMENT: u32 = u32::from_be_bytes(*b"aumu"); +/// The component types SoundCraft hosts. +pub const SUPPORTED_TYPES: [u32; 3] = [TYPE_EFFECT, TYPE_MUSIC_EFFECT, TYPE_INSTRUMENT]; + +/// True when a component type is one SoundCraft hosts (effects, music effects, instruments). +pub fn is_supported_type(ty: u32) -> bool { + SUPPORTED_TYPES.contains(&ty) +} + +/// Writes a four-char code: its 4 characters when all are printable ASCII (`"dls "`), else `#` +/// followed by 8 upper-case hex digits. +pub fn fourcc_string(code: u32) -> String { + let b = code.to_be_bytes(); + if b.iter().all(|c| (0x20..=0x7e).contains(c)) { b.iter().map(|&c| char::from(c)).collect() } else { format!("#{code:08X}") } +} + +/// Parses one four-char code written by [`fourcc_string`]. +pub fn parse_fourcc(s: &str) -> Option { + match parse_field(s)? { + (code, "") => Some(code), + _ => None, + } +} + +/// Parses one code at the start of `s`, returning it and the rest. +fn parse_field(s: &str) -> Option<(u32, &str)> { + let bytes = s.as_bytes(); + // `#XXXXXXXX`, only when followed by the end or a field separator (a printable code may + // itself start with `#`). + if bytes.first() == Some(&b'#') + && let Some(hex) = s.get(1..9) + && hex.bytes().all(|c| c.is_ascii_hexdigit()) + && matches!(bytes.get(9), None | Some(b':')) + && let Ok(code) = u32::from_str_radix(hex, 16) + { + return Some((code, s.get(9..)?)); + } + let four = bytes.get(..4)?; + if !four.iter().all(|c| (0x20..=0x7e).contains(c)) { + return None; + } + let code = u32::from_be_bytes([*four.first()?, *four.get(1)?, *four.get(2)?, *four.get(3)?]); + Some((code, s.get(4..)?)) +} + +/// The SoundCraft id of a component: `au:::`. +pub fn format_id(ty: u32, subtype: u32, manufacturer: u32) -> String { + format!("{ID_PREFIX}{}:{}:{}", fourcc_string(ty), fourcc_string(subtype), fourcc_string(manufacturer)) +} + +/// The `(type, subtype, manufacturer)` codes of an `au:…` id (`None` for other ids or a +/// component type SoundCraft does not host). +pub fn parse_id(id: &str) -> Option<(u32, u32, u32)> { + let rest = id.strip_prefix(ID_PREFIX)?; + let (ty, rest) = parse_field(rest)?; + let (sub, rest) = parse_field(rest.strip_prefix(':')?)?; + let (manu, rest) = parse_field(rest.strip_prefix(':')?)?; + (rest.is_empty() && is_supported_type(ty)).then_some((ty, sub, manu)) +} + +/// The insert-menu category of a component, from its type, subtype and name. +pub fn category_for(ty: u32, subtype: u32, name: &str) -> Category { + if ty == TYPE_INSTRUMENT { + return Category::Instrument; + } + let by_sub = match &subtype.to_be_bytes() { + b"dely" | b"sdly" | b"mdly" => Some(Category::Delay), + b"lpas" | b"hpas" | b"bpas" | b"hshf" | b"lshf" | b"pmeq" | b"nbeq" | b"geq " => Some(Category::Eq), + b"dcmp" | b"mcmp" | b"lmtr" | b"xpnd" | b"ngte" => Some(Category::Dynamics), + b"rvb2" | b"mrev" => Some(Category::Reverb), + b"dist" => Some(Category::Harmonic), + b"tmpt" | b"pitc" | b"nutp" => Some(Category::PitchShift), + _ => None, + }; + if let Some(c) = by_sub { + return c; + } + let n = name.to_ascii_lowercase(); + let has = |words: &[&str]| words.iter().any(|w| n.contains(w)); + if has(&["reverb", "verb", "room", "hall", "plate"]) { + Category::Reverb + } else if has(&["delay", "echo"]) { + Category::Delay + } else if has(&["eq", "filter", "lowpass", "highpass", "bandpass", "shelf"]) { + Category::Eq + } else if has(&["compress", "limit", "gate", "expand", "dynamic", "de-ess"]) { + Category::Dynamics + } else if has(&["chorus", "flang", "phase", "tremolo", "vibrato", "rotary", "ring mod"]) { + Category::Modulation + } else if has(&["distort", "satur", "overdrive", "fuzz", "bitcrush"]) { + Category::Harmonic + } else if has(&["pitch", "tune", "shift"]) { + Category::PitchShift + } else { + Category::Other + } +} + +/// Splits a component name `"Vendor: Plugin"` into `(vendor, name)`. +pub fn split_component_name(full: &str) -> (String, String) { + match full.split_once(':') { + Some((v, n)) if !n.trim().is_empty() => (v.trim().to_string(), n.trim().to_string()), + _ => (String::new(), full.trim().to_string()), + } +} + +/// One Audio Unit found by a scan. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct AuDescriptor { + /// SoundCraft id: `au:::`. + pub id: String, + pub name: String, + pub vendor: String, + /// `major.minor.bugfix` from the component version. + pub version: String, + /// The four-char codes as written in the id. + pub component_type: String, + pub subtype: String, + pub manufacturer: String, + pub category: Category, + pub is_instrument: bool, +} + +/// Why an Audio Unit operation failed. +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum AuError { + #[error("no Audio Unit `{0}`")] + NotFound(String), + #[error("cannot create Audio Unit `{0}`: {1}")] + Instantiate(String, String), + #[error("Audio Unit processing: {0}")] + Process(String), + #[error("Audio Units are not supported on this platform")] + Unsupported, +} + +#[cfg(target_os = "macos")] +mod api { + use super::*; + + pub fn scan() -> Vec { + registry::scan() + } + pub fn rescan() -> Vec { + registry::rescan() + } + pub fn instantiate(id: &str) -> Result, AuError> { + registry::instantiate(id) + } + pub fn plugin_info(id: &str) -> Option<&'static PluginInfo> { + registry::plugin_info(id) + } +} + +#[cfg(not(target_os = "macos"))] +mod api { + use super::*; + + pub fn scan() -> Vec { + Vec::new() + } + pub fn rescan() -> Vec { + Vec::new() + } + pub fn instantiate(_id: &str) -> Result, AuError> { + Err(AuError::Unsupported) + } + pub fn plugin_info(_id: &str) -> Option<&'static PluginInfo> { + None + } +} + +/// Every hostable Audio Unit registered on the system (effects, music effects, instruments). +/// Listed on first call (reads the component registry, creates no instances), then cached. +pub fn scan() -> Vec { + api::scan() +} + +/// Forgets the cached scan (and remembered failures) and lists again. +pub fn rescan() -> Vec { + api::rescan() +} + +/// Creates a plugin by `au:::`, prepared for 48 kHz stereo with +/// 1024-frame blocks (like `soundcraft_dsp::create`), with the reason when it fails. +pub fn instantiate(id: &str) -> Result, AuError> { + api::instantiate(id) +} + +/// Like [`instantiate`], but returns the concrete [`AuPlugin`]. +#[cfg(target_os = "macos")] +pub fn instantiate_plugin(id: &str) -> Result { + registry::instantiate_plugin(id) +} + +/// Creates a plugin by `au:…` id; `None` if unknown or it fails to load. +pub fn create(id: &str) -> Option> { + parse_id(id)?; + match api::instantiate(id) { + Ok(p) => Some(p), + Err(e) => { + log::warn!("{e}"); + None + } + } +} + +/// The description of an `au:…` plugin. The first call for a type instantiates it once to read +/// its parameters; the result is cached (and leaked, see the crate docs). +pub fn plugin_info(id: &str) -> Option<&'static PluginInfo> { + api::plugin_info(id) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fourccs_round_trip() { + assert_eq!(parse_fourcc("aufx"), Some(TYPE_EFFECT)); + assert_eq!(fourcc_string(TYPE_INSTRUMENT), "aumu"); + assert_eq!(fourcc_string(u32::from_be_bytes(*b"dls ")), "dls "); + assert_eq!(fourcc_string(0x0102_0304), "#01020304"); + assert_eq!(parse_fourcc("#01020304"), Some(0x0102_0304)); + assert_eq!(parse_fourcc("#0102030"), None); + assert_eq!(parse_fourcc("#abc"), Some(u32::from_be_bytes(*b"#abc"))); + assert_eq!(parse_fourcc("abc"), None); + assert_eq!(parse_fourcc("abcde"), None); + assert_eq!(parse_fourcc("ab\u{e9}"), None); + assert_eq!(parse_fourcc(""), None); + for code in [0u32, 0x7f7f_7f7f, 0x2020_2020, u32::MAX, u32::from_be_bytes(*b"a:b:")] { + assert_eq!(parse_fourcc(&fourcc_string(code)), Some(code), "{code:08X}"); + } + } + + #[test] + fn ids_parse_and_format() { + let dly = (TYPE_EFFECT, u32::from_be_bytes(*b"dely"), u32::from_be_bytes(*b"appl")); + assert_eq!(format_id(dly.0, dly.1, dly.2), "au:aufx:dely:appl"); + assert_eq!(parse_id("au:aufx:dely:appl"), Some(dly)); + let dls = (TYPE_INSTRUMENT, u32::from_be_bytes(*b"dls "), u32::from_be_bytes(*b"appl")); + assert_eq!(parse_id("au:aumu:dls :appl"), Some(dls)); + // Colons inside a code are fine: fields have fixed widths. + let odd = (TYPE_MUSIC_EFFECT, u32::from_be_bytes(*b"a:b:"), 0x0000_00FF); + let s = format_id(odd.0, odd.1, odd.2); + assert_eq!(s, "au:aumf:a:b::#000000FF"); + assert_eq!(parse_id(&s), Some(odd)); + for bad in [ + "au:", + "au:aufx", + "au:aufx:dely", + "au:aufx:dely:", + "au:aufx:dely:appl:", + "au:aufx:dely:appl ", + "au:aufxdelyappl", + "au:aufx;dely;appl", + "AU:aufx:dely:appl", + "au:aufc:dely:appl", + "au:auou:def :appl", + "au:aufx:dé:appl", + "vst3:0123", + "eq_7band", + "", + ] { + assert_eq!(parse_id(bad), None, "{bad:?}"); + } + } + + #[test] + fn categories_and_names() { + let c = |ty: &[u8; 4], sub: &[u8; 4], name: &str| category_for(u32::from_be_bytes(*ty), u32::from_be_bytes(*sub), name); + assert_eq!(c(b"aumu", b"dls ", "DLSMusicDevice"), Category::Instrument); + assert_eq!(c(b"aufx", b"dely", "AUDelay"), Category::Delay); + assert_eq!(c(b"aufx", b"lpas", "AULowpass"), Category::Eq); + assert_eq!(c(b"aufx", b"dcmp", "AUDynamicsProcessor"), Category::Dynamics); + assert_eq!(c(b"aufx", b"mrev", "AUMatrixReverb"), Category::Reverb); + assert_eq!(c(b"aufx", b"xxxx", "Big Plate Verb"), Category::Reverb); + assert_eq!(c(b"aufx", b"xxxx", "Tape Echo"), Category::Delay); + assert_eq!(c(b"aufx", b"xxxx", "Thing"), Category::Other); + assert_eq!(split_component_name("Apple: AUDelay"), ("Apple".into(), "AUDelay".into())); + assert_eq!(split_component_name("NoVendor"), (String::new(), "NoVendor".into())); + assert_eq!(split_component_name("Odd:"), (String::new(), "Odd:".into())); + } + + #[test] + fn foreign_ids_are_rejected_without_scanning() { + assert!(create("eq_7band").is_none()); + assert!(create("vst3:0123456789ABCDEF0123456789ABCDEF").is_none()); + assert!(plugin_info("reverb").is_none()); + assert!(plugin_info("au:aufx").is_none()); + } + + #[test] + fn unknown_components_are_errors_not_crashes() { + let missing = "au:aufx:zzzz:zzzz"; + assert!(create(missing).is_none()); + assert!(plugin_info(missing).is_none()); + assert!(instantiate(missing).is_err()); + } +} diff --git a/crates/au-host/src/plugin.rs b/crates/au-host/src/plugin.rs new file mode 100644 index 00000000..c76aab00 --- /dev/null +++ b/crates/au-host/src/plugin.rs @@ -0,0 +1,251 @@ +//! `AuPlugin`: a loaded Audio Unit behind the safe `soundcraft_dsp::Plugin` trait. + +use crate::ffi::Instance; +use soundcraft_dsp::{Plugin, PluginInfo}; + +/// Most MIDI events queued between two blocks (extra events are dropped). +const EVENT_CAP: usize = 1024; +/// Most channels a host strip may ask for. +const MAX_CHANNELS: usize = 64; + +/// A queued MIDI channel message at a frame offset within the next `process` call. +#[derive(Debug, Clone, Copy, PartialEq)] +struct MidiEvent { + time: u32, + status: u8, + data1: u8, + data2: u8, +} + +/// A hosted Audio Unit. Parameter ids are global-scope `AudioUnitParameterID`s as decimal +/// strings; values are the unit's own values, with names and ranges in `info().params`. +pub struct AuPlugin { + inst: Instance, + info: &'static PluginInfo, + /// Audio Unit id of each `info.params` entry (same order). + ids: Vec, + values: Vec, + /// Pending MIDI, kept sorted by time. Never grows past `EVENT_CAP`. + events: Vec, + held: [bool; 128], + /// Takes audio input (effects); instruments replace the buffer contents. + has_input: bool, + /// Accepts MIDI (instruments and music effects). + takes_midi: bool, + channels: usize, + latency: usize, + tail: usize, +} + +impl AuPlugin { + pub(crate) fn new(inst: Instance, info: &'static PluginInfo, has_input: bool, takes_midi: bool) -> AuPlugin { + let ids: Vec = info.params.iter().map(|p| p.id.parse::().unwrap_or(u32::MAX)).collect(); + let values = info.params.iter().zip(&ids).map(|(p, &id)| inst.get_param(id).map_or(p.default, |v| p.clamp(v))).collect(); + AuPlugin { + inst, + info, + ids, + values, + events: Vec::with_capacity(EVENT_CAP), + held: [false; 128], + has_input, + takes_midi, + channels: 2, + latency: 0, + tail: 0, + } + } + + /// Re-reads every parameter value from the unit (after a state load). + fn refresh_values(&mut self) { + for ((slot, p), &id) in self.values.iter_mut().zip(self.info.params).zip(&self.ids) { + if let Some(v) = self.inst.get_param(id) { + *slot = p.clamp(v); + } + } + } + + /// Inserts keeping the queue time-sorted; drops the event when the queue is full (never + /// allocates on the audio thread). + fn push_event(&mut self, e: MidiEvent) { + if !self.takes_midi || self.events.len() >= self.events.capacity() { + return; + } + let pos = self.events.iter().rposition(|q| q.time <= e.time).map_or(0, |i| i + 1); + self.events.insert(pos, e); + } +} + +impl Plugin for AuPlugin { + fn info(&self) -> &'static PluginInfo { + self.info + } + + fn prepare(&mut self, sample_rate: f32, max_block: usize, channels: usize) { + let sr = if sample_rate.is_finite() && sample_rate > 0.0 { f64::from(sample_rate) } else { 48_000.0 }; + self.channels = channels.clamp(1, MAX_CHANNELS); + match self.inst.configure(sr, max_block, self.channels) { + Ok(_) => { + self.latency = self.inst.latency(); + self.tail = self.inst.tail(); + } + Err(e) => { + log::warn!("{}: {e}", self.info.id); + self.latency = 0; + self.tail = 0; + } + } + // Re-send every value so a re-initialized unit matches the session. + for (&id, &v) in self.ids.iter().zip(&self.values) { + self.inst.set_param(id, v); + } + } + + fn reset(&mut self) { + self.events.clear(); + self.held = [false; 128]; + self.inst.reset(); + } + + fn set_param(&mut self, id: &str, value: f32) -> bool { + let Some(i) = self.info.params.iter().position(|p| p.id == id) else { return false }; + let (Some(pi), Some(&aid)) = (self.info.params.get(i), self.ids.get(i)) else { return false }; + let v = pi.clamp(value); + if let Some(slot) = self.values.get_mut(i) { + *slot = v; + } + self.inst.set_param(aid, v); + true + } + + fn param(&self, id: &str) -> Option { + let i = self.info.params.iter().position(|p| p.id == id)?; + self.values.get(i).copied() + } + + fn process(&mut self, io: &mut [Vec], frames: usize) { + let ch = self.channels.min(io.len()); + let frames = io.iter().take(ch).map(Vec::len).fold(frames, usize::min); + if frames == 0 || ch == 0 { + return; + } + if !self.inst.is_ready() { + // Effects pass through; instruments are silent. + if !self.has_input { + for c in io.iter_mut().take(ch) { + c.iter_mut().take(frames).for_each(|s| *s = 0.0); + } + } + self.events.clear(); + return; + } + let max = self.inst.max_frames(); + let au_ch = self.inst.channels(); + let last = u32::try_from(frames - 1).unwrap_or(u32::MAX); + for e in &mut self.events { + e.time = e.time.min(last); + } + let mut start = 0usize; + while start < frames { + let n = (frames - start).min(max); + if self.has_input { + for c in 0..au_ch { + let src = io.get(c.min(ch - 1)).and_then(|s| s.get(start..start + n)); + if let (Some(src), Some(dst)) = (src, self.inst.input_mut(c).and_then(|d| d.get_mut(..n))) { + dst.copy_from_slice(src); + } + } + } + let end = start + n; + let k = self.events.iter().position(|e| e.time as usize >= end).unwrap_or(self.events.len()); + for e in self.events.iter().take(k) { + let offset = u32::try_from((e.time as usize).saturating_sub(start)).unwrap_or(0); + self.inst.midi(e.status, e.data1, e.data2, offset); + } + self.events.drain(..k); + match self.inst.render(n) { + Ok(()) => { + for (c, dst) in io.iter_mut().take(ch).enumerate() { + let Some(dst) = dst.get_mut(start..end) else { continue }; + if ch == 1 && au_ch >= 2 { + if let (Some(l), Some(r)) = (self.inst.output(0, n), self.inst.output(1, n)) { + for ((d, a), b) in dst.iter_mut().zip(l).zip(r) { + let v = 0.5 * (a + b); + *d = if v.is_finite() { v } else { 0.0 }; + } + } + } else if let Some(src) = self.inst.output(c.min(au_ch.saturating_sub(1)), n) { + for (d, &s) in dst.iter_mut().zip(src) { + *d = if s.is_finite() { s } else { 0.0 }; + } + } + } + } + Err(e) => { + log::debug!("{}: render failed ({e})", self.info.id); + if !self.has_input { + for c in io.iter_mut().take(ch) { + if let Some(d) = c.get_mut(start..end) { + d.iter_mut().for_each(|s| *s = 0.0); + } + } + } + } + } + start = end; + } + } + + fn latency(&self) -> usize { + self.latency + } + + fn tail_samples(&self) -> usize { + self.tail + } + + fn note_on(&mut self, offset: usize, note: u8, velocity: u8) { + if note > 127 { + return; + } + let time = u32::try_from(offset).unwrap_or(u32::MAX); + // Velocity 0 would be a note-off. + self.push_event(MidiEvent { time, status: 0x90, data1: note, data2: velocity.clamp(1, 127) }); + if let Some(h) = self.held.get_mut(note as usize) { + *h = true; + } + } + + fn note_off(&mut self, offset: usize, note: u8) { + if note > 127 { + return; + } + let time = u32::try_from(offset).unwrap_or(u32::MAX); + self.push_event(MidiEvent { time, status: 0x80, data1: note, data2: 0 }); + if let Some(h) = self.held.get_mut(note as usize) { + *h = false; + } + } + + fn all_notes_off(&mut self) { + for k in 0..128u8 { + if self.held.get(k as usize).copied().unwrap_or(false) { + self.note_off(0, k); + } + } + // CC 123: All Notes Off (catches anything the unit still sustains). + self.push_event(MidiEvent { time: 0, status: 0xB0, data1: 123, data2: 0 }); + } + + fn save_state(&mut self) -> Option> { + self.inst.save_state() + } + + fn load_state(&mut self, data: &[u8]) -> bool { + let ok = self.inst.load_state(data); + if ok { + self.refresh_values(); + } + ok + } +} diff --git a/crates/au-host/src/registry.rs b/crates/au-host/src/registry.rs new file mode 100644 index 00000000..fae31379 --- /dev/null +++ b/crates/au-host/src/registry.rs @@ -0,0 +1,281 @@ +//! The process-wide Audio Unit registry: scan cache, leaked `PluginInfo`s, instantiation. + +use crate::ffi::{self, Instance, PARAM_FLAG_DISPLAY_LOG, PARAM_FLAG_METER_READ_ONLY, PARAM_FLAG_WRITABLE, RawParam}; +use crate::plugin::AuPlugin; +use crate::{AuDescriptor, AuError, SUPPORTED_TYPES, TYPE_EFFECT, TYPE_INSTRUMENT, TYPE_MUSIC_EFFECT, parse_id}; +use soundcraft_dsp::{ParamInfo, Plugin, PluginInfo, Taper, Unit}; +use std::collections::{HashMap, HashSet}; +use std::sync::{Mutex, OnceLock, PoisonError}; + +/// Most labels for an indexed parameter shown as a choice (more become a plain range). +const MAX_CHOICES: f32 = 128.0; + +// AudioUnitParameterUnit values we map. +const U_BOOLEAN: u32 = 2; +const U_PERCENT: u32 = 3; +const U_SECONDS: u32 = 4; +const U_HERTZ: u32 = 8; +const U_CENTS: u32 = 9; +const U_SEMITONES: u32 = 10; +const U_DECIBELS: u32 = 13; +const U_METERS: u32 = 19; +const U_ABSOLUTE_CENTS: u32 = 20; +const U_MILLISECONDS: u32 = 24; +const U_RATIO: u32 = 25; + +#[derive(Default)] +struct Registry { + scanned: Option>, + /// One leaked `PluginInfo` per plugin type that was actually instantiated. + infos: HashMap, + /// Ids that failed to instantiate (not retried every UI frame). + failed: HashSet, +} + +fn registry() -> &'static Mutex { + static R: OnceLock> = OnceLock::new(); + R.get_or_init(|| Mutex::new(Registry::default())) +} + +fn version_string(v: u32) -> String { + format!("{}.{}.{}", v >> 16, (v >> 8) & 0xff, v & 0xff) +} + +fn list() -> Vec { + let mut out: Vec = Vec::new(); + for ty in SUPPORTED_TYPES { + for c in ffi::components(ty) { + let id = crate::format_id(c.ty, c.subtype, c.manufacturer); + if out.iter().any(|o| o.id == id) { + continue; + } + let (vendor, name) = crate::split_component_name(&c.name); + let name = if name.is_empty() { "Audio Unit".to_string() } else { name }; + out.push(AuDescriptor { + category: crate::category_for(c.ty, c.subtype, &name), + is_instrument: c.ty == TYPE_INSTRUMENT, + component_type: crate::fourcc_string(c.ty), + subtype: crate::fourcc_string(c.subtype), + manufacturer: crate::fourcc_string(c.manufacturer), + version: version_string(c.version), + id, + name, + vendor, + }); + } + } + out.sort_by_key(|d| d.name.to_lowercase()); + out +} + +fn ensure_scanned(r: &mut Registry) -> &Vec { + if r.scanned.is_none() { + r.scanned = Some(list()); + } + r.scanned.get_or_insert_with(Vec::new) +} + +pub fn scan() -> Vec { + let mut r = registry().lock().unwrap_or_else(PoisonError::into_inner); + ensure_scanned(&mut r).clone() +} + +pub fn rescan() -> Vec { + let mut r = registry().lock().unwrap_or_else(PoisonError::into_inner); + r.scanned = None; + r.failed.clear(); + ensure_scanned(&mut r).clone() +} + +fn leak(s: String) -> &'static str { + Box::leak(s.into_boxed_str()) +} + +/// Maps an Audio Unit parameter unit (and a custom unit label) to a SoundCraft display unit. +pub(crate) fn unit_for(unit: u32, custom: &str) -> Unit { + match unit { + U_DECIBELS => Unit::Db, + U_HERTZ => Unit::Hz, + U_MILLISECONDS => Unit::Ms, + U_PERCENT => Unit::Percent, + U_SECONDS => Unit::Seconds, + U_SEMITONES => Unit::Semitones, + U_CENTS | U_ABSOLUTE_CENTS => Unit::Cents, + U_RATIO => Unit::Ratio, + ffi::UNIT_CUSTOM => match custom.trim().to_ascii_lowercase().as_str() { + "db" => Unit::Db, + "hz" => Unit::Hz, + "ms" => Unit::Ms, + "%" => Unit::Percent, + "s" | "sec" | "secs" => Unit::Seconds, + _ => Unit::None, + }, + _ => Unit::None, + } +} + +/// The SoundCraft description of one parameter (`None` for read-only/meter/broken ones). +pub(crate) fn param_info(p: &RawParam) -> Option { + if p.flags & PARAM_FLAG_WRITABLE == 0 || p.flags & PARAM_FLAG_METER_READ_ONLY != 0 || p.unit == U_METERS { + return None; + } + if !(p.min.is_finite() && p.max.is_finite() && p.max > p.min) { + return None; + } + let name = if p.name.trim().is_empty() { format!("Param {}", p.id) } else { p.name.trim().to_string() }; + let default = if p.default.is_finite() { p.default.clamp(p.min, p.max) } else { p.min }; + let base = ParamInfo { + id: leak(p.id.to_string()), + name: leak(name), + min: p.min, + max: p.max, + default, + unit: Unit::None, + taper: Taper::Linear, + choices: &[], + }; + if p.unit == U_BOOLEAN && p.min == 0.0 && p.max == 1.0 { + return Some(ParamInfo { unit: Unit::Toggle, default: default.round(), ..base }); + } + if p.unit == ffi::UNIT_INDEXED && p.min == 0.0 && p.max.fract() == 0.0 && p.max <= MAX_CHOICES { + let n = p.max as usize; + let labels: Vec<&'static str> = + (0..=n).map(|i| leak(p.value_strings.get(i).filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| i.to_string()))).collect(); + return Some(ParamInfo { unit: Unit::Choice, default: default.round(), choices: Box::leak(labels.into_boxed_slice()), ..base }); + } + let unit = unit_for(p.unit, &p.custom_unit); + let log = p.min > 0.0 && (p.flags & PARAM_FLAG_DISPLAY_LOG != 0 || (unit == Unit::Hz && p.max / p.min >= 10.0)); + Some(ParamInfo { unit, taper: if log { Taper::Log } else { Taper::Linear }, ..base }) +} + +/// Builds and leaks the `PluginInfo` for one plugin type. Called at most once per id (cached). +fn build_info(d: &AuDescriptor, inst: &Instance) -> &'static PluginInfo { + let mut seen = HashSet::new(); + let params: Vec = inst.params().iter().filter(|p| seen.insert(p.id)).filter_map(param_info).collect(); + let short: String = d.name.chars().take(8).collect(); + Box::leak(Box::new(PluginInfo { + id: leak(d.id.clone()), + name: leak(d.name.clone()), + short_name: leak(short), + category: d.category, + params: Box::leak(params.into_boxed_slice()), + is_instrument: d.is_instrument, + audiosuite: false, + })) +} + +/// Finds the descriptor and creates an instance plus its (cached) info. +fn instantiate_raw(id: &str) -> Result<(Instance, &'static PluginInfo, u32), AuError> { + let (ty, sub, manu) = parse_id(id).ok_or_else(|| AuError::NotFound(id.to_string()))?; + let desc = { + let mut r = registry().lock().unwrap_or_else(PoisonError::into_inner); + if r.failed.contains(id) { + return Err(AuError::Instantiate(id.to_string(), "failed before".into())); + } + ensure_scanned(&mut r).iter().find(|d| d.id == id).cloned().ok_or_else(|| AuError::NotFound(id.to_string()))? + }; + let has_input = ty == TYPE_EFFECT || ty == TYPE_MUSIC_EFFECT; + let made = Instance::new(ty, sub, manu, has_input); + let mut r = registry().lock().unwrap_or_else(PoisonError::into_inner); + match made { + Ok(inst) => { + let info = match r.infos.get(id) { + Some(i) => *i, + None => { + let i = build_info(&desc, &inst); + r.infos.insert(id.to_string(), i); + i + } + }; + Ok((inst, info, ty)) + } + Err(e) => { + r.failed.insert(id.to_string()); + Err(AuError::Instantiate(id.to_string(), e)) + } + } +} + +pub fn instantiate(id: &str) -> Result, AuError> { + Ok(Box::new(instantiate_plugin(id)?)) +} + +pub fn instantiate_plugin(id: &str) -> Result { + let (inst, info, ty) = instantiate_raw(id)?; + let has_input = ty == TYPE_EFFECT || ty == TYPE_MUSIC_EFFECT; + let takes_midi = ty == TYPE_INSTRUMENT || ty == TYPE_MUSIC_EFFECT; + let mut p = AuPlugin::new(inst, info, has_input, takes_midi); + p.prepare(48_000.0, 1024, 2); + Ok(p) +} + +pub fn plugin_info(id: &str) -> Option<&'static PluginInfo> { + parse_id(id)?; + { + let r = registry().lock().unwrap_or_else(PoisonError::into_inner); + if let Some(i) = r.infos.get(id) { + return Some(*i); + } + if r.failed.contains(id) { + return None; + } + } + // Instantiate once to learn the parameters. + instantiate_raw(id).ok().map(|(_, info, _)| info) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw(unit: u32, min: f32, max: f32, default: f32, flags: u32) -> RawParam { + RawParam { + id: 7, + name: "Cutoff".into(), + unit, + custom_unit: String::new(), + min, + max, + default, + flags: flags | PARAM_FLAG_WRITABLE | ffi::PARAM_FLAG_READABLE, + value_strings: Vec::new(), + } + } + + #[test] + fn units_map() { + assert_eq!(unit_for(U_DECIBELS, ""), Unit::Db); + assert_eq!(unit_for(U_HERTZ, ""), Unit::Hz); + assert_eq!(unit_for(U_MILLISECONDS, ""), Unit::Ms); + assert_eq!(unit_for(U_CENTS, ""), Unit::Cents); + assert_eq!(unit_for(ffi::UNIT_CUSTOM, " dB "), Unit::Db); + assert_eq!(unit_for(ffi::UNIT_CUSTOM, "furlongs"), Unit::None); + assert_eq!(unit_for(999, ""), Unit::None); + } + + #[test] + fn params_map_and_hostile_ones_are_dropped() { + let p = param_info(&raw(U_HERTZ, 10.0, 22_050.0, 6900.0, 0)).unwrap(); + assert_eq!((p.id, p.name, p.unit, p.taper, p.default), ("7", "Cutoff", Unit::Hz, Taper::Log, 6900.0)); + let t = param_info(&raw(U_BOOLEAN, 0.0, 1.0, 1.0, 0)).unwrap(); + assert_eq!(t.unit, Unit::Toggle); + let mut ix = raw(ffi::UNIT_INDEXED, 0.0, 2.0, 1.0, 0); + ix.value_strings = vec!["Low".into(), String::new()]; + let c = param_info(&ix).unwrap(); + assert_eq!((c.unit, c.choices), (Unit::Choice, &["Low", "1", "2"][..])); + // Indexed not starting at 0 stays a plain range. + assert_eq!(param_info(&raw(ffi::UNIT_INDEXED, 1.0, 4.0, 1.0, 0)).unwrap().unit, Unit::None); + // Hostile defaults clamp; broken ranges, meters and read-only params are skipped. + assert_eq!(param_info(&raw(U_DECIBELS, -20.0, 20.0, f32::NAN, 0)).unwrap().default, -20.0); + assert_eq!(param_info(&raw(U_DECIBELS, -20.0, 20.0, 99.0, 0)).unwrap().default, 20.0); + assert!(param_info(&raw(U_DECIBELS, 1.0, 1.0, 1.0, 0)).is_none()); + assert!(param_info(&raw(U_DECIBELS, f32::NEG_INFINITY, 0.0, 0.0, 0)).is_none()); + assert!(param_info(&raw(U_DECIBELS, 0.0, f32::NAN, 0.0, 0)).is_none()); + assert!(param_info(&raw(U_METERS, 0.0, 1.0, 0.0, 0)).is_none()); + assert!(param_info(&raw(U_DECIBELS, 0.0, 1.0, 0.0, PARAM_FLAG_METER_READ_ONLY)).is_none()); + let mut ro = raw(U_DECIBELS, 0.0, 1.0, 0.0, 0); + ro.flags &= !PARAM_FLAG_WRITABLE; + assert!(param_info(&ro).is_none()); + assert_eq!(version_string(0x0001_0203), "1.2.3"); + } +} diff --git a/crates/au-host/tests/system_units.rs b/crates/au-host/tests/system_units.rs new file mode 100644 index 00000000..2e246758 --- /dev/null +++ b/crates/au-host/tests/system_units.rs @@ -0,0 +1,170 @@ +//! End to end against the Audio Units Apple ships with macOS: scan → instantiate → params → +//! process → state, plus an instrument playing a note. Skips (passes) when a unit is missing. +#![cfg(target_os = "macos")] + +use soundcraft_dsp::{Category, Plugin}; + +const LOWPASS: &str = "au:aufx:lpas:appl"; +const DELAY: &str = "au:aufx:dely:appl"; +const DLS: &str = "au:aumu:dls :appl"; + +fn available(id: &str) -> bool { + let ok = soundcraft_au_host::scan().iter().any(|d| d.id == id); + if !ok { + eprintln!("{id} not installed; skipping"); + } + ok +} + +fn sine(freq: f32, sr: f32, n: usize, phase0: usize) -> Vec { + (0..n).map(|i| (std::f32::consts::TAU * freq * (i + phase0) as f32 / sr).sin() * 0.5).collect() +} + +fn rms(x: &[f32]) -> f32 { + (x.iter().map(|v| v * v).sum::() / x.len().max(1) as f32).sqrt() +} + +/// Runs `freq` through the plugin in 512-frame blocks for ~0.5 s; RMS of the last half. +fn run_tone(p: &mut dyn Plugin, freq: f32) -> f32 { + let sr = 48_000.0; + let block = 512; + let blocks = 48; + let mut tail = Vec::new(); + for b in 0..blocks { + let s = sine(freq, sr, block, b * block); + let mut io = vec![s.clone(), s]; + p.process(&mut io, block); + for c in &io { + assert!(c.iter().all(|v| v.is_finite())); + } + if b >= blocks / 2 { + tail.extend_from_slice(&io[0]); + } + } + rms(&tail) +} + +#[test] +fn scan_lists_apple_units() { + let all = soundcraft_au_host::scan(); + for d in &all { + assert!(d.id.starts_with(soundcraft_au_host::ID_PREFIX), "{d:?}"); + assert_eq!(soundcraft_au_host::parse_id(&d.id).map(|(t, s, m)| soundcraft_au_host::format_id(t, s, m)), Some(d.id.clone())); + assert_eq!(d.is_instrument, d.component_type == "aumu"); + } + if let Some(d) = all.iter().find(|d| d.id == LOWPASS) { + assert_eq!(d.name, "AULowpass"); + assert_eq!(d.vendor, "Apple"); + assert_eq!(d.category, Category::Eq); + assert!(!d.is_instrument); + } + if let Some(d) = all.iter().find(|d| d.id == DLS) { + assert!(d.is_instrument); + assert_eq!(d.category, Category::Instrument); + } + assert_eq!(soundcraft_au_host::rescan().len(), all.len()); +} + +#[test] +fn lowpass_attenuates_highs() { + if !available(LOWPASS) { + return; + } + let info = soundcraft_au_host::plugin_info(LOWPASS).unwrap(); + assert_eq!(info.id, LOWPASS); + assert!(!info.params.is_empty(), "AULowpass has a cutoff"); + let mut p = soundcraft_au_host::create(LOWPASS).unwrap(); + p.prepare(48_000.0, 512, 2); + let cutoff = info.params.iter().find(|q| q.unit == soundcraft_dsp::Unit::Hz).unwrap(); + assert!(p.set_param(cutoff.id, 500.0)); + assert!((p.param(cutoff.id).unwrap() - 500.0).abs() < 1e-3); + assert!(!p.set_param("999999", 1.0)); + assert!(p.set_param(cutoff.id, f32::NAN), "NaN becomes the default, never reaches the unit"); + assert!(p.set_param(cutoff.id, 500.0)); + let low = run_tone(p.as_mut(), 100.0); + p.reset(); + let high = run_tone(p.as_mut(), 12_000.0); + eprintln!("lowpass rms: 100 Hz {low}, 12 kHz {high}"); + assert!(low > 0.2, "low tone passes: {low}"); + assert!(high < low * 0.1, "high tone attenuated: {high} vs {low}"); + + // State round trip restores the cutoff. + let state = p.save_state().expect("ClassInfo"); + assert!(p.set_param(cutoff.id, 5000.0)); + assert!(p.load_state(&state)); + assert!((p.param(cutoff.id).unwrap() - 500.0).abs() < 1.0, "{:?}", p.param(cutoff.id)); + // Hostile state is rejected without harm. + assert!(!p.load_state(b"")); + assert!(!p.load_state(b"bplist00 garbage \x00\x01\x02\xff")); + assert!(!p.load_state(&[0xffu8; 1000])); + assert!(run_tone(p.as_mut(), 100.0).is_finite()); +} + +#[test] +fn delay_runs_mono_stereo_and_odd_blocks() { + if !available(DELAY) { + return; + } + let mut p = soundcraft_au_host::instantiate(DELAY).unwrap(); + for (ch, block) in [(2usize, 1024usize), (1, 300), (2, 5000)] { + p.prepare(44_100.0, block, ch); + let mut io: Vec> = (0..ch).map(|_| sine(440.0, 44_100.0, block, 0)).collect(); + // Larger than prepared and empty blocks are clamped, not crashes. + p.process(&mut io, block * 4); + p.process(&mut io, 0); + p.process(&mut [], block); + for _ in 0..20 { + p.process(&mut io, block); + } + assert!(io.iter().flatten().all(|v| v.is_finite())); + assert!(rms(&io[0]) > 0.0); + } + // Hostile prepare values. + p.prepare(f32::NAN, 0, 0); + let mut io = vec![vec![0.25f32; 64]]; + p.process(&mut io, 64); + assert!(io[0].iter().all(|v| v.is_finite())); +} + +#[test] +fn dls_synth_plays_a_note() { + if !available(DLS) { + return; + } + let mut p = match soundcraft_au_host::instantiate(DLS) { + Ok(p) => p, + Err(e) => { + eprintln!("DLSMusicDevice unavailable: {e}"); + return; + } + }; + assert!(p.info().is_instrument); + p.prepare(48_000.0, 512, 2); + let mut silent = vec![vec![1.0f32; 512]; 2]; + p.process(&mut silent, 512); + assert!(silent.iter().flatten().all(|v| v.is_finite() && v.abs() < 1e-3), "instruments replace the buffer"); + p.note_on(10, 60, 110); + p.note_on(0, 200, 110); // ignored + let mut energy = 0.0f32; + for _ in 0..40 { + let mut io = vec![vec![0.0f32; 512]; 2]; + p.process(&mut io, 512); + assert!(io.iter().flatten().all(|v| v.is_finite())); + energy = energy.max(rms(&io[0])); + } + eprintln!("DLS rms {energy}"); + assert!(energy > 1e-3, "note-on produced sound: {energy}"); + p.all_notes_off(); + // Mono output downmixes. + p.prepare(48_000.0, 256, 1); + p.note_on(0, 64, 100); + let mut mono = vec![vec![0.0f32; 256]]; + let mut e = 0.0f32; + for _ in 0..40 { + p.process(&mut mono, 256); + e = e.max(rms(&mono[0])); + } + assert!(e > 1e-3, "{e}"); + p.note_off(0, 64); + p.reset(); +} diff --git a/crates/clap-host/Cargo.toml b/crates/clap-host/Cargo.toml index b0e05544..e0ff7424 100644 --- a/crates/clap-host/Cargo.toml +++ b/crates/clap-host/Cargo.toml @@ -13,6 +13,10 @@ serde.workspace = true thiserror.workspace = true log.workspace = true +[[test]] +name = "editor_main_thread" +harness = false + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] clap-sys = "0.5" libloading = "0.8" diff --git a/crates/clap-host/src/ffi.rs b/crates/clap-host/src/ffi.rs index 42150385..6cbac6f6 100644 --- a/crates/clap-host/src/ffi.rs +++ b/crates/clap-host/src/ffi.rs @@ -18,17 +18,20 @@ use clap_sys::events::{ clap_event_note, clap_event_param_value, clap_input_events, clap_output_events, }; use clap_sys::ext::audio_ports::{CLAP_AUDIO_PORT_IS_MAIN, CLAP_EXT_AUDIO_PORTS, clap_audio_port_info, clap_plugin_audio_ports}; +use clap_sys::ext::gui::{CLAP_EXT_GUI, clap_host_gui, clap_plugin_gui}; use clap_sys::ext::latency::{CLAP_EXT_LATENCY, clap_host_latency, clap_plugin_latency}; use clap_sys::ext::log::{CLAP_EXT_LOG, CLAP_LOG_ERROR, CLAP_LOG_FATAL, CLAP_LOG_WARNING, clap_host_log, clap_log_severity}; use clap_sys::ext::note_ports::{CLAP_EXT_NOTE_PORTS, clap_note_port_info, clap_plugin_note_ports}; use clap_sys::ext::params::{ CLAP_EXT_PARAMS, clap_host_params, clap_param_clear_flags, clap_param_info, clap_param_rescan_flags, clap_plugin_params, }; +use clap_sys::ext::state::{CLAP_EXT_STATE, clap_host_state, clap_plugin_state}; use clap_sys::factory::plugin_factory::{CLAP_PLUGIN_FACTORY_ID, clap_plugin_factory}; use clap_sys::host::clap_host; use clap_sys::id::clap_id; use clap_sys::plugin::clap_plugin; use clap_sys::process::{CLAP_PROCESS_ERROR, clap_process}; +use clap_sys::stream::{clap_istream, clap_ostream}; use clap_sys::version::{CLAP_VERSION, clap_version_is_compatible}; use std::collections::HashMap; use std::ffi::{CStr, CString, c_char, c_void}; @@ -48,6 +51,10 @@ pub(crate) const MAX_PORT_CHANNELS: u32 = 64; const MAX_FEATURES: usize = 64; /// Capacity of the per-block event list (preallocated: `process` never allocates). pub(crate) const EVENT_CAP: usize = 1024; +/// Largest plugin state we accept from `clap_plugin_state.save` (hostile-input cap). +pub(crate) const MAX_STATE_BYTES: usize = 64 << 20; +/// Most editor parameter edits buffered between two UI frames. +const EDIT_CAP: usize = 256; // ---- plugin strings ------------------------------------------------------------------------ @@ -212,10 +219,10 @@ impl Bundle { let err = |m: &str| ClapError::Instantiate(plugin_id.to_string(), m.to_string()); let create = self.factory().create_plugin.ok_or_else(|| err("factory cannot create plugins"))?; let cid = CString::new(plugin_id).map_err(|_| err("id contains NUL"))?; - let state = Box::new(HostState::default()); + let state = Arc::new(HostState::default()); let host = Box::new(clap_host { clap_version: CLAP_VERSION, - host_data: (&*state as *const HostState).cast_mut().cast::(), + host_data: Arc::as_ptr(&state).cast_mut().cast::(), name: c"SoundCraft".as_ptr(), vendor: c"SoundCraft contributors".as_ptr(), url: c"https://github.com/storytold/soundcraft".as_ptr(), @@ -237,7 +244,10 @@ impl Bundle { ports: std::ptr::null(), latency: std::ptr::null(), notes: std::ptr::null(), + state_ext: std::ptr::null(), + gui: Arc::new(GuiLink::default()), raw: Vec::with_capacity(EVENT_CAP), + out_params: Vec::with_capacity(EVENT_CAP), active: false, processing: false, max_frames: 0, @@ -261,6 +271,12 @@ impl Bundle { inst.ports = inst.extension::(CLAP_EXT_AUDIO_PORTS); inst.latency = inst.extension::(CLAP_EXT_LATENCY); inst.notes = inst.extension::(CLAP_EXT_NOTE_PORTS); + inst.state_ext = inst.extension::(CLAP_EXT_STATE); + let gui = inst.extension::(CLAP_EXT_GUI); + inst.gui = Arc::new(GuiLink { + inner: Mutex::new(GuiInner { plugin, gui, alive: !gui.is_null(), created: false, visible: false }), + host: Arc::clone(&inst.state), + }); Ok(inst) } } @@ -280,13 +296,40 @@ impl Drop for Bundle { // ---- host side ----------------------------------------------------------------------------- /// Flags the plugin raises through the host callbacks (any thread). -#[derive(Default)] pub(crate) struct HostState { pub restart: AtomicBool, pub process: AtomicBool, pub callback: AtomicBool, pub latency_changed: AtomicBool, pub params_rescan: AtomicBool, + /// `clap_host_state.mark_dirty`. + pub dirty: AtomicBool, + /// `clap_host_gui` requests, serviced by the editor's `idle` on the main thread. + pub gui_closed: AtomicBool, + pub gui_destroyed: AtomicBool, + pub gui_show: AtomicBool, + pub gui_hide: AtomicBool, + /// Parameter changes the plugin reported (output events: typically edits in its GUI), + /// waiting for the UI. Bounded by `EDIT_CAP`; the audio thread only `try_lock`s it. + pub edits: Mutex>, +} + +impl Default for HostState { + fn default() -> Self { + HostState { + restart: AtomicBool::new(false), + process: AtomicBool::new(false), + callback: AtomicBool::new(false), + latency_changed: AtomicBool::new(false), + params_rescan: AtomicBool::new(false), + dirty: AtomicBool::new(false), + gui_closed: AtomicBool::new(false), + gui_destroyed: AtomicBool::new(false), + gui_show: AtomicBool::new(false), + gui_hide: AtomicBool::new(false), + edits: Mutex::new(Vec::with_capacity(EDIT_CAP)), + } + } } /// SAFETY contract: `host` is null or one of our `clap_host` structs, whose `host_data` points @@ -355,6 +398,60 @@ unsafe extern "C" fn host_latency_changed(host: *const clap_host) { } } +unsafe extern "C" fn host_state_mark_dirty(host: *const clap_host) { + // SAFETY: the plugin passes back the host pointer we gave it. + if let Some(s) = unsafe { state(host) } { + s.dirty.store(true, Ordering::Relaxed); + } +} + +unsafe extern "C" fn host_gui_resize_hints_changed(_host: *const clap_host) {} + +unsafe extern "C" fn host_gui_request_resize(_host: *const clap_host, _width: u32, _height: u32) -> bool { + // Floating windows belong to the plugin, which resizes them itself. + true +} + +unsafe extern "C" fn host_gui_request_show(host: *const clap_host) -> bool { + // SAFETY: the plugin passes back the host pointer we gave it. + match unsafe { state(host) } { + Some(s) => { + s.gui_show.store(true, Ordering::Relaxed); + true + } + None => false, + } +} + +unsafe extern "C" fn host_gui_request_hide(host: *const clap_host) -> bool { + // SAFETY: the plugin passes back the host pointer we gave it. + match unsafe { state(host) } { + Some(s) => { + s.gui_hide.store(true, Ordering::Relaxed); + true + } + None => false, + } +} + +unsafe extern "C" fn host_gui_closed(host: *const clap_host, was_destroyed: bool) { + // SAFETY: the plugin passes back the host pointer we gave it. + if let Some(s) = unsafe { state(host) } { + if was_destroyed { + s.gui_destroyed.store(true, Ordering::Relaxed); + } + s.gui_closed.store(true, Ordering::Relaxed); + } +} + +static HOST_STATE: clap_host_state = clap_host_state { mark_dirty: Some(host_state_mark_dirty) }; +static HOST_GUI: clap_host_gui = clap_host_gui { + resize_hints_changed: Some(host_gui_resize_hints_changed), + request_resize: Some(host_gui_request_resize), + request_show: Some(host_gui_request_show), + request_hide: Some(host_gui_request_hide), + closed: Some(host_gui_closed), +}; static HOST_LOG: clap_host_log = clap_host_log { log: Some(host_log) }; static HOST_PARAMS: clap_host_params = clap_host_params { rescan: Some(host_params_rescan), clear: Some(host_params_clear), request_flush: Some(host_params_request_flush) }; @@ -372,8 +469,12 @@ unsafe extern "C" fn host_get_extension(_host: *const clap_host, id: *const c_ch (&HOST_PARAMS as *const clap_host_params).cast() } else if id == CLAP_EXT_LATENCY { (&HOST_LATENCY as *const clap_host_latency).cast() + } else if id == CLAP_EXT_STATE { + (&HOST_STATE as *const clap_host_state).cast() + } else if id == CLAP_EXT_GUI { + (&HOST_GUI as *const clap_host_gui).cast() } else { - // Everything else (gui, state, thread-check, timers, …) is unsupported: CLAP allows null. + // Everything else (thread-check, timers, fd support, …) is unsupported: CLAP allows null. std::ptr::null() } } @@ -495,12 +596,287 @@ unsafe extern "C" fn in_events_get(list: *const clap_input_events, index: u32) - v.and_then(|v| v.get(index as usize)).map_or(std::ptr::null(), RawEvent::header) } -unsafe extern "C" fn out_events_try_push(_list: *const clap_output_events, _event: *const clap_event_header) -> bool { - // Plugin → host events (e.g. GUI parameter edits) are accepted and ignored: SoundCraft's - // session is the source of truth for parameter values. +unsafe extern "C" fn out_events_try_push(list: *const clap_output_events, event: *const clap_event_header) -> bool { + // Plugin → host events are accepted. Parameter values (typically edits in the plugin's GUI) + // are kept, while there is room, so the UI can write them into the session; the rest is + // ignored. + if list.is_null() || event.is_null() { + return true; + } + // SAFETY: CLAP passes a valid event header for the duration of the call. + let h = unsafe { &*event }; + if h.space_id != CLAP_CORE_EVENT_SPACE_ID || h.type_ != CLAP_EVENT_PARAM_VALUE || (h.size as usize) < size_of::() { + return true; + } + // SAFETY: the header says this is a complete `clap_event_param_value` (type and size checked). + let e = unsafe { &*event.cast::() }; + // SAFETY: `ctx` is null or the instance's preallocated `out_params` Vec, which `process` + // borrows exclusively for the duration of the plugin call. + if let Some(v) = unsafe { ((*list).ctx as *mut Vec<(u32, f64)>).as_mut() } + && v.len() < v.capacity() + && e.value.is_finite() + { + v.push((e.param_id, e.value)); + } true } +// ---- state streams ------------------------------------------------------------------------- + +/// Read side of an in-memory state blob. +struct ReadCursor<'a> { + data: &'a [u8], + pos: usize, +} + +unsafe extern "C" fn istream_read(stream: *const clap_istream, buffer: *mut c_void, size: u64) -> i64 { + if stream.is_null() || buffer.is_null() { + return -1; + } + // SAFETY: `ctx` is the `ReadCursor` on `load_state`'s stack frame, alive during the call. + let Some(c) = (unsafe { ((*stream).ctx as *mut ReadCursor<'_>).as_mut() }) else { return -1 }; + let rest = c.data.get(c.pos..).unwrap_or(&[]); + let n = rest.len().min(usize::try_from(size).unwrap_or(usize::MAX)); + if let Some(src) = rest.get(..n) { + // SAFETY: the plugin provides `size` ≥ `n` writable bytes at `buffer`; `src` is `n` + // readable bytes of our own slice; they cannot overlap. + unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), buffer.cast::(), n) }; + } + c.pos += n; + i64::try_from(n).unwrap_or(-1) +} + +unsafe extern "C" fn ostream_write(stream: *const clap_ostream, buffer: *const c_void, size: u64) -> i64 { + if stream.is_null() || buffer.is_null() { + return -1; + } + // SAFETY: `ctx` is the output Vec on `save_state`'s stack frame, alive during the call. + let Some(v) = (unsafe { ((*stream).ctx as *mut Vec).as_mut() }) else { return -1 }; + let Ok(n) = usize::try_from(size) else { return -1 }; + if v.len().saturating_add(n) > MAX_STATE_BYTES { + return -1; + } + // SAFETY: the plugin provides `size` readable bytes at `buffer` for the call. + let src = unsafe { std::slice::from_raw_parts(buffer.cast::(), n) }; + v.extend_from_slice(src); + i64::try_from(n).unwrap_or(-1) +} + +// ---- editor (clap.gui) --------------------------------------------------------------------- + +/// True on the process's main thread (AppKit requires GUI calls there). +#[cfg(target_os = "macos")] +pub(crate) fn on_main_thread() -> bool { + unsafe extern "C" { + fn pthread_main_np() -> std::ffi::c_int; + } + // SAFETY: a libSystem function without arguments or preconditions. + unsafe { pthread_main_np() == 1 } +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn on_main_thread() -> bool { + true +} + +/// The windowing API of this platform. +fn window_api() -> Option<&'static CStr> { + if cfg!(target_os = "macos") { + Some(clap_sys::ext::gui::CLAP_WINDOW_API_COCOA) + } else if cfg!(windows) { + Some(clap_sys::ext::gui::CLAP_WINDOW_API_WIN32) + } else if cfg!(all(unix, not(target_os = "android"))) { + Some(clap_sys::ext::gui::CLAP_WINDOW_API_X11) + } else { + None + } +} + +struct GuiInner { + plugin: *const clap_plugin, + gui: *const clap_plugin_gui, + /// The plugin instance exists (cleared by `Instance::drop` before it destroys the plugin). + alive: bool, + created: bool, + visible: bool, +} + +/// The link between an instance and its editor handles. The instance may process on the audio +/// thread while the editor is driven from the main thread: CLAP allows `clap.gui` (main-thread) +/// calls concurrently with audio-thread calls. Every GUI call happens under this lock, and the +/// instance clears `alive` (destroying any GUI) under the same lock before it destroys the +/// plugin, so a handle can never call into a destroyed plugin. +pub(crate) struct GuiLink { + inner: Mutex, + host: Arc, +} + +impl Default for GuiLink { + fn default() -> Self { + GuiLink { + inner: Mutex::new(GuiInner { plugin: std::ptr::null(), gui: std::ptr::null(), alive: false, created: false, visible: false }), + host: Arc::new(HostState::default()), + } + } +} + +// SAFETY: the raw pointers are only dereferenced under the mutex, while `alive` guarantees the +// plugin exists; CLAP's threading rules (main-thread GUI calls) are checked at each call. +unsafe impl Send for GuiInner {} + +impl GuiLink { + fn lock(&self) -> std::sync::MutexGuard<'_, GuiInner> { + self.inner.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub fn available(&self) -> bool { + self.lock().alive + } + + pub fn host(&self) -> &HostState { + &self.host + } + + fn vt(g: &GuiInner) -> Option { + // SAFETY: `gui` is null or the plugin's static extension vtable, valid while `alive`. + (g.alive).then(|| unsafe { g.gui.as_ref() }.copied()).flatten() + } + + /// Opens (creates and shows) the plugin's floating editor window. + pub fn open(&self, title: &str) -> Result<(), String> { + if !on_main_thread() { + return Err("plugin editors must be opened on the main thread".into()); + } + let mut g = self.lock(); + let vt = GuiLink::vt(&g).ok_or_else(|| "this plugin has no editor".to_string())?; + if g.created { + if let Some(show) = vt.show { + // SAFETY: live plugin with a created GUI; main thread (checked). + unsafe { show(g.plugin) }; + } + g.visible = true; + return Ok(()); + } + let api = window_api().ok_or_else(|| "plugin editors are not supported on this platform".to_string())?; + let (Some(supported), Some(create)) = (vt.is_api_supported, vt.create) else { return Err("incomplete clap.gui extension".into()) }; + // SAFETY: live plugin; `api` is a static NUL-terminated string; main thread. + if !unsafe { supported(g.plugin, api.as_ptr(), true) } { + return Err("this plugin offers only an embedded editor; SoundCraft hosts floating CLAP editors".into()); + } + self.host.gui_closed.store(false, Ordering::Relaxed); + self.host.gui_destroyed.store(false, Ordering::Relaxed); + // SAFETY: as above; no GUI exists yet. + if !unsafe { create(g.plugin, api.as_ptr(), true) } { + return Err("the plugin could not create its editor".into()); + } + g.created = true; + if !cfg!(target_os = "macos") + && let Some(f) = vt.set_scale + { + // SAFETY: live plugin with a created GUI. (Cocoa uses logical pixels: not called.) + unsafe { f(g.plugin, 1.0) }; + } + if let (Some(f), Ok(t)) = (vt.suggest_title, CString::new(title)) { + // SAFETY: live plugin with a created floating GUI; `t` outlives the call. + unsafe { f(g.plugin, t.as_ptr()) }; + } + let shown = match vt.show { + // SAFETY: as above. + Some(f) => unsafe { f(g.plugin) }, + None => false, + }; + if !shown { + GuiLink::destroy(&mut g); + return Err("the plugin could not show its editor".into()); + } + g.visible = true; + Ok(()) + } + + fn destroy(g: &mut GuiInner) { + if g.created + && let Some(f) = GuiLink::vt(g).and_then(|v| v.destroy) + { + // SAFETY: live plugin with a created GUI; destroyed exactly once. + unsafe { f(g.plugin) }; + } + g.created = false; + g.visible = false; + } + + /// Closes (destroys) the editor. + pub fn close(&self) { + if !on_main_thread() { + return; + } + GuiLink::destroy(&mut self.lock()); + } + + pub fn is_open(&self) -> bool { + let g = self.lock(); + g.alive && g.created && g.visible + } + + /// Main-thread housekeeping: runs a requested `on_main_thread`, follows the plugin's + /// show/hide/closed requests, and returns the parameter edits reported since the last call. + pub fn idle(&self) -> Vec<(u32, f64)> { + if !on_main_thread() { + return Vec::new(); + } + let mut g = self.lock(); + if !g.alive { + return Vec::new(); + } + if self.host.callback.swap(false, Ordering::Relaxed) { + // SAFETY: `plugin` is valid while `alive`; we copy its vtable struct. + if let Some(f) = unsafe { (*g.plugin).on_main_thread } { + // SAFETY: live plugin, main thread, requested via `request_callback`. + unsafe { f(g.plugin) }; + } + } + if g.created { + if self.host.gui_closed.swap(false, Ordering::Relaxed) { + // The user closed the floating window (or the plugin lost its GUI): free it. + self.host.gui_destroyed.store(false, Ordering::Relaxed); + GuiLink::destroy(&mut g); + } else if let Some(vt) = GuiLink::vt(&g) { + if self.host.gui_show.swap(false, Ordering::Relaxed) + && let Some(f) = vt.show + { + // SAFETY: live plugin with a created GUI, main thread. + g.visible = unsafe { f(g.plugin) }; + } + if self.host.gui_hide.swap(false, Ordering::Relaxed) + && let Some(f) = vt.hide + { + // SAFETY: as above. + unsafe { f(g.plugin) }; + g.visible = false; + } + } + } + drop(g); + let mut e = self.host.edits.lock().unwrap_or_else(PoisonError::into_inner); + let out = e.clone(); + e.clear(); + out + } + + /// Called by the instance before it destroys the plugin. + fn kill(&self) { + let mut g = self.lock(); + if g.created { + if on_main_thread() { + GuiLink::destroy(&mut g); + } else { + log::warn!("clap: editor still open while its plugin is destroyed off the main thread"); + g.created = false; + } + } + g.alive = false; + } +} + // ---- audio buffers ------------------------------------------------------------------------- /// Planar f32 buffers for every audio port in one direction, with the pointer tables CLAP needs. @@ -590,7 +966,11 @@ pub(crate) struct Instance { ports: *const clap_plugin_audio_ports, latency: *const clap_plugin_latency, notes: *const clap_plugin_note_ports, + state_ext: *const clap_plugin_state, + gui: Arc, raw: Vec, + /// Parameter values the plugin reported through output events in the current `process`. + out_params: Vec<(u32, f64)>, active: bool, processing: bool, max_frames: usize, @@ -598,7 +978,7 @@ pub(crate) struct Instance { // Field order matters for drop: the plugin is destroyed in `Drop::drop` first; then the host // struct and its state, then the bundle (which keeps the code mapped) go. _host: Box, - state: Box, + state: Arc, _bundle: Arc, } @@ -626,6 +1006,31 @@ impl Instance { &self.state } + /// The editor link shared with editor handles (`None` when the plugin has no `clap.gui`). + pub fn gui(&self) -> Option> { + self.gui.available().then(|| Arc::clone(&self.gui)) + } + + /// `clap_plugin_state.save` into memory (`None` without the extension or on failure). + pub fn save_state(&self) -> Option> { + // SAFETY: the extension pointer, when non-null, points to a static vtable in the plugin. + let save = unsafe { self.state_ext.as_ref() }?.save?; + let mut out: Vec = Vec::new(); + let stream = clap_ostream { ctx: (&mut out as *mut Vec).cast::(), write: Some(ostream_write) }; + // SAFETY: valid instance; the stream and its context live on this frame for the call. + unsafe { save(self.plugin, &stream) }.then_some(out) + } + + /// `clap_plugin_state.load` from memory. `false` without the extension or when refused. + pub fn load_state(&self, data: &[u8]) -> bool { + // SAFETY: see `save_state`. + let Some(load) = (unsafe { self.state_ext.as_ref() }).and_then(|s| s.load) else { return false }; + let mut cursor = ReadCursor { data, pos: 0 }; + let stream = clap_istream { ctx: (&mut cursor as *mut ReadCursor<'_>).cast::(), read: Some(istream_read) }; + // SAFETY: valid instance; the stream and its cursor live on this frame for the call. + unsafe { load(self.plugin, &stream) } + } + /// Runs `on_main_thread` if the plugin asked for a callback. pub fn service_callback(&mut self) { if self.state.callback.swap(false, Ordering::Relaxed) @@ -810,7 +1215,9 @@ impl Instance { size: Some(in_events_size), get: Some(in_events_get), }; - let out_events = clap_output_events { ctx: std::ptr::null_mut(), try_push: Some(out_events_try_push) }; + self.out_params.clear(); + let out_events = + clap_output_events { ctx: (&mut self.out_params as *mut Vec<(u32, f64)>).cast::(), try_push: Some(out_events_try_push) }; let p = clap_process { steady_time, frames_count: u32::try_from(frames).unwrap_or(0), @@ -827,12 +1234,25 @@ impl Instance { // lists and their contexts live on this stack frame / in `self.raw`, untouched until the // call returns. let status = unsafe { process(self.plugin, &p) }; + if !self.out_params.is_empty() + && let Ok(mut edits) = self.state.edits.try_lock() + { + // Never blocks or grows: a busy UI or a full queue drops the edits. + for &e in &self.out_params { + if edits.len() < edits.capacity() { + edits.push(e); + } + } + } + self.out_params.clear(); if status == CLAP_PROCESS_ERROR { Err(ClapError::Process("plugin reported a processing error".into())) } else { Ok(status) } } } impl Drop for Instance { fn drop(&mut self) { + // Editors first (CLAP: destroy the GUI before the plugin), and no handle may call in after. + self.gui.kill(); self.deactivate(); if self.created && let Some(f) = self.vt().destroy diff --git a/crates/clap-host/src/lib.rs b/crates/clap-host/src/lib.rs index 13008268..2740f49a 100644 --- a/crates/clap-host/src/lib.rs +++ b/crates/clap-host/src/lib.rs @@ -17,7 +17,13 @@ //! //! On `wasm32` the crate compiles to stubs: scans are empty and nothing can be created. //! -//! Plugin GUIs (`clap.gui`) are not hosted yet; SoundCraft shows its generic parameter editor. +//! Plugin state (`clap.state`) round-trips through [`Plugin::save_state`] / +//! [`Plugin::load_state`] as an opaque blob. Plugin editors (`clap.gui`) open as the plugin's own +//! floating window ([`Plugin::open_editor`], or [`Plugin::editor`] for a handle usable on the main +//! thread while the instance processes on the audio thread); plugins that offer only embedded +//! editors are not hosted yet, and SoundCraft's generic parameter editor still works for them. +//! Parameter changes made in the plugin's GUI arrive as output events while the plugin processes +//! and are reported by the editor's `idle`. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] diff --git a/crates/clap-host/src/plugin.rs b/crates/clap-host/src/plugin.rs index 6dbd45d8..6fe1abcf 100644 --- a/crates/clap-host/src/plugin.rs +++ b/crates/clap-host/src/plugin.rs @@ -1,8 +1,9 @@ //! `ClapPlugin`: a loaded CLAP instance behind the safe `soundcraft_dsp::Plugin` trait. -use crate::ffi::{EVENT_CAP, Event, Instance, NotePort, PortBuffers}; +use crate::ffi::{EVENT_CAP, Event, GuiLink, Instance, NotePort, PortBuffers}; use clap_sys::ext::note_ports::{CLAP_NOTE_DIALECT_CLAP, CLAP_NOTE_DIALECT_MIDI}; -use soundcraft_dsp::{Plugin, PluginInfo}; +use soundcraft_dsp::{Plugin, PluginEditor, PluginInfo}; +use std::sync::Arc; use std::sync::atomic::Ordering; /// Largest block we activate a plugin for. @@ -62,6 +63,17 @@ impl ClapPlugin { } } + /// Re-reads every parameter value from the plugin (after a state load). + fn refresh_values(&mut self) { + for ((v, p), id) in self.values.iter_mut().zip(self.info.params).zip(&self.ids) { + if let Some(x) = self.inst.param_value(*id) { + *v = p.clamp(x as f32); + } + } + // Pending changes from before the load would undo it. + self.queue.retain(|e| !matches!(e, Event::Param { .. })); + } + /// Inserts keeping the queue time-sorted; drops the event when the queue is full (never /// allocates on the audio thread). fn push(&mut self, e: Event) { @@ -243,4 +255,72 @@ impl Plugin for ClapPlugin { } } } + + fn save_state(&mut self) -> Option> { + self.inst.save_state() + } + + fn load_state(&mut self, data: &[u8]) -> bool { + let ok = self.inst.load_state(data); + if ok { + self.refresh_values(); + } + ok + } + + fn open_editor(&mut self) -> Result<(), String> { + let link = self.inst.gui().ok_or_else(|| "this plugin has no editor".to_string())?; + link.open(self.info.name) + } + + fn close_editor(&mut self) { + if let Some(link) = self.inst.gui() { + link.close(); + } + } + + fn editor(&mut self) -> Option> { + let link = self.inst.gui()?; + Some(Box::new(ClapEditor { link, info: self.info, ids: self.ids.clone() })) + } +} + +/// A CLAP plugin's floating editor, driven from the main thread (see [`GuiLink`]). +struct ClapEditor { + link: Arc, + info: &'static PluginInfo, + ids: Vec, +} + +impl PluginEditor for ClapEditor { + fn open(&mut self) -> Result<(), String> { + // Edits reported before the editor opened are stale. + let _ = self.link.idle(); + self.link.open(self.info.name) + } + + fn close(&mut self) { + self.link.close(); + } + + fn is_open(&self) -> bool { + self.link.is_open() + } + + fn idle(&mut self) -> Vec<(String, f32)> { + let raw = self.link.idle(); + let _ = self.link.host().dirty.swap(false, Ordering::Relaxed); + let mut out: Vec<(String, f32)> = Vec::new(); + for (cid, v) in raw { + let Some(i) = self.ids.iter().position(|&x| x == cid) else { continue }; + let Some(p) = self.info.params.get(i) else { continue }; + let v = p.clamp(v as f32); + // Keep only the latest value per parameter. + match out.iter_mut().find(|(id, _)| id == p.id) { + Some(e) => e.1 = v, + None => out.push((p.id.to_string(), v)), + } + } + out + } } diff --git a/crates/clap-host/tests/common/mod.rs b/crates/clap-host/tests/common/mod.rs new file mode 100644 index 00000000..2da23955 --- /dev/null +++ b/crates/clap-host/tests/common/mod.rs @@ -0,0 +1,46 @@ +//! Shared by the integration tests: builds the fixture plugin (a cdylib outside the workspace) +//! and installs it in a temp folder. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; + +/// Builds the fixture once and returns a folder containing `Fixture.clap`. +pub fn fixture_dir() -> &'static Path { + static DIR: OnceLock = OnceLock::new(); + DIR.get_or_init(|| { + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture/Cargo.toml"); + let target = Path::new(env!("CARGO_TARGET_TMPDIR")).join("clap-fixture-target"); + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); + let st = Command::new(cargo) + .args(["build", "--quiet", "--manifest-path"]) + .arg(&manifest) + .arg("--target-dir") + .arg(&target) + .env_remove("RUSTFLAGS") + .status() + .expect("run cargo"); + assert!(st.success(), "building the CLAP fixture failed"); + let lib = if cfg!(target_os = "macos") { + "libclap_fixture.dylib" + } else if cfg!(windows) { + "clap_fixture.dll" + } else { + "libclap_fixture.so" + }; + let built = target.join("debug").join(lib); + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("clap-fixture-install-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + if cfg!(target_os = "macos") { + // Install as a real macOS bundle to exercise bundle resolution. + let exe = dir.join("vendor/Fixture.clap/Contents/MacOS"); + std::fs::create_dir_all(&exe).unwrap(); + std::fs::copy(&built, exe.join("Fixture")).unwrap(); + } else { + std::fs::create_dir_all(dir.join("vendor")).unwrap(); + std::fs::copy(&built, dir.join("vendor/Fixture.clap")).unwrap(); + } + assert_eq!(soundcraft_clap_host::add_search_dir(&dir), 2); + dir + }) +} diff --git a/crates/clap-host/tests/editor_main_thread.rs b/crates/clap-host/tests/editor_main_thread.rs new file mode 100644 index 00000000..016d74ff --- /dev/null +++ b/crates/clap-host/tests/editor_main_thread.rs @@ -0,0 +1,48 @@ +//! Plugin editors must be driven from the process main thread, which the standard test harness +//! never uses, so this test binary has its own `main` (`harness = false`). The fixture's GUI is +//! window-less: nothing appears on screen. + +mod common; + +use soundcraft_dsp::Plugin; + +const GAIN: &str = "clap:org.soundcraft.test.gain"; + +fn block(p: &mut Box) -> Vec> { + let mut io = vec![vec![1.0f32; 64]; 2]; + p.process(&mut io, 64); + io +} + +fn main() { + common::fixture_dir(); + let mut p = soundcraft_clap_host::create(GAIN).expect("fixture"); + p.prepare(48_000.0, 64, 2); + let mut ed = p.editor().expect("editor handle"); + assert!(!ed.is_open()); + ed.open().expect("open on the main thread"); + assert!(ed.is_open()); + ed.open().expect("opening again just raises it"); + // The fixture "edits" the gain in its GUI; the edit arrives as an output event of the next + // block and is reported (in SoundCraft units) by idle. + let io = block(&mut p); + assert!((io[0][10] - 0.25).abs() < 1e-6, "{}", io[0][10]); + assert_eq!(ed.idle(), vec![("7".to_string(), 0.25)]); + assert!(ed.idle().is_empty(), "drained"); + ed.close(); + assert!(!ed.is_open()); + // Through the Plugin trait as well. + p.open_editor().expect("open via the trait"); + assert!(ed.is_open(), "the handle sees the same editor"); + p.close_editor(); + assert!(!ed.is_open()); + // Dropping the instance with its editor open destroys the GUI first (the fixture asserts + // that); the handle then refuses politely. + ed.open().expect("reopen"); + drop(p); + assert!(!ed.is_open()); + assert!(ed.open().is_err()); + assert!(ed.idle().is_empty()); + ed.close(); + println!("editor_main_thread: ok"); +} diff --git a/crates/clap-host/tests/fixture/src/lib.rs b/crates/clap-host/tests/fixture/src/lib.rs index 43811233..40de13ce 100644 --- a/crates/clap-host/tests/fixture/src/lib.rs +++ b/crates/clap-host/tests/fixture/src/lib.rs @@ -4,12 +4,19 @@ //! (stepped enum A/B/C), hidden param 9, latency 3 samples. //! - `org.soundcraft.test.synth`: instrument, stereo out, CLAP note port; outputs a constant equal //! to the velocity of the held note (0 when none), switching at the event's sample offset. +//! +//! The gain plugin also implements `clap.state` (gain and mode as two little-endian f64s) and a +//! floating-only `clap.gui` without a real window: showing it "edits" the gain to 0.25 in the GUI, +//! reported as an output parameter event by the next `process`. #![allow(clippy::missing_safety_doc, non_upper_case_globals)] use clap_sys::audio_buffer::clap_audio_buffer; use clap_sys::entry::clap_plugin_entry; use clap_sys::events::*; use clap_sys::ext::audio_ports::*; +use clap_sys::ext::gui::*; +use clap_sys::ext::state::*; +use clap_sys::stream::*; use clap_sys::ext::latency::*; use clap_sys::ext::log::*; use clap_sys::ext::note_ports::*; @@ -63,6 +70,9 @@ struct Fx { level: f64, active: bool, processing: bool, + gui_created: bool, + gui_visible: bool, + gui_edit_pending: bool, } unsafe fn fx<'a>(p: *const clap_plugin) -> &'a mut Fx { @@ -87,6 +97,7 @@ unsafe extern "C" fn init(p: *const clap_plugin) -> bool { } unsafe extern "C" fn destroy(p: *const clap_plugin) { + assert!(!unsafe { fx(p) }.gui_created, "plugin destroyed with its GUI alive"); let data = unsafe { (*p).plugin_data } as *mut Fx; drop(unsafe { Box::from_raw(data) }); } @@ -165,6 +176,44 @@ unsafe extern "C" fn process(p: *const clap_plugin, pr: *const clap_process) -> } Some(unsafe { &*pr.audio_inputs }) }; + if f.gui_edit_pending { + f.gui_edit_pending = false; + f.gain = 0.25; + let e = clap_event_param_value { + header: clap_event_header { + size: std::mem::size_of::() as u32, + time: 0, + space_id: CLAP_CORE_EVENT_SPACE_ID, + type_: CLAP_EVENT_PARAM_VALUE, + flags: 0, + }, + param_id: 7, + cookie: std::ptr::null_mut(), + note_id: -1, + port_index: -1, + channel: -1, + key: -1, + value: 0.25, + }; + let out = unsafe { &*pr.out_events }; + assert!(unsafe { (out.try_push.unwrap())(pr.out_events, &e.header) }); + // Other event types are accepted and ignored by the host. + let note = clap_event_note { + header: clap_event_header { + size: std::mem::size_of::() as u32, + time: 0, + space_id: CLAP_CORE_EVENT_SPACE_ID, + type_: CLAP_EVENT_NOTE_END, + flags: 0, + }, + note_id: -1, + port_index: 0, + channel: 0, + key: 60, + velocity: 0.0, + }; + assert!(unsafe { (out.try_push.unwrap())(pr.out_events, ¬e.header) }); + } let mut next = 0u32; let mut last_time = 0u32; for i in 0..n { @@ -311,6 +360,112 @@ unsafe extern "C" fn notes_get(_p: *const clap_plugin, index: u32, is_input: boo static NOTES: clap_plugin_note_ports = clap_plugin_note_ports { count: Some(notes_count), get: Some(notes_get) }; +unsafe extern "C" fn state_save(p: *const clap_plugin, stream: *const clap_ostream) -> bool { + let f = unsafe { fx(p) }; + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&f.gain.to_le_bytes()); + bytes[8..].copy_from_slice(&f.mode.to_le_bytes()); + let s = unsafe { &*stream }; + // Write in two pieces to exercise appending. + let a = unsafe { (s.write.unwrap())(stream, bytes.as_ptr().cast(), 5) }; + let b = unsafe { (s.write.unwrap())(stream, bytes[5..].as_ptr().cast(), 11) }; + a == 5 && b == 11 +} + +unsafe extern "C" fn state_load(p: *const clap_plugin, stream: *const clap_istream) -> bool { + let f = unsafe { fx(p) }; + let s = unsafe { &*stream }; + let mut bytes = [0u8; 17]; + let mut got = 0usize; + loop { + let n = unsafe { (s.read.unwrap())(stream, bytes[got..].as_mut_ptr().cast(), (17 - got) as u64) }; + if n <= 0 { + break; + } + got += n as usize; + if got == 17 { + break; + } + } + if got != 16 { + return false; + } + let gain = f64::from_le_bytes(bytes[..8].try_into().unwrap()); + let mode = f64::from_le_bytes(bytes[8..16].try_into().unwrap()); + if !(0.0..=2.0).contains(&gain) || !(0.0..=2.0).contains(&mode) { + return false; + } + f.gain = gain; + f.mode = mode; + true +} + +static STATE: clap_plugin_state = clap_plugin_state { save: Some(state_save), load: Some(state_load) }; + +unsafe extern "C" fn gui_is_api_supported(_p: *const clap_plugin, api: *const c_char, floating: bool) -> bool { + assert!(!api.is_null()); + floating +} + +unsafe extern "C" fn gui_create(p: *const clap_plugin, _api: *const c_char, floating: bool) -> bool { + let f = unsafe { fx(p) }; + assert!(floating && !f.gui_created, "create"); + f.gui_created = true; + true +} + +unsafe extern "C" fn gui_destroy(p: *const clap_plugin) { + let f = unsafe { fx(p) }; + assert!(f.gui_created, "destroy without create"); + f.gui_created = false; + f.gui_visible = false; +} + +unsafe extern "C" fn gui_set_scale(_p: *const clap_plugin, scale: f64) -> bool { + scale > 0.0 +} + +unsafe extern "C" fn gui_suggest_title(p: *const clap_plugin, title: *const c_char) { + assert!(unsafe { fx(p) }.gui_created); + assert!(!unsafe { CStr::from_ptr(title) }.to_bytes().is_empty()); +} + +unsafe extern "C" fn gui_show(p: *const clap_plugin) -> bool { + let f = unsafe { fx(p) }; + assert!(f.gui_created, "show without create"); + f.gui_visible = true; + f.gui_edit_pending = true; + // Exercise the host GUI extension. + let host = unsafe { &*f.host }; + let hg = unsafe { (host.get_extension.unwrap())(f.host, CLAP_EXT_GUI.as_ptr()) } as *const clap_host_gui; + let hg = unsafe { hg.as_ref() }.expect("host gui extension"); + assert!(unsafe { (hg.request_resize.unwrap())(f.host, 300, 200) }); + true +} + +unsafe extern "C" fn gui_hide(p: *const clap_plugin) -> bool { + unsafe { fx(p) }.gui_visible = false; + true +} + +static GUI: clap_plugin_gui = clap_plugin_gui { + is_api_supported: Some(gui_is_api_supported), + get_preferred_api: None, + create: Some(gui_create), + destroy: Some(gui_destroy), + set_scale: Some(gui_set_scale), + get_size: None, + can_resize: None, + get_resize_hints: None, + adjust_size: None, + set_size: None, + set_parent: None, + set_transient: None, + suggest_title: Some(gui_suggest_title), + show: Some(gui_show), + hide: Some(gui_hide), +}; + unsafe extern "C" fn get_extension(p: *const clap_plugin, id: *const c_char) -> *const c_void { let id = unsafe { CStr::from_ptr(id) }; let synth = unsafe { fx(p) }.synth; @@ -322,6 +477,10 @@ unsafe extern "C" fn get_extension(p: *const clap_plugin, id: *const c_char) -> &LATENCY as *const _ as *const c_void } else if id == CLAP_EXT_NOTE_PORTS && synth { &NOTES as *const _ as *const c_void + } else if id == CLAP_EXT_STATE && !synth { + &STATE as *const _ as *const c_void + } else if id == CLAP_EXT_GUI && !synth { + &GUI as *const _ as *const c_void } else { std::ptr::null() } @@ -374,6 +533,9 @@ unsafe extern "C" fn factory_create(_f: *const clap_plugin_factory, host: *const level: 0.0, active: false, processing: false, + gui_created: false, + gui_visible: false, + gui_edit_pending: false, }); let raw = Box::into_raw(b); unsafe { diff --git a/crates/clap-host/tests/fixture_e2e.rs b/crates/clap-host/tests/fixture_e2e.rs index 7f295141..22164875 100644 --- a/crates/clap-host/tests/fixture_e2e.rs +++ b/crates/clap-host/tests/fixture_e2e.rs @@ -2,49 +2,9 @@ //! install it as a `.clap` in a temp folder, then scan → instantiate → process → params → notes. use soundcraft_dsp::{Category, Unit}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::OnceLock; -/// Builds the fixture once and returns a folder containing `Fixture.clap`. -fn fixture_dir() -> &'static Path { - static DIR: OnceLock = OnceLock::new(); - DIR.get_or_init(|| { - let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture/Cargo.toml"); - let target = Path::new(env!("CARGO_TARGET_TMPDIR")).join("clap-fixture-target"); - let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); - let st = Command::new(cargo) - .args(["build", "--quiet", "--manifest-path"]) - .arg(&manifest) - .arg("--target-dir") - .arg(&target) - .env_remove("RUSTFLAGS") - .status() - .expect("run cargo"); - assert!(st.success(), "building the CLAP fixture failed"); - let lib = if cfg!(target_os = "macos") { - "libclap_fixture.dylib" - } else if cfg!(windows) { - "clap_fixture.dll" - } else { - "libclap_fixture.so" - }; - let built = target.join("debug").join(lib); - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("clap-fixture-install-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - if cfg!(target_os = "macos") { - // Install as a real macOS bundle to exercise bundle resolution. - let exe = dir.join("vendor/Fixture.clap/Contents/MacOS"); - std::fs::create_dir_all(&exe).unwrap(); - std::fs::copy(&built, exe.join("Fixture")).unwrap(); - } else { - std::fs::create_dir_all(dir.join("vendor")).unwrap(); - std::fs::copy(&built, dir.join("vendor/Fixture.clap")).unwrap(); - } - assert_eq!(soundcraft_clap_host::add_search_dir(&dir), 2); - dir - }) -} +mod common; +use common::fixture_dir; const GAIN: &str = "clap:org.soundcraft.test.gain"; const SYNTH: &str = "clap:org.soundcraft.test.synth"; @@ -158,3 +118,50 @@ fn many_instances_create_and_drop_cleanly() { drop(b); } } + +#[test] +fn state_saves_and_restores_into_a_new_instance() { + fixture_dir(); + let mut p = soundcraft_clap_host::create(GAIN).unwrap(); + p.prepare(48_000.0, 64, 2); + assert!(p.set_param("7", 1.5)); + assert!(p.set_param("3", 2.0)); + let mut io = vec![vec![0.5f32; 64]; 2]; + p.process(&mut io, 64); + let blob = p.save_state().expect("the gain fixture has clap.state"); + assert_eq!(blob.len(), 16); + let mut q = soundcraft_clap_host::create(GAIN).unwrap(); + q.prepare(48_000.0, 64, 2); + assert_eq!(q.param("7"), Some(1.0)); + assert!(q.load_state(&blob)); + assert_eq!((q.param("7"), q.param("3")), (Some(1.5), Some(2.0)), "values re-read after the load"); + let mut io = vec![vec![0.5f32; 64]; 2]; + q.process(&mut io, 64); + assert!(io[0].iter().all(|&x| (x - 0.75).abs() < 1e-6), "{:?}", &io[0][..4]); + // Hostile blobs are refused and leave the plugin as it was. + assert!(!q.load_state(b"short")); + assert!(!q.load_state(&[0xff; 16]), "NaN gain rejected by the plugin"); + assert!(!q.load_state(&[0u8; 4096])); + assert_eq!(q.param("7"), Some(1.5)); + // Plugins without clap.state. + let mut s = soundcraft_clap_host::create(SYNTH).unwrap(); + assert!(s.save_state().is_none()); + assert!(!s.load_state(&blob)); + assert!(s.editor().is_none(), "the synth has no GUI"); + assert!(s.open_editor().is_err()); +} + +#[test] +fn editors_refuse_to_open_off_the_main_thread() { + fixture_dir(); + // Test threads are never the process main thread. + let mut p = soundcraft_clap_host::create(GAIN).unwrap(); + let mut ed = p.editor().expect("the gain fixture has clap.gui"); + if cfg!(target_os = "macos") { + assert!(ed.open().unwrap_err().contains("main thread")); + assert!(p.open_editor().is_err()); + assert!(!ed.is_open()); + } + ed.close(); + p.close_editor(); +} diff --git a/crates/dsp/src/lib.rs b/crates/dsp/src/lib.rs index 0ad45deb..6b6c0235 100644 --- a/crates/dsp/src/lib.rs +++ b/crates/dsp/src/lib.rs @@ -197,6 +197,47 @@ pub trait Plugin: Send { fn note_on(&mut self, _offset: usize, _note: u8, _velocity: u8) {} fn note_off(&mut self, _offset: usize, _note: u8) {} fn all_notes_off(&mut self) {} + /// The plugin's complete internal state as an opaque blob (third-party plugins), for saving + /// in the session. `None` when the plugin has no state beyond its parameters (built-ins) or + /// the plugin refused. May allocate: never call it in steady-state audio processing. + fn save_state(&mut self) -> Option> { + None + } + /// Restores a blob from [`Plugin::save_state`]. `false` if unsupported or the plugin rejected + /// it (the plugin keeps working with its previous state). The data is hostile input. + fn load_state(&mut self, _data: &[u8]) -> bool { + false + } + /// Opens the plugin's own editor window (GUI). Main (UI) thread only. + fn open_editor(&mut self) -> Result<(), String> { + Err("no editor".into()) + } + /// Closes the editor opened with [`Plugin::open_editor`] (no-op when none is open). + fn close_editor(&mut self) {} + /// A handle to this instance's editor that can be used on the UI thread while the instance + /// itself is processing on the audio thread (`None` when the plugin has no editor). The + /// handle stays valid after the instance is dropped (its calls then do nothing). + fn editor(&mut self) -> Option> { + None + } +} + +/// A plugin's editor (GUI), driven from the main (UI) thread. Obtained from +/// [`Plugin::editor`]; the plugin instance may meanwhile run on the audio thread (plugin formats +/// allow GUI calls on the main thread concurrently with processing on the audio thread). +pub trait PluginEditor: Send { + /// Opens (or raises) the editor window. Main thread only; an error says why it can't. + fn open(&mut self) -> Result<(), String>; + /// Closes the editor window. + fn close(&mut self); + fn is_open(&self) -> bool; + /// Call every UI frame while open: services the plugin's GUI requests, notices a window the + /// user closed, and returns parameter edits made in the editor as `(param id, value)` in + /// SoundCraft units (ids and ranges of [`PluginInfo::params`]). + fn idle(&mut self) -> Vec<(String, f32)>; + /// Tells the editor that a parameter changed elsewhere (generic editor, automation), so the + /// GUI shows it. Values are in SoundCraft units. + fn set_param(&mut self, _id: &str, _value: f32) {} } /// The plugin registry. @@ -236,3 +277,20 @@ pub fn gain_to_db(g: f32) -> f32 { } (20.0 * a.log10()).max(MIN_DB) } + +#[cfg(test)] +mod trait_default_tests { + use super::*; + + #[test] + fn built_ins_have_no_state_or_editor() { + for info in plugins() { + let mut p = create(info.id).unwrap(); + assert!(p.save_state().is_none(), "{}", info.id); + assert!(!p.load_state(b"anything"), "{}", info.id); + assert!(p.open_editor().is_err(), "{}", info.id); + p.close_editor(); + assert!(p.editor().is_none(), "{}", info.id); + } + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 5e24d6b2..18dacdc0 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -16,6 +16,7 @@ soundcraft-clap-host.workspace = true soundcraft-vst3-host.workspace = true soundcraft-model.workspace = true soundcraft-mix.workspace = true +soundcraft-video.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/crates/engine/src/cmd/clip_more.rs b/crates/engine/src/cmd/clip_more.rs index d8cce74a..cca8102b 100644 --- a/crates/engine/src/cmd/clip_more.rs +++ b/crates/engine/src/cmd/clip_more.rs @@ -104,6 +104,7 @@ fn same_material(a: &Clip, b: &Clip) -> bool { && match (&a.content, &b.content) { (ClipContent::Audio { source: s1, offset: o1 }, ClipContent::Audio { source: s2, offset: o2 }) => s1 == s2 && o1 == o2, (ClipContent::Midi { sequence: q1 }, ClipContent::Midi { sequence: q2 }) => q1 == q2, + (ClipContent::Video { source: s1, offset: o1 }, ClipContent::Video { source: s2, offset: o2 }) => s1 == s2 && o1 == o2, _ => false, } } diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index d6b9738f..86bb4b58 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -37,6 +37,7 @@ pub fn specs() -> Vec { let mut s = e.session().clone(); for t in &mut s.tracks { for pl in &mut t.playlists { pl.clips.clear() } } s.sources.clear(); + s.videos.clear(); let text = s.to_json().map_err(|err| EngineError::Io(err.to_string()))?; std::fs::write(&path, text).map_err(|err| EngineError::Io(format!("{path}: {err}")))?; Ok(json!({"path": path})) @@ -67,19 +68,13 @@ pub fn specs() -> Vec { "Video...", ["File", "Import"], Some("Cmd+Ctrl+I"), - "{path, at?} — imports the movie's audio onto a new track (picture playback is not supported yet)", + "{path, at?} — adds a Video track clip for the movie's picture (H.264, ProRes, Motion JPEG) and imports its audio onto a new track", always, |e, p| { let path = str_param(p, "path").ok_or_else(|| bad("file.import_video", "`path` required"))?.to_string(); let at = position_param(e, "file.import_video", p, "at")?.unwrap_or(0).max(0); let bytes = std::fs::read(&path).map_err(|err| EngineError::Io(format!("{path}: {err}")))?; - let stem = std::path::Path::new(&path).file_stem().and_then(|n| n.to_str()).unwrap_or("Video").to_string(); - let ext = std::path::Path::new(&path).extension().and_then(|x| x.to_str()).unwrap_or("mp4").to_ascii_lowercase(); - let name = format!("{stem} audio.{}", if ext == "mov" { "mp4" } else { ext.as_str() }); - let r = crate::io::import_audio_bytes(e, &name, &bytes, Some(path.as_str()), None, at) - .map_err(|err| EngineError::Io(format!("{path}: no usable audio stream ({err})")))?; - e.message(format!("{stem}: imported the audio; video picture is not shown yet")); - Ok(r) + crate::io::import_video_bytes(e, &path, bytes, at) } ), cmd!(noundo "file.bounce_mix", "Bounce Mix...", ["File"], Some("Cmd+Alt+B"), "{path, format?: wav|aiff|flac, bit_depth?: 16|24|32f, start?, end?, source?: main|bus name, normalize?: false, dither?: true, fold_down?: stereo}", always, bounce), diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index da716a32..665056ae 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -20,12 +20,14 @@ mod midi; mod mix; mod more_util; mod options; +mod plugin_state; mod query; mod setup_more; mod surround; mod track; mod track_more; mod transport; +mod video; mod view; mod view_more; mod vst3; @@ -116,6 +118,8 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(clap::specs()); v.extend(vst3::specs()); v.extend(surround::specs()); + v.extend(plugin_state::specs()); + v.extend(video::specs()); v }) } diff --git a/crates/engine/src/cmd/plugin_state.rs b/crates/engine/src/cmd/plugin_state.rs new file mode 100644 index 00000000..be533aa1 --- /dev/null +++ b/crates/engine/src/cmd/plugin_state.rs @@ -0,0 +1,118 @@ +//! Third-party plugin state in the session (`mix.insert_state`): the UI reads each live plugin's +//! state from the audio engine right before saving and stores it on its insert (base64), from +//! where the mixer restores it into new instances. + +use super::*; +use serde_json::json; +use soundcraft_model::{INSERT_SLOTS, b64}; + +pub fn specs() -> Vec { + vec![CommandSpec { + id: "mix.insert_state", + label: "Store Plugin State", + menu: &[], + shortcut: None, + params: "{track, slot: 0..9 | \"a\"..\"j\" | \"instrument\", state: base64 | null} → {bytes}. Not undoable and not journaled (states can be large).", + enabled: has_tracks, + run: insert_state, + journal: false, + undoable: false, + }] +} + +const CMD: &str = "mix.insert_state"; + +/// `None` = the instrument slot. +fn slot_param(p: &Value) -> Result> { + let v = p.get("slot").ok_or_else(|| bad(CMD, "`slot` required"))?; + if let Some(n) = v.as_u64() { + return usize::try_from(n).ok().filter(|&n| n < INSERT_SLOTS).map(Some).ok_or_else(|| bad(CMD, "slot out of range")); + } + let s = v.as_str().map(|s| s.trim().to_ascii_lowercase()).ok_or_else(|| bad(CMD, "`slot` must be a number or a letter"))?; + if s == "instrument" { + return Ok(None); + } + match s.as_bytes() { + [c @ b'a'..=b'z'] if usize::from(c - b'a') < INSERT_SLOTS => Ok(Some(usize::from(c - b'a'))), + _ => s.parse::().ok().filter(|&n| n < INSERT_SLOTS).map(Some).ok_or_else(|| bad(CMD, "slot out of range")), + } +} + +fn insert_state(e: &mut Engine, p: &Value) -> Result { + let t = track_param(e, CMD, p, "track")?.ok_or_else(|| bad(CMD, "`track` required"))?; + let slot = slot_param(p)?; + let state = match p.get("state") { + None | Some(Value::Null) => None, + Some(Value::String(s)) => Some(b64::decode(s).ok_or_else(|| bad(CMD, "`state` is not valid base64 (or is too large)"))?), + Some(_) => return Err(bad(CMD, "`state` must be a base64 string or null")), + }; + let s = e.session_mut(); + let tr = s.track_mut(t).ok_or_else(|| bad(CMD, "no track"))?; + let ins = match slot { + Some(i) => tr.mixer.inserts.get_mut(i).and_then(Option::as_mut), + None => tr.instrument.as_mut(), + } + .ok_or_else(|| bad(CMD, "empty slot"))?; + let n = state.as_ref().map_or(0, Vec::len); + match state { + Some(bytes) => ins.set_state_bytes(&bytes), + None => ins.state = None, + } + Ok(json!({"bytes": n})) +} + +#[cfg(test)] +mod tests { + use crate::Engine; + use serde_json::json; + use soundcraft_model::b64; + + #[test] + fn stores_and_clears_state_without_undo() { + let mut e = Engine::default(); + e.execute("track.new", &json!({})).unwrap(); + let t = e.session().tracks[0].id.0; + e.execute("mix.insert", &json!({"track": t, "plugin": "gain", "slot": 2})).unwrap(); + let undo_before = e.can_undo(); + let blob = b64::encode(b"\x00\x01plugin state\xff"); + let r = e.execute("mix.insert_state", &json!({"track": t, "slot": "c", "state": blob})).unwrap(); + assert_eq!(r["bytes"], 15); + let ins = e.session().tracks[0].mixer.inserts[2].clone().unwrap(); + assert_eq!(ins.state_bytes().unwrap(), b"\x00\x01plugin state\xff"); + assert_eq!(e.can_undo(), undo_before, "not undoable"); + assert!(e.journal.iter().all(|(id, _)| id != "mix.insert_state"), "not journaled"); + // Survives save/load. + let dir = std::env::temp_dir().join(format!("soundcraft-insert-state-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("s.scraft"); + e.execute("session.save", &json!({"path": path.to_string_lossy()})).unwrap(); + let mut f = Engine::default(); + f.execute("session.open", &json!({"path": path.to_string_lossy()})).unwrap(); + assert_eq!(f.session().tracks[0].mixer.inserts[2].as_ref().unwrap().state, ins.state); + let _ = std::fs::remove_dir_all(&dir); + e.execute("mix.insert_state", &json!({"track": t, "slot": 2, "state": null})).unwrap(); + assert_eq!(e.session().tracks[0].mixer.inserts[2].as_ref().unwrap().state, None); + } + + #[test] + fn hostile_params_are_errors() { + let mut e = Engine::default(); + e.execute("track.new", &json!({})).unwrap(); + let t = e.session().tracks[0].id.0; + e.execute("mix.insert", &json!({"track": t, "plugin": "gain", "slot": 0})).unwrap(); + for p in [ + json!({"track": t, "slot": 0, "state": "!!!"}), + json!({"track": t, "slot": 0, "state": 5}), + json!({"track": t, "slot": 1, "state": "AAAA"}), + json!({"track": t, "slot": 99, "state": "AAAA"}), + json!({"track": t, "slot": "instrument", "state": "AAAA"}), + json!({"track": t, "slot": -1}), + json!({"track": t}), + json!({"track": 999, "slot": 0}), + json!({"slot": 0}), + ] { + assert!(e.execute("mix.insert_state", &p).is_err(), "{p}"); + } + } +} diff --git a/crates/engine/src/cmd/video.rs b/crates/engine/src/cmd/video.rs new file mode 100644 index 00000000..5277ca5a --- /dev/null +++ b/crates/engine/src/cmd/video.rs @@ -0,0 +1,190 @@ +//! Video (picture) commands: Video track online/offline, picture offset, the movie list. +//! +//! Movies are imported with `file.import_video`; the session only references them +//! (`Session::videos`). Decoding happens in the UI (`soundcraft-video`), never in the engine. + +use super::*; +use crate::cmd; +use serde_json::json; +use soundcraft_model::{ClipContent, TrackKind}; + +/// `edit.flags` key set while the Video track is offline (online is the default). +pub const OFFLINE_FLAG: &str = "video.offline"; + +pub fn specs() -> Vec { + vec![ + cmd!(noundo "video.online", "Video Track Online", ["Options"], None, "{value?: bool} (toggles when omitted) — when offline the Video window and the Video track stop decoding pictures", always, |e, p| { + let s = e.session_mut(); + let online = p.get("value").and_then(Value::as_bool).unwrap_or(s.edit.flag(OFFLINE_FLAG)); + s.edit.set_flag(OFFLINE_FLAG, !online); + Ok(json!({"value": online})) + }), + cmd!( + "video.offset", + "Set Video Start", + [], + None, + "{track?: video track (default: the first), at: position} — moves the track's video clips so the picture starts at `at` (e.g. '01:00:00:00')", + always, + offset + ), + cmd!(query "video.list", "Movies", [], None, "{} — movies used by Video tracks (path, size, rate, codec, online file)", always, |e, _| { + let s = e.session(); + let rows: Vec = s + .videos + .iter() + .map(|v| { + let clips = s.tracks.iter().flat_map(|t| t.clips()).filter(|c| c.video_source() == Some(v.id)).count(); + json!({"id": v.id, "name": v.name, "path": v.path, "width": v.width, "height": v.height, "frame_rate": v.frame_rate, + "duration": v.duration, "codec": v.codec, "clips": clips, "file_found": std::path::Path::new(&v.path).is_file()}) + }) + .collect(); + Ok(json!({"videos": rows, "online": !s.edit.flag(OFFLINE_FLAG)})) + }), + ] +} + +fn offset(e: &mut Engine, p: &Value) -> Result { + let at = position_param(e, "video.offset", p, "at")?.ok_or_else(|| bad("video.offset", "`at` required"))?.max(0); + let track = match track_param(e, "video.offset", p, "track")? { + Some(t) => t, + None => e + .session() + .tracks + .iter() + .find(|t| t.kind == TrackKind::Video) + .map(|t| t.id) + .ok_or_else(|| bad("video.offset", "the session has no Video track"))?, + }; + let s = e.session_mut(); + let tr = s.track_mut(track).ok_or_else(|| bad("video.offset", "no such track"))?; + if tr.kind != TrackKind::Video { + return Err(bad("video.offset", format!("`{}` is not a Video track", tr.name))); + } + let Some(pl) = tr.playlist_mut() else { return Err(bad("video.offset", "the track has no playlist")) }; + let first = pl.clips.iter().filter(|c| matches!(c.content, ClipContent::Video { .. })).map(|c| c.start).min(); + let Some(first) = first else { return Err(bad("video.offset", "the Video track has no clips")) }; + let delta = at.saturating_sub(first); + for c in &mut pl.clips { + c.start = c.start.saturating_add(delta).clamp(0, MAX_POSITION); + } + pl.sort(); + Ok(json!({"track": track, "start": at, "moved_by": delta})) +} + +#[cfg(test)] +mod tests { + use crate::Engine; + use serde_json::json; + use soundcraft_model::{ChannelFormat, Clip, ClipContent, SourceId, TrackKind, VideoSource}; + + fn video_engine() -> (Engine, soundcraft_model::TrackId) { + let mut e = Engine::default(); + let s = e.session_mut(); + let t = s.add_track(TrackKind::Video, ChannelFormat::Mono, None); + let vid = SourceId(s.alloc()); + s.videos.push(VideoSource { + id: vid, + name: "m.mov".into(), + path: "/nonexistent/m.mov".into(), + width: 64, + height: 48, + frame_rate: 24.0, + duration: 2.0, + codec: "h264".into(), + }); + let c = s.new_clip_id(); + crate::edit::place_clip(s, t, Clip::video(c, "m", vid, 0, 1000, 96_000)); + (e, t) + } + + #[test] + fn offset_moves_the_picture_and_undoes() { + let (mut e, t) = video_engine(); + e.execute("video.offset", &json!({"at": 48_000})).unwrap(); + assert_eq!(e.session().track(t).unwrap().clips()[0].start, 48_000); + e.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(e.session().track(t).unwrap().clips()[0].start, 1000); + assert!(e.execute("video.offset", &json!({})).is_err()); + let a = e.session_mut().add_track(TrackKind::Audio, ChannelFormat::Mono, Some("A")); + assert!(e.execute("video.offset", &json!({"track": a.0, "at": 0})).is_err()); + assert!(e.execute("video.offset", &json!({"at": i64::MAX})).is_ok()); + } + + #[test] + fn online_toggles_and_list_reports_missing_files() { + let (mut e, _) = video_engine(); + assert_eq!(e.execute("video.online", &json!({})).unwrap()["value"], false); + assert_eq!(e.execute("video.online", &json!({})).unwrap()["value"], true); + assert_eq!(e.execute("video.online", &json!({"value": false})).unwrap()["value"], false); + let l = e.execute("video.list", &json!({})).unwrap(); + assert_eq!(l["online"], false); + assert_eq!(l["videos"][0]["file_found"], false); + assert_eq!(l["videos"][0]["clips"], 1); + } + + #[test] + fn video_clips_are_silent_in_the_mix() { + let (mut e, t) = video_engine(); + assert!(matches!(e.session().track(t).unwrap().clips()[0].content, ClipContent::Video { .. })); + e.session_mut().add_track(TrackKind::Master, ChannelFormat::Stereo, None); + let dir = std::env::temp_dir().join(format!("sc-video-bounce-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let out = dir.join("mix.wav"); + e.execute("file.bounce_mix", &json!({"path": out.to_string_lossy(), "start": 0, "end": 96_000, "dither": false})).unwrap(); + let (_, buf) = soundcraft_audio_io::decode(&std::fs::read(&out).unwrap(), Some("wav")).unwrap(); + assert!(buf.channels.iter().flatten().all(|x| *x == 0.0)); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn import_video_adds_a_video_track_and_survives_save_and_open() { + let dir = std::env::temp_dir().join(format!("sc-video-import-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let movie = dir.join("picture.mp4"); + std::fs::write(&movie, soundcraft_video::synth::h264_mp4(30, 10).unwrap()).unwrap(); + let mut e = Engine::default(); + let r = e.execute("file.import_video", &json!({"path": movie.to_string_lossy(), "at": 4800})).unwrap(); + assert!(r["audio"].is_null() && r["audio_error"].is_string(), "{r}"); + let s = e.session(); + assert_eq!(s.tracks[0].kind, TrackKind::Video); + let clip = s.tracks[0].clips()[0].clone(); + assert_eq!((clip.start, clip.length), (4800, 60_000)); + let v = s.video(clip.video_source().unwrap()).unwrap().clone(); + assert_eq!((v.width, v.height, v.codec.as_str()), (64, 48, "h264")); + assert!((v.frame_rate - 24.0).abs() < 1e-9); + // A second import does not overlap: it goes onto the same Video track after the first. + e.execute("file.import_video", &json!({"path": movie.to_string_lossy(), "at": 96_000})).unwrap(); + assert_eq!(e.session().tracks.iter().filter(|t| t.kind == TrackKind::Video).count(), 1); + e.execute("edit.undo", &json!({})).unwrap(); + // Save, reopen: the clip and the movie reference round-trip; nothing is missing. + let sess = dir.join("s.scraft"); + e.execute("session.save_as", &json!({"path": sess.to_string_lossy()})).unwrap(); + let mut e2 = Engine::default(); + let missing = crate::io::open_session(&mut e2, &sess.to_string_lossy()).unwrap(); + assert!(missing.is_empty(), "{missing:?}"); + assert_eq!(e2.session().tracks[0].clips()[0], clip); + assert_eq!(e2.session().videos, e.session().videos); + // Movie moved next to the session's "Video Files" folder: relinked on open. + std::fs::create_dir_all(dir.join("Video Files")).unwrap(); + std::fs::rename(&movie, dir.join("Video Files").join("picture.mp4")).unwrap(); + let mut e3 = Engine::default(); + assert!(crate::io::open_session(&mut e3, &sess.to_string_lossy()).unwrap().is_empty()); + assert!(e3.session().videos[0].path.contains("Video Files")); + // Movie gone: the session still opens and reports it. + std::fs::remove_file(dir.join("Video Files").join("picture.mp4")).unwrap(); + let mut e4 = Engine::default(); + let missing = crate::io::open_session(&mut e4, &sess.to_string_lossy()).unwrap(); + assert!(missing.iter().any(|m| m.contains("picture.mp4")), "{missing:?}"); + assert!(e4.messages.iter().any(|m| m.contains("missing media"))); + assert_eq!(e4.session().tracks[0].clips().len(), 1); + // Neither picture nor audio: an error, and nothing changes. + let junk = dir.join("junk.mov"); + std::fs::write(&junk, b"not a movie at all").unwrap(); + let before = e4.session().tracks.len(); + assert!(e4.execute("file.import_video", &json!({"path": junk.to_string_lossy()})).is_err()); + assert_eq!(e4.session().tracks.len(), before); + let _ = std::fs::remove_dir_all(dir); + } +} diff --git a/crates/engine/src/edit.rs b/crates/engine/src/edit.rs index 4c8b434c..05cc0e69 100644 --- a/crates/engine/src/edit.rs +++ b/crates/engine/src/edit.rs @@ -319,6 +319,10 @@ pub fn create_fades(s: &mut Session, track: TrackId, range: Range, shape: FadeSh let total = s.source(source).map_or(0, |src| i64::try_from(src.frames).unwrap_or(i64::MAX)); total.saturating_sub(offset).saturating_sub(c.length) } + ClipContent::Video { source, offset } => { + let total = s.video(source).map_or(0, |v| s.sample_rate.samples(v.duration.clamp(0.0, 1e7))); + total.saturating_sub(offset).saturating_sub(c.length) + } ClipContent::Midi { .. } => 0, }; (c.id, avail) diff --git a/crates/engine/src/inspect.rs b/crates/engine/src/inspect.rs index 9e1fdd21..2ca264ba 100644 --- a/crates/engine/src/inspect.rs +++ b/crates/engine/src/inspect.rs @@ -21,6 +21,10 @@ fn clip_json(e: &Engine, c: &Clip) -> Value { ClipContent::Midi { sequence } => { v["notes"] = json!(sequence.notes.len()); } + ClipContent::Video { source, offset } => { + v["video"] = json!(source); + v["offset"] = json!(offset); + } } if !c.gain_env.is_empty() { v["gain_envelope"] = json!(c.gain_env); diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 24096a1e..53d38e23 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -65,6 +65,88 @@ pub fn import_audio_bytes(e: &mut Engine, name: &str, bytes: &[u8], path: Option Ok(json!({"track": track, "clip": cid, "source": src, "frames": frames, "channels": channels})) } +/// Import a movie: a Video track clip spanning the picture (the movie is referenced by `path`, +/// never copied) plus its audio on a new audio track when it has a usable audio stream. Fails +/// only when the file has neither a readable picture nor audio. +pub fn import_video_bytes(e: &mut Engine, path: &str, bytes: Vec, at: Samples) -> Result { + let stem = Path::new(path).file_stem().and_then(|n| n.to_str()).unwrap_or("Video").to_string(); + let ext = Path::new(path).extension().and_then(|x| x.to_str()).unwrap_or("mp4").to_ascii_lowercase(); + let bytes: Arc<[u8]> = bytes.into(); + let movie = soundcraft_video::Movie::open(Arc::clone(&bytes)).map(|m| m.info().clone()); + let audio_name = format!("{stem} audio.{}", if ext == "mov" { "mp4" } else { ext.as_str() }); + let audio = import_audio_bytes(e, &audio_name, &bytes, Some(path), None, at); + let info = match (movie, &audio) { + (Ok(info), _) => info, + (Err(verr), Ok(a)) => { + e.message(format!("{stem}: imported the audio; no picture ({verr})")); + return Ok(json!({"audio": a, "video": Value::Null, "video_error": verr.to_string()})); + } + (Err(verr), Err(aerr)) => return Err(EngineError::Io(format!("{path}: {verr}; no usable audio stream ({aerr})"))), + }; + let abs = std::fs::canonicalize(path).map_or_else(|_| path.to_string(), |p| p.to_string_lossy().into_owned()); + let s = e.session_mut(); + let sr = s.sample_rate.as_f64(); + let len = (info.duration * sr).round(); + let len = if len.is_finite() { (len as Samples).clamp(1, sr as Samples * 3600 * 48) } else { 1 }; + let vid = SourceId(s.alloc()); + s.videos.push(soundcraft_model::VideoSource { + id: vid, + name: Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or("movie").to_string(), + path: abs, + width: info.width, + height: info.height, + frame_rate: info.frame_rate, + duration: info.duration, + codec: info.codec.clone(), + }); + // Reuse the first Video track when the new clip does not overlap its clips; else add one at the top. + let range = Range::new(at, at.saturating_add(len)); + let track = match s.tracks.iter().find(|t| t.kind == TrackKind::Video) { + Some(t) if !t.clips().iter().any(|c| c.range().overlaps(&range)) => t.id, + _ => { + let id = s.add_track(TrackKind::Video, ChannelFormat::Mono, Some(&stem)); + if let Some(i) = s.track_index(id) { + let t = s.tracks.remove(i); + s.tracks.insert(0, t); + } + id + } + }; + let cid = s.new_clip_id(); + crate::edit::place_clip(s, track, Clip::video(cid, stem.clone(), vid, 0, at, len)); + if !info.decodable { + e.message(format!("{stem}: {} pictures cannot be shown yet; the Video track shows the clip only", info.codec_detail)); + } + if let Err(err) = &audio { + e.message(format!("{stem}: no usable audio stream ({err})")); + } + Ok(json!({ + "video": {"track": track, "clip": cid, "source": vid, "length": len, "info": info}, + "audio": audio.as_ref().ok(), + "audio_error": audio.as_ref().err().map(ToString::to_string), + })) +} + +/// Resolve each referenced movie: keep it when its path exists, else look for a file of the same +/// name next to the session (moved projects) and relink. Returns the movies still missing. +pub fn relink_videos(s: &mut Session, session_dir: &Path) -> Vec { + let mut missing = Vec::new(); + for v in &mut s.videos { + let p = if Path::new(&v.path).is_absolute() { PathBuf::from(&v.path) } else { session_dir.join(&v.path) }; + if p.is_file() { + continue; + } + let name = Path::new(&v.path).file_name().map(PathBuf::from).unwrap_or_default(); + let near = + [session_dir.join(&name), session_dir.join("Video Files").join(&name)].into_iter().find(|c| !name.as_os_str().is_empty() && c.is_file()); + match near { + Some(c) => v.path = c.to_string_lossy().into_owned(), + None => missing.push(format!("{} (video)", p.display())), + } + } + missing +} + /// Import a Standard MIDI File as MIDI tracks (one per SMF track). pub fn import_midi_bytes(e: &mut Engine, bytes: &[u8], at: Samples, tempo_map: bool) -> Result> { let smf = soundcraft_midi::read_smf(bytes).map_err(|err| EngineError::Io(err.to_string()))?; @@ -457,6 +539,7 @@ pub fn open_session(e: &mut Engine, path: &str) -> Result> { Err(_) => missing.push(p.display().to_string()), } } + missing.extend(relink_videos(&mut s, &dir)); e.replace_session(s); e.path = Some(path.to_string()); for m in &missing { diff --git a/crates/engine/tests/clap_e2e.rs b/crates/engine/tests/clap_e2e.rs index dc92a2da..9a559f18 100644 --- a/crates/engine/tests/clap_e2e.rs +++ b/crates/engine/tests/clap_e2e.rs @@ -12,6 +12,11 @@ use std::process::Command; use std::sync::Arc; fn install_fixture() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(install_fixture_now); +} + +fn install_fixture_now() { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clap-host/tests/fixture/Cargo.toml"); let target = Path::new(env!("CARGO_TARGET_TMPDIR")).join("clap-fixture-target"); let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); @@ -63,3 +68,53 @@ fn clap_insert_renders_through_the_mix() { assert!((half - 0.5 * dry).abs() < 1e-4, "dry {dry} half {half}"); assert!(e.execute("mix.insert_param", &json!({"track": t.0, "slot": 0, "param": "Gain", "value": 1.0})).is_err()); } + +#[test] +fn clap_state_is_captured_stored_and_restored_before_the_first_block() { + install_fixture(); + let gain = "clap:org.soundcraft.test.gain"; + let mut e = Engine::default(); + let s = e.session_mut(); + let t = s.add_track(TrackKind::Audio, ChannelFormat::Mono, None); + s.pool.insert(SourceId(78), Arc::new(SourceAudio::new(AudioBuffer { sample_rate: 48_000, channels: vec![vec![0.25; 4800]] }))); + let id = s.new_clip_id(); + s.track_mut(t).unwrap().playlist_mut().unwrap().clips.push(Clip::audio(id, "dc", SourceId(78), 0, 0, 4800)); + e.execute("mix.insert", &json!({"track": t.0, "plugin": gain, "params": {"7": 2.0}})).unwrap(); + + // A realtime-style engine: the instance is created "on the UI thread" from the specs and + // adopted; its live state (gain 1.5, changed behind the session's back as a plugin GUI + // would) is captured and stored on the insert. + let mut eng = soundcraft_mix::MixEngine::new(48_000.0, 512); + eng.set_external_instances(true); + let specs = soundcraft_mix::instance_specs(e.session()); + assert_eq!(specs.len(), 1); + let mut p = soundcraft_mix::create_instance(&specs[0].id, None, 48_000.0, 512, specs[0].ch).unwrap(); + p.set_param("7", 1.5); + let mut io = vec![vec![0.0f32; 64]]; + p.process(&mut io, 64); + eng.adopt(vec![soundcraft_mix::PreparedInstance { track: t, slot: 0, id: gain.into(), ch: 1, plugin: p }]); + eng.ensure_synced(e.session()); + let states = eng.capture_states(); + assert_eq!(states.len(), 1); + let (tt, slot, blob) = &states[0]; + assert_eq!((*tt, *slot), (t, 0)); + e.execute("mix.insert_state", &json!({"track": t.0, "slot": slot, "state": soundcraft_model::b64::encode(blob)})).unwrap(); + assert_eq!(eng.take_retired().len(), 0); + + // Offline render (synchronous creation): the state is restored before the first block. + // With the gain absent from the session's params, the state's 1.5 is what plays. + let gain_of = |s: &soundcraft_model::Session| soundcraft_mix::render_range(s, Range::new(0, 4800), 512)[0][2000]; + let mut s2 = e.session().clone(); + s2.track_mut(t).unwrap().mixer.inserts[0].as_mut().unwrap().params.remove("7"); + let restored = gain_of(&s2); + let mut s3 = e.session().clone(); + let ins = s3.track_mut(t).unwrap().mixer.inserts[0].as_mut().unwrap(); + ins.state = None; + ins.params.insert("7".into(), 1.5); + let reference = gain_of(&s3); + assert!((restored - reference).abs() < 1e-5, "state {restored} vs param 1.5 {reference}"); + // Session params apply on top of the state. + e.execute("mix.insert_param", &json!({"track": t.0, "slot": 0, "param": "7", "value": 0.5})).unwrap(); + let half = gain_of(e.session()); + assert!((half - reference / 3.0).abs() < 1e-5, "{half} vs {}", reference / 3.0); +} diff --git a/crates/engine/tests/hostile_sessions.proptest-regressions b/crates/engine/tests/hostile_sessions.proptest-regressions new file mode 100644 index 00000000..7032c3e3 --- /dev/null +++ b/crates/engine/tests/hostile_sessions.proptest-regressions @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 03712deacbb9401dbbdda3badfe083e2ffd4c554d3eb4c5f2131e5595a288594 # shrinks to picks = [(3185898794, -1e18)] +cc 40a5a9de57271049d0fd89dce6889b738bb88a6bb29a5f83a1793ca5fc87d60e # shrinks to picks = [(3418328157, 1e-300)] diff --git a/crates/mix/Cargo.toml b/crates/mix/Cargo.toml index c0825157..1c32f529 100644 --- a/crates/mix/Cargo.toml +++ b/crates/mix/Cargo.toml @@ -14,6 +14,7 @@ soundcraft-midi.workspace = true soundcraft-dsp.workspace = true soundcraft-clap-host.workspace = true soundcraft-vst3-host.workspace = true +soundcraft-au-host.workspace = true soundcraft-model.workspace = true log.workspace = true diff --git a/crates/mix/src/lib.rs b/crates/mix/src/lib.rs index 7b657de6..ecc15af9 100644 --- a/crates/mix/src/lib.rs +++ b/crates/mix/src/lib.rs @@ -19,6 +19,31 @@ //! - sources whose format equals the destination's pass straight through, channel for channel; //! - other multichannel sources fold to stereo with ITU-R BS.775 coefficients (C and surrounds at //! -3 dB, LFE dropped), or map speaker by speaker into other multichannel layouts. +//! +//! ## Plugin instances and threads +//! [`MixEngine::sync`] brings plugin instances in line with the session. Built-in plugins are +//! cheap (allocation only) and are always created there, on whichever thread renders. Hosted +//! third-party plugins (`clap:` / `vst3:` ids, see [`is_third_party`]) are expensive and have +//! main-thread rules, so a realtime host keeps them off the audio thread: +//! - with [`MixEngine::set_external_instances`]`(true)` (realtime playback), `sync` never creates a +//! third-party plugin. The host's control (UI) thread creates them with [`create_instance`] from +//! [`instance_specs`] (restoring each insert's stored [`Insert::state`] and calling `prepare` +//! there), and hands them over with [`MixEngine::adopt`]; `sync` then moves the matching one +//! into its slot. A slot whose instance has not arrived yet passes audio through. +//! - every instance `sync` replaces or removes (built-in or not) goes to a retired list instead of +//! being dropped; the host takes it with [`MixEngine::take_retired`] and drops it on the control +//! thread, so plugin destruction never runs on the audio thread either. +//! - without external instances (the default: offline [`render_range`], bounces, tests) `sync` +//! creates third-party plugins itself, synchronously, restores their stored state before the +//! first `process`, and drops retired instances at the end of `sync`. +//! +//! An instance starts from its stored state; the insert's parameter values are then applied on +//! top before the first block (the session's values are kept in step with the plugin, including +//! edits made in its own editor, so both agree; the state carries everything else). +//! [`MixEngine::capture_states`] reads every live instance's state back (for saving the session); +//! it calls into the plugins and allocates, so a realtime host runs it only on request. +//! +//! [`Insert::state`]: soundcraft_model::Insert::state #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] use soundcraft_dsp::pan::{PanLaw, SpeakerPos, SurroundParams, gains}; @@ -38,6 +63,76 @@ pub fn main_channels(s: &Session) -> usize { s.main_format().channels().clamp(MAIN_CHANNELS, MAX_CHANNELS) } +/// Slot index used for a track's instrument plugin in [`InstanceSpec`], [`PreparedInstance`] and +/// [`MixEngine::capture_states`] (insert slots use their index). +pub const INSTRUMENT_SLOT: usize = usize::MAX; + +/// Hosted third-party plugin ids (CLAP or VST3): created off the audio thread in realtime use. +pub fn is_third_party(id: &str) -> bool { + id.starts_with(soundcraft_clap_host::ID_PREFIX) + || id.starts_with(soundcraft_vst3_host::ID_PREFIX) + || id.starts_with(soundcraft_au_host::ID_PREFIX) +} + +/// A third-party plugin instance a session needs: which slot, which plugin, how many channels, +/// and the state to restore. +#[derive(Debug, Clone, PartialEq)] +pub struct InstanceSpec { + pub track: TrackId, + /// Insert slot index, or [`INSTRUMENT_SLOT`]. + pub slot: usize, + pub id: String, + /// Channel count the instance is prepared for. + pub ch: usize, + /// Decoded [`Insert::state`](soundcraft_model::Insert::state). + pub state: Option>, +} + +/// A plugin instance created and prepared off the audio thread, handed to [`MixEngine::adopt`]. +pub struct PreparedInstance { + pub track: TrackId, + pub slot: usize, + pub id: String, + pub ch: usize, + pub plugin: Box, +} + +/// Every third-party plugin slot (inserts and instruments) of a session, with the channel count +/// [`MixEngine::sync`] prepares it for. Inactive and master tracks are included (they have strips). +pub fn instance_specs(s: &Session) -> Vec { + let main_ch = main_channels(s); + let mut out = Vec::new(); + for t in &s.tracks { + let ch = strip_channels(t, main_ch); + for (i, ins) in t.mixer.inserts.iter().enumerate() { + if let Some(ins) = ins + && is_third_party(&ins.plugin) + { + out.push(InstanceSpec { track: t.id, slot: i, id: ins.plugin.clone(), ch, state: ins.state_bytes() }); + } + } + if let Some(ins) = &t.instrument + && is_third_party(&ins.plugin) + { + out.push(InstanceSpec { track: t.id, slot: INSTRUMENT_SLOT, id: ins.plugin.clone(), ch: ch.max(2), state: ins.state_bytes() }); + } + } + out +} + +/// Creates a plugin (built-in or hosted), restores `state` (if any) and prepares it. Expensive +/// for third-party plugins: in realtime use, call it on the control thread. +pub fn create_instance(id: &str, state: Option<&[u8]>, sample_rate: f32, max_block: usize, ch: usize) -> Option> { + let mut p = create_plugin(id)?; + if let Some(st) = state + && !p.load_state(st) + { + log::warn!("{id}: the plugin did not accept its stored state"); + } + p.prepare(sample_rate, max_block, ch); + Some(p) +} + /// Peak meter values for one strip (linear, per channel, max over the last block). #[derive(Debug, Clone, Default, PartialEq)] pub struct StripMeter { @@ -204,6 +299,48 @@ pub struct MixEngine { pub monitor_only: bool, /// Recording: record-armed tracks hear their input (auto input). pub recording: bool, + /// Third-party instances come from [`MixEngine::adopt`] instead of being created in `sync`. + external: bool, + /// Instances handed over by the host, waiting for `sync` to place them. + adopt: HashMap<(TrackId, usize), PreparedInstance>, + /// Instances `sync` took out of service, for the host to drop (see [`MixEngine::take_retired`]). + retired: Vec>, +} + +/// What `sync` needs to fill a slot. +struct SlotCtx<'a> { + sr: f32, + mb: usize, + external: bool, + adopt: &'a mut HashMap<(TrackId, usize), PreparedInstance>, + retired: &'a mut Vec>, +} + +impl SlotCtx<'_> { + fn retire(&mut self, slot: Option) { + if let Some(s) = slot { + self.retired.push(s.plugin); + } + } + + /// A new instance for `ins` on track `t`, slot `idx`, prepared for `ch` channels. + fn make(&mut self, t: TrackId, idx: usize, ins: &soundcraft_model::Insert, ch: usize) -> Option { + let third = is_third_party(&ins.plugin); + let plugin = if third && self.external { + match self.adopt.remove(&(t, idx)) { + Some(p) if p.id == ins.plugin && p.ch == ch => Some(p.plugin), + Some(p) => { + self.retired.push(p.plugin); + None + } + None => None, + } + } else { + let state = if third { ins.state_bytes() } else { None }; + create_instance(&ins.plugin, state.as_deref(), self.sr, self.mb, ch) + }?; + Some(PluginSlot { id: ins.plugin.clone(), ch, plugin, applied: Vec::new() }) + } } impl MixEngine { @@ -226,9 +363,68 @@ impl MixEngine { input: Vec::new(), monitor_only: false, recording: false, + external: false, + adopt: HashMap::new(), + retired: Vec::new(), } } + /// Realtime mode: third-party plugins are created by the host and handed over with + /// [`MixEngine::adopt`] instead of being created in `sync` (see the crate docs). + pub fn set_external_instances(&mut self, on: bool) { + self.external = on; + } + + /// Takes instances created off the audio thread; the next render's `sync` places them. + /// An earlier instance waiting for the same slot is retired. + pub fn adopt(&mut self, instances: Vec) { + for p in instances { + if let Some(old) = self.adopt.insert((p.track, p.slot), p) { + self.retired.push(old.plugin); + } + } + // Force a re-sync so waiting slots pick their instance up. + self.synced = (usize::MAX, usize::MAX, usize::MAX); + } + + /// Instances taken out of service since the last call (drop them off the audio thread). + /// Does not allocate. + pub fn take_retired(&mut self) -> Vec> { + std::mem::take(&mut self.retired) + } + + /// Every instance this engine holds (strips, instruments, clip effects excluded) plus the + /// retired and waiting ones, leaving the engine empty: hand them to another thread to drop + /// before replacing the engine. + pub fn drain_plugins(&mut self) -> Vec> { + let mut out = std::mem::take(&mut self.retired); + for st in self.strips.values_mut() { + out.extend(st.plugins.iter_mut().filter_map(Option::take).map(|s| s.plugin)); + out.extend(st.instrument.take().map(|s| s.plugin)); + } + out.extend(self.adopt.drain().map(|(_, p)| p.plugin)); + self.synced = (usize::MAX, usize::MAX, usize::MAX); + out + } + + /// The state of every live plugin that has one (third-party plugins), as + /// `(track, slot or INSTRUMENT_SLOT, state)`. Calls into the plugins and allocates. + pub fn capture_states(&mut self) -> Vec<(TrackId, usize, Vec)> { + let mut out = Vec::new(); + for (tid, st) in self.strips.iter_mut() { + for (i, slot) in st.plugins.iter_mut().enumerate() { + if let Some(data) = slot.as_mut().and_then(|s| s.plugin.save_state()) { + out.push((*tid, i, data)); + } + } + if let Some(data) = st.instrument.as_mut().and_then(|s| s.plugin.save_state()) { + out.push((*tid, INSTRUMENT_SLOT, data)); + } + } + out.sort_by_key(|(t, s, _)| (*t, *s)); + out + } + pub fn max_block(&self) -> usize { self.max_block } @@ -262,43 +458,65 @@ impl MixEngine { /// Bring plugin instances in line with the session (creates/destroys as needed). Allocates; /// call outside the realtime thread when the document changes, or let `render` do it. pub fn sync(&mut self, s: &Session) { - let sr = self.sample_rate; let mb = self.max_block; let main_ch = main_channels(s); self.main_fmt = s.main_format(); if self.main.len() != main_ch { self.main = vec![vec![0.0; mb]; main_ch]; } - self.strips.retain(|id, _| s.track(*id).is_some()); + let mut cx = SlotCtx { sr: self.sample_rate, mb, external: self.external, adopt: &mut self.adopt, retired: &mut self.retired }; + let gone: Vec = self.strips.keys().filter(|id| s.track(**id).is_none()).copied().collect(); + for id in gone { + if let Some(mut st) = self.strips.remove(&id) { + for p in st.plugins.drain(..) { + cx.retire(p); + } + cx.retire(st.instrument.take()); + } + } for t in &s.tracks { let ch = strip_channels(t, main_ch); let strip = self.strips.entry(t.id).or_insert_with(Strip::new); if strip.buf.len() != ch || strip.buf.first().map_or(0, Vec::len) != mb { strip.buf = vec![vec![0.0; mb]; ch]; } + while strip.plugins.len() > t.mixer.inserts.len() { + let p = strip.plugins.pop().flatten(); + cx.retire(p); + } strip.plugins.resize_with(t.mixer.inserts.len(), || None); - for (slot, ins) in strip.plugins.iter_mut().zip(t.mixer.inserts.iter()) { + for (idx, (slot, ins)) in strip.plugins.iter_mut().zip(t.mixer.inserts.iter()).enumerate() { match ins { Some(i) if slot.as_ref().is_none_or(|p| p.id != i.plugin || p.ch != ch) => { - *slot = create_plugin(&i.plugin).map(|mut p| { - p.prepare(sr, mb, ch); - PluginSlot { id: i.plugin.clone(), ch, plugin: p, applied: Vec::new() } - }); + let old = slot.take(); + cx.retire(old); + *slot = cx.make(t.id, idx, i, ch); + } + None => { + let old = slot.take(); + cx.retire(old); } - None => *slot = None, _ => {} } } - let want = t.instrument.as_ref().map(|i| i.plugin.as_str()); - if strip.instrument.as_ref().map(|p| p.id.as_str()) != want { - strip.instrument = want.and_then(|id| { - create_plugin(id).map(|mut p| { - p.prepare(sr, mb, ch.max(2)); - PluginSlot { id: id.to_string(), ch: ch.max(2), plugin: p, applied: Vec::new() } - }) - }); + let want = t.instrument.as_ref(); + let ich = ch.max(2); + if strip.instrument.as_ref().map(|p| (p.id.as_str(), p.ch)) != want.map(|i| (i.plugin.as_str(), ich)) { + let old = strip.instrument.take(); + cx.retire(old); + strip.instrument = want.and_then(|i| cx.make(t.id, INSTRUMENT_SLOT, i, ich)); } } + // Instances nobody claimed (the session changed again before they arrived). + let leftovers: Vec<(TrackId, usize)> = cx.adopt.keys().copied().collect(); + for k in leftovers { + if let Some(p) = cx.adopt.remove(&k) { + cx.retired.push(p.plugin); + } + } + if !self.external { + self.retired.clear(); + } for b in &s.busses { let n = b.format.channels().clamp(1, MAX_CHANNELS); let bus = self.busses.entry(b.id).or_insert_with(|| BusBuf { fmt: b.format, buf: Vec::new() }); @@ -310,6 +528,18 @@ impl MixEngine { self.busses.retain(|id, _| s.bus(*id).is_some()); } + /// Re-syncs plugin instances and the routing order when the session's structure changed + /// since the last sync (or instances were adopted). `render` calls it; a realtime host may + /// also call it right after swapping sessions so slots are filled before playback starts. + pub fn ensure_synced(&mut self, s: &Session) { + let key = (structure_fingerprint(s), s.tracks.len(), s.busses.len()); + if key != self.synced { + self.sync(s); + self.order = processing_order(s); + self.synced = key; + } + } + /// Render `frames` (≤ max_block) samples starting at `pos` into `out` (≥ 2 channels). pub fn render(&mut self, s: &Session, pos: Samples, frames: usize, out: &mut [Vec]) { let frames = frames.min(self.max_block); @@ -323,13 +553,7 @@ impl MixEngine { } } self.last_end = pos.saturating_add(frames as i64); - // Re-sync plugin instances and routing order only when the session snapshot changes. - let key = (structure_fingerprint(s), s.tracks.len(), s.busses.len()); - if key != self.synced { - self.sync(s); - self.order = processing_order(s); - self.synced = key; - } + self.ensure_synced(s); for b in self.busses.values_mut() { for c in b.buf.iter_mut() { c.iter_mut().take(frames).for_each(|x| *x = 0.0); @@ -870,21 +1094,21 @@ fn render_clip(s: &Session, clip: &Clip, pos: Samples, frames: usize, buf: &mut let mut i = i0; while i < i1 { let seg_end = (i + STEP).min(i1); - let rel0 = pos + i as i64 - clip.start; - let rel1 = pos + seg_end as i64 - clip.start; + let rel0 = pos.saturating_add(i as i64).saturating_sub(clip.start); + let rel1 = pos.saturating_add(seg_end as i64).saturating_sub(clip.start); let g0 = clip.gain_at(rel0); let g1 = clip.gain_at(rel1.min(clip.length)); let n = (seg_end - i) as f32; for k in i..seg_end { - let rel = pos + k as i64 - clip.start; + let rel = pos.saturating_add(k as i64).saturating_sub(clip.start); let v = if (stretch - 1.0).abs() < 1e-9 { - usize::try_from(offset + rel).ok().and_then(|x| src.get(x)).copied().unwrap_or(0.0) + usize::try_from(offset.saturating_add(rel)).ok().and_then(|x| src.get(x)).copied().unwrap_or(0.0) } else { let f = offset as f64 + rel as f64 / stretch; let x0 = f.floor(); let fr = (f - x0) as f32; let a = if x0 >= 0.0 { src.get(x0 as usize).copied().unwrap_or(0.0) } else { 0.0 }; - let b = if x0 + 1.0 >= 0.0 { src.get(x0 as usize + 1).copied().unwrap_or(0.0) } else { 0.0 }; + let b = if x0 + 1.0 >= 0.0 { src.get((x0 as usize).saturating_add(1)).copied().unwrap_or(0.0) } else { 0.0 }; a + (b - a) * fr }; let t = (k - i) as f32 / n; @@ -927,9 +1151,9 @@ fn schedule_notes(s: &Session, t: &Track, pos: Samples, frames: usize, inst: &mu n.velocity = (i64::from(n.velocity) + dv).clamp(1, 127) as u8; n.pitch = (i64::from(n.pitch) + dp).clamp(0, 127) as u8; n.length = ((n.length as f64 * dur) as i64).max(1); - n.start = (n.start + delay).max(0); - let on = s.tempo.tick_to_samples(base + n.start, sr); - let off = s.tempo.tick_to_samples(base + n.start + n.length, sr).min(clip.end()); + n.start = n.start.saturating_add(delay).max(0); + let on = s.tempo.tick_to_samples(base.saturating_add(n.start), sr); + let off = s.tempo.tick_to_samples(base.saturating_add(n.start).saturating_add(n.length), sr).min(clip.end()); if on >= clip.end() { continue; } @@ -1412,7 +1636,10 @@ pub fn render_track_pre_fader(s: &Session, track: TrackId, range: Range) -> Vec< /// A built-in plugin, else a hosted CLAP plugin (`clap:`). fn create_plugin(id: &str) -> Option> { - soundcraft_dsp::create(id).or_else(|| soundcraft_clap_host::create(id)).or_else(|| soundcraft_vst3_host::create(id)) + soundcraft_dsp::create(id) + .or_else(|| soundcraft_clap_host::create(id)) + .or_else(|| soundcraft_vst3_host::create(id)) + .or_else(|| soundcraft_au_host::create(id)) } #[cfg(test)] @@ -1528,6 +1755,113 @@ mod tests { let _ = render_range(&s, Range::new(0, 4800), 512); } + /// A built-in gain behind a third-party id, with a state blob and a log of parameter sets. + struct Stateful { + inner: Box, + blob: Vec, + sets: Arc, + } + + impl Plugin for Stateful { + fn info(&self) -> &'static soundcraft_dsp::PluginInfo { + self.inner.info() + } + fn prepare(&mut self, sr: f32, mb: usize, ch: usize) { + self.inner.prepare(sr, mb, ch); + } + fn reset(&mut self) { + self.inner.reset(); + } + fn set_param(&mut self, id: &str, v: f32) -> bool { + self.sets.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.inner.set_param(id, v) + } + fn param(&self, id: &str) -> Option { + self.inner.param(id) + } + fn process(&mut self, io: &mut [Vec], frames: usize) { + self.inner.process(io, frames); + } + fn save_state(&mut self) -> Option> { + Some(self.blob.clone()) + } + } + + fn render_block(eng: &mut MixEngine, s: &Session, pos: i64) -> f32 { + let mut out = vec![vec![0.0f32; 512]; 2]; + eng.render(s, pos, 512, &mut out); + out[0][500] + } + + #[test] + fn external_instances_are_adopted_retired_and_captured() { + let (mut s, t) = session_with_dc(0.5, 48_000); + let mut ins = Insert::new("clap:test.fake"); + ins.params.insert("gain".into(), -6.0); + s.track_mut(t).unwrap().mixer.inserts[0] = Some(ins); + let specs = instance_specs(&s); + assert_eq!(specs, vec![InstanceSpec { track: t, slot: 0, id: "clap:test.fake".into(), ch: 1, state: None }]); + let mut eng = MixEngine::new(48_000.0, 512); + eng.set_external_instances(true); + // Nothing to adopt yet: the slot passes audio through (and sync never tried to load a + // CLAP plugin on this thread). + let dry = render_block(&mut eng, &s, 0); + assert!(dry > 0.3, "{dry}"); + let sets = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut gain = soundcraft_dsp::create("gain").unwrap(); + gain.prepare(48_000.0, 512, 1); + let fake = Stateful { inner: gain, blob: b"live state".to_vec(), sets: Arc::clone(&sets) }; + // An instance for the wrong channel count is retired, not used. + let wrong = Stateful { inner: soundcraft_dsp::create("gain").unwrap(), blob: Vec::new(), sets: Arc::clone(&sets) }; + eng.adopt(vec![PreparedInstance { track: t, slot: 0, id: "clap:test.fake".into(), ch: 2, plugin: Box::new(wrong) }]); + assert!(render_block(&mut eng, &s, 512) > 0.3); + assert_eq!(eng.take_retired().len(), 1); + eng.adopt(vec![PreparedInstance { track: t, slot: 0, id: "clap:test.fake".into(), ch: 1, plugin: Box::new(fake) }]); + let mut wet = 0.0; + for b in 2..20 { + wet = render_block(&mut eng, &s, b * 512); + } + assert!((wet / dry - 0.501).abs() < 0.01, "{wet} vs {dry}"); + assert!(sets.load(std::sync::atomic::Ordering::Relaxed) > 0, "no stored state: session params are pushed"); + assert_eq!(eng.capture_states(), vec![(t, 0, b"live state".to_vec())]); + assert!(eng.take_retired().is_empty()); + // Removing the insert retires the instance instead of dropping it here. + s.track_mut(t).unwrap().mixer.inserts[0] = None; + render_block(&mut eng, &s, 20 * 512); + assert_eq!(eng.take_retired().len(), 1); + assert!(eng.capture_states().is_empty()); + // Instances nobody claims are retired too; drain_plugins empties the engine. + eng.adopt(vec![PreparedInstance { track: t, slot: 3, id: "clap:x".into(), ch: 1, plugin: soundcraft_dsp::create("gain").unwrap() }]); + render_block(&mut eng, &s, 21 * 512); + assert_eq!(eng.take_retired().len(), 1); + assert!(eng.drain_plugins().is_empty()); + } + + #[test] + fn stored_state_comes_first_then_session_params() { + let (mut s, t) = session_with_dc(0.5, 48_000); + let mut ins = Insert::new("vst3:00000000000000000000000000000001"); + ins.params.insert("gain".into(), -6.0); + ins.set_state_bytes(b"stored"); + s.track_mut(t).unwrap().mixer.inserts[0] = Some(ins); + assert_eq!(instance_specs(&s)[0].state.as_deref(), Some(&b"stored"[..])); + let mut eng = MixEngine::new(48_000.0, 512); + eng.set_external_instances(true); + let sets = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let fake = Stateful { inner: soundcraft_dsp::create("gain").unwrap(), blob: Vec::new(), sets: Arc::clone(&sets) }; + eng.adopt(vec![PreparedInstance { track: t, slot: 0, id: "vst3:00000000000000000000000000000001".into(), ch: 1, plugin: Box::new(fake) }]); + render_block(&mut eng, &s, 0); + assert_eq!(sets.load(std::sync::atomic::Ordering::Relaxed), 1, "session params apply once"); + render_block(&mut eng, &s, 512); + assert_eq!(sets.load(std::sync::atomic::Ordering::Relaxed), 1); + s.track_mut(t).unwrap().mixer.inserts[0].as_mut().unwrap().params.insert("gain".into(), -12.0); + render_block(&mut eng, &s, 1024); + assert_eq!(sets.load(std::sync::atomic::Ordering::Relaxed), 2, "later session edits apply"); + // Offline (not external): unknown third-party plugins are simply absent. + let out = render_range(&s, Range::new(0, 1024), 512); + assert!(out[0][100] > 0.3); + } + #[test] fn clip_effects_gain_and_trim_automation_apply() { let (mut s, t) = session_with_dc(0.5, 4800); diff --git a/crates/model/src/b64.rs b/crates/model/src/b64.rs new file mode 100644 index 00000000..8288c466 --- /dev/null +++ b/crates/model/src/b64.rs @@ -0,0 +1,84 @@ +//! Standard base64 (RFC 4648, `+/` alphabet, `=` padding), used to keep opaque plugin state +//! blobs in the JSON session file. Decoding treats its input as hostile: it rejects bad +//! characters and bad padding, ignores ASCII whitespace, and caps the output size. + +/// Largest blob [`decode`] will produce (64 MiB): bigger session fields are rejected. +pub const MAX_DECODED_BYTES: usize = 64 << 20; + +const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/// Encodes bytes as padded standard base64. +pub fn encode(data: &[u8]) -> String { + let mut out = String::with_capacity(data.len().div_ceil(3).saturating_mul(4)); + for chunk in data.chunks(3) { + let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)]; + let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + let sym = |shift: u32| char::from(ALPHABET.get(((n >> shift) & 63) as usize).copied().unwrap_or(b'A')); + out.push(sym(18)); + out.push(sym(12)); + out.push(if chunk.len() > 1 { sym(6) } else { '=' }); + out.push(if chunk.len() > 2 { sym(0) } else { '=' }); + } + out +} + +fn value(c: u8) -> Option { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' => 62, + b'/' => 63, + _ => return None, + }; + Some(u32::from(v)) +} + +/// Decodes padded (or unpadded) standard base64. `None` for malformed input or a result larger +/// than [`MAX_DECODED_BYTES`]. +pub fn decode(text: &str) -> Option> { + let syms: Vec = text.bytes().filter(|c| !c.is_ascii_whitespace()).collect(); + let body = syms.strip_suffix(b"==").or_else(|| syms.strip_suffix(b"=")).unwrap_or(&syms); + if body.contains(&b'=') || (syms.len() != body.len() && !syms.len().is_multiple_of(4)) || body.len() % 4 == 1 { + return None; + } + if body.len() / 4 * 3 > MAX_DECODED_BYTES { + return None; + } + let mut out = Vec::with_capacity(body.len() / 4 * 3 + 2); + for chunk in body.chunks(4) { + let mut n = 0u32; + for (i, &c) in chunk.iter().enumerate() { + n |= value(c)? << (18 - 6 * i as u32); + } + let bytes = [(n >> 16) as u8, (n >> 8) as u8, n as u8]; + out.extend_from_slice(bytes.get(..chunk.len() - 1)?); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rfc4648_vectors_round_trip() { + for (plain, enc) in + [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] + { + assert_eq!(encode(plain.as_bytes()), enc); + assert_eq!(decode(enc).unwrap(), plain.as_bytes()); + } + let all: Vec = (0..=255).collect(); + assert_eq!(decode(&encode(&all)).unwrap(), all); + assert_eq!(decode("Zm9v\nYmE=").unwrap(), b"fooba", "whitespace is ignored"); + assert_eq!(decode("Zm9vYmE").unwrap(), b"fooba", "padding is optional"); + } + + #[test] + fn hostile_input_is_rejected() { + for bad in ["Z", "Zm9=v", "Zm9v!", "====", "Zg=", "é", "Zg===", "Z==="] { + assert!(decode(bad).is_none(), "{bad}"); + } + } +} diff --git a/crates/model/src/clip.rs b/crates/model/src/clip.rs index 046b3df4..4d30bea1 100644 --- a/crates/model/src/clip.rs +++ b/crates/model/src/clip.rs @@ -60,6 +60,9 @@ pub enum ClipContent { Audio { source: SourceId, offset: Samples }, /// MIDI notes, with ticks relative to the clip start. Midi { sequence: Sequence }, + /// Picture from the movie `source` ([`crate::VideoSource`], `Session::videos`), starting + /// `offset` samples into the movie. Video clips carry no audio. + Video { source: SourceId, offset: Samples }, } /// A clip on a playlist. @@ -137,6 +140,13 @@ impl Clip { c } + /// A clip showing the movie `source` from `offset` samples into it. + pub fn video(id: ClipId, name: impl Into, source: SourceId, offset: Samples, start: Samples, length: Samples) -> Clip { + let mut c = Clip::audio(id, name, SourceId(0), 0, start, length); + c.content = ClipContent::Video { source, offset }; + c + } + pub fn end(&self) -> Samples { self.start.saturating_add(self.length) } @@ -149,16 +159,30 @@ impl Clip { matches!(self.content, ClipContent::Audio { .. }) } + pub fn is_video(&self) -> bool { + matches!(self.content, ClipContent::Video { .. }) + } + + /// The audio source this clip plays (None for MIDI and video clips). pub fn source(&self) -> Option { match self.content { ClipContent::Audio { source, .. } => Some(source), - ClipContent::Midi { .. } => None, + ClipContent::Midi { .. } | ClipContent::Video { .. } => None, } } + /// The movie this clip shows (video clips only). + pub fn video_source(&self) -> Option { + match self.content { + ClipContent::Video { source, .. } => Some(source), + ClipContent::Audio { .. } | ClipContent::Midi { .. } => None, + } + } + + /// Offset into the audio file or movie (0 for MIDI). pub fn source_offset(&self) -> Samples { match self.content { - ClipContent::Audio { offset, .. } => offset, + ClipContent::Audio { offset, .. } | ClipContent::Video { offset, .. } => offset, ClipContent::Midi { .. } => 0, } } @@ -184,7 +208,7 @@ impl Clip { pub fn trim_start_to(&mut self, new_start: Samples) { let new_start = new_start.min(self.end() - 1); let delta = new_start - self.start; - if let ClipContent::Audio { offset, .. } = &mut self.content { + if let ClipContent::Audio { offset, .. } | ClipContent::Video { offset, .. } = &mut self.content { *offset = offset.saturating_add(delta); } // MIDI note ticks are re-anchored by the engine, which knows the tempo map. diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index 7bf81afd..55d605d0 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -6,6 +6,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod automation; +pub mod b64; mod clip; mod ids; mod markers; @@ -22,7 +23,7 @@ pub use mixer::{ Bus, ChannelFormat, INSERT_SLOTS, Insert, Mixer, OutputPath, Route, SEND_SLOTS, SendSlot, Speaker, SurroundPan, fader_db_to_pos, fader_pos_to_db, }; pub use session::{BitDepthSetting, EditMode, EditState, SESSION_EXTENSION, Session, SessionError, Tool, TrackView, ZoomState}; -pub use source::{Source, SourceAudio, SourcePool}; +pub use source::{Source, SourceAudio, SourcePool, VideoSource}; pub use track::{Playlist, TRACK_COLORS, Track, TrackHeight, TrackKind}; pub use soundcraft_time as time; diff --git a/crates/model/src/mixer.rs b/crates/model/src/mixer.rs index d1f33464..e9a78673 100644 --- a/crates/model/src/mixer.rs +++ b/crates/model/src/mixer.rs @@ -434,6 +434,10 @@ pub struct Insert { /// Preset name shown in the plugin window header. #[serde(default)] pub preset: String, + /// Opaque plugin state (third-party plugins: CLAP state / VST3 component + controller + /// state), standard base64. Restored into a new instance before it first processes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, } fn yes() -> bool { @@ -442,7 +446,17 @@ fn yes() -> bool { impl Insert { pub fn new(plugin: impl Into) -> Self { - Insert { plugin: plugin.into(), params: BTreeMap::new(), bypass: false, active: true, preset: String::from("") } + Insert { plugin: plugin.into(), params: BTreeMap::new(), bypass: false, active: true, preset: String::from(""), state: None } + } + + /// The decoded plugin state (`None` when absent or malformed). + pub fn state_bytes(&self) -> Option> { + self.state.as_deref().and_then(crate::b64::decode) + } + + /// Stores a plugin state blob (base64-encoded); an empty blob clears it. + pub fn set_state_bytes(&mut self, data: &[u8]) { + self.state = (!data.is_empty()).then(|| crate::b64::encode(data)); } } @@ -590,6 +604,26 @@ pub fn fader_db_to_pos(db: f32) -> f32 { mod tests { use super::*; + #[test] + fn insert_state_round_trips_through_json() { + let mut i = Insert::new("vst3:00"); + assert_eq!(i.state_bytes(), None); + let blob: Vec = (0..=255u8).cycle().take(1000).collect(); + i.set_state_bytes(&blob); + let json = serde_json::to_string(&i).unwrap(); + let back: Insert = serde_json::from_str(&json).unwrap(); + assert_eq!(back.state_bytes().unwrap(), blob); + i.set_state_bytes(&[]); + assert_eq!(i.state, None); + // Old sessions have no `state`; stateless inserts don't write one. + let old: Insert = serde_json::from_str(r#"{"plugin":"eq_7band"}"#).unwrap(); + assert_eq!(old.state, None); + assert!(!serde_json::to_string(&old).unwrap().contains("state")); + // A hostile state string decodes to nothing rather than failing the load. + let bad: Insert = serde_json::from_str(r#"{"plugin":"x","state":"!!!"}"#).unwrap(); + assert_eq!(bad.state_bytes(), None); + } + #[test] fn fader_taper_round_trips() { for db in [-100.0f32, -60.0, -24.0, -10.0, 0.0, 6.0, 12.0] { diff --git a/crates/model/src/session.rs b/crates/model/src/session.rs index ec94b1f9..f3537d7c 100644 --- a/crates/model/src/session.rs +++ b/crates/model/src/session.rs @@ -2,7 +2,7 @@ use crate::{ Bus, BusId, ChannelFormat, Clip, ClipId, Group, GroupId, MarkerId, MarkerKind, MemoryLocation, OutputPath, Source, SourceId, SourcePool, - TRACK_COLORS, Track, TrackId, TrackKind, + TRACK_COLORS, Track, TrackId, TrackKind, VideoSource, }; use soundcraft_time::{FrameRate, GridValue, Range, SampleRate, Samples, TempoMap, TimeFormat}; @@ -291,6 +291,9 @@ pub struct Session { pub key_signatures: Vec<(Samples, String)>, pub tracks: Vec, pub sources: Vec, + /// Movies referenced by Video track clips. + #[serde(default)] + pub videos: Vec, pub busses: Vec, pub outputs: Vec, pub markers: Vec, @@ -328,6 +331,7 @@ impl Session { key_signatures: Vec::new(), tracks: Vec::new(), sources: Vec::new(), + videos: Vec::new(), busses: Vec::new(), outputs: vec![OutputPath { name: "Out 1-2".into(), first_channel: 0, format: ChannelFormat::Stereo }], markers: Vec::new(), @@ -370,6 +374,9 @@ impl Session { pub fn source(&self, id: SourceId) -> Option<&Source> { self.sources.iter().find(|s| s.id == id) } + pub fn video(&self, id: SourceId) -> Option<&VideoSource> { + self.videos.iter().find(|v| v.id == id) + } pub fn bus(&self, id: BusId) -> Option<&Bus> { self.busses.iter().find(|b| b.id == id) } @@ -497,6 +504,7 @@ impl Session { .map(|t| t.id.0) .chain(self.tracks.iter().flat_map(|t| t.playlists.iter().flat_map(|p| p.clips.iter().map(|c| c.id.0)))) .chain(self.sources.iter().map(|s| s.id.0)) + .chain(self.videos.iter().map(|v| v.id.0)) .chain(self.busses.iter().map(|b| b.id.0)) .chain(self.markers.iter().map(|m| m.id.0)) .chain(self.groups.iter().map(|g| g.id.0)) @@ -530,6 +538,14 @@ impl Session { l.points.sort_by_key(|p| p.at); } } + for v in &mut self.videos { + if !v.frame_rate.is_finite() || v.frame_rate < 0.0 { + v.frame_rate = 0.0; + } + if !v.duration.is_finite() || v.duration < 0.0 { + v.duration = 0.0; + } + } // Drop performances whose clip no longer exists anywhere (any playlist). let clip_ids: std::collections::BTreeSet = self.tracks.iter().flat_map(|t| t.playlists.iter().flat_map(|p| p.clips.iter().map(|c| c.id.0))).collect(); @@ -573,6 +589,41 @@ mod tests { assert_eq!(back, s); } + #[test] + fn video_clip_round_trips_and_old_sessions_load() { + let mut s = Session::default(); + let t = s.add_track(TrackKind::Video, ChannelFormat::Mono, None); + let vid = SourceId(s.alloc()); + s.videos.push(VideoSource { + id: vid, + name: "cut.mov".into(), + path: "/films/cut.mov".into(), + width: 1920, + height: 1080, + frame_rate: 23.976, + duration: 12.5, + codec: "prores".into(), + }); + let cid = s.new_clip_id(); + if let Some(p) = s.track_mut(t).and_then(Track::playlist_mut) { + p.clips.push(Clip::video(cid, "cut", vid, 480, 0, 600_000)); + } + let back = Session::from_json(&s.to_json().unwrap()).unwrap(); + assert_eq!(back, s); + let c = &back.track(t).unwrap().clips()[0]; + assert_eq!((c.video_source(), c.source(), c.source_offset(), c.is_audio()), (Some(vid), None, 480, false)); + assert_eq!(back.track(t).unwrap().kind.label(), "Video Track"); + assert!(back.video(vid).is_some()); + // A session written before video support has no `videos` key. + let mut v = serde_json::to_value(Session::default()).unwrap(); + v.as_object_mut().unwrap().remove("videos"); + assert!(Session::from_json(&v.to_string()).unwrap().videos.is_empty()); + // Trimming the start keeps the picture anchored. + let mut c = c.clone(); + c.trim_start_to(100); + assert_eq!(c.source_offset(), 580); + } + #[test] fn rejects_garbage_and_future_versions() { assert!(Session::from_json("not json").is_err()); diff --git a/crates/model/src/source.rs b/crates/model/src/source.rs index 9c8ac6c2..5da35e76 100644 --- a/crates/model/src/source.rs +++ b/crates/model/src/source.rs @@ -1,4 +1,5 @@ -//! Audio sources (files in the session's Audio Files folder) and their decoded data. +//! Audio sources (files in the session's Audio Files folder) and their decoded data, and the +//! movies video clips reference. use crate::SourceId; use soundcraft_audio_io::{AudioBuffer, FileFormat, peaks::Peaks}; @@ -24,6 +25,36 @@ pub struct Source { pub unsaved: bool, } +/// A movie used by Video track clips (serialised). The pictures themselves are never stored in +/// the session: the UI opens the movie from `path` and decodes frames on demand. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct VideoSource { + pub id: SourceId, + pub name: String, + /// Absolute path of the movie (referenced, never copied into the session folder). + pub path: String, + #[serde(default)] + pub width: u32, + #[serde(default)] + pub height: u32, + /// Frames per second (0 = unknown). + #[serde(default)] + pub frame_rate: f64, + /// Picture length in seconds. + #[serde(default)] + pub duration: f64, + /// Codec id (`h264`, `prores`, `mjpeg`, `hevc`, …). + #[serde(default)] + pub codec: String, +} + +impl VideoSource { + /// Display aspect ratio (width / height), 16:9 when unknown. + pub fn aspect(&self) -> f64 { + if self.width == 0 || self.height == 0 { 16.0 / 9.0 } else { f64::from(self.width) / f64::from(self.height) } + } +} + /// Decoded audio plus waveform overviews (not serialised). #[derive(Debug)] pub struct SourceAudio { diff --git a/crates/model/src/track.rs b/crates/model/src/track.rs index 085d9083..aa2b64f4 100644 --- a/crates/model/src/track.rs +++ b/crates/model/src/track.rs @@ -14,11 +14,21 @@ pub enum TrackKind { Instrument, Vca, Folder, + /// Picture: clips reference a movie ([`crate::VideoSource`]); no audio path. + Video, } impl TrackKind { - pub const ALL: [TrackKind; 7] = - [TrackKind::Audio, TrackKind::Aux, TrackKind::Master, TrackKind::Midi, TrackKind::Instrument, TrackKind::Vca, TrackKind::Folder]; + pub const ALL: [TrackKind; 8] = [ + TrackKind::Audio, + TrackKind::Aux, + TrackKind::Master, + TrackKind::Midi, + TrackKind::Instrument, + TrackKind::Vca, + TrackKind::Folder, + TrackKind::Video, + ]; pub fn label(self) -> &'static str { match self { TrackKind::Audio => "Audio Track", @@ -28,6 +38,7 @@ impl TrackKind { TrackKind::Instrument => "Instrument Track", TrackKind::Vca => "VCA Master", TrackKind::Folder => "Routing Folder Track", + TrackKind::Video => "Video Track", } } pub fn default_name(self) -> &'static str { @@ -39,6 +50,7 @@ impl TrackKind { TrackKind::Instrument => "Inst", TrackKind::Vca => "VCA", TrackKind::Folder => "Folder", + TrackKind::Video => "Video", } } pub fn id(self) -> &'static str { @@ -50,6 +62,7 @@ impl TrackKind { TrackKind::Instrument => "instrument", TrackKind::Vca => "vca", TrackKind::Folder => "folder", + TrackKind::Video => "video", } } pub fn from_id(s: &str) -> Option { @@ -58,7 +71,7 @@ impl TrackKind { } /// Track kinds that hold clips on a timeline. pub fn has_playlist(self) -> bool { - matches!(self, TrackKind::Audio | TrackKind::Midi | TrackKind::Instrument) + matches!(self, TrackKind::Audio | TrackKind::Midi | TrackKind::Instrument | TrackKind::Video) } pub fn is_midi(self) -> bool { matches!(self, TrackKind::Midi | TrackKind::Instrument) diff --git a/crates/playback/src/lib.rs b/crates/playback/src/lib.rs index 0ac1113e..6bc6b29e 100644 --- a/crates/playback/src/lib.rs +++ b/crates/playback/src/lib.rs @@ -11,11 +11,21 @@ //! the session's main format. Main channels (SMPTE/WAV order) go to device channels in order; when //! the device has fewer channels than the main mix, the mix is folded down to stereo (ITU-R //! BS.775) on the first two device channels. +//! +//! Plugins and threads: hosted third-party plugins (CLAP/VST3) are created, given their stored +//! state and prepared on the thread that calls [`Player::update_session`] (the UI thread), and +//! travel to the audio thread together with the session snapshot that needs them; the mix +//! engine adopts them (see `soundcraft_mix`'s crate docs). Instances the engine retires come back +//! on a return channel and are dropped on the UI thread ([`Player::idle`]). Plugin states for +//! saving are read on request ([`Player::capture_states`]: a command with a reply; the audio +//! thread answers between two blocks, never blocking). Plugin editors are driven on the UI thread +//! through handles taken from each instance before it leaves ([`Player::open_editor`]). #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod record; -use soundcraft_mix::{MAX_CHANNELS, MixEngine, StripMeter, itu_stereo_fold, main_channels}; +use soundcraft_dsp::{Plugin, PluginEditor}; +use soundcraft_mix::{MAX_CHANNELS, MixEngine, PreparedInstance, StripMeter, itu_stereo_fold, main_channels}; use soundcraft_model::{Session, TrackId}; use soundcraft_time::{Range, Samples}; use std::collections::HashMap; @@ -32,11 +42,24 @@ pub struct MeterSnapshot { pub main: StripMeter, } +/// Plugin states read from the live instances: `(track, slot or INSTRUMENT_SLOT, state)`. +pub type PluginStates = Vec<(TrackId, usize, Vec)>; + +/// A parameter edit made in a plugin's own editor: `(track, slot, param id, value)`. +pub type EditorEdit = (TrackId, usize, String, f32); + enum Cmd { - Session(Arc), + /// A new snapshot plus the third-party instances it needs (created on the UI thread). + Session(Arc, Vec), + /// Read every plugin state and reply. + CaptureStates(Sender), Input(Arc), Recording(bool), - Play { from: Samples, end: Option, looped: Option }, + Play { + from: Samples, + end: Option, + looped: Option, + }, Stop, Locate(Samples), } @@ -69,17 +92,26 @@ struct AudioState { device_rate: f64, input: Option>, recording: bool, + /// Retired plugin instances go back to the UI thread to be dropped there. + garbage: Sender>>, +} + +fn new_engine(sr: f32) -> MixEngine { + let mut m = MixEngine::new(sr, BLOCK); + m.set_external_instances(true); + m } impl AudioState { - fn new(rx: Receiver, shared: Arc, session: Arc, device_rate: f64) -> Self { + fn new(rx: Receiver, shared: Arc, session: Arc, device_rate: f64, garbage: Sender>>) -> Self { let sr = session.sample_rate.as_f64() as f32; let nch = main_channels(&session); AudioState { + garbage, fold: fold_gains(&session), rx, shared, - mix: MixEngine::new(sr, BLOCK), + mix: new_engine(sr), session, pos: 0, playing: false, @@ -96,14 +128,22 @@ impl AudioState { } fn drain(&mut self) { + let mut changed = false; while let Ok(c) = self.rx.try_recv() { match c { Cmd::Input(r) => self.input = Some(r), Cmd::Recording(r) => self.recording = r, - Cmd::Session(s) => { + Cmd::CaptureStates(reply) => { + self.mix.ensure_synced(&self.session); + let _ = reply.send(self.mix.capture_states()); + } + Cmd::Session(s, instances) => { + changed = true; if (s.sample_rate.as_f64() - self.session.sample_rate.as_f64()).abs() > 0.5 { - self.mix = MixEngine::new(s.sample_rate.as_f64() as f32, BLOCK); + let mut old = std::mem::replace(&mut self.mix, new_engine(s.sample_rate.as_f64() as f32)); + let _ = self.garbage.send(old.drain_plugins()); } + self.mix.adopt(instances); // The main format changed (a document edit, not steady state): resize. let nch = main_channels(&s); if self.block.len() != nch { @@ -135,9 +175,22 @@ impl AudioState { } } } + if changed { + // Place adopted instances now (not only once playback starts), so state captures and + // editors see them; `sync` creates no third-party plugins here. + self.mix.ensure_synced(&self.session); + } + self.send_garbage(); self.shared.playing.store(self.playing, Ordering::Relaxed); } + fn send_garbage(&mut self) { + let g = self.mix.take_retired(); + if !g.is_empty() { + let _ = self.garbage.send(g); + } + } + /// Render the next block at the session rate into `pending`. fn render_next(&mut self) { let mut n = BLOCK; @@ -174,6 +227,7 @@ impl AudioState { self.mix.monitor_only = !self.playing; self.mix.recording = self.recording; self.mix.render(&self.session, self.pos, n, &mut self.block); + self.send_garbage(); for (d, s) in self.pending.iter_mut().zip(self.block.iter()) { if let (Some(dd), Some(ss)) = (d.get_mut(..n), s.get(..n)) { dd.copy_from_slice(ss); @@ -262,10 +316,38 @@ impl AudioState { } } +/// UI-thread bookkeeping of the third-party instances the audio thread holds. +struct Side { + /// What was last created for each slot: (plugin id, channels, created ok). + shadow: HashMap<(TrackId, usize), (String, usize, bool)>, + /// Session sample rate the instances were prepared for. + rate: f32, + /// Editor handles of the live instances. + editors: HashMap<(TrackId, usize), Box>, + garbage: Receiver>>, +} + +impl Side { + /// Drops retired instances (closing their editors first, inside the plugin's drop). + fn collect_garbage(&mut self) { + while let Ok(g) = self.garbage.try_recv() { + drop(g); + } + } + + fn forget(&mut self, key: (TrackId, usize)) { + self.shadow.remove(&key); + if let Some(mut e) = self.editors.remove(&key) { + e.close(); + } + } +} + /// The transport and audio device. pub struct Player { shared: Arc, tx: Sender, + side: Mutex, #[cfg(not(target_arch = "wasm32"))] _stream: Option, #[cfg(target_arch = "wasm32")] @@ -281,6 +363,19 @@ pub struct Player { impl Player { /// Open the default output device. Never fails: without a device it runs a silent clock. pub fn new(session: Arc) -> Player { + let p = Player::open(Arc::clone(&session)); + // Hand the initial session's third-party plugins over. + p.update_session(session); + p + } + + fn side(&self) -> std::sync::MutexGuard<'_, Side> { + self.side.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn open(session: Arc) -> Player { + let (gtx, grx) = channel(); + let side = Mutex::new(Side { shadow: HashMap::new(), rate: session.sample_rate.as_f64() as f32, editors: HashMap::new(), garbage: grx }); let shared = Arc::new(Shared { position: AtomicI64::new(0), playing: AtomicBool::new(false), @@ -294,7 +389,7 @@ impl Player { Ok((device, config, name)) => { let rate = config.sample_rate.0; let channels = usize::from(config.channels); - let mut state = AudioState::new(rx, Arc::clone(&shared), session, f64::from(rate)); + let mut state = AudioState::new(rx, Arc::clone(&shared), session, f64::from(rate), gtx); let err_fn = |e: cpal::StreamError| log::warn!("audio stream error: {e}"); use cpal::traits::{DeviceTrait, StreamTrait}; match device.build_output_stream( @@ -311,6 +406,7 @@ impl Player { Player { shared, tx, + side, _stream: Some(stream), device_name: name, device_rate: rate, @@ -321,28 +417,45 @@ impl Player { Err(e) => { log::warn!("cannot open audio stream: {e}; using a silent clock"); let (tx2, rx2) = channel(); - // The state (and its receiver) moved into the failed closure; rebuild. - Player::silent_clock(shared, tx2, rx2, want) + let (gtx2, grx2) = channel(); + // The state (and its receivers) moved into the failed closure; rebuild. + let mut side = side; + side.get_mut().unwrap_or_else(PoisonError::into_inner).garbage = grx2; + Player::silent_clock(shared, tx2, rx2, want, side, gtx2) } } } Err(e) => { log::warn!("no audio output: {e}; using a silent clock"); let s2 = Arc::clone(&shared); - Player::silent_clock_with(s2, tx, rx, session) + Player::silent_clock_with(s2, tx, rx, session, side, gtx) } } } - fn silent_clock(shared: Arc, tx: Sender, rx: Receiver, rate: u32) -> Player { + fn silent_clock( + shared: Arc, + tx: Sender, + rx: Receiver, + rate: u32, + side: Mutex, + gtx: Sender>>, + ) -> Player { let s = Arc::new(Session::new("Untitled", soundcraft_time::SampleRate::new(rate).unwrap_or_default())); - Player::silent_clock_with(shared, tx, rx, s) + Player::silent_clock_with(shared, tx, rx, s, side, gtx) } #[cfg(not(target_arch = "wasm32"))] - fn silent_clock_with(shared: Arc, tx: Sender, rx: Receiver, session: Arc) -> Player { + fn silent_clock_with( + shared: Arc, + tx: Sender, + rx: Receiver, + session: Arc, + side: Mutex, + gtx: Sender>>, + ) -> Player { let rate = session.sample_rate.hz(); - let mut state = AudioState::new(rx, Arc::clone(&shared), session, f64::from(rate)); + let mut state = AudioState::new(rx, Arc::clone(&shared), session, f64::from(rate), gtx); let spawn = std::thread::Builder::new().name("soundcraft-clock".into()).spawn(move || { let mut buf = vec![0.0f32; 4096]; let mut last = std::time::Instant::now(); @@ -363,14 +476,31 @@ impl Player { if let Err(e) = spawn { log::warn!("clock thread failed: {e}"); } - Player { shared, tx, _stream: None, device_name: "No audio device (silent)".into(), device_rate: rate, device_channels: 0, silent: true } - } - - #[cfg(target_arch = "wasm32")] - fn silent_clock_with(shared: Arc, tx: Sender, _rx: Receiver, session: Arc) -> Player { Player { shared, tx, + side, + _stream: None, + device_name: "No audio device (silent)".into(), + device_rate: rate, + device_channels: 0, + silent: true, + } + } + + #[cfg(target_arch = "wasm32")] + fn silent_clock_with( + shared: Arc, + tx: Sender, + _rx: Receiver, + session: Arc, + side: Mutex, + _gtx: Sender>>, + ) -> Player { + Player { + shared, + tx, + side, _stream: None, device_name: "No audio device".into(), device_rate: session.sample_rate.hz(), @@ -379,8 +509,73 @@ impl Player { } } + /// Sends a new session snapshot to the audio thread, with freshly created (state-restored, + /// prepared) instances of any third-party plugin it newly needs. Call on the UI thread. pub fn update_session(&self, s: Arc) { - let _ = self.tx.send(Cmd::Session(s)); + let instances = self.side().prepare(&s); + let _ = self.tx.send(Cmd::Session(s, instances)); + } + + /// Reads the state of every live third-party plugin (for saving the session). Waits briefly + /// for the audio thread to answer between two blocks; empty when it does not answer. + pub fn capture_states(&self) -> PluginStates { + self.side().collect_garbage(); + #[cfg(not(target_arch = "wasm32"))] + { + let (tx, rx) = channel(); + if self.tx.send(Cmd::CaptureStates(tx)).is_err() { + return Vec::new(); + } + rx.recv_timeout(std::time::Duration::from_secs(2)).unwrap_or_default() + } + #[cfg(target_arch = "wasm32")] + { + Vec::new() + } + } + + /// Per-UI-frame housekeeping: drops retired instances and services plugin editors. Returns + /// the parameter edits made in open editors since the last call. + pub fn idle(&self) -> Vec { + let mut side = self.side(); + side.collect_garbage(); + let mut out = Vec::new(); + for (&(t, slot), e) in side.editors.iter_mut() { + for (id, v) in e.idle() { + out.push((t, slot, id, v)); + } + } + out + } + + /// True when the plugin in this slot has its own editor. + pub fn has_editor(&self, track: TrackId, slot: usize) -> bool { + self.side().editors.contains_key(&(track, slot)) + } + + /// Opens the plugin's own editor window. Main thread only. + pub fn open_editor(&self, track: TrackId, slot: usize) -> Result<(), String> { + match self.side().editors.get_mut(&(track, slot)) { + Some(e) => e.open(), + None => Err("this plugin has no editor (or it is not loaded)".into()), + } + } + + pub fn close_editor(&self, track: TrackId, slot: usize) { + if let Some(e) = self.side().editors.get_mut(&(track, slot)) { + e.close(); + } + } + + pub fn editor_open(&self, track: TrackId, slot: usize) -> bool { + self.side().editors.get(&(track, slot)).is_some_and(|e| e.is_open()) + } + + /// Shows a parameter change made elsewhere in the plugin's open editor. + pub fn editor_set_param(&self, track: TrackId, slot: usize, id: &str, value: f32) { + if let Some(e) = self.side().editors.get_mut(&(track, slot)) { + e.set_param(id, value); + } } /// Start playback at `from`, stopping at `end`, or looping `looped`. @@ -427,6 +622,44 @@ impl Player { } } +impl Side { + /// Creates the third-party instances `s` needs that the audio thread does not have yet. + fn prepare(&mut self, s: &Session) -> Vec { + self.collect_garbage(); + let sr = s.sample_rate.as_f64() as f32; + if (sr - self.rate).abs() > 0.5 { + // The audio thread rebuilds its engine: everything is created anew. + let keys: Vec<(TrackId, usize)> = self.shadow.keys().copied().collect(); + for k in keys { + self.forget(k); + } + self.rate = sr; + } + let specs = soundcraft_mix::instance_specs(s); + let stale: Vec<(TrackId, usize)> = self.shadow.keys().filter(|k| !specs.iter().any(|x| (x.track, x.slot) == **k)).copied().collect(); + for k in stale { + self.forget(k); + } + let mut out = Vec::new(); + for spec in specs { + let key = (spec.track, spec.slot); + if self.shadow.get(&key).is_some_and(|(id, ch, _)| *id == spec.id && *ch == spec.ch) { + continue; + } + self.forget(key); + let made = soundcraft_mix::create_instance(&spec.id, spec.state.as_deref(), sr, BLOCK, spec.ch); + self.shadow.insert(key, (spec.id.clone(), spec.ch, made.is_some())); + if let Some(mut plugin) = made { + if let Some(e) = plugin.editor() { + self.editors.insert(key, e); + } + out.push(PreparedInstance { track: spec.track, slot: spec.slot, id: spec.id, ch: spec.ch, plugin }); + } + } + out + } +} + /// ITU stereo fold-down gains for each main channel of a session. fn fold_gains(s: &Session) -> [(f32, f32); MAX_CHANNELS] { let mut g = [(0.0, 0.0); MAX_CHANNELS]; @@ -494,7 +727,7 @@ mod tests { meters: Mutex::new(MeterSnapshot::default()), }); let (tx, rx) = channel(); - let mut st = AudioState::new(rx, Arc::clone(&shared), s, 48_000.0); + let mut st = AudioState::new(rx, Arc::clone(&shared), s, 48_000.0, channel().0); tx.send(Cmd::Play { from: 0, end: Some(1000), looped: None }).unwrap(); let mut out = vec![0.0f32; 4096]; st.fill(&mut out, 2); @@ -520,7 +753,7 @@ mod tests { meters: Mutex::new(MeterSnapshot::default()), }); let (tx, rx) = channel(); - let mut st = AudioState::new(rx, Arc::clone(&shared), Arc::clone(&s), 48_000.0); + let mut st = AudioState::new(rx, Arc::clone(&shared), Arc::clone(&s), 48_000.0, channel().0); tx.send(Cmd::Play { from: 0, end: None, looped: None }).unwrap(); // An 8-channel device: channels in order, the rest silent. let mut out = vec![0.0f32; 8 * 256]; @@ -539,6 +772,99 @@ mod tests { assert_eq!(shared.meters.lock().unwrap().main.peaks.len(), 6); } + struct Stateful(Box); + + impl Plugin for Stateful { + fn info(&self) -> &'static soundcraft_dsp::PluginInfo { + self.0.info() + } + fn prepare(&mut self, sr: f32, mb: usize, ch: usize) { + self.0.prepare(sr, mb, ch); + } + fn reset(&mut self) { + self.0.reset(); + } + fn set_param(&mut self, id: &str, v: f32) -> bool { + self.0.set_param(id, v) + } + fn param(&self, id: &str) -> Option { + self.0.param(id) + } + fn process(&mut self, io: &mut [Vec], frames: usize) { + self.0.process(io, frames); + } + fn save_state(&mut self) -> Option> { + Some(b"blob".to_vec()) + } + } + + #[test] + fn instances_arrive_with_the_session_and_leave_through_the_return_channel() { + use soundcraft_model::{ChannelFormat, Insert, TrackKind}; + let mut s = soundcraft_model::Session::default(); + let t = s.add_track(TrackKind::Audio, ChannelFormat::Stereo, None); + s.track_mut(t).unwrap().mixer.inserts[1] = Some(Insert::new("clap:test.fake")); + let s = Arc::new(s); + let shared = Arc::new(Shared { + position: AtomicI64::new(0), + playing: AtomicBool::new(false), + speed: AtomicU32::new(1000), + meters: Mutex::new(MeterSnapshot::default()), + }); + let (tx, rx) = channel(); + let (gtx, grx) = channel(); + let mut st = AudioState::new(rx, Arc::clone(&shared), Arc::new(soundcraft_model::Session::default()), 48_000.0, gtx); + let plugin: Box = Box::new(Stateful(soundcraft_dsp::create("gain").unwrap())); + tx.send(Cmd::Session(Arc::clone(&s), vec![PreparedInstance { track: t, slot: 1, id: "clap:test.fake".into(), ch: 2, plugin }])).unwrap(); + let (rtx, rrx) = channel(); + tx.send(Cmd::CaptureStates(rtx)).unwrap(); + // Stopped: the callback only drains commands (adopting the instance) and answers. + let mut out = vec![0.0f32; 256]; + st.fill(&mut out, 2); + assert_eq!(rrx.try_recv().unwrap(), vec![(t, 1, b"blob".to_vec())]); + assert!(grx.try_recv().is_err(), "nothing retired yet"); + // A session without the insert retires the instance to the return channel. + tx.send(Cmd::Session(Arc::new(soundcraft_model::Session::default()), Vec::new())).unwrap(); + st.fill(&mut out, 2); + assert_eq!(grx.try_recv().unwrap().len(), 1); + // A sample-rate change rebuilds the engine without dropping plugins on this thread. + tx.send(Cmd::Session( + Arc::clone(&s), + vec![PreparedInstance { + track: t, + slot: 1, + id: "clap:test.fake".into(), + ch: 2, + plugin: Box::new(Stateful(soundcraft_dsp::create("gain").unwrap())), + }], + )) + .unwrap(); + st.fill(&mut out, 2); + let mut s44 = (*s).clone(); + s44.sample_rate = soundcraft_time::SampleRate::new(44_100).unwrap(); + tx.send(Cmd::Session(Arc::new(s44), Vec::new())).unwrap(); + st.fill(&mut out, 2); + assert_eq!(grx.try_recv().unwrap().len(), 1); + } + + #[test] + fn side_creates_each_missing_instance_once() { + use soundcraft_model::{ChannelFormat, Insert, TrackKind}; + let (_gtx, grx) = channel(); + let mut side = Side { shadow: HashMap::new(), rate: 48_000.0, editors: HashMap::new(), garbage: grx }; + let mut s = soundcraft_model::Session::default(); + let t = s.add_track(TrackKind::Audio, ChannelFormat::Mono, None); + s.track_mut(t).unwrap().mixer.inserts[0] = Some(Insert::new("clap:does.not.exist")); + s.track_mut(t).unwrap().mixer.inserts[1] = Some(Insert::new("gain")); + assert!(side.prepare(&s).is_empty(), "the plugin is not installed; built-ins stay with the engine"); + assert_eq!(side.shadow.get(&(t, 0)), Some(&("clap:does.not.exist".to_string(), 1, false))); + assert!(side.prepare(&s).is_empty()); + assert_eq!(side.shadow.len(), 1, "failures are remembered, not retried every update"); + s.track_mut(t).unwrap().mixer.inserts[0] = None; + side.prepare(&s); + assert!(side.shadow.is_empty()); + } + #[test] fn looping_wraps() { let s = Arc::new(soundcraft_model::Session::default()); @@ -549,7 +875,7 @@ mod tests { meters: Mutex::new(MeterSnapshot::default()), }); let (tx, rx) = channel(); - let mut st = AudioState::new(rx, Arc::clone(&shared), s, 48_000.0); + let mut st = AudioState::new(rx, Arc::clone(&shared), s, 48_000.0, channel().0); tx.send(Cmd::Play { from: 0, end: None, looped: Some(Range::new(0, 1000)) }).unwrap(); let mut out = vec![0.0f32; 20_000]; st.fill(&mut out, 2); diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index ca4e5f85..4b7c4c08 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -14,10 +14,12 @@ soundcraft-midi.workspace = true soundcraft-dsp.workspace = true soundcraft-clap-host.workspace = true soundcraft-vst3-host.workspace = true +soundcraft-au-host.workspace = true soundcraft-model.workspace = true soundcraft-mix.workspace = true soundcraft-engine.workspace = true soundcraft-playback.workspace = true +soundcraft-video.workspace = true egui.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/ui-egui/examples/parity_missing.rs b/crates/ui-egui/examples/parity_missing.rs new file mode 100644 index 00000000..4b100835 --- /dev/null +++ b/crates/ui-egui/examples/parity_missing.rs @@ -0,0 +1,6 @@ +//! Prints the app-level parity summary and the menu items not yet implemented. + +fn main() { + let v = soundcraft_ui_egui::menus::parity(); + println!("{}", serde_json::to_string_pretty(&v).unwrap_or_default()); +} diff --git a/crates/ui-egui/src/edit_window.rs b/crates/ui-egui/src/edit_window.rs index 4ef7b483..17a54b08 100644 --- a/crates/ui-egui/src/edit_window.rs +++ b/crates/ui-egui/src/edit_window.rs @@ -645,6 +645,7 @@ fn track_row(app: &mut SoundApp, ui: &mut Ui, id: TrackId, row: Rect, tl: Rect) } draw_clip(app, &painter, &s, &track, clip, tl, lane, auto_view.is_some()); } + crate::video_track::draw_lane(app, &painter, &s, &track, tl, lane); if let Some(p) = &auto_view { draw_automation(&painter, &s, &track, p, tl, lane); } @@ -829,6 +830,7 @@ fn track_header(app: &mut SoundApp, ui: &mut Ui, track: &Track, head: Rect, sele TrackKind::Instrument => "INST", TrackKind::Vca => "VCA", TrackKind::Folder => "FOLDER", + TrackKind::Video => "VIDEO", }; if !kind_label.is_empty() { ui.painter().text(pos2(pl_r.max.x + 4.0, name_r.center().y), Align2::LEFT_CENTER, kind_label, bold(9.0), t.text_dim); @@ -1174,6 +1176,7 @@ fn draw_clip(app: &SoundApp, painter: &egui::Painter, s: &Session, track: &Track painter.text(body_r.center(), Align2::CENTER_CENTER, "media offline", regular(10.0), Color32::from_rgb(220, 120, 120)); } } + ClipContent::Video { .. } => {} // thumbnails: video_track::draw_lane ClipContent::Midi { sequence } => { let (lo, hi) = sequence.notes.iter().fold((127u8, 0u8), |(lo, hi), n| (lo.min(n.pitch), hi.max(n.pitch))); let (lo, hi) = if lo > hi { (48, 72) } else { (lo.saturating_sub(2), hi.saturating_add(2)) }; diff --git a/crates/ui-egui/src/extra_windows.rs b/crates/ui-egui/src/extra_windows.rs new file mode 100644 index 00000000..7737a965 --- /dev/null +++ b/crates/ui-egui/src/extra_windows.rs @@ -0,0 +1,174 @@ +//! Window menu extras: the Renderer (speaker layout of the main output with live levels and the +//! object tracks' positions), UI Customization, and auto-updating the active window configuration. + +use crate::SoundApp; +use crate::theme::{Tokens, bold, regular}; +use egui::{Align2, Color32, Sense, Stroke, pos2, vec2}; +use serde_json::{Value, json}; +use soundcraft_model::{Speaker, TrackKind}; + +/// How often an auto-updating configuration is refreshed (seconds). +const AUTO_UPDATE_SECS: f64 = 2.0; + +#[derive(Debug, Clone)] +pub struct ExtraState { + pub show_renderer: bool, + pub show_ui_customization: bool, + pub auto_update_config: bool, + pub ui_scale: f32, + pub reduce_motion: bool, + pub tooltips: bool, + last_auto_update: f64, + applied: Option<(f32, bool)>, +} + +impl Default for ExtraState { + fn default() -> Self { + Self { + show_renderer: false, + show_ui_customization: false, + auto_update_config: false, + ui_scale: 1.0, + reduce_motion: false, + tooltips: true, + last_auto_update: 0.0, + applied: None, + } + } +} + +/// UI commands handled here (`window.renderer`, `window.ui_customization`, `window.config_auto_update`). +pub fn run(app: &mut SoundApp, id: &str, p: &Value) -> Option { + let flag = |v: &mut bool| { + *v = p.get("value").and_then(Value::as_bool).unwrap_or(!*v); + json!({"value": *v}) + }; + Some(match id { + "window.renderer" => flag(&mut app.extra.show_renderer), + "window.ui_customization" => flag(&mut app.extra.show_ui_customization), + "window.config_auto_update" => flag(&mut app.extra.auto_update_config), + _ => return None, + }) +} + +pub fn show(app: &mut SoundApp, ctx: &egui::Context) { + apply_customization(app, ctx); + auto_update(app, ctx); + renderer(app, ctx); + ui_customization(app, ctx); +} + +fn apply_customization(app: &mut SoundApp, ctx: &egui::Context) { + let want = (app.extra.ui_scale.clamp(0.75, 1.5), app.extra.reduce_motion); + if app.extra.applied == Some(want) { + return; + } + // Only touch the zoom once the user has changed it, so platform scaling stays untouched. + if app.extra.applied.is_some() || (want.0 - 1.0).abs() > f32::EPSILON { + ctx.set_zoom_factor(want.0); + } + ctx.global_style_mut(|s| s.animation_time = if want.1 { 0.0 } else { 1.0 / 12.0 }); + app.extra.applied = Some(want); +} + +fn auto_update(app: &mut SoundApp, ctx: &egui::Context) { + if !app.extra.auto_update_config || app.ui.configurations.is_empty() { + return; + } + let now = ctx.input(|i| i.time); + if now - app.extra.last_auto_update >= AUTO_UPDATE_SECS { + app.extra.last_auto_update = now; + let _ = crate::menus::run_ui_command(app, "window.config_update", &json!({})); + } +} + +fn renderer(app: &mut SoundApp, ctx: &egui::Context) { + let mut open = app.extra.show_renderer; + if !open { + return; + } + let t = Tokens::DARK; + let fmt = app.engine.session().main_format(); + let peaks: Vec = app.player.as_ref().map(|p| p.meters().main.peaks).unwrap_or_default(); + let objects: Vec<(String, f32, f32, f32)> = app + .engine + .session() + .tracks + .iter() + .filter(|tr| tr.kind != TrackKind::Master && app.engine.session().edit.flag(&format!("object.{}", tr.id.0))) + .map(|tr| { + let sp = tr.mixer.surround.unwrap_or_default(); + (tr.name.clone(), sp.x, sp.y, sp.z) + }) + .collect(); + egui::Window::new(egui::RichText::new("Renderer").font(bold(13.0))).open(&mut open).default_size(vec2(380.0, 440.0)).resizable(true).show( + ctx, + |ui| { + ui.label(egui::RichText::new(format!("Main output: {}", fmt.label())).font(regular(11.0)).color(t.text)); + let side = ui.available_width().min(360.0); + let (r, _) = ui.allocate_exact_size(vec2(side, side), Sense::hover()); + let p = ui.painter_at(r); + p.rect_filled(r, 6.0, Color32::from_rgb(16, 19, 23)); + let c = r.center(); + let rad = side * 0.40; + p.circle_stroke(c, rad, Stroke::new(1.0, Color32::from_rgb(60, 68, 78))); + p.circle_stroke(c, rad * 0.55, Stroke::new(1.0, Color32::from_rgb(44, 50, 58))); + p.text(pos2(c.x, r.min.y + 8.0), Align2::CENTER_TOP, "front", regular(9.0), t.text_dim); + for (i, sp) in fmt.speakers().iter().enumerate() { + let level = peaks.get(i).copied().unwrap_or(0.0).clamp(0.0, 1.0); + let ring = if sp.is_height() { rad * 0.55 } else { rad }; + let a = sp.azimuth().to_radians(); + let pos = if *sp == Speaker::Lfe { pos2(c.x, c.y + rad * 0.2) } else { pos2(c.x + ring * a.sin(), c.y - ring * a.cos()) }; + let glow = Color32::from_rgb(40 + (level * 60.0) as u8, 90 + (level * 140.0) as u8, 70 + (level * 40.0) as u8); + p.circle_filled(pos, 9.0 + level * 6.0, glow); + p.circle_stroke(pos, 9.0, Stroke::new(1.0, Color32::from_rgb(150, 160, 170))); + p.text(pos, Align2::CENTER_CENTER, format!("{sp:?}"), bold(8.0), Color32::WHITE); + } + for (name, x, y, z) in &objects { + let pos = pos2(c.x + x.clamp(-1.0, 1.0) * rad, c.y - y.clamp(-1.0, 1.0) * rad); + p.circle_filled(pos, 5.0 + z.clamp(0.0, 1.0) * 3.0, Color32::from_rgb(240, 180, 60)); + p.text(pos2(pos.x + 8.0, pos.y), Align2::LEFT_CENTER, name, regular(9.0), Color32::from_rgb(240, 200, 120)); + } + ui.add_space(4.0); + if objects.is_empty() { + ui.label( + egui::RichText::new("No object tracks. Use the Object view in the Mix window to route tracks as objects.") + .small() + .color(t.text_dim), + ); + } else { + ui.label(egui::RichText::new(format!("{} object track(s)", objects.len())).small().color(t.text_dim)); + } + }, + ); + app.extra.show_renderer = open; +} + +fn ui_customization(app: &mut SoundApp, ctx: &egui::Context) { + let mut open = app.extra.show_ui_customization; + if !open { + return; + } + egui::Window::new(egui::RichText::new("UI Customization").font(bold(13.0))) + .open(&mut open) + .default_size(vec2(320.0, 180.0)) + .resizable(false) + .show(ctx, |ui| { + egui::Grid::new("ui_custom").num_columns(2).spacing(vec2(12.0, 8.0)).show(ui, |ui| { + ui.label("Interface scale"); + ui.add(egui::Slider::new(&mut app.extra.ui_scale, 0.75..=1.5).step_by(0.05).custom_formatter(|v, _| format!("{:.0} %", v * 100.0))); + ui.end_row(); + ui.label("Reduce motion"); + ui.checkbox(&mut app.extra.reduce_motion, ""); + ui.end_row(); + ui.label("Auto-update configuration"); + ui.checkbox(&mut app.extra.auto_update_config, ""); + ui.end_row(); + }); + if ui.button("Reset").clicked() { + app.extra.ui_scale = 1.0; + app.extra.reduce_motion = false; + } + }); + app.extra.show_ui_customization = open; +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f8649378..3be7de39 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -7,6 +7,7 @@ pub mod control; pub mod dialogs; pub mod edit_window; +pub mod extra_windows; pub mod fonts; pub mod icons; pub mod menus; @@ -19,6 +20,8 @@ pub mod score_editor; pub mod shortcuts; pub mod theme; pub mod toolbar; +pub mod video_track; +pub mod video_window; pub mod widgets; pub mod windows; @@ -84,6 +87,10 @@ pub struct UiState { pub show_rtp: bool, pub show_score: bool, pub show_search: bool, + /// Window › Video, Window › Video Universe, timecode burn-in in the Video window. + pub show_video: bool, + pub show_video_universe: bool, + pub video_burn_in: bool, pub workspace_dir: String, pub configurations: Vec<(String, Value)>, } @@ -129,6 +136,9 @@ impl Default for UiState { show_rtp: false, show_score: false, show_search: false, + show_video: false, + show_video_universe: false, + video_burn_in: false, workspace_dir: String::new(), configurations: Vec::new(), } @@ -198,6 +208,7 @@ pub struct SoundApp { pub edit_layout: edit_window::EditLayout, pub midi: midi_editor::MidiEditorState, pub ops: ops_windows::OpsState, + pub extra: extra_windows::ExtraState, pub palette: palette::PaletteState, pub synthetic: Vec, synthetic_grace: u32, @@ -216,6 +227,8 @@ pub struct SoundApp { last_write_at: Samples, pub frame_ms: f32, last_frame: Option, + /// Movies for the Video window and Video track (decoded in the background). + pub video: video_track::VideoPool, } impl SoundApp { @@ -239,6 +252,7 @@ impl SoundApp { edit_layout: edit_window::EditLayout::default(), midi: midi_editor::MidiEditorState::default(), ops: ops_windows::OpsState::default(), + extra: extra_windows::ExtraState::default(), palette: palette::PaletteState::default(), synthetic: Vec::new(), synthetic_grace: 0, @@ -254,6 +268,7 @@ impl SoundApp { last_write_at: i64::MIN, frame_ms: 0.0, last_frame: None, + video: video_track::VideoPool::default(), } } @@ -267,6 +282,10 @@ impl SoundApp { if let Some(r) = menus::run_ui_command(self, id, ¶ms) { return r; } + if id.starts_with("session.save") || id.starts_with("file.bounce") { + // Third-party plugins keep state the session can't see: store it first. + self.capture_plugin_states(); + } match self.engine.execute(id, ¶ms) { Ok(v) => { if id == "app.quit" { @@ -362,6 +381,31 @@ impl SoundApp { } } + /// Reads every live third-party plugin's state from the audio engine and stores it on its + /// insert (`mix.insert_state`), so saves and bounces include it. + pub fn capture_plugin_states(&mut self) { + let Some(p) = &self.player else { return }; + for (t, slot, data) in p.capture_states() { + let slot = if slot == soundcraft_mix::INSTRUMENT_SLOT { json!("instrument") } else { json!(slot) }; + let state = soundcraft_model::b64::encode(&data); + if let Err(e) = self.engine.execute("mix.insert_state", &json!({"track": t.0, "slot": slot, "state": state})) { + log::warn!("storing plugin state: {e}"); + } + } + } + + /// Parameter edits made in plugin editors go into the session (one undo step per gesture). + fn apply_editor_edits(&mut self) { + let Some(p) = &self.player else { return }; + for (t, slot, param, value) in p.idle() { + if slot == soundcraft_mix::INSTRUMENT_SLOT { + continue; + } + let key = format!("param:{}:{slot}:{param}", t.0); + let _ = self.engine.execute_merged("mix.insert_param", &json!({"track": t.0, "slot": slot, "param": param, "value": value}), &key); + } + } + /// Every two minutes, copy a modified session to the autosave folder (not while recording). fn autosave(&mut self, now: f64) { let Some(dir) = self.autosave_dir.clone() else { return }; @@ -372,6 +416,7 @@ impl SoundApp { return; } self.last_autosave = now; + self.capture_plugin_states(); let name = soundcraft_engine::io::sanitize_name(&self.engine.session().name); let path = dir.join(&name).join(format!("{name}.scraft")); if let Some(parent) = path.parent() { @@ -645,6 +690,7 @@ impl SoundApp { self.last_frame = Some(now); self.frame_ms = self.frame_ms * 0.9 + dt * 1000.0 * 0.1; control::drain(self, ctx); + self.apply_editor_edits(); if !ctx.input(|i| i.pointer.any_down()) { if matches!(self.gesture, Some(Gesture::Fader { .. })) { self.gesture = None; @@ -727,7 +773,9 @@ impl SoundApp { } panels::floating(self, &ctx); windows::show(self, &ctx); + video_window::show(self, &ctx); ops_windows::show(self, &ctx); + extra_windows::show(self, &ctx); score_editor::show(self, &ctx); palette::show(self, &ctx); dialogs::show(self, &ctx); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index dc99dbe5..949d29f3 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -24,6 +24,8 @@ pub const UI_COMMANDS: &[(&str, &str, &str, Option<&str>)] = &[ ("window.hide_floating", "Hide All Floating Windows", "Window > Hide All Floating Windows", Some("Cmd+Ctrl+W")), ("window.midi_editor", "MIDI Editor", "Window > MIDI Editor", None), ("window.universe", "Universe", "View > Other Displays > Universe", None), + ("window.video", "Video", "Window > Video", Some("Cmd+9")), + ("window.video_universe", "Video Universe", "Window > Video Universe", None), ("window.automation", "Automation", "Window > Automation", Some("Cmd+4")), ("window.color_palette", "Color Palette", "Window > Color Palette", None), ("window.disk_usage", "Disk Usage", "Window > Disk Usage", None), @@ -40,6 +42,9 @@ pub const UI_COMMANDS: &[(&str, &str, &str, Option<&str>)] = &[ ("window.configurations", "Window Configuration List", "Window > Configurations > Window Configuration List", None), ("window.config_new", "New Configuration", "Window > Configurations > New Configuration...", None), ("window.config_update", "Update Active Configuration", "Window > Configurations > Update Active Configuration", None), + ("window.config_auto_update", "Auto-Update Active Configuration", "Window > Configurations > Auto-Update Active Configuration", None), + ("window.renderer", "Renderer", "Window > Renderer", None), + ("window.ui_customization", "UI Customization", "Window > UI Customization", None), ("window.arrange_tile", "Tile", "Window > Arrange > Tile", None), ("window.arrange_tile_h", "Tile Horizontal", "Window > Arrange > Tile Horizontal", None), ("window.arrange_tile_v", "Tile Vertical", "Window > Arrange > Tile Vertical", None), @@ -267,6 +272,9 @@ fn checked_state(app: &SoundApp, path: &str, id: Option<&str>) -> bool { Some("window.track_list") => app.ui.show_tracks_list, Some("window.clip_list_view") | Some("window.clip_list") => app.ui.show_clip_list, Some("window.narrow_mix") => app.ui.narrow_mix, + Some("window.renderer") => app.extra.show_renderer, + Some("window.ui_customization") => app.extra.show_ui_customization, + Some("window.config_auto_update") => app.extra.auto_update_config, Some("view.mix_section") => path.rsplit(" > ").next().map(mix_section_id).is_some_and(|s| app.ui.mix_views.iter().any(|v| v == s)), _ => false, } @@ -374,6 +382,9 @@ fn wants_dialog(id: &str) -> bool { /// Handle UI-layer commands. Returns None when `id` is not a UI command. pub fn run_ui_command(app: &mut SoundApp, id: &str, p: &Value) -> Option> { + if let Some(v) = crate::extra_windows::run(app, id, p) { + return Some(Ok(v)); + } let toggle = |v: &mut bool| { *v = p.get("value").and_then(Value::as_bool).unwrap_or(!*v); json!({"value": *v}) @@ -452,6 +463,8 @@ pub fn run_ui_command(app: &mut SoundApp, id: &str, p: &Value) -> Option toggle(&mut app.ui.show_universe), + "window.video" => toggle(&mut app.ui.show_video), + "window.video_universe" => toggle(&mut app.ui.show_video_universe), "window.close" | "window.hide_floating" => { app.ui.show_transport = false; app.ui.show_big_counter = false; diff --git a/crates/ui-egui/src/mix_window.rs b/crates/ui-egui/src/mix_window.rs index a65a2e51..f7a9aace 100644 --- a/crates/ui-egui/src/mix_window.rs +++ b/crates/ui-egui/src/mix_window.rs @@ -23,7 +23,7 @@ pub fn show(app: &mut SoundApp, ui: &mut Ui) { .show(ui, |ui| panels::tracks_and_groups(app, ui)); } egui::CentralPanel::default().frame(egui::Frame::NONE.fill(t.window_bg)).show(ui, |ui| { - let ids: Vec = app.engine.session().tracks.iter().filter(|x| !x.hidden).map(|x| x.id).collect(); + let ids: Vec = app.engine.session().tracks.iter().filter(|x| !x.hidden && x.kind != TrackKind::Video).map(|x| x.id).collect(); // Per-channel peaks for multichannel meters (read once per frame). let snap = app .player @@ -376,6 +376,7 @@ fn strip(app: &mut SoundApp, ui: &mut Ui, id: TrackId, snap: Option<&MeterSnapsh TrackKind::Instrument => "inst", TrackKind::Vca => "vca", TrackKind::Folder => "folder", + TrackKind::Video => "video", }; ui.painter().text(pos2(vr.center().x, vr.max.y + 9.0), Align2::CENTER_CENTER, kind, regular(9.5), t.text_dim); let nr = Rect::from_min_size(pos2(x0, r.max.y - 26.0), vec2(inner_w, 18.0)); @@ -480,11 +481,29 @@ pub fn plugin_menu(app: &mut SoundApp, ui: &mut Ui, id: TrackId, slot: usize, oc } } }); + // Audio Units (macOS; scanned once, cached by soundcraft-au-host). + if cfg!(target_os = "macos") { + ui.menu_button("Audio Units", |ui| { + let found: Vec<_> = soundcraft_au_host::scan().into_iter().filter(|d| !d.is_instrument).collect(); + if found.is_empty() { + ui.label("No Audio Units found"); + } + for d in found { + if ui.button(format!("{} ({})", d.name, d.vendor)).clicked() { + let _ = app.run("mix.insert", json!({"track": id.0, "slot": slot, "plugin": d.id})); + } + } + }); + } } -/// A built-in plugin's description, else a hosted CLAP (`clap:`) or VST3 (`vst3:`) plugin's. +/// A built-in plugin's description, else a hosted CLAP (`clap:`), VST3 (`vst3:`) or +/// Audio Unit (`au:::`) plugin's. pub fn plugin_info(id: &str) -> Option<&'static soundcraft_dsp::PluginInfo> { - soundcraft_dsp::plugin_info(id).or_else(|| soundcraft_clap_host::plugin_info(id)).or_else(|| soundcraft_vst3_host::plugin_info(id)) + soundcraft_dsp::plugin_info(id) + .or_else(|| soundcraft_clap_host::plugin_info(id)) + .or_else(|| soundcraft_vst3_host::plugin_info(id)) + .or_else(|| soundcraft_au_host::plugin_info(id)) } fn send_slot(app: &mut SoundApp, ui: &mut Ui, track: &Track, slot: usize, r: Rect) { diff --git a/crates/ui-egui/src/panels.rs b/crates/ui-egui/src/panels.rs index 4dade8fa..eff4e512 100644 --- a/crates/ui-egui/src/panels.rs +++ b/crates/ui-egui/src/panels.rs @@ -393,6 +393,23 @@ fn plugin_windows(app: &mut SoundApp, ctx: &egui::Context) { let _ = app.run("mix.insert_bypass", json!({"track": tid.0, "slot": slot})); } ui.menu_button("Presets ▾", |ui| preset_menu(app, ui, tid, slot, info, &ins)); + if soundcraft_mix::is_third_party(&ins.plugin) { + let open = app.player.as_ref().is_some_and(|p| p.editor_open(tid, slot)); + let available = app.player.as_ref().is_some_and(|p| p.has_editor(tid, slot)); + let label = if open { "Close Plugin Editor" } else { "Open Plugin Editor" }; + let resp = ui.add_enabled(available, egui::Button::new(label)); + let resp = + if available { resp } else { resp.on_disabled_hover_text("This plugin has no editor of its own, or it is not loaded") }; + if resp.clicked() + && let Some(p) = &app.player + { + if open { + p.close_editor(tid, slot); + } else if let Err(e) = p.open_editor(tid, slot) { + app.ui.status = format!("{}: {e}", info.name); + } + } + } }); if info.id == "eq_7band" || info.id == "eq_1band" { eq_curve(ui, info.id, &ins); @@ -426,6 +443,10 @@ fn plugin_windows(app: &mut SoundApp, ctx: &egui::Context) { &json!({"track": tid.0, "slot": slot, "param": p.id, "value": x}), &format!("param:{}:{slot}:{}", tid.0, p.id), ); + if let Some(pl) = &app.player { + // Keep the plugin's own editor in step. + pl.editor_set_param(tid, slot, p.id, x); + } } let _ = resp; ui.label(egui::RichText::new(p.format(x)).font(mono(11.0))); diff --git a/crates/ui-egui/src/video_track.rs b/crates/ui-egui/src/video_track.rs new file mode 100644 index 00000000..fdf5d87b --- /dev/null +++ b/crates/ui-egui/src/video_track.rs @@ -0,0 +1,487 @@ +//! Video track: the movie pool (opened from `Session::videos` paths, decoded off the UI thread) +//! and thumbnails drawn along video clips in the Edit window. +//! +//! Decoding runs on one background thread on native targets: the playhead frame request always +//! replaces the previous one (latest wins), thumbnails queue behind it. On the web there are no +//! threads, so the same worker runs inline on the UI thread with a per-frame budget. + +use crate::SoundApp; +use crate::edit_window::{sample_at, x_of}; +use crate::theme::regular; +use egui::{Color32, Painter, Rect, pos2}; +use soundcraft_model::{ClipContent, Session, SourceId, Track}; +use soundcraft_time::Samples; +use soundcraft_video::{Frame, Movie, MovieInfo}; +use std::collections::HashMap; +use std::sync::Arc; + +/// `edit.flags` key set while the Video track is offline (Options › Video Track Online). +pub const OFFLINE_FLAG: &str = "video.offline"; +/// Most thumbnail textures kept; beyond that the cache is cleared and refilled on demand. +const MAX_THUMBS: usize = 600; + +/// Identity of a movie: session source id plus a hash of its path (ids repeat across sessions). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MovieKey(pub SourceId, pub u64); + +impl MovieKey { + pub fn new(id: SourceId, path: &str) -> MovieKey { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + path.hash(&mut h); + MovieKey(id, h.finish()) + } +} + +/// A thumbnail request key: movie, time (ms) and pixel height. +type ThumbKey = (MovieKey, i64, u32); + +enum Job { + Frame { key: MovieKey, path: String, secs: f64 }, + Thumb { key: MovieKey, path: String, secs: f64, h: u32, tk: ThumbKey }, +} + +enum Done { + Info(MovieKey, Result), + Frame(MovieKey, f64, Result, String>), + Thumb(ThumbKey, Result, String>), +} + +/// Owns the open movies; runs jobs (on the worker thread, or inline on wasm). +#[derive(Default)] +struct Worker { + movies: HashMap>, +} + +impl Worker { + fn open(&mut self, key: MovieKey, path: &str, out: &mut Vec) -> Option<&mut Movie> { + if !self.movies.contains_key(&key) { + let m = open_movie(path); + out.push(Done::Info(key, m.as_ref().map(|m| m.info().clone()).map_err(Clone::clone))); + // Keep a handful of movies open. + if self.movies.len() >= 8 { + self.movies.clear(); + } + self.movies.insert(key, m); + } + self.movies.get_mut(&key).and_then(|m| m.as_mut().ok()) + } + + fn run(&mut self, job: Job) -> Vec { + let mut out = Vec::new(); + match job { + Job::Frame { key, path, secs } => { + let r = match self.open(key, &path, &mut out) { + Some(m) => m.frame_at(secs).map_err(|e| e.to_string()), + None => Err("movie offline".to_string()), + }; + out.push(Done::Frame(key, secs, r)); + } + Job::Thumb { key, path, secs, h, tk } => { + let r = match self.open(key, &path, &mut out) { + Some(m) => m.thumbnail_at(secs, h).map_err(|e| e.to_string()), + None => Err("movie offline".to_string()), + }; + out.push(Done::Thumb(tk, r)); + } + } + out + } +} + +fn open_movie(path: &str) -> Result { + #[cfg(not(target_arch = "wasm32"))] + { + let f = std::fs::File::open(path).map_err(|e| format!("{path}: {e}"))?; + Movie::open_source(Arc::new(f)).map_err(|e| e.to_string()) + } + #[cfg(target_arch = "wasm32")] + { + let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?; + Movie::open(bytes).map_err(|e| e.to_string()) + } +} + +#[cfg(not(target_arch = "wasm32"))] +mod backend { + use super::{Done, Job, Worker}; + use std::collections::VecDeque; + use std::sync::mpsc::{Receiver, Sender, channel}; + use std::sync::{Arc, Condvar, Mutex}; + + #[derive(Default)] + struct Queue { + frame: Option, + thumbs: VecDeque, + quit: bool, + } + + /// The decoding thread and its job queue. + pub struct Backend { + shared: Arc<(Mutex, Condvar)>, + rx: Receiver, + } + + impl Backend { + pub fn new(ctx: egui::Context) -> Option { + let shared = Arc::new((Mutex::new(Queue::default()), Condvar::new())); + let (tx, rx) = channel(); + let s2 = Arc::clone(&shared); + std::thread::Builder::new().name("video-decode".into()).spawn(move || run(&s2, &tx, &ctx)).ok()?; + Some(Backend { shared, rx }) + } + + pub fn submit(&mut self, job: Job) { + let (lock, cv) = &*self.shared; + if let Ok(mut q) = lock.lock() { + match job { + Job::Frame { .. } => q.frame = Some(job), + Job::Thumb { .. } => { + q.thumbs.push_back(job); + // Stale requests (scrolled away) are dropped; their slots are freed so + // they can be asked again. + while q.thumbs.len() > 96 { + q.thumbs.pop_front(); + } + } + } + cv.notify_one(); + } + } + + /// Finished jobs. + pub fn poll(&mut self) -> Vec { + self.rx.try_iter().collect() + } + + /// Thumbnail jobs still waiting (pending slots not listed here were dropped or are running). + pub fn queued_thumbs(&self) -> Vec { + let (lock, _) = &*self.shared; + lock.lock() + .map(|q| q.thumbs.iter().filter_map(|j| if let Job::Thumb { tk, .. } = j { Some(*tk) } else { None }).collect()) + .unwrap_or_default() + } + } + + impl Drop for Backend { + fn drop(&mut self) { + let (lock, cv) = &*self.shared; + if let Ok(mut q) = lock.lock() { + q.quit = true; + } + cv.notify_all(); + } + } + + fn run(shared: &(Mutex, Condvar), tx: &Sender, ctx: &egui::Context) { + let mut worker = Worker::default(); + loop { + let job = { + let (lock, cv) = shared; + let Ok(mut q) = lock.lock() else { return }; + while q.frame.is_none() && q.thumbs.is_empty() && !q.quit { + q = match cv.wait(q) { + Ok(q) => q, + Err(_) => return, + }; + } + if q.quit { + return; + } + match q.frame.take() { + Some(j) => j, + None => match q.thumbs.pop_back() { + // Newest first: what is on screen now. + Some(j) => j, + None => continue, + }, + } + }; + // A decoder bug must not take the app down: contain it to this job. + let done = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| worker.run(job))); + match done { + Ok(done) => { + for d in done { + if tx.send(d).is_err() { + return; + } + } + } + Err(_) => worker = Worker::default(), + } + ctx.request_repaint(); + } + } +} + +#[cfg(target_arch = "wasm32")] +mod backend { + use super::{Done, Job, Worker}; + use std::collections::VecDeque; + + /// Inline decoding with a per-frame budget (no threads on the web). + pub struct Backend { + worker: Worker, + frame: Option, + thumbs: VecDeque, + ctx: egui::Context, + } + + impl Backend { + pub fn new(ctx: egui::Context) -> Option { + Some(Backend { worker: Worker::default(), frame: None, thumbs: VecDeque::new(), ctx }) + } + pub fn submit(&mut self, job: Job) { + match job { + Job::Frame { .. } => self.frame = Some(job), + Job::Thumb { .. } => { + self.thumbs.push_back(job); + while self.thumbs.len() > 96 { + self.thumbs.pop_front(); + } + } + } + } + /// Run at most one frame and two thumbnails per UI frame. + pub fn poll(&mut self) -> Vec { + let mut out = Vec::new(); + if let Some(j) = self.frame.take() { + out.extend(self.worker.run(j)); + } + for _ in 0..2 { + let Some(j) = self.thumbs.pop_back() else { break }; + out.extend(self.worker.run(j)); + } + if !self.thumbs.is_empty() { + self.ctx.request_repaint(); + } + out + } + pub fn queued_thumbs(&self) -> Vec { + self.thumbs.iter().filter_map(|j| if let Job::Thumb { tk, .. } = j { Some(*tk) } else { None }).collect() + } + } +} + +enum Slot { + /// Requested at this UI time (seconds). + Pending(f64), + Ready(egui::TextureHandle), + Failed, +} + +/// The UI's movie cache: decoded pictures as textures (never serialised; rebuilt from paths). +#[derive(Default)] +pub struct VideoPool { + backend: Option, + backend_failed: bool, + infos: HashMap>, + thumbs: HashMap, + /// The latest playhead picture: movie, its frame index, texture. + current: Option<(MovieKey, usize, egui::TextureHandle)>, + current_err: Option<(MovieKey, String)>, + /// Last frame request (to avoid re-sending identical requests). + last_req: Option<(MovieKey, i64)>, + /// Pictures shown in the Video window since start (diagnostics). + pub frames_shown: u64, +} + +impl VideoPool { + fn backend(&mut self, ctx: &egui::Context) -> Option<&mut backend::Backend> { + if self.backend.is_none() && !self.backend_failed { + self.backend = backend::Backend::new(ctx.clone()); + self.backend_failed = self.backend.is_none(); + } + self.backend.as_mut() + } + + /// Collect finished work (call once per frame before drawing). + pub fn poll(&mut self, ctx: &egui::Context) { + let Some(b) = self.backend.as_mut() else { return }; + let done = b.poll(); + let queued: std::collections::HashSet = b.queued_thumbs().into_iter().collect(); + for d in done { + match d { + Done::Info(k, info) => { + self.infos.insert(k, info); + } + Done::Frame(k, _secs, r) => match r { + Ok(f) => { + let img = color_image(&f); + match &mut self.current { + Some((ck, ci, tex)) => { + tex.set(img, egui::TextureOptions::LINEAR); + *ck = k; + *ci = f.index; + } + None => self.current = Some((k, f.index, ctx.load_texture("video-frame", img, egui::TextureOptions::LINEAR))), + } + self.current_err = None; + self.frames_shown += 1; + } + Err(e) => self.current_err = Some((k, e)), + }, + Done::Thumb(tk, r) => { + let slot = match r { + Ok(f) => Slot::Ready(ctx.load_texture(format!("video-thumb-{tk:?}"), color_image(&f), egui::TextureOptions::LINEAR)), + Err(_) => Slot::Failed, + }; + self.thumbs.insert(tk, slot); + } + } + } + // Pending slots whose job fell off the queue (scrolled away) may be asked for again; the + // one being decoded right now is not queued either, so give it a moment. + let now = ctx.input(|i| i.time); + self.thumbs.retain(|k, s| !matches!(s, Slot::Pending(t) if now - *t > 2.0 && !queued.contains(k))); + } + + pub fn info(&self, key: MovieKey) -> Option<&Result> { + self.infos.get(&key) + } + + /// Ask for the picture at `secs` of the movie (the Video window). + pub fn request_frame(&mut self, ctx: &egui::Context, key: MovieKey, path: &str, secs: f64, fps: f64) { + let q = (secs * fps.max(1.0) * 4.0).floor() as i64; + if self.last_req == Some((key, q)) { + return; + } + self.last_req = Some((key, q)); + let path = path.to_string(); + if let Some(b) = self.backend(ctx) { + b.submit(Job::Frame { key, path, secs }); + } + } + + /// The current playhead picture for `key`, if one has been decoded. + pub fn current(&self, key: MovieKey) -> Option<&egui::TextureHandle> { + self.current.as_ref().filter(|c| c.0 == key).map(|c| &c.2) + } + + pub fn current_error(&self, key: MovieKey) -> Option<&str> { + self.current_err.as_ref().filter(|c| c.0 == key).map(|c| c.1.as_str()) + } + + /// A thumbnail at `secs`, `h` pixels high; requested in the background when missing. + pub fn thumb(&mut self, ctx: &egui::Context, key: MovieKey, path: &str, secs: f64, h: u32) -> Option { + let tk = (key, (secs * 1000.0).round() as i64, h); + match self.thumbs.get(&tk) { + Some(Slot::Ready(t)) => return Some(t.clone()), + Some(Slot::Pending(_) | Slot::Failed) => return None, + None => {} + } + if self.thumbs.len() >= MAX_THUMBS { + self.thumbs.retain(|_, s| matches!(s, Slot::Pending(_))); + } + self.thumbs.insert(tk, Slot::Pending(ctx.input(|i| i.time))); + let path = path.to_string(); + if let Some(b) = self.backend(ctx) { + b.submit(Job::Thumb { key, path, secs, h, tk }); + } + None + } +} + +fn color_image(f: &Frame) -> egui::ColorImage { + let (w, h) = (f.width as usize, f.height as usize); + if f.rgba.len() == w * h * 4 { + egui::ColorImage::from_rgba_unmultiplied([w, h], &f.rgba) + } else { + egui::ColorImage::new([1, 1], vec![Color32::BLACK]) + } +} + +/// True unless Options › Video Track Online is off. +pub fn online(s: &Session) -> bool { + !s.edit.flag(OFFLINE_FLAG) +} + +/// Movie time (seconds) shown at timeline position `at` by a video clip, including the session's +/// Video Sync Offset (positive delays the picture). +pub fn movie_secs(s: &Session, clip_start: Samples, offset: Samples, at: Samples) -> f64 { + let sync = s.edit.value("video.sync_offset", 0.0); + let sync = if sync.is_finite() { sync } else { 0.0 }; + (at.saturating_sub(clip_start).saturating_add(offset) as f64 - sync) / s.sample_rate.as_f64().max(1.0) +} + +/// The picture at `at`: the topmost active Video track with an unmuted clip there. +pub fn picture_at(s: &Session, at: Samples) -> Option<(MovieKey, String, f64, f64)> { + for t in s.tracks.iter().filter(|t| t.kind == soundcraft_model::TrackKind::Video && !t.inactive) { + for c in t.clips() { + if c.muted || !c.range().contains(at) { + continue; + } + if let ClipContent::Video { source, offset } = c.content + && let Some(v) = s.video(source) + { + return Some((MovieKey::new(source, &v.path), v.path.clone(), movie_secs(s, c.start, offset, at), v.frame_rate)); + } + } + } + None +} + +/// Draw thumbnails inside the video clips of `track` (called from the Edit window's track row, +/// after the clip blocks are painted). +pub fn draw_lane(app: &mut SoundApp, painter: &Painter, s: &Session, track: &Track, tl: Rect, lane: Rect) { + if !track.clips().iter().any(|c| c.is_video()) { + return; + } + let ctx = painter.ctx().clone(); + app.video.poll(&ctx); + let on = online(s); + let name_h = if lane.height() >= 40.0 && s.edit.flag("view.clip.name") { 13.0 } else { 0.0 }; + let ppp = ctx.pixels_per_point(); + for clip in track.clips() { + let ClipContent::Video { source, offset } = clip.content else { continue }; + let x0 = x_of(s, tl, clip.start); + let x1 = x_of(s, tl, clip.end()).max(x0 + 1.0); + let body = Rect::from_min_max(pos2(x0 + 1.0, lane.min.y + 2.0 + name_h), pos2(x1 - 1.0, lane.max.y - 2.0)); + let vis = body.intersect(lane); + if vis.width() < 2.0 || vis.height() < 6.0 { + continue; + } + let p = painter.with_clip_rect(vis); + let Some(v) = s.video(source) else { + p.text(vis.center(), egui::Align2::CENTER_CENTER, "movie missing from session", regular(10.0), Color32::from_rgb(220, 120, 120)); + continue; + }; + if !on { + p.text(vis.center(), egui::Align2::CENTER_CENTER, "Video Track Offline", regular(10.0), Color32::from_gray(170)); + continue; + } + let key = MovieKey::new(source, &v.path); + if let Some(Err(e)) = app.video.info(key) { + p.text(vis.center(), egui::Align2::CENTER_CENTER, format!("media offline: {e}"), regular(10.0), Color32::from_rgb(220, 120, 120)); + continue; + } + let aspect = app.video.info(key).and_then(|i| i.as_ref().ok()).map_or(v.aspect(), MovieInfo::display_aspect) as f32; + let th = body.height(); + let tw = (th * aspect).clamp(8.0, 600.0); + // Pixel height bucket (multiples of 16) so zooming the track does not re-decode constantly. + let hpx = (((th * ppp) / 16.0).ceil() * 16.0).clamp(16.0, 256.0) as u32; + let first = ((vis.min.x - body.min.x) / tw).floor().max(0.0) as i64; + let mut k = first; + loop { + let tx = body.min.x + k as f32 * tw; + if tx > vis.max.x || k > first + 400 { + break; + } + let at = sample_at(s, tl, tx); + let secs = movie_secs(s, clip.start, offset, at.max(clip.start)); + let r = Rect::from_min_max(pos2(tx, body.min.y), pos2((tx + tw).min(body.max.x), body.max.y)); + match app.video.thumb(&ctx, key, &v.path, secs.max(0.0), hpx) { + Some(tex) => { + let full = Rect::from_min_size(r.min, egui::vec2(tw, th)); + let uv = Rect::from_min_max(pos2(0.0, 0.0), pos2((r.width() / tw).clamp(0.0, 1.0), 1.0)); + p.image(tex.id(), Rect::from_min_max(full.min, pos2(r.max.x, full.max.y)), uv, Color32::WHITE); + } + None => { + p.rect_filled(r.shrink(1.0), 0.0, Color32::from_rgb(24, 24, 26)); + } + } + p.line_segment([pos2(tx, body.min.y), pos2(tx, body.max.y)], egui::Stroke::new(1.0, Color32::from_black_alpha(160))); + k += 1; + } + } +} diff --git a/crates/ui-egui/src/video_window.rs b/crates/ui-egui/src/video_window.rs new file mode 100644 index 00000000..d282b95c --- /dev/null +++ b/crates/ui-egui/src/video_window.rs @@ -0,0 +1,166 @@ +//! Window › Video (the picture at the playhead, sized to the movie's aspect, optional timecode +//! burn-in) and Window › Video Universe (the whole Video track as a strip; click to locate). + +use crate::SoundApp; +use crate::theme::{mono, regular}; +use crate::video_track::{MovieKey, movie_secs, online, picture_at}; +use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; +use serde_json::json; +use soundcraft_model::{ClipContent, TrackKind}; +use soundcraft_time::{TimeFormat, format_position}; + +pub fn show(app: &mut SoundApp, ctx: &egui::Context) { + if app.ui.show_video { + video(app, ctx); + } + if app.ui.show_video_universe { + universe(app, ctx); + } +} + +fn video(app: &mut SoundApp, ctx: &egui::Context) { + app.video.poll(ctx); + let pos = app.position(); + let s = app.engine.session_arc(); + let pic = picture_at(&s, pos); + let on = online(&s); + if on && let Some((key, path, secs, fps)) = &pic { + app.video.request_frame(ctx, *key, path, *secs, *fps); + } + let aspect = pic + .as_ref() + .and_then(|(k, ..)| app.video.info(*k).and_then(|i| i.as_ref().ok()).map(|i| i.display_aspect() as f32)) + .or_else(|| first_movie_aspect(&s)) + .unwrap_or(16.0 / 9.0) + .clamp(0.2, 5.0); + let mut open = true; + let mut burn = app.ui.video_burn_in; + egui::Window::new("Video").open(&mut open).default_size(vec2(480.0, 480.0 / aspect + 28.0)).resizable(true).show(ctx, |ui| { + ui.horizontal(|ui| { + ui.checkbox(&mut burn, "Timecode burn-in"); + let text = match &pic { + Some((k, ..)) => match app.video.info(*k) { + Some(Ok(i)) => format!("{}x{} {:.3} fps {}", i.width, i.height, i.frame_rate, i.codec_detail), + Some(Err(e)) => e.clone(), + None => "opening…".into(), + }, + None => String::new(), + }; + ui.label(egui::RichText::new(text).font(regular(10.5)).color(Color32::from_gray(150))); + }); + let avail = ui.available_rect_before_wrap(); + let (r, _) = ui.allocate_exact_size(avail.size().max(vec2(64.0, 36.0)), Sense::hover()); + let painter = ui.painter_at(r); + painter.rect_filled(r, 0.0, Color32::BLACK); + // Fit the picture inside the area, keeping its aspect. + let (mut w, mut h) = (r.width(), r.width() / aspect); + if h > r.height() { + h = r.height(); + w = h * aspect; + } + let img = Rect::from_center_size(r.center(), vec2(w, h)); + let msg = if !on { + Some("Video Track Offline".to_string()) + } else { + match &pic { + None => None, + Some((k, ..)) => match (app.video.current(*k), app.video.current_error(*k)) { + (Some(tex), _) => { + painter.image(tex.id(), img, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + None + } + (None, Some(e)) => Some(e.to_string()), + (None, None) => match app.video.info(*k) { + Some(Err(e)) => Some(e.clone()), + _ => None, + }, + }, + } + }; + if let Some(m) = msg { + painter.text(img.center(), Align2::CENTER_CENTER, m, regular(12.0), Color32::from_gray(170)); + } + if burn { + let tc = format_position(pos, TimeFormat::Timecode, s.sample_rate, &s.tempo, s.frame_rate, s.timecode_start); + let size = (h / 12.0).clamp(10.0, 36.0); + let galley = painter.layout_no_wrap(tc, mono(size), Color32::WHITE); + let at = pos2(img.center().x - galley.size().x / 2.0, img.max.y - galley.size().y - size * 0.6); + painter.rect_filled(Rect::from_min_size(at, galley.size()).expand2(vec2(6.0, 2.0)), 2.0, Color32::from_black_alpha(200)); + painter.galley(at, galley, Color32::WHITE); + } + }); + app.ui.video_burn_in = burn; + if !open { + app.ui.show_video = false; + } +} + +fn first_movie_aspect(s: &soundcraft_model::Session) -> Option { + s.videos.first().map(|v| v.aspect() as f32) +} + +fn universe(app: &mut SoundApp, ctx: &egui::Context) { + app.video.poll(ctx); + let s = app.engine.session_arc(); + let pos = app.position(); + let mut open = true; + let mut locate = None; + egui::Window::new("Video Universe").open(&mut open).default_size(vec2(640.0, 90.0)).resizable(true).show(ctx, |ui| { + let avail = ui.available_rect_before_wrap(); + let (r, resp) = ui.allocate_exact_size(vec2(avail.width().max(120.0), 64.0), Sense::click_and_drag()); + let painter = ui.painter_at(r); + painter.rect_filled(r, 0.0, Color32::from_rgb(14, 14, 15)); + let end = s.content_end().max(s.sample_rate.samples(10.0)).max(1); + let x_at = |at: i64| r.min.x + r.width() * (at as f32 / end as f32); + let on = online(&s); + let Some(track) = s.tracks.iter().find(|t| t.kind == TrackKind::Video) else { + painter.text(r.center(), Align2::CENTER_CENTER, "No Video track — File › Import › Video…", regular(11.0), Color32::from_gray(150)); + return; + }; + let ppp = ctx.pixels_per_point(); + for c in track.clips() { + let ClipContent::Video { source, offset } = c.content else { continue }; + let Some(v) = s.video(source) else { continue }; + let key = MovieKey::new(source, &v.path); + let cr = Rect::from_min_max(pos2(x_at(c.start), r.min.y + 4.0), pos2(x_at(c.end()).max(x_at(c.start) + 2.0), r.max.y - 4.0)); + painter.rect_filled(cr, 0.0, Color32::from_rgb(30, 30, 34)); + if !on { + continue; + } + let tw = (cr.height() * v.aspect() as f32).max(8.0); + let hpx = ((cr.height() * ppp / 16.0).ceil() * 16.0).clamp(16.0, 256.0) as u32; + let mut x = cr.min.x; + while x < cr.max.x { + let at = c.start + ((x - r.min.x) / r.width() * end as f32) as i64 - c.start.min(0); + let secs = movie_secs(&s, c.start, offset, at.max(c.start)).max(0.0); + if let Some(tex) = app.video.thumb(ctx, key, &v.path, secs, hpx) { + let tr = Rect::from_min_size(pos2(x, cr.min.y), vec2(tw, cr.height())); + let vis = tr.intersect(cr); + let uv = Rect::from_min_max(pos2(0.0, 0.0), pos2((vis.width() / tw).clamp(0.0, 1.0), 1.0)); + painter.image(tex.id(), vis, uv, Color32::WHITE); + } + x += tw; + } + } + // The Edit window's visible range and the playhead. + let spp = s.edit.zoom.samples_per_px.max(1e-6); + let vs = s.edit.zoom.scroll.max(0); + let ve = vs.saturating_add((spp * f64::from(ui.ctx().content_rect().width())) as i64); + let view = Rect::from_min_max(pos2(x_at(vs), r.min.y + 1.0), pos2(x_at(ve).min(r.max.x), r.max.y - 1.0)); + painter.rect_stroke(view, 0.0, egui::Stroke::new(1.0, Color32::from_rgb(230, 200, 80)), egui::StrokeKind::Inside); + let px = x_at(pos); + painter.line_segment([pos2(px, r.min.y), pos2(px, r.max.y)], egui::Stroke::new(1.5, Color32::from_rgb(240, 240, 240))); + if (resp.clicked() || resp.dragged()) + && let Some(p) = resp.interact_pointer_pos() + { + locate = Some((((p.x - r.min.x) / r.width()).clamp(0.0, 1.0) * end as f32) as i64); + } + }); + if let Some(at) = locate { + let _ = app.run("transport.locate", json!({"at": at})); + let _ = app.run("edit.select", json!({"start": at, "end": at})); + } + if !open { + app.ui.show_video_universe = false; + } +} diff --git a/crates/video/Cargo.toml b/crates/video/Cargo.toml new file mode 100644 index 00000000..d8165ab2 --- /dev/null +++ b/crates/video/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "soundcraft-video" +description = "SoundCraft video: MP4/MOV demux, H.264 / ProRes / Motion JPEG decoding to RGBA, timeline thumbnails" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[features] +default = [] +# Frame-parallel H.264 and slice-parallel ProRes decoding (rayon). Off by default: SoundCraft +# decodes on its own background thread and only needs one picture at a time. +threads = ["dep:rayon"] + +[dependencies] +thiserror.workspace = true +serde.workspace = true +image = { workspace = true, features = ["jpeg"] } +rayon = { workspace = true, optional = true } + +[dev-dependencies] +proptest.workspace = true + +[lints] +workspace = true diff --git a/crates/video/examples/test_movie.rs b/crates/video/examples/test_movie.rs new file mode 100644 index 00000000..189961ad --- /dev/null +++ b/crates/video/examples/test_movie.rs @@ -0,0 +1,14 @@ +//! Write a synthetic colour-bar test movie (H.264 in MP4, 320x192, 24 fps). +//! +//! `cargo run -p soundcraft-video --example test_movie -- out.mp4 [seconds]` + +fn main() { + let mut args = std::env::args().skip(1); + let out = args.next().unwrap_or_else(|| "test_pattern.mp4".into()); + let secs = args.next().and_then(|s| s.parse().ok()).unwrap_or(4); + match soundcraft_video::synth::test_pattern_mp4(secs).map(|b| std::fs::write(&out, b)) { + Ok(Ok(())) => println!("wrote {out}"), + Ok(Err(e)) => eprintln!("{out}: {e}"), + Err(e) => eprintln!("{e}"), + } +} diff --git a/crates/video/src/bitstream.rs b/crates/video/src/bitstream.rs new file mode 100644 index 00000000..779970f1 --- /dev/null +++ b/crates/video/src/bitstream.rs @@ -0,0 +1,431 @@ +//! Bit-level reading and writing shared by FilmCraft's codec crates. +//! +//! - [`BitReader`]: MSB-first reader with Exp-Golomb (`ue(v)`, `se(v)`), as used by H.264/HEVC/AV1 headers. +//! - [`BitWriter`]: the matching writer (for encoders and header rewriting). +//! - [`unescape_rbsp`] / [`escape_rbsp`]: remove/insert H.264/HEVC emulation-prevention bytes. +//! - [`annexb_nals`] / [`length_prefixed_nals`]: split Annex-B byte streams and AVCC/HVCC samples into NAL units. + +use std::fmt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BitError { + /// Tried to read past the end of the buffer. + Eof, + /// An Exp-Golomb code longer than 32 leading zeros. + InvalidExpGolomb, +} + +impl fmt::Display for BitError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BitError::Eof => f.write_str("unexpected end of bitstream"), + BitError::InvalidExpGolomb => f.write_str("invalid Exp-Golomb code"), + } + } +} +impl std::error::Error for BitError {} + +pub type Result = std::result::Result; + +/// MSB-first bit reader over a byte slice. +#[derive(Clone)] +pub struct BitReader<'a> { + data: &'a [u8], + /// Position in bits. + pos: usize, +} + +impl<'a> BitReader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + pub fn data(&self) -> &'a [u8] { + self.data + } + /// Bit position from the start. + pub fn position(&self) -> usize { + self.pos + } + pub fn bits_left(&self) -> usize { + (self.data.len() * 8).saturating_sub(self.pos) + } + pub fn is_byte_aligned(&self) -> bool { + self.pos.is_multiple_of(8) + } + pub fn byte_align(&mut self) { + self.pos = self.pos.div_ceil(8) * 8; + } + /// Byte offset of the current position (rounded down). + pub fn byte_pos(&self) -> usize { + self.pos / 8 + } + pub fn seek_bits(&mut self, pos: usize) { + self.pos = pos; + } + + /// Peek up to 32 bits without consuming (zero-padded past the end). + #[inline] + pub fn peek(&self, n: u32) -> u32 { + debug_assert!(n <= 32); + if n == 0 { + return 0; + } + let byte = self.pos / 8; + let shift = self.pos % 8; + let mut v: u64 = 0; + for i in 0..5 { + v = (v << 8) | *self.data.get(byte + i).unwrap_or(&0) as u64; + } + // v holds 40 bits starting at `byte` + ((v << (24 + shift)) >> (64 - n)) as u32 + } + + #[inline] + pub fn read_bits(&mut self, n: u32) -> Result { + if n as usize > self.bits_left() { + return Err(BitError::Eof); + } + let v = self.peek(n); + self.pos += n as usize; + Ok(v) + } + + pub fn read_bits_u64(&mut self, n: u32) -> Result { + if n <= 32 { + return self.read_bits(n).map(u64::from); + } + let hi = self.read_bits(n - 32)? as u64; + let lo = self.read_bits(32)? as u64; + Ok((hi << 32) | lo) + } + + #[inline] + pub fn read_bit(&mut self) -> Result { + self.read_bits(1).map(|b| b == 1) + } + pub fn read_flag(&mut self) -> Result { + self.read_bit() + } + + pub fn skip(&mut self, n: usize) -> Result<()> { + if n > self.bits_left() { + return Err(BitError::Eof); + } + self.pos += n; + Ok(()) + } + + /// Unsigned Exp-Golomb `ue(v)`. + pub fn read_ue(&mut self) -> Result { + let mut zeros = 0u32; + while !self.read_bit()? { + zeros += 1; + if zeros > 32 { + return Err(BitError::InvalidExpGolomb); + } + } + if zeros == 0 { + return Ok(0); + } + let v = self.read_bits_u64(zeros)?; + let r = (1u64 << zeros) - 1 + v; + u32::try_from(r).map_err(|_| BitError::InvalidExpGolomb) + } + + /// Signed Exp-Golomb `se(v)`. + pub fn read_se(&mut self) -> Result { + let k = self.read_ue()? as i64; + Ok(if k & 1 == 1 { ((k + 1) / 2) as i32 } else { -((k / 2) as i32) }) + } + + /// Truncated Exp-Golomb `te(v)` with range `max`. + pub fn read_te(&mut self, max: u32) -> Result { + if max > 1 { self.read_ue() } else { Ok(!self.read_bit()? as u32) } + } + + /// H.264/HEVC `more_rbsp_data()`: true if there is data before the rbsp trailing bits. + pub fn more_rbsp_data(&self) -> bool { + let total = self.data.len() * 8; + if self.pos >= total { + return false; + } + // find last set bit (the rbsp_stop_one_bit) + let mut last = None; + for (i, &b) in self.data.iter().enumerate().rev() { + if b != 0 { + last = Some(i * 8 + 7 - b.trailing_zeros() as usize); + break; + } + } + match last { + Some(stop) => self.pos < stop, + None => false, + } + } +} + +/// MSB-first bit writer. +#[derive(Default, Clone)] +pub struct BitWriter { + buf: Vec, + acc: u64, + nbits: u32, +} + +impl BitWriter { + pub fn new() -> Self { + Self::default() + } + pub fn write_bits(&mut self, value: u32, n: u32) { + debug_assert!(n <= 32); + if n == 0 { + return; + } + let v = (value as u64) & ((1u64 << n) - 1); + self.acc = (self.acc << n) | v; + self.nbits += n; + while self.nbits >= 8 { + self.nbits -= 8; + self.buf.push((self.acc >> self.nbits) as u8); + } + self.acc &= (1u64 << self.nbits) - 1; + } + pub fn write_bit(&mut self, b: bool) { + self.write_bits(b as u32, 1); + } + pub fn write_ue(&mut self, v: u32) { + let x = v as u64 + 1; + let len = 64 - x.leading_zeros(); + let zeros = len - 1; + for _ in 0..zeros { + self.write_bit(false); + } + if len > 32 { + self.write_bits((x >> 32) as u32, len - 32); + self.write_bits(x as u32, 32); + } else { + self.write_bits(x as u32, len); + } + } + pub fn write_se(&mut self, v: i32) { + let k = if v > 0 { (v as u32) * 2 - 1 } else { (-(v as i64) as u32) * 2 }; + self.write_ue(k); + } + pub fn bit_len(&self) -> usize { + self.buf.len() * 8 + self.nbits as usize + } + pub fn is_byte_aligned(&self) -> bool { + self.nbits == 0 + } + /// Pad with zero bits to a byte boundary. + pub fn align_zero(&mut self) { + if self.nbits > 0 { + self.write_bits(0, 8 - self.nbits); + } + } + /// `rbsp_trailing_bits()`: a one bit then zeros to alignment. + pub fn rbsp_trailing(&mut self) { + self.write_bit(true); + self.align_zero(); + } + pub fn write_bytes(&mut self, bytes: &[u8]) { + if self.nbits == 0 { + self.buf.extend_from_slice(bytes); + } else { + for &b in bytes { + self.write_bits(b as u32, 8); + } + } + } + pub fn finish(mut self) -> Vec { + self.align_zero(); + self.buf + } + pub fn bytes(&self) -> &[u8] { + &self.buf + } +} + +/// Remove emulation-prevention bytes (`00 00 03` → `00 00`) from a NAL payload. +pub fn unescape_rbsp(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len()); + let mut zeros = 0; + for &b in data { + if zeros >= 2 && b == 3 { + zeros = 0; + continue; + } + out.push(b); + zeros = if b == 0 { zeros + 1 } else { 0 }; + } + out +} + +/// Insert emulation-prevention bytes so no `00 00 0x` (x ≤ 3) sequence appears. +pub fn escape_rbsp(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len() + data.len() / 64); + let mut zeros = 0; + for &b in data { + if zeros >= 2 && b <= 3 { + out.push(3); + zeros = 0; + } + out.push(b); + zeros = if b == 0 { zeros + 1 } else { 0 }; + } + out +} + +/// Split an Annex-B byte stream (start codes `00 00 01` / `00 00 00 01`) into NAL units (escaped payloads). +pub fn annexb_nals(data: &[u8]) -> Vec<&[u8]> { + let mut starts = Vec::new(); + let mut i = 0; + while i + 3 <= data.len() { + if data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 { + starts.push(i + 3); + i += 3; + } else { + i += 1; + } + } + let mut out = Vec::with_capacity(starts.len()); + for (k, &s) in starts.iter().enumerate() { + let mut e = if k + 1 < starts.len() { starts[k + 1] - 3 } else { data.len() }; + // trim trailing zero bytes (part of a 4-byte start code or trailing_zero_8bits) + while e > s && data[e - 1] == 0 { + e -= 1; + } + if e > s { + out.push(&data[s..e]); + } + } + out +} + +/// Split a length-prefixed sample (AVCC/HVCC, `length_size` 1/2/4 bytes) into NAL units. +pub fn length_prefixed_nals(data: &[u8], length_size: usize) -> Result> { + let mut out = Vec::new(); + let mut i = 0; + while i < data.len() { + if i + length_size > data.len() { + return Err(BitError::Eof); + } + let mut len = 0usize; + for k in 0..length_size { + len = (len << 8) | data[i + k] as usize; + } + i += length_size; + if i + len > data.len() { + return Err(BitError::Eof); + } + out.push(&data[i..i + len]); + i += len; + } + Ok(out) +} + +/// CRC-32 (IEEE, reflected) — used by containers (e.g. Matroska CRC-32 elements, PNG). +pub fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xFFFF_FFFFu32; + for &b in data { + crc ^= b as u32; + for _ in 0..8 { + crc = if crc & 1 != 0 { (crc >> 1) ^ 0xEDB8_8320 } else { crc >> 1 }; + } + } + !crc +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + #[test] + fn reads_bits() { + let d = [0b1010_1100, 0b0101_0011]; + let mut r = BitReader::new(&d); + assert_eq!(r.read_bits(1).unwrap(), 1); + assert_eq!(r.read_bits(3).unwrap(), 0b010); + assert_eq!(r.read_bits(8).unwrap(), 0b1100_0101); + assert_eq!(r.bits_left(), 4); + assert_eq!(r.read_bits(5), Err(BitError::Eof)); + } + + #[test] + fn exp_golomb_known() { + // 1 | 010 | 011 | 00100 | 00101 -> 0,1,2,3,4 + let mut w = BitWriter::new(); + for v in 0..5 { + w.write_ue(v); + } + let b = w.finish(); + let mut r = BitReader::new(&b); + for v in 0..5 { + assert_eq!(r.read_ue().unwrap(), v); + } + let mut w = BitWriter::new(); + for v in [0, 1, -1, 2, -2, 1000, -1000] { + w.write_se(v); + } + let b = w.finish(); + let mut r = BitReader::new(&b); + for v in [0, 1, -1, 2, -2, 1000, -1000] { + assert_eq!(r.read_se().unwrap(), v); + } + } + + #[test] + fn emulation_prevention() { + let raw = [0, 0, 1, 0, 0, 0, 5, 0, 0, 3]; + let esc = escape_rbsp(&raw); + assert_eq!(esc, vec![0, 0, 3, 1, 0, 0, 3, 0, 5, 0, 0, 3, 3]); + assert_eq!(unescape_rbsp(&esc), raw); + } + + #[test] + fn annexb_split() { + let s = [0, 0, 0, 1, 0x67, 1, 2, 0, 0, 1, 0x68, 3, 0, 0, 0, 1, 0x65, 9, 9]; + let n = annexb_nals(&s); + assert_eq!(n, vec![&[0x67, 1, 2][..], &[0x68, 3][..], &[0x65, 9, 9][..]]); + let l = [0, 0, 0, 2, 0x67, 1, 0, 0, 0, 1, 0x68]; + assert_eq!(length_prefixed_nals(&l, 4).unwrap(), vec![&[0x67, 1][..], &[0x68][..]]); + } + + #[test] + fn more_rbsp() { + let d = [0b1100_0000]; + let mut r = BitReader::new(&d); + assert!(r.more_rbsp_data()); + r.read_bit().unwrap(); + assert!(!r.more_rbsp_data()); + } + + #[test] + fn crc() { + assert_eq!(crc32(b"123456789"), 0xCBF4_3926); + } + + proptest! { + #[test] + fn roundtrip(vals in proptest::collection::vec((0u32..u32::MAX, 1u32..=32), 0..200), ues in proptest::collection::vec(0u32..u32::MAX - 1, 0..50)) { + let mut w = BitWriter::new(); + for &(v, n) in &vals { w.write_bits(v, n); } + for &u in &ues { w.write_ue(u); } + let b = w.finish(); + let mut r = BitReader::new(&b); + for &(v, n) in &vals { + let mask = if n == 32 { u32::MAX } else { (1 << n) - 1 }; + prop_assert_eq!(r.read_bits(n).unwrap(), v & mask); + } + for &u in &ues { prop_assert_eq!(r.read_ue().unwrap(), u); } + } + + #[test] + fn escape_roundtrip(data in proptest::collection::vec(0u8..4, 0..300)) { + let e = escape_rbsp(&data); + prop_assert_eq!(unescape_rbsp(&e), data); + for w in e.windows(3) { prop_assert!(!(w[0] == 0 && w[1] == 0 && w[2] < 3)); } + } + } +} diff --git a/crates/video/src/convert.rs b/crates/video/src/convert.rs new file mode 100644 index 00000000..e9605409 --- /dev/null +++ b/crates/video/src/convert.rs @@ -0,0 +1,171 @@ +//! Pixel conversion: Y'CbCr → RGBA (BT.601 / BT.709, limited or full range) and RGBA downscaling. + +/// Y'CbCr → R'G'B' matrix coefficients in 1/65536 units: (Cr→R, Cb→G, Cr→G, Cb→B). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Matrix { + cr_r: i32, + cb_g: i32, + cr_g: i32, + cb_b: i32, + full_range: bool, +} + +impl Matrix { + /// BT.601 (SD) or BT.709 (HD) coefficients. + pub fn new(bt709: bool, full_range: bool) -> Matrix { + // Kr/Kb: 601 = 0.299/0.114, 709 = 0.2126/0.0722. + let (kr, kb) = if bt709 { (0.2126f64, 0.0722f64) } else { (0.299, 0.114) }; + let kg = 1.0 - kr - kb; + let s = 65536.0; + Matrix { + cr_r: (2.0 * (1.0 - kr) * s).round() as i32, + cb_b: (2.0 * (1.0 - kb) * s).round() as i32, + cb_g: (2.0 * (1.0 - kb) * kb / kg * s).round() as i32, + cr_g: (2.0 * (1.0 - kr) * kr / kg * s).round() as i32, + full_range, + } + } + + /// Choose by ITU-T H.273 matrix code (1 = 709, 5/6 = 601), else by picture height. + pub fn for_code(matrix: u8, height: u32, full_range: bool) -> Matrix { + let bt709 = match matrix { + 1 => true, + 5 | 6 => false, + _ => height >= 600, + }; + Matrix::new(bt709, full_range) + } + + /// Convert one 8-bit sample triple. + #[inline] + pub fn rgb(&self, y: u8, cb: u8, cr: u8) -> [u8; 3] { + let (y, cb, cr) = (i32::from(y), i32::from(cb) - 128, i32::from(cr) - 128); + let (y, cb, cr) = if self.full_range { + (y << 16, cb, cr) + } else { + // Expand 16..235 / 16..240 to full range. + ((y - 16) * 76309, (cb * 74_606) >> 16, (cr * 74_606) >> 16) + }; + let y = y + (1 << 15); // round to nearest + let r = (y + self.cr_r * cr) >> 16; + let g = (y - self.cb_g * cb - self.cr_g * cr) >> 16; + let b = (y + self.cb_b * cb) >> 16; + [r.clamp(0, 255) as u8, g.clamp(0, 255) as u8, b.clamp(0, 255) as u8] + } +} + +/// Planar 8-bit Y'CbCr with chroma subsampled by `cx` horizontally and `cy` vertically +/// (1 or 2 each) → packed RGBA. Missing samples (short planes) read as black / neutral. +pub fn planar_to_rgba( + width: usize, + height: usize, + y: &[u8], + y_stride: usize, + u: &[u8], + v: &[u8], + uv_stride: usize, + cx: usize, + cy: usize, + m: &Matrix, +) -> Vec { + let (cx, cy) = (cx.max(1), cy.max(1)); + let mut out = vec![0u8; width.saturating_mul(height).saturating_mul(4)]; + for (row, line) in out.chunks_exact_mut(width.max(1) * 4).enumerate().take(height) { + let yl = y.get(row * y_stride..).unwrap_or(&[]); + let crow = (row / cy) * uv_stride; + let ul = u.get(crow..).unwrap_or(&[]); + let vl = v.get(crow..).unwrap_or(&[]); + for (col, px) in line.as_chunks_mut::<4>().0.iter_mut().enumerate() { + let yy = yl.get(col).copied().unwrap_or(16); + let cb = ul.get(col / cx).copied().unwrap_or(128); + let cr = vl.get(col / cx).copied().unwrap_or(128); + let [r, g, b] = m.rgb(yy, cb, cr); + px.copy_from_slice(&[r, g, b, 255]); + } + } + out +} + +/// Area-average downscale (or nearest upscale) of packed RGBA. +pub fn scale_rgba(src: &[u8], sw: usize, sh: usize, dw: usize, dh: usize) -> Vec { + let (dw, dh) = (dw.max(1), dh.max(1)); + let mut out = vec![0u8; dw * dh * 4]; + if sw == 0 || sh == 0 || src.len() < sw * sh * 4 { + return out; + } + for dy in 0..dh { + let y0 = dy * sh / dh; + let y1 = ((dy + 1) * sh / dh).max(y0 + 1).min(sh); + for dx in 0..dw { + let x0 = dx * sw / dw; + let x1 = ((dx + 1) * sw / dw).max(x0 + 1).min(sw); + // Sample at most a 4x4 grid inside the source box (keeps big reductions cheap). + let (sx, sy) = ((x1 - x0).div_ceil(4).max(1), (y1 - y0).div_ceil(4).max(1)); + let mut acc = [0u32; 4]; + let mut n = 0u32; + let mut y = y0; + while y < y1 { + let mut x = x0; + while x < x1 { + if let Some(p) = src.get((y * sw + x) * 4..(y * sw + x) * 4 + 4) { + for (a, v) in acc.iter_mut().zip(p) { + *a += u32::from(*v); + } + n += 1; + } + x += sx; + } + y += sy; + } + if let Some(o) = out.get_mut((dy * dw + dx) * 4..(dy * dw + dx) * 4 + 4) { + for (o, a) in o.iter_mut().zip(acc) { + *o = (a / n.max(1)) as u8; + } + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn limited_range_black_white_and_grey() { + for bt709 in [false, true] { + let m = Matrix::new(bt709, false); + assert_eq!(m.rgb(16, 128, 128), [0, 0, 0]); + assert_eq!(m.rgb(235, 128, 128), [255, 255, 255]); + let g = m.rgb(126, 128, 128); + assert!(g[0] == g[1] && g[1] == g[2]); + } + let f = Matrix::new(true, true); + assert_eq!(f.rgb(0, 128, 128), [0, 0, 0]); + assert_eq!(f.rgb(255, 128, 128), [255, 255, 255]); + } + + #[test] + fn red_is_red() { + // BT.601 limited-range red: Y 81, Cb 90, Cr 240. + let [r, g, b] = Matrix::new(false, false).rgb(81, 90, 240); + assert!(r > 250 && g < 5 && b < 5, "{r} {g} {b}"); + } + + #[test] + fn scaling_averages_and_survives_bad_input() { + let src = [255u8, 255, 255, 255, 0, 0, 0, 255]; + let out = scale_rgba(&src, 2, 1, 1, 1); + assert_eq!(out, vec![127, 127, 127, 255]); + assert_eq!(scale_rgba(&[], 10, 10, 2, 2), vec![0; 16]); + assert_eq!(scale_rgba(&src, 2, 1, 4, 2).len(), 32); + } + + #[test] + fn planar_conversion_tolerates_short_planes() { + let m = Matrix::new(true, false); + let out = planar_to_rgba(4, 4, &[235; 3], 4, &[], &[], 2, 2, 2, &m); + assert_eq!(out.len(), 64); + assert_eq!(&out[..4], &[255, 255, 255, 255]); + } +} diff --git a/crates/video/src/h264/cabac.rs b/crates/video/src/h264/cabac.rs new file mode 100644 index 00000000..a7747d6b --- /dev/null +++ b/crates/video/src/h264/cabac.rs @@ -0,0 +1,335 @@ +//! CABAC arithmetic decoding engine (9.3.1.2, 9.3.3.2) and context initialisation (9.3.1.1). + +use crate::h264::cabac_tables::CABAC_INIT_MN; +use crate::h264::error::{Result, ensure}; + +/// Table 9-44 rangeTabLPS[pStateIdx][qCodIRangeIdx]. +#[rustfmt::skip] +pub(crate) static RANGE_TAB_LPS: [[u8; 4]; 64] = [ + [128, 176, 208, 240], [128, 167, 197, 227], [128, 158, 187, 216], [123, 150, 178, 205], + [116, 142, 169, 195], [111, 135, 160, 185], [105, 128, 152, 175], [100, 122, 144, 166], + [95, 116, 137, 158], [90, 110, 130, 150], [85, 104, 123, 142], [81, 99, 117, 135], + [77, 94, 111, 128], [73, 89, 105, 122], [69, 85, 100, 116], [66, 80, 95, 110], + [62, 76, 90, 104], [59, 72, 86, 99], [56, 69, 81, 94], [53, 65, 77, 89], + [51, 62, 73, 85], [48, 59, 69, 80], [46, 56, 66, 76], [43, 53, 63, 72], + [41, 50, 59, 69], [39, 48, 56, 65], [37, 45, 54, 62], [35, 43, 51, 59], + [33, 41, 48, 56], [32, 39, 46, 53], [30, 37, 43, 50], [29, 35, 41, 48], + [27, 33, 39, 45], [26, 31, 37, 43], [24, 30, 35, 41], [23, 28, 33, 39], + [22, 27, 32, 37], [21, 26, 30, 35], [20, 24, 29, 33], [19, 23, 27, 31], + [18, 22, 26, 30], [17, 21, 25, 28], [16, 20, 23, 27], [15, 19, 22, 25], + [14, 18, 21, 24], [14, 17, 20, 23], [13, 16, 19, 22], [12, 15, 18, 21], + [12, 14, 17, 20], [11, 14, 16, 19], [11, 13, 15, 18], [10, 12, 15, 17], + [10, 12, 14, 16], [9, 11, 13, 15], [9, 11, 12, 14], [8, 10, 12, 14], + [8, 9, 11, 13], [7, 9, 11, 12], [7, 9, 10, 12], [7, 8, 10, 11], + [6, 8, 9, 11], [6, 7, 9, 10], [6, 7, 8, 9], [2, 2, 2, 2], +]; + +/// Table 9-45 transIdxLPS. +#[rustfmt::skip] +static TRANS_IDX_LPS: [u8; 64] = [ + 0, 0, 1, 2, 2, 4, 4, 5, 6, 7, 8, 9, 9, 11, 11, 12, + 13, 13, 15, 15, 16, 16, 18, 18, 19, 19, 21, 21, 22, 22, 23, 24, + 24, 25, 26, 26, 27, 27, 28, 29, 29, 30, 30, 30, 31, 32, 32, 33, + 33, 33, 34, 34, 35, 35, 35, 36, 36, 36, 37, 37, 37, 38, 38, 63, +]; + +/// Table 9-45 transIdxMPS. +#[rustfmt::skip] +static TRANS_IDX_MPS: [u8; 64] = [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 62, 63, +]; + +/// Combined state transition: index = (pStateIdx << 1 | valMPS) * 2 + bin_is_lps. +/// Each context is stored as `pStateIdx << 1 | valMPS`. +pub(crate) static NEXT_STATE: [[u8; 2]; 128] = { + let mut t = [[0u8; 2]; 128]; + let mut s = 0; + while s < 128 { + let p = s >> 1; + let mps = s & 1; + // MPS path + t[s][0] = (TRANS_IDX_MPS[p] << 1) | mps as u8; + // LPS path + let new_mps = if p == 0 { 1 - mps } else { mps }; + t[s][1] = (TRANS_IDX_LPS[p] << 1) | new_mps as u8; + s += 1; + } + t +}; + +/// Number of context variables used (0..=1023). +pub const NUM_CTX: usize = 1024; + +pub struct Cabac<'a> { + data: &'a [u8], + /// Byte position of the next byte to load into `value`. + next_byte: usize, + range: u32, + /// codIOffset scaled by 2^bits: value = (codIOffset << bits) | (bits already loaded but not yet + /// consumed by the arithmetic decoder). + value: u64, + bits: u32, + pub ctx: [u8; NUM_CTX], +} + +impl<'a> Cabac<'a> { + /// Create an engine over `data` (slice RBSP) starting at bit `pos` (must be byte aligned), + /// with contexts initialised for `slice_qp` and `init` (0..=2 cabac_init_idc, 3 = I slice). + pub fn new(data: &'a [u8], pos: usize, slice_qp: i32, init: usize) -> Result { + let mut c = Cabac { data, next_byte: pos / 8, range: 0, value: 0, bits: 0, ctx: [0; NUM_CTX] }; + c.init_contexts(slice_qp, init); + c.init_engine()?; + Ok(c) + } + + pub fn init_contexts(&mut self, slice_qp: i32, init: usize) { + let qp = slice_qp.clamp(0, 51); + let tab = &CABAC_INIT_MN[init]; + for (c, &(m, n)) in self.ctx.iter_mut().zip(tab.iter()) { + let pre = (((m as i32) * qp) >> 4) + n as i32; + let pre = pre.clamp(1, 126); + *c = if pre <= 63 { ((63 - pre) << 1) as u8 } else { (((pre - 64) << 1) | 1) as u8 }; + } + // ctxIdx 276 (end_of_slice / I_PCM) is handled by decode_terminate. + } + + /// 9.3.1.2: codIRange = 510, codIOffset = read_bits(9) (from the current byte-aligned position). + pub fn init_engine(&mut self) -> Result<()> { + self.range = 510; + self.value = 0; + self.bits = 0; + self.refill(); + // the top 9 loaded bits form codIOffset; the remaining ones stay buffered below it + self.bits -= 9; + ensure!((self.value >> self.bits) < 510, "invalid CABAC offset at init"); + Ok(()) + } + + /// Load 32 more bits (zeros past the end of the data). + #[inline(always)] + fn refill(&mut self) { + let b = self.next_byte; + let mut w = [0u8; 4]; + if let Some(src) = self.data.get(b..b + 4) { + w.copy_from_slice(src); + } else if b < self.data.len() { + let n = self.data.len() - b; + w[..n].copy_from_slice(&self.data[b..]); + } + self.next_byte += 4; + self.value = (self.value << 32) | u32::from_be_bytes(w) as u64; + self.bits += 32; + } + + /// Bit position of the next bit not consumed by the arithmetic decoder (for I_PCM alignment + /// after decode_terminate returned 1). + pub fn bit_pos(&self) -> usize { + self.next_byte * 8 - self.bits as usize + } + /// Continue reading at byte-aligned bit position `pos` (call `init_engine` afterwards). + pub fn set_bit_pos(&mut self, pos: usize) { + self.next_byte = pos / 8; + self.value = 0; + self.bits = 0; + } + pub fn data(&self) -> &'a [u8] { + self.data + } + /// True when the reader has run past the end of the data (corrupt stream). + pub fn overrun(&self) -> bool { + self.next_byte > self.data.len() + 16 + } + + #[inline(always)] + fn renorm(&mut self) { + // codIRange < 512 always, so the shift is 0 when no renormalisation is needed: no branch on + // the (unpredictable) range, only on the rare refill. + let shift = self.range.leading_zeros() - 23; + self.range <<= shift; + if self.bits < shift { + self.refill(); + } + self.bits -= shift; + } + + /// 9.3.3.2.1, branch-free: the MPS / LPS outcome of a context-coded bin is close to random + /// at high bit rates, so it selects values instead of jumping. + #[inline(always)] + pub fn decode_decision(&mut self, ctx_idx: usize) -> u32 { + // Every ctxIdx is < 1024 and every state < 128: the masks only let the compiler drop the + // bounds checks from the per-bin path. + let ctx_idx = ctx_idx & (NUM_CTX - 1); + let s = (self.ctx[ctx_idx] & 127) as usize; + let q = ((self.range >> 6) & 3) as usize; + let lps = RANGE_TAB_LPS[s >> 1][q] as u32; + let mps_range = self.range - lps; + let scaled = (mps_range as u64) << self.bits; + let is_lps = self.value >= scaled; + self.value -= scaled & (is_lps as u64).wrapping_neg(); + self.range = if is_lps { lps } else { mps_range }; + self.ctx[ctx_idx] = NEXT_STATE[s][is_lps as usize]; + self.renorm(); + (s & 1) as u32 ^ is_lps as u32 + } + + #[inline(always)] + pub fn decode_bypass(&mut self) -> u32 { + if self.bits == 0 { + self.refill(); + } + self.bits -= 1; + let scaled = (self.range as u64) << self.bits; + let one = self.value >= scaled; + self.value -= scaled & (one as u64).wrapping_neg(); + one as u32 + } + + pub fn decode_terminate(&mut self) -> u32 { + self.range -= 2; + let scaled = (self.range as u64) << self.bits; + if self.value >= scaled { + 1 + } else { + self.renorm(); + 0 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Minimal CABAC encoder (9.3.4, informative) used to round-trip test the decoder. + struct Enc { + low: u32, + range: u32, + outstanding: u32, + first: bool, + bits: Vec, + ctx: Vec, + } + impl Enc { + fn new(ctx: &[u8]) -> Self { + Enc { low: 0, range: 510, outstanding: 0, first: true, bits: vec![], ctx: ctx.to_vec() } + } + fn put(&mut self, b: bool) { + if self.first { + self.first = false; + } else { + self.bits.push(b); + } + while self.outstanding > 0 { + self.bits.push(!b); + self.outstanding -= 1; + } + } + fn renorm(&mut self) { + while self.range < 256 { + if self.low < 256 { + self.put(false); + } else if self.low >= 512 { + self.low -= 512; + self.put(true); + } else { + self.low -= 256; + self.outstanding += 1; + } + self.range <<= 1; + self.low <<= 1; + } + } + fn encode(&mut self, ctx_idx: usize, bin: u32) { + let s = self.ctx[ctx_idx] as usize; + let p = s >> 1; + let mps = (s & 1) as u32; + let q = ((self.range >> 6) & 3) as usize; + let lps = RANGE_TAB_LPS[p][q] as u32; + self.range -= lps; + if bin != mps { + self.low += self.range; + self.range = lps; + self.ctx[ctx_idx] = NEXT_STATE[s][1]; + } else { + self.ctx[ctx_idx] = NEXT_STATE[s][0]; + } + self.renorm(); + } + fn bypass(&mut self, bin: u32) { + self.low <<= 1; + if bin != 0 { + self.low += self.range; + } + if self.low >= 1024 { + self.put(true); + self.low -= 1024; + } else if self.low < 512 { + self.put(false); + } else { + self.low -= 512; + self.outstanding += 1; + } + } + fn terminate_flush(&mut self) -> Vec { + // encode terminate bin = 1 then flush (9.3.4.5) + self.range -= 2; + self.low += self.range; + self.range = 2; + self.renorm(); + self.put((self.low >> 9) & 1 != 0); + self.bits.push((self.low >> 8) & 1 != 0); + self.bits.push(true); // rbsp_stop_one_bit + while !self.bits.len().is_multiple_of(8) { + self.bits.push(false); + } + self.bits.chunks(8).map(|c| c.iter().fold(0u8, |a, &b| (a << 1) | b as u8)).collect() + } + } + + #[test] + fn context_init_formula() { + let c = Cabac::new(&[0, 0, 0, 0], 0, 26, 3).unwrap(); + // ctxIdx 0: m=20, n=-15 at QP 26: ((20*26)>>4) - 15 = 32 - 15 = 17 -> pState 46, MPS 0 + assert_eq!(c.ctx[0], 46 << 1); + // ctxIdx 60: m=0,n=41 -> pre 41 -> pState 22 MPS 0 + assert_eq!(c.ctx[60], 22 << 1); + // ctxIdx 70 (I): m=0 n=11 -> pState 52 MPS 0 + assert_eq!(c.ctx[70], 52 << 1); + } + + #[test] + fn round_trip_random_bins() { + let mut seed = 12345u32; + let mut rnd = || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed + }; + let dummy = Cabac::new(&[0, 0, 0, 0], 0, 30, 0).unwrap(); + let mut enc = Enc::new(&dummy.ctx); + let mut ops = Vec::new(); + for _ in 0..5000 { + let kind = rnd() % 4; + let ctx = (rnd() % 16) as usize; + // skewed bins so contexts adapt + let bin = if rnd() % 10 < 8 { (ctx & 1) as u32 } else { 1 - (ctx & 1) as u32 }; + if kind == 0 { + enc.bypass(bin); + } else { + enc.encode(ctx, bin); + } + ops.push((kind, ctx, bin)); + } + let bytes = enc.terminate_flush(); + let mut dec = Cabac::new(&bytes, 0, 30, 0).unwrap(); + for (i, &(kind, ctx, bin)) in ops.iter().enumerate() { + let got = if kind == 0 { dec.decode_bypass() } else { dec.decode_decision(ctx) }; + assert_eq!(got, bin, "bin {i}"); + } + assert_eq!(dec.decode_terminate(), 1); + } +} diff --git a/crates/video/src/h264/cabac_mb.rs b/crates/video/src/h264/cabac_mb.rs new file mode 100644 index 00000000..4a3125e5 --- /dev/null +++ b/crates/video/src/h264/cabac_mb.rs @@ -0,0 +1,689 @@ +//! CABAC slice data and macroblock layer parsing (7.3.4, 7.3.5 with 9.3 binarisations and context +//! index derivations). + +use crate::h264::cabac::Cabac; +use crate::h264::error::{Result, ensure, invalid}; +use crate::h264::mbtypes::*; +use crate::h264::picture::MbKind; +use crate::h264::slice::SliceType; +use crate::h264::slicedec::{MbCur, SliceDecoder}; + +/// Table 9-43: significant_coeff_flag ctxIdxInc for 8x8 frame blocks. +#[rustfmt::skip] +const SIG8_FRAME: [u8; 63] = [ + 0, 1, 2, 3, 4, 5, 5, 4, 4, 3, 3, 4, 4, 4, 5, 5, 4, 4, 4, 4, 3, 3, 6, 7, 7, 7, 8, 9, 10, 9, 8, 7, + 7, 6, 11, 12, 13, 11, 6, 7, 8, 9, 14, 10, 9, 8, 6, 11, 12, 13, 11, 6, 9, 14, 10, 9, 11, 12, 13, 11, 14, 10, 12, +]; +/// Table 9-43: last_significant_coeff_flag ctxIdxInc for 8x8 blocks. +#[rustfmt::skip] +const LAST8: [u8; 63] = [ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, +]; + +/// ctxIdxInc = levelListIdx (ctxBlockCat 0, 1, 2, 4). +const IDENT_CTX: [u8; 64] = { + let mut t = [0u8; 64]; + let mut i = 0; + while i < 64 { + t[i] = i as u8; + i += 1; + } + t +}; +/// ctxIdxInc = Min(levelListIdx / NumC8x8, 2) for 4:2:0 chroma DC (ctxBlockCat 3). +const CHROMA_DC_CTX: [u8; 4] = [0, 1, 2, 2]; + +/// Table 9-40 ctxIdxBlockCatOffset for ctxBlockCat 0..=5: (coded_block_flag, significant/last, abs level). +const CAT_OFFSET: [(usize, usize, usize); 6] = [(0, 0, 0), (4, 15, 10), (8, 29, 20), (12, 44, 30), (16, 47, 39), (0, 0, 0)]; + +impl SliceDecoder<'_> { + /// Decode slice_data() with CABAC. `rbsp` is the whole slice RBSP. + pub fn decode_cabac(&mut self, rbsp: &[u8]) -> Result<()> { + let start = self.sh.header_bits.div_ceil(8) * 8; + let init = if self.sh.slice_type.is_intra() { 3 } else { self.sh.cabac_init_idc as usize }; + let mut c = Cabac::new(rbsp, start, self.sh.qp(self.pps), init)?; + let total = self.mb_w * self.mb_h; + let mut addr = self.sh.first_mb_in_slice as usize; + let intra_slice = self.sh.slice_type.is_intra(); + loop { + ensure!(addr < total, "macroblock address {addr} out of range"); + self.start_mb(addr); + let skip = !intra_slice && self.decode_skip_flag(&mut c); + if skip { + self.decode_skip()?; + } else { + self.decode_mb_cabac(&mut c)?; + } + self.pic.mb_done(addr); + ensure!(!c.overrun(), "CABAC read past end of slice data"); + if c.decode_terminate() == 1 { + break; + } + addr += 1; + } + Ok(()) + } + + fn decode_skip_flag(&mut self, c: &mut Cabac) -> bool { + let cond = |n: Option| n.map(|m| !self.pic.mbs[m].kind_is_skip()).unwrap_or(false) as usize; + let inc = cond(self.nb[0]) + cond(self.nb[1]); + let base = if self.sh.slice_type.is_b() { 24 } else { 11 }; + c.decode_decision(base + inc) == 1 + } + + /// Intra mb_type suffix / I-slice mb_type (Table 9-36). `base` is 3 (I slice) or 17/32 (prefix + /// offsets in P/B slices). Returns the I-slice mb_type value 0..=25. + fn decode_mb_type_i(&mut self, c: &mut Cabac, base: usize) -> u32 { + let islice = base == 3; + let b0 = if islice { + let cond = |n: Option| n.map(|m| !matches!(self.pic.mbs[m].kind, MbKind::I4x4 | MbKind::I8x8)).unwrap_or(false) as usize; + let inc = cond(self.nb[0]) + cond(self.nb[1]); + c.decode_decision(base + inc) + } else { + c.decode_decision(base) + }; + if b0 == 0 { + return 0; + } + if c.decode_terminate() == 1 { + return 25; + } + let (c_luma, c_chroma, c_chroma2, c_pred0, c_pred1) = + if islice { (base + 3, base + 4, base + 5, base + 6, base + 7) } else { (base + 1, base + 2, base + 2, base + 3, base + 3) }; + let luma = c.decode_decision(c_luma); + let mut chroma = c.decode_decision(c_chroma); + if chroma != 0 { + chroma += c.decode_decision(c_chroma2); + } + let p0 = c.decode_decision(c_pred0); + let p1 = c.decode_decision(c_pred1); + 1 + (p0 * 2 + p1) + 4 * chroma + 12 * luma + } + + fn decode_mb_type(&mut self, c: &mut Cabac) -> Result { + let info = match self.sh.slice_type { + SliceType::I | SliceType::Si => intra_mb_type(self.decode_mb_type_i(c, 3)), + SliceType::P | SliceType::Sp => { + if c.decode_decision(14) == 1 { + intra_mb_type(self.decode_mb_type_i(c, 17)) + } else if c.decode_decision(15) == 0 { + p_mb_type(if c.decode_decision(16) == 0 { 0 } else { 3 }) + } else { + p_mb_type(if c.decode_decision(17) == 1 { 1 } else { 2 }) + } + } + SliceType::B => { + let cond = + |n: Option| n.map(|m| !matches!(self.pic.mbs[m].kind, MbKind::BSkip | MbKind::BDirect16x16)).unwrap_or(false) as usize; + let inc = cond(self.nb[0]) + cond(self.nb[1]); + if c.decode_decision(27 + inc) == 0 { + b_mb_type(0) + } else if c.decode_decision(27 + 3) == 0 { + b_mb_type(1 + c.decode_decision(27 + 5)) + } else { + let b2 = c.decode_decision(27 + 4); + let b3 = c.decode_decision(27 + 5); + let b4 = c.decode_decision(27 + 5); + let b5 = c.decode_decision(27 + 5); + let v = (b2 << 3) | (b3 << 2) | (b4 << 1) | b5; + match v { + 0b1101 => intra_mb_type(self.decode_mb_type_i(c, 32)), + 0b1111 => b_mb_type(22), + 0b1110 => b_mb_type(11), + _ if b2 == 0 => b_mb_type(3 + v), + _ => { + let b6 = c.decode_decision(27 + 5); + b_mb_type(12 + (((v & 7) << 1) | b6)) + } + } + } + } + }; + match info { + Some(i) => Ok(i), + None => invalid("invalid CABAC mb_type"), + } + } + + fn decode_sub_mb_type(&mut self, c: &mut Cabac) -> Result { + let r = if self.sh.slice_type.is_b() { + if c.decode_decision(36) == 0 { + b_sub_mb_type(0) + } else if c.decode_decision(37) == 0 { + b_sub_mb_type(1 + c.decode_decision(39)) + } else if c.decode_decision(38) == 1 { + if c.decode_decision(39) == 1 { + b_sub_mb_type(11 + c.decode_decision(39)) + } else { + let b4 = c.decode_decision(39); + let b5 = c.decode_decision(39); + b_sub_mb_type(7 + (b4 << 1) + b5) + } + } else { + let b3 = c.decode_decision(39); + let b4 = c.decode_decision(39); + b_sub_mb_type(3 + (b3 << 1) + b4) + } + } else if c.decode_decision(21) == 1 { + p_sub_mb_type(0) + } else if c.decode_decision(22) == 0 { + p_sub_mb_type(1) + } else { + p_sub_mb_type(if c.decode_decision(23) == 1 { 2 } else { 3 }) + }; + match r { + Some(s) => Ok(s), + None => invalid("invalid CABAC sub_mb_type"), + } + } + + /// ref_idx_lX for the partition whose top-left 4x4 block is (x, y) (9.3.3.1.1.6). + fn decode_ref_idx(&mut self, c: &mut Cabac, list: usize, x: usize, y: usize) -> u32 { + let cond = |xx: i32, yy: i32| -> usize { + let (mb, bx, by) = if xx < 0 { + (self.nb[0], 3usize, yy as usize) + } else if yy < 0 { + (self.nb[1], xx as usize, 3usize) + } else { + (Some(self.mb_addr), xx as usize, yy as usize) + }; + let Some(m) = mb else { return 0 }; + let st = &self.pic.mbs[m]; + let b8 = (by >> 1) * 2 + (bx >> 1); + if st.kind.is_intra() || st.kind_is_skip() || st.direct8x8 & (1 << b8) != 0 { + return 0; + } + (st.ref_idx[list][b8] > 0) as usize + }; + let inc = cond(x as i32 - 1, y as i32) + 2 * cond(x as i32, y as i32 - 1); + let mut v = 0u32; + if c.decode_decision(54 + inc) == 1 { + v = 1; + if c.decode_decision(54 + 4) == 1 { + v = 2; + while c.decode_decision(54 + 5) == 1 { + v += 1; + if v > 32 { + break; + } + } + } + } + v + } + + /// One mvd component for the (sub-)partition with top-left 4x4 block (x, y) (9.3.3.1.1.7, UEG3). + fn decode_mvd(&mut self, c: &mut Cabac, list: usize, comp: usize, x: usize, y: usize) -> i32 { + let absn = |xx: i32, yy: i32| -> u32 { + let (mb, bx, by) = if xx < 0 { + (self.nb[0], 3usize, yy as usize) + } else if yy < 0 { + (self.nb[1], xx as usize, 3usize) + } else { + (Some(self.mb_addr), xx as usize, yy as usize) + }; + match mb { + Some(m) => self.pic.mbs[m].mvd[list][by * 4 + bx][comp] as u32, + None => 0, + } + }; + let sum = absn(x as i32 - 1, y as i32) + absn(x as i32, y as i32 - 1); + let inc = if sum > 32 { + 2 + } else if sum > 2 { + 1 + } else { + 0 + }; + let base = if comp == 0 { 40 } else { 47 }; + if c.decode_decision(base + inc) == 0 { + return 0; + } + let mut prefix = 1; + while prefix < 9 && c.decode_decision(base + (prefix + 2).min(6)) == 1 { + prefix += 1; + } + let mut val = prefix as i32; + if prefix >= 9 { + // EG3 suffix + let mut k = 3; + while c.decode_bypass() == 1 { + val += 1 << k; + k += 1; + if k > 24 { + break; + } + } + while k > 0 { + k -= 1; + val += (c.decode_bypass() as i32) << k; + } + } + if c.decode_bypass() == 1 { -val } else { val } + } + + fn set_cur_ref(&mut self, list: usize, x: usize, y: usize, w: usize, h: usize, r: i8) { + let st = self.mb_mut(); + for yy in (y..y + h).step_by(2) { + for xx in (x..x + w).step_by(2) { + st.ref_idx[list][(yy >> 1) * 2 + (xx >> 1)] = r; + } + } + } + + fn set_cur_mvd(&mut self, list: usize, x: usize, y: usize, w: usize, h: usize, d: [i32; 2]) { + let a = [d[0].unsigned_abs().min(127) as u8, d[1].unsigned_abs().min(127) as u8]; + let st = self.mb_mut(); + for yy in y..y + h { + for xx in x..x + w { + st.mvd[list][yy * 4 + xx] = a; + } + } + } + + fn decode_cbp(&mut self, c: &mut Cabac) -> u8 { + let mut luma = 0u8; + for b8 in 0..4usize { + let (bx, by) = ((b8 & 1) * 2, (b8 >> 1) * 2); + // A + let ca = if bx > 0 { ((luma >> (b8 - 1)) & 1 == 0) as usize } else { self.cbp_luma_cond(self.nb[0], b8 + 1) }; + let cb = if by > 0 { ((luma >> (b8 - 2)) & 1 == 0) as usize } else { self.cbp_luma_cond(self.nb[1], b8 + 2) }; + let _ = (bx, by); + let bin = c.decode_decision(73 + ca + 2 * cb); + luma |= (bin as u8) << b8; + } + let cond = |n: Option, bin: usize| -> usize { + let Some(m) = n else { return 0 }; + let st = &self.pic.mbs[m]; + if st.kind == MbKind::IPcm { + return 1; + } + if st.kind_is_skip() { + return 0; + } + let ch = st.cbp >> 4; + if bin == 0 { (ch != 0) as usize } else { (ch == 2) as usize } + }; + let inc0 = cond(self.nb[0], 0) + 2 * cond(self.nb[1], 0); + let mut chroma = 0u8; + if c.decode_decision(77 + inc0) == 1 { + let inc1 = cond(self.nb[0], 1) + 2 * cond(self.nb[1], 1) + 4; + chroma = 1 + c.decode_decision(77 + inc1) as u8; + } + luma | (chroma << 4) + } + + /// condTermFlagN for coded_block_pattern luma bin from a neighbouring MB; `b8n` is the neighbouring + /// 8x8 block index within that MB. + fn cbp_luma_cond(&self, n: Option, b8n: usize) -> usize { + let Some(m) = n else { return 0 }; + let st = &self.pic.mbs[m]; + if st.kind == MbKind::IPcm { + return 0; + } + if st.kind_is_skip() { + return 1; + } + ((st.cbp >> (b8n & 3)) & 1 == 0) as usize + } + + fn decode_qp_delta(&mut self, c: &mut Cabac) -> Result { + let inc = self.prev_qp_delta_nz as usize; + if c.decode_decision(60 + inc) == 0 { + return Ok(0); + } + let mut k = 1u32; + let mut ctx = 62; + while c.decode_decision(ctx) == 1 { + k += 1; + ctx = 63; + ensure!(k <= 128, "mb_qp_delta too large"); + } + let v = k.div_ceil(2) as i32; + Ok(if k % 2 == 1 { v } else { -v }) + } + + /// coded_block_flag ctxIdxInc condition for a neighbouring luma 4x4 block (cat 0..2) or chroma. + fn cbf_luma_cond(&self, bx: i32, by: i32, intra_cur: bool) -> usize { + if bx >= 0 && by >= 0 { + return (self.mb().nnz[(by * 4 + bx) as usize] != 0) as usize; + } + let n = if bx < 0 { self.nb[0] } else { self.nb[1] }; + let Some(m) = n else { return intra_cur as usize }; + let st = &self.pic.mbs[m]; + let (x, y) = if bx < 0 { (3, by as usize) } else { (bx as usize, 3) }; + (st.nnz[y * 4 + x] != 0) as usize + } + + fn cbf_dc_cond(&self, n: Option, bit: u8, intra_cur: bool) -> usize { + let Some(m) = n else { return intra_cur as usize }; + let st = &self.pic.mbs[m]; + if st.kind == MbKind::IPcm { + return 1; + } + (st.cbf_dc & (1 << bit) != 0) as usize + } + + fn cbf_chroma_ac_cond(&self, comp: usize, bx: i32, by: i32, intra_cur: bool) -> usize { + if bx >= 0 && by >= 0 { + return (self.mb().nnz_c[comp][(by * 2 + bx) as usize] != 0) as usize; + } + let n = if bx < 0 { self.nb[0] } else { self.nb[1] }; + let Some(m) = n else { return intra_cur as usize }; + let st = &self.pic.mbs[m]; + let (x, y) = if bx < 0 { (1, by as usize) } else { (bx as usize, 1) }; + (st.nnz_c[comp][y * 2 + x] != 0) as usize + } + + /// residual_block_cabac for ctxBlockCat `cat` (0..=5). Coefficients are written to `coeff` + /// in list order (indices 0..max_num). `cbf_inc` is the coded_block_flag ctxIdxInc (ignored for cat 5). + /// Returns the number of non-zero coefficients. + fn residual_block_cabac(&mut self, c: &mut Cabac, coeff: &mut [i32], cat: usize, max_num: usize, cbf_inc: usize) -> u8 { + if cat != 5 && c.decode_decision(85 + CAT_OFFSET[cat].0 + cbf_inc) == 0 { + return 0; + } + let (sig_base, last_base, abs_base) = + if cat == 5 { (402, 417, 426) } else { (105 + CAT_OFFSET[cat].1, 166 + CAT_OFFSET[cat].1, 227 + CAT_OFFSET[cat].2) }; + let (sig_tab, last_tab): (&[u8], &[u8]) = match cat { + 3 => (&CHROMA_DC_CTX, &CHROMA_DC_CTX), + 5 => (&SIG8_FRAME, &LAST8), + _ => (&IDENT_CTX, &IDENT_CTX), + }; + // significance map: positions of the significant coefficients in scan order + let mut pos = [0u8; 64]; + let mut n = 0usize; + let last_i = max_num - 1; + let mut found_last = false; + for i in 0..last_i { + if c.decode_decision(sig_base + sig_tab[i] as usize) == 1 { + pos[n] = i as u8; + n += 1; + if c.decode_decision(last_base + last_tab[i] as usize) == 1 { + found_last = true; + break; + } + } + } + if !found_last { + pos[n] = last_i as u8; + n += 1; + } + let mut eq1 = 0usize; + let mut gt1 = 0usize; + let gt1_cap = if cat == 3 { 3 } else { 4 }; + for &p in pos[..n].iter().rev() { + let inc0 = if gt1 != 0 { 0 } else { (1 + eq1).min(4) }; + let mut abs = 1i32; + if c.decode_decision(abs_base + inc0) == 1 { + let ctx = abs_base + 5 + gt1.min(gt1_cap); + let mut prefix = 1; + while prefix < 14 && c.decode_decision(ctx) == 1 { + prefix += 1; + } + abs = prefix + 1; + if prefix == 14 { + let mut k = 0; + let mut suf = 0i32; + while c.decode_bypass() == 1 { + suf += 1 << k; + k += 1; + if k > 24 { + break; + } + } + while k > 0 { + k -= 1; + suf += (c.decode_bypass() as i32) << k; + } + abs += suf; + } + gt1 += 1; + } else { + eq1 += 1; + } + coeff[p as usize] = if c.decode_bypass() == 1 { -abs } else { abs }; + } + n as u8 + } + + fn residual_cabac(&mut self, c: &mut Cabac, cbp: u8) -> Result<()> { + let kind = self.mb().kind; + let intra = kind.is_intra(); + let t8 = self.mb().transform_8x8; + if kind == MbKind::I16x16 { + let inc = self.cbf_dc_cond(self.nb[0], 0, intra) + 2 * self.cbf_dc_cond(self.nb[1], 0, intra); + let mut lv = [0i32; 16]; + if self.residual_block_cabac(c, &mut lv, 0, 16, inc) > 0 { + self.mb_mut().cbf_dc |= 1; + self.put_luma_dc(&lv); + } + } + for b8 in 0..4 { + let (bx8, by8) = ((b8 & 1) * 2, (b8 >> 1) * 2); + if cbp & (1 << b8) == 0 { + continue; + } + if t8 { + let mut lv = [0i32; 64]; + self.residual_block_cabac(c, &mut lv, 5, 64, 0); + let st = self.mb_mut(); + for r in [by8 * 4 + bx8, by8 * 4 + bx8 + 1, (by8 + 1) * 4 + bx8, (by8 + 1) * 4 + bx8 + 1] { + st.nnz[r] = 1; + } + self.put_luma8(b8, &lv); + } else { + for i4 in 0..4 { + let (bx, by) = (bx8 + (i4 & 1), by8 + (i4 >> 1)); + let raster = by * 4 + bx; + let inc = self.cbf_luma_cond(bx as i32 - 1, by as i32, intra) + 2 * self.cbf_luma_cond(bx as i32, by as i32 - 1, intra); + let mut lv = [0i32; 16]; + let n = if kind == MbKind::I16x16 { + self.residual_block_cabac(c, &mut lv[1..], 1, 15, inc) + } else { + self.residual_block_cabac(c, &mut lv, 2, 16, inc) + }; + self.mb_mut().nnz[raster] = n; + if n > 0 { + self.put_luma4(raster, &lv, kind == MbKind::I16x16); + } + } + } + } + let cbp_c = cbp >> 4; + if cbp_c != 0 { + for comp in 0..2 { + let bit = 1 + comp as u8; + let inc = self.cbf_dc_cond(self.nb[0], bit, intra) + 2 * self.cbf_dc_cond(self.nb[1], bit, intra); + let mut lv = [0i32; 4]; + if self.residual_block_cabac(c, &mut lv, 3, 4, inc) > 0 { + self.mb_mut().cbf_dc |= 1 << bit; + self.put_chroma_dc(comp, &lv); + } + } + } + if cbp_c == 2 { + for comp in 0..2 { + for b in 0..4 { + let (bx, by) = ((b & 1) as i32, (b >> 1) as i32); + let inc = self.cbf_chroma_ac_cond(comp, bx - 1, by, intra) + 2 * self.cbf_chroma_ac_cond(comp, bx, by - 1, intra); + let mut lv = [0i32; 16]; + let n = self.residual_block_cabac(c, &mut lv[1..], 4, 15, inc); + self.mb_mut().nnz_c[comp][b] = n; + if n > 0 { + self.put_chroma_ac(comp, b, &lv); + } + } + } + } + Ok(()) + } + + /// Parse and reconstruct one macroblock_layer() with CABAC. + fn decode_mb_cabac(&mut self, c: &mut Cabac) -> Result<()> { + let mut info = self.decode_mb_type(c)?; + self.cur = MbCur { info, ..Default::default() }; + if info.kind == MbKind::IPcm { + // pcm alignment + samples, then re-initialise the arithmetic decoder + let pos = c.bit_pos().div_ceil(8); + let data = c.data(); + ensure!(data.len() >= pos + 384, "truncated I_PCM macroblock"); + let Some(&samples) = data.get(pos..).and_then(|d| d.first_chunk::<384>()) else { + return invalid("truncated I_PCM macroblock"); + }; + c.set_bit_pos((pos + 384) * 8); + c.init_engine()?; + self.finish_pcm(); + self.write_pcm(&samples); + self.prev_qp_delta_nz = false; + return Ok(()); + } + let intra = info.kind.is_intra(); + if info.kind == MbKind::I4x4 && self.pps.transform_8x8_mode && self.decode_t8x8_flag(c) { + info.kind = MbKind::I8x8; + self.cur.info.kind = MbKind::I8x8; + self.mb_mut().transform_8x8 = true; + } + self.mb_mut().kind = info.kind; + let n_ref = self.sh.num_ref_idx_active; + match info.kind { + MbKind::I4x4 | MbKind::I8x8 => { + let n = if info.kind == MbKind::I8x8 { 4 } else { 16 }; + for i in 0..n { + self.cur.rem_mode[i] = if c.decode_decision(68) == 1 { + -1 + } else { + let b0 = c.decode_decision(69); + let b1 = c.decode_decision(69); + let b2 = c.decode_decision(69); + (b0 | (b1 << 1) | (b2 << 2)) as i8 + }; + } + self.derive_intra_modes(info.kind == MbKind::I8x8); + let m = self.decode_chroma_mode(c); + self.mb_mut().intra_chroma_mode = m; + } + MbKind::I16x16 => { + let m = self.decode_chroma_mode(c); + self.mb_mut().intra_chroma_mode = m; + } + MbKind::BDirect16x16 => { + self.mb_mut().direct8x8 = 0xf; + } + _ => { + if info.part == Part::P8x8 { + for p in 0..4 { + self.cur.sub[p] = self.decode_sub_mb_type(c)?; + } + let direct_mask: u8 = (0..4).filter(|&p| self.cur.sub[p].direct).map(|p| 1u8 << p).sum(); + self.mb_mut().direct8x8 = direct_mask; + for l in 0..2 { + for p in 0..4 { + let sub = self.cur.sub[p]; + if sub.direct || sub.pred & (1 << l) == 0 { + continue; + } + let (x, y) = ((p & 1) * 2, (p >> 1) * 2); + let r = if n_ref[l] > 1 && !info.ref0 { self.decode_ref_idx(c, l, x, y) as i8 } else { 0 }; + self.cur.ref_idx[l][p] = r; + self.set_cur_ref(l, x, y, 2, 2, r); + } + } + for l in 0..2 { + for p in 0..4 { + let sub = self.cur.sub[p]; + if sub.direct || sub.pred & (1 << l) == 0 { + continue; + } + let (x8, y8) = ((p & 1) * 2, (p >> 1) * 2); + for j in 0..sub.shape.num_parts() { + let (sx, sy, w, h) = sub.shape.rect(j); + let (x, y) = (x8 + sx, y8 + sy); + let dx = self.decode_mvd(c, l, 0, x, y); + let dy = self.decode_mvd(c, l, 1, x, y); + self.set_cur_mvd(l, x, y, w, h, [dx, dy]); + self.cur.mvd[l][p * 4 + j] = [dx as i16, dy as i16]; + } + } + } + } else { + let np = info.part.num_parts(); + for l in 0..2 { + for p in 0..np { + if info.pred[p] & (1 << l) == 0 { + continue; + } + let (x, y, w, h) = info.part.rect(p); + let r = if n_ref[l] > 1 { self.decode_ref_idx(c, l, x, y) as i8 } else { 0 }; + self.cur.ref_idx[l][p] = r; + self.set_cur_ref(l, x, y, w, h, r); + } + } + for l in 0..2 { + for p in 0..np { + if info.pred[p] & (1 << l) == 0 { + continue; + } + let (x, y, w, h) = info.part.rect(p); + let dx = self.decode_mvd(c, l, 0, x, y); + let dy = self.decode_mvd(c, l, 1, x, y); + self.set_cur_mvd(l, x, y, w, h, [dx, dy]); + self.cur.mvd[l][p * 4] = [dx as i16, dy as i16]; + } + } + } + } + } + let cbp = if info.kind == MbKind::I16x16 { info.i16_cbp } else { self.decode_cbp(c) }; + self.cur.cbp = cbp; + self.mb_mut().cbp = cbp; + if !intra && cbp & 15 != 0 && self.pps.transform_8x8_mode && self.no_sub_8x8_lt() { + let t = self.decode_t8x8_flag(c); + self.mb_mut().transform_8x8 = t; + } + if !intra { + // ref_idx written during parsing is replaced by the derived motion + self.derive_inter_motion()?; + } + if cbp != 0 || info.kind == MbKind::I16x16 { + let d = self.decode_qp_delta(c)?; + self.apply_qp_delta(d)?; + self.prev_qp_delta_nz = d != 0; + } else { + self.prev_qp_delta_nz = false; + } + self.store_qp(); + self.residual_cabac(c, cbp)?; + if intra { + self.reconstruct_intra(); + } else { + self.predict_inter(); + self.reconstruct_inter_residual(); + } + Ok(()) + } + + fn decode_t8x8_flag(&mut self, c: &mut Cabac) -> bool { + let cond = |n: Option| n.map(|m| self.pic.mbs[m].transform_8x8).unwrap_or(false) as usize; + let inc = cond(self.nb[0]) + cond(self.nb[1]); + c.decode_decision(399 + inc) == 1 + } + + fn decode_chroma_mode(&mut self, c: &mut Cabac) -> u8 { + let cond = |n: Option| { + n.map(|m| { + let st = &self.pic.mbs[m]; + st.kind.is_intra() && st.kind != MbKind::IPcm && st.intra_chroma_mode != 0 + }) + .unwrap_or(false) as usize + }; + let inc = cond(self.nb[0]) + cond(self.nb[1]); + if c.decode_decision(64 + inc) == 0 { + return 0; + } + if c.decode_decision(67) == 0 { + return 1; + } + if c.decode_decision(67) == 0 { 2 } else { 3 } + } +} diff --git a/crates/video/src/h264/cabac_tables.rs b/crates/video/src/h264/cabac_tables.rs new file mode 100644 index 00000000..c5c7205f --- /dev/null +++ b/crates/video/src/h264/cabac_tables.rs @@ -0,0 +1,525 @@ +// Generated from ITU-T H.264 (06/2026) Tables 9-12..9-33 (m, n) by a table-extraction script. +// Index: [0..=2] = cabac_init_idc for P/SP/B slices, [3] = I/SI slices. (0, 0) marks "na". +#[rustfmt::skip] +pub(crate) static CABAC_INIT_MN: [[(i8, i8); 1024]; 4] = [ + [ + (20, -15), (2, 54), (3, 74), (20, -15), (2, 54), (3, 74), (-28, 127), (-23, 104), + (-6, 53), (-1, 54), (7, 51), (23, 33), (23, 2), (21, 0), (1, 9), (0, 49), + (-37, 118), (5, 57), (-13, 78), (-11, 65), (1, 62), (12, 49), (-4, 73), (17, 50), + (18, 64), (9, 43), (29, 0), (26, 67), (16, 90), (9, 104), (-46, 127), (-20, 104), + (1, 67), (-13, 78), (-11, 65), (1, 62), (-6, 86), (-17, 95), (-6, 61), (9, 45), + (-3, 69), (-6, 81), (-11, 96), (6, 55), (7, 67), (-5, 86), (2, 88), (0, 58), + (-3, 76), (-10, 94), (5, 54), (4, 69), (-3, 81), (0, 88), (-7, 67), (-5, 74), + (-4, 74), (-5, 80), (-7, 72), (1, 58), (0, 41), (0, 63), (0, 63), (0, 63), + (-9, 83), (4, 86), (0, 97), (-7, 72), (13, 41), (3, 62), (0, 45), (-4, 78), + (-3, 96), (-27, 126), (-28, 98), (-25, 101), (-23, 67), (-28, 82), (-20, 94), (-16, 83), + (-22, 110), (-21, 91), (-18, 102), (-13, 93), (-29, 127), (-7, 92), (-5, 89), (-7, 96), + (-13, 108), (-3, 46), (-1, 65), (-1, 57), (-9, 93), (-3, 74), (-9, 92), (-8, 87), + (-23, 126), (5, 54), (6, 60), (6, 59), (6, 69), (-1, 48), (0, 68), (-4, 69), + (-8, 88), (-2, 85), (-6, 78), (-1, 75), (-7, 77), (2, 54), (5, 50), (-3, 68), + (1, 50), (6, 42), (-4, 81), (1, 63), (-4, 70), (0, 67), (2, 57), (-2, 76), + (11, 35), (4, 64), (1, 61), (11, 35), (18, 25), (12, 24), (13, 29), (13, 36), + (-10, 93), (-7, 73), (-2, 73), (13, 46), (9, 49), (-7, 100), (9, 53), (2, 53), + (5, 53), (-2, 61), (0, 56), (0, 56), (-13, 63), (-5, 60), (-1, 62), (4, 57), + (-6, 69), (4, 57), (14, 39), (4, 51), (13, 68), (3, 64), (1, 61), (9, 63), + (7, 50), (16, 39), (5, 44), (4, 52), (11, 48), (-5, 60), (-1, 59), (0, 59), + (22, 33), (5, 44), (14, 43), (-1, 78), (0, 60), (9, 69), (11, 28), (2, 40), + (3, 44), (0, 49), (0, 46), (2, 44), (2, 51), (0, 47), (4, 39), (2, 62), + (6, 46), (0, 54), (3, 54), (2, 58), (4, 63), (6, 51), (6, 57), (7, 53), + (6, 52), (6, 55), (11, 45), (14, 36), (8, 53), (-1, 82), (7, 55), (-3, 78), + (15, 46), (22, 31), (-1, 84), (25, 7), (30, -7), (28, 3), (28, 4), (32, 0), + (34, -1), (30, 6), (30, 6), (32, 9), (31, 19), (26, 27), (26, 30), (37, 20), + (28, 34), (17, 70), (1, 67), (5, 59), (9, 67), (16, 30), (18, 32), (18, 35), + (22, 29), (24, 31), (23, 38), (18, 43), (20, 41), (11, 63), (9, 59), (9, 64), + (-1, 94), (-2, 89), (-9, 108), (-6, 76), (-2, 44), (0, 45), (0, 52), (-3, 64), + (-2, 59), (-4, 70), (-4, 75), (-8, 82), (-17, 102), (-9, 77), (3, 24), (0, 42), + (0, 48), (0, 55), (-6, 59), (-7, 71), (-12, 83), (-11, 87), (-30, 119), (1, 58), + (-3, 29), (-1, 36), (1, 38), (2, 43), (-6, 55), (0, 58), (0, 64), (-3, 74), + (-10, 90), (0, 70), (-4, 29), (5, 31), (7, 42), (1, 59), (-2, 58), (-3, 72), + (-3, 81), (-11, 97), (0, 58), (8, 5), (10, 14), (14, 18), (13, 27), (2, 40), + (0, 58), (-3, 70), (-6, 79), (-8, 85), (0, 0), (-13, 106), (-16, 106), (-10, 87), + (-21, 114), (-18, 110), (-14, 98), (-22, 110), (-21, 106), (-18, 103), (-21, 107), (-23, 108), + (-26, 112), (-10, 96), (-12, 95), (-5, 91), (-9, 93), (-22, 94), (-5, 86), (9, 67), + (-4, 80), (-10, 85), (-1, 70), (7, 60), (9, 58), (5, 61), (12, 50), (15, 50), + (18, 49), (17, 54), (10, 41), (7, 46), (-1, 51), (7, 49), (8, 52), (9, 41), + (6, 47), (2, 55), (13, 41), (10, 44), (6, 50), (5, 53), (13, 49), (4, 63), + (6, 64), (-2, 69), (-2, 59), (6, 70), (10, 44), (9, 31), (12, 43), (3, 53), + (14, 34), (10, 38), (-3, 52), (13, 40), (17, 32), (7, 44), (7, 38), (13, 50), + (10, 57), (26, 43), (14, 11), (11, 14), (9, 11), (18, 11), (21, 9), (23, -2), + (32, -15), (32, -15), (34, -21), (39, -23), (42, -33), (41, -31), (46, -28), (38, -12), + (21, 29), (45, -24), (53, -45), (48, -26), (65, -43), (43, -19), (39, -10), (30, 9), + (18, 26), (20, 27), (0, 57), (-14, 82), (-5, 75), (-19, 97), (-35, 125), (27, 0), + (28, 0), (31, -4), (27, 6), (34, 8), (30, 10), (24, 22), (33, 19), (22, 32), + (26, 31), (21, 41), (26, 44), (23, 47), (16, 65), (14, 71), (8, 60), (6, 63), + (17, 65), (21, 24), (23, 20), (26, 23), (27, 32), (28, 23), (28, 24), (23, 40), + (24, 32), (28, 29), (23, 42), (19, 57), (22, 53), (22, 61), (11, 86), (12, 40), + (11, 51), (14, 59), (-4, 79), (-7, 71), (-5, 69), (-9, 70), (-8, 66), (-10, 68), + (-19, 73), (-12, 69), (-16, 70), (-15, 67), (-20, 62), (-19, 70), (-16, 66), (-22, 65), + (-20, 63), (9, -2), (26, -9), (33, -9), (39, -7), (41, -2), (45, 3), (49, 9), + (45, 27), (36, 59), (-6, 66), (-7, 35), (-7, 42), (-8, 45), (-5, 48), (-12, 56), + (-6, 60), (-5, 62), (-8, 66), (-8, 76), (-5, 85), (-6, 81), (-10, 77), (-7, 81), + (-17, 80), (-18, 73), (-4, 74), (-10, 83), (-9, 71), (-9, 67), (-1, 61), (-8, 66), + (-14, 66), (0, 59), (2, 59), (21, -13), (33, -14), (39, -7), (46, -2), (51, 2), + (60, 6), (61, 17), (55, 34), (42, 62), (-7, 92), (-5, 89), (-7, 96), (-13, 108), + (-3, 46), (-1, 65), (-1, 57), (-9, 93), (-3, 74), (-9, 92), (-8, 87), (-23, 126), + (-7, 92), (-5, 89), (-7, 96), (-13, 108), (-3, 46), (-1, 65), (-1, 57), (-9, 93), + (-3, 74), (-9, 92), (-8, 87), (-23, 126), (-2, 85), (-6, 78), (-1, 75), (-7, 77), + (2, 54), (5, 50), (-3, 68), (1, 50), (6, 42), (-4, 81), (1, 63), (-4, 70), + (0, 67), (2, 57), (-2, 76), (11, 35), (4, 64), (1, 61), (11, 35), (18, 25), + (12, 24), (13, 29), (13, 36), (-10, 93), (-7, 73), (-2, 73), (13, 46), (9, 49), + (-7, 100), (9, 53), (2, 53), (5, 53), (-2, 61), (0, 56), (0, 56), (-13, 63), + (-5, 60), (-1, 62), (4, 57), (-6, 69), (4, 57), (14, 39), (4, 51), (13, 68), + (-2, 85), (-6, 78), (-1, 75), (-7, 77), (2, 54), (5, 50), (-3, 68), (1, 50), + (6, 42), (-4, 81), (1, 63), (-4, 70), (0, 67), (2, 57), (-2, 76), (11, 35), + (4, 64), (1, 61), (11, 35), (18, 25), (12, 24), (13, 29), (13, 36), (-10, 93), + (-7, 73), (-2, 73), (13, 46), (9, 49), (-7, 100), (9, 53), (2, 53), (5, 53), + (-2, 61), (0, 56), (0, 56), (-13, 63), (-5, 60), (-1, 62), (4, 57), (-6, 69), + (4, 57), (14, 39), (4, 51), (13, 68), (11, 28), (2, 40), (3, 44), (0, 49), + (0, 46), (2, 44), (2, 51), (0, 47), (4, 39), (2, 62), (6, 46), (0, 54), + (3, 54), (2, 58), (4, 63), (6, 51), (6, 57), (7, 53), (6, 52), (6, 55), + (11, 45), (14, 36), (8, 53), (-1, 82), (7, 55), (-3, 78), (15, 46), (22, 31), + (-1, 84), (25, 7), (30, -7), (28, 3), (28, 4), (32, 0), (34, -1), (30, 6), + (30, 6), (32, 9), (31, 19), (26, 27), (26, 30), (37, 20), (28, 34), (17, 70), + (11, 28), (2, 40), (3, 44), (0, 49), (0, 46), (2, 44), (2, 51), (0, 47), + (4, 39), (2, 62), (6, 46), (0, 54), (3, 54), (2, 58), (4, 63), (6, 51), + (6, 57), (7, 53), (6, 52), (6, 55), (11, 45), (14, 36), (8, 53), (-1, 82), + (7, 55), (-3, 78), (15, 46), (22, 31), (-1, 84), (25, 7), (30, -7), (28, 3), + (28, 4), (32, 0), (34, -1), (30, 6), (30, 6), (32, 9), (31, 19), (26, 27), + (26, 30), (37, 20), (28, 34), (17, 70), (-4, 79), (-7, 71), (-5, 69), (-9, 70), + (-8, 66), (-10, 68), (-19, 73), (-12, 69), (-16, 70), (-15, 67), (-20, 62), (-19, 70), + (-16, 66), (-22, 65), (-20, 63), (-5, 85), (-6, 81), (-10, 77), (-7, 81), (-17, 80), + (-18, 73), (-4, 74), (-10, 83), (-9, 71), (-9, 67), (-1, 61), (-8, 66), (-14, 66), + (0, 59), (2, 59), (9, -2), (26, -9), (33, -9), (39, -7), (41, -2), (45, 3), + (49, 9), (45, 27), (36, 59), (21, -13), (33, -14), (39, -7), (46, -2), (51, 2), + (60, 6), (61, 17), (55, 34), (42, 62), (-6, 66), (-7, 35), (-7, 42), (-8, 45), + (-5, 48), (-12, 56), (-6, 60), (-5, 62), (-8, 66), (-8, 76), (-4, 79), (-7, 71), + (-5, 69), (-9, 70), (-8, 66), (-10, 68), (-19, 73), (-12, 69), (-16, 70), (-15, 67), + (-20, 62), (-19, 70), (-16, 66), (-22, 65), (-20, 63), (-5, 85), (-6, 81), (-10, 77), + (-7, 81), (-17, 80), (-18, 73), (-4, 74), (-10, 83), (-9, 71), (-9, 67), (-1, 61), + (-8, 66), (-14, 66), (0, 59), (2, 59), (9, -2), (26, -9), (33, -9), (39, -7), + (41, -2), (45, 3), (49, 9), (45, 27), (36, 59), (21, -13), (33, -14), (39, -7), + (46, -2), (51, 2), (60, 6), (61, 17), (55, 34), (42, 62), (-6, 66), (-7, 35), + (-7, 42), (-8, 45), (-5, 48), (-12, 56), (-6, 60), (-5, 62), (-8, 66), (-8, 76), + (-13, 106), (-16, 106), (-10, 87), (-21, 114), (-18, 110), (-14, 98), (-22, 110), (-21, 106), + (-18, 103), (-21, 107), (-23, 108), (-26, 112), (-10, 96), (-12, 95), (-5, 91), (-9, 93), + (-22, 94), (-5, 86), (9, 67), (-4, 80), (-10, 85), (-1, 70), (7, 60), (9, 58), + (5, 61), (12, 50), (15, 50), (18, 49), (17, 54), (10, 41), (7, 46), (-1, 51), + (7, 49), (8, 52), (9, 41), (6, 47), (2, 55), (13, 41), (10, 44), (6, 50), + (5, 53), (13, 49), (4, 63), (6, 64), (-13, 106), (-16, 106), (-10, 87), (-21, 114), + (-18, 110), (-14, 98), (-22, 110), (-21, 106), (-18, 103), (-21, 107), (-23, 108), (-26, 112), + (-10, 96), (-12, 95), (-5, 91), (-9, 93), (-22, 94), (-5, 86), (9, 67), (-4, 80), + (-10, 85), (-1, 70), (7, 60), (9, 58), (5, 61), (12, 50), (15, 50), (18, 49), + (17, 54), (10, 41), (7, 46), (-1, 51), (7, 49), (8, 52), (9, 41), (6, 47), + (2, 55), (13, 41), (10, 44), (6, 50), (5, 53), (13, 49), (4, 63), (6, 64), + (14, 11), (11, 14), (9, 11), (18, 11), (21, 9), (23, -2), (32, -15), (32, -15), + (34, -21), (39, -23), (42, -33), (41, -31), (46, -28), (38, -12), (21, 29), (45, -24), + (53, -45), (48, -26), (65, -43), (43, -19), (39, -10), (30, 9), (18, 26), (20, 27), + (0, 57), (-14, 82), (-5, 75), (-19, 97), (-35, 125), (27, 0), (28, 0), (31, -4), + (27, 6), (34, 8), (30, 10), (24, 22), (33, 19), (22, 32), (26, 31), (21, 41), + (26, 44), (23, 47), (16, 65), (14, 71), (14, 11), (11, 14), (9, 11), (18, 11), + (21, 9), (23, -2), (32, -15), (32, -15), (34, -21), (39, -23), (42, -33), (41, -31), + (46, -28), (38, -12), (21, 29), (45, -24), (53, -45), (48, -26), (65, -43), (43, -19), + (39, -10), (30, 9), (18, 26), (20, 27), (0, 57), (-14, 82), (-5, 75), (-19, 97), + (-35, 125), (27, 0), (28, 0), (31, -4), (27, 6), (34, 8), (30, 10), (24, 22), + (33, 19), (22, 32), (26, 31), (21, 41), (26, 44), (23, 47), (16, 65), (14, 71), + (-6, 76), (-2, 44), (0, 45), (0, 52), (-3, 64), (-2, 59), (-4, 70), (-4, 75), + (-8, 82), (-17, 102), (-9, 77), (3, 24), (0, 42), (0, 48), (0, 55), (-6, 59), + (-7, 71), (-12, 83), (-11, 87), (-30, 119), (1, 58), (-3, 29), (-1, 36), (1, 38), + (2, 43), (-6, 55), (0, 58), (0, 64), (-3, 74), (-10, 90), (-6, 76), (-2, 44), + (0, 45), (0, 52), (-3, 64), (-2, 59), (-4, 70), (-4, 75), (-8, 82), (-17, 102), + (-9, 77), (3, 24), (0, 42), (0, 48), (0, 55), (-6, 59), (-7, 71), (-12, 83), + (-11, 87), (-30, 119), (1, 58), (-3, 29), (-1, 36), (1, 38), (2, 43), (-6, 55), + (0, 58), (0, 64), (-3, 74), (-10, 90), (-3, 74), (-9, 92), (-8, 87), (-23, 126), + (-3, 74), (-9, 92), (-8, 87), (-23, 126), (-3, 74), (-9, 92), (-8, 87), (-23, 126), + ], + [ + (20, -15), (2, 54), (3, 74), (20, -15), (2, 54), (3, 74), (-28, 127), (-23, 104), + (-6, 53), (-1, 54), (7, 51), (22, 25), (34, 0), (16, 0), (-2, 9), (4, 41), + (-29, 118), (2, 65), (-6, 71), (-13, 79), (5, 52), (9, 50), (-3, 70), (10, 54), + (26, 34), (19, 22), (40, 0), (57, 2), (41, 36), (26, 69), (-45, 127), (-15, 101), + (-4, 76), (-6, 71), (-13, 79), (5, 52), (6, 69), (-13, 90), (0, 52), (8, 43), + (-2, 69), (-5, 82), (-10, 96), (2, 59), (2, 75), (-3, 87), (-3, 100), (1, 56), + (-3, 74), (-6, 85), (0, 59), (-3, 81), (-7, 86), (-5, 95), (-1, 66), (-1, 77), + (1, 70), (-2, 86), (-5, 72), (0, 61), (0, 41), (0, 63), (0, 63), (0, 63), + (-9, 83), (4, 86), (0, 97), (-7, 72), (13, 41), (3, 62), (13, 15), (7, 51), + (2, 80), (-39, 127), (-18, 91), (-17, 96), (-26, 81), (-35, 98), (-24, 102), (-23, 97), + (-27, 119), (-24, 99), (-21, 110), (-18, 102), (-36, 127), (0, 80), (-5, 89), (-7, 94), + (-4, 92), (0, 39), (0, 65), (-15, 84), (-35, 127), (-2, 73), (-12, 104), (-9, 91), + (-31, 127), (3, 55), (7, 56), (7, 55), (8, 61), (-3, 53), (0, 68), (-7, 74), + (-9, 88), (-13, 103), (-13, 91), (-9, 89), (-14, 92), (-8, 76), (-12, 87), (-23, 110), + (-24, 105), (-10, 78), (-20, 112), (-17, 99), (-78, 127), (-70, 127), (-50, 127), (-46, 127), + (-4, 66), (-5, 78), (-4, 71), (-8, 72), (2, 59), (-1, 55), (-7, 70), (-6, 75), + (-8, 89), (-34, 119), (-3, 75), (32, 20), (30, 22), (-44, 127), (0, 54), (-5, 61), + (0, 58), (-1, 60), (-3, 61), (-8, 67), (-25, 84), (-14, 74), (-5, 65), (5, 52), + (2, 57), (0, 61), (-9, 69), (-11, 70), (18, 55), (-4, 71), (0, 58), (7, 61), + (9, 41), (18, 25), (9, 32), (5, 43), (9, 47), (0, 44), (0, 51), (2, 46), + (19, 38), (-4, 66), (15, 38), (12, 42), (9, 34), (0, 89), (4, 45), (10, 28), + (10, 31), (33, -11), (52, -43), (18, 15), (28, 0), (35, -22), (38, -25), (34, 0), + (39, -18), (32, -12), (102, -94), (0, 0), (56, -15), (33, -4), (29, 10), (37, -5), + (51, -29), (39, -9), (52, -34), (69, -58), (67, -63), (44, -5), (32, 7), (55, -29), + (32, 1), (0, 0), (27, 36), (33, -25), (34, -30), (36, -28), (38, -28), (38, -27), + (34, -18), (35, -16), (34, -14), (32, -8), (37, -6), (35, 0), (30, 10), (28, 18), + (26, 25), (29, 41), (0, 75), (2, 72), (8, 77), (14, 35), (18, 31), (17, 35), + (21, 30), (17, 45), (20, 42), (18, 45), (27, 26), (16, 54), (7, 66), (16, 56), + (11, 73), (10, 67), (-10, 116), (-23, 112), (-15, 71), (-7, 61), (0, 53), (-5, 66), + (-11, 77), (-9, 80), (-9, 84), (-10, 87), (-34, 127), (-21, 101), (-3, 39), (-5, 53), + (-7, 61), (-11, 75), (-15, 77), (-17, 91), (-25, 107), (-25, 111), (-28, 122), (-11, 76), + (-10, 44), (-10, 52), (-10, 57), (-9, 58), (-16, 72), (-7, 69), (-4, 69), (-5, 74), + (-9, 86), (2, 66), (-9, 34), (1, 32), (11, 31), (5, 52), (-2, 55), (-2, 67), + (0, 73), (-8, 89), (3, 52), (7, 4), (10, 8), (17, 8), (16, 19), (3, 37), + (-1, 61), (-5, 73), (-1, 70), (-4, 78), (0, 0), (-21, 126), (-23, 124), (-20, 110), + (-26, 126), (-25, 124), (-17, 105), (-27, 121), (-27, 117), (-17, 102), (-26, 117), (-27, 116), + (-33, 122), (-10, 95), (-14, 100), (-8, 95), (-17, 111), (-28, 114), (-6, 89), (-2, 80), + (-4, 82), (-9, 85), (-8, 81), (-1, 72), (5, 64), (1, 67), (9, 56), (0, 69), + (1, 69), (7, 69), (-7, 69), (-6, 67), (-16, 77), (-2, 64), (2, 61), (-6, 67), + (-3, 64), (2, 57), (-3, 65), (-3, 66), (0, 62), (9, 51), (-1, 66), (-2, 71), + (-2, 75), (-1, 70), (-9, 72), (14, 60), (16, 37), (0, 47), (18, 35), (11, 37), + (12, 41), (10, 41), (2, 48), (12, 41), (13, 41), (0, 59), (3, 50), (19, 40), + (3, 66), (18, 50), (19, -6), (18, -6), (14, 0), (26, -12), (31, -16), (33, -25), + (33, -22), (37, -28), (39, -30), (42, -30), (47, -42), (45, -36), (49, -34), (41, -17), + (32, 9), (69, -71), (63, -63), (66, -64), (77, -74), (54, -39), (52, -35), (41, -10), + (36, 0), (40, -1), (30, 14), (28, 26), (23, 37), (12, 55), (11, 65), (37, -33), + (39, -36), (40, -37), (38, -30), (46, -33), (42, -30), (40, -24), (49, -29), (38, -12), + (40, -10), (38, -3), (46, -5), (31, 20), (29, 30), (25, 44), (12, 48), (11, 49), + (26, 45), (22, 22), (23, 22), (27, 21), (33, 20), (26, 28), (30, 24), (27, 34), + (18, 42), (25, 39), (18, 50), (12, 70), (21, 54), (14, 71), (11, 83), (25, 32), + (21, 49), (21, 54), (-5, 85), (-6, 81), (-10, 77), (-7, 81), (-17, 80), (-18, 73), + (-4, 74), (-10, 83), (-9, 71), (-9, 67), (-1, 61), (-8, 66), (-14, 66), (0, 59), + (2, 59), (17, -10), (32, -13), (42, -9), (49, -5), (53, 0), (64, 3), (68, 10), + (66, 27), (47, 57), (-5, 71), (0, 24), (-1, 36), (-2, 42), (-2, 52), (-9, 57), + (-6, 63), (-4, 65), (-4, 67), (-7, 82), (-3, 81), (-3, 76), (-7, 72), (-6, 78), + (-12, 72), (-14, 68), (-3, 70), (-6, 76), (-5, 66), (-5, 62), (0, 57), (-4, 61), + (-9, 60), (1, 54), (2, 58), (17, -10), (32, -13), (42, -9), (49, -5), (53, 0), + (64, 3), (68, 10), (66, 27), (47, 57), (0, 80), (-5, 89), (-7, 94), (-4, 92), + (0, 39), (0, 65), (-15, 84), (-35, 127), (-2, 73), (-12, 104), (-9, 91), (-31, 127), + (0, 80), (-5, 89), (-7, 94), (-4, 92), (0, 39), (0, 65), (-15, 84), (-35, 127), + (-2, 73), (-12, 104), (-9, 91), (-31, 127), (-13, 103), (-13, 91), (-9, 89), (-14, 92), + (-8, 76), (-12, 87), (-23, 110), (-24, 105), (-10, 78), (-20, 112), (-17, 99), (-78, 127), + (-70, 127), (-50, 127), (-46, 127), (-4, 66), (-5, 78), (-4, 71), (-8, 72), (2, 59), + (-1, 55), (-7, 70), (-6, 75), (-8, 89), (-34, 119), (-3, 75), (32, 20), (30, 22), + (-44, 127), (0, 54), (-5, 61), (0, 58), (-1, 60), (-3, 61), (-8, 67), (-25, 84), + (-14, 74), (-5, 65), (5, 52), (2, 57), (0, 61), (-9, 69), (-11, 70), (18, 55), + (-13, 103), (-13, 91), (-9, 89), (-14, 92), (-8, 76), (-12, 87), (-23, 110), (-24, 105), + (-10, 78), (-20, 112), (-17, 99), (-78, 127), (-70, 127), (-50, 127), (-46, 127), (-4, 66), + (-5, 78), (-4, 71), (-8, 72), (2, 59), (-1, 55), (-7, 70), (-6, 75), (-8, 89), + (-34, 119), (-3, 75), (32, 20), (30, 22), (-44, 127), (0, 54), (-5, 61), (0, 58), + (-1, 60), (-3, 61), (-8, 67), (-25, 84), (-14, 74), (-5, 65), (5, 52), (2, 57), + (0, 61), (-9, 69), (-11, 70), (18, 55), (4, 45), (10, 28), (10, 31), (33, -11), + (52, -43), (18, 15), (28, 0), (35, -22), (38, -25), (34, 0), (39, -18), (32, -12), + (102, -94), (0, 0), (56, -15), (33, -4), (29, 10), (37, -5), (51, -29), (39, -9), + (52, -34), (69, -58), (67, -63), (44, -5), (32, 7), (55, -29), (32, 1), (0, 0), + (27, 36), (33, -25), (34, -30), (36, -28), (38, -28), (38, -27), (34, -18), (35, -16), + (34, -14), (32, -8), (37, -6), (35, 0), (30, 10), (28, 18), (26, 25), (29, 41), + (4, 45), (10, 28), (10, 31), (33, -11), (52, -43), (18, 15), (28, 0), (35, -22), + (38, -25), (34, 0), (39, -18), (32, -12), (102, -94), (0, 0), (56, -15), (33, -4), + (29, 10), (37, -5), (51, -29), (39, -9), (52, -34), (69, -58), (67, -63), (44, -5), + (32, 7), (55, -29), (32, 1), (0, 0), (27, 36), (33, -25), (34, -30), (36, -28), + (38, -28), (38, -27), (34, -18), (35, -16), (34, -14), (32, -8), (37, -6), (35, 0), + (30, 10), (28, 18), (26, 25), (29, 41), (-5, 85), (-6, 81), (-10, 77), (-7, 81), + (-17, 80), (-18, 73), (-4, 74), (-10, 83), (-9, 71), (-9, 67), (-1, 61), (-8, 66), + (-14, 66), (0, 59), (2, 59), (-3, 81), (-3, 76), (-7, 72), (-6, 78), (-12, 72), + (-14, 68), (-3, 70), (-6, 76), (-5, 66), (-5, 62), (0, 57), (-4, 61), (-9, 60), + (1, 54), (2, 58), (17, -10), (32, -13), (42, -9), (49, -5), (53, 0), (64, 3), + (68, 10), (66, 27), (47, 57), (17, -10), (32, -13), (42, -9), (49, -5), (53, 0), + (64, 3), (68, 10), (66, 27), (47, 57), (-5, 71), (0, 24), (-1, 36), (-2, 42), + (-2, 52), (-9, 57), (-6, 63), (-4, 65), (-4, 67), (-7, 82), (-5, 85), (-6, 81), + (-10, 77), (-7, 81), (-17, 80), (-18, 73), (-4, 74), (-10, 83), (-9, 71), (-9, 67), + (-1, 61), (-8, 66), (-14, 66), (0, 59), (2, 59), (-3, 81), (-3, 76), (-7, 72), + (-6, 78), (-12, 72), (-14, 68), (-3, 70), (-6, 76), (-5, 66), (-5, 62), (0, 57), + (-4, 61), (-9, 60), (1, 54), (2, 58), (17, -10), (32, -13), (42, -9), (49, -5), + (53, 0), (64, 3), (68, 10), (66, 27), (47, 57), (17, -10), (32, -13), (42, -9), + (49, -5), (53, 0), (64, 3), (68, 10), (66, 27), (47, 57), (-5, 71), (0, 24), + (-1, 36), (-2, 42), (-2, 52), (-9, 57), (-6, 63), (-4, 65), (-4, 67), (-7, 82), + (-21, 126), (-23, 124), (-20, 110), (-26, 126), (-25, 124), (-17, 105), (-27, 121), (-27, 117), + (-17, 102), (-26, 117), (-27, 116), (-33, 122), (-10, 95), (-14, 100), (-8, 95), (-17, 111), + (-28, 114), (-6, 89), (-2, 80), (-4, 82), (-9, 85), (-8, 81), (-1, 72), (5, 64), + (1, 67), (9, 56), (0, 69), (1, 69), (7, 69), (-7, 69), (-6, 67), (-16, 77), + (-2, 64), (2, 61), (-6, 67), (-3, 64), (2, 57), (-3, 65), (-3, 66), (0, 62), + (9, 51), (-1, 66), (-2, 71), (-2, 75), (-21, 126), (-23, 124), (-20, 110), (-26, 126), + (-25, 124), (-17, 105), (-27, 121), (-27, 117), (-17, 102), (-26, 117), (-27, 116), (-33, 122), + (-10, 95), (-14, 100), (-8, 95), (-17, 111), (-28, 114), (-6, 89), (-2, 80), (-4, 82), + (-9, 85), (-8, 81), (-1, 72), (5, 64), (1, 67), (9, 56), (0, 69), (1, 69), + (7, 69), (-7, 69), (-6, 67), (-16, 77), (-2, 64), (2, 61), (-6, 67), (-3, 64), + (2, 57), (-3, 65), (-3, 66), (0, 62), (9, 51), (-1, 66), (-2, 71), (-2, 75), + (19, -6), (18, -6), (14, 0), (26, -12), (31, -16), (33, -25), (33, -22), (37, -28), + (39, -30), (42, -30), (47, -42), (45, -36), (49, -34), (41, -17), (32, 9), (69, -71), + (63, -63), (66, -64), (77, -74), (54, -39), (52, -35), (41, -10), (36, 0), (40, -1), + (30, 14), (28, 26), (23, 37), (12, 55), (11, 65), (37, -33), (39, -36), (40, -37), + (38, -30), (46, -33), (42, -30), (40, -24), (49, -29), (38, -12), (40, -10), (38, -3), + (46, -5), (31, 20), (29, 30), (25, 44), (19, -6), (18, -6), (14, 0), (26, -12), + (31, -16), (33, -25), (33, -22), (37, -28), (39, -30), (42, -30), (47, -42), (45, -36), + (49, -34), (41, -17), (32, 9), (69, -71), (63, -63), (66, -64), (77, -74), (54, -39), + (52, -35), (41, -10), (36, 0), (40, -1), (30, 14), (28, 26), (23, 37), (12, 55), + (11, 65), (37, -33), (39, -36), (40, -37), (38, -30), (46, -33), (42, -30), (40, -24), + (49, -29), (38, -12), (40, -10), (38, -3), (46, -5), (31, 20), (29, 30), (25, 44), + (-23, 112), (-15, 71), (-7, 61), (0, 53), (-5, 66), (-11, 77), (-9, 80), (-9, 84), + (-10, 87), (-34, 127), (-21, 101), (-3, 39), (-5, 53), (-7, 61), (-11, 75), (-15, 77), + (-17, 91), (-25, 107), (-25, 111), (-28, 122), (-11, 76), (-10, 44), (-10, 52), (-10, 57), + (-9, 58), (-16, 72), (-7, 69), (-4, 69), (-5, 74), (-9, 86), (-23, 112), (-15, 71), + (-7, 61), (0, 53), (-5, 66), (-11, 77), (-9, 80), (-9, 84), (-10, 87), (-34, 127), + (-21, 101), (-3, 39), (-5, 53), (-7, 61), (-11, 75), (-15, 77), (-17, 91), (-25, 107), + (-25, 111), (-28, 122), (-11, 76), (-10, 44), (-10, 52), (-10, 57), (-9, 58), (-16, 72), + (-7, 69), (-4, 69), (-5, 74), (-9, 86), (-2, 73), (-12, 104), (-9, 91), (-31, 127), + (-2, 73), (-12, 104), (-9, 91), (-31, 127), (-2, 73), (-12, 104), (-9, 91), (-31, 127), + ], + [ + (20, -15), (2, 54), (3, 74), (20, -15), (2, 54), (3, 74), (-28, 127), (-23, 104), + (-6, 53), (-1, 54), (7, 51), (29, 16), (25, 0), (14, 0), (-10, 51), (-3, 62), + (-27, 99), (26, 16), (-4, 85), (-24, 102), (5, 57), (6, 57), (-17, 73), (14, 57), + (20, 40), (20, 10), (29, 0), (54, 0), (37, 42), (12, 97), (-32, 127), (-22, 117), + (-2, 74), (-4, 85), (-24, 102), (5, 57), (-6, 93), (-14, 88), (-6, 44), (4, 55), + (-11, 89), (-15, 103), (-21, 116), (19, 57), (20, 58), (4, 84), (6, 96), (1, 63), + (-5, 85), (-13, 106), (5, 63), (6, 75), (-3, 90), (-1, 101), (3, 55), (-4, 79), + (-2, 75), (-12, 97), (-7, 50), (1, 60), (0, 41), (0, 63), (0, 63), (0, 63), + (-9, 83), (4, 86), (0, 97), (-7, 72), (13, 41), (3, 62), (7, 34), (-9, 88), + (-20, 127), (-36, 127), (-17, 91), (-14, 95), (-25, 84), (-25, 86), (-12, 89), (-17, 91), + (-31, 127), (-14, 76), (-18, 103), (-13, 90), (-37, 127), (11, 80), (5, 76), (2, 84), + (5, 78), (-6, 55), (4, 61), (-14, 83), (-37, 127), (-5, 79), (-11, 104), (-11, 91), + (-30, 127), (0, 65), (-2, 79), (0, 72), (-4, 92), (-6, 56), (3, 68), (-8, 71), + (-13, 98), (-4, 86), (-12, 88), (-5, 82), (-3, 72), (-4, 67), (-8, 72), (-16, 89), + (-9, 69), (-1, 59), (5, 66), (4, 57), (-4, 71), (-2, 71), (2, 58), (-1, 74), + (-4, 44), (-1, 69), (0, 62), (-7, 51), (-4, 47), (-6, 42), (-3, 41), (-6, 53), + (8, 76), (-9, 78), (-11, 83), (9, 52), (0, 67), (-5, 90), (1, 67), (-15, 72), + (-5, 75), (-8, 80), (-21, 83), (-21, 64), (-13, 31), (-25, 64), (-29, 94), (9, 75), + (17, 63), (-8, 74), (-5, 35), (-2, 27), (13, 91), (3, 65), (-7, 69), (8, 77), + (-10, 66), (3, 62), (-3, 68), (-20, 81), (0, 30), (1, 7), (-3, 23), (-21, 74), + (16, 66), (-23, 124), (17, 37), (44, -18), (50, -34), (-22, 127), (4, 39), (0, 42), + (7, 34), (11, 29), (8, 31), (6, 37), (7, 42), (3, 40), (8, 33), (13, 43), + (13, 36), (4, 47), (3, 55), (2, 58), (6, 60), (8, 44), (11, 44), (14, 42), + (7, 48), (4, 56), (4, 52), (13, 37), (9, 49), (19, 58), (10, 48), (12, 45), + (0, 69), (20, 33), (8, 63), (35, -18), (33, -25), (28, -3), (24, 10), (27, 0), + (34, -14), (52, -44), (39, -24), (19, 17), (31, 25), (36, 29), (24, 33), (34, 15), + (30, 20), (22, 73), (20, 34), (19, 31), (27, 44), (19, 16), (15, 36), (15, 36), + (21, 28), (25, 21), (30, 20), (31, 12), (27, 16), (24, 42), (0, 93), (14, 56), + (15, 57), (26, 38), (-24, 127), (-24, 115), (-22, 82), (-9, 62), (0, 53), (0, 59), + (-14, 85), (-13, 89), (-13, 94), (-11, 92), (-29, 127), (-21, 100), (-14, 57), (-12, 67), + (-11, 71), (-10, 77), (-21, 85), (-16, 88), (-23, 104), (-15, 98), (-37, 127), (-10, 82), + (-8, 48), (-8, 61), (-8, 66), (-7, 70), (-14, 75), (-10, 79), (-9, 83), (-12, 92), + (-18, 108), (-4, 79), (-22, 69), (-16, 75), (-2, 58), (1, 58), (-13, 78), (-9, 83), + (-4, 81), (-13, 99), (-13, 81), (-6, 38), (-13, 62), (-6, 58), (-2, 59), (-16, 73), + (-10, 76), (-13, 86), (-9, 83), (-10, 87), (0, 0), (-22, 127), (-25, 127), (-25, 120), + (-27, 127), (-19, 114), (-23, 117), (-25, 118), (-26, 117), (-24, 113), (-28, 118), (-31, 120), + (-37, 124), (-10, 94), (-15, 102), (-10, 99), (-13, 106), (-50, 127), (-5, 92), (17, 57), + (-5, 86), (-13, 94), (-12, 91), (-2, 77), (0, 71), (-1, 73), (4, 64), (-7, 81), + (5, 64), (15, 57), (1, 67), (0, 68), (-10, 67), (1, 68), (0, 77), (2, 64), + (0, 68), (-5, 78), (7, 55), (5, 59), (2, 65), (14, 54), (15, 44), (5, 60), + (2, 70), (-2, 76), (-18, 86), (12, 70), (5, 64), (-12, 70), (11, 55), (5, 56), + (0, 69), (2, 65), (-6, 74), (5, 54), (7, 54), (-6, 76), (-11, 82), (-2, 77), + (-2, 77), (25, 42), (17, -13), (16, -9), (17, -12), (27, -21), (37, -30), (41, -40), + (42, -41), (48, -47), (39, -32), (46, -40), (52, -51), (46, -41), (52, -39), (43, -19), + (32, 11), (61, -55), (56, -46), (62, -50), (81, -67), (45, -20), (35, -2), (28, 15), + (34, 1), (39, 1), (30, 17), (20, 38), (18, 45), (15, 54), (0, 79), (36, -16), + (37, -14), (37, -17), (32, 1), (34, 15), (29, 15), (24, 25), (34, 22), (31, 16), + (35, 18), (31, 28), (33, 41), (36, 28), (27, 47), (21, 62), (18, 31), (19, 26), + (36, 24), (24, 23), (27, 16), (24, 30), (31, 29), (22, 41), (22, 42), (16, 60), + (15, 52), (14, 60), (3, 78), (-16, 123), (21, 53), (22, 56), (25, 61), (21, 33), + (19, 50), (17, 61), (-3, 78), (-8, 74), (-9, 72), (-10, 72), (-18, 75), (-12, 71), + (-11, 63), (-5, 70), (-17, 75), (-14, 72), (-16, 67), (-8, 53), (-14, 59), (-9, 52), + (-11, 68), (9, -2), (30, -10), (31, -4), (33, -1), (33, 7), (31, 12), (37, 23), + (31, 38), (20, 64), (-9, 71), (-7, 37), (-8, 44), (-11, 49), (-10, 56), (-12, 59), + (-8, 63), (-9, 67), (-6, 68), (-10, 79), (-3, 78), (-8, 74), (-9, 72), (-10, 72), + (-18, 75), (-12, 71), (-11, 63), (-5, 70), (-17, 75), (-14, 72), (-16, 67), (-8, 53), + (-14, 59), (-9, 52), (-11, 68), (9, -2), (30, -10), (31, -4), (33, -1), (33, 7), + (31, 12), (37, 23), (31, 38), (20, 64), (11, 80), (5, 76), (2, 84), (5, 78), + (-6, 55), (4, 61), (-14, 83), (-37, 127), (-5, 79), (-11, 104), (-11, 91), (-30, 127), + (11, 80), (5, 76), (2, 84), (5, 78), (-6, 55), (4, 61), (-14, 83), (-37, 127), + (-5, 79), (-11, 104), (-11, 91), (-30, 127), (-4, 86), (-12, 88), (-5, 82), (-3, 72), + (-4, 67), (-8, 72), (-16, 89), (-9, 69), (-1, 59), (5, 66), (4, 57), (-4, 71), + (-2, 71), (2, 58), (-1, 74), (-4, 44), (-1, 69), (0, 62), (-7, 51), (-4, 47), + (-6, 42), (-3, 41), (-6, 53), (8, 76), (-9, 78), (-11, 83), (9, 52), (0, 67), + (-5, 90), (1, 67), (-15, 72), (-5, 75), (-8, 80), (-21, 83), (-21, 64), (-13, 31), + (-25, 64), (-29, 94), (9, 75), (17, 63), (-8, 74), (-5, 35), (-2, 27), (13, 91), + (-4, 86), (-12, 88), (-5, 82), (-3, 72), (-4, 67), (-8, 72), (-16, 89), (-9, 69), + (-1, 59), (5, 66), (4, 57), (-4, 71), (-2, 71), (2, 58), (-1, 74), (-4, 44), + (-1, 69), (0, 62), (-7, 51), (-4, 47), (-6, 42), (-3, 41), (-6, 53), (8, 76), + (-9, 78), (-11, 83), (9, 52), (0, 67), (-5, 90), (1, 67), (-15, 72), (-5, 75), + (-8, 80), (-21, 83), (-21, 64), (-13, 31), (-25, 64), (-29, 94), (9, 75), (17, 63), + (-8, 74), (-5, 35), (-2, 27), (13, 91), (4, 39), (0, 42), (7, 34), (11, 29), + (8, 31), (6, 37), (7, 42), (3, 40), (8, 33), (13, 43), (13, 36), (4, 47), + (3, 55), (2, 58), (6, 60), (8, 44), (11, 44), (14, 42), (7, 48), (4, 56), + (4, 52), (13, 37), (9, 49), (19, 58), (10, 48), (12, 45), (0, 69), (20, 33), + (8, 63), (35, -18), (33, -25), (28, -3), (24, 10), (27, 0), (34, -14), (52, -44), + (39, -24), (19, 17), (31, 25), (36, 29), (24, 33), (34, 15), (30, 20), (22, 73), + (4, 39), (0, 42), (7, 34), (11, 29), (8, 31), (6, 37), (7, 42), (3, 40), + (8, 33), (13, 43), (13, 36), (4, 47), (3, 55), (2, 58), (6, 60), (8, 44), + (11, 44), (14, 42), (7, 48), (4, 56), (4, 52), (13, 37), (9, 49), (19, 58), + (10, 48), (12, 45), (0, 69), (20, 33), (8, 63), (35, -18), (33, -25), (28, -3), + (24, 10), (27, 0), (34, -14), (52, -44), (39, -24), (19, 17), (31, 25), (36, 29), + (24, 33), (34, 15), (30, 20), (22, 73), (-3, 78), (-8, 74), (-9, 72), (-10, 72), + (-18, 75), (-12, 71), (-11, 63), (-5, 70), (-17, 75), (-14, 72), (-16, 67), (-8, 53), + (-14, 59), (-9, 52), (-11, 68), (-3, 78), (-8, 74), (-9, 72), (-10, 72), (-18, 75), + (-12, 71), (-11, 63), (-5, 70), (-17, 75), (-14, 72), (-16, 67), (-8, 53), (-14, 59), + (-9, 52), (-11, 68), (9, -2), (30, -10), (31, -4), (33, -1), (33, 7), (31, 12), + (37, 23), (31, 38), (20, 64), (9, -2), (30, -10), (31, -4), (33, -1), (33, 7), + (31, 12), (37, 23), (31, 38), (20, 64), (-9, 71), (-7, 37), (-8, 44), (-11, 49), + (-10, 56), (-12, 59), (-8, 63), (-9, 67), (-6, 68), (-10, 79), (-3, 78), (-8, 74), + (-9, 72), (-10, 72), (-18, 75), (-12, 71), (-11, 63), (-5, 70), (-17, 75), (-14, 72), + (-16, 67), (-8, 53), (-14, 59), (-9, 52), (-11, 68), (-3, 78), (-8, 74), (-9, 72), + (-10, 72), (-18, 75), (-12, 71), (-11, 63), (-5, 70), (-17, 75), (-14, 72), (-16, 67), + (-8, 53), (-14, 59), (-9, 52), (-11, 68), (9, -2), (30, -10), (31, -4), (33, -1), + (33, 7), (31, 12), (37, 23), (31, 38), (20, 64), (9, -2), (30, -10), (31, -4), + (33, -1), (33, 7), (31, 12), (37, 23), (31, 38), (20, 64), (-9, 71), (-7, 37), + (-8, 44), (-11, 49), (-10, 56), (-12, 59), (-8, 63), (-9, 67), (-6, 68), (-10, 79), + (-22, 127), (-25, 127), (-25, 120), (-27, 127), (-19, 114), (-23, 117), (-25, 118), (-26, 117), + (-24, 113), (-28, 118), (-31, 120), (-37, 124), (-10, 94), (-15, 102), (-10, 99), (-13, 106), + (-50, 127), (-5, 92), (17, 57), (-5, 86), (-13, 94), (-12, 91), (-2, 77), (0, 71), + (-1, 73), (4, 64), (-7, 81), (5, 64), (15, 57), (1, 67), (0, 68), (-10, 67), + (1, 68), (0, 77), (2, 64), (0, 68), (-5, 78), (7, 55), (5, 59), (2, 65), + (14, 54), (15, 44), (5, 60), (2, 70), (-22, 127), (-25, 127), (-25, 120), (-27, 127), + (-19, 114), (-23, 117), (-25, 118), (-26, 117), (-24, 113), (-28, 118), (-31, 120), (-37, 124), + (-10, 94), (-15, 102), (-10, 99), (-13, 106), (-50, 127), (-5, 92), (17, 57), (-5, 86), + (-13, 94), (-12, 91), (-2, 77), (0, 71), (-1, 73), (4, 64), (-7, 81), (5, 64), + (15, 57), (1, 67), (0, 68), (-10, 67), (1, 68), (0, 77), (2, 64), (0, 68), + (-5, 78), (7, 55), (5, 59), (2, 65), (14, 54), (15, 44), (5, 60), (2, 70), + (17, -13), (16, -9), (17, -12), (27, -21), (37, -30), (41, -40), (42, -41), (48, -47), + (39, -32), (46, -40), (52, -51), (46, -41), (52, -39), (43, -19), (32, 11), (61, -55), + (56, -46), (62, -50), (81, -67), (45, -20), (35, -2), (28, 15), (34, 1), (39, 1), + (30, 17), (20, 38), (18, 45), (15, 54), (0, 79), (36, -16), (37, -14), (37, -17), + (32, 1), (34, 15), (29, 15), (24, 25), (34, 22), (31, 16), (35, 18), (31, 28), + (33, 41), (36, 28), (27, 47), (21, 62), (17, -13), (16, -9), (17, -12), (27, -21), + (37, -30), (41, -40), (42, -41), (48, -47), (39, -32), (46, -40), (52, -51), (46, -41), + (52, -39), (43, -19), (32, 11), (61, -55), (56, -46), (62, -50), (81, -67), (45, -20), + (35, -2), (28, 15), (34, 1), (39, 1), (30, 17), (20, 38), (18, 45), (15, 54), + (0, 79), (36, -16), (37, -14), (37, -17), (32, 1), (34, 15), (29, 15), (24, 25), + (34, 22), (31, 16), (35, 18), (31, 28), (33, 41), (36, 28), (27, 47), (21, 62), + (-24, 115), (-22, 82), (-9, 62), (0, 53), (0, 59), (-14, 85), (-13, 89), (-13, 94), + (-11, 92), (-29, 127), (-21, 100), (-14, 57), (-12, 67), (-11, 71), (-10, 77), (-21, 85), + (-16, 88), (-23, 104), (-15, 98), (-37, 127), (-10, 82), (-8, 48), (-8, 61), (-8, 66), + (-7, 70), (-14, 75), (-10, 79), (-9, 83), (-12, 92), (-18, 108), (-24, 115), (-22, 82), + (-9, 62), (0, 53), (0, 59), (-14, 85), (-13, 89), (-13, 94), (-11, 92), (-29, 127), + (-21, 100), (-14, 57), (-12, 67), (-11, 71), (-10, 77), (-21, 85), (-16, 88), (-23, 104), + (-15, 98), (-37, 127), (-10, 82), (-8, 48), (-8, 61), (-8, 66), (-7, 70), (-14, 75), + (-10, 79), (-9, 83), (-12, 92), (-18, 108), (-5, 79), (-11, 104), (-11, 91), (-30, 127), + (-5, 79), (-11, 104), (-11, 91), (-30, 127), (-5, 79), (-11, 104), (-11, 91), (-30, 127), + ], + [ + (20, -15), (2, 54), (3, 74), (20, -15), (2, 54), (3, 74), (-28, 127), (-23, 104), + (-6, 53), (-1, 54), (7, 51), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), + (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), + (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), + (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), + (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), + (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), + (0, 0), (0, 0), (0, 0), (0, 0), (0, 41), (0, 63), (0, 63), (0, 63), + (-9, 83), (4, 86), (0, 97), (-7, 72), (13, 41), (3, 62), (0, 11), (1, 55), + (0, 69), (-17, 127), (-13, 102), (0, 82), (-7, 74), (-21, 107), (-27, 127), (-31, 127), + (-24, 127), (-18, 95), (-27, 127), (-21, 114), (-30, 127), (-17, 123), (-12, 115), (-16, 122), + (-11, 115), (-12, 63), (-2, 68), (-15, 84), (-13, 104), (-3, 70), (-8, 93), (-10, 90), + (-30, 127), (-1, 74), (-6, 97), (-7, 91), (-20, 127), (-4, 56), (-5, 82), (-7, 76), + (-22, 125), (-7, 93), (-11, 87), (-3, 77), (-5, 71), (-4, 63), (-4, 68), (-12, 84), + (-7, 62), (-7, 65), (8, 61), (5, 56), (-2, 66), (1, 64), (0, 61), (-2, 78), + (1, 50), (7, 52), (10, 35), (0, 44), (11, 38), (1, 45), (0, 46), (5, 44), + (31, 17), (1, 51), (7, 50), (28, 19), (16, 33), (14, 62), (-13, 108), (-15, 100), + (-13, 101), (-13, 91), (-12, 94), (-10, 88), (-16, 84), (-10, 86), (-7, 83), (-13, 87), + (-19, 94), (1, 70), (0, 72), (-5, 74), (18, 59), (-8, 102), (-15, 100), (0, 95), + (-4, 75), (2, 72), (-11, 75), (-3, 71), (15, 46), (-13, 69), (0, 62), (0, 65), + (21, 37), (-15, 72), (9, 57), (16, 54), (0, 62), (12, 72), (24, 0), (15, 9), + (8, 25), (13, 18), (15, 9), (13, 19), (10, 37), (12, 18), (6, 29), (20, 33), + (15, 30), (4, 45), (1, 58), (0, 62), (7, 61), (12, 38), (11, 45), (15, 39), + (11, 42), (13, 44), (16, 45), (12, 41), (10, 49), (30, 34), (18, 42), (10, 55), + (17, 51), (17, 46), (0, 89), (26, -19), (22, -17), (26, -17), (30, -25), (28, -20), + (33, -23), (37, -27), (33, -23), (40, -28), (38, -17), (33, -11), (40, -15), (41, -6), + (38, 1), (41, 17), (30, -6), (27, 3), (26, 22), (37, -16), (35, -4), (38, -8), + (38, -3), (37, 3), (38, 5), (42, 0), (35, 16), (39, 22), (14, 48), (27, 37), + (21, 60), (12, 68), (2, 97), (-3, 71), (-6, 42), (-5, 50), (-3, 54), (-2, 62), + (0, 58), (1, 63), (-2, 72), (-1, 74), (-9, 91), (-5, 67), (-5, 27), (-3, 39), + (-2, 44), (0, 46), (-16, 64), (-8, 68), (-10, 78), (-6, 77), (-10, 86), (-12, 92), + (-15, 55), (-10, 60), (-6, 62), (-4, 65), (-12, 73), (-8, 76), (-7, 80), (-9, 88), + (-17, 110), (-11, 97), (-20, 84), (-11, 79), (-6, 73), (-4, 74), (-13, 86), (-13, 96), + (-11, 97), (-19, 117), (-8, 78), (-5, 33), (-4, 48), (-2, 53), (-3, 62), (-13, 71), + (-10, 79), (-12, 86), (-13, 90), (-14, 97), (0, 0), (-6, 93), (-6, 84), (-8, 79), + (0, 66), (-1, 71), (0, 62), (-2, 60), (-2, 59), (-5, 75), (-3, 62), (-4, 58), + (-9, 66), (-1, 79), (0, 71), (3, 68), (10, 44), (-7, 62), (15, 36), (14, 40), + (16, 27), (12, 29), (1, 44), (20, 36), (18, 32), (5, 42), (1, 48), (10, 62), + (17, 46), (9, 64), (-12, 104), (-11, 97), (-16, 96), (-7, 88), (-8, 85), (-7, 85), + (-9, 85), (-13, 88), (4, 66), (-3, 77), (-3, 76), (-6, 76), (10, 58), (-1, 76), + (-1, 83), (-7, 99), (-14, 95), (2, 95), (0, 76), (-5, 74), (0, 70), (-11, 75), + (1, 68), (0, 65), (-14, 73), (3, 62), (4, 62), (-1, 68), (-13, 75), (11, 55), + (5, 64), (12, 70), (15, 6), (6, 19), (7, 16), (12, 14), (18, 13), (13, 11), + (13, 15), (15, 16), (12, 23), (13, 23), (15, 20), (14, 26), (14, 44), (17, 40), + (17, 47), (24, 17), (21, 21), (25, 22), (31, 27), (22, 29), (19, 35), (14, 50), + (10, 57), (7, 63), (-2, 77), (-4, 82), (-3, 94), (9, 69), (-12, 109), (36, -35), + (36, -34), (32, -26), (37, -30), (44, -32), (34, -18), (34, -15), (40, -15), (33, -7), + (35, -5), (33, 0), (38, 2), (33, 13), (23, 35), (13, 58), (29, -3), (26, 0), + (22, 30), (31, -7), (35, -15), (34, -3), (34, 3), (36, -1), (34, 5), (32, 11), + (35, 5), (34, 12), (39, 11), (30, 29), (34, 26), (29, 39), (19, 66), (31, 21), + (31, 31), (25, 50), (-17, 120), (-20, 112), (-18, 114), (-11, 85), (-15, 92), (-14, 89), + (-26, 71), (-15, 81), (-14, 80), (0, 68), (-14, 70), (-24, 56), (-23, 68), (-24, 50), + (-11, 74), (23, -13), (26, -13), (40, -15), (49, -14), (44, 3), (45, 6), (44, 34), + (33, 54), (19, 82), (-3, 75), (-1, 23), (1, 34), (1, 43), (0, 54), (-2, 55), + (0, 61), (1, 64), (0, 68), (-9, 92), (-14, 106), (-13, 97), (-15, 90), (-12, 90), + (-18, 88), (-10, 73), (-9, 79), (-14, 86), (-10, 73), (-10, 70), (-10, 69), (-5, 66), + (-9, 64), (-5, 58), (2, 59), (21, -10), (24, -11), (28, -8), (28, -1), (29, 3), + (29, 9), (35, 20), (29, 36), (14, 67), (-17, 123), (-12, 115), (-16, 122), (-11, 115), + (-12, 63), (-2, 68), (-15, 84), (-13, 104), (-3, 70), (-8, 93), (-10, 90), (-30, 127), + (-17, 123), (-12, 115), (-16, 122), (-11, 115), (-12, 63), (-2, 68), (-15, 84), (-13, 104), + (-3, 70), (-8, 93), (-10, 90), (-30, 127), (-7, 93), (-11, 87), (-3, 77), (-5, 71), + (-4, 63), (-4, 68), (-12, 84), (-7, 62), (-7, 65), (8, 61), (5, 56), (-2, 66), + (1, 64), (0, 61), (-2, 78), (1, 50), (7, 52), (10, 35), (0, 44), (11, 38), + (1, 45), (0, 46), (5, 44), (31, 17), (1, 51), (7, 50), (28, 19), (16, 33), + (14, 62), (-13, 108), (-15, 100), (-13, 101), (-13, 91), (-12, 94), (-10, 88), (-16, 84), + (-10, 86), (-7, 83), (-13, 87), (-19, 94), (1, 70), (0, 72), (-5, 74), (18, 59), + (-7, 93), (-11, 87), (-3, 77), (-5, 71), (-4, 63), (-4, 68), (-12, 84), (-7, 62), + (-7, 65), (8, 61), (5, 56), (-2, 66), (1, 64), (0, 61), (-2, 78), (1, 50), + (7, 52), (10, 35), (0, 44), (11, 38), (1, 45), (0, 46), (5, 44), (31, 17), + (1, 51), (7, 50), (28, 19), (16, 33), (14, 62), (-13, 108), (-15, 100), (-13, 101), + (-13, 91), (-12, 94), (-10, 88), (-16, 84), (-10, 86), (-7, 83), (-13, 87), (-19, 94), + (1, 70), (0, 72), (-5, 74), (18, 59), (24, 0), (15, 9), (8, 25), (13, 18), + (15, 9), (13, 19), (10, 37), (12, 18), (6, 29), (20, 33), (15, 30), (4, 45), + (1, 58), (0, 62), (7, 61), (12, 38), (11, 45), (15, 39), (11, 42), (13, 44), + (16, 45), (12, 41), (10, 49), (30, 34), (18, 42), (10, 55), (17, 51), (17, 46), + (0, 89), (26, -19), (22, -17), (26, -17), (30, -25), (28, -20), (33, -23), (37, -27), + (33, -23), (40, -28), (38, -17), (33, -11), (40, -15), (41, -6), (38, 1), (41, 17), + (24, 0), (15, 9), (8, 25), (13, 18), (15, 9), (13, 19), (10, 37), (12, 18), + (6, 29), (20, 33), (15, 30), (4, 45), (1, 58), (0, 62), (7, 61), (12, 38), + (11, 45), (15, 39), (11, 42), (13, 44), (16, 45), (12, 41), (10, 49), (30, 34), + (18, 42), (10, 55), (17, 51), (17, 46), (0, 89), (26, -19), (22, -17), (26, -17), + (30, -25), (28, -20), (33, -23), (37, -27), (33, -23), (40, -28), (38, -17), (33, -11), + (40, -15), (41, -6), (38, 1), (41, 17), (-17, 120), (-20, 112), (-18, 114), (-11, 85), + (-15, 92), (-14, 89), (-26, 71), (-15, 81), (-14, 80), (0, 68), (-14, 70), (-24, 56), + (-23, 68), (-24, 50), (-11, 74), (-14, 106), (-13, 97), (-15, 90), (-12, 90), (-18, 88), + (-10, 73), (-9, 79), (-14, 86), (-10, 73), (-10, 70), (-10, 69), (-5, 66), (-9, 64), + (-5, 58), (2, 59), (23, -13), (26, -13), (40, -15), (49, -14), (44, 3), (45, 6), + (44, 34), (33, 54), (19, 82), (21, -10), (24, -11), (28, -8), (28, -1), (29, 3), + (29, 9), (35, 20), (29, 36), (14, 67), (-3, 75), (-1, 23), (1, 34), (1, 43), + (0, 54), (-2, 55), (0, 61), (1, 64), (0, 68), (-9, 92), (-17, 120), (-20, 112), + (-18, 114), (-11, 85), (-15, 92), (-14, 89), (-26, 71), (-15, 81), (-14, 80), (0, 68), + (-14, 70), (-24, 56), (-23, 68), (-24, 50), (-11, 74), (-14, 106), (-13, 97), (-15, 90), + (-12, 90), (-18, 88), (-10, 73), (-9, 79), (-14, 86), (-10, 73), (-10, 70), (-10, 69), + (-5, 66), (-9, 64), (-5, 58), (2, 59), (23, -13), (26, -13), (40, -15), (49, -14), + (44, 3), (45, 6), (44, 34), (33, 54), (19, 82), (21, -10), (24, -11), (28, -8), + (28, -1), (29, 3), (29, 9), (35, 20), (29, 36), (14, 67), (-3, 75), (-1, 23), + (1, 34), (1, 43), (0, 54), (-2, 55), (0, 61), (1, 64), (0, 68), (-9, 92), + (-6, 93), (-6, 84), (-8, 79), (0, 66), (-1, 71), (0, 62), (-2, 60), (-2, 59), + (-5, 75), (-3, 62), (-4, 58), (-9, 66), (-1, 79), (0, 71), (3, 68), (10, 44), + (-7, 62), (15, 36), (14, 40), (16, 27), (12, 29), (1, 44), (20, 36), (18, 32), + (5, 42), (1, 48), (10, 62), (17, 46), (9, 64), (-12, 104), (-11, 97), (-16, 96), + (-7, 88), (-8, 85), (-7, 85), (-9, 85), (-13, 88), (4, 66), (-3, 77), (-3, 76), + (-6, 76), (10, 58), (-1, 76), (-1, 83), (-6, 93), (-6, 84), (-8, 79), (0, 66), + (-1, 71), (0, 62), (-2, 60), (-2, 59), (-5, 75), (-3, 62), (-4, 58), (-9, 66), + (-1, 79), (0, 71), (3, 68), (10, 44), (-7, 62), (15, 36), (14, 40), (16, 27), + (12, 29), (1, 44), (20, 36), (18, 32), (5, 42), (1, 48), (10, 62), (17, 46), + (9, 64), (-12, 104), (-11, 97), (-16, 96), (-7, 88), (-8, 85), (-7, 85), (-9, 85), + (-13, 88), (4, 66), (-3, 77), (-3, 76), (-6, 76), (10, 58), (-1, 76), (-1, 83), + (15, 6), (6, 19), (7, 16), (12, 14), (18, 13), (13, 11), (13, 15), (15, 16), + (12, 23), (13, 23), (15, 20), (14, 26), (14, 44), (17, 40), (17, 47), (24, 17), + (21, 21), (25, 22), (31, 27), (22, 29), (19, 35), (14, 50), (10, 57), (7, 63), + (-2, 77), (-4, 82), (-3, 94), (9, 69), (-12, 109), (36, -35), (36, -34), (32, -26), + (37, -30), (44, -32), (34, -18), (34, -15), (40, -15), (33, -7), (35, -5), (33, 0), + (38, 2), (33, 13), (23, 35), (13, 58), (15, 6), (6, 19), (7, 16), (12, 14), + (18, 13), (13, 11), (13, 15), (15, 16), (12, 23), (13, 23), (15, 20), (14, 26), + (14, 44), (17, 40), (17, 47), (24, 17), (21, 21), (25, 22), (31, 27), (22, 29), + (19, 35), (14, 50), (10, 57), (7, 63), (-2, 77), (-4, 82), (-3, 94), (9, 69), + (-12, 109), (36, -35), (36, -34), (32, -26), (37, -30), (44, -32), (34, -18), (34, -15), + (40, -15), (33, -7), (35, -5), (33, 0), (38, 2), (33, 13), (23, 35), (13, 58), + (-3, 71), (-6, 42), (-5, 50), (-3, 54), (-2, 62), (0, 58), (1, 63), (-2, 72), + (-1, 74), (-9, 91), (-5, 67), (-5, 27), (-3, 39), (-2, 44), (0, 46), (-16, 64), + (-8, 68), (-10, 78), (-6, 77), (-10, 86), (-12, 92), (-15, 55), (-10, 60), (-6, 62), + (-4, 65), (-12, 73), (-8, 76), (-7, 80), (-9, 88), (-17, 110), (-3, 71), (-6, 42), + (-5, 50), (-3, 54), (-2, 62), (0, 58), (1, 63), (-2, 72), (-1, 74), (-9, 91), + (-5, 67), (-5, 27), (-3, 39), (-2, 44), (0, 46), (-16, 64), (-8, 68), (-10, 78), + (-6, 77), (-10, 86), (-12, 92), (-15, 55), (-10, 60), (-6, 62), (-4, 65), (-12, 73), + (-8, 76), (-7, 80), (-9, 88), (-17, 110), (-3, 70), (-8, 93), (-10, 90), (-30, 127), + (-3, 70), (-8, 93), (-10, 90), (-30, 127), (-3, 70), (-8, 93), (-10, 90), (-30, 127), + ], +]; diff --git a/crates/video/src/h264/cavlc.rs b/crates/video/src/h264/cavlc.rs new file mode 100644 index 00000000..2126a451 --- /dev/null +++ b/crates/video/src/h264/cavlc.rs @@ -0,0 +1,281 @@ +//! CAVLC residual block parsing (9.2). + +use crate::bitstream::BitReader; +use crate::h264::cavlc_tables::*; +use crate::h264::error::{Result, ensure}; +use std::sync::OnceLock; + +/// Prefix-code lookup keyed by (number of leading zeros, next 5 bits after the first one bit). +pub(crate) struct VlcTable { + /// entry = (len << 8) | value, 0 = invalid. + entries: Box<[u16; 17 * 32]>, +} + +impl VlcTable { + pub fn build(codes: &[(&str, u8)]) -> Self { + let mut entries = Box::new([0u16; 17 * 32]); + for &(code, value) in codes { + if code.is_empty() { + continue; + } + let len = code.len(); + let lz = code.bytes().take_while(|&b| b == b'0').count(); + let val = ((len as u16) << 8) | value as u16; + if lz == len { + // all-zero code: any continuation with at least `len` zeros + for l in lz..=16 { + for s in 0..32 { + entries[l * 32 + s] = val; + } + } + continue; + } + let suffix = &code[lz + 1..]; + assert!(suffix.len() <= 5, "suffix too long in {code}"); + let sbits = suffix.bytes().fold(0u32, |a, b| (a << 1) | u32::from(b == b'1')); + let free = 5 - suffix.len(); + for ext in 0..(1u32 << free) { + let s = (sbits << free) | ext; + entries[lz * 32 + s as usize] = val; + } + } + Self { entries } + } + + #[inline] + pub fn decode(&self, r: &mut BitReader) -> Result { + let bits = r.peek(32); + let lz = bits.leading_zeros().min(16); + let s = if lz >= 16 { 0 } else { ((bits << lz) << 1) >> 27 }; + let e = self.entries[(lz * 32 + s) as usize]; + ensure!(e != 0, "invalid VLC code"); + r.skip((e >> 8) as usize)?; + Ok(e as u8) + } +} + +struct Tables { + /// coeff_token per nC class; value = (TrailingOnes << 5) | TotalCoeff + coeff_token: [VlcTable; 6], + total_zeros_4x4: [VlcTable; 15], + total_zeros_2x2: [VlcTable; 3], + total_zeros_2x4: [VlcTable; 7], + run_before: [VlcTable; 7], +} + +fn tables() -> &'static Tables { + static T: OnceLock = OnceLock::new(); + T.get_or_init(|| { + let coeff_token = std::array::from_fn(|col| { + let codes: Vec<(&str, u8)> = COEFF_TOKEN.iter().map(|&(t1, tc, ref c)| (c[col], (t1 << 5) | tc)).collect(); + VlcTable::build(&codes) + }); + fn simple(row: &[&'static str; N]) -> VlcTable { + let codes: Vec<(&str, u8)> = row.iter().enumerate().map(|(i, &c)| (c, i as u8)).collect(); + VlcTable::build(&codes) + } + Tables { + coeff_token, + total_zeros_4x4: std::array::from_fn(|i| simple(&TOTAL_ZEROS_4X4[i])), + total_zeros_2x2: std::array::from_fn(|i| simple(&TOTAL_ZEROS_2X2[i])), + total_zeros_2x4: std::array::from_fn(|i| simple(&TOTAL_ZEROS_2X4[i])), + run_before: std::array::from_fn(|i| simple(&RUN_BEFORE[i])), + } + }) +} + +/// Force table construction (so the first slice does not pay for it). +pub(crate) fn init_tables() { + let _ = tables(); +} + +/// Parse residual_block_cavlc(). Coefficients are written to `coeff[start_idx..=end_idx]` in scan order +/// (entries outside the coded range are left untouched; the caller zeroes them). +/// `nc` is the nC value (-1 for 4:2:0 chroma DC, -2 for 4:2:2 chroma DC). Returns TotalCoeff. +pub(crate) fn residual_block(r: &mut BitReader, coeff: &mut [i32], start_idx: usize, end_idx: usize, max_num_coeff: usize, nc: i32) -> Result { + let t = tables(); + let col = match nc { + -1 => 4, + -2 => 5, + 0..=1 => 0, + 2..=3 => 1, + 4..=7 => 2, + _ => 3, + }; + let tok = t.coeff_token[col].decode(r)?; + let total_coeff = (tok & 31) as usize; + let trailing_ones = (tok >> 5) as usize; + if total_coeff == 0 { + return Ok(0); + } + ensure!(total_coeff <= max_num_coeff, "TotalCoeff exceeds maxNumCoeff"); + let mut level = [0i32; 16]; + let mut suffix_length: u32 = if total_coeff > 10 && trailing_ones < 3 { 1 } else { 0 }; + for (i, lv) in level.iter_mut().enumerate().take(total_coeff) { + if i < trailing_ones { + *lv = if r.read_bit()? { -1 } else { 1 }; + continue; + } + // level_prefix + let bits = r.peek(32); + let level_prefix = bits.leading_zeros(); + ensure!(level_prefix <= 25, "level_prefix too large"); + r.skip(level_prefix as usize + 1)?; + let level_suffix_size = if level_prefix == 14 && suffix_length == 0 { + 4 + } else if level_prefix >= 15 { + level_prefix - 3 + } else { + suffix_length + }; + let level_suffix = if level_suffix_size > 0 { r.read_bits(level_suffix_size)? as i32 } else { 0 }; + let mut level_code = ((level_prefix.min(15) as i32) << suffix_length) + level_suffix; + if level_prefix >= 15 && suffix_length == 0 { + level_code += 15; + } + if level_prefix >= 16 { + level_code += (1 << (level_prefix - 3)) - 4096; + } + if i == trailing_ones && trailing_ones < 3 { + level_code += 2; + } + *lv = if level_code % 2 == 0 { (level_code + 2) >> 1 } else { (-level_code - 1) >> 1 }; + if suffix_length == 0 { + suffix_length = 1; + } + if lv.abs() > (3 << (suffix_length - 1)) && suffix_length < 6 { + suffix_length += 1; + } + } + let mut zeros_left = if total_coeff < end_idx - start_idx + 1 { + let tz_idx = total_coeff - 1; + let tz = match max_num_coeff { + 4 => t.total_zeros_2x2[tz_idx].decode(r)?, + 8 => t.total_zeros_2x4[tz_idx].decode(r)?, + _ => t.total_zeros_4x4[tz_idx].decode(r)?, + } as usize; + ensure!(tz + total_coeff <= end_idx - start_idx + 1, "total_zeros out of range"); + tz + } else { + 0 + }; + // runs and placement: highest-frequency coefficient first + let mut pos = start_idx + total_coeff + zeros_left - 1; + for (i, &lv) in level.iter().enumerate().take(total_coeff) { + coeff[pos] = lv; + if i + 1 == total_coeff { + break; + } + let run = if zeros_left > 0 { + let run = t.run_before[zeros_left.min(7) - 1].decode(r)? as usize; + ensure!(run <= zeros_left, "run_before out of range"); + zeros_left -= run; + run + } else { + 0 + }; + pos -= 1 + run; + } + Ok(total_coeff as u8) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bitstream::BitWriter; + + fn kraft(codes: &[&str]) -> f64 { + codes.iter().filter(|c| !c.is_empty()).map(|c| 0.5f64.powi(c.len() as i32)).sum() + } + + fn prefix_free(codes: &[&str]) -> bool { + let c: Vec<&&str> = codes.iter().filter(|c| !c.is_empty()).collect(); + for (i, a) in c.iter().enumerate() { + for (j, b) in c.iter().enumerate() { + if i != j && b.starts_with(**a) { + return false; + } + } + } + true + } + + #[test] + fn coeff_token_tables_are_prefix_codes() { + for col in 0..6 { + let codes: Vec<&str> = COEFF_TOKEN.iter().map(|e| e.2[col]).collect(); + assert!(prefix_free(&codes), "column {col}"); + let k = kraft(&codes); + assert!(k <= 1.0 + 1e-12, "column {col} kraft {k}"); + } + // nC>=8 is a complete 6-bit fixed code except the unused 000011 pattern + let codes: Vec<&str> = COEFF_TOKEN.iter().map(|e| e.2[3]).collect(); + assert!(codes.iter().all(|c| c.len() == 6)); + } + + #[test] + fn total_zeros_and_run_before_are_complete_prefix_codes() { + for (i, row) in TOTAL_ZEROS_4X4.iter().enumerate() { + assert!(prefix_free(row)); + // tzVlcIndex 1 leaves the all-zero 9-bit code unused; all others are complete + let k = kraft(row); + if i == 0 { + assert!((k - (1.0 - 1.0 / 512.0)).abs() < 1e-12, "{row:?}"); + } else { + assert!((k - 1.0).abs() < 1e-12, "{row:?}"); + } + } + for row in TOTAL_ZEROS_2X2.iter() { + assert!(prefix_free(row)); + assert!((kraft(row) - 1.0).abs() < 1e-12); + } + for row in TOTAL_ZEROS_2X4.iter() { + assert!(prefix_free(row)); + assert!((kraft(row) - 1.0).abs() < 1e-12); + } + for row in RUN_BEFORE.iter() { + assert!(prefix_free(row)); + } + } + + fn write_code(w: &mut BitWriter, code: &str) { + for b in code.bytes() { + w.write_bit(b == b'1'); + } + } + + #[test] + fn decodes_spec_style_example() { + // Block: 0 3 -1 0 | 0 -1 1 0 | 1 0 0 0 | 0 0 0 0 (zig-zag order), nC = 0. + // TotalCoeff 5, TrailingOnes 3: coeff_token 0000 100. + // trailing ones (reverse order): +1, -1, -1 -> sign bits 0 1 1 + // level -1? No: remaining levels (reverse): 1 (idx 4?) ... use the classic example: + // coefficients in scan order: [0, 3, -1, 0, 0, -1, 1, 0, 1, 0, ...] + // reverse order non-zero: 1 (pos 8), 1 (pos 6), -1 (pos 5), -1 (pos 2), 3 (pos 1) + // TotalCoeff 5, TrailingOnes 3 (1, 1, -1). + let mut w = BitWriter::new(); + write_code(&mut w, "0000100"); // coeff_token TC=5 T1=3, nC 0..2 + write_code(&mut w, "0"); // +1 + write_code(&mut w, "0"); // +1 + write_code(&mut w, "1"); // -1 + // level -1 with suffixLength 0: levelCode = 1 (T1 == 3 so no +2) -> prefix 1 => "01" + write_code(&mut w, "01"); + // level 3 with suffixLength 1: levelCode = 4 -> prefix 2, suffix 0 => "001" "0" + write_code(&mut w, "0010"); + // total_zeros = 4 (positions 0, 3, 4, 7) with tzVlcIndex 5 -> "110" + write_code(&mut w, "110"); + // run_before: zerosLeft 4 at pos 8: run 1 -> "10"; zerosLeft 3 at pos 6: run 0 -> "11"; + // zerosLeft 3 at pos 5: run 2 -> "01"; zerosLeft 1 at pos 2: run 0 -> "1" + write_code(&mut w, "10"); + write_code(&mut w, "11"); + write_code(&mut w, "01"); + write_code(&mut w, "1"); + w.rbsp_trailing(); + let b = w.finish(); + let mut r = BitReader::new(&b); + let mut c = [0i32; 16]; + let tc = residual_block(&mut r, &mut c, 0, 15, 16, 0).unwrap(); + assert_eq!(tc, 5); + assert_eq!(&c[..10], &[0, 3, -1, 0, 0, -1, 1, 0, 1, 0]); + } +} diff --git a/crates/video/src/h264/cavlc_tables.rs b/crates/video/src/h264/cavlc_tables.rs new file mode 100644 index 00000000..ba509e2f --- /dev/null +++ b/crates/video/src/h264/cavlc_tables.rs @@ -0,0 +1,117 @@ +// Generated from ITU-T H.264 (06/2026) Tables 9-5, 9-7..9-10 by a table-extraction script. +// Codes are bit strings; "" marks an unused entry. + +/// (TrailingOnes, TotalCoeff, [nC 0..2, 2..4, 4..8, 8.., -1, -2]) +#[rustfmt::skip] +pub(crate) static COEFF_TOKEN: [(u8, u8, [&str; 6]); 62] = [ + (0, 0, ["1", "11", "1111", "000011", "01", "1"]), + (0, 1, ["000101", "001011", "001111", "000000", "000111", "0001111"]), + (1, 1, ["01", "10", "1110", "000001", "1", "01"]), + (0, 2, ["00000111", "000111", "001011", "000100", "000100", "0001110"]), + (1, 2, ["000100", "00111", "01111", "000101", "000110", "0001101"]), + (2, 2, ["001", "011", "1101", "000110", "001", "001"]), + (0, 3, ["000000111", "0000111", "001000", "001000", "000011", "000000111"]), + (1, 3, ["00000110", "001010", "01100", "001001", "0000011", "0001100"]), + (2, 3, ["0000101", "001001", "01110", "001010", "0000010", "0001011"]), + (3, 3, ["00011", "0101", "1100", "001011", "000101", "00001"]), + (0, 4, ["0000000111", "00000111", "0001111", "001100", "000010", "000000110"]), + (1, 4, ["000000110", "000110", "01010", "001101", "00000011", "000000101"]), + (2, 4, ["00000101", "000101", "01011", "001110", "00000010", "0001010"]), + (3, 4, ["000011", "0100", "1011", "001111", "0000000", "000001"]), + (0, 5, ["00000000111", "00000100", "0001011", "010000", "", "0000000111"]), + (1, 5, ["0000000110", "0000110", "01000", "010001", "", "0000000110"]), + (2, 5, ["000000101", "0000101", "01001", "010010", "", "000000100"]), + (3, 5, ["0000100", "00110", "1010", "010011", "", "0001001"]), + (0, 6, ["0000000001111", "000000111", "0001001", "010100", "", "00000000111"]), + (1, 6, ["00000000110", "00000110", "001110", "010101", "", "00000000110"]), + (2, 6, ["0000000101", "00000101", "001101", "010110", "", "0000000101"]), + (3, 6, ["00000100", "001000", "1001", "010111", "", "0001000"]), + (0, 7, ["0000000001011", "00000001111", "0001000", "011000", "", "000000000111"]), + (1, 7, ["0000000001110", "000000110", "001010", "011001", "", "000000000110"]), + (2, 7, ["00000000101", "000000101", "001001", "011010", "", "00000000101"]), + (3, 7, ["000000100", "000100", "1000", "011011", "", "0000000100"]), + (0, 8, ["0000000001000", "00000001011", "00001111", "011100", "", "0000000000111"]), + (1, 8, ["0000000001010", "00000001110", "0001110", "011101", "", "000000000101"]), + (2, 8, ["0000000001101", "00000001101", "0001101", "011110", "", "000000000100"]), + (3, 8, ["0000000100", "0000100", "01101", "011111", "", "00000000100"]), + (0, 9, ["00000000001111", "000000001111", "00001011", "100000", "", ""]), + (1, 9, ["00000000001110", "00000001010", "00001110", "100001", "", ""]), + (2, 9, ["0000000001001", "00000001001", "0001010", "100010", "", ""]), + (3, 9, ["00000000100", "000000100", "001100", "100011", "", ""]), + (0, 10, ["00000000001011", "000000001011", "000001111", "100100", "", ""]), + (1, 10, ["00000000001010", "000000001110", "00001010", "100101", "", ""]), + (2, 10, ["00000000001101", "000000001101", "00001101", "100110", "", ""]), + (3, 10, ["0000000001100", "00000001100", "0001100", "100111", "", ""]), + (0, 11, ["000000000001111", "000000001000", "000001011", "101000", "", ""]), + (1, 11, ["000000000001110", "000000001010", "000001110", "101001", "", ""]), + (2, 11, ["00000000001001", "000000001001", "00001001", "101010", "", ""]), + (3, 11, ["00000000001100", "00000001000", "00001100", "101011", "", ""]), + (0, 12, ["000000000001011", "0000000001111", "000001000", "101100", "", ""]), + (1, 12, ["000000000001010", "0000000001110", "000001010", "101101", "", ""]), + (2, 12, ["000000000001101", "0000000001101", "000001101", "101110", "", ""]), + (3, 12, ["00000000001000", "000000001100", "00001000", "101111", "", ""]), + (0, 13, ["0000000000001111", "0000000001011", "0000001101", "110000", "", ""]), + (1, 13, ["000000000000001", "0000000001010", "000000111", "110001", "", ""]), + (2, 13, ["000000000001001", "0000000001001", "000001001", "110010", "", ""]), + (3, 13, ["000000000001100", "0000000001100", "000001100", "110011", "", ""]), + (0, 14, ["0000000000001011", "0000000000111", "0000001001", "110100", "", ""]), + (1, 14, ["0000000000001110", "00000000001011", "0000001100", "110101", "", ""]), + (2, 14, ["0000000000001101", "0000000000110", "0000001011", "110110", "", ""]), + (3, 14, ["000000000001000", "0000000001000", "0000001010", "110111", "", ""]), + (0, 15, ["0000000000000111", "00000000001001", "0000000101", "111000", "", ""]), + (1, 15, ["0000000000001010", "00000000001000", "0000001000", "111001", "", ""]), + (2, 15, ["0000000000001001", "00000000001010", "0000000111", "111010", "", ""]), + (3, 15, ["0000000000001100", "0000000000001", "0000000110", "111011", "", ""]), + (0, 16, ["0000000000000100", "00000000000111", "0000000001", "111100", "", ""]), + (1, 16, ["0000000000000110", "00000000000110", "0000000100", "111101", "", ""]), + (2, 16, ["0000000000000101", "00000000000101", "0000000011", "111110", "", ""]), + (3, 16, ["0000000000001000", "00000000000100", "0000000010", "111111", "", ""]), +]; +/// total_zeros for 4x4 blocks: [tzVlcIndex-1][total_zeros] +#[rustfmt::skip] +pub(crate) static TOTAL_ZEROS_4X4: [[&str; 16]; 15] = [ + ["1", "011", "010", "0011", "0010", "00011", "00010", "000011", "000010", "0000011", "0000010", "00000011", "00000010", "000000011", "000000010", "000000001"], + ["111", "110", "101", "100", "011", "0101", "0100", "0011", "0010", "00011", "00010", "000011", "000010", "000001", "000000", ""], + ["0101", "111", "110", "101", "0100", "0011", "100", "011", "0010", "00011", "00010", "000001", "00001", "000000", "", ""], + ["00011", "111", "0101", "0100", "110", "101", "100", "0011", "011", "0010", "00010", "00001", "00000", "", "", ""], + ["0101", "0100", "0011", "111", "110", "101", "100", "011", "0010", "00001", "0001", "00000", "", "", "", ""], + ["000001", "00001", "111", "110", "101", "100", "011", "010", "0001", "001", "000000", "", "", "", "", ""], + ["000001", "00001", "101", "100", "011", "11", "010", "0001", "001", "000000", "", "", "", "", "", ""], + ["000001", "0001", "00001", "011", "11", "10", "010", "001", "000000", "", "", "", "", "", "", ""], + ["000001", "000000", "0001", "11", "10", "001", "01", "00001", "", "", "", "", "", "", "", ""], + ["00001", "00000", "001", "11", "10", "01", "0001", "", "", "", "", "", "", "", "", ""], + ["0000", "0001", "001", "010", "1", "011", "", "", "", "", "", "", "", "", "", ""], + ["0000", "0001", "01", "1", "001", "", "", "", "", "", "", "", "", "", "", ""], + ["000", "001", "1", "01", "", "", "", "", "", "", "", "", "", "", "", ""], + ["00", "01", "1", "", "", "", "", "", "", "", "", "", "", "", "", ""], + ["0", "1", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], +]; +/// total_zeros for chroma DC 2x2: [tzVlcIndex-1][total_zeros] +#[rustfmt::skip] +pub(crate) static TOTAL_ZEROS_2X2: [[&str; 4]; 3] = [ + ["1", "01", "001", "000"], + ["1", "01", "00", ""], + ["1", "0", "", ""], +]; +/// total_zeros for chroma DC 2x4: [tzVlcIndex-1][total_zeros] +#[rustfmt::skip] +pub(crate) static TOTAL_ZEROS_2X4: [[&str; 8]; 7] = [ + ["1", "010", "011", "0010", "0011", "0001", "00001", "00000"], + ["000", "01", "001", "100", "101", "110", "111", ""], + ["000", "001", "01", "10", "110", "111", "", ""], + ["110", "00", "01", "10", "111", "", "", ""], + ["00", "01", "10", "11", "", "", "", ""], + ["00", "01", "1", "", "", "", "", ""], + ["0", "1", "", "", "", "", "", ""], +]; +/// run_before: [min(zerosLeft,7)-1][run_before] +#[rustfmt::skip] +pub(crate) static RUN_BEFORE: [[&str; 15]; 7] = [ + ["1", "0", "", "", "", "", "", "", "", "", "", "", "", "", ""], + ["1", "01", "00", "", "", "", "", "", "", "", "", "", "", "", ""], + ["11", "10", "01", "00", "", "", "", "", "", "", "", "", "", "", ""], + ["11", "10", "01", "001", "000", "", "", "", "", "", "", "", "", "", ""], + ["11", "10", "011", "010", "001", "000", "", "", "", "", "", "", "", "", ""], + ["11", "000", "001", "011", "010", "101", "100", "", "", "", "", "", "", "", ""], + ["111", "110", "101", "100", "011", "010", "001", "0001", "00001", "000001", "0000001", "00000001", "000000001", "0000000001", "00000000001"], +]; diff --git a/crates/video/src/h264/deblock.rs b/crates/video/src/h264/deblock.rs new file mode 100644 index 00000000..887c93e5 --- /dev/null +++ b/crates/video/src/h264/deblock.rs @@ -0,0 +1,554 @@ +//! Deblocking filter (8.7) for progressive frames, 4:2:0, 8-bit. + +use crate::h264::picture::{MbKind, MbState}; +use crate::h264::slicedec::{PicState, SliceInfo}; +use crate::h264::tables::{ALPHA, BETA, TC0}; + +/// Reference picture identity + mvs of one 4x4 block, for bS = 1 decisions. +#[derive(Clone, Copy)] +struct Motion { + ids: [u32; 2], + mv: [[i16; 2]; 2], + count: u8, +} + +/// Referenced picture ids per 8x8 block and list (u32::MAX = list unused). +fn ref_ids8(st: &MbState, sl: &SliceInfo) -> [[u32; 2]; 4] { + std::array::from_fn(|b8| { + std::array::from_fn(|l| { + let r = st.ref_idx[l][b8]; + if r >= 0 { sl.ref_ids[l].get(r as usize).copied().unwrap_or(u32::MAX - 1) } else { u32::MAX } + }) + }) +} + +#[inline(always)] +fn block_motion(st: &MbState, ids8: &[[u32; 2]; 4], raster: usize) -> Motion { + let b8 = (raster >> 3) * 2 + ((raster & 3) >> 1); + let ids = ids8[b8]; + let count = (ids[0] != u32::MAX) as u8 + (ids[1] != u32::MAX) as u8; + Motion { ids, mv: [st.mv[0][raster], st.mv[1][raster]], count } +} + +#[inline(always)] +fn mv_far(a: [i16; 2], b: [i16; 2]) -> bool { + (a[0] as i32 - b[0] as i32).abs() >= 4 || (a[1] as i32 - b[1] as i32).abs() >= 4 +} + +#[inline] +fn motion_bs(p: &Motion, q: &Motion) -> u8 { + if p.ids == q.ids && p.mv == q.mv { + return 0; + } + if p.count != q.count { + return 1; + } + if p.count == 1 { + let (pi, pm) = if p.ids[0] != u32::MAX { (p.ids[0], p.mv[0]) } else { (p.ids[1], p.mv[1]) }; + let (qi, qm) = if q.ids[0] != u32::MAX { (q.ids[0], q.mv[0]) } else { (q.ids[1], q.mv[1]) }; + if pi != qi { + return 1; + } + return mv_far(pm, qm) as u8; + } + if p.count == 0 { + return 0; + } + // two motion vectors each + let same_set = (p.ids[0] == q.ids[0] && p.ids[1] == q.ids[1]) || (p.ids[0] == q.ids[1] && p.ids[1] == q.ids[0]); + if !same_set { + return 1; + } + if p.ids[0] != p.ids[1] { + // two different reference pictures: compare mvs referring to the same picture + if p.ids[0] == q.ids[0] { + (mv_far(p.mv[0], q.mv[0]) || mv_far(p.mv[1], q.mv[1])) as u8 + } else { + (mv_far(p.mv[0], q.mv[1]) || mv_far(p.mv[1], q.mv[0])) as u8 + } + } else { + // both mvs refer to the same picture + ((mv_far(p.mv[0], q.mv[0]) || mv_far(p.mv[1], q.mv[1])) && (mv_far(p.mv[0], q.mv[1]) || mv_far(p.mv[1], q.mv[0]))) as u8 + } +} + +#[cfg(test)] +fn clip3(lo: i32, hi: i32, v: i32) -> i32 { + v.clamp(lo, hi) +} + +/// Filter one set of eight samples p3 p2 p1 p0 | q0 q1 q2 q3 across an edge (8.7.2.3 / 8.7.2.4): +/// the per-line reference for [`filter_lanes`]. +#[cfg(test)] +fn filter8(v: &mut [u8; 8], bs: u8, alpha: i32, beta: i32, tc0: i32, chroma: bool) { + let p0 = v[3] as i32; + let q0 = v[4] as i32; + let p1 = v[2] as i32; + let q1 = v[5] as i32; + if (p0 - q0).abs() >= alpha || (p1 - p0).abs() >= beta || (q1 - q0).abs() >= beta { + return; + } + if chroma { + if bs < 4 { + let tc = tc0 + 1; + let delta = clip3(-tc, tc, (((q0 - p0) << 2) + (p1 - q1) + 4) >> 3); + v[3] = (p0 + delta).clamp(0, 255) as u8; + v[4] = (q0 - delta).clamp(0, 255) as u8; + } else { + v[3] = ((2 * p1 + p0 + q1 + 2) >> 2) as u8; + v[4] = ((2 * q1 + q0 + p1 + 2) >> 2) as u8; + } + return; + } + let p2 = v[1] as i32; + let q2 = v[6] as i32; + let ap = (p2 - p0).abs(); + let aq = (q2 - q0).abs(); + if bs < 4 { + let tc = tc0 + (ap < beta) as i32 + (aq < beta) as i32; + let delta = clip3(-tc, tc, (((q0 - p0) << 2) + (p1 - q1) + 4) >> 3); + v[3] = (p0 + delta).clamp(0, 255) as u8; + v[4] = (q0 - delta).clamp(0, 255) as u8; + if ap < beta { + v[2] = (p1 + clip3(-tc0, tc0, (p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1)) as u8; + } + if aq < beta { + v[5] = (q1 + clip3(-tc0, tc0, (q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1)) as u8; + } + } else { + let strong = (p0 - q0).abs() < ((alpha >> 2) + 2); + if ap < beta && strong { + let p3 = v[0] as i32; + v[3] = ((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) as u8; + v[2] = ((p2 + p1 + p0 + q0 + 2) >> 2) as u8; + v[1] = ((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) as u8; + } else { + v[3] = ((2 * p1 + p0 + q1 + 2) >> 2) as u8; + } + if aq < beta && strong { + let q3 = v[7] as i32; + v[4] = ((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) as u8; + v[5] = ((p0 + q0 + q1 + q2 + 2) >> 2) as u8; + v[6] = ((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) as u8; + } else { + v[4] = ((2 * q1 + q0 + p1 + 2) >> 2) as u8; + } + } +} + +/// Edge filter parameters for `n` lines; `bs[i]` / `tc0[i]` apply to lines `i * n / 4 .. (i + 1) * n / 4`. +struct EdgeParams { + bs: [u8; 4], + tc0: [i32; 4], + alpha: i32, + beta: i32, +} + +/// Filter one edge for `N` lines at once (lane `i` = line `i` across the edge). `s[k][i]` holds +/// sample p3 p2 p1 p0 q0 q1 q2 q3 (k = 0..8) of line `i`. Branch-free per lane, so the loops +/// vectorise; bit-exact with [`filter8`] applied to every line (8.7.2.3 / 8.7.2.4). +#[inline(always)] +fn filter_lanes(s: &mut [[i16; N]; 8], ep: &EdgeParams, chroma: bool) { + let per = N / 4; + let (alpha, beta) = (ep.alpha as i16, ep.beta as i16); + let mut bs_on = [false; N]; + let mut tc0 = [0i16; N]; + for k in 0..4 { + for i in k * per..(k + 1) * per { + bs_on[i] = ep.bs[k] != 0; + tc0[i] = ep.tc0[k] as i16; + } + } + let [p3, p2, p1, p0, q0, q1, q2, q3] = *s; + // Masks are 0 / -1 per lane, selects are bitwise, and clipping to per-lane bounds uses + // min / max (`clamp` would assert its bounds per lane), so every loop below vectorises. + let lt = |x: i16, y: i16| -((x < y) as i16); + let sel = |m: i16, a: i16, b: i16| (a & m) | (b & !m); + let mut on = [0i16; N]; + let mut any = 0i16; + for i in 0..N { + on[i] = -(bs_on[i] as i16) & lt((p0[i] - q0[i]).abs(), alpha) & lt((p1[i] - p0[i]).abs(), beta) & lt((q1[i] - q0[i]).abs(), beta); + any |= on[i]; + } + if any == 0 { + return; + } + // bS 4 is only ever assigned to all four segments of an edge + let strong_edge = ep.bs[0] >= 4; + if chroma { + for i in 0..N { + let (np0, nq0) = if strong_edge { + ((2 * p1[i] + p0[i] + q1[i] + 2) >> 2, (2 * q1[i] + q0[i] + p1[i] + 2) >> 2) + } else { + let tc = tc0[i] + 1; + let delta = ((((q0[i] - p0[i]) << 2) + (p1[i] - q1[i]) + 4) >> 3).max(-tc).min(tc); + ((p0[i] + delta).clamp(0, 255), (q0[i] - delta).clamp(0, 255)) + }; + s[3][i] = sel(on[i], np0, p0[i]); + s[4][i] = sel(on[i], nq0, q0[i]); + } + return; + } + if strong_edge { + let lim = (alpha >> 2) + 2; + for i in 0..N { + let (a, b, c, d, e, f, g, h) = (p3[i], p2[i], p1[i], p0[i], q0[i], q1[i], q2[i], q3[i]); + let strong = lt((d - e).abs(), lim); + let sp = on[i] & lt((b - d).abs(), beta) & strong; + let sq = on[i] & lt((g - e).abs(), beta) & strong; + let wp0 = (2 * c + d + f + 2) >> 2; + let wq0 = (2 * f + e + c + 2) >> 2; + let sp0 = (b + 2 * c + 2 * d + 2 * e + f + 4) >> 3; + let sq0 = (c + 2 * d + 2 * e + 2 * f + g + 4) >> 3; + s[3][i] = sel(sp, sp0, sel(on[i], wp0, d)); + s[2][i] = sel(sp, (b + c + d + e + 2) >> 2, c); + s[1][i] = sel(sp, (2 * a + 3 * b + c + d + e + 4) >> 3, b); + s[4][i] = sel(sq, sq0, sel(on[i], wq0, e)); + s[5][i] = sel(sq, (d + e + f + g + 2) >> 2, f); + s[6][i] = sel(sq, (2 * h + 3 * g + f + e + d + 4) >> 3, g); + } + } else { + for i in 0..N { + let ap = lt((p2[i] - p0[i]).abs(), beta); + let aq = lt((q2[i] - q0[i]).abs(), beta); + let t0 = tc0[i]; + let tc = t0 - ap - aq; + let delta = ((((q0[i] - p0[i]) << 2) + (p1[i] - q1[i]) + 4) >> 3).max(-tc).min(tc); + let avg = (p0[i] + q0[i] + 1) >> 1; + let np1 = p1[i] + ((p2[i] + avg - (p1[i] << 1)) >> 1).max(-t0).min(t0); + let nq1 = q1[i] + ((q2[i] + avg - (q1[i] << 1)) >> 1).max(-t0).min(t0); + s[3][i] = sel(on[i], (p0[i] + delta).clamp(0, 255), p0[i]); + s[4][i] = sel(on[i], (q0[i] - delta).clamp(0, 255), q0[i]); + s[2][i] = sel(on[i] & ap, np1, p1[i]); + s[5][i] = sel(on[i] & aq, nq1, q1[i]); + } + } +} + +/// Filter a vertical edge whose q0 column is at `x`, for `N` lines starting at row `y`. +#[inline(always)] +fn filter_vertical(pix: &mut [u8], stride: usize, x: usize, y: usize, ep: &EdgeParams, chroma: bool) { + // Each line's eight samples as one little-endian word: byte k = sample k (p3 .. q3). + let mut w = [0u64; N]; + for (i, w) in w.iter_mut().enumerate() { + let o = (y + i) * stride + x - 4; + let Some(&b) = pix.get(o..).and_then(|p| p.first_chunk::<8>()) else { + return; + }; + *w = u64::from_le_bytes(b); + } + let mut s = [[0i16; N]; 8]; + for (k, row) in s.iter_mut().enumerate() { + for i in 0..N { + row[i] = ((w[i] >> (8 * k)) & 0xff) as i16; + } + } + filter_lanes(&mut s, ep, chroma); + for (i, w) in w.iter_mut().enumerate() { + *w = (0..8).fold(0u64, |acc, k| acc | ((s[k][i] as u8 as u64) << (8 * k))); + } + for (i, w) in w.iter().enumerate() { + let o = (y + i) * stride + x - 4; + pix[o..o + 8].copy_from_slice(&w.to_le_bytes()); + } +} + +/// Filter a horizontal edge whose q0 row is `y`, for `N` columns starting at `x`. +#[inline(always)] +fn filter_horizontal(pix: &mut [u8], stride: usize, x: usize, y: usize, ep: &EdgeParams, chroma: bool) { + let base = (y - 4) * stride + x; + let mut s = [[0i16; N]; 8]; + for (k, row) in s.iter_mut().enumerate() { + let Some(src) = pix.get(base + k * stride..).and_then(|p| p.first_chunk::()) else { + return; + }; + for i in 0..N { + row[i] = src[i] as i16; + } + } + filter_lanes(&mut s, ep, chroma); + let (k0, k1) = if chroma { (3, 5) } else { (1, 7) }; + for (k, row) in s.iter().enumerate().take(k1).skip(k0) { + let Some(dst) = pix.get_mut(base + k * stride..).and_then(|p| p.first_chunk_mut::()) else { + return; + }; + for i in 0..N { + dst[i] = row[i] as u8; + } + } +} + +/// Deblock one macroblock (macroblocks must be processed in raster order). +#[allow(clippy::needless_range_loop)] +pub fn deblock_mb(pic: &mut PicState, addr: usize, mb_w: usize) { + let q = &pic.mbs[addr]; + if q.slice_num == u32::MAX { + return; + } + let sq = &pic.slices[q.slice_num as usize]; + if sq.disable_deblocking_filter_idc == 1 { + return; + } + let (mx, my) = (addr % mb_w, addr / mb_w); + let usable = |n: Option| -> Option { + let n = n?; + let st = &pic.mbs[n]; + if st.slice_num == u32::MAX || (sq.disable_deblocking_filter_idc == 2 && st.slice_num != q.slice_num) { + return None; + } + Some(n) + }; + let left = usable(if mx > 0 { Some(addr - 1) } else { None }); + let top = usable(if my > 0 { Some(addr - mb_w) } else { None }); + let alpha_off = sq.alpha_offset; + let beta_off = sq.beta_offset; + let t8 = q.transform_8x8; + // bS[dir][edge][segment]; dir 0 = vertical edges (x), 1 = horizontal edges (y) + let mut bs = [[[0u8; 4]; 4]; 2]; + if q.kind.is_intra() { + for dir in 0..2 { + if (if dir == 0 { left } else { top }).is_some() { + bs[dir][0] = [4; 4]; + } + for e in 1..4 { + if !(t8 && e % 2 == 1) { + bs[dir][e] = [3; 4]; + } + } + } + } else { + let qids = ref_ids8(q, sq); + let q_uniform = (0..2).all(|l| { + let r = q.ref_idx[l]; + r[1] == r[0] && r[2] == r[0] && r[3] == r[0] && q.mv[l].iter().all(|m| *m == q.mv[l][0]) + }); + let qm: [Motion; 16] = std::array::from_fn(|r| block_motion(q, &qids, r)); + for dir in 0..2 { + let neighbor = if dir == 0 { left } else { top }; + if let Some(n) = neighbor { + let p = &pic.mbs[n]; + if p.kind.is_intra() { + bs[dir][0] = [4; 4]; + } else { + let pids = ref_ids8(p, &pic.slices[p.slice_num as usize]); + for k in 0..4 { + let (rq, rp) = if dir == 0 { (k * 4, k * 4 + 3) } else { (k, 12 + k) }; + bs[dir][0][k] = if (p.nz_mask >> rp) & 1 != 0 || (q.nz_mask >> rq) & 1 != 0 { + 2 + } else { + motion_bs(&block_motion(p, &pids, rp), &qm[rq]) + }; + } + } + } + if q_uniform && q.nz_mask == 0 { + // one motion for the whole MB and no coefficients: all internal edges have bS 0 + continue; + } + for e in 1..4 { + if t8 && e % 2 == 1 { + continue; + } + for k in 0..4 { + let (rq, rp) = if dir == 0 { (k * 4 + e, k * 4 + e - 1) } else { (e * 4 + k, e * 4 + k - 4) }; + bs[dir][e][k] = if (q.nz_mask >> rp) & 1 != 0 || (q.nz_mask >> rq) & 1 != 0 { 2 } else { motion_bs(&qm[rp], &qm[rq]) }; + } + } + } + } + let qp_of = |st: &MbState| if st.kind == MbKind::IPcm { 0 } else { st.qp as i32 }; + let q_qp = qp_of(q); + let q_qpc = q.qpc; + let p_qp = [left.map(|n| (qp_of(&pic.mbs[n]), pic.mbs[n].qpc)), top.map(|n| (qp_of(&pic.mbs[n]), pic.mbs[n].qpc))]; + let params = |qpp: i32, qpq: i32, b: [u8; 4]| { + let qpav = (qpp + qpq + 1) >> 1; + let index_a = (qpav + alpha_off).clamp(0, 51) as usize; + let index_b = (qpav + beta_off).clamp(0, 51) as usize; + let tc = |b: u8| if (1..4).contains(&b) { TC0[index_a][b as usize - 1] as i32 } else { 0 }; + EdgeParams { bs: b, tc0: [tc(b[0]), tc(b[1]), tc(b[2]), tc(b[3])], alpha: ALPHA[index_a] as i32, beta: BETA[index_b] as i32 } + }; + let width = pic.planes.width; + let cwidth = pic.planes.cwidth; + // luma + for dir in 0..2 { + for e in 0..4 { + if bs[dir][e] == [0; 4] { + continue; + } + let qpp = if e == 0 { p_qp[dir].map(|p| p.0).unwrap_or(q_qp) } else { q_qp }; + let ep = params(qpp, q_qp, bs[dir][e]); + if ep.alpha == 0 || ep.beta == 0 { + // filterSamplesFlag needs |p0 - q0| < alpha and |p1 - p0| < beta (8.7.2.2): no + // sample of the edge changes (low QPs) + continue; + } + if dir == 0 { + filter_vertical::<16>(&mut pic.planes.y, width, mx * 16 + e * 4, my * 16, &ep, false); + } else { + filter_horizontal::<16>(&mut pic.planes.y, width, mx * 16, my * 16 + e * 4, &ep, false); + } + } + } + // chroma (4:2:0): edges at chroma samples 0 and 4 use the bS of luma edges 0 and 2 + for c in 0..2 { + for dir in 0..2 { + for ce in 0..2 { + let e = ce * 2; + if bs[dir][e] == [0; 4] { + continue; + } + let qpp = if e == 0 { p_qp[dir].map(|p| p.1[c]).unwrap_or(q_qpc[c]) } else { q_qpc[c] } as i32; + let ep = params(qpp, q_qpc[c] as i32, bs[dir][e]); + if ep.alpha == 0 || ep.beta == 0 { + continue; + } + let plane = if c == 0 { &mut pic.planes.cb } else { &mut pic.planes.cr }; + if dir == 0 { + filter_vertical::<8>(plane, cwidth, mx * 8 + ce * 4, my * 8, &ep, true); + } else { + filter_horizontal::<8>(plane, cwidth, mx * 8, my * 8 + ce * 4, &ep, true); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The vectorised edge filter equals the per-line reference [`filter8`] on random edges + /// (small differences across the edge so that every filter branch is taken). + #[test] + fn lane_filter_matches_per_line_reference() { + let mut rng = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + rng + }; + for iter in 0..20_000 { + let chroma = iter % 3 == 0; + let strong = iter % 4 == 1; + let bs: [u8; 4] = if strong { [4; 4] } else { std::array::from_fn(|_| (next() % 4) as u8) }; + let index_a = (next() % 52) as usize; + let index_b = (next() % 52) as usize; + let tc = |b: u8| if (1..4).contains(&b) { TC0[index_a][b as usize - 1] as i32 } else { 0 }; + let ep = EdgeParams { bs, tc0: bs.map(tc), alpha: ALPHA[index_a] as i32, beta: BETA[index_b] as i32 }; + let base = (next() % 256) as i32; + let spread = 1 + (next() % 24) as i32; + let mut lines = [[0u8; 8]; 16]; + for l in lines.iter_mut() { + for v in l.iter_mut() { + *v = (base + (next() % (2 * spread as u64 + 1)) as i32 - spread).clamp(0, 255) as u8; + } + } + let n = if chroma { 8 } else { 16 }; + let mut want = lines; + for (i, l) in want.iter_mut().enumerate().take(n) { + let k = i / (n / 4); + if bs[k] != 0 { + filter8(l, bs[k], ep.alpha, ep.beta, ep.tc0[k], chroma); + } + } + let got: Vec<[u8; 8]> = if chroma { + let mut s = [[0i16; 8]; 8]; + for i in 0..8 { + for k in 0..8 { + s[k][i] = lines[i][k] as i16; + } + } + filter_lanes(&mut s, &ep, true); + (0..8).map(|i| std::array::from_fn(|k| s[k][i] as u8)).collect() + } else { + let mut s = [[0i16; 16]; 8]; + for i in 0..16 { + for k in 0..8 { + s[k][i] = lines[i][k] as i16; + } + } + filter_lanes(&mut s, &ep, false); + (0..16).map(|i| std::array::from_fn(|k| s[k][i] as u8)).collect() + }; + assert_eq!(&got[..], &want[..n], "iteration {iter}: bs {bs:?} indexA {index_a} indexB {index_b}"); + } + } +} + +#[cfg(test)] +mod speed { + use super::*; + + /// Rough timing of the edge filters (`cargo test --release -p soundcraft-video --lib edge_filter_speed -- --ignored --nocapture`). + #[test] + #[ignore] + fn edge_filter_speed() { + let (w, h) = (1024usize, 256usize); + let mut pix: Vec = (0..w * h).map(|i| (128 + ((i * 7919) % 13) as i32 - 6) as u8).collect(); + let eps = [ + EdgeParams { bs: [4; 4], tc0: [0; 4], alpha: 40, beta: 9 }, + EdgeParams { bs: [3; 4], tc0: [2; 4], alpha: 40, beta: 9 }, + EdgeParams { bs: [2, 1, 0, 2], tc0: [1, 1, 0, 1], alpha: 40, beta: 9 }, + ]; + let t0 = std::time::Instant::now(); + let mut n = 0; + for _ in 0..20 { + for my in 1..h / 16 { + for mx in 1..w / 16 { + for (j, ep) in eps.iter().enumerate() { + filter_horizontal::<16>(&mut pix, w, mx * 16, my * 16 + j * 4, ep, false); + n += 1; + } + } + } + } + let dt = t0.elapsed(); + let tv = std::time::Instant::now(); + for _ in 0..20 { + for my in 1..h / 16 { + for mx in 1..w / 16 { + for (j, ep) in eps.iter().enumerate() { + filter_vertical::<16>(&mut pix, w, mx * 16 + j * 4, my * 16, ep, false); + } + } + } + } + println!("vertical: {:.1} ns/edge", tv.elapsed().as_nanos() as f64 / n as f64); + let t1 = std::time::Instant::now(); + for _ in 0..20 { + for my in 1..h / 16 { + for mx in 1..w / 16 { + for ep in eps.iter() { + for (vert, x, y) in [(true, mx * 16, my * 16), (false, mx * 16, my * 16)] { + for i in 0..16 { + let b = ep.bs[i / 4]; + if b == 0 { + continue; + } + let mut v = [0u8; 8]; + for k in 0..8 { + v[k] = if vert { pix[(y + i) * w + x - 4 + k] } else { pix[(y - 4 + k) * w + x + i] }; + } + filter8(&mut v, b, ep.alpha, ep.beta, ep.tc0[i / 4], false); + for k in 1..7 { + if vert { + pix[(y + i) * w + x - 4 + k] = v[k]; + } else { + pix[(y - 4 + k) * w + x + i] = v[k]; + } + } + } + } + } + } + } + } + println!("scalar reference (both): {:.1} ns/edge", t1.elapsed().as_nanos() as f64 / (2 * n) as f64); + println!( + "horizontal: {n} luma edges in {dt:?}: {:.1} ns/edge (checksum {})", + dt.as_nanos() as f64 / n as f64, + pix.iter().map(|&v| v as u64).sum::() + ); + } +} diff --git a/crates/video/src/h264/decoder.rs b/crates/video/src/h264/decoder.rs new file mode 100644 index 00000000..a80c945b --- /dev/null +++ b/crates/video/src/h264/decoder.rs @@ -0,0 +1,645 @@ +//! Top-level decoder: NAL dispatch, picture boundaries, POC, DPB and output. +//! +//! The calling thread parses headers and does all DPB / reference list bookkeeping (which only needs +//! header information). Macroblock decoding of each picture runs as a job — on a thread pool with the +//! `threads` feature — that publishes finished macroblock rows into the picture's shared [`Frame`]; +//! jobs of later pictures block per row on the reference data they need, so several pictures decode +//! concurrently (frame-level parallelism). + +use crate::bitstream::{BitReader, annexb_nals, length_prefixed_nals, unescape_rbsp}; +use crate::h264::dpb::{Dpb, Output, OutputMeta}; +use crate::h264::error::{Error, Result, ensure, invalid, unsupported}; +use crate::h264::params::{Pps, Sps}; +use crate::h264::picture::{Frame, FrameRef, MbKind, MbState, Planes, RefPic}; +use crate::h264::slice::{NalHeader, Poc, PocState, SliceHeader, SliceType, nal_type}; +use crate::h264::slicedec::{PicState, SliceDecoder}; +use crate::h264::transform::LevelScale; +use crate::h264::{ColorInfo, Picture}; +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; + +/// One slice of a picture job. +struct SliceJob { + sh: SliceHeader, + pps: Arc, + sps: Arc, + rbsp: Vec, + refs: [Vec; 2], + ls: Arc, +} + +/// A picture whose slices are being collected. +struct PendingPic { + frame: FrameRef, + sps: Arc, + first: SliceHeader, + poc: Poc, + pts: i64, + key: bool, + slices: Vec, +} + +/// Counters describing what the decoder has seen (useful to check test coverage). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DecodeStats { + pub pictures: u64, + pub slices_cavlc: u64, + pub slices_cabac: u64, + pub slices_i: u64, + pub slices_p: u64, + pub slices_b: u64, + pub mb_i4x4: u64, + pub mb_i8x8: u64, + pub mb_i16x16: u64, + pub mb_pcm: u64, + pub mb_p_skip: u64, + pub mb_b_skip: u64, + pub mb_b_direct16x16: u64, + pub mb_inter: u64, + /// Inter macroblocks using the 8x8 transform. + pub mb_inter_8x8_transform: u64, + pub mmco_ops: u64, + pub long_term_marks: u64, + pub frame_num_gaps: u64, + pub weighted_slices: u64, + pub temporal_direct_slices: u64, + /// Non-reference pictures decoded in draft mode (deblocking skipped). + pub draft_pictures: u64, +} + +/// State shared with decoding jobs. +#[derive(Default)] +struct Shared { + error: Mutex>, + stats: Mutex, + /// Recycled picture buffers. + pool: Mutex>, +} + +/// H.264 decoder. +pub struct Decoder { + spss: Vec>>, + ppss: Vec>>, + nal_length_size: Option, + dpb: Dpb, + poc_state: PocState, + prev_ref_frame_num: u32, + pending: Option, + next_id: u32, + active_sps: Option>, + ls_cache: Vec<(Arc, Arc)>, + /// Pictures leaving the DPB, converted once their decoding job has finished. + out_queue: VecDeque, + shared: Arc, + #[cfg(feature = "threads")] + pool: Option, + in_flight: VecDeque, + max_in_flight: usize, + threads: usize, + /// Draft mode: skip deblocking of non-reference pictures ([`Decoder::set_draft`]). + draft: bool, +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +/// Is `sh` the first slice of a new primary coded picture relative to `prev` (7.4.1.2.4)? +fn is_new_picture(prev: &SliceHeader, sh: &SliceHeader, sps: &Sps) -> bool { + if sh.frame_num != prev.frame_num + || sh.pps_id != prev.pps_id + || sh.field_pic != prev.field_pic + || sh.bottom_field != prev.bottom_field + || (sh.nal_ref_idc == 0) != (prev.nal_ref_idc == 0) + || sh.idr != prev.idr + || (sh.idr && prev.idr && sh.idr_pic_id != prev.idr_pic_id) + { + return true; + } + if sps.pic_order_cnt_type == 0 + && (sh.pic_order_cnt_lsb != prev.pic_order_cnt_lsb || sh.delta_pic_order_cnt_bottom != prev.delta_pic_order_cnt_bottom) + { + return true; + } + if sps.pic_order_cnt_type == 1 && sh.delta_pic_order_cnt != prev.delta_pic_order_cnt { + return true; + } + false +} + +/// The worker threads [`Decoder::new`] uses: the available cores (at most 16) with the `threads` +/// feature on native targets, 1 otherwise. +pub fn default_threads() -> usize { + #[cfg(all(feature = "threads", not(target_arch = "wasm32")))] + { + std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1).min(16) + } + #[cfg(not(all(feature = "threads", not(target_arch = "wasm32"))))] + { + 1 + } +} + +impl Decoder { + /// A decoder using all available cores (with the `threads` feature). + pub fn new() -> Self { + Self::with_threads(default_threads()) + } + + /// A decoder using up to `threads` worker threads (1 = decode on the calling thread). + pub fn with_threads(threads: usize) -> Self { + crate::h264::cavlc::init_tables(); + #[cfg(feature = "threads")] + let pool = if threads > 1 && cfg!(not(target_arch = "wasm32")) { + rayon::ThreadPoolBuilder::new().num_threads(threads).thread_name(|i| format!("h264-{i}")).build().ok() + } else { + None + }; + #[cfg(feature = "threads")] + let workers = if pool.is_some() { threads } else { 1 }; + #[cfg(not(feature = "threads"))] + let workers = 1; + Decoder { + spss: vec![None; 32], + ppss: vec![None; 256], + nal_length_size: None, + dpb: Dpb::new(), + poc_state: PocState::default(), + prev_ref_frame_num: 0, + pending: None, + next_id: 1, + active_sps: None, + ls_cache: Vec::new(), + out_queue: VecDeque::new(), + shared: Arc::new(Shared::default()), + #[cfg(feature = "threads")] + pool, + in_flight: VecDeque::new(), + max_in_flight: threads.max(1) + 2, + threads: workers, + draft: false, + } + } + + /// Worker threads decoding pictures (1: on the calling thread). + pub fn threads(&self) -> usize { + self.threads + } + + /// Draft mode (off by default) for reduced-resolution playback: pictures submitted from now + /// on that are non-reference pictures (`nal_ref_idc` 0) skip the deblocking filter (8.7). + /// Nothing predicts from a non-reference picture (and deblocking never changes the motion + /// data used for direct prediction), so every other picture stays bit-exact; the draft + /// pictures themselves are approximate and flagged [`Picture::draft`]. Never use it for + /// export or for a frame shown while paused. + pub fn set_draft(&mut self, on: bool) { + self.draft = on; + } + + pub fn draft(&self) -> bool { + self.draft + } + + /// Configure from an `avcC` (AVCDecoderConfigurationRecord) box payload. + pub fn from_avcc(avcc: &[u8]) -> Result { + let mut d = Decoder::new(); + d.configure_avcc(avcc)?; + Ok(d) + } + + /// Parse an `avcC` record: SPS/PPS and the NAL length size used by [`Decoder::decode`]. + pub fn configure_avcc(&mut self, avcc: &[u8]) -> Result<()> { + ensure!(avcc.len() >= 7, "avcC too short"); + ensure!(avcc[0] == 1, "unsupported avcC version {}", avcc[0]); + self.nal_length_size = Some((avcc[4] & 3) as usize + 1); + let mut pos = 5; + let read_sets = |pos: &mut usize, count: usize, d: &mut Decoder| -> Result<()> { + for _ in 0..count { + ensure!(*pos + 2 <= avcc.len(), "avcC truncated"); + let len = u16::from_be_bytes([avcc[*pos], avcc[*pos + 1]]) as usize; + *pos += 2; + ensure!(*pos + len <= avcc.len(), "avcC truncated"); + d.handle_nal(&avcc[*pos..*pos + len], 0)?; + *pos += len; + } + Ok(()) + }; + let nsps = (avcc[pos] & 0x1f) as usize; + pos += 1; + read_sets(&mut pos, nsps, self)?; + ensure!(pos < avcc.len(), "avcC truncated"); + let npps = avcc[pos] as usize; + pos += 1; + read_sets(&mut pos, npps, self) + } + + /// Length of NAL length prefixes (from avcC), or None for Annex-B input. + pub fn nal_length_size(&self) -> Option { + self.nal_length_size + } + + /// Decode one access unit (all NAL units of one picture, Annex-B or length-prefixed per + /// configuration; a byte stream with several complete access units is accepted too). Returns + /// pictures that became ready for output, in output order; `pts` travels with the picture of + /// this access unit through reordering. + pub fn decode(&mut self, data: &[u8], pts: i64) -> Result> { + let nals = match self.nal_length_size { + Some(n) => length_prefixed_nals(data, n)?, + None => annexb_nals(data), + }; + let mut result = Ok(()); + for nal in nals { + if let Err(e) = self.handle_nal(nal, pts) { + result = Err(e); + break; + } + } + // The access unit is complete: hand its picture to a decoding job. + self.submit_pending(); + if let Some(e) = self.take_error() { + result = result.and(Err(e)); + } + result?; + // Return finished pictures without blocking on frames still being decoded (bounded latency). + let max_queue = self.max_in_flight * 2; + let mut pics = Vec::new(); + while let Some(o) = self.out_queue.front() { + if !o.frame.is_published(o.frame.mb_h() - 1) && self.out_queue.len() <= max_queue { + break; + } + let Some(o) = self.out_queue.pop_front() else { break }; + pics.push(make_picture(&o)); + } + Ok(pics) + } + + /// Output all remaining pictures (end of stream). + pub fn flush(&mut self) -> Vec { + self.submit_pending(); + let mut outs = Vec::new(); + self.dpb.flush(&mut outs); + self.emit(outs); + let pics = self.out_queue.drain(..).map(|o| make_picture(&o)).collect(); + while let Some(f) = self.in_flight.pop_front() { + f.wait_complete(); + } + pics + } + + /// Statistics accumulated so far (macroblock counts include finished pictures only). + pub fn stats(&self) -> DecodeStats { + self.shared.stats.lock().map(|s| s.clone()).unwrap_or_default() + } + + fn stat(&self, f: impl FnOnce(&mut DecodeStats)) { + if let Ok(mut s) = self.shared.stats.lock() { + f(&mut s); + } + } + + /// First error reported by a decoding job since the last call (also returned by + /// [`Decoder::decode`]); useful after [`Decoder::flush`]. + pub fn take_error(&mut self) -> Option { + self.shared.error.lock().ok().and_then(|mut e| e.take()) + } + + fn handle_nal(&mut self, nal: &[u8], pts: i64) -> Result<()> { + if nal.is_empty() { + return Ok(()); + } + let hdr = NalHeader::parse(nal[0])?; + match hdr.nal_unit_type { + nal_type::SPS => { + let sps = Sps::parse(&unescape_rbsp(&nal[1..]))?; + let id = sps.id as usize; + self.spss[id] = Some(Arc::new(sps)); + } + nal_type::PPS => { + let rbsp = unescape_rbsp(&nal[1..]); + let spss: Vec> = self.spss.iter().map(|s| s.as_ref().map(|s| (**s).clone())).collect(); + let pps = Pps::parse(&rbsp, &spss)?; + let id = pps.id as usize; + self.ppss[id] = Some(Arc::new(pps)); + } + nal_type::SLICE | nal_type::IDR => self.handle_slice(nal, hdr, pts)?, + nal_type::SLICE_DPA | nal_type::SLICE_DPB | nal_type::SLICE_DPC => { + return unsupported("data partitioning (Extended profile)"); + } + nal_type::END_SEQ | nal_type::END_STREAM => self.submit_pending(), + // SEI, AUD, filler, SPS extension, prefix NAL, subset SPS, auxiliary and extension slices: skipped. + _ => {} + } + Ok(()) + } + + fn level_scale(&mut self, pps: &Arc) -> Arc { + if let Some((_, ls)) = self.ls_cache.iter().find(|(p, _)| Arc::ptr_eq(p, pps)) { + return ls.clone(); + } + let ls: Arc = Arc::from(LevelScale::new(&pps.scaling)); + self.ls_cache.retain(|(p, _)| self.ppss.iter().flatten().any(|q| Arc::ptr_eq(p, q))); + self.ls_cache.push((pps.clone(), ls.clone())); + ls + } + + fn handle_slice(&mut self, nal: &[u8], hdr: NalHeader, pts: i64) -> Result<()> { + let rbsp = unescape_rbsp(&nal[1..]); + let ppss = &self.ppss; + let spss = &self.spss; + let mut found: Option<(Arc, Arc)> = None; + let (sh, _, _) = SliceHeader::parse(&rbsp, hdr, |id| { + let pps = ppss[id as usize].as_ref().ok_or_else(|| Error::MissingParameterSet(format!("PPS {id}")))?; + let sps = spss[pps.sps_id as usize].as_ref().ok_or_else(|| Error::MissingParameterSet(format!("SPS {}", pps.sps_id)))?; + found = Some((pps.clone(), sps.clone())); + Ok((&**pps, &**sps)) + })?; + let Some((pps, sps)) = found else { + return Err(Error::MissingParameterSet("PPS".into())); + }; + sps.check_supported()?; + if pps.num_slice_groups > 1 { + return unsupported("slice groups (FMO)"); + } + if sh.field_pic { + return unsupported("field pictures"); + } + if matches!(sh.slice_type, SliceType::Sp | SliceType::Si) { + return unsupported("SP/SI slices"); + } + if sh.redundant_pic_cnt > 0 { + return Ok(()); // redundant slices are ignored + } + let new_pic = match &self.pending { + None => true, + Some(p) => is_new_picture(&p.first, &sh, &sps) || sh.first_mb_in_slice == 0 || !Arc::ptr_eq(&p.sps, &sps), + }; + if new_pic { + self.submit_pending(); + if sh.first_mb_in_slice != 0 && self.active_sps.is_none() { + return invalid("stream does not start with the first slice of a picture"); + } + self.start_picture(&sh, &sps, pts)?; + } + let ls = self.level_scale(&pps); + let Some(pending) = self.pending.as_mut() else { + return invalid("slice without a started picture"); + }; + let refs = self.dpb.build_ref_lists(&sh, pending.poc.frame(), sps.max_frame_num())?; + if !sh.slice_type.is_intra() { + ensure!(!refs[0].is_empty(), "no reference pictures available for inter slice"); + if sh.slice_type.is_b() { + ensure!(!refs[1].is_empty(), "empty RefPicList1 in B slice"); + } + } + let weighted = (pps.weighted_pred && sh.slice_type.is_p()) || (pps.weighted_bipred_idc != 0 && sh.slice_type.is_b()); + let temporal = sh.slice_type.is_b() && !sh.direct_spatial_mv_pred; + let (cabac, st, mmcos, lt) = ( + pps.entropy_coding_mode, + sh.slice_type, + sh.mmcos.len() as u64, + sh.long_term_reference as u64 + sh.mmcos.iter().filter(|m| m.op == 3 || m.op == 6).count() as u64, + ); + pending.slices.push(SliceJob { sh, pps, sps, rbsp, refs, ls }); + self.stat(|s| { + if cabac { + s.slices_cabac += 1; + } else { + s.slices_cavlc += 1; + } + match st { + SliceType::I | SliceType::Si => s.slices_i += 1, + SliceType::P | SliceType::Sp => s.slices_p += 1, + SliceType::B => s.slices_b += 1, + } + s.weighted_slices += weighted as u64; + s.temporal_direct_slices += temporal as u64; + s.mmco_ops += mmcos; + s.long_term_marks += lt; + }); + Ok(()) + } + + fn start_picture(&mut self, sh: &SliceHeader, sps: &Arc, pts: i64) -> Result<()> { + // Activate SPS; a resolution change flushes the DPB. + let changed = match &self.active_sps { + None => true, + Some(a) => a.width() != sps.width() || a.height() != sps.height() || a.max_dpb_frames() != sps.max_dpb_frames(), + }; + if changed && self.active_sps.is_some() { + let mut outs = Vec::new(); + self.dpb.flush(&mut outs); + self.emit(outs); + self.dpb.entries.clear(); + } + self.active_sps = Some(sps.clone()); + self.dpb.capacity = sps.max_dpb_frames(); + self.dpb.max_reorder = sps.max_num_reorder_frames().min(self.dpb.capacity); + let mb_w = sps.pic_width_in_mbs as usize; + let mb_h = sps.frame_height_in_mbs() as usize; + if sh.idr { + let mut outs = Vec::new(); + self.dpb.idr(sh.no_output_of_prior_pics, &mut outs); + self.emit(outs); + self.prev_ref_frame_num = 0; + } else { + let max = sps.max_frame_num(); + if sh.frame_num != self.prev_ref_frame_num && sh.frame_num != (self.prev_ref_frame_num + 1) % max { + // frame_num gap (8.2.5.2): insert "non-existing" frames. Their samples are never used by + // conforming streams; copy the latest decoded frame for concealment. + let meta = Arc::new(output_meta(sps, pts, false)); + let next_id = &mut self.next_id; + let poc_state = &mut self.poc_state; + let last = self.dpb.entries.iter().filter(|e| !e.non_existing).max_by_key(|e| e.frame.id).map(|e| e.frame.clone()); + let mut make = |_fnum: u32| { + let id = *next_id; + *next_id += 1; + let planes = match &last { + Some(f) if f.mb_w == mb_w && f.mb_h() == mb_h => { + let mut p = Planes::new(mb_w * 16, mb_h * 16); + let (y, u, v) = f.copy_cropped((0, 0, mb_w * 16, mb_h * 16)); + p.y = y; + p.cb = u; + p.cr = v; + p + } + _ => Planes::gray(mb_w * 16, mb_h * 16), + }; + (Arc::new(Frame::from_planes(id, 0, &planes)), meta.clone()) + }; + let mut upd = |fnum: u32| poc_state.update_gap_frame(fnum, sps); + self.shared.stats.lock().map(|mut s| s.frame_num_gaps += 1).ok(); + self.dpb.fill_frame_num_gap(self.prev_ref_frame_num, sh.frame_num, max, sps.max_num_ref_frames as usize, &mut make, &mut upd); + self.prev_ref_frame_num = (sh.frame_num + max - 1) % max; + } + } + let poc = self.poc_state.compute(sh, sps); + // MMCO5: the picture's POC becomes relative to itself (tempPicOrderCnt, 8.2.1). + let frame_poc = if sh.has_mmco5() { 0 } else { poc.frame() }; + let id = self.next_id; + self.next_id += 1; + let frame = Arc::new(Frame::new(id, frame_poc, mb_w, mb_h)); + self.pending = Some(PendingPic { frame, sps: sps.clone(), first: sh.clone(), poc, pts, key: sh.idr, slices: Vec::new() }); + Ok(()) + } + + /// Start decoding the pending picture and do its reference marking / DPB insertion. + fn submit_pending(&mut self) { + let Some(p) = self.pending.take() else { return }; + let PendingPic { frame, sps, first, poc, pts, key, slices } = p; + let draft = self.draft && first.nal_ref_idc == 0; + self.dispatch(frame.clone(), slices, draft); + self.poc_state.update(&first, &poc); + if first.nal_ref_idc != 0 { + self.prev_ref_frame_num = if first.has_mmco5() { 0 } else { first.frame_num }; + } + let meta = Arc::new(OutputMeta { draft, ..output_meta(&sps, pts, key) }); + let mut outs = Vec::new(); + let fpoc = frame.poc; + self.dpb.store_picture(&first, frame, fpoc, sps.max_frame_num(), sps.max_num_ref_frames as usize, meta, &mut outs); + self.emit(outs); + } + + fn dispatch(&mut self, frame: FrameRef, slices: Vec, draft: bool) { + let shared = self.shared.clone(); + #[cfg(feature = "threads")] + if let Some(pool) = &self.pool { + self.in_flight.retain(|f| !f.is_published(f.mb_h() - 1)); + while self.in_flight.len() >= self.max_in_flight { + if let Some(f) = self.in_flight.pop_front() { + f.wait_complete(); + } + } + self.in_flight.push_back(frame.clone()); + pool.spawn_fifo(move || run_job(frame, slices, &shared, draft)); + return; + } + run_job(frame, slices, &shared, draft); + } + + fn emit(&mut self, outs: Vec) { + self.out_queue.extend(outs); + } +} + +/// Decode all slices of one picture and publish its rows. +fn run_job(frame: FrameRef, slices: Vec, shared: &Shared, draft: bool) { + let reuse = shared.pool.lock().ok().and_then(|mut p| p.pop()); + let mut pic = match reuse { + Some(mut p) if p.mb_w == frame.mb_w && p.mb_h == frame.mb_h() => { + p.reset(frame.clone()); + p + } + _ => PicState::new(frame.clone()), + }; + pic.skip_deblock = draft; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut first_err = None; + for s in &slices { + if let Err(e) = decode_slice(s, &mut pic) { + first_err.get_or_insert(e); + } + } + pic.flush_rows(true); + first_err + })); + let err = match result { + Ok(e) => e, + Err(_) => { + // Make sure readers never block on this frame. + let fallback = Planes::gray(frame.width, frame.height); + let blank = vec![MbState::default(); frame.mb_w * frame.mb_h()]; + for r in 0..frame.mb_h() { + if !frame.is_published(r) { + frame.publish(r, Frame::make_row(&fallback, r, &blank, &|_, _, _| u32::MAX)); + } + } + pic = PicState::new(frame.clone()); + Some(Error::Invalid("internal error while decoding a picture".into())) + } + }; + if let Ok(mut s) = shared.stats.lock() { + s.pictures += 1; + s.draft_pictures += draft as u64; + for st in &pic.mbs { + match st.kind { + MbKind::I4x4 => s.mb_i4x4 += 1, + MbKind::I8x8 => s.mb_i8x8 += 1, + MbKind::I16x16 => s.mb_i16x16 += 1, + MbKind::IPcm => s.mb_pcm += 1, + MbKind::PSkip => s.mb_p_skip += 1, + MbKind::BSkip => s.mb_b_skip += 1, + MbKind::BDirect16x16 => s.mb_b_direct16x16 += 1, + MbKind::Inter => s.mb_inter += 1, + MbKind::None => {} + } + if st.transform_8x8 && !st.kind.is_intra() { + s.mb_inter_8x8_transform += 1; + } + } + } + if let (Some(e), Ok(mut slot)) = (err, shared.error.lock()) { + slot.get_or_insert(e); + } + if let Ok(mut p) = shared.pool.lock() + && p.len() < 4 + { + p.push(pic); + } +} + +fn decode_slice(s: &SliceJob, pic: &mut PicState) -> Result<()> { + let mut sd = SliceDecoder::new(&s.sh, &s.pps, &s.sps, pic, &s.refs, &s.ls)?; + if s.pps.entropy_coding_mode { + sd.decode_cabac(&s.rbsp) + } else { + let mut r = BitReader::new(&s.rbsp); + r.seek_bits(s.sh.header_bits); + sd.decode_cavlc(&mut r) + } +} + +fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta { + let vui = sps.vui.clone().unwrap_or_default(); + OutputMeta { + pts, + key, + crop: sps.crop_rect(), + full_range: vui.full_range, + colour_primaries: vui.colour_primaries, + transfer_characteristics: vui.transfer_characteristics, + matrix_coefficients: vui.matrix_coefficients, + sar: vui.sar, + draft: false, + } +} + +fn make_picture(o: &Output) -> Picture { + let (cx, cy, cw, ch) = o.meta.crop; + let (cx, cy, cw, ch) = (cx as usize, cy as usize, cw as usize, ch as usize); + let (y, u, v) = o.frame.copy_cropped((cx, cy, cw, ch)); + let (ccw, cch) = (cw.div_ceil(2), ch.div_ceil(2)); + Picture { + width: cw as u32, + height: ch as u32, + chroma_width: ccw as u32, + chroma_height: cch as u32, + y, + u, + v, + y_stride: cw, + uv_stride: ccw, + pts: o.meta.pts, + poc: o.poc, + key: o.meta.key, + color: ColorInfo { + full_range: o.meta.full_range, + primaries: o.meta.colour_primaries, + transfer: o.meta.transfer_characteristics, + matrix: o.meta.matrix_coefficients, + }, + sar: o.meta.sar, + draft: o.meta.draft, + } +} diff --git a/crates/video/src/h264/dpb.rs b/crates/video/src/h264/dpb.rs new file mode 100644 index 00000000..5afc169b --- /dev/null +++ b/crates/video/src/h264/dpb.rs @@ -0,0 +1,420 @@ +//! Decoded picture buffer: reference marking (8.2.5), reference list construction (8.2.4) and output +//! ordering / bumping (C.4). + +use crate::h264::error::{Result, ensure}; +use crate::h264::picture::{FrameRef, RefPic}; +use crate::h264::slice::{Mmco, SliceHeader, SliceType}; +use std::sync::Arc; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefMark { + Unused, + Short, + Long, +} + +/// Output-related metadata attached to a decoded frame. +#[derive(Clone, Debug)] +pub struct OutputMeta { + pub pts: i64, + pub key: bool, + pub crop: (u32, u32, u32, u32), + pub full_range: bool, + pub colour_primaries: u8, + pub transfer_characteristics: u8, + pub matrix_coefficients: u8, + pub sar: (u16, u16), + /// Decoded without deblocking (draft mode, non-reference picture). + pub draft: bool, +} + +pub struct DpbEntry { + pub frame: FrameRef, + pub poc: i32, + pub frame_num: u32, + pub long_term_frame_idx: u32, + pub mark: RefMark, + pub needed_for_output: bool, + pub non_existing: bool, + pub meta: Arc, +} + +pub struct Dpb { + pub entries: Vec, + /// None = "no long-term frame indices". + pub max_long_term_frame_idx: Option, + pub capacity: usize, + pub max_reorder: usize, +} + +/// A picture leaving the DPB for output. +pub struct Output { + pub frame: FrameRef, + pub poc: i32, + pub meta: Arc, +} + +impl Dpb { + pub fn new() -> Self { + Dpb { entries: Vec::new(), max_long_term_frame_idx: None, capacity: 16, max_reorder: 16 } + } + + fn frame_num_wrap(e: &DpbEntry, cur_frame_num: u32, max_frame_num: u32) -> i32 { + if e.frame_num > cur_frame_num { e.frame_num as i32 - max_frame_num as i32 } else { e.frame_num as i32 } + } + + fn num_refs(&self) -> usize { + self.entries.iter().filter(|e| e.mark != RefMark::Unused).count() + } + + /// Remove entries not needed for output and not used for reference. + fn prune(&mut self) { + self.entries.retain(|e| e.needed_for_output || e.mark != RefMark::Unused); + } + + /// Output the picture with the smallest POC (C.4.5.3 "bumping"). Returns false if none waits. + fn bump(&mut self, out: &mut Vec) -> bool { + let Some(i) = self.entries.iter().enumerate().filter(|(_, e)| e.needed_for_output).min_by_key(|(_, e)| e.poc).map(|(i, _)| i) else { + return false; + }; + let e = &mut self.entries[i]; + e.needed_for_output = false; + out.push(Output { frame: e.frame.clone(), poc: e.poc, meta: e.meta.clone() }); + self.prune(); + true + } + + /// Output everything in POC order and clear non-reference entries. + pub fn flush(&mut self, out: &mut Vec) { + while self.bump(out) {} + self.prune(); + } + + /// IDR: mark all references unused; output (or drop) prior pictures. + pub fn idr(&mut self, no_output_of_prior_pics: bool, out: &mut Vec) { + for e in &mut self.entries { + e.mark = RefMark::Unused; + } + if no_output_of_prior_pics { + self.entries.clear(); + } else { + self.flush(out); + } + self.max_long_term_frame_idx = None; + } + + /// Sliding window marking (8.2.5.3). + fn sliding_window(&mut self, cur_frame_num: u32, max_frame_num: u32, max_num_ref_frames: usize) { + let limit = max_num_ref_frames.max(1); + while self.num_refs() >= limit { + let victim = self + .entries + .iter() + .enumerate() + .filter(|(_, e)| e.mark == RefMark::Short) + .min_by_key(|(_, e)| Self::frame_num_wrap(e, cur_frame_num, max_frame_num)) + .map(|(i, _)| i); + match victim { + Some(i) => self.entries[i].mark = RefMark::Unused, + None => break, + } + } + self.prune(); + } + + /// Insert "non-existing" frames for a frame_num gap (8.2.5.2). + #[allow(clippy::too_many_arguments)] + pub fn fill_frame_num_gap( + &mut self, + prev_ref_frame_num: u32, + frame_num: u32, + max_frame_num: u32, + max_num_ref_frames: usize, + make_frame: &mut dyn FnMut(u32) -> (FrameRef, Arc), + poc_state: &mut dyn FnMut(u32), + ) { + let mut unused = (prev_ref_frame_num + 1) % max_frame_num; + let mut guard = 0; + while unused != frame_num && guard < max_frame_num { + self.sliding_window(unused, max_frame_num, max_num_ref_frames); + let (frame, meta) = make_frame(unused); + poc_state(unused); + self.entries.push(DpbEntry { + poc: frame.poc, + frame, + frame_num: unused, + long_term_frame_idx: 0, + mark: RefMark::Short, + needed_for_output: false, + non_existing: true, + meta, + }); + unused = (unused + 1) % max_frame_num; + guard += 1; + } + } + + /// Initial reference picture lists (8.2.4.2) + modification (8.2.4.3) for a frame slice. + pub fn build_ref_lists(&self, sh: &SliceHeader, cur_poc: i32, max_frame_num: u32) -> Result<[Vec; 2]> { + let mut lists: [Vec; 2] = [Vec::new(), Vec::new()]; + if sh.slice_type.is_intra() { + return Ok(lists); + } + let cur_fn = sh.frame_num; + let pic_num = |e: &DpbEntry| Self::frame_num_wrap(e, cur_fn, max_frame_num); + let short: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Short).collect(); + let mut long: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Long).collect(); + long.sort_by_key(|e| e.long_term_frame_idx); + let to_ref = |e: &&DpbEntry| RefPic { frame: e.frame.clone(), long_term: e.mark == RefMark::Long }; + let mut entry_lists: [Vec<&DpbEntry>; 2] = [Vec::new(), Vec::new()]; + if sh.slice_type != SliceType::B { + let mut s = short.clone(); + s.sort_by_key(|e| std::cmp::Reverse(pic_num(e))); + entry_lists[0] = s.into_iter().chain(long.iter().copied()).collect(); + } else { + let mut before: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc < cur_poc).collect(); + let mut after: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc > cur_poc).collect(); + before.sort_by_key(|e| std::cmp::Reverse(e.poc)); + after.sort_by_key(|e| e.poc); + entry_lists[0] = before.iter().chain(after.iter()).chain(long.iter()).copied().collect(); + entry_lists[1] = after.iter().chain(before.iter()).chain(long.iter()).copied().collect(); + if entry_lists[1].len() > 1 && entry_lists[0].len() == entry_lists[1].len() { + let same = entry_lists[0].iter().zip(entry_lists[1].iter()).all(|(a, b)| std::ptr::eq(*a, *b)); + if same { + entry_lists[1].swap(0, 1); + } + } + } + let nlists = if sh.slice_type == SliceType::B { 2 } else { 1 }; + for l in 0..nlists { + let n = sh.num_ref_idx_active[l] as usize; + let mut list: Vec> = entry_lists[l].iter().copied().map(Some).collect(); + list.truncate(n); + list.resize(n, None); + if !sh.ref_pic_list_mod[l].is_empty() { + let max_pic_num = max_frame_num as i32; + let curr_pic_num = cur_fn as i32; + let mut pred = curr_pic_num; + let mut ref_idx = 0usize; + for m in &sh.ref_pic_list_mod[l] { + ensure!(ref_idx < n, "too many ref_pic_list_modification entries"); + list.push(None); // temporarily one longer + let (target, is_long): (Option<&DpbEntry>, bool) = match m.idc { + 0 | 1 => { + let d = m.value as i32 + 1; + let no_wrap = if m.idc == 0 { + if pred - d < 0 { pred - d + max_pic_num } else { pred - d } + } else if pred + d >= max_pic_num { + pred + d - max_pic_num + } else { + pred + d + }; + pred = no_wrap; + let pn = if no_wrap > curr_pic_num { no_wrap - max_pic_num } else { no_wrap }; + (short.iter().copied().find(|e| pic_num(e) == pn), false) + } + 2 => (long.iter().copied().find(|e| e.long_term_frame_idx == m.value), true), + _ => continue, + }; + let Some(target) = target else { + // Missing picture: keep the list unchanged (error concealment). + list.pop(); + ref_idx += 1; + continue; + }; + for c in (ref_idx + 1..=n).rev() { + list[c] = list[c - 1]; + } + list[ref_idx] = Some(target); + ref_idx += 1; + let mut nidx = ref_idx; + for c in ref_idx..=n { + let keep = match list[c] { + None => true, + Some(e) => { + if is_long { + !(e.mark == RefMark::Long && e.long_term_frame_idx == target.long_term_frame_idx) + } else { + !(e.mark == RefMark::Short && std::ptr::eq(e, target)) + } + } + }; + if keep { + list[nidx] = list[c]; + nidx += 1; + } + } + list.truncate(n); + } + } + // Fill missing entries (should not happen in conforming streams) with the first valid entry. + let fallback = list.iter().flatten().next().copied().or_else(|| self.entries.last()); + let out: Vec = list.iter().filter_map(|e| e.or(fallback)).map(|e| to_ref(&e)).collect(); + lists[l] = out; + } + Ok(lists) + } + + /// Adaptive memory control (8.2.5.4). Returns true if MMCO 5 was executed. + fn apply_mmcos(&mut self, mmcos: &[Mmco], cur_frame_num: u32, max_frame_num: u32, cur_long: &mut Option) -> bool { + let mut had5 = false; + let curr_pic_num = cur_frame_num as i32; + for m in mmcos { + match m.op { + 1 => { + let pn = curr_pic_num - (m.difference_of_pic_nums_minus1 as i32 + 1); + for e in &mut self.entries { + if e.mark == RefMark::Short && Self::frame_num_wrap(e, cur_frame_num, max_frame_num) == pn { + e.mark = RefMark::Unused; + } + } + } + 2 => { + for e in &mut self.entries { + if e.mark == RefMark::Long && e.long_term_frame_idx == m.long_term_pic_num { + e.mark = RefMark::Unused; + } + } + } + 3 => { + let pn = curr_pic_num - (m.difference_of_pic_nums_minus1 as i32 + 1); + for e in &mut self.entries { + if e.mark == RefMark::Long && e.long_term_frame_idx == m.long_term_frame_idx { + e.mark = RefMark::Unused; + } + } + for e in &mut self.entries { + if e.mark == RefMark::Short && Self::frame_num_wrap(e, cur_frame_num, max_frame_num) == pn { + e.mark = RefMark::Long; + e.long_term_frame_idx = m.long_term_frame_idx; + } + } + } + 4 => { + let max = m.max_long_term_frame_idx_plus1; + for e in &mut self.entries { + if e.mark == RefMark::Long && (max == 0 || e.long_term_frame_idx > max - 1) { + e.mark = RefMark::Unused; + } + } + self.max_long_term_frame_idx = if max == 0 { None } else { Some(max - 1) }; + } + 5 => { + for e in &mut self.entries { + e.mark = RefMark::Unused; + } + self.max_long_term_frame_idx = None; + had5 = true; + } + 6 => { + for e in &mut self.entries { + if e.mark == RefMark::Long && e.long_term_frame_idx == m.long_term_frame_idx { + e.mark = RefMark::Unused; + } + } + *cur_long = Some(m.long_term_frame_idx); + } + _ => {} + } + } + self.prune(); + had5 + } + + /// Reference marking for the just-decoded picture and insertion into the DPB with output bumping. + /// Returns pictures ready for output. + #[allow(clippy::too_many_arguments)] + pub fn store_picture( + &mut self, + sh: &SliceHeader, + frame: FrameRef, + poc: i32, + max_frame_num: u32, + max_num_ref_frames: usize, + meta: Arc, + out: &mut Vec, + ) { + let mut mark = RefMark::Unused; + let mut long_idx = 0; + let mut had5 = false; + if sh.nal_ref_idc != 0 { + if sh.idr { + if sh.long_term_reference { + mark = RefMark::Long; + self.max_long_term_frame_idx = Some(0); + } else { + mark = RefMark::Short; + self.max_long_term_frame_idx = None; + } + } else { + let mut cur_long = None; + if sh.adaptive_ref_pic_marking { + had5 = self.apply_mmcos(&sh.mmcos, sh.frame_num, max_frame_num, &mut cur_long); + } else { + self.sliding_window(sh.frame_num, max_frame_num, max_num_ref_frames); + } + if let Some(i) = cur_long { + mark = RefMark::Long; + long_idx = i; + } else { + mark = RefMark::Short; + // make room if MMCOs left the DPB over-full (non-conforming streams) + if self.num_refs() >= max_num_ref_frames.max(1) && sh.adaptive_ref_pic_marking { + self.sliding_window(sh.frame_num, max_frame_num, max_num_ref_frames); + } + } + } + } + if had5 { + // All earlier pictures are output before the current one (C.4.4). + self.flush(out); + } + let frame_num = if had5 { 0 } else { sh.frame_num }; + // Bumping until there is space (C.4.5.1 / C.4.5.2). + if mark == RefMark::Unused { + loop { + let waiting = self.entries.len(); + if waiting < self.capacity { + break; + } + // non-reference picture with the smallest POC is output directly + let min_poc = self.entries.iter().filter(|e| e.needed_for_output).map(|e| e.poc).min(); + if min_poc.is_none_or(|m| poc < m) { + out.push(Output { frame, poc, meta }); + return; + } + if !self.bump(out) { + break; + } + } + } else { + while self.entries.len() >= self.capacity { + if !self.bump(out) { + break; + } + } + } + self.entries.push(DpbEntry { + frame, + poc, + frame_num, + long_term_frame_idx: long_idx, + mark, + needed_for_output: true, + non_existing: false, + meta, + }); + // Low-delay output: honour max_num_reorder_frames. + while self.entries.iter().filter(|e| e.needed_for_output).count() > self.max_reorder { + if !self.bump(out) { + break; + } + } + } +} + +impl Default for Dpb { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/video/src/h264/error.rs b/crates/video/src/h264/error.rs new file mode 100644 index 00000000..81d76656 --- /dev/null +++ b/crates/video/src/h264/error.rs @@ -0,0 +1,38 @@ +use crate::bitstream::BitError; + +/// Errors produced by the decoder. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The bitstream ended early or contained an invalid Exp-Golomb code. + #[error("bitstream error: {0}")] + Bitstream(#[from] BitError), + /// A syntax element had a value outside its legal range. + #[error("invalid bitstream: {0}")] + Invalid(String), + /// A valid feature this decoder does not implement (yet). + #[error("unsupported: {0}")] + Unsupported(String), + /// A slice referenced a parameter set that has not been received. + #[error("missing parameter set: {0}")] + MissingParameterSet(String), +} + +pub type Result = std::result::Result; + +pub(crate) fn invalid(msg: impl Into) -> Result { + Err(Error::Invalid(msg.into())) +} + +pub(crate) fn unsupported(msg: impl Into) -> Result { + Err(Error::Unsupported(msg.into())) +} + +/// `ensure!(cond, "message")` returns `Error::Invalid` when `cond` is false. +macro_rules! ensure { + ($cond:expr, $($arg:tt)+) => { + if !$cond { + return Err($crate::h264::error::Error::Invalid(format!($($arg)+))); + } + }; +} +pub(crate) use ensure; diff --git a/crates/video/src/h264/inter.rs b/crates/video/src/h264/inter.rs new file mode 100644 index 00000000..030d5ddd --- /dev/null +++ b/crates/video/src/h264/inter.rs @@ -0,0 +1,513 @@ +//! Inter prediction sample interpolation (8.4.2.2) and weighted sample prediction (8.4.2.3). + +/// A reference plane (used by the plane-based helpers in tests). +#[cfg(test)] +#[derive(Clone, Copy)] +pub struct PlaneRef<'a> { + pub data: &'a [u8], + pub width: usize, + pub height: usize, + pub stride: usize, +} + +const WIN: usize = 16 + 5; + +/// Copy the (bw+5)x(bh+5) window whose top-left is (x-2, y-2) into `win` (stride WIN), clamping +/// coordinates to the picture. +#[cfg(test)] +#[inline] +fn fetch_window(p: PlaneRef, x: i32, y: i32, bw: usize, bh: usize, win: &mut [u8; WIN * WIN]) { + let x0 = x - 2; + let y0 = y - 2; + let ww = bw + 5; + let wh = bh + 5; + let maxx = p.width as i32 - 1; + let maxy = p.height as i32 - 1; + if x0 >= 0 && y0 >= 0 && x0 + ww as i32 - 1 <= maxx && y0 + wh as i32 - 1 <= maxy { + for r in 0..wh { + let src = (y0 as usize + r) * p.stride + x0 as usize; + win[r * WIN..r * WIN + ww].copy_from_slice(&p.data[src..src + ww]); + } + } else { + for r in 0..wh { + let yy = (y0 + r as i32).clamp(0, maxy) as usize; + let row = &p.data[yy * p.stride..yy * p.stride + p.width]; + for c in 0..ww { + let xx = (x0 + c as i32).clamp(0, maxx) as usize; + win[r * WIN + c] = row[xx]; + } + } + } +} + +#[inline(always)] +fn tap6(a: i32, b: i32, c: i32, d: i32, e: i32, f: i32) -> i32 { + a - 5 * b + 20 * c + 20 * d - 5 * e + f +} + +#[inline(always)] +fn clip1(v: i32) -> u8 { + v.clamp(0, 255) as u8 +} + +/// Copy `n` bytes with a fixed-size move for the common block widths. +#[inline(always)] +fn copy_n(dst: &mut [u8], src: &[u8], n: usize) { + fn fixed(dst: &mut [u8], src: &[u8]) { + if let (Some(d), Some(s)) = (dst.first_chunk_mut::(), src.first_chunk::()) { + *d = *s; + } + } + match n { + 16 => fixed::<16>(dst, src), + 8 => fixed::<8>(dst, src), + 4 => fixed::<4>(dst, src), + 2 => fixed::<2>(dst, src), + _ => dst[..n].copy_from_slice(&src[..n]), + } +} + +#[inline(always)] +fn avg(a: u8, b: u8) -> u8 { + ((a as u16 + b as u16 + 1) >> 1) as u8 +} + +/// Horizontal 6-tap (unclipped) at every position of `row` for `n` outputs: out[c] uses row[c..c + 6]. +#[inline(always)] +fn htap_row(row: &[u8], out: &mut [i32], n: usize) { + let row = &row[..n + 5]; + for (o, w) in out[..n].iter_mut().zip(row.windows(6)) { + *o = tap6(w[0] as i32, w[1] as i32, w[2] as i32, w[3] as i32, w[4] as i32, w[5] as i32); + } +} + +/// Vertical 6-tap (unclipped) over six rows at columns 0..n. +#[inline(always)] +fn vtap_row(rows: [&[u8]; 6], out: &mut [i32], n: usize) { + let [a, b, c, d, e, f] = rows.map(|r| &r[..n]); + for i in 0..n { + out[i] = tap6(a[i] as i32, b[i] as i32, c[i] as i32, d[i] as i32, e[i] as i32, f[i] as i32); + } +} + +/// Luma sample interpolation for a bw x bh block (8.4.2.2.1). (x, y) is the integer sample position +/// (block position + (mv >> 2)); (fx, fy) the quarter-sample fraction. Output stride is `os`. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub fn mc_luma(p: PlaneRef, x: i32, y: i32, fx: u32, fy: u32, bw: usize, bh: usize, out: &mut [u8], os: usize) { + let mut win = [0u8; WIN * WIN]; + let (x0, y0) = (x - 2, y - 2); + let inside = x0 >= 0 && y0 >= 0 && x0 as usize + bw + 5 <= p.width && y0 as usize + bh + 5 <= p.height; + let (src, ss): (&[u8], usize) = if inside { + (&p.data[y0 as usize * p.stride + x0 as usize..], p.stride) + } else { + fetch_window(p, x, y, bw, bh, &mut win); + (&win[..], WIN) + }; + mc_luma_win(src, ss, fx, fy, bw, bh, out, os); +} + +/// Luma interpolation from a prepared window: `src[0]` is the sample at (x - 2, y - 2) and `ss` the +/// window stride; the window covers (bw + 5) x (bh + 5) samples. +#[allow(clippy::too_many_arguments)] +pub fn mc_luma_win(src: &[u8], ss: usize, fx: u32, fy: u32, bw: usize, bh: usize, out: &mut [u8], os: usize) { + // window row r (0..bh+5) starts at src[r * ss]; sample G of block (r, c) is at window (r + 2, c + 2) + let wrow = |r: usize| &src[r * ss..r * ss + bw + 5]; + let mut t = [0i32; 16]; + match (fx, fy) { + (0, 0) => { + for r in 0..bh { + copy_n(&mut out[r * os..], &wrow(r + 2)[2..], bw); + } + } + (_, 0) => { + for r in 0..bh { + let row = wrow(r + 2); + htap_row(row, &mut t, bw); + let o = &mut out[r * os..r * os + bw]; + match fx { + 1 => { + for c in 0..bw { + o[c] = avg(row[c + 2], clip1((t[c] + 16) >> 5)); + } + } + 2 => { + for c in 0..bw { + o[c] = clip1((t[c] + 16) >> 5); + } + } + _ => { + for c in 0..bw { + o[c] = avg(row[c + 3], clip1((t[c] + 16) >> 5)); + } + } + } + } + } + (0, _) => { + for r in 0..bh { + let rows = [wrow(r), wrow(r + 1), wrow(r + 2), wrow(r + 3), wrow(r + 4), wrow(r + 5)].map(|x| &x[2..]); + vtap_row(rows, &mut t, bw); + let g = if fy == 1 { rows[2] } else { rows[3] }; + let o = &mut out[r * os..r * os + bw]; + if fy == 2 { + for c in 0..bw { + o[c] = clip1((t[c] + 16) >> 5); + } + } else { + for c in 0..bw { + o[c] = avg(g[c], clip1((t[c] + 16) >> 5)); + } + } + } + } + (2, _) | (_, 2) => { + // unclipped horizontal half samples b1 for window rows 0..bh+5 + let mut b1 = [[0i32; 16]; WIN]; + for (r, bt) in b1.iter_mut().enumerate().take(bh + 5) { + htap_row(wrow(r), bt, bw); + } + let mut v = [0i32; 16]; + for r in 0..bh { + let o = &mut out[r * os..r * os + bw]; + let (b0, b1r, b2, b3, b4, b5) = (&b1[r], &b1[r + 1], &b1[r + 2], &b1[r + 3], &b1[r + 4], &b1[r + 5]); + for c in 0..bw { + t[c] = tap6(b0[c], b1r[c], b2[c], b3[c], b4[c], b5[c]); + } + match (fx, fy) { + (2, 2) => { + for c in 0..bw { + o[c] = clip1((t[c] + 512) >> 10); + } + } + (2, 1) | (2, 3) => { + let bh_ = if fy == 1 { b2 } else { b3 }; + for c in 0..bw { + o[c] = avg(clip1((bh_[c] + 16) >> 5), clip1((t[c] + 512) >> 10)); + } + } + _ => { + let off = if fx == 1 { 2 } else { 3 }; + let rows = [wrow(r), wrow(r + 1), wrow(r + 2), wrow(r + 3), wrow(r + 4), wrow(r + 5)].map(|x| &x[off..]); + vtap_row(rows, &mut v, bw); + for c in 0..bw { + o[c] = avg(clip1((v[c] + 16) >> 5), clip1((t[c] + 512) >> 10)); + } + } + } + } + } + _ => { + // e, g, p, r: average of a horizontal half sample (row r or r + 1) and a vertical one (col c or c + 1) + let hoff = if fy == 1 { 2 } else { 3 }; + let voff = if fx == 1 { 2 } else { 3 }; + let mut v = [0i32; 16]; + for r in 0..bh { + htap_row(wrow(r + hoff), &mut t, bw); + let rows = [wrow(r), wrow(r + 1), wrow(r + 2), wrow(r + 3), wrow(r + 4), wrow(r + 5)].map(|x| &x[voff..]); + vtap_row(rows, &mut v, bw); + let o = &mut out[r * os..r * os + bw]; + for c in 0..bw { + o[c] = avg(clip1((t[c] + 16) >> 5), clip1((v[c] + 16) >> 5)); + } + } + } + } +} + +/// Chroma sample interpolation (8.4.2.2.2) for a bw x bh block at integer position (x, y) with +/// eighth-sample fraction (fx, fy). +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub fn mc_chroma(p: PlaneRef, x: i32, y: i32, fx: u32, fy: u32, bw: usize, bh: usize, out: &mut [u8], os: usize) { + let maxx = p.width as i32 - 1; + let maxy = p.height as i32 - 1; + let inside = x >= 0 && y >= 0 && x + bw as i32 <= maxx && y + bh as i32 <= maxy; + // (bw+1) x (bh+1) source window; copied with edge clamping when it crosses the picture border + let mut win = [0u8; 9 * 9]; + let (src, ss): (&[u8], usize) = if inside { + (&p.data[y as usize * p.stride + x as usize..], p.stride) + } else { + for r in 0..=bh { + let yy = (y + r as i32).clamp(0, maxy) as usize; + for c in 0..=bw { + win[r * 9 + c] = p.data[yy * p.stride + (x + c as i32).clamp(0, maxx) as usize]; + } + } + (&win[..], 9) + }; + mc_chroma_win(src, ss, fx, fy, bw, bh, out, os); +} + +/// Chroma interpolation from a prepared (bw + 1) x (bh + 1) window whose first sample is the integer +/// position of the block. +#[allow(clippy::too_many_arguments)] +pub fn mc_chroma_win(src: &[u8], ss: usize, fx: u32, fy: u32, bw: usize, bh: usize, out: &mut [u8], os: usize) { + let (fx, fy) = (fx as u16, fy as u16); + let (w00, w10, w01, w11) = ((8 - fx) * (8 - fy), fx * (8 - fy), (8 - fx) * fy, fx * fy); + if fx == 0 && fy == 0 { + for r in 0..bh { + copy_n(&mut out[r * os..], &src[r * ss..], bw); + } + return; + } + for r in 0..bh { + let r0 = &src[r * ss..r * ss + bw + 1]; + let r1 = &src[(r + 1) * ss..(r + 1) * ss + bw + 1]; + let o = &mut out[r * os..r * os + bw]; + for (c, o) in o.iter_mut().enumerate() { + let v = w00 * r0[c] as u16 + w10 * r0[c + 1] as u16 + w01 * r1[c] as u16 + w11 * r1[c + 1] as u16; + *o = ((v + 32) >> 6) as u8; + } + } +} + +/// Weights for one prediction direction or pair. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Weight { + /// Default: copy (single list) or rounded average (bi). + Default, + /// Explicit/implicit weighting: log2 denominator, weights and offsets for L0 and L1. + Weighted { log_wd: i32, w0: i32, w1: i32, o0: i32, o1: i32 }, +} + +/// Combine prediction(s) into `dst` (stride `ds`). `p0`/`p1` have stride `ps`. +#[allow(clippy::too_many_arguments)] +pub fn weighted_store(dst: &mut [u8], ds: usize, p0: Option<&[u8]>, p1: Option<&[u8]>, ps: usize, bw: usize, bh: usize, w: Weight) { + match (p0, p1, w) { + (Some(a), Some(b), Weight::Default) => { + for r in 0..bh { + let d = &mut dst[r * ds..r * ds + bw]; + let ra = &a[r * ps..r * ps + bw]; + let rb = &b[r * ps..r * ps + bw]; + for ((d, &x), &y) in d.iter_mut().zip(ra).zip(rb) { + *d = ((x as u32 + y as u32 + 1) >> 1) as u8; + } + } + } + (Some(a), Some(b), Weight::Weighted { log_wd, w0, w1, o0, o1 }) => { + let round = 1 << log_wd; + let off = (o0 + o1 + 1) >> 1; + let sh = log_wd + 1; + for r in 0..bh { + let d = &mut dst[r * ds..r * ds + bw]; + let ra = &a[r * ps..r * ps + bw]; + let rb = &b[r * ps..r * ps + bw]; + for ((d, &x), &y) in d.iter_mut().zip(ra).zip(rb) { + *d = clip1(((x as i32 * w0 + y as i32 * w1 + round) >> sh) + off); + } + } + } + (Some(a), None, w) | (None, Some(a), w) => { + let (wt, o, log_wd) = match (w, p0.is_some()) { + (Weight::Default, _) => { + for r in 0..bh { + copy_n(&mut dst[r * ds..], &a[r * ps..], bw); + } + return; + } + (Weight::Weighted { w0, o0, log_wd, .. }, true) => (w0, o0, log_wd), + (Weight::Weighted { w1, o1, log_wd, .. }, false) => (w1, o1, log_wd), + }; + let round = if log_wd >= 1 { 1 << (log_wd - 1) } else { 0 }; + for r in 0..bh { + let d = &mut dst[r * ds..r * ds + bw]; + let ra = &a[r * ps..r * ps + bw]; + for (d, &x) in d.iter_mut().zip(ra) { + *d = clip1(((x as i32 * wt + round) >> log_wd) + o); + } + } + } + (None, None, _) => {} + } +} + +#[cfg(test)] +mod reference { + use super::*; + + /// Luma sample interpolation for a bw x bh block. (x, y) is the integer sample position + /// (block position + (mv >> 2)); (fx, fy) the quarter-sample fraction. Output stride is `os`. + #[allow(clippy::too_many_arguments)] + pub fn mc_luma_ref(p: PlaneRef, x: i32, y: i32, fx: u32, fy: u32, bw: usize, bh: usize, out: &mut [u8], os: usize) { + let mut win = [0u8; WIN * WIN]; + fetch_window(p, x, y, bw, bh, &mut win); + // sample G at (r, c) in block coordinates is win[(r + 2) * WIN + c + 2] + let g = |r: usize, c: usize| win[(r + 2) * WIN + c + 2] as i32; + // unclipped horizontal half-sample b1 at block row r (-2..bh+3 via offset) and column c + let b1 = |r: isize, c: usize| { + let base = ((r + 2) as usize) * WIN + c; + tap6(win[base] as i32, win[base + 1] as i32, win[base + 2] as i32, win[base + 3] as i32, win[base + 4] as i32, win[base + 5] as i32) + }; + let h1 = |r: usize, c: isize| { + let col = (c + 2) as usize; + tap6( + win[r * WIN + col] as i32, + win[(r + 1) * WIN + col] as i32, + win[(r + 2) * WIN + col] as i32, + win[(r + 3) * WIN + col] as i32, + win[(r + 4) * WIN + col] as i32, + win[(r + 5) * WIN + col] as i32, + ) + }; + match (fx, fy) { + (0, 0) => { + for r in 0..bh { + out[r * os..r * os + bw].copy_from_slice(&win[(r + 2) * WIN + 2..(r + 2) * WIN + 2 + bw]); + } + } + (_, 0) => { + // a, b, c + for r in 0..bh { + for c in 0..bw { + let b = clip1((b1(r as isize, c) + 16) >> 5) as i32; + out[r * os + c] = match fx { + 1 => ((g(r, c) + b + 1) >> 1) as u8, + 2 => b as u8, + _ => ((g(r, c + 1) + b + 1) >> 1) as u8, + }; + } + } + } + (0, _) => { + // d, h, n + for r in 0..bh { + for c in 0..bw { + let h = clip1((h1(r, c as isize) + 16) >> 5) as i32; + out[r * os + c] = match fy { + 1 => ((g(r, c) + h + 1) >> 1) as u8, + 2 => h as u8, + _ => ((g(r + 1, c) + h + 1) >> 1) as u8, + }; + } + } + } + (2, _) | (_, 2) => { + // j-based: j needs b1 for rows -2..bh+3 + let mut bcol = [0i32; WIN * WIN]; + for r in 0..bh + 5 { + for c in 0..bw { + bcol[r * WIN + c] = b1(r as isize - 2, c); + } + } + for r in 0..bh { + for c in 0..bw { + let j1 = tap6( + bcol[r * WIN + c], + bcol[(r + 1) * WIN + c], + bcol[(r + 2) * WIN + c], + bcol[(r + 3) * WIN + c], + bcol[(r + 4) * WIN + c], + bcol[(r + 5) * WIN + c], + ); + let j = clip1((j1 + 512) >> 10) as i32; + let v = match (fx, fy) { + (2, 2) => j, + (2, 1) => (clip1((bcol[(r + 2) * WIN + c] + 16) >> 5) as i32 + j + 1) >> 1, // f + (2, 3) => (clip1((bcol[(r + 3) * WIN + c] + 16) >> 5) as i32 + j + 1) >> 1, // q + (1, 2) => (clip1((h1(r, c as isize) + 16) >> 5) as i32 + j + 1) >> 1, // i + _ => (clip1((h1(r, c as isize + 1) + 16) >> 5) as i32 + j + 1) >> 1, // k + }; + out[r * os + c] = v as u8; + } + } + } + _ => { + // e, g, p, r: average of a horizontal half-sample (b or s) and a vertical one (h or m) + for r in 0..bh { + for c in 0..bw { + let hr = if fy == 1 { r as isize } else { r as isize + 1 }; + let vc = if fx == 1 { c as isize } else { c as isize + 1 }; + let bh_ = clip1((b1(hr, c) + 16) >> 5) as i32; + let vv = clip1((h1(r, vc) + 16) >> 5) as i32; + out[r * os + c] = ((bh_ + vv + 1) >> 1) as u8; + } + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn plane(w: usize, h: usize, f: impl Fn(usize, usize) -> u8) -> Vec { + (0..w * h).map(|i| f(i % w, i / w)).collect() + } + + #[test] + fn full_sample_copy_and_clamp() { + let d = plane(16, 16, |x, y| (x + 16 * y) as u8); + let p = PlaneRef { data: &d, width: 16, height: 16, stride: 16 }; + let mut out = [0u8; 16]; + mc_luma(p, 2, 3, 0, 0, 4, 4, &mut out, 4); + assert_eq!(&out[..4], &[50, 51, 52, 53]); + // clamped: far outside to the top-left gives sample (0,0) + mc_luma(p, -40, -40, 0, 0, 4, 4, &mut out, 4); + assert!(out.iter().all(|&v| v == 0)); + } + + #[test] + fn half_sample_constant_is_constant() { + let d = vec![100u8; 32 * 32]; + let p = PlaneRef { data: &d, width: 32, height: 32, stride: 32 }; + for fx in 0..4 { + for fy in 0..4 { + let mut out = [0u8; 64]; + mc_luma(p, 8, 8, fx, fy, 8, 8, &mut out, 8); + assert!(out.iter().all(|&v| v == 100), "{fx},{fy}"); + mc_chroma(p, 8, 8, fx * 2, fy * 2, 8, 8, &mut out, 8); + assert!(out.iter().all(|&v| v == 100)); + } + } + } + + #[test] + fn half_sample_horizontal_ramp() { + // linear ramp: 6-tap filter of a linear function is exact: b = (G + H) / 2 (rounded) + let d = plane(32, 32, |x, _| (x * 4) as u8); + let p = PlaneRef { data: &d, width: 32, height: 32, stride: 32 }; + let mut out = [0u8; 16]; + mc_luma(p, 8, 8, 2, 0, 4, 4, &mut out, 4); + assert_eq!(&out[..4], &[34, 38, 42, 46]); + mc_luma(p, 8, 8, 1, 0, 4, 4, &mut out, 4); + assert_eq!(&out[..4], &[33, 37, 41, 45]); + } + + #[test] + fn fast_luma_matches_reference() { + let mut seed = 7u32; + let d: Vec = (0..40 * 36) + .map(|_| { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + (seed >> 24) as u8 + }) + .collect(); + let p = PlaneRef { data: &d, width: 40, height: 36, stride: 40 }; + for &(bw, bh) in &[(16, 16), (16, 8), (8, 16), (8, 8), (8, 4), (4, 8), (4, 4)] { + for fx in 0..4 { + for fy in 0..4 { + for &(x, y) in &[(10, 9), (-3, -5), (30, 30), (0, 0), (2, 2), (38, 1)] { + let mut a = [0u8; 256]; + let mut b = [0u8; 256]; + mc_luma(p, x, y, fx, fy, bw, bh, &mut a, 16); + reference::mc_luma_ref(p, x, y, fx, fy, bw, bh, &mut b, 16); + assert_eq!(a, b, "{bw}x{bh} frac {fx},{fy} at {x},{y}"); + } + } + } + } + } + + #[test] + fn weighting() { + let a = [100u8; 4]; + let b = [50u8; 4]; + let mut d = [0u8; 4]; + weighted_store(&mut d, 4, Some(&a), Some(&b), 4, 4, 1, Weight::Default); + assert_eq!(d, [75; 4]); + weighted_store(&mut d, 4, Some(&a), None, 4, 4, 1, Weight::Weighted { log_wd: 5, w0: 16, w1: 0, o0: 3, o1: 0 }); + assert_eq!(d, [53; 4]); + } +} diff --git a/crates/video/src/h264/intra.rs b/crates/video/src/h264/intra.rs new file mode 100644 index 00000000..decf2965 --- /dev/null +++ b/crates/video/src/h264/intra.rs @@ -0,0 +1,462 @@ +//! Intra prediction (8.3): Intra_4x4, Intra_8x8 (with reference filtering), Intra_16x16 and chroma. + +/// Neighbouring sample availability for one block. +#[derive(Clone, Copy, Debug, Default)] +pub struct Avail { + pub left: bool, + pub top: bool, + pub top_left: bool, + pub top_right: bool, +} + +/// Neighbouring samples of an NxN block: p[x,-1] for x = 0..2N-1, p[-1,y] for y = 0..N-1, p[-1,-1]. +#[derive(Clone, Copy)] +struct Edge { + top: [i32; 16], + left: [i32; 8], + tl: i32, +} + +impl Edge { + /// p[x, -1] for x >= -1 + #[inline(always)] + fn t(&self, x: i32) -> i32 { + if x < 0 { self.tl } else { self.top[x as usize] } + } + /// p[-1, y] for y >= -1 + #[inline(always)] + fn l(&self, y: i32) -> i32 { + if y < 0 { self.tl } else { self.left[y as usize] } + } +} + +fn load_edge(plane: &[u8], stride: usize, x: usize, y: usize, n: usize, a: Avail) -> Edge { + let mut e = Edge { top: [128; 16], left: [128; 8], tl: 128 }; + if a.top { + let row = &plane[(y - 1) * stride + x..]; + for i in 0..n { + e.top[i] = row[i] as i32; + } + if a.top_right { + for i in n..2 * n { + e.top[i] = row[i] as i32; + } + } else { + let v = e.top[n - 1]; + for t in &mut e.top[n..2 * n] { + *t = v; + } + } + } + if a.left { + for i in 0..n { + e.left[i] = plane[(y + i) * stride + x - 1] as i32; + } + } + if a.top_left { + e.tl = plane[(y - 1) * stride + x - 1] as i32; + } + e +} + +/// Directional / DC prediction shared by Intra_4x4 and Intra_8x8 (8.3.1.2.x, 8.3.2.2.x). +fn pred_nxn(mode: u8, e: &Edge, n: i32, a: Avail, out: &mut [u8], stride: usize) { + let mut put = |x: i32, y: i32, v: i32| out[y as usize * stride + x as usize] = v as u8; + match mode { + 0 => { + for y in 0..n { + for x in 0..n { + put(x, y, e.t(x)); + } + } + } + 1 => { + for y in 0..n { + for x in 0..n { + put(x, y, e.l(y)); + } + } + } + 2 => { + let shift = if n == 4 { 2 } else { 3 }; + let st: i32 = (0..n).map(|x| e.t(x)).sum(); + let sl: i32 = (0..n).map(|y| e.l(y)).sum(); + let dc = match (a.top, a.left) { + (true, true) => (st + sl + n) >> (shift + 1), + (false, true) => (sl + (n >> 1)) >> shift, + (true, false) => (st + (n >> 1)) >> shift, + _ => 128, + }; + for y in 0..n { + for x in 0..n { + put(x, y, dc); + } + } + } + 3 => { + for y in 0..n { + for x in 0..n { + let v = if x == n - 1 && y == n - 1 { + (e.t(2 * n - 2) + 3 * e.t(2 * n - 1) + 2) >> 2 + } else { + (e.t(x + y) + 2 * e.t(x + y + 1) + e.t(x + y + 2) + 2) >> 2 + }; + put(x, y, v); + } + } + } + 4 => { + for y in 0..n { + for x in 0..n { + let v = if x > y { + (e.t(x - y - 2) + 2 * e.t(x - y - 1) + e.t(x - y) + 2) >> 2 + } else if x < y { + (e.l(y - x - 2) + 2 * e.l(y - x - 1) + e.l(y - x) + 2) >> 2 + } else { + (e.t(0) + 2 * e.tl + e.l(0) + 2) >> 2 + }; + put(x, y, v); + } + } + } + 5 => { + for y in 0..n { + for x in 0..n { + let z = 2 * x - y; + let v = if z >= 0 && z % 2 == 0 { + (e.t(x - (y >> 1) - 1) + e.t(x - (y >> 1)) + 1) >> 1 + } else if z > 0 { + (e.t(x - (y >> 1) - 2) + 2 * e.t(x - (y >> 1) - 1) + e.t(x - (y >> 1)) + 2) >> 2 + } else if z == -1 { + (e.l(0) + 2 * e.tl + e.t(0) + 2) >> 2 + } else { + (e.l(y - 2 * x - 1) + 2 * e.l(y - 2 * x - 2) + e.l(y - 2 * x - 3) + 2) >> 2 + }; + put(x, y, v); + } + } + } + 6 => { + for y in 0..n { + for x in 0..n { + let z = 2 * y - x; + let v = if z >= 0 && z % 2 == 0 { + (e.l(y - (x >> 1) - 1) + e.l(y - (x >> 1)) + 1) >> 1 + } else if z > 0 { + (e.l(y - (x >> 1) - 2) + 2 * e.l(y - (x >> 1) - 1) + e.l(y - (x >> 1)) + 2) >> 2 + } else if z == -1 { + (e.l(0) + 2 * e.tl + e.t(0) + 2) >> 2 + } else { + (e.t(x - 2 * y - 1) + 2 * e.t(x - 2 * y - 2) + e.t(x - 2 * y - 3) + 2) >> 2 + }; + put(x, y, v); + } + } + } + 7 => { + for y in 0..n { + for x in 0..n { + let i = x + (y >> 1); + let v = if y % 2 == 0 { (e.t(i) + e.t(i + 1) + 1) >> 1 } else { (e.t(i) + 2 * e.t(i + 1) + e.t(i + 2) + 2) >> 2 }; + put(x, y, v); + } + } + } + _ => { + let zmax = 2 * n - 3; + for y in 0..n { + for x in 0..n { + let z = x + 2 * y; + let i = y + (x >> 1); + let v = if z < zmax && z % 2 == 0 { + (e.l(i) + e.l(i + 1) + 1) >> 1 + } else if z < zmax { + (e.l(i) + 2 * e.l(i + 1) + e.l(i + 2) + 2) >> 2 + } else if z == zmax { + (e.l(n - 2) + 3 * e.l(n - 1) + 2) >> 2 + } else { + e.l(n - 1) + }; + put(x, y, v); + } + } + } + } +} + +/// Intra_4x4 prediction written into `plane` at (x, y). +pub fn pred4x4(plane: &mut [u8], stride: usize, x: usize, y: usize, mode: u8, a: Avail) { + let e = load_edge(plane, stride, x, y, 4, a); + pred_nxn(mode, &e, 4, a, &mut plane[y * stride + x..], stride); +} + +/// Intra_8x8 prediction (with reference sample filtering) written into `plane` at (x, y). +pub fn pred8x8(plane: &mut [u8], stride: usize, x: usize, y: usize, mode: u8, a: Avail) { + let p = load_edge(plane, stride, x, y, 8, a); + let mut f = p; + if a.top { + f.top[0] = if a.top_left { (p.tl + 2 * p.top[0] + p.top[1] + 2) >> 2 } else { (3 * p.top[0] + p.top[1] + 2) >> 2 }; + for i in 1..15 { + f.top[i] = (p.top[i - 1] + 2 * p.top[i] + p.top[i + 1] + 2) >> 2; + } + f.top[15] = (p.top[14] + 3 * p.top[15] + 2) >> 2; + } + if a.top_left { + f.tl = match (a.top, a.left) { + (true, true) => (p.top[0] + 2 * p.tl + p.left[0] + 2) >> 2, + (true, false) => (3 * p.tl + p.top[0] + 2) >> 2, + (false, true) => (3 * p.tl + p.left[0] + 2) >> 2, + _ => p.tl, + }; + } + if a.left { + f.left[0] = if a.top_left { (p.tl + 2 * p.left[0] + p.left[1] + 2) >> 2 } else { (3 * p.left[0] + p.left[1] + 2) >> 2 }; + for i in 1..7 { + f.left[i] = (p.left[i - 1] + 2 * p.left[i] + p.left[i + 1] + 2) >> 2; + } + f.left[7] = (p.left[6] + 3 * p.left[7] + 2) >> 2; + } + pred_nxn(mode, &f, 8, a, &mut plane[y * stride + x..], stride); +} + +#[inline(always)] +fn clip(v: i32) -> u8 { + v.clamp(0, 255) as u8 +} + +/// Intra_16x16 prediction (8.3.3) for the macroblock at luma (x, y). +pub fn pred16x16(plane: &mut [u8], stride: usize, x: usize, y: usize, mode: u8, a: Avail) { + let mut top = [0i32; 16]; + let mut left = [0i32; 16]; + if a.top { + for i in 0..16 { + top[i] = plane[(y - 1) * stride + x + i] as i32; + } + } + if a.left { + for i in 0..16 { + left[i] = plane[(y + i) * stride + x - 1] as i32; + } + } + let base = y * stride + x; + match mode { + 0 => { + for r in 0..16 { + for c in 0..16 { + plane[base + r * stride + c] = top[c] as u8; + } + } + } + 1 => { + for r in 0..16 { + plane[base + r * stride..base + r * stride + 16].fill(left[r] as u8); + } + } + 2 => { + let st: i32 = top.iter().sum(); + let sl: i32 = left.iter().sum(); + let dc = match (a.top, a.left) { + (true, true) => (st + sl + 16) >> 5, + (false, true) => (sl + 8) >> 4, + (true, false) => (st + 8) >> 4, + _ => 128, + } as u8; + for r in 0..16 { + plane[base + r * stride..base + r * stride + 16].fill(dc); + } + } + _ => { + let tl = if a.top_left { plane[(y - 1) * stride + x - 1] as i32 } else { 128 }; + let t = |i: i32| if i < 0 { tl } else { top[i as usize] }; + let l = |i: i32| if i < 0 { tl } else { left[i as usize] }; + let mut h = 0; + let mut v = 0; + for k in 0..8 { + h += (k + 1) * (t(8 + k) - t(6 - k)); + v += (k + 1) * (l(8 + k) - l(6 - k)); + } + let aa = 16 * (left[15] + top[15]); + let b = (5 * h + 32) >> 6; + let c = (5 * v + 32) >> 6; + for r in 0..16 { + for cc in 0..16 { + plane[base + r * stride + cc] = clip((aa + b * (cc as i32 - 7) + c * (r as i32 - 7) + 16) >> 5); + } + } + } + } +} + +/// Chroma intra prediction for a 4:2:0 macroblock (8x8 block at chroma (x, y)), 8.3.4. +/// Modes: 0 DC, 1 horizontal, 2 vertical, 3 plane. +pub fn pred_chroma(plane: &mut [u8], stride: usize, x: usize, y: usize, mode: u8, a: Avail) { + const W: usize = 8; + const H: usize = 8; + let mut top = [0i32; W]; + let mut left = [0i32; H]; + if a.top { + for i in 0..W { + top[i] = plane[(y - 1) * stride + x + i] as i32; + } + } + if a.left { + for i in 0..H { + left[i] = plane[(y + i) * stride + x - 1] as i32; + } + } + let base = y * stride + x; + match mode { + 0 => { + for by in 0..H / 4 { + for bx in 0..W / 4 { + let st: i32 = top[bx * 4..bx * 4 + 4].iter().sum(); + let sl: i32 = left[by * 4..by * 4 + 4].iter().sum(); + let dc = if (bx == 0 && by == 0) || (bx > 0 && by > 0) { + match (a.top, a.left) { + (true, true) => (st + sl + 4) >> 3, + (false, true) => (sl + 2) >> 2, + (true, false) => (st + 2) >> 2, + _ => 128, + } + } else if bx > 0 { + if a.top { + (st + 2) >> 2 + } else if a.left { + (sl + 2) >> 2 + } else { + 128 + } + } else if a.left { + (sl + 2) >> 2 + } else if a.top { + (st + 2) >> 2 + } else { + 128 + }; + for r in 0..4 { + let o = base + (by * 4 + r) * stride + bx * 4; + plane[o..o + 4].fill(dc as u8); + } + } + } + } + 1 => { + for r in 0..H { + plane[base + r * stride..base + r * stride + W].fill(left[r] as u8); + } + } + 2 => { + for r in 0..H { + for c in 0..W { + plane[base + r * stride + c] = top[c] as u8; + } + } + } + _ => { + let tl = if a.top_left { plane[(y - 1) * stride + x - 1] as i32 } else { 128 }; + let t = |i: i32| if i < 0 { tl } else { top[i as usize] }; + let l = |i: i32| if i < 0 { tl } else { left[i as usize] }; + let mut h = 0; + let mut v = 0; + for k in 0..4 { + h += (k + 1) * (t(4 + k) - t(2 - k)); + v += (k + 1) * (l(4 + k) - l(2 - k)); + } + let aa = 16 * (left[H - 1] + top[W - 1]); + let b = (34 * h + 32) >> 6; + let c = (34 * v + 32) >> 6; + for r in 0..H { + for cc in 0..W { + plane[base + r * stride + cc] = clip((aa + b * (cc as i32 - 3) + c * (r as i32 - 3) + 16) >> 5); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn setup() -> (Vec, usize) { + // 12x12 plane; block at (4,4); top row y=3 = 10,20,..; left column x=3 = 1,2,3,... + let stride = 16; + let mut p = vec![0u8; stride * 16]; + for i in 0..12 { + p[3 * stride + 4 + i] = (10 * (i + 1)) as u8; + } + for i in 0..8 { + p[(4 + i) * stride + 3] = (i + 1) as u8; + } + p[3 * stride + 3] = 5; + (p, stride) + } + const ALL: Avail = Avail { left: true, top: true, top_left: true, top_right: true }; + + #[test] + fn i4_vertical_horizontal_dc() { + let (mut p, s) = setup(); + pred4x4(&mut p, s, 4, 4, 0, ALL); + assert_eq!(&p[4 * s + 4..4 * s + 8], &[10, 20, 30, 40]); + assert_eq!(&p[7 * s + 4..7 * s + 8], &[10, 20, 30, 40]); + pred4x4(&mut p, s, 4, 4, 1, ALL); + assert_eq!(&p[5 * s + 4..5 * s + 8], &[2, 2, 2, 2]); + pred4x4(&mut p, s, 4, 4, 2, ALL); + // (10+20+30+40 + 1+2+3+4 + 4) >> 3 = 114 >> 3 = 14 + assert_eq!(p[4 * s + 4], 14); + } + + #[test] + fn i4_diag_down_left_hand_computed() { + let (mut p, s) = setup(); + pred4x4(&mut p, s, 4, 4, 3, ALL); + // pred[0,0] = (t0 + 2 t1 + t2 + 2) >> 2 = (10 + 40 + 30 + 2) >> 2 = 20 + assert_eq!(p[4 * s + 4], 20); + // pred[3,3] = (t6 + 3 t7 + 2) >> 2 = (70 + 240 + 2) >> 2 = 78 + assert_eq!(p[7 * s + 7], 78); + } + + #[test] + fn i4_diag_down_right_hand_computed() { + let (mut p, s) = setup(); + pred4x4(&mut p, s, 4, 4, 4, ALL); + // x==y: (t0 + 2 tl + l0 + 2) >> 2 = (10 + 10 + 1 + 2) >> 2 = 5 + assert_eq!(p[4 * s + 4], 5); + // x=1,y=0: (tl + 2 t0 + t1 + 2)>>2 = (5 + 20 + 20 + 2) >> 2 = 11 + assert_eq!(p[4 * s + 5], 11); + // x=0,y=1: (tl + 2 l0 + l1 + 2) >> 2 = (5 + 2 + 2 + 2) >> 2 = 2 + assert_eq!(p[5 * s + 4], 2); + } + + #[test] + fn i4_top_right_substitution() { + let (mut p, s) = setup(); + let a = Avail { top_right: false, ..ALL }; + pred4x4(&mut p, s, 4, 4, 3, a); + // top-right replaced by t3 = 40: pred[3,3] = (40 + 120 + 2) >> 2 = 40 + assert_eq!(p[7 * s + 7], 40); + } + + #[test] + fn i16_plane_flat_is_flat() { + let s = 32; + let mut p = vec![77u8; s * 32]; + pred16x16(&mut p, s, 8, 8, 3, ALL); + assert!(p[8 * s + 8..8 * s + 24].iter().all(|&v| v == 77)); + } + + #[test] + fn chroma_dc_rules() { + let s = 16; + let mut p = vec![0u8; s * 16]; + for i in 0..8 { + p[3 * s + 4 + i] = if i < 4 { 40 } else { 80 }; + p[(4 + i) * s + 3] = if i < 4 { 8 } else { 16 }; + } + pred_chroma(&mut p, s, 4, 4, 0, ALL); + assert_eq!(p[4 * s + 4], (160 + 32 + 4) >> 3); // top-left block: both + assert_eq!(p[4 * s + 8], 80); // top-right block: top only + assert_eq!(p[8 * s + 4], 16); // bottom-left: left only + assert_eq!(p[8 * s + 8], ((320 + 64 + 4) >> 3) as u8); // bottom-right: both + } +} diff --git a/crates/video/src/h264/mbtypes.rs b/crates/video/src/h264/mbtypes.rs new file mode 100644 index 00000000..aaaf96e9 --- /dev/null +++ b/crates/video/src/h264/mbtypes.rs @@ -0,0 +1,235 @@ +//! Macroblock / sub-macroblock type tables (7.4.5, Tables 7-11 .. 7-18). + +use crate::h264::picture::MbKind; + +pub const PRED_L0: u8 = 1; +pub const PRED_L1: u8 = 2; +pub const PRED_BI: u8 = 3; + +/// Macroblock partitioning for inter macroblocks. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum Part { + #[default] + P16x16, + P16x8, + P8x16, + P8x8, +} + +impl Part { + pub fn num_parts(self) -> usize { + match self { + Part::P16x16 => 1, + Part::P16x8 | Part::P8x16 => 2, + Part::P8x8 => 4, + } + } + /// (x, y, w, h) of partition `i` in 4x4-block units. + pub fn rect(self, i: usize) -> (usize, usize, usize, usize) { + match self { + Part::P16x16 => (0, 0, 4, 4), + Part::P16x8 => (0, 2 * i, 4, 2), + Part::P8x16 => (2 * i, 0, 2, 4), + Part::P8x8 => ((i & 1) * 2, (i >> 1) * 2, 2, 2), + } + } +} + +/// Sub-macroblock partitioning. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum SubPart { + #[default] + S8x8, + S8x4, + S4x8, + S4x4, +} + +impl SubPart { + pub fn num_parts(self) -> usize { + match self { + SubPart::S8x8 => 1, + SubPart::S8x4 | SubPart::S4x8 => 2, + SubPart::S4x4 => 4, + } + } + /// (x, y, w, h) of sub-partition `j` relative to the 8x8 block, in 4x4 units. + pub fn rect(self, j: usize) -> (usize, usize, usize, usize) { + match self { + SubPart::S8x8 => (0, 0, 2, 2), + SubPart::S8x4 => (0, j, 2, 1), + SubPart::S4x8 => (j, 0, 1, 2), + SubPart::S4x4 => (j & 1, j >> 1, 1, 1), + } + } +} + +/// Decoded meaning of mb_type. +#[derive(Clone, Copy, Debug, Default)] +pub struct MbTypeInfo { + pub kind: MbKind, + /// Intra16x16PredMode. + pub i16_mode: u8, + /// coded_block_pattern implied by an Intra_16x16 mb_type. + pub i16_cbp: u8, + pub part: Part, + /// Prediction list flags per macroblock partition. + pub pred: [u8; 2], + /// P_8x8ref0. + pub ref0: bool, +} + +/// Intra mb_type (I-slice numbering 0..=25). +pub fn intra_mb_type(t: u32) -> Option { + let mut m = MbTypeInfo::default(); + match t { + 0 => m.kind = MbKind::I4x4, // or I8x8, decided by transform_size_8x8_flag + 1..=24 => { + m.kind = MbKind::I16x16; + m.i16_mode = ((t - 1) % 4) as u8; + let chroma = ((t - 1) / 4) % 3; + let luma = if t >= 13 { 15 } else { 0 }; + m.i16_cbp = (luma | (chroma << 4)) as u8; + } + 25 => m.kind = MbKind::IPcm, + _ => return None, + } + Some(m) +} + +/// P-slice mb_type 0..=4 (5.. are intra, handled by the caller). +pub fn p_mb_type(t: u32) -> Option { + let mut m = MbTypeInfo { kind: MbKind::Inter, pred: [PRED_L0; 2], ..Default::default() }; + m.part = match t { + 0 => Part::P16x16, + 1 => Part::P16x8, + 2 => Part::P8x16, + 3 => Part::P8x8, + 4 => { + m.ref0 = true; + Part::P8x8 + } + _ => return None, + }; + Some(m) +} + +/// B-slice mb_type 0..=22 (23.. are intra). +pub fn b_mb_type(t: u32) -> Option { + let mut m = MbTypeInfo { kind: MbKind::Inter, ..Default::default() }; + match t { + 0 => { + m.kind = MbKind::BDirect16x16; + m.pred = [PRED_BI; 2]; + } + 1..=3 => { + m.part = Part::P16x16; + m.pred = [t as u8; 2]; + } + 4..=21 => { + const PAIRS: [(u8, u8); 9] = [ + (PRED_L0, PRED_L0), + (PRED_L1, PRED_L1), + (PRED_L0, PRED_L1), + (PRED_L1, PRED_L0), + (PRED_L0, PRED_BI), + (PRED_L1, PRED_BI), + (PRED_BI, PRED_L0), + (PRED_BI, PRED_L1), + (PRED_BI, PRED_BI), + ]; + let (a, b) = PAIRS[((t - 4) / 2) as usize]; + m.pred = [a, b]; + m.part = if t.is_multiple_of(2) { Part::P16x8 } else { Part::P8x16 }; + } + 22 => m.part = Part::P8x8, + _ => return None, + } + Some(m) +} + +/// Sub-macroblock type: (shape, prediction flags, direct). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SubMbInfo { + pub shape: SubPart, + pub pred: u8, + pub direct: bool, +} + +pub fn p_sub_mb_type(t: u32) -> Option { + let shape = match t { + 0 => SubPart::S8x8, + 1 => SubPart::S8x4, + 2 => SubPart::S4x8, + 3 => SubPart::S4x4, + _ => return None, + }; + Some(SubMbInfo { shape, pred: PRED_L0, direct: false }) +} + +pub fn b_sub_mb_type(t: u32) -> Option { + let (shape, pred, direct) = match t { + 0 => (SubPart::S8x8, PRED_BI, true), + 1 => (SubPart::S8x8, PRED_L0, false), + 2 => (SubPart::S8x8, PRED_L1, false), + 3 => (SubPart::S8x8, PRED_BI, false), + 4 => (SubPart::S8x4, PRED_L0, false), + 5 => (SubPart::S4x8, PRED_L0, false), + 6 => (SubPart::S8x4, PRED_L1, false), + 7 => (SubPart::S4x8, PRED_L1, false), + 8 => (SubPart::S8x4, PRED_BI, false), + 9 => (SubPart::S4x8, PRED_BI, false), + 10 => (SubPart::S4x4, PRED_L0, false), + 11 => (SubPart::S4x4, PRED_L1, false), + 12 => (SubPart::S4x4, PRED_BI, false), + _ => return None, + }; + Some(SubMbInfo { shape, pred, direct }) +} + +/// Table 9-4 (a): codeNum -> coded_block_pattern for (Intra_4x4/Intra_8x8, Inter), ChromaArrayType 1 or 2. +#[rustfmt::skip] +pub const CBP_ME: [(u8, u8); 48] = [ + (47, 0), (31, 16), (15, 1), (0, 2), (23, 4), (27, 8), (29, 32), (30, 3), + (7, 5), (11, 10), (13, 12), (14, 15), (39, 47), (43, 7), (45, 11), (46, 13), + (16, 14), (3, 6), (5, 9), (10, 31), (12, 35), (19, 37), (21, 42), (26, 44), + (28, 33), (35, 34), (37, 36), (42, 40), (44, 39), (1, 43), (2, 45), (4, 46), + (8, 17), (17, 18), (18, 20), (20, 24), (24, 19), (6, 21), (9, 26), (22, 28), + (25, 23), (32, 27), (33, 29), (34, 30), (36, 22), (40, 25), (38, 38), (41, 41), +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cbp_table_is_permutation() { + let mut a = [false; 48]; + let mut b = [false; 48]; + for &(i, p) in &CBP_ME { + a[i as usize] = true; + b[p as usize] = true; + } + assert!(a.iter().all(|&x| x) && b.iter().all(|&x| x)); + } + + #[test] + fn i16_types() { + let m = intra_mb_type(1).unwrap(); + assert_eq!((m.kind, m.i16_mode, m.i16_cbp), (MbKind::I16x16, 0, 0)); + let m = intra_mb_type(24).unwrap(); + assert_eq!((m.i16_mode, m.i16_cbp), (3, 15 | (2 << 4))); + let m = intra_mb_type(13).unwrap(); + assert_eq!((m.i16_mode, m.i16_cbp), (0, 15)); + } + + #[test] + fn b_types() { + let m = b_mb_type(12).unwrap(); + assert_eq!((m.part, m.pred), (Part::P16x8, [PRED_L0, PRED_BI])); + let m = b_mb_type(21).unwrap(); + assert_eq!((m.part, m.pred), (Part::P8x16, [PRED_BI, PRED_BI])); + let m = b_mb_type(11).unwrap(); + assert_eq!((m.part, m.pred), (Part::P8x16, [PRED_L1, PRED_L0])); + } +} diff --git a/crates/video/src/h264/mod.rs b/crates/video/src/h264/mod.rs new file mode 100644 index 00000000..315b2643 --- /dev/null +++ b/crates/video/src/h264/mod.rs @@ -0,0 +1,83 @@ +//! Clean-room pure-Rust H.264 / AVC decoder, implemented from ITU-T Rec. H.264 (ISO/IEC 14496-10). +//! +//! Supported: progressive (frame) coding, 8-bit 4:2:0, Baseline / Main / High profiles — CAVLC and CABAC, +//! I/P/B slices, 8x8 transform, custom scaling matrices, weighted prediction, spatial/temporal direct, +//! multiple slices, long-term references and all MMCOs, deblocking. +//! +//! ```no_run +//! let mut dec = soundcraft_video::h264::Decoder::new(); +//! let stream = std::fs::read("video.h264").unwrap(); +//! for pic in dec.decode(&stream, 0).unwrap() { +//! println!("{}x{} poc {}", pic.width, pic.height, pic.poc); +//! } +//! for pic in dec.flush() { +//! let _ = pic; +//! } +//! ``` + +// Index loops over fixed-size blocks read more clearly than iterator chains in codec code. +#![allow(clippy::needless_range_loop, clippy::too_many_arguments)] + +mod cabac; +mod cabac_mb; +#[rustfmt::skip] +mod cabac_tables; +mod cavlc; +#[rustfmt::skip] +mod cavlc_tables; +mod deblock; +mod decoder; +mod dpb; +mod error; +mod inter; +mod intra; +mod mbtypes; +pub mod params; +mod picture; +pub use picture::set_plane_allocator; +pub mod slice; +mod slicedec; +mod tables; +mod transform; + +pub use decoder::{DecodeStats, Decoder, default_threads}; +pub use error::{Error, Result}; + +/// Colour description from the VUI (ITU-T H.273 code points). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ColorInfo { + /// `video_full_range_flag`. + pub full_range: bool, + pub primaries: u8, + pub transfer: u8, + pub matrix: u8, +} + +/// A decoded, cropped picture in 8-bit planar 4:2:0. +#[derive(Clone, Debug)] +pub struct Picture { + /// Cropped luma width. + pub width: u32, + /// Cropped luma height. + pub height: u32, + pub chroma_width: u32, + pub chroma_height: u32, + pub y: Vec, + pub u: Vec, + pub v: Vec, + pub y_stride: usize, + pub uv_stride: usize, + /// Presentation timestamp passed to [`Decoder::decode`] with the access unit of this picture. + pub pts: i64, + /// Picture order count. + pub poc: i32, + /// IDR picture. + pub key: bool, + pub color: ColorInfo, + /// Sample aspect ratio (0, 0 when unspecified). + pub sar: (u16, u16), + /// Decoded in draft mode ([`Decoder::set_draft`]): a non-reference picture whose deblocking + /// filter was skipped. Its samples are approximate (not the conforming output); no other + /// picture is affected. + pub draft: bool, +} diff --git a/crates/video/src/h264/params.rs b/crates/video/src/h264/params.rs new file mode 100644 index 00000000..b99404a1 --- /dev/null +++ b/crates/video/src/h264/params.rs @@ -0,0 +1,719 @@ +//! Sequence and picture parameter sets (7.3.2.1, 7.3.2.2) including VUI and scaling lists. + +use crate::bitstream::BitReader; +use crate::h264::error::{Result, ensure, unsupported}; +use crate::h264::tables::*; + +/// Scaling matrices after applying fall-back rules, stored in raster order (weightScale). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScalingMatrices { + /// [Intra Y, Intra Cb, Intra Cr, Inter Y, Inter Cb, Inter Cr], raster order. + pub m4: [[u8; 16]; 6], + /// [Intra Y, Inter Y, Intra Cb, Inter Cb, Intra Cr, Inter Cr], raster order. + pub m8: [[u8; 64]; 6], +} + +impl ScalingMatrices { + pub fn flat() -> Self { + Self { m4: [[16; 16]; 6], m8: [[16; 64]; 6] } + } + pub fn is_flat(&self) -> bool { + self.m4.iter().all(|m| m.iter().all(|&v| v == 16)) && self.m8.iter().all(|m| m.iter().all(|&v| v == 16)) + } +} + +/// Lists as parsed, in zig-zag order, before inverse scanning. +#[derive(Clone, Debug)] +struct ZzLists { + l4: [[u8; 16]; 6], + l8: [[u8; 64]; 6], +} + +fn zz4_to_raster(l: &[u8; 16]) -> [u8; 16] { + let mut out = [0; 16]; + for (k, &v) in l.iter().enumerate() { + out[ZIGZAG4[k] as usize] = v; + } + out +} +fn zz8_to_raster(l: &[u8; 64]) -> [u8; 64] { + let mut out = [0; 64]; + for (k, &v) in l.iter().enumerate() { + out[ZIGZAG8[k] as usize] = v; + } + out +} + +impl ZzLists { + fn to_matrices(&self) -> ScalingMatrices { + let mut m = ScalingMatrices::flat(); + for i in 0..6 { + m.m4[i] = zz4_to_raster(&self.l4[i]); + m.m8[i] = zz8_to_raster(&self.l8[i]); + } + m + } + fn flat() -> Self { + Self { l4: [[16; 16]; 6], l8: [[16; 64]; 6] } + } +} + +/// Result of parsing one scaling_list(): Some(list) or None meaning "use default". +fn parse_scaling_list(r: &mut BitReader) -> Result> { + let mut list = [0u8; N]; + let mut last = 8i32; + let mut next = 8i32; + for (j, item) in list.iter_mut().enumerate() { + if next != 0 { + let delta = r.read_se()?; + ensure!((-128..=127).contains(&delta), "delta_scale out of range"); + next = (last + delta + 256) % 256; + if j == 0 && next == 0 { + return Ok(None); + } + } + *item = if next == 0 { last as u8 } else { next as u8 }; + last = *item as i32; + } + Ok(Some(list)) +} + +/// Parse scaling lists with a fall-back rule. `fallback` gives the lists used for rule B +/// (sequence-level) or None for rule A. +fn parse_scaling_lists(r: &mut BitReader, count: usize, fallback: Option<&ZzLists>) -> Result { + let mut out = ZzLists::flat(); + for i in 0..count.max(8) { + let present = if i < count { r.read_flag()? } else { false }; + if i < 6 { + let parsed = if present { Some(parse_scaling_list::<16>(r)?) } else { None }; + out.l4[i] = match parsed { + Some(Some(l)) => l, + Some(None) => { + if i < 3 { + DEFAULT_4X4_INTRA + } else { + DEFAULT_4X4_INTER + } + } + None => match (i, fallback) { + (0, None) => DEFAULT_4X4_INTRA, + (3, None) => DEFAULT_4X4_INTER, + (0 | 3, Some(f)) => f.l4[i], + _ => out.l4[i - 1], + }, + }; + } else { + let k = i - 6; + if k >= 6 { + break; + } + let parsed = if present { Some(parse_scaling_list::<64>(r)?) } else { None }; + out.l8[k] = match parsed { + Some(Some(l)) => l, + Some(None) => { + if k % 2 == 0 { + DEFAULT_8X8_INTRA + } else { + DEFAULT_8X8_INTER + } + } + None => match (k, fallback) { + (0, None) => DEFAULT_8X8_INTRA, + (1, None) => DEFAULT_8X8_INTER, + (0 | 1, Some(f)) => f.l8[k], + _ => out.l8[k - 2], + }, + }; + } + } + // 8x8 chroma lists that were never signalled (count == 8) follow rule A/B chains as well. + if count <= 8 { + for k in 2..6 { + out.l8[k] = out.l8[k - 2]; + } + } + Ok(out) +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HrdParameters { + pub cpb_cnt: u32, + pub bit_rate_scale: u8, + pub cpb_size_scale: u8, + pub bit_rate_value_minus1: Vec, + pub cpb_size_value_minus1: Vec, + pub cbr_flag: Vec, + pub initial_cpb_removal_delay_length: u8, + pub cpb_removal_delay_length: u8, + pub dpb_output_delay_length: u8, + pub time_offset_length: u8, +} + +fn parse_hrd(r: &mut BitReader) -> Result { + let cpb_cnt = r.read_ue()? + 1; + ensure!(cpb_cnt <= 32, "cpb_cnt_minus1 out of range"); + let mut h = HrdParameters { cpb_cnt, bit_rate_scale: r.read_bits(4)? as u8, cpb_size_scale: r.read_bits(4)? as u8, ..Default::default() }; + for _ in 0..cpb_cnt { + h.bit_rate_value_minus1.push(r.read_ue()?); + h.cpb_size_value_minus1.push(r.read_ue()?); + h.cbr_flag.push(r.read_flag()?); + } + h.initial_cpb_removal_delay_length = r.read_bits(5)? as u8 + 1; + h.cpb_removal_delay_length = r.read_bits(5)? as u8 + 1; + h.dpb_output_delay_length = r.read_bits(5)? as u8 + 1; + h.time_offset_length = r.read_bits(5)? as u8; + Ok(h) +} + +/// Video usability information (Annex E). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Vui { + pub aspect_ratio_idc: u8, + pub sar: (u16, u16), + pub overscan_appropriate: Option, + pub video_format: u8, + pub full_range: bool, + pub colour_primaries: u8, + pub transfer_characteristics: u8, + pub matrix_coefficients: u8, + pub chroma_loc: Option<(u32, u32)>, + pub timing: Option<(u32, u32, bool)>, + pub nal_hrd: Option, + pub vcl_hrd: Option, + pub low_delay_hrd: bool, + pub pic_struct_present: bool, + pub bitstream_restriction: Option, +} + +impl Default for Vui { + fn default() -> Self { + Self { + aspect_ratio_idc: 0, + sar: (0, 0), + overscan_appropriate: None, + video_format: 5, + full_range: false, + colour_primaries: 2, + transfer_characteristics: 2, + matrix_coefficients: 2, + chroma_loc: None, + timing: None, + nal_hrd: None, + vcl_hrd: None, + low_delay_hrd: false, + pic_struct_present: false, + bitstream_restriction: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BitstreamRestriction { + pub motion_vectors_over_pic_boundaries: bool, + pub max_bytes_per_pic_denom: u32, + pub max_bits_per_mb_denom: u32, + pub log2_max_mv_length_horizontal: u32, + pub log2_max_mv_length_vertical: u32, + pub max_num_reorder_frames: u32, + pub max_dec_frame_buffering: u32, +} + +/// Table E-1 sample aspect ratios. +const SAR_TABLE: [(u16, u16); 17] = [ + (0, 0), + (1, 1), + (12, 11), + (10, 11), + (16, 11), + (40, 33), + (24, 11), + (20, 11), + (32, 11), + (80, 33), + (18, 11), + (15, 11), + (64, 33), + (160, 99), + (4, 3), + (3, 2), + (2, 1), +]; + +fn parse_vui(r: &mut BitReader) -> Result { + let mut v = Vui::default(); + if r.read_flag()? { + v.aspect_ratio_idc = r.read_bits(8)? as u8; + if v.aspect_ratio_idc == 255 { + v.sar = (r.read_bits(16)? as u16, r.read_bits(16)? as u16); + } else if (v.aspect_ratio_idc as usize) < SAR_TABLE.len() { + v.sar = SAR_TABLE[v.aspect_ratio_idc as usize]; + } + } + if r.read_flag()? { + v.overscan_appropriate = Some(r.read_flag()?); + } + if r.read_flag()? { + v.video_format = r.read_bits(3)? as u8; + v.full_range = r.read_flag()?; + if r.read_flag()? { + v.colour_primaries = r.read_bits(8)? as u8; + v.transfer_characteristics = r.read_bits(8)? as u8; + v.matrix_coefficients = r.read_bits(8)? as u8; + } + } + if r.read_flag()? { + v.chroma_loc = Some((r.read_ue()?, r.read_ue()?)); + } + if r.read_flag()? { + let n = r.read_bits(32)?; + let t = r.read_bits(32)?; + let fixed = r.read_flag()?; + v.timing = Some((n, t, fixed)); + } + if r.read_flag()? { + v.nal_hrd = Some(parse_hrd(r)?); + } + if r.read_flag()? { + v.vcl_hrd = Some(parse_hrd(r)?); + } + if v.nal_hrd.is_some() || v.vcl_hrd.is_some() { + v.low_delay_hrd = r.read_flag()?; + } + v.pic_struct_present = r.read_flag()?; + // Some streams truncate the VUI after this point; treat EOF as "absent". + if r.bits_left() > 0 && r.read_flag()? { + let br = (|| -> Result { + Ok(BitstreamRestriction { + motion_vectors_over_pic_boundaries: r.read_flag()?, + max_bytes_per_pic_denom: r.read_ue()?, + max_bits_per_mb_denom: r.read_ue()?, + log2_max_mv_length_horizontal: r.read_ue()?, + log2_max_mv_length_vertical: r.read_ue()?, + max_num_reorder_frames: r.read_ue()?, + max_dec_frame_buffering: r.read_ue()?, + }) + })(); + v.bitstream_restriction = br.ok(); + } + Ok(v) +} + +/// Sequence parameter set. +#[derive(Clone, Debug)] +pub struct Sps { + pub profile_idc: u8, + pub constraint_flags: u8, + pub level_idc: u8, + pub id: u32, + pub chroma_format_idc: u32, + pub separate_colour_plane: bool, + pub bit_depth_luma: u32, + pub bit_depth_chroma: u32, + pub qpprime_y_zero_transform_bypass: bool, + pub seq_scaling_matrix_present: bool, + scaling_lists: ZzLists, + pub scaling: ScalingMatrices, + pub log2_max_frame_num: u32, + pub pic_order_cnt_type: u32, + pub log2_max_poc_lsb: u32, + pub delta_pic_order_always_zero: bool, + pub offset_for_non_ref_pic: i32, + pub offset_for_top_to_bottom_field: i32, + pub offset_for_ref_frame: Vec, + pub max_num_ref_frames: u32, + pub gaps_in_frame_num_allowed: bool, + pub pic_width_in_mbs: u32, + pub pic_height_in_map_units: u32, + pub frame_mbs_only: bool, + pub mb_adaptive_frame_field: bool, + pub direct_8x8_inference: bool, + /// (left, right, top, bottom) in crop units as coded. + pub frame_crop: Option<(u32, u32, u32, u32)>, + pub vui: Option, +} + +impl Sps { + pub fn parse(rbsp: &[u8]) -> Result { + let mut r = BitReader::new(rbsp); + let profile_idc = r.read_bits(8)? as u8; + let constraint_flags = r.read_bits(8)? as u8; + let level_idc = r.read_bits(8)? as u8; + let id = r.read_ue()?; + ensure!(id < 32, "seq_parameter_set_id {id} out of range"); + let mut chroma_format_idc = 1; + let mut separate_colour_plane = false; + let mut bit_depth_luma = 8; + let mut bit_depth_chroma = 8; + let mut bypass = false; + let mut seq_scaling_matrix_present = false; + let mut scaling_lists = ZzLists::flat(); + if matches!(profile_idc, 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135) { + chroma_format_idc = r.read_ue()?; + ensure!(chroma_format_idc <= 3, "chroma_format_idc out of range"); + if chroma_format_idc == 3 { + separate_colour_plane = r.read_flag()?; + } + bit_depth_luma = r.read_ue()? + 8; + bit_depth_chroma = r.read_ue()? + 8; + ensure!(bit_depth_luma <= 14 && bit_depth_chroma <= 14, "bit depth out of range"); + bypass = r.read_flag()?; + seq_scaling_matrix_present = r.read_flag()?; + if seq_scaling_matrix_present { + let count = if chroma_format_idc != 3 { 8 } else { 12 }; + scaling_lists = parse_scaling_lists(&mut r, count, None)?; + } + } + let log2_max_frame_num = r.read_ue()? + 4; + ensure!(log2_max_frame_num <= 16, "log2_max_frame_num out of range"); + let pic_order_cnt_type = r.read_ue()?; + ensure!(pic_order_cnt_type <= 2, "pic_order_cnt_type out of range"); + let mut log2_max_poc_lsb = 0; + let mut delta_pic_order_always_zero = false; + let mut offset_for_non_ref_pic = 0; + let mut offset_for_top_to_bottom_field = 0; + let mut offset_for_ref_frame = Vec::new(); + if pic_order_cnt_type == 0 { + log2_max_poc_lsb = r.read_ue()? + 4; + ensure!(log2_max_poc_lsb <= 16, "log2_max_pic_order_cnt_lsb out of range"); + } else if pic_order_cnt_type == 1 { + delta_pic_order_always_zero = r.read_flag()?; + offset_for_non_ref_pic = r.read_se()?; + offset_for_top_to_bottom_field = r.read_se()?; + let n = r.read_ue()?; + ensure!(n <= 255, "num_ref_frames_in_pic_order_cnt_cycle out of range"); + for _ in 0..n { + offset_for_ref_frame.push(r.read_se()?); + } + } + let max_num_ref_frames = r.read_ue()?; + ensure!(max_num_ref_frames <= 16, "max_num_ref_frames out of range"); + let gaps_in_frame_num_allowed = r.read_flag()?; + let pic_width_in_mbs = r.read_ue()? + 1; + let pic_height_in_map_units = r.read_ue()? + 1; + ensure!(pic_width_in_mbs <= 1024 && pic_height_in_map_units <= 1024, "picture too large"); + let frame_mbs_only = r.read_flag()?; + let mb_adaptive_frame_field = if !frame_mbs_only { r.read_flag()? } else { false }; + let direct_8x8_inference = r.read_flag()?; + let frame_crop = if r.read_flag()? { Some((r.read_ue()?, r.read_ue()?, r.read_ue()?, r.read_ue()?)) } else { None }; + let vui = if r.read_flag()? { Some(parse_vui(&mut r)?) } else { None }; + let scaling = if seq_scaling_matrix_present { scaling_lists.to_matrices() } else { ScalingMatrices::flat() }; + let sps = Sps { + profile_idc, + constraint_flags, + level_idc, + id, + chroma_format_idc, + separate_colour_plane, + bit_depth_luma, + bit_depth_chroma, + qpprime_y_zero_transform_bypass: bypass, + seq_scaling_matrix_present, + scaling_lists, + scaling, + log2_max_frame_num, + pic_order_cnt_type, + log2_max_poc_lsb, + delta_pic_order_always_zero, + offset_for_non_ref_pic, + offset_for_top_to_bottom_field, + offset_for_ref_frame, + max_num_ref_frames, + gaps_in_frame_num_allowed, + pic_width_in_mbs, + pic_height_in_map_units, + frame_mbs_only, + mb_adaptive_frame_field, + direct_8x8_inference, + frame_crop, + vui, + }; + if let Some((l, rr, t, b)) = sps.frame_crop { + let (cx, cy) = sps.crop_unit(); + ensure!((l + rr) * cx < sps.width() && (t + b) * cy < sps.height(), "frame cropping exceeds picture size"); + } + Ok(sps) + } + + /// ChromaArrayType. + pub fn chroma_array_type(&self) -> u32 { + if self.separate_colour_plane { 0 } else { self.chroma_format_idc } + } + pub fn frame_height_in_mbs(&self) -> u32 { + (2 - self.frame_mbs_only as u32) * self.pic_height_in_map_units + } + /// Decoded (uncropped) luma width. + pub fn width(&self) -> u32 { + self.pic_width_in_mbs * 16 + } + pub fn height(&self) -> u32 { + self.frame_height_in_mbs() * 16 + } + pub fn max_frame_num(&self) -> u32 { + 1 << self.log2_max_frame_num + } + /// (CropUnitX, CropUnitY). + pub fn crop_unit(&self) -> (u32, u32) { + let (sub_w, sub_h) = match self.chroma_array_type() { + 1 => (2, 2), + 2 => (2, 1), + _ => (1, 1), + }; + if self.chroma_array_type() == 0 { (1, 2 - self.frame_mbs_only as u32) } else { (sub_w, sub_h * (2 - self.frame_mbs_only as u32)) } + } + /// Cropping rectangle in luma samples: (x, y, width, height). + pub fn crop_rect(&self) -> (u32, u32, u32, u32) { + match self.frame_crop { + None => (0, 0, self.width(), self.height()), + Some((l, r, t, b)) => { + let (cx, cy) = self.crop_unit(); + (l * cx, t * cy, self.width() - (l + r) * cx, self.height() - (t + b) * cy) + } + } + } + + /// MaxDpbFrames from Table A-1 (and the VUI max_dec_frame_buffering when present). + pub fn max_dpb_frames(&self) -> usize { + let max_dpb_mbs: u32 = match self.level_idc { + 9 => 396, + 10 => 396, + 11 => { + if self.constraint_flags & 0x10 != 0 && !matches!(self.profile_idc, 100 | 110 | 122 | 244 | 44) { + 396 + } else { + 900 + } + } + 12 | 13 | 20 => 2376, + 21 => 4752, + 22 | 30 => 8100, + 31 => 18000, + 32 => 20480, + 40 | 41 => 32768, + 42 => 34816, + 50 => 110400, + 51 | 52 => 184320, + 60..=62 => 696320, + _ => 184320, + }; + let frame_mbs = self.pic_width_in_mbs * self.frame_height_in_mbs(); + let mut n = (max_dpb_mbs / frame_mbs.max(1)).clamp(1, 16) as usize; + if let Some(br) = self.vui.as_ref().and_then(|v| v.bitstream_restriction.as_ref()) { + n = (br.max_dec_frame_buffering as usize).min(16); + } + n.max(self.max_num_ref_frames as usize).max(1) + } + + /// Number of frames that may precede a frame in decoding order and follow it in output order. + pub fn max_num_reorder_frames(&self) -> usize { + if let Some(br) = self.vui.as_ref().and_then(|v| v.bitstream_restriction.as_ref()) { + return br.max_num_reorder_frames as usize; + } + // Profiles that cannot reorder (no B slices) with constraint_set3 (intra) etc. + if matches!(self.profile_idc, 44 | 86 | 100 | 110 | 122 | 244) && self.constraint_flags & 0x10 != 0 { + return 0; + } + self.max_dpb_frames() + } + + pub fn check_supported(&self) -> Result<()> { + if !self.frame_mbs_only { + return unsupported("interlaced (field / MBAFF) coding"); + } + if self.chroma_array_type() != 1 { + return unsupported(format!("chroma_format_idc {} (only 4:2:0 is implemented)", self.chroma_format_idc)); + } + if self.bit_depth_luma != 8 || self.bit_depth_chroma != 8 { + return unsupported(format!("bit depth luma {} chroma {} (only 8-bit is implemented)", self.bit_depth_luma, self.bit_depth_chroma)); + } + if self.qpprime_y_zero_transform_bypass { + return unsupported("qpprime_y_zero_transform_bypass (lossless)"); + } + Ok(()) + } +} + +/// Picture parameter set. +#[derive(Clone, Debug)] +pub struct Pps { + pub id: u32, + pub sps_id: u32, + pub entropy_coding_mode: bool, + pub bottom_field_pic_order_in_frame_present: bool, + pub num_slice_groups: u32, + pub slice_group_map_type: u32, + pub num_ref_idx_l0_default_active: u32, + pub num_ref_idx_l1_default_active: u32, + pub weighted_pred: bool, + pub weighted_bipred_idc: u32, + pub pic_init_qp: i32, + pub pic_init_qs: i32, + pub chroma_qp_index_offset: i32, + pub deblocking_filter_control_present: bool, + pub constrained_intra_pred: bool, + pub redundant_pic_cnt_present: bool, + pub transform_8x8_mode: bool, + pub pic_scaling_matrix_present: bool, + pub second_chroma_qp_index_offset: i32, + /// Effective scaling matrices (after PPS/SPS fall-back), raster order. + pub scaling: ScalingMatrices, +} + +impl Pps { + /// Parse a PPS. Needs the SPS table because the tail depends on chroma_format_idc. + pub fn parse(rbsp: &[u8], spss: &[Option]) -> Result { + let mut r = BitReader::new(rbsp); + let id = r.read_ue()?; + ensure!(id < 256, "pic_parameter_set_id out of range"); + let sps_id = r.read_ue()?; + ensure!(sps_id < 32, "seq_parameter_set_id out of range"); + let sps = spss + .get(sps_id as usize) + .and_then(|s| s.as_ref()) + .ok_or_else(|| crate::h264::Error::MissingParameterSet(format!("SPS {sps_id} for PPS {id}")))?; + let entropy_coding_mode = r.read_flag()?; + let bottom_field_pic_order_in_frame_present = r.read_flag()?; + let num_slice_groups = r.read_ue()? + 1; + ensure!(num_slice_groups <= 8, "num_slice_groups out of range"); + let mut slice_group_map_type = 0; + if num_slice_groups > 1 { + // Parsed for completeness; FMO decoding is unsupported. + slice_group_map_type = r.read_ue()?; + match slice_group_map_type { + 0 => { + for _ in 0..num_slice_groups { + r.read_ue()?; + } + } + 2 => { + for _ in 0..num_slice_groups - 1 { + r.read_ue()?; + r.read_ue()?; + } + } + 3..=5 => { + r.read_flag()?; + r.read_ue()?; + } + 6 => { + let n = r.read_ue()? + 1; + let bits = 32 - (num_slice_groups - 1).leading_zeros(); + for _ in 0..n { + r.read_bits(bits)?; + } + } + _ => {} + } + } + let num_ref_idx_l0_default_active = r.read_ue()? + 1; + let num_ref_idx_l1_default_active = r.read_ue()? + 1; + ensure!(num_ref_idx_l0_default_active <= 32 && num_ref_idx_l1_default_active <= 32, "num_ref_idx_default_active out of range"); + let weighted_pred = r.read_flag()?; + let weighted_bipred_idc = r.read_bits(2)?; + let pic_init_qp = 26 + r.read_se()?; + let pic_init_qs = 26 + r.read_se()?; + let chroma_qp_index_offset = r.read_se()?; + ensure!((-12..=12).contains(&chroma_qp_index_offset), "chroma_qp_index_offset out of range"); + let deblocking_filter_control_present = r.read_flag()?; + let constrained_intra_pred = r.read_flag()?; + let redundant_pic_cnt_present = r.read_flag()?; + let mut transform_8x8_mode = false; + let mut pic_scaling_matrix_present = false; + let mut second_chroma_qp_index_offset = chroma_qp_index_offset; + let mut scaling = sps.scaling.clone(); + if r.more_rbsp_data() { + transform_8x8_mode = r.read_flag()?; + pic_scaling_matrix_present = r.read_flag()?; + if pic_scaling_matrix_present { + let count = 6 + if sps.chroma_format_idc != 3 { 2 } else { 6 } * transform_8x8_mode as usize; + // Fall-back rule B uses the sequence-level lists (or Flat when the SPS has none, but then + // rule A applies per 7.4.2.2). + let lists = if sps.seq_scaling_matrix_present { + parse_scaling_lists(&mut r, count, Some(&sps.scaling_lists))? + } else { + parse_scaling_lists(&mut r, count, None)? + }; + scaling = lists.to_matrices(); + } + second_chroma_qp_index_offset = r.read_se()?; + ensure!((-12..=12).contains(&second_chroma_qp_index_offset), "second_chroma_qp_index_offset out of range"); + } + Ok(Pps { + id, + sps_id, + entropy_coding_mode, + bottom_field_pic_order_in_frame_present, + num_slice_groups, + slice_group_map_type, + num_ref_idx_l0_default_active, + num_ref_idx_l1_default_active, + weighted_pred, + weighted_bipred_idc, + pic_init_qp, + pic_init_qs, + chroma_qp_index_offset, + deblocking_filter_control_present, + constrained_intra_pred, + redundant_pic_cnt_present, + transform_8x8_mode, + pic_scaling_matrix_present, + second_chroma_qp_index_offset, + scaling, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bitstream::BitWriter; + + #[test] + fn scaling_list_default_signal() { + // delta_scale = -8 on the first entry => nextScale 0 => use default. + let mut w = BitWriter::new(); + w.write_se(-8); + w.rbsp_trailing(); + let b = w.finish(); + let mut r = BitReader::new(&b); + assert_eq!(parse_scaling_list::<16>(&mut r).unwrap(), None); + } + + #[test] + fn scaling_list_repeat_last() { + // 8 -> +2 = 10, then +0 = 10, then -10 => 0 => repeat 10 for the rest. + let mut w = BitWriter::new(); + w.write_se(2); + w.write_se(0); + w.write_se(-10); + w.rbsp_trailing(); + let b = w.finish(); + let mut r = BitReader::new(&b); + let l = parse_scaling_list::<16>(&mut r).unwrap().unwrap(); + assert_eq!(l, [10; 16]); + } + + #[test] + fn parse_minimal_baseline_sps() { + // profile 66, level 30, sps_id 0, log2_max_frame_num_minus4 0, poc type 2, 1 ref, 11x9 MBs. + let mut w = BitWriter::new(); + w.write_bits(66, 8); + w.write_bits(0xC0, 8); + w.write_bits(30, 8); + w.write_ue(0); + w.write_ue(0); + w.write_ue(2); + w.write_ue(1); + w.write_bit(false); + w.write_ue(10); + w.write_ue(8); + w.write_bit(true); // frame_mbs_only + w.write_bit(true); // direct_8x8 + w.write_bit(false); // cropping + w.write_bit(false); // vui + w.rbsp_trailing(); + let sps = Sps::parse(&w.finish()).unwrap(); + assert_eq!((sps.width(), sps.height()), (176, 144)); + assert_eq!(sps.pic_order_cnt_type, 2); + assert_eq!(sps.crop_rect(), (0, 0, 176, 144)); + } +} diff --git a/crates/video/src/h264/picture.rs b/crates/video/src/h264/picture.rs new file mode 100644 index 00000000..5e52f074 --- /dev/null +++ b/crates/video/src/h264/picture.rs @@ -0,0 +1,361 @@ +//! Picture buffers and per-macroblock state. + +use std::sync::{Arc, OnceLock}; + +/// Planar 4:2:0 8-bit sample buffers (MB-aligned dimensions, stride == width). +#[derive(Clone)] +pub struct Planes { + pub y: Vec, + pub cb: Vec, + pub cr: Vec, + pub width: usize, + pub height: usize, + pub cwidth: usize, +} + +impl Planes { + pub fn new(width: usize, height: usize) -> Self { + let (cw, ch) = (width / 2, height / 2); + Planes { y: vec![0; width * height], cb: vec![128; cw * ch], cr: vec![128; cw * ch], width, height, cwidth: cw } + } + pub fn gray(width: usize, height: usize) -> Self { + let mut p = Self::new(width, height); + p.y.fill(128); + p + } +} + +/// Macroblock coding category. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum MbKind { + #[default] + None, + I4x4, + I8x8, + I16x16, + IPcm, + PSkip, + BSkip, + BDirect16x16, + /// Any other P or B inter macroblock. + Inter, +} + +impl MbKind { + #[inline] + pub fn is_intra(self) -> bool { + matches!(self, MbKind::I4x4 | MbKind::I8x8 | MbKind::I16x16 | MbKind::IPcm) + } +} + +/// Per-macroblock state kept for the whole picture (neighbour derivations, deblocking, co-located data). +#[derive(Clone, Copy)] +pub struct MbState { + /// Slice number within the picture; `u32::MAX` = not (yet) decoded. + pub slice_num: u32, + pub kind: MbKind, + pub transform_8x8: bool, + /// coded_block_pattern: bits 0..3 luma, bits 4..5 chroma. + pub cbp: u8, + pub qp: u8, + /// QPc for Cb / Cr (used by deblocking). + pub qpc: [u8; 2], + pub intra_chroma_mode: u8, + /// Intra4x4PredMode / Intra8x8PredMode per 4x4 block (raster order). + pub intra_modes: [u8; 16], + /// CAVLC: TotalCoeff per luma 4x4 block (raster). CABAC: coded_block_flag. + pub nnz: [u8; 16], + /// Same for chroma AC blocks [Cb, Cr][raster 2x2]. + pub nnz_c: [[u8; 4]; 2], + /// CABAC coded_block_flag of DC blocks: bit0 luma (I16x16), bit1 Cb, bit2 Cr. + pub cbf_dc: u8, + /// Luma 4x4 blocks (raster bit) with non-zero coefficients, for deblocking bS = 2. + pub nz_mask: u16, + pub ref_idx: [[i8; 4]; 2], + /// Motion vectors per 4x4 block (raster order). + pub mv: [[[i16; 2]; 16]; 2], + /// |mvd| per 4x4 block (raster), saturated, for CABAC context selection. + pub mvd: [[[u8; 2]; 16]; 2], + /// Bit per 8x8 block predicted in direct mode. + pub direct8x8: u8, +} + +impl MbState { + #[inline] + pub fn kind_is_skip(&self) -> bool { + matches!(self.kind, MbKind::PSkip | MbKind::BSkip) + } +} + +impl Default for MbState { + fn default() -> Self { + MbState { + slice_num: u32::MAX, + kind: MbKind::None, + transform_8x8: false, + cbp: 0, + qp: 0, + qpc: [0; 2], + intra_chroma_mode: 0, + intra_modes: [2; 16], + nnz: [0; 16], + nnz_c: [[0; 4]; 2], + cbf_dc: 0, + nz_mask: 0, + ref_idx: [[-1; 4]; 2], + mv: [[[0; 2]; 16]; 2], + mvd: [[[0; 2]; 16]; 2], + direct8x8: 0, + } + } +} + +/// One published macroblock row of a decoded frame: final (deblocked) samples plus the motion data +/// needed as co-located information for direct prediction. +pub struct FrameRow { + /// 16 luma lines. + pub y: Box<[u8]>, + /// 8 lines per chroma component. + pub cb: Box<[u8]>, + pub cr: Box<[u8]>, + /// Per 4x4 block: index mb_x * 16 + raster. + pub mv: [Box<[[i16; 2]]>; 2], + /// Per 8x8 block: index mb_x * 4 + b8. + pub ref_idx: [Box<[i8]>; 2], + /// Unique id of the referenced picture per 8x8 block (u32::MAX = none). + pub ref_id: [Box<[u32]>; 2], + pub intra: Box<[bool]>, +} + +impl FrameRow { + /// A mid-grey intra row with no motion, `mb_w` macroblocks wide. + #[cfg(not(feature = "threads"))] + fn grey(mb_w: usize) -> FrameRow { + FrameRow { + y: vec![128u8; mb_w * 16 * 16].into(), + cb: vec![128u8; mb_w * 8 * 8].into(), + cr: vec![128u8; mb_w * 8 * 8].into(), + mv: [vec![[0i16; 2]; mb_w * 16].into(), vec![[0i16; 2]; mb_w * 16].into()], + ref_idx: [vec![-1i8; mb_w * 4].into(), vec![-1i8; mb_w * 4].into()], + ref_id: [vec![u32::MAX; mb_w * 4].into(), vec![u32::MAX; mb_w * 4].into()], + intra: vec![true; mb_w].into(), + } + } +} + +/// A decoded (or in-progress) frame as stored in the DPB and referenced by later pictures. +/// +/// Rows are published in order by the thread decoding the picture; readers block (per row) until the +/// data they need is available, which lets several pictures decode concurrently. +pub struct Frame { + pub id: u32, + pub poc: i32, + pub width: usize, + pub height: usize, + pub cwidth: usize, + pub cheight: usize, + pub mb_w: usize, + rows: Box<[OnceLock]>, +} + +impl Frame { + pub fn new(id: u32, poc: i32, mb_w: usize, mb_h: usize) -> Self { + Frame { + id, + poc, + width: mb_w * 16, + height: mb_h * 16, + cwidth: mb_w * 8, + cheight: mb_h * 8, + mb_w, + rows: (0..mb_h).map(|_| OnceLock::new()).collect(), + } + } + + pub fn mb_h(&self) -> usize { + self.rows.len() + } + + /// Publish MB row `r` (ignored if already published). + pub fn publish(&self, r: usize, row: FrameRow) { + let _ = self.rows[r].set(row); + } + + pub fn is_published(&self, r: usize) -> bool { + self.rows[r].get().is_some() + } + + /// MB row `r`, waiting until it has been published. + #[inline] + pub fn row(&self, r: usize) -> &FrameRow { + #[cfg(feature = "threads")] + { + self.rows[r].wait() + } + #[cfg(not(feature = "threads"))] + { + // Rows are decoded before use on the calling thread; a missing one (never expected) + // reads as mid-grey intra rather than aborting. (SoundCraft change: was `expect`.) + self.rows[r].get_or_init(|| FrameRow::grey(self.mb_w)) + } + } + + /// Wait until the whole frame is available. + pub fn wait_complete(&self) { + for r in 0..self.rows.len() { + self.row(r); + } + } + + /// Build a published row from a contiguous plane set (rows of MB row `r`) and motion data. + pub fn make_row(planes: &Planes, r: usize, mbs: &[MbState], ref_ids: &dyn Fn(&MbState, usize, i8) -> u32) -> FrameRow { + let w = planes.width; + let cw = planes.cwidth; + let mb_w = w / 16; + let y = planes.y[r * 16 * w..(r + 1) * 16 * w].into(); + let cb = planes.cb[r * 8 * cw..(r + 1) * 8 * cw].into(); + let cr = planes.cr[r * 8 * cw..(r + 1) * 8 * cw].into(); + let row_mbs = &mbs[r * mb_w..(r + 1) * mb_w]; + let mut mv = [vec![[0i16; 2]; mb_w * 16], vec![[0i16; 2]; mb_w * 16]]; + let mut ref_idx = [vec![-1i8; mb_w * 4], vec![-1i8; mb_w * 4]]; + let mut ref_id = [vec![u32::MAX; mb_w * 4], vec![u32::MAX; mb_w * 4]]; + let mut intra = vec![true; mb_w]; + for (x, st) in row_mbs.iter().enumerate() { + if st.slice_num == u32::MAX { + continue; + } + intra[x] = st.kind.is_intra(); + for l in 0..2 { + mv[l][x * 16..x * 16 + 16].copy_from_slice(&st.mv[l]); + for b in 0..4 { + let ri = st.ref_idx[l][b]; + ref_idx[l][x * 4 + b] = ri; + if ri >= 0 { + ref_id[l][x * 4 + b] = ref_ids(st, l, ri); + } + } + } + } + let [mv0, mv1] = mv; + let [ri0, ri1] = ref_idx; + let [id0, id1] = ref_id; + FrameRow { y, cb, cr, mv: [mv0.into(), mv1.into()], ref_idx: [ri0.into(), ri1.into()], ref_id: [id0.into(), id1.into()], intra: intra.into() } + } + + /// A frame with every row published from `planes` and no motion (intra), e.g. "non-existing" + /// frames inferred for frame_num gaps. + pub fn from_planes(id: u32, poc: i32, planes: &Planes) -> Self { + let (mb_w, mb_h) = (planes.width / 16, planes.height / 16); + let f = Frame::new(id, poc, mb_w, mb_h); + let blank = vec![MbState::default(); mb_w * mb_h]; + for r in 0..mb_h { + f.publish(r, Frame::make_row(planes, r, &blank, &|_, _, _| u32::MAX)); + } + f + } + + /// Copy the w x h luma window at (x0, y0) (clamped to the picture) into `out` (stride `os`). + #[inline] + pub fn luma_window(&self, x0: i32, y0: i32, w: usize, h: usize, out: &mut [u8], os: usize) { + let (width, height) = (self.width, self.height); + for r in 0..h { + let yy = (y0 + r as i32).clamp(0, height as i32 - 1) as usize; + let row = self.row(yy >> 4); + let line = &row.y[(yy & 15) * width..(yy & 15) * width + width]; + copy_line::<24>(line, x0, &mut out[r * os..], w); + } + } + + /// Copy the w x h window of chroma component `c` at (x0, y0) (clamped) into `out`. + #[inline] + pub fn chroma_window(&self, c: usize, x0: i32, y0: i32, w: usize, h: usize, out: &mut [u8], os: usize) { + let (cw, ch) = (self.cwidth, self.cheight); + for r in 0..h { + let yy = (y0 + r as i32).clamp(0, ch as i32 - 1) as usize; + let row = self.row(yy >> 3); + let plane = if c == 0 { &row.cb } else { &row.cr }; + let line = &plane[(yy & 7) * cw..(yy & 7) * cw + cw]; + copy_line::<16>(line, x0, &mut out[r * os..], w); + } + } + + /// Crop and copy the frame into planar buffers (waits for completion). + pub fn copy_cropped(&self, crop: (usize, usize, usize, usize)) -> (Vec, Vec, Vec) { + let (cx, cy, cw, ch) = crop; + let alloc = plane_allocator(); + let mut y = alloc(cw * ch); + for r in cy..cy + ch { + let row = self.row(r >> 4); + y.extend_from_slice(&row.y[(r & 15) * self.width + cx..(r & 15) * self.width + cx + cw]); + } + let (ccx, ccy, ccw, cch) = (cx / 2, cy / 2, cw.div_ceil(2), ch.div_ceil(2)); + let mut u = alloc(ccw * cch); + let mut v = alloc(ccw * cch); + for r in ccy..ccy + cch { + let row = self.row(r >> 3); + let o = (r & 7) * self.cwidth + ccx; + u.extend_from_slice(&row.cb[o..o + ccw]); + v.extend_from_slice(&row.cr[o..o + ccw]); + } + (y, u, v) + } +} + +static PLANE_ALLOCATOR: std::sync::OnceLock Vec> = std::sync::OnceLock::new(); + +/// Where output planes come from: an empty buffer with room for the given number of samples. +/// The host sets it once to recycle the planes of pictures it is done with; the first call wins. +pub fn set_plane_allocator(alloc: fn(usize) -> Vec) { + let _ = PLANE_ALLOCATOR.set(alloc); +} + +fn plane_allocator() -> fn(usize) -> Vec { + PLANE_ALLOCATOR.get().copied().unwrap_or(Vec::with_capacity) +} + +/// Copy `w` samples of `line` starting at x0 into `out`; when possible a fixed-size block of `N` +/// bytes is copied instead (cheaper than a variable-length copy; `out` must then have room for `N`). +#[inline(always)] +fn copy_line(line: &[u8], x0: i32, out: &mut [u8], w: usize) { + if w <= N + && x0 >= 0 + && let Some(src) = line.get(x0 as usize..).and_then(|l| l.first_chunk::()) + && let Some(dst) = out.first_chunk_mut::() + { + *dst = *src; + } else { + copy_clamped(line, x0, &mut out[..w]); + } +} + +/// Copy `out.len()` samples of `line` starting at x0, replicating edge samples outside the line. +#[inline(always)] +fn copy_clamped(line: &[u8], x0: i32, out: &mut [u8]) { + let w = out.len(); + if x0 >= 0 && x0 as usize + w <= line.len() { + out.copy_from_slice(&line[x0 as usize..x0 as usize + w]); + } else { + let max = line.len() as i32 - 1; + for (c, o) in out.iter_mut().enumerate() { + *o = line[(x0 + c as i32).clamp(0, max) as usize]; + } + } +} + +pub type FrameRef = Arc; + +/// An entry of RefPicList0/1 for the current slice. +#[derive(Clone)] +pub struct RefPic { + pub frame: FrameRef, + pub long_term: bool, +} + +impl RefPic { + pub fn poc(&self) -> i32 { + self.frame.poc + } + pub fn id(&self) -> u32 { + self.frame.id + } +} diff --git a/crates/video/src/h264/slice.rs b/crates/video/src/h264/slice.rs new file mode 100644 index 00000000..bd641d5a --- /dev/null +++ b/crates/video/src/h264/slice.rs @@ -0,0 +1,515 @@ +//! NAL unit header and slice header parsing (7.3.1, 7.3.3). + +use crate::bitstream::BitReader; +use crate::h264::error::{Result, ensure}; +use crate::h264::params::{Pps, Sps}; + +/// NAL unit types used by the decoder. +pub mod nal_type { + pub const SLICE: u8 = 1; + pub const SLICE_DPA: u8 = 2; + pub const SLICE_DPB: u8 = 3; + pub const SLICE_DPC: u8 = 4; + pub const IDR: u8 = 5; + pub const SEI: u8 = 6; + pub const SPS: u8 = 7; + pub const PPS: u8 = 8; + pub const AUD: u8 = 9; + pub const END_SEQ: u8 = 10; + pub const END_STREAM: u8 = 11; + pub const FILLER: u8 = 12; + pub const SPS_EXT: u8 = 13; + pub const PREFIX: u8 = 14; + pub const SUBSET_SPS: u8 = 15; + pub const AUX_SLICE: u8 = 19; + pub const SLICE_EXT: u8 = 20; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NalHeader { + pub nal_ref_idc: u8, + pub nal_unit_type: u8, +} + +impl NalHeader { + pub fn parse(b: u8) -> Result { + ensure!(b & 0x80 == 0, "forbidden_zero_bit set"); + Ok(Self { nal_ref_idc: (b >> 5) & 3, nal_unit_type: b & 0x1f }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SliceType { + P, + B, + I, + Sp, + Si, +} + +impl SliceType { + fn from_raw(v: u32) -> Result { + Ok(match v % 5 { + 0 => SliceType::P, + 1 => SliceType::B, + 2 => SliceType::I, + 3 => SliceType::Sp, + _ => SliceType::Si, + }) + } + pub fn is_intra(self) -> bool { + matches!(self, SliceType::I | SliceType::Si) + } + pub fn is_b(self) -> bool { + self == SliceType::B + } + pub fn is_p(self) -> bool { + matches!(self, SliceType::P | SliceType::Sp) + } +} + +/// One entry of ref_pic_list_modification(). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RefPicListMod { + pub idc: u32, + /// abs_diff_pic_num_minus1 or long_term_pic_num. + pub value: u32, +} + +/// memory_management_control_operation entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Mmco { + pub op: u32, + pub difference_of_pic_nums_minus1: u32, + pub long_term_pic_num: u32, + pub long_term_frame_idx: u32, + pub max_long_term_frame_idx_plus1: u32, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct WeightEntry { + pub luma_weight: i32, + pub luma_offset: i32, + pub luma_flag: bool, + pub chroma_weight: [i32; 2], + pub chroma_offset: [i32; 2], + pub chroma_flag: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PredWeightTable { + pub luma_log2_denom: u32, + pub chroma_log2_denom: u32, + pub l0: Vec, + pub l1: Vec, +} + +#[derive(Clone, Debug)] +pub struct SliceHeader { + pub nal_ref_idc: u8, + pub nal_unit_type: u8, + pub idr: bool, + pub first_mb_in_slice: u32, + pub slice_type_raw: u32, + pub slice_type: SliceType, + pub pps_id: u32, + pub colour_plane_id: u32, + pub frame_num: u32, + pub field_pic: bool, + pub bottom_field: bool, + pub idr_pic_id: u32, + pub pic_order_cnt_lsb: u32, + pub delta_pic_order_cnt_bottom: i32, + pub delta_pic_order_cnt: [i32; 2], + pub redundant_pic_cnt: u32, + pub direct_spatial_mv_pred: bool, + pub num_ref_idx_active: [u32; 2], + pub ref_pic_list_mod: [Vec; 2], + pub pred_weight_table: Option, + pub no_output_of_prior_pics: bool, + pub long_term_reference: bool, + pub adaptive_ref_pic_marking: bool, + pub mmcos: Vec, + pub cabac_init_idc: u32, + pub slice_qp_delta: i32, + pub sp_for_switch: bool, + pub slice_qs_delta: i32, + pub disable_deblocking_filter_idc: u32, + pub slice_alpha_c0_offset_div2: i32, + pub slice_beta_offset_div2: i32, + pub slice_group_change_cycle: u32, + /// Bit offset of slice_data() within the RBSP. + pub header_bits: usize, +} + +impl SliceHeader { + pub fn qp(&self, pps: &Pps) -> i32 { + pps.pic_init_qp + self.slice_qp_delta + } + pub fn has_mmco5(&self) -> bool { + self.mmcos.iter().any(|m| m.op == 5) + } + + /// Parse the slice header. `pps_lookup` resolves pic_parameter_set_id to (PPS, SPS). + pub fn parse<'a>( + rbsp: &[u8], + nal: NalHeader, + pps_lookup: impl FnOnce(u32) -> Result<(&'a Pps, &'a Sps)>, + ) -> Result<(SliceHeader, &'a Pps, &'a Sps)> { + let mut r = BitReader::new(rbsp); + let idr = nal.nal_unit_type == nal_type::IDR; + let first_mb_in_slice = r.read_ue()?; + let slice_type_raw = r.read_ue()?; + ensure!(slice_type_raw <= 9, "slice_type out of range"); + let slice_type = SliceType::from_raw(slice_type_raw)?; + let pps_id = r.read_ue()?; + ensure!(pps_id < 256, "pic_parameter_set_id out of range"); + let (pps, sps) = pps_lookup(pps_id)?; + let mut colour_plane_id = 0; + if sps.separate_colour_plane { + colour_plane_id = r.read_bits(2)?; + } + let frame_num = r.read_bits(sps.log2_max_frame_num)?; + let mut field_pic = false; + let mut bottom_field = false; + if !sps.frame_mbs_only { + field_pic = r.read_flag()?; + if field_pic { + bottom_field = r.read_flag()?; + } + } + let idr_pic_id = if idr { r.read_ue()? } else { 0 }; + let mut pic_order_cnt_lsb = 0; + let mut delta_pic_order_cnt_bottom = 0; + let mut delta_pic_order_cnt = [0; 2]; + if sps.pic_order_cnt_type == 0 { + pic_order_cnt_lsb = r.read_bits(sps.log2_max_poc_lsb)?; + if pps.bottom_field_pic_order_in_frame_present && !field_pic { + delta_pic_order_cnt_bottom = r.read_se()?; + } + } + if sps.pic_order_cnt_type == 1 && !sps.delta_pic_order_always_zero { + delta_pic_order_cnt[0] = r.read_se()?; + if pps.bottom_field_pic_order_in_frame_present && !field_pic { + delta_pic_order_cnt[1] = r.read_se()?; + } + } + let redundant_pic_cnt = if pps.redundant_pic_cnt_present { r.read_ue()? } else { 0 }; + let direct_spatial_mv_pred = if slice_type.is_b() { r.read_flag()? } else { false }; + let mut num_ref_idx_active = [pps.num_ref_idx_l0_default_active, pps.num_ref_idx_l1_default_active]; + if !slice_type.is_intra() { + if r.read_flag()? { + num_ref_idx_active[0] = r.read_ue()? + 1; + if slice_type.is_b() { + num_ref_idx_active[1] = r.read_ue()? + 1; + } + } + let max = if field_pic { 32 } else { 16 }; + ensure!(num_ref_idx_active[0] <= max && num_ref_idx_active[1] <= max, "num_ref_idx_active out of range"); + } + if !slice_type.is_b() { + num_ref_idx_active[1] = 0; + } + if slice_type.is_intra() { + num_ref_idx_active[0] = 0; + } + ensure!(nal.nal_unit_type != 20 && nal.nal_unit_type != 21, "MVC slices are not supported"); + let mut ref_pic_list_mod = [Vec::new(), Vec::new()]; + for (list, mods) in ref_pic_list_mod.iter_mut().enumerate() { + let present = if list == 0 { !slice_type.is_intra() } else { slice_type.is_b() }; + if present && r.read_flag()? { + loop { + let idc = r.read_ue()?; + ensure!(idc <= 5, "modification_of_pic_nums_idc out of range"); + if idc == 3 { + break; + } + let value = r.read_ue()?; + mods.push(RefPicListMod { idc, value }); + ensure!(mods.len() <= 66, "too many ref_pic_list_modification entries"); + } + } + } + let mut pred_weight_table = None; + if (pps.weighted_pred && slice_type.is_p()) || (pps.weighted_bipred_idc == 1 && slice_type.is_b()) { + let luma_log2_denom = r.read_ue()?; + ensure!(luma_log2_denom <= 7, "luma_log2_weight_denom out of range"); + let mut chroma_log2_denom = 0; + if sps.chroma_array_type() != 0 { + chroma_log2_denom = r.read_ue()?; + ensure!(chroma_log2_denom <= 7, "chroma_log2_weight_denom out of range"); + } + let mut lists = [Vec::new(), Vec::new()]; + let nlists = if slice_type.is_b() { 2 } else { 1 }; + for (l, entries) in lists.iter_mut().enumerate().take(nlists) { + for _ in 0..num_ref_idx_active[l] { + let mut e = WeightEntry { luma_weight: 1 << luma_log2_denom, chroma_weight: [1 << chroma_log2_denom; 2], ..Default::default() }; + if r.read_flag()? { + e.luma_flag = true; + e.luma_weight = r.read_se()?; + e.luma_offset = r.read_se()?; + ensure!((-128..=127).contains(&e.luma_weight) && (-128..=127).contains(&e.luma_offset), "luma weight out of range"); + } + if sps.chroma_array_type() != 0 && r.read_flag()? { + e.chroma_flag = true; + for j in 0..2 { + e.chroma_weight[j] = r.read_se()?; + e.chroma_offset[j] = r.read_se()?; + ensure!( + (-128..=127).contains(&e.chroma_weight[j]) && (-128..=127).contains(&e.chroma_offset[j]), + "chroma weight out of range" + ); + } + } + entries.push(e); + } + } + let [l0, l1] = lists; + pred_weight_table = Some(PredWeightTable { luma_log2_denom, chroma_log2_denom, l0, l1 }); + } + let mut no_output_of_prior_pics = false; + let mut long_term_reference = false; + let mut adaptive_ref_pic_marking = false; + let mut mmcos = Vec::new(); + if nal.nal_ref_idc != 0 { + if idr { + no_output_of_prior_pics = r.read_flag()?; + long_term_reference = r.read_flag()?; + } else { + adaptive_ref_pic_marking = r.read_flag()?; + if adaptive_ref_pic_marking { + loop { + let op = r.read_ue()?; + ensure!(op <= 6, "memory_management_control_operation out of range"); + if op == 0 { + break; + } + let mut m = Mmco { + op, + difference_of_pic_nums_minus1: 0, + long_term_pic_num: 0, + long_term_frame_idx: 0, + max_long_term_frame_idx_plus1: 0, + }; + if op == 1 || op == 3 { + m.difference_of_pic_nums_minus1 = r.read_ue()?; + } + if op == 2 { + m.long_term_pic_num = r.read_ue()?; + } + if op == 3 || op == 6 { + m.long_term_frame_idx = r.read_ue()?; + } + if op == 4 { + m.max_long_term_frame_idx_plus1 = r.read_ue()?; + } + mmcos.push(m); + ensure!(mmcos.len() <= 66, "too many MMCOs"); + } + } + } + } + let mut cabac_init_idc = 0; + if pps.entropy_coding_mode && !slice_type.is_intra() { + cabac_init_idc = r.read_ue()?; + ensure!(cabac_init_idc <= 2, "cabac_init_idc out of range"); + } + let slice_qp_delta = r.read_se()?; + let qp = pps.pic_init_qp + slice_qp_delta; + ensure!((0..=51).contains(&qp), "slice QP {qp} out of range"); + let mut sp_for_switch = false; + let mut slice_qs_delta = 0; + if matches!(slice_type, SliceType::Sp | SliceType::Si) { + if slice_type == SliceType::Sp { + sp_for_switch = r.read_flag()?; + } + slice_qs_delta = r.read_se()?; + } + let mut disable_deblocking_filter_idc = 0; + let mut slice_alpha_c0_offset_div2 = 0; + let mut slice_beta_offset_div2 = 0; + if pps.deblocking_filter_control_present { + disable_deblocking_filter_idc = r.read_ue()?; + ensure!(disable_deblocking_filter_idc <= 2, "disable_deblocking_filter_idc out of range"); + if disable_deblocking_filter_idc != 1 { + slice_alpha_c0_offset_div2 = r.read_se()?; + slice_beta_offset_div2 = r.read_se()?; + ensure!( + (-6..=6).contains(&slice_alpha_c0_offset_div2) && (-6..=6).contains(&slice_beta_offset_div2), + "deblocking offsets out of range" + ); + } + } + let mut slice_group_change_cycle = 0; + if pps.num_slice_groups > 1 && (3..=5).contains(&pps.slice_group_map_type) { + // Ceil(Log2(PicSizeInMapUnits / SliceGroupChangeRate + 1)); rate is not retained, so read + // conservatively using the picture size (FMO is unsupported for decoding anyway). + let pic_size = sps.pic_width_in_mbs * sps.pic_height_in_map_units; + let bits = 32 - pic_size.leading_zeros(); + slice_group_change_cycle = r.read_bits(bits)?; + } + let header_bits = r.position(); + Ok(( + SliceHeader { + nal_ref_idc: nal.nal_ref_idc, + nal_unit_type: nal.nal_unit_type, + idr, + first_mb_in_slice, + slice_type_raw, + slice_type, + pps_id, + colour_plane_id, + frame_num, + field_pic, + bottom_field, + idr_pic_id, + pic_order_cnt_lsb, + delta_pic_order_cnt_bottom, + delta_pic_order_cnt, + redundant_pic_cnt, + direct_spatial_mv_pred, + num_ref_idx_active, + ref_pic_list_mod, + pred_weight_table, + no_output_of_prior_pics, + long_term_reference, + adaptive_ref_pic_marking, + mmcos, + cabac_init_idc, + slice_qp_delta, + sp_for_switch, + slice_qs_delta, + disable_deblocking_filter_idc, + slice_alpha_c0_offset_div2, + slice_beta_offset_div2, + slice_group_change_cycle, + header_bits, + }, + pps, + sps, + )) + } +} + +/// POC decoding state carried between pictures (8.2.1). +#[derive(Clone, Debug, Default)] +pub struct PocState { + /// PicOrderCntMsb / pic_order_cnt_lsb of the previous reference picture (type 0). + prev_ref_msb: i32, + prev_ref_lsb: i32, + /// FrameNumOffset / frame_num of the previous picture (types 1 and 2). + prev_frame_num_offset: i32, + prev_frame_num: u32, +} + +/// Output of POC computation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Poc { + pub top: i32, + pub bottom: i32, + msb: i32, + frame_num_offset: i32, +} + +impl Poc { + pub fn frame(&self) -> i32 { + self.top.min(self.bottom) + } +} + +impl PocState { + /// Compute TopFieldOrderCnt/BottomFieldOrderCnt of a frame (8.2.1.1-8.2.1.3). + pub fn compute(&self, sh: &SliceHeader, sps: &Sps) -> Poc { + match sps.pic_order_cnt_type { + 0 => { + let (prev_msb, prev_lsb) = if sh.idr { (0, 0) } else { (self.prev_ref_msb, self.prev_ref_lsb) }; + let max_lsb = 1i32 << sps.log2_max_poc_lsb; + let lsb = sh.pic_order_cnt_lsb as i32; + let msb = if lsb < prev_lsb && prev_lsb - lsb >= max_lsb / 2 { + prev_msb + max_lsb + } else if lsb > prev_lsb && lsb - prev_lsb > max_lsb / 2 { + prev_msb - max_lsb + } else { + prev_msb + }; + let top = msb + lsb; + let bottom = top + sh.delta_pic_order_cnt_bottom; + Poc { top, bottom, msb, frame_num_offset: 0 } + } + 1 => { + let frame_num_offset = self.frame_num_offset(sh.idr, sh.frame_num, sps); + let n = sps.offset_for_ref_frame.len() as i32; + let mut abs_frame_num = if n != 0 { frame_num_offset + sh.frame_num as i32 } else { 0 }; + if sh.nal_ref_idc == 0 && abs_frame_num > 0 { + abs_frame_num -= 1; + } + let mut expected = 0i32; + if abs_frame_num > 0 { + let cycle_cnt = (abs_frame_num - 1) / n; + let in_cycle = (abs_frame_num - 1) % n; + let delta_cycle: i32 = sps.offset_for_ref_frame.iter().sum(); + expected = cycle_cnt.wrapping_mul(delta_cycle); + for &o in &sps.offset_for_ref_frame[..=in_cycle as usize] { + expected += o; + } + } + if sh.nal_ref_idc == 0 { + expected += sps.offset_for_non_ref_pic; + } + let top = expected + sh.delta_pic_order_cnt[0]; + let bottom = top + sps.offset_for_top_to_bottom_field + sh.delta_pic_order_cnt[1]; + Poc { top, bottom, msb: 0, frame_num_offset } + } + _ => { + let frame_num_offset = self.frame_num_offset(sh.idr, sh.frame_num, sps); + let temp = if sh.idr { + 0 + } else if sh.nal_ref_idc == 0 { + 2 * (frame_num_offset + sh.frame_num as i32) - 1 + } else { + 2 * (frame_num_offset + sh.frame_num as i32) + }; + Poc { top: temp, bottom: temp, msb: 0, frame_num_offset } + } + } + } + + fn frame_num_offset(&self, idr: bool, frame_num: u32, sps: &Sps) -> i32 { + if idr { + 0 + } else if self.prev_frame_num > frame_num { + self.prev_frame_num_offset + sps.max_frame_num() as i32 + } else { + self.prev_frame_num_offset + } + } + + /// Update after a picture has been decoded. `poc` is the value returned by [`PocState::compute`] + /// (before any MMCO5 adjustment). + pub fn update(&mut self, sh: &SliceHeader, poc: &Poc) { + let mmco5 = sh.has_mmco5(); + if sh.nal_ref_idc != 0 { + if mmco5 { + self.prev_ref_msb = 0; + self.prev_ref_lsb = poc.top - poc.frame(); + } else { + self.prev_ref_msb = poc.msb; + self.prev_ref_lsb = sh.pic_order_cnt_lsb as i32; + } + } + if mmco5 { + self.prev_frame_num_offset = 0; + self.prev_frame_num = 0; + } else { + self.prev_frame_num_offset = poc.frame_num_offset; + self.prev_frame_num = sh.frame_num; + } + } + + /// Account for a "non-existing" frame inferred by the frame_num gap process (8.2.5.2). + pub fn update_gap_frame(&mut self, frame_num: u32, sps: &Sps) { + let off = self.frame_num_offset(false, frame_num, sps); + self.prev_frame_num_offset = off; + self.prev_frame_num = frame_num; + } +} diff --git a/crates/video/src/h264/slicedec.rs b/crates/video/src/h264/slicedec.rs new file mode 100644 index 00000000..a71a6b4e --- /dev/null +++ b/crates/video/src/h264/slicedec.rs @@ -0,0 +1,1469 @@ +//! Slice data decoding: macroblock parsing (CAVLC), motion vector derivation, and reconstruction. + +use crate::bitstream::BitReader; +use crate::h264::cavlc; +use crate::h264::deblock; +use crate::h264::error::{Result, ensure, invalid}; +use crate::h264::inter::{self, Weight}; +use crate::h264::intra::{self, Avail}; +use crate::h264::mbtypes::*; +use crate::h264::params::{Pps, Sps}; +use crate::h264::picture::*; +use crate::h264::slice::{SliceHeader, SliceType}; +use crate::h264::tables::*; +use crate::h264::transform::{self, LevelScale}; + +/// Per-slice data kept with the picture (deblocking, motion field export). +#[derive(Clone, Debug, Default)] +pub struct SliceInfo { + pub disable_deblocking_filter_idc: u32, + pub alpha_offset: i32, + pub beta_offset: i32, + /// Unique ids of the pictures in RefPicList0/1. + pub ref_ids: [Vec; 2], +} + +/// The picture under construction: private reconstruction buffers plus the shared [`Frame`] that +/// finished (deblocked) macroblock rows are published into. +pub struct PicState { + pub planes: Planes, + pub mbs: Vec, + pub slices: Vec, + pub mb_w: usize, + pub mb_h: usize, + pub poc: i32, + pub decoded_mbs: usize, + pub frame: FrameRef, + /// Decoded macroblocks per MB row. + row_count: Vec, + /// Leading MB rows that are fully reconstructed / deblocked / published. + rows_recon: usize, + rows_deblocked: usize, + rows_published: usize, + /// Draft mode for a non-reference picture: rows are published without deblocking. + pub skip_deblock: bool, +} + +impl PicState { + pub fn new(frame: FrameRef) -> Self { + let (mb_w, mb_h) = (frame.mb_w, frame.mb_h()); + PicState { + planes: Planes::new(mb_w * 16, mb_h * 16), + mbs: vec![MbState::default(); mb_w * mb_h], + slices: Vec::new(), + mb_w, + mb_h, + poc: frame.poc, + decoded_mbs: 0, + frame, + row_count: vec![0; mb_h], + rows_recon: 0, + rows_deblocked: 0, + rows_published: 0, + skip_deblock: false, + } + } + + /// Reuse the buffers of a finished picture of the same size for a new frame. + pub fn reset(&mut self, frame: FrameRef) { + debug_assert_eq!((self.mb_w, self.mb_h), (frame.mb_w, frame.mb_h())); + // start_mb() re-initialises each macroblock before use; only availability must be reset + for m in &mut self.mbs { + m.slice_num = u32::MAX; + } + self.slices.clear(); + self.poc = frame.poc; + self.decoded_mbs = 0; + self.frame = frame; + self.row_count.fill(0); + self.rows_recon = 0; + self.rows_deblocked = 0; + self.rows_published = 0; + self.skip_deblock = false; + } + + /// Record a decoded macroblock and deblock / publish rows that became final. + #[inline] + pub fn mb_done(&mut self, addr: usize) { + self.decoded_mbs += 1; + let r = addr / self.mb_w; + self.row_count[r] += 1; + if self.row_count[r] as usize >= self.mb_w && r == self.rows_recon { + while self.rows_recon < self.mb_h && self.row_count[self.rows_recon] as usize >= self.mb_w { + self.rows_recon += 1; + } + self.flush_rows(false); + } + } + + /// Deblock and publish rows. With `finished`, everything remaining is processed (also rows with + /// missing macroblocks, so that waiting readers never block forever). + pub fn flush_rows(&mut self, finished: bool) { + // Deblocking row r must wait until row r + 1 is reconstructed (its intra prediction reads the + // unfiltered bottom line of row r); row r is final once row r + 1 is deblocked. + let deblock_limit = if finished { self.mb_h } else { self.rows_recon.saturating_sub(1) }; + while self.rows_deblocked < deblock_limit { + let r = self.rows_deblocked; + if !self.skip_deblock { + for addr in r * self.mb_w..(r + 1) * self.mb_w { + deblock::deblock_mb(self, addr, self.mb_w); + } + } + self.rows_deblocked += 1; + } + let publish_limit = if finished { self.mb_h } else { self.rows_deblocked.saturating_sub(1) }; + while self.rows_published < publish_limit { + let r = self.rows_published; + let slices = &self.slices; + let ids = |st: &MbState, l: usize, ri: i8| { + slices.get(st.slice_num as usize).and_then(|s| s.ref_ids[l].get(ri as usize).copied()).unwrap_or(u32::MAX) + }; + let row = Frame::make_row(&self.planes, r, &self.mbs, &ids); + self.frame.publish(r, row); + self.rows_published += 1; + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum WpMode { + Default, + Explicit, + Implicit, +} + +/// Prediction weight tables of a slice. +struct Weights { + mode: WpMode, + luma_denom: i32, + chroma_denom: i32, + /// [list][ref] -> (weight, offset) for luma and two chroma components. + luma: [Vec<(i32, i32)>; 2], + chroma: [Vec<[(i32, i32); 2]>; 2], + /// Implicit weights (w0, w1) indexed by ref0 * n1 + ref1. + implicit: Vec<(i32, i32)>, + n1: usize, +} + +/// Current macroblock syntax (prediction part). +#[derive(Clone, Copy, Default)] +pub(crate) struct MbCur { + pub info: MbTypeInfo, + pub sub: [SubMbInfo; 4], + /// rem_intra_pred_mode per block, or -1 when prev_intra_pred_mode_flag was set. + pub rem_mode: [i8; 16], + pub cbp: u8, + pub ref_idx: [[i8; 4]; 2], + /// mvd per (partition * 4 + sub-partition). + pub mvd: [[[i16; 2]; 16]; 2], +} + +/// Coefficient scratch for one macroblock. +pub(crate) struct Scratch { + /// Luma: 16 4x4 blocks (raster block index * 16 + raster coefficient) or 4 8x8 blocks (b8 * 64 + raster). + pub coef: [i32; 256], + /// Chroma: [Cb, Cr][raster 2x2 block * 16 + raster coefficient]. + pub coef_c: [[i32; 64]; 2], + /// Luma blocks with coefficients: raster 4x4 bits, or b8 bits in 8x8 mode. + pub blk_nz: u16, + /// Chroma blocks with coefficients (bit per raster 2x2 block). + pub c_nz: [u8; 2], + pub pred: [[u8; 256]; 2], + pub pred_c: [[[u8; 64]; 2]; 2], +} + +impl Scratch { + pub fn new() -> Box { + Box::new(Scratch { coef: [0; 256], coef_c: [[0; 64]; 2], blk_nz: 0, c_nz: [0; 2], pred: [[0; 256]; 2], pred_c: [[[0; 64]; 2]; 2] }) + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Nb { + pub avail: bool, + pub ref_idx: i8, + pub mv: [i16; 2], +} + +const NB_NA: Nb = Nb { avail: false, ref_idx: -1, mv: [0, 0] }; + +pub struct SliceDecoder<'a> { + pub sh: &'a SliceHeader, + pub pps: &'a Pps, + pub sps: &'a Sps, + pub pic: &'a mut PicState, + pub refs: &'a [Vec; 2], + pub ls: &'a LevelScale, + pub slice_num: u32, + pub mb_w: usize, + pub mb_h: usize, + pub qp: i32, + pub chroma_qp_offset: [i32; 2], + // current macroblock + pub mb_addr: usize, + pub mb_x: usize, + pub mb_y: usize, + /// Neighbouring macroblocks A (left), B (above), C (above right), D (above left) when available. + pub nb: [Option; 4], + pub mv_done: u16, + pub cur: MbCur, + pub s: Box, + weights: Weights, + /// Temporal direct: DistScaleFactor per refIdxL0, and whether pic0 is long-term. + tdirect: Vec<(i32, bool)>, + /// CABAC: previous MB in decoding order had a non-zero mb_qp_delta. + pub prev_qp_delta_nz: bool, +} + +/// Interpolate a bw x bh luma block at (px, py) with motion vector `mv` from reference `f` into `y`, +/// and the corresponding chroma blocks into `c` (stride `cs`). +#[allow(clippy::too_many_arguments)] +#[inline] +fn predict_block(f: &Frame, mv: [i16; 2], px: usize, py: usize, bw: usize, bh: usize, y: (&mut [u8], usize), c: [&mut [u8]; 2], cs: usize) { + const LW: usize = 24; + let mut win = [0u8; LW * 21]; + let ix = px as i32 + (mv[0] as i32 >> 2); + let iy = py as i32 + (mv[1] as i32 >> 2); + f.luma_window(ix - 2, iy - 2, bw + 5, bh + 5, &mut win, LW); + inter::mc_luma_win(&win, LW, (mv[0] & 3) as u32, (mv[1] & 3) as u32, bw, bh, y.0, y.1); + let cx = (px / 2) as i32 + (mv[0] as i32 >> 3); + let cy = (py / 2) as i32 + (mv[1] as i32 >> 3); + let (cw, ch) = (bw / 2, bh / 2); + let (fx, fy) = ((mv[0] & 7) as u32, (mv[1] & 7) as u32); + for (comp, out) in c.into_iter().enumerate() { + let mut w = [0u8; 16 * 9]; + f.chroma_window(comp, cx, cy, cw + 1, ch + 1, &mut w, 16); + inter::mc_chroma_win(&w, 16, fx, fy, cw, ch, out, cs); + } +} + +#[inline(always)] +fn median(a: i16, b: i16, c: i16) -> i16 { + a.max(b).min(a.min(b).max(c)) +} + +impl<'a> SliceDecoder<'a> { + pub fn new( + sh: &'a SliceHeader, + pps: &'a Pps, + sps: &'a Sps, + pic: &'a mut PicState, + refs: &'a [Vec; 2], + ls: &'a LevelScale, + ) -> Result { + let slice_num = pic.slices.len() as u32; + let chroma_qp_offset = [pps.chroma_qp_index_offset, pps.second_chroma_qp_index_offset]; + pic.slices.push(SliceInfo { + disable_deblocking_filter_idc: sh.disable_deblocking_filter_idc, + alpha_offset: sh.slice_alpha_c0_offset_div2 * 2, + beta_offset: sh.slice_beta_offset_div2 * 2, + ref_ids: [refs[0].iter().map(|r| r.id()).collect(), refs[1].iter().map(|r| r.id()).collect()], + }); + let weights = Self::build_weights(sh, pps, refs, pic.poc); + let mut tdirect = Vec::new(); + if sh.slice_type == SliceType::B && !sh.direct_spatial_mv_pred && !refs[1].is_empty() { + let poc1 = refs[1][0].poc(); + for r0 in refs[0].iter() { + let tb = (pic.poc - r0.poc()).clamp(-128, 127); + let td = (poc1 - r0.poc()).clamp(-128, 127); + if td == 0 || r0.long_term { + tdirect.push((256, true)); + } else { + let tx = (16384 + (td / 2).abs()) / td; + tdirect.push((((tb * tx + 32) >> 6).clamp(-1024, 1023), false)); + } + } + } + let (mb_w, mb_h) = (pic.mb_w, pic.mb_h); + Ok(SliceDecoder { + sh, + pps, + sps, + pic, + refs, + ls, + slice_num, + mb_w, + mb_h, + qp: sh.qp(pps), + chroma_qp_offset, + mb_addr: 0, + mb_x: 0, + mb_y: 0, + nb: [None; 4], + mv_done: 0, + cur: MbCur::default(), + s: Scratch::new(), + weights, + tdirect, + prev_qp_delta_nz: false, + }) + } + + fn build_weights(sh: &SliceHeader, pps: &Pps, refs: &[Vec; 2], cur_poc: i32) -> Weights { + let mut w = Weights { + mode: WpMode::Default, + luma_denom: 0, + chroma_denom: 0, + luma: [Vec::new(), Vec::new()], + chroma: [Vec::new(), Vec::new()], + implicit: Vec::new(), + n1: refs[1].len(), + }; + let explicit = (pps.weighted_pred && sh.slice_type.is_p()) || (pps.weighted_bipred_idc == 1 && sh.slice_type.is_b()); + if explicit { + if let Some(t) = &sh.pred_weight_table { + w.mode = WpMode::Explicit; + w.luma_denom = t.luma_log2_denom as i32; + w.chroma_denom = t.chroma_log2_denom as i32; + for (l, entries) in [&t.l0, &t.l1].into_iter().enumerate() { + w.luma[l] = entries.iter().map(|e| (e.luma_weight, e.luma_offset)).collect(); + w.chroma[l] = + entries.iter().map(|e| [(e.chroma_weight[0], e.chroma_offset[0]), (e.chroma_weight[1], e.chroma_offset[1])]).collect(); + } + } + } else if pps.weighted_bipred_idc == 2 && sh.slice_type.is_b() { + w.mode = WpMode::Implicit; + for r0 in refs[0].iter() { + for r1 in refs[1].iter() { + let tb = (cur_poc - r0.poc()).clamp(-128, 127); + let td = (r1.poc() - r0.poc()).clamp(-128, 127); + let pair = if td == 0 || r0.long_term || r1.long_term { + (32, 32) + } else { + let tx = (16384 + (td / 2).abs()) / td; + let dsf = ((tb * tx + 32) >> 6).clamp(-1024, 1023); + if (dsf >> 2) < -64 || (dsf >> 2) > 128 { (32, 32) } else { (64 - (dsf >> 2), dsf >> 2) } + }; + w.implicit.push(pair); + } + } + } + w + } + + // ------------------------------------------------------------------------------------------ + // Neighbour helpers + // ------------------------------------------------------------------------------------------ + + /// Set up the current macroblock address and neighbour availability. + pub fn start_mb(&mut self, addr: usize) { + self.mb_addr = addr; + self.mb_x = addr % self.mb_w; + self.mb_y = addr / self.mb_w; + let sn = self.slice_num; + let mbs = &self.pic.mbs; + let w = self.mb_w; + let (x, y) = (self.mb_x, self.mb_y); + let ok = |a: usize| if mbs[a].slice_num == sn { Some(a) } else { None }; + self.nb = [ + if x > 0 { ok(addr - 1) } else { None }, + if y > 0 { ok(addr - w) } else { None }, + if y > 0 && x + 1 < w { ok(addr - w + 1) } else { None }, + if y > 0 && x > 0 { ok(addr - w - 1) } else { None }, + ]; + self.mv_done = 0; + self.pic.mbs[addr] = MbState { slice_num: sn, ..MbState::default() }; + } + + #[inline] + pub fn mb(&self) -> &MbState { + &self.pic.mbs[self.mb_addr] + } + #[inline] + pub fn mb_mut(&mut self) -> &mut MbState { + &mut self.pic.mbs[self.mb_addr] + } + + /// Neighbour MB usable for intra prediction (constrained_intra_pred aware). + fn intra_nb(&self, n: usize) -> bool { + match self.nb[n] { + None => false, + Some(a) => !self.pps.constrained_intra_pred || self.pic.mbs[a].kind.is_intra(), + } + } + + fn chroma_qp(&self, qp: i32, c: usize) -> i32 { + QPC_TABLE[(qp + self.chroma_qp_offset[c]).clamp(0, 51) as usize] as i32 + } + + /// nC for a luma 4x4 block (raster index) - 9.2.1. + fn nc_luma(&self, raster: usize) -> i32 { + let (bx, by) = (raster & 3, raster >> 2); + let cur = self.mb(); + let a = if bx > 0 { Some(cur.nnz[raster - 1]) } else { self.nb[0].map(|m| self.pic.mbs[m].nnz[by * 4 + 3]) }; + let b = if by > 0 { Some(cur.nnz[raster - 4]) } else { self.nb[1].map(|m| self.pic.mbs[m].nnz[12 + bx]) }; + match (a, b) { + (Some(a), Some(b)) => (a as i32 + b as i32 + 1) >> 1, + (Some(a), None) => a as i32, + (None, Some(b)) => b as i32, + _ => 0, + } + } + + fn nc_chroma(&self, c: usize, raster: usize) -> i32 { + let (bx, by) = (raster & 1, raster >> 1); + let cur = self.mb(); + let a = if bx > 0 { Some(cur.nnz_c[c][raster - 1]) } else { self.nb[0].map(|m| self.pic.mbs[m].nnz_c[c][by * 2 + 1]) }; + let b = if by > 0 { Some(cur.nnz_c[c][raster - 2]) } else { self.nb[1].map(|m| self.pic.mbs[m].nnz_c[c][2 + bx]) }; + match (a, b) { + (Some(a), Some(b)) => (a as i32 + b as i32 + 1) >> 1, + (Some(a), None) => a as i32, + (None, Some(b)) => b as i32, + _ => 0, + } + } + + /// Motion data of the neighbouring 4x4 block at (x, y) (4x4 units, relative to the current MB). + pub(crate) fn nb_motion(&self, list: usize, x: i32, y: i32) -> Nb { + let (mb, bx, by) = if x < 0 { + if y < 0 { + (self.nb[3], 3, 3) + } else if y > 3 { + (None, 0, 0) + } else { + (self.nb[0], 3, y as usize) + } + } else if x > 3 { + if y < 0 { (self.nb[2], 0, 3) } else { (None, 0, 0) } + } else if y < 0 { + (self.nb[1], x as usize, 3) + } else { + let r = (y * 4 + x) as usize; + if self.mv_done & (1 << r) == 0 { + return NB_NA; + } + (Some(self.mb_addr), x as usize, y as usize) + }; + let Some(m) = mb else { return NB_NA }; + let st = &self.pic.mbs[m]; + let b8 = (by >> 1) * 2 + (bx >> 1); + let r = st.ref_idx[list][b8]; + if r < 0 { + return Nb { avail: true, ref_idx: -1, mv: [0, 0] }; + } + Nb { avail: true, ref_idx: r, mv: st.mv[list][by * 4 + bx] } + } + + /// Returns (A, B, C) neighbour motion for a partition at (x, y) of width w (4x4 units), with C + /// replaced by D when unavailable. + fn nb_abc(&self, list: usize, x: usize, y: usize, w: usize) -> (Nb, Nb, Nb) { + let (x, y, w) = (x as i32, y as i32, w as i32); + let a = self.nb_motion(list, x - 1, y); + let b = self.nb_motion(list, x, y - 1); + let mut c = self.nb_motion(list, x + w, y - 1); + if !c.avail { + c = self.nb_motion(list, x - 1, y - 1); + } + (a, b, c) + } + + /// Luma motion vector prediction (8.4.1.3). `shape`: 1 = 16x8, 2 = 8x16, 0 = other. + #[allow(clippy::too_many_arguments)] + fn mvp(&self, list: usize, x: usize, y: usize, w: usize, ref_idx: i8, shape: u8, part: usize) -> [i16; 2] { + let (a, b, c) = self.nb_abc(list, x, y, w); + if shape == 1 { + if part == 0 && b.ref_idx == ref_idx { + return b.mv; + } + if part == 1 && a.ref_idx == ref_idx { + return a.mv; + } + } else if shape == 2 { + if part == 0 && a.ref_idx == ref_idx { + return a.mv; + } + if part == 1 && c.ref_idx == ref_idx { + return c.mv; + } + } + Self::median_pred(a, b, c, ref_idx) + } + + fn median_pred(a: Nb, mut b: Nb, mut c: Nb, ref_idx: i8) -> [i16; 2] { + if !b.avail && !c.avail && a.avail { + b = a; + c = a; + } + let ma = a.ref_idx == ref_idx; + let mb = b.ref_idx == ref_idx; + let mc = c.ref_idx == ref_idx; + match (ma, mb, mc) { + (true, false, false) => a.mv, + (false, true, false) => b.mv, + (false, false, true) => c.mv, + _ => [median(a.mv[0], b.mv[0], c.mv[0]), median(a.mv[1], b.mv[1], c.mv[1])], + } + } + + /// Store motion for a rectangle (4x4 units) of the current MB and mark it decoded. + fn set_motion(&mut self, x: usize, y: usize, w: usize, h: usize, refs: [i8; 2], mvs: [[i16; 2]; 2]) { + let st = &mut self.pic.mbs[self.mb_addr]; + for yy in y..y + h { + for xx in x..x + w { + let r = yy * 4 + xx; + for l in 0..2 { + st.mv[l][r] = if refs[l] >= 0 { mvs[l] } else { [0, 0] }; + } + self.mv_done |= 1 << r; + } + } + for l in 0..2 { + // ref_idx is per 8x8; rectangles never straddle 8x8 blocks with different refs + for yy in (y..y + h).step_by(1) { + for xx in x..x + w { + st.ref_idx[l][(yy >> 1) * 2 + (xx >> 1)] = refs[l]; + } + } + } + } + + // ------------------------------------------------------------------------------------------ + // Direct prediction (8.4.1.2) + // ------------------------------------------------------------------------------------------ + + /// Co-located (mvCol, refIdxCol, referenced picture id) for 4x4 block `raster` of the current MB. + fn colocated(&self, raster: usize) -> ([i16; 2], i8, u32) { + let Some(col) = self.refs[1].first() else { return ([0, 0], -1, u32::MAX) }; + let raster = if self.sps.direct_8x8_inference { + let (bx, by) = (raster & 3, raster >> 2); + (if by >= 2 { 12 } else { 0 }) + if bx >= 2 { 3 } else { 0 } + } else { + raster + }; + if col.frame.mb_w != self.mb_w || self.mb_y >= col.frame.mb_h() { + return ([0, 0], -1, u32::MAX); + } + let m = col.frame.row(self.mb_y); + let mb = self.mb_x; + if m.intra[mb] { + return ([0, 0], -1, u32::MAX); + } + let b8 = mb * 4 + (raster >> 3) * 2 + ((raster & 3) >> 1); + let l = if m.ref_idx[0][b8] >= 0 { 0 } else { 1 }; + (m.mv[l][mb * 16 + raster], m.ref_idx[l][b8], m.ref_id[l][b8]) + } + + /// Spatial direct: MB-level reference indices, predicted mvs and directZeroPrediction (8.4.1.2.2). + fn spatial_direct_base(&self) -> ([i8; 2], [[i16; 2]; 2], bool) { + let mut refs = [-1i8; 2]; + let mut nbs = [(NB_NA, NB_NA, NB_NA); 2]; + let min_pos = |x: i8, y: i8| if x >= 0 && y >= 0 { x.min(y) } else { x.max(y) }; + for (l, nb) in nbs.iter_mut().enumerate() { + *nb = self.nb_abc(l, 0, 0, 4); + refs[l] = min_pos(nb.0.ref_idx, min_pos(nb.1.ref_idx, nb.2.ref_idx)); + } + let mut mvs = [[0i16; 2]; 2]; + if refs[0] < 0 && refs[1] < 0 { + return ([0, 0], mvs, true); + } + for l in 0..2 { + if refs[l] >= 0 { + let (a, b, c) = nbs[l]; + mvs[l] = Self::median_pred(a, b, c, refs[l]); + } + } + (refs, mvs, false) + } + + /// Derive direct-mode motion for the 8x8 blocks in `b8_mask` of the current MB. + fn derive_direct(&mut self, b8_mask: u8) -> Result<()> { + let spatial = self.sh.direct_spatial_mv_pred; + let (base_refs, base_mvs, direct_zero) = if spatial { self.spatial_direct_base() } else { ([0, 0], [[0, 0]; 2], false) }; + let col_short_term = self.refs[1].first().map(|r| !r.long_term).unwrap_or(false); + for b8 in 0..4 { + if b8_mask & (1 << b8) == 0 { + continue; + } + let (x8, y8) = ((b8 & 1) * 2, (b8 >> 1) * 2); + let unit = if self.sps.direct_8x8_inference { 2 } else { 1 }; + for sub in (0..4).filter(|s| unit == 1 || *s == 0) { + let (x, y) = (x8 + (sub & 1), y8 + (sub >> 1)); + let raster = y * 4 + x; + let (mv_col, ref_col, id_col) = self.colocated(raster); + let (refs, mvs) = if spatial { + let col_zero = col_short_term && ref_col == 0 && (-1..=1).contains(&mv_col[0]) && (-1..=1).contains(&mv_col[1]); + let mut mvs = base_mvs; + for l in 0..2 { + if direct_zero || base_refs[l] < 0 || (base_refs[l] == 0 && col_zero) { + mvs[l] = [0, 0]; + } + } + (base_refs, mvs) + } else { + let ref0 = if ref_col < 0 { + 0 + } else { + match self.refs[0].iter().position(|r| r.id() == id_col) { + Some(i) => i as i8, + None => 0, + } + }; + let (dsf, ltr) = self.tdirect.get(ref0 as usize).copied().unwrap_or((256, true)); + let (mv0, mv1) = if ltr { + (mv_col, [0, 0]) + } else { + let s = |c: i16| ((dsf * c as i32 + 128) >> 8) as i16; + let m0 = [s(mv_col[0]), s(mv_col[1])]; + (m0, [m0[0] - mv_col[0], m0[1] - mv_col[1]]) + }; + ([ref0, 0], [mv0, mv1]) + }; + self.set_motion(x, y, unit, unit, refs, mvs); + } + } + let st = &mut self.pic.mbs[self.mb_addr]; + st.direct8x8 |= b8_mask; + Ok(()) + } + + // ------------------------------------------------------------------------------------------ + // Motion vector derivation for a parsed inter macroblock + // ------------------------------------------------------------------------------------------ + + /// P_Skip motion (8.4.1.1). + pub fn derive_p_skip(&mut self) { + let a = self.nb_motion(0, -1, 0); + let b = self.nb_motion(0, 0, -1); + let mv = if self.nb[0].is_none() || self.nb[1].is_none() || (a.ref_idx == 0 && a.mv == [0, 0]) || (b.ref_idx == 0 && b.mv == [0, 0]) { + [0, 0] + } else { + self.mvp(0, 0, 0, 4, 0, 0, 0) + }; + self.set_motion(0, 0, 4, 4, [0, -1], [mv, [0, 0]]); + } + + /// Derive mvs of the current (non-skip) inter MB from `self.cur` (ref_idx, mvd). + pub fn derive_inter_motion(&mut self) -> Result<()> { + let info = self.cur.info; + if info.kind == MbKind::BDirect16x16 || info.kind == MbKind::BSkip { + return self.derive_direct(0xf); + } + let n = [self.refs[0].len(), self.refs[1].len()]; + if info.part != Part::P8x8 { + let shape = match info.part { + Part::P16x8 => 1, + Part::P8x16 => 2, + _ => 0, + }; + for p in 0..info.part.num_parts() { + let (x, y, w, h) = info.part.rect(p); + let mut refs = [-1i8; 2]; + let mut mvs = [[0i16; 2]; 2]; + for l in 0..2 { + if info.pred[p] & (1 << l) != 0 { + let r = self.cur.ref_idx[l][p]; + ensure!((r as usize) < n[l], "ref_idx {r} out of range (list {l}, size {})", n[l]); + let mvp = self.mvp(l, x, y, w, r, shape, p); + let d = self.cur.mvd[l][p * 4]; + refs[l] = r; + mvs[l] = [mvp[0].wrapping_add(d[0]), mvp[1].wrapping_add(d[1])]; + } + } + self.set_motion(x, y, w, h, refs, mvs); + } + return Ok(()); + } + for p in 0..4 { + let sub = self.cur.sub[p]; + let (x8, y8) = ((p & 1) * 2, (p >> 1) * 2); + if sub.direct { + self.derive_direct(1 << p)?; + continue; + } + let mut refs = [-1i8; 2]; + for (l, r) in refs.iter_mut().enumerate() { + if sub.pred & (1 << l) != 0 { + let v = self.cur.ref_idx[l][p]; + ensure!((v as usize) < n[l], "ref_idx {v} out of range (list {l}, size {})", n[l]); + *r = v; + } + } + for j in 0..sub.shape.num_parts() { + let (sx, sy, w, h) = sub.shape.rect(j); + let (x, y) = (x8 + sx, y8 + sy); + let mut mvs = [[0i16; 2]; 2]; + for l in 0..2 { + if refs[l] >= 0 { + let mvp = self.mvp(l, x, y, w, refs[l], 0, 0); + let d = self.cur.mvd[l][p * 4 + j]; + mvs[l] = [mvp[0].wrapping_add(d[0]), mvp[1].wrapping_add(d[1])]; + } + } + self.set_motion(x, y, w, h, refs, mvs); + } + } + Ok(()) + } + + // ------------------------------------------------------------------------------------------ + // Inter prediction + // ------------------------------------------------------------------------------------------ + + fn weight_for(&self, refs: [i8; 2], comp: usize) -> Weight { + let w = &self.weights; + match w.mode { + WpMode::Default => Weight::Default, + WpMode::Implicit => { + if refs[0] >= 0 && refs[1] >= 0 { + let (w0, w1) = w.implicit[refs[0] as usize * w.n1 + refs[1] as usize]; + Weight::Weighted { log_wd: 5, w0, w1, o0: 0, o1: 0 } + } else { + Weight::Default + } + } + WpMode::Explicit => { + let get = |l: usize| -> (i32, i32) { + if refs[l] < 0 { + return (0, 0); + } + let r = refs[l] as usize; + if comp == 0 { + w.luma[l].get(r).copied().unwrap_or((1 << w.luma_denom, 0)) + } else { + w.chroma[l].get(r).map(|c| c[comp - 1]).unwrap_or((1 << w.chroma_denom, 0)) + } + }; + let (w0, o0) = get(0); + let (w1, o1) = get(1); + let log_wd = if comp == 0 { w.luma_denom } else { w.chroma_denom }; + Weight::Weighted { log_wd, w0, w1, o0, o1 } + } + } + } + + /// Motion-compensated prediction of a rectangle (4x4 units) written into the picture. + fn mc_rect(&mut self, x4: usize, y4: usize, w4: usize, h4: usize) { + let st = &self.pic.mbs[self.mb_addr]; + let r = y4 * 4 + x4; + let b8 = (y4 >> 1) * 2 + (x4 >> 1); + let refs = [st.ref_idx[0][b8], st.ref_idx[1][b8]]; + let mvs = [st.mv[0][r], st.mv[1][r]]; + let (bw, bh) = (w4 * 4, h4 * 4); + let wts = [self.weight_for(refs, 0), self.weight_for(refs, 1), self.weight_for(refs, 2)]; + let px = self.mb_x * 16 + x4 * 4; + let py = self.mb_y * 16 + y4 * 4; + let use0 = refs[0] >= 0 && self.refs[0].get(refs[0] as usize).is_some(); + let use1 = refs[1] >= 0 && self.refs[1].get(refs[1] as usize).is_some(); + let ys = self.pic.planes.width; + let cs = self.pic.planes.cwidth; + let (cx, cy) = (px / 2, py / 2); + if use0 != use1 && wts.iter().all(|w| *w == Weight::Default) { + // single-list unweighted prediction: interpolate straight into the picture + let l = if use0 { 0 } else { 1 }; + let f = &self.refs[l][refs[l] as usize].frame; + let planes = &mut self.pic.planes; + predict_block( + f, + mvs[l], + px, + py, + bw, + bh, + (&mut planes.y[py * ys + px..], ys), + [&mut planes.cb[cy * cs + cx..], &mut planes.cr[cy * cs + cx..]], + cs, + ); + return; + } + let s = &mut *self.s; + for l in 0..2 { + if refs[l] < 0 { + continue; + } + let Some(rp) = self.refs[l].get(refs[l] as usize) else { continue }; + let [pc0, pc1] = &mut s.pred_c[l]; + predict_block(&rp.frame, mvs[l], px, py, bw, bh, (&mut s.pred[l][..], 16), [&mut pc0[..], &mut pc1[..]], 8); + } + inter::weighted_store( + &mut self.pic.planes.y[py * ys + px..], + ys, + if use0 { Some(&s.pred[0][..]) } else { None }, + if use1 { Some(&s.pred[1][..]) } else { None }, + 16, + bw, + bh, + wts[0], + ); + for c in 0..2 { + let wc = wts[c + 1]; + let a = if use0 { Some(&s.pred_c[0][c][..]) } else { None }; + let b = if use1 { Some(&s.pred_c[1][c][..]) } else { None }; + let plane = if c == 0 { &mut self.pic.planes.cb } else { &mut self.pic.planes.cr }; + inter::weighted_store(&mut plane[cy * cs + cx..], cs, a, b, 8, bw / 2, bh / 2, wc); + } + } + + /// All 16 blocks of the current MB share references and motion vectors. + fn uniform_motion(&self) -> bool { + let st = self.mb(); + (0..2).all(|l| { + let r = st.ref_idx[l]; + r[1] == r[0] && r[2] == r[0] && r[3] == r[0] && st.mv[l].iter().all(|m| *m == st.mv[l][0]) + }) + } + + /// Inter prediction for the whole current MB, using the stored motion. + pub fn predict_inter(&mut self) { + let info = self.cur.info; + let direct_unit = if self.sps.direct_8x8_inference { 2 } else { 1 }; + match info.kind { + MbKind::PSkip => self.mc_rect(0, 0, 4, 4), + MbKind::BSkip | MbKind::BDirect16x16 if self.uniform_motion() => self.mc_rect(0, 0, 4, 4), + MbKind::BSkip | MbKind::BDirect16x16 => { + for y in (0..4).step_by(direct_unit) { + for x in (0..4).step_by(direct_unit) { + self.mc_rect(x, y, direct_unit, direct_unit); + } + } + } + _ => { + if info.part != Part::P8x8 { + for p in 0..info.part.num_parts() { + let (x, y, w, h) = info.part.rect(p); + self.mc_rect(x, y, w, h); + } + } else { + for p in 0..4 { + let sub = self.cur.sub[p]; + let (x8, y8) = ((p & 1) * 2, (p >> 1) * 2); + if sub.direct { + for y in (0..2).step_by(direct_unit) { + for x in (0..2).step_by(direct_unit) { + self.mc_rect(x8 + x, y8 + y, direct_unit, direct_unit); + } + } + } else { + for j in 0..sub.shape.num_parts() { + let (sx, sy, w, h) = sub.shape.rect(j); + self.mc_rect(x8 + sx, y8 + sy, w, h); + } + } + } + } + } + } + } + + // ------------------------------------------------------------------------------------------ + // Intra prediction mode derivation (8.3.1.1 / 8.3.2.1) + // ------------------------------------------------------------------------------------------ + + fn intra_mode_nb(&self, n: usize, bx: usize, by: usize) -> Option { + // returns Some(mode) or None if dcPredModePredictedFlag must be set + let m = self.nb[n]?; + let st = &self.pic.mbs[m]; + if self.pps.constrained_intra_pred && !st.kind.is_intra() { + return None; + } + Some(match st.kind { + MbKind::I4x4 | MbKind::I8x8 => st.intra_modes[by * 4 + bx], + _ => 2, + }) + } + + /// Resolve Intra4x4PredMode / Intra8x8PredMode for the current MB from `self.cur.rem_mode`. + pub fn derive_intra_modes(&mut self, is8x8: bool) { + let step = if is8x8 { 2 } else { 1 }; + let nblk = if is8x8 { 4 } else { 16 }; + for i in 0..nblk { + let (bx, by) = if is8x8 { + ((i & 1) * 2, (i >> 1) * 2) + } else { + let (x, y) = BLK4_XY[i]; + (x as usize, y as usize) + }; + let a = if bx > 0 { Some(self.mb().intra_modes[by * 4 + bx - 1]) } else { self.intra_mode_nb(0, 3, by) }; + let b = if by > 0 { Some(self.mb().intra_modes[(by - 1) * 4 + bx]) } else { self.intra_mode_nb(1, bx, 3) }; + let pred = match (a, b) { + (Some(a), Some(b)) => a.min(b), + _ => 2, + }; + let rem = self.cur.rem_mode[i]; + let mode = if rem < 0 { + pred + } else if (rem as u8) < pred { + rem as u8 + } else { + rem as u8 + 1 + }; + let st = self.mb_mut(); + for yy in by..by + step { + for xx in bx..bx + step { + st.intra_modes[yy * 4 + xx] = mode; + } + } + } + } + + // ------------------------------------------------------------------------------------------ + // Reconstruction + // ------------------------------------------------------------------------------------------ + + fn add_luma_block4(&mut self, raster: usize) { + if self.s.blk_nz & (1 << raster) == 0 { + return; + } + let stride = self.pic.planes.width; + let px = self.mb_x * 16 + (raster & 3) * 4; + let py = self.mb_y * 16 + (raster >> 2) * 4; + let Some(blk) = self.s.coef.get_mut(raster * 16..).and_then(|c| c.first_chunk_mut::<16>()) else { + return; + }; + transform::idct4_add(blk, &mut self.pic.planes.y[py * stride + px..], stride); + blk.fill(0); + } + + fn add_luma_block8(&mut self, b8: usize) { + if self.s.blk_nz & (1 << b8) == 0 { + return; + } + let stride = self.pic.planes.width; + let px = self.mb_x * 16 + (b8 & 1) * 8; + let py = self.mb_y * 16 + (b8 >> 1) * 8; + let Some(blk) = self.s.coef.get_mut(b8 * 64..).and_then(|c| c.first_chunk_mut::<64>()) else { + return; + }; + transform::idct8_add(blk, &mut self.pic.planes.y[py * stride + px..], stride); + blk.fill(0); + } + + fn add_chroma(&mut self) { + let stride = self.pic.planes.cwidth; + for c in 0..2 { + if self.s.c_nz[c] == 0 { + continue; + } + for b in 0..4 { + if self.s.c_nz[c] & (1 << b) == 0 { + continue; + } + let px = self.mb_x * 8 + (b & 1) * 4; + let py = self.mb_y * 8 + (b >> 1) * 4; + let Some(blk) = self.s.coef_c[c].get_mut(b * 16..).and_then(|c| c.first_chunk_mut::<16>()) else { + continue; + }; + let plane = if c == 0 { &mut self.pic.planes.cb } else { &mut self.pic.planes.cr }; + transform::idct4_add(blk, &mut plane[py * stride + px..], stride); + blk.fill(0); + } + self.s.c_nz[c] = 0; + } + } + + fn luma_avail4(&self, bx: usize, by: usize) -> Avail { + let a = self.intra_nb(0); + let b = self.intra_nb(1); + let c = self.intra_nb(2); + let d = self.intra_nb(3); + let left = if bx > 0 { true } else { a }; + let top = if by > 0 { true } else { b }; + let top_left = match (bx > 0, by > 0) { + (true, true) => true, + (false, true) => a, + (true, false) => b, + (false, false) => d, + }; + let top_right = if by == 0 { + if bx < 3 { b } else { c } + } else if bx == 3 { + false + } else { + RASTER_TO_BLK4[(by - 1) * 4 + bx + 1] < RASTER_TO_BLK4[by * 4 + bx] + }; + Avail { left, top, top_left, top_right } + } + + /// Reconstruct an intra MB (prediction + residual) after its syntax has been parsed. + pub fn reconstruct_intra(&mut self) { + let kind = self.mb().kind; + let stride = self.pic.planes.width; + let (x0, y0) = (self.mb_x * 16, self.mb_y * 16); + match kind { + MbKind::I4x4 => { + for (blk, &(bx, by)) in BLK4_XY.iter().enumerate() { + let (bx, by) = (bx as usize, by as usize); + let _ = blk; + let av = self.luma_avail4(bx, by); + let mode = self.mb().intra_modes[by * 4 + bx]; + intra::pred4x4(&mut self.pic.planes.y, stride, x0 + bx * 4, y0 + by * 4, mode, av); + self.add_luma_block4(by * 4 + bx); + } + } + MbKind::I8x8 => { + for b8 in 0..4 { + let (bx, by) = ((b8 & 1) * 2, (b8 >> 1) * 2); + let mut av = self.luma_avail4(bx, by); + av.top_right = match b8 { + 0 => self.intra_nb(1), + 1 => self.intra_nb(2), + 2 => true, + _ => false, + }; + let mode = self.mb().intra_modes[by * 4 + bx]; + intra::pred8x8(&mut self.pic.planes.y, stride, x0 + bx * 4, y0 + by * 4, mode, av); + self.add_luma_block8(b8); + } + } + MbKind::I16x16 => { + let av = Avail { left: self.intra_nb(0), top: self.intra_nb(1), top_left: self.intra_nb(3), top_right: false }; + intra::pred16x16(&mut self.pic.planes.y, stride, x0, y0, self.cur.info.i16_mode, av); + for r in 0..16 { + self.add_luma_block4(r); + } + } + _ => {} + } + let av = Avail { left: self.intra_nb(0), top: self.intra_nb(1), top_left: self.intra_nb(3), top_right: false }; + let cs = self.pic.planes.cwidth; + let mode = self.mb().intra_chroma_mode; + let (cx, cy) = (self.mb_x * 8, self.mb_y * 8); + intra::pred_chroma(&mut self.pic.planes.cb, cs, cx, cy, mode, av); + intra::pred_chroma(&mut self.pic.planes.cr, cs, cx, cy, mode, av); + self.add_chroma(); + self.s.blk_nz = 0; + } + + /// Add the residual of an inter MB (prediction already written). + pub fn reconstruct_inter_residual(&mut self) { + if self.mb().transform_8x8 { + for b8 in 0..4 { + self.add_luma_block8(b8); + } + } else if self.s.blk_nz != 0 { + for r in 0..16 { + self.add_luma_block4(r); + } + } + self.add_chroma(); + self.s.blk_nz = 0; + } + + /// Write I_PCM samples. + pub fn write_pcm(&mut self, samples: &[u8]) { + let stride = self.pic.planes.width; + let (x0, y0) = (self.mb_x * 16, self.mb_y * 16); + for r in 0..16 { + self.pic.planes.y[(y0 + r) * stride + x0..(y0 + r) * stride + x0 + 16].copy_from_slice(&samples[r * 16..r * 16 + 16]); + } + let cs = self.pic.planes.cwidth; + let (cx, cy) = (self.mb_x * 8, self.mb_y * 8); + for c in 0..2 { + let src = &samples[256 + c * 64..256 + c * 64 + 64]; + let plane = if c == 0 { &mut self.pic.planes.cb } else { &mut self.pic.planes.cr }; + for r in 0..8 { + plane[(cy + r) * cs + cx..(cy + r) * cs + cx + 8].copy_from_slice(&src[r * 8..r * 8 + 8]); + } + } + } + + /// Common state for an I_PCM MB. + pub fn finish_pcm(&mut self) { + let qpc = [self.chroma_qp(0, 0) as u8, self.chroma_qp(0, 1) as u8]; + let st = self.mb_mut(); + st.kind = MbKind::IPcm; + st.qp = 0; + st.qpc = qpc; + st.nnz = [16; 16]; + st.nnz_c = [[16; 4]; 2]; + st.cbf_dc = 7; + st.cbp = 0x2f; + st.nz_mask = 0xffff; + st.ref_idx = [[-1; 4]; 2]; + } + + /// Update QP after mb_qp_delta. + pub fn apply_qp_delta(&mut self, delta: i32) -> Result<()> { + ensure!((-26..=25).contains(&delta), "mb_qp_delta {delta} out of range"); + self.qp = (self.qp + delta + 52) % 52; + Ok(()) + } + + pub fn store_qp(&mut self) { + let qp = self.qp; + let qpc = [self.chroma_qp(qp, 0) as u8, self.chroma_qp(qp, 1) as u8]; + let st = self.mb_mut(); + st.qp = qp as u8; + st.qpc = qpc; + } + + // ------------------------------------------------------------------------------------------ + // Residual storage helpers (shared by CAVLC and CABAC) + // ------------------------------------------------------------------------------------------ + + fn luma_list4(&self) -> usize { + if self.mb().kind.is_intra() { 0 } else { 3 } + } + + /// Store a 4x4 luma block given in scan order (levels[0..16]); `ac_only` skips position 0. + pub fn put_luma4(&mut self, raster: usize, levels: &[i32; 16], ac_only: bool) { + let qp = self.qp; + let ls = &self.ls.ls4[self.luma_list4()][(qp % 6) as usize]; + let blk = &mut self.s.coef[raster * 16..raster * 16 + 16]; + let mut any = false; + for (k, &lv) in levels.iter().enumerate().skip(ac_only as usize) { + if lv != 0 { + let p = ZIGZAG4[k] as usize; + blk[p] = transform::scale4(lv, ls[p], qp); + any = true; + } + } + if any { + self.s.blk_nz |= 1 << raster; + self.pic.mbs[self.mb_addr].nz_mask |= 1 << raster; + } + } + + /// Store an 8x8 luma block given in scan order. + pub fn put_luma8(&mut self, b8: usize, levels: &[i32; 64]) { + let qp = self.qp; + let list = if self.mb().kind.is_intra() { 0 } else { 1 }; + let ls = &self.ls.ls8[list][(qp % 6) as usize]; + let blk = &mut self.s.coef[b8 * 64..b8 * 64 + 64]; + let mut any = false; + for (k, &lv) in levels.iter().enumerate() { + if lv != 0 { + let p = ZIGZAG8[k] as usize; + blk[p] = transform::scale8(lv, ls[p], qp); + any = true; + } + } + if any { + self.s.blk_nz |= 1 << b8; + let (bx, by) = ((b8 & 1) * 2, (b8 >> 1) * 2); + let m = (0b11u16 << (by * 4 + bx)) | (0b11u16 << ((by + 1) * 4 + bx)); + self.pic.mbs[self.mb_addr].nz_mask |= m; + } + } + + /// Intra16x16 DC levels in scan order -> scaled DC of each 4x4 block. + pub fn put_luma_dc(&mut self, levels: &[i32; 16]) { + let mut c = [0i32; 16]; + for (k, &lv) in levels.iter().enumerate() { + c[ZIGZAG4[k] as usize] = lv; + } + let qp = self.qp; + let ls00 = self.ls.ls4[0][(qp % 6) as usize][0]; + transform::luma_dc_dequant(&mut c, qp, ls00); + for (r, &dc) in c.iter().enumerate() { + if dc != 0 { + self.s.coef[r * 16] = dc; + self.s.blk_nz |= 1 << r; + } + } + } + + /// Chroma DC levels (4:2:0: 4 values in raster order) for component `c`. + pub fn put_chroma_dc(&mut self, c: usize, levels: &[i32; 4]) { + let mut v = *levels; + let qpc = self.chroma_qp(self.qp, c); + let list = if self.mb().kind.is_intra() { 1 + c } else { 4 + c }; + let ls00 = self.ls.ls4[list][(qpc % 6) as usize][0]; + transform::chroma_dc_dequant_420(&mut v, qpc, ls00); + for (b, &dc) in v.iter().enumerate() { + if dc != 0 { + self.s.coef_c[c][b * 16] = dc; + self.s.c_nz[c] |= 1 << b; + } + } + } + + /// Chroma AC levels (scan positions 1..16 in levels[1..]) for block `b` of component `c`. + pub fn put_chroma_ac(&mut self, c: usize, b: usize, levels: &[i32; 16]) { + let qpc = self.chroma_qp(self.qp, c); + let list = if self.mb().kind.is_intra() { 1 + c } else { 4 + c }; + let ls = &self.ls.ls4[list][(qpc % 6) as usize]; + let blk = &mut self.s.coef_c[c][b * 16..b * 16 + 16]; + let mut any = false; + for (k, &lv) in levels.iter().enumerate().skip(1) { + if lv != 0 { + let p = ZIGZAG4[k] as usize; + blk[p] = transform::scale4(lv, ls[p], qpc); + any = true; + } + } + if any { + self.s.c_nz[c] |= 1 << b; + } + } + + // ------------------------------------------------------------------------------------------ + // CAVLC macroblock layer + // ------------------------------------------------------------------------------------------ + + /// Decode a P_Skip or B_Skip macroblock at the current address. + pub fn decode_skip(&mut self) -> Result<()> { + let b = self.sh.slice_type.is_b(); + self.cur = MbCur::default(); + self.cur.info.kind = if b { MbKind::BSkip } else { MbKind::PSkip }; + self.mb_mut().kind = self.cur.info.kind; + self.store_qp(); + if b { + self.derive_direct(0xf)?; + } else { + ensure!(!self.refs[0].is_empty(), "P_Skip with empty reference list"); + self.derive_p_skip(); + } + self.predict_inter(); + self.prev_qp_delta_nz = false; + Ok(()) + } + + fn read_te(r: &mut BitReader, range: u32) -> Result { + Ok(if range > 1 { r.read_ue()? } else { !r.read_bit()? as u32 }) + } + + /// Parse and reconstruct one macroblock_layer() with CAVLC. + pub fn decode_mb_cavlc(&mut self, r: &mut BitReader) -> Result<()> { + let mb_type = r.read_ue()?; + let st = self.sh.slice_type; + let info = match st { + SliceType::I | SliceType::Si => intra_mb_type(mb_type), + SliceType::P | SliceType::Sp => { + if mb_type < 5 { + p_mb_type(mb_type) + } else { + intra_mb_type(mb_type - 5) + } + } + SliceType::B => { + if mb_type < 23 { + b_mb_type(mb_type) + } else { + intra_mb_type(mb_type - 23) + } + } + }; + let Some(mut info) = info else { return invalid(format!("mb_type {mb_type} out of range")) }; + self.cur = MbCur { info, ..Default::default() }; + if info.kind == MbKind::IPcm { + r.byte_align(); + let pos = r.byte_pos(); + let data = r.data(); + ensure!(data.len() >= pos + 384, "truncated I_PCM macroblock"); + let Some(&samples) = data.get(pos..).and_then(|d| d.first_chunk::<384>()) else { + return invalid("truncated I_PCM macroblock"); + }; + r.skip(384 * 8)?; + self.finish_pcm(); + self.write_pcm(&samples); + return Ok(()); + } + let mut transform_8x8 = false; + if info.kind == MbKind::I4x4 && self.pps.transform_8x8_mode && r.read_flag()? { + transform_8x8 = true; + info.kind = MbKind::I8x8; + self.cur.info.kind = MbKind::I8x8; + } + self.mb_mut().kind = info.kind; + self.mb_mut().transform_8x8 = transform_8x8; + let n_ref = [self.sh.num_ref_idx_active[0], self.sh.num_ref_idx_active[1]]; + match info.kind { + MbKind::I4x4 | MbKind::I8x8 => { + let n = if info.kind == MbKind::I8x8 { 4 } else { 16 }; + for i in 0..n { + self.cur.rem_mode[i] = if r.read_flag()? { -1 } else { r.read_bits(3)? as i8 }; + } + self.derive_intra_modes(info.kind == MbKind::I8x8); + let m = r.read_ue()?; + ensure!(m <= 3, "intra_chroma_pred_mode out of range"); + self.mb_mut().intra_chroma_mode = m as u8; + } + MbKind::I16x16 => { + let m = r.read_ue()?; + ensure!(m <= 3, "intra_chroma_pred_mode out of range"); + self.mb_mut().intra_chroma_mode = m as u8; + } + MbKind::BDirect16x16 => {} + _ => { + if info.part == Part::P8x8 { + for p in 0..4 { + let t = r.read_ue()?; + let sub = if st.is_b() { b_sub_mb_type(t) } else { p_sub_mb_type(t) }; + let Some(sub) = sub else { return invalid(format!("sub_mb_type {t} out of range")) }; + self.cur.sub[p] = sub; + } + for l in 0..2 { + for p in 0..4 { + let sub = self.cur.sub[p]; + if sub.direct || sub.pred & (1 << l) == 0 { + continue; + } + self.cur.ref_idx[l][p] = if n_ref[l] > 1 && !info.ref0 { Self::read_te(r, n_ref[l] - 1)? as i8 } else { 0 }; + } + } + for l in 0..2 { + for p in 0..4 { + let sub = self.cur.sub[p]; + if sub.direct || sub.pred & (1 << l) == 0 { + continue; + } + for j in 0..sub.shape.num_parts() { + let x = r.read_se()?; + let y = r.read_se()?; + self.cur.mvd[l][p * 4 + j] = [x as i16, y as i16]; + } + } + } + } else { + let np = info.part.num_parts(); + for l in 0..2 { + for p in 0..np { + if info.pred[p] & (1 << l) != 0 { + self.cur.ref_idx[l][p] = if n_ref[l] > 1 { Self::read_te(r, n_ref[l] - 1)? as i8 } else { 0 }; + } + } + } + for l in 0..2 { + for p in 0..np { + if info.pred[p] & (1 << l) != 0 { + let x = r.read_se()?; + let y = r.read_se()?; + self.cur.mvd[l][p * 4] = [x as i16, y as i16]; + } + } + } + } + } + } + // coded_block_pattern + let cbp = if info.kind == MbKind::I16x16 { + info.i16_cbp + } else { + let code = r.read_ue()?; + ensure!(code < 48, "coded_block_pattern out of range"); + let (i, p) = CBP_ME[code as usize]; + if info.kind.is_intra() { i } else { p } + }; + self.cur.cbp = cbp; + self.mb_mut().cbp = cbp; + if !info.kind.is_intra() && cbp & 15 != 0 && self.pps.transform_8x8_mode && self.no_sub_8x8_lt() { + let t = r.read_flag()?; + self.mb_mut().transform_8x8 = t; + } + if !info.kind.is_intra() { + self.derive_inter_motion()?; + } + if cbp != 0 || info.kind == MbKind::I16x16 { + let d = r.read_se()?; + self.apply_qp_delta(d)?; + } + self.store_qp(); + self.residual_cavlc(r, cbp)?; + if info.kind.is_intra() { + self.reconstruct_intra(); + } else { + self.predict_inter(); + self.reconstruct_inter_residual(); + } + Ok(()) + } + + /// Whether transform_size_8x8_flag may be present for an inter MB (NoSubMbPartSizeLessThan8x8 etc.). + pub fn no_sub_8x8_lt(&self) -> bool { + let info = self.cur.info; + match info.kind { + MbKind::BDirect16x16 => self.sps.direct_8x8_inference, + MbKind::Inter if info.part == Part::P8x8 => { + self.cur.sub.iter().all(|s| if s.direct { self.sps.direct_8x8_inference } else { s.shape == SubPart::S8x8 }) + } + _ => true, + } + } + + fn residual_cavlc(&mut self, r: &mut BitReader, cbp: u8) -> Result<()> { + let kind = self.mb().kind; + let t8 = self.mb().transform_8x8; + if kind == MbKind::I16x16 { + let nc = self.nc_luma(0); + let mut lv = [0i32; 16]; + cavlc::residual_block(r, &mut lv, 0, 15, 16, nc)?; + self.put_luma_dc(&lv); + } + for b8 in 0..4 { + let (bx8, by8) = ((b8 & 1) * 2, (b8 >> 1) * 2); + if cbp & (1 << b8) == 0 { + continue; + } + if t8 { + let mut lv8 = [0i32; 64]; + for i4 in 0..4 { + let raster = (by8 + (i4 >> 1)) * 4 + bx8 + (i4 & 1); + let nc = self.nc_luma(raster); + let mut lv = [0i32; 16]; + let tc = cavlc::residual_block(r, &mut lv, 0, 15, 16, nc)?; + self.mb_mut().nnz[raster] = tc; + for (i, &v) in lv.iter().enumerate() { + lv8[4 * i + i4] = v; + } + } + self.put_luma8(b8, &lv8); + } else { + for i4 in 0..4 { + let raster = (by8 + (i4 >> 1)) * 4 + bx8 + (i4 & 1); + let nc = self.nc_luma(raster); + let mut lv = [0i32; 16]; + let tc = if kind == MbKind::I16x16 { + cavlc::residual_block(r, &mut lv[1..], 0, 14, 15, nc)? + } else { + cavlc::residual_block(r, &mut lv, 0, 15, 16, nc)? + }; + self.mb_mut().nnz[raster] = tc; + self.put_luma4(raster, &lv, kind == MbKind::I16x16); + } + } + } + let cbp_c = cbp >> 4; + if cbp_c & 3 != 0 { + for c in 0..2 { + let mut lv = [0i32; 4]; + cavlc::residual_block(r, &mut lv, 0, 3, 4, -1)?; + self.put_chroma_dc(c, &lv); + } + } + if cbp_c & 2 != 0 { + for c in 0..2 { + for b in 0..4 { + let nc = self.nc_chroma(c, b); + let mut lv = [0i32; 16]; + let tc = cavlc::residual_block(r, &mut lv[1..], 0, 14, 15, nc)?; + self.mb_mut().nnz_c[c][b] = tc; + self.put_chroma_ac(c, b, &lv); + } + } + } + Ok(()) + } + + /// Decode slice_data() with CAVLC. `r` is positioned after the slice header. + pub fn decode_cavlc(&mut self, r: &mut BitReader) -> Result<()> { + let total = self.mb_w * self.mb_h; + let mut addr = self.sh.first_mb_in_slice as usize; + let intra_slice = self.sh.slice_type.is_intra(); + loop { + ensure!(addr < total, "macroblock address {addr} out of range"); + if !intra_slice { + let run = r.read_ue()? as usize; + ensure!(addr + run <= total, "mb_skip_run out of range"); + for _ in 0..run { + self.start_mb(addr); + self.decode_skip()?; + self.pic.mb_done(addr); + addr += 1; + } + if run > 0 && !r.more_rbsp_data() { + break; + } + ensure!(addr < total, "macroblock address {addr} out of range"); + } + self.start_mb(addr); + self.decode_mb_cavlc(r)?; + self.pic.mb_done(addr); + addr += 1; + if !r.more_rbsp_data() { + break; + } + } + Ok(()) + } +} diff --git a/crates/video/src/h264/tables.rs b/crates/video/src/h264/tables.rs new file mode 100644 index 00000000..d6e0d588 --- /dev/null +++ b/crates/video/src/h264/tables.rs @@ -0,0 +1,175 @@ +//! Small constant tables from the H.264 specification. + +/// 4x4 frame zig-zag scan: scan position -> raster index (row * 4 + col). +pub const ZIGZAG4: [u8; 16] = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]; + +/// 8x8 frame zig-zag scan: scan position -> raster index (row * 8 + col). +pub const ZIGZAG8: [u8; 64] = zigzag8(); + +const fn zigzag8() -> [u8; 64] { + let mut out = [0u8; 64]; + let mut k = 0; + let mut d: i32 = 0; // anti-diagonal index = row + col + while d < 15 { + // even diagonals go up-right (row decreasing), odd go down-left + let mut i: i32 = 0; + while i < 8 { + let (row, col) = if d % 2 == 0 { (d - i, i) } else { (i, d - i) }; + if row >= 0 && row < 8 && col >= 0 && col < 8 { + out[k] = (row * 8 + col) as u8; + k += 1; + } + i += 1; + } + d += 1; + } + out +} + +/// Table 7-3. +pub const DEFAULT_4X4_INTRA: [u8; 16] = [6, 13, 13, 20, 20, 20, 28, 28, 28, 28, 32, 32, 32, 37, 37, 42]; +pub const DEFAULT_4X4_INTER: [u8; 16] = [10, 14, 14, 20, 20, 20, 24, 24, 24, 24, 27, 27, 27, 30, 30, 34]; +/// Table 7-4. +#[rustfmt::skip] +pub const DEFAULT_8X8_INTRA: [u8; 64] = [ + 6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23, + 23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, + 27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31, + 31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42, +]; +#[rustfmt::skip] +pub const DEFAULT_8X8_INTER: [u8; 64] = [ + 9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21, + 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24, + 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27, + 27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35, +]; + +/// Table 8-15: QPc as a function of qPI (0..=51). +pub const QPC_TABLE: [u8; 52] = { + let mut t = [0u8; 52]; + let tail = [29u8, 30, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39]; + let mut i = 0; + while i < 52 { + t[i] = if i < 30 { i as u8 } else { tail[i - 30] }; + i += 1; + } + t +}; + +/// (8-315) v matrix for normAdjust4x4. +pub const NORM_ADJUST4: [[u16; 3]; 6] = [[10, 16, 13], [11, 18, 14], [13, 20, 16], [14, 23, 18], [16, 25, 20], [18, 29, 23]]; +/// (8-318) v matrix for normAdjust8x8. +pub const NORM_ADJUST8: [[u16; 6]; 6] = [ + [20, 18, 32, 19, 25, 24], + [22, 19, 35, 21, 28, 26], + [26, 23, 42, 24, 33, 31], + [28, 25, 45, 26, 35, 33], + [32, 28, 51, 30, 40, 38], + [36, 32, 58, 34, 46, 43], +]; + +/// normAdjust4x4(m, i, j) for raster position `pos` (i = row, j = col). +pub const fn norm_adjust4(m: usize, pos: usize) -> u16 { + let (i, j) = (pos / 4, pos % 4); + if i % 2 == 0 && j % 2 == 0 { + NORM_ADJUST4[m][0] + } else if i % 2 == 1 && j % 2 == 1 { + NORM_ADJUST4[m][1] + } else { + NORM_ADJUST4[m][2] + } +} + +/// normAdjust8x8(m, i, j) for raster position `pos` (i = row, j = col). +pub const fn norm_adjust8(m: usize, pos: usize) -> u16 { + let (i, j) = (pos / 8, pos % 8); + let k = if i % 4 == 0 && j % 4 == 0 { + 0 + } else if i % 2 == 1 && j % 2 == 1 { + 1 + } else if i % 4 == 2 && j % 4 == 2 { + 2 + } else if (i % 4 == 0 && j % 2 == 1) || (i % 2 == 1 && j % 4 == 0) { + 3 + } else if (i % 4 == 0 && j % 4 == 2) || (i % 4 == 2 && j % 4 == 0) { + 4 + } else { + 5 + }; + NORM_ADJUST8[m][k] +} + +/// Table 8-16 alpha' (indexA 0..=51). +#[rustfmt::skip] +pub const ALPHA: [u8; 52] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 5, 6, 7, 8, 9, 10, 12, 13, + 15, 17, 20, 22, 25, 28, 32, 36, 40, 45, 50, 56, 63, 71, 80, 90, 101, 113, 127, 144, 162, 182, 203, 226, 255, 255, +]; +/// Table 8-16 beta' (indexB 0..=51). +#[rustfmt::skip] +pub const BETA: [u8; 52] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, +]; +/// Table 8-17 tC0' for bS = 1, 2, 3 (indexA 0..=51). +#[rustfmt::skip] +pub const TC0: [[u8; 3]; 52] = { + let b1: [u8; 52] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 7, 8, 9, 10, 11, 13, + ]; + let b2: [u8; 52] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 10, 11, 12, 13, 15, 17, + ]; + let b3: [u8; 52] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 7, 8, 9, 10, 11, 13, 14, 16, 18, 20, 23, 25, + ]; + let mut t = [[0u8; 3]; 52]; + let mut i = 0; + while i < 52 { + t[i] = [b1[i], b2[i], b3[i]]; + i += 1; + } + t +}; + +/// Luma 4x4 block index (decoding order, 6.4.3) -> (x, y) in 4-sample units. +pub const BLK4_XY: [(u8, u8); 16] = + [(0, 0), (1, 0), (0, 1), (1, 1), (2, 0), (3, 0), (2, 1), (3, 1), (0, 2), (1, 2), (0, 3), (1, 3), (2, 2), (3, 2), (2, 3), (3, 3)]; + +/// Raster 4x4 position (y * 4 + x) -> luma4x4BlkIdx. +pub const RASTER_TO_BLK4: [u8; 16] = [0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn zigzag8_known_prefix() { + assert_eq!(&ZIGZAG8[..16], &[0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5]); + assert_eq!(&ZIGZAG8[48..], &[58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63]); + let mut seen = [false; 64]; + for &z in &ZIGZAG8 { + seen[z as usize] = true; + } + assert!(seen.iter().all(|&s| s)); + } + + #[test] + fn blk4_maps_consistent() { + for (b, &(x, y)) in BLK4_XY.iter().enumerate() { + let r = (y * 4 + x) as usize; + assert_eq!(RASTER_TO_BLK4[r] as usize, b); + } + } + + #[test] + fn qpc() { + assert_eq!(QPC_TABLE[29], 29); + assert_eq!(QPC_TABLE[30], 29); + assert_eq!(QPC_TABLE[51], 39); + } +} diff --git a/crates/video/src/h264/transform.rs b/crates/video/src/h264/transform.rs new file mode 100644 index 00000000..1ad55f7b --- /dev/null +++ b/crates/video/src/h264/transform.rs @@ -0,0 +1,307 @@ +//! Scaling (dequantisation) and inverse transforms (8.5.10 - 8.5.13). + +use crate::h264::params::ScalingMatrices; +use crate::h264::tables::{norm_adjust4, norm_adjust8}; + +/// LevelScale4x4 / LevelScale8x8 for all six scaling lists and qP % 6, raster order. +#[derive(Clone)] +pub struct LevelScale { + /// [list 0..6 (IntraY, IntraCb, IntraCr, InterY, InterCb, InterCr)][qp % 6][pos] + pub ls4: [[[i32; 16]; 6]; 6], + /// [list 0..6 (IntraY, InterY, IntraCb, InterCb, IntraCr, InterCr)][qp % 6][pos] + pub ls8: [[[i32; 64]; 6]; 6], +} + +impl LevelScale { + pub fn new(m: &ScalingMatrices) -> Box { + let mut s = Box::new(LevelScale { ls4: [[[0; 16]; 6]; 6], ls8: [[[0; 64]; 6]; 6] }); + for l in 0..6 { + for q in 0..6 { + for p in 0..16 { + s.ls4[l][q][p] = m.m4[l][p] as i32 * norm_adjust4(q, p) as i32; + } + for p in 0..64 { + s.ls8[l][q][p] = m.m8[l][p] as i32 * norm_adjust8(q, p) as i32; + } + } + } + s + } +} + +/// Scale one 4x4 AC coefficient (8.5.12.1, not the DC of Intra16x16 / chroma blocks). +#[inline(always)] +pub fn scale4(c: i32, ls: i32, qp: i32) -> i32 { + if qp >= 24 { c.wrapping_mul(ls) << (qp / 6 - 4) } else { c.wrapping_mul(ls).wrapping_add(1 << (3 - qp / 6)) >> (4 - qp / 6) } +} + +/// Scale one 8x8 coefficient (8.5.13.1). +#[inline(always)] +pub fn scale8(c: i32, ls: i32, qp: i32) -> i32 { + if qp >= 36 { c.wrapping_mul(ls) << (qp / 6 - 6) } else { c.wrapping_mul(ls).wrapping_add(1 << (5 - qp / 6)) >> (6 - qp / 6) } +} + +/// Intra16x16 luma DC: inverse Hadamard + scaling (8.5.10). `c` is raster order (row*4+col); +/// output replaces `c` with dcY (raster: row i -> 4x4 block row, col j -> block column). +pub fn luma_dc_dequant(c: &mut [i32; 16], qp: i32, ls00: i32) { + let mut t = [0i32; 16]; + // rows + for i in 0..4 { + let r = &c[i * 4..i * 4 + 4]; + let s01 = r[0] + r[1]; + let d01 = r[0] - r[1]; + let s23 = r[2] + r[3]; + let d23 = r[2] - r[3]; + t[i * 4] = s01 + s23; + t[i * 4 + 1] = s01 - s23; + t[i * 4 + 2] = d01 - d23; + t[i * 4 + 3] = d01 + d23; + } + // columns + for j in 0..4 { + let s01 = t[j] + t[4 + j]; + let d01 = t[j] - t[4 + j]; + let s23 = t[8 + j] + t[12 + j]; + let d23 = t[8 + j] - t[12 + j]; + let f = [s01 + s23, s01 - s23, d01 - d23, d01 + d23]; + for i in 0..4 { + c[i * 4 + j] = if qp >= 36 { + f[i].wrapping_mul(ls00) << (qp / 6 - 6) + } else { + f[i].wrapping_mul(ls00).wrapping_add(1 << (5 - qp / 6)) >> (6 - qp / 6) + }; + } + } +} + +/// 4:2:0 chroma DC: 2x2 transform + scaling (8.5.11). `c` = [c00, c01, c10, c11]. +pub fn chroma_dc_dequant_420(c: &mut [i32; 4], qp: i32, ls00: i32) { + let s0 = c[0].wrapping_add(c[1]); + let d0 = c[0].wrapping_sub(c[1]); + let s1 = c[2].wrapping_add(c[3]); + let d1 = c[2].wrapping_sub(c[3]); + let f = [s0.wrapping_add(s1), d0.wrapping_add(d1), s0.wrapping_sub(s1), d0.wrapping_sub(d1)]; + for k in 0..4 { + c[k] = (f[k].wrapping_mul(ls00) << (qp / 6)) >> 5; + } +} + +#[inline(always)] +fn clip_u8(v: i32) -> u8 { + v.clamp(0, 255) as u8 +} + +/// Inverse 4x4 transform of raster-order `d` and add to `dst` (8.5.12.2, 8.5.14). +/// +/// Rows first, then columns (the spec's order: the `>> 1` steps make it matter). The column +/// pass and the reconstruction run across all columns at once so they vectorise. +pub fn idct4_add(d: &[i32; 16], dst: &mut [u8], stride: usize) { + let mut t = [[0i32; 4]; 4]; + for (i, row) in t.iter_mut().enumerate() { + let r = &d[i * 4..i * 4 + 4]; + let e = r[0] + r[2]; + let f = r[0] - r[2]; + let g = (r[1] >> 1) - r[3]; + let h = r[1] + (r[3] >> 1); + *row = [e + h, f + g, f - g, e - h]; + } + let mut o = [[0i32; 4]; 4]; + for j in 0..4 { + let e = t[0][j] + t[2][j]; + let f = t[0][j] - t[2][j]; + let g = (t[1][j] >> 1) - t[3][j]; + let h = t[1][j] + (t[3][j] >> 1); + o[0][j] = e + h; + o[1][j] = f + g; + o[2][j] = f - g; + o[3][j] = e - h; + } + for (i, row) in o.iter().enumerate() { + let Some(line) = dst.get_mut(i * stride..).and_then(|d| d.first_chunk_mut::<4>()) else { + return; + }; + for (p, &v) in line.iter_mut().zip(row) { + *p = clip_u8(*p as i32 + ((v + 32) >> 6)); + } + } +} + +/// DC-only shortcut: all AC coefficients zero. +#[allow(dead_code)] +pub fn idct4_dc_add(dc: i32, dst: &mut [u8], stride: usize) { + let v = (dc + 32) >> 6; + for i in 0..4 { + for p in &mut dst[i * stride..i * stride + 4] { + *p = clip_u8(*p as i32 + v); + } + } +} + +#[inline(always)] +fn idct8_1d(d: [i32; 8]) -> [i32; 8] { + let a0 = d[0] + d[4]; + let a4 = d[0] - d[4]; + let a2 = (d[2] >> 1) - d[6]; + let a6 = d[2] + (d[6] >> 1); + let b0 = a0 + a6; + let b2 = a4 + a2; + let b4 = a4 - a2; + let b6 = a0 - a6; + let a1 = -d[3] + d[5] - d[7] - (d[7] >> 1); + let a3 = d[1] + d[7] - d[3] - (d[3] >> 1); + let a5 = -d[1] + d[7] + d[5] + (d[5] >> 1); + let a7 = d[3] + d[5] + d[1] + (d[1] >> 1); + let b1 = a1 + (a7 >> 2); + let b7 = a7 - (a1 >> 2); + let b3 = a3 + (a5 >> 2); + let b5 = (a3 >> 2) - a5; + [b0 + b7, b2 + b5, b4 + b3, b6 + b1, b6 - b1, b4 - b3, b2 - b5, b0 - b7] +} + +/// Inverse 8x8 transform of raster-order `d` and add to `dst` (8.5.13.2): rows, then columns, +/// the column pass and reconstruction across all columns at once (they vectorise). +pub fn idct8_add(d: &[i32; 64], dst: &mut [u8], stride: usize) { + let mut t = [[0i32; 8]; 8]; + for (row, src) in t.iter_mut().zip(d.as_chunks::<8>().0) { + *row = idct8_1d(*src); + } + let mut o = [[0i32; 8]; 8]; + for j in 0..8 { + let col = idct8_1d(std::array::from_fn(|i| t[i][j])); + for i in 0..8 { + o[i][j] = col[i]; + } + } + for (i, row) in o.iter().enumerate() { + let Some(line) = dst.get_mut(i * stride..).and_then(|d| d.first_chunk_mut::<8>()) else { + return; + }; + for (p, &v) in line.iter_mut().zip(row) { + *p = clip_u8(*p as i32 + ((v + 32) >> 6)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The straightforward per-column formulation (8.5.12.2 / 8.5.13.2) the vectorised + /// transforms must equal. + fn reference_add(d: &[i32], n: usize, dst: &mut [u8], stride: usize) { + let one = |v: &[i32]| -> Vec { + if n == 4 { + let (e, f, g, h) = (v[0] + v[2], v[0] - v[2], (v[1] >> 1) - v[3], v[1] + (v[3] >> 1)); + vec![e + h, f + g, f - g, e - h] + } else { + idct8_1d(v.try_into().unwrap()).to_vec() + } + }; + let rows: Vec> = (0..n).map(|i| one(&d[i * n..i * n + n])).collect(); + for j in 0..n { + let col = one(&(0..n).map(|i| rows[i][j]).collect::>()); + for i in 0..n { + let p = &mut dst[i * stride + j]; + *p = (*p as i32 + ((col[i] + 32) >> 6)).clamp(0, 255) as u8; + } + } + } + + #[test] + fn vectorised_transforms_match_reference() { + let mut rng = 0x1234_5678_9abc_def1u64; + let mut next = move || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + rng + }; + for iter in 0..5000 { + let range = [64u64, 1024, 8192, 1 << 15][iter % 4]; + let d4: [i32; 16] = std::array::from_fn(|_| (next() % (2 * range + 1)) as i32 - range as i32); + let d8: [i32; 64] = std::array::from_fn(|_| (next() % (2 * range + 1)) as i32 - range as i32); + let base: Vec = (0..24 * 8).map(|_| next() as u8).collect(); + let (mut a, mut b) = (base.clone(), base.clone()); + idct4_add(&d4, &mut a[3..], 24); + reference_add(&d4, 4, &mut b[3..], 24); + assert_eq!(a, b, "4x4 iteration {iter}"); + let (mut a, mut b) = (base.clone(), base); + idct8_add(&d8, &mut a[5..], 24); + reference_add(&d8, 8, &mut b[5..], 24); + assert_eq!(a, b, "8x8 iteration {iter}"); + } + } + + #[test] + fn idct4_dc_only_matches_shortcut() { + let mut d = [0i32; 16]; + d[0] = 640; // -> +10 everywhere + let mut a = [100u8; 16]; + let mut b = [100u8; 16]; + idct4_add(&d, &mut a, 4); + idct4_dc_add(640, &mut b, 4); + assert_eq!(a, b); + assert!(a.iter().all(|&v| v == 110)); + } + + #[test] + fn idct4_hand_example() { + // d = [[64, 64, 0, 0], 0...]: row0: e=64,f=64,g=32,h=64 -> [128, 96, 32, 0]; columns: each col only row0 set + // col j: e=f=t0j, g=0, h=0 -> all rows = t0j. Result (t+32)>>6 = [2, 2, 1, 0] for every row. + let mut d = [0i32; 16]; + d[0] = 64; + d[1] = 64; + let mut out = [0u8; 16]; + idct4_add(&d, &mut out, 4); + for i in 0..4 { + assert_eq!(&out[i * 4..i * 4 + 4], &[2, 2, 1, 0]); + } + } + + #[test] + fn idct8_dc() { + let mut d = [0i32; 64]; + d[0] = 64 * 5; + let mut out = [10u8; 64]; + idct8_add(&d, &mut out, 8); + assert!(out.iter().all(|&v| v == 15)); + } + + #[test] + fn luma_dc_flat() { + // single DC coefficient spreads equally to all 16 blocks + let mut c = [0i32; 16]; + c[0] = 4; + luma_dc_dequant(&mut c, 36, 16 * 16); + assert!(c.iter().all(|&v| v == 4 * 256)); + } + + #[test] + fn chroma_dc() { + let mut c = [1, 0, 0, 0]; + chroma_dc_dequant_420(&mut c, 6, 16 * 16); + // f = [1,1,1,1] -> ((1*256) << 1) >> 5 = 16 + assert_eq!(c, [16, 16, 16, 16]); + } + + #[test] + fn scale_functions() { + assert_eq!(scale4(2, 160, 24), 2 * 160); + assert_eq!(scale4(2, 160, 12), (2 * 160 + 2) >> 2); + assert_eq!(scale8(1, 320, 36), 320); + assert_eq!(scale8(1, 320, 30), (320 + 1) >> 1); + } + + /// Corrupt streams can carry huge coefficient levels: dequantisation wraps (as release + /// builds always did) instead of panicking with "attempt to multiply with overflow" in + /// debug builds (found by mutation fuzzing an H.264 transport stream). + #[test] + fn huge_levels_dequantise_without_overflow_panics() { + let mut c = [i32::MAX / 2, i32::MAX / 3, -i32::MAX / 2, 7]; + chroma_dc_dequant_420(&mut c, 51, 16 * 25); + let _ = scale4(i32::MAX, 16 * 25, 51); + let _ = scale4(i32::MAX, 16 * 25, 0); + let _ = scale8(i32::MIN, 16 * 25, 51); + let _ = scale8(i32::MIN, 16 * 25, 0); + } +} diff --git a/crates/video/src/isobmff/bytes.rs b/crates/video/src/isobmff/bytes.rs new file mode 100644 index 00000000..7f59443e --- /dev/null +++ b/crates/video/src/isobmff/bytes.rs @@ -0,0 +1,361 @@ +//! Low-level helpers: four-character codes, a bounds-checked big-endian cursor, box iteration +//! over in-memory payloads, and a box builder for the muxers. + +use crate::isobmff::error::{Error, Result}; +use std::fmt; + +/// A four-character code (box type, sample entry format, brand, handler type…). +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +pub struct FourCc(pub [u8; 4]); + +impl FourCc { + pub const fn new(b: &[u8; 4]) -> Self { + FourCc(*b) + } + pub fn as_u32(self) -> u32 { + u32::from_be_bytes(self.0) + } + /// Lossy string form; `©` (0xA9) is mapped to U+00A9, other non-printables to `?`. + pub fn to_string_lossy(self) -> String { + self.0 + .iter() + .map(|&b| match b { + 0xA9 => '©', + 0x20..=0x7E => b as char, + _ => '?', + }) + .collect() + } +} + +impl fmt::Display for FourCc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_string_lossy()) + } +} +impl fmt::Debug for FourCc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "FourCc({:?})", self.to_string_lossy()) + } +} +impl From<&[u8; 4]> for FourCc { + fn from(b: &[u8; 4]) -> Self { + FourCc(*b) + } +} +impl PartialEq<[u8; 4]> for FourCc { + fn eq(&self, other: &[u8; 4]) -> bool { + &self.0 == other + } +} +impl PartialEq<&[u8; 4]> for FourCc { + fn eq(&self, other: &&[u8; 4]) -> bool { + &self.0 == *other + } +} +impl PartialEq<&&[u8; 4]> for FourCc { + fn eq(&self, other: &&&[u8; 4]) -> bool { + &self.0 == **other + } +} + +/// Bounds-checked big-endian cursor over a byte slice. +#[derive(Clone)] +pub(crate) struct Cur<'a> { + pub data: &'a [u8], + pub pos: usize, + what: &'static str, +} + +impl<'a> Cur<'a> { + pub fn new(data: &'a [u8], what: &'static str) -> Self { + Cur { data, pos: 0, what } + } + pub fn remaining(&self) -> usize { + self.data.len().saturating_sub(self.pos) + } + pub fn rest(&self) -> &'a [u8] { + self.data.get(self.pos..).unwrap_or(&[]) + } + pub fn bytes(&mut self, n: usize) -> Result<&'a [u8]> { + let end = self.pos.checked_add(n).ok_or(Error::Truncated(self.what))?; + let s = self.data.get(self.pos..end).ok_or(Error::Truncated(self.what))?; + self.pos = end; + Ok(s) + } + pub fn skip(&mut self, n: usize) -> Result<()> { + self.bytes(n).map(|_| ()) + } + pub fn array(&mut self) -> Result<[u8; N]> { + let s = self.bytes(N)?; + let mut a = [0u8; N]; + a.copy_from_slice(s); + Ok(a) + } + pub fn u8(&mut self) -> Result { + Ok(self.array::<1>()?[0]) + } + pub fn u16(&mut self) -> Result { + Ok(u16::from_be_bytes(self.array()?)) + } + pub fn i16(&mut self) -> Result { + Ok(i16::from_be_bytes(self.array()?)) + } + pub fn u24(&mut self) -> Result { + let a = self.array::<3>()?; + Ok(((a[0] as u32) << 16) | ((a[1] as u32) << 8) | a[2] as u32) + } + pub fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes(self.array()?)) + } + pub fn i32(&mut self) -> Result { + Ok(i32::from_be_bytes(self.array()?)) + } + pub fn u64(&mut self) -> Result { + Ok(u64::from_be_bytes(self.array()?)) + } + pub fn i64(&mut self) -> Result { + Ok(i64::from_be_bytes(self.array()?)) + } + pub fn f64(&mut self) -> Result { + Ok(f64::from_be_bytes(self.array()?)) + } + pub fn fourcc(&mut self) -> Result { + Ok(FourCc(self.array()?)) + } + /// Full-box header: (version, flags). + pub fn full_header(&mut self) -> Result<(u8, u32)> { + let v = self.u32()?; + Ok(((v >> 24) as u8, v & 0x00FF_FFFF)) + } + /// Read a count and verify that `count * entry_size` bytes remain (allocation guard). + pub fn count(&mut self, entry_size: usize) -> Result { + let n = self.u32()? as usize; + self.check_count(n, entry_size)?; + Ok(n) + } + pub fn check_count(&self, n: usize, entry_size: usize) -> Result<()> { + match n.checked_mul(entry_size) { + Some(b) if b <= self.remaining() => Ok(()), + _ => Err(Error::Truncated(self.what)), + } + } +} + +/// Header of one box as found in a byte stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct BoxHeader { + pub kind: FourCc, + /// Header length including largesize and uuid extended type. + pub header_len: u64, + /// Total box size including header; `None` means "extends to end of the enclosing container". + pub size: Option, + pub uuid: Option<[u8; 16]>, +} + +/// Parse a box header from `buf` (needs up to 32 bytes; fewer if available). +pub(crate) fn parse_box_header(buf: &[u8]) -> Result { + let mut c = Cur::new(buf, "box header"); + let size32 = c.u32()?; + let kind = c.fourcc()?; + let mut header_len = 8u64; + let size = match size32 { + 0 => None, + 1 => { + header_len += 8; + let s = c.u64()?; + Some(s) + } + s => Some(s as u64), + }; + let uuid = if kind == b"uuid" { + header_len += 16; + Some(c.array::<16>()?) + } else { + None + }; + if let Some(s) = size + && s < header_len + { + return Err(Error::Invalid(format!("box '{kind}' size {s} smaller than its header"))); + } + Ok(BoxHeader { kind, header_len, size, uuid }) +} + +/// One child box inside an in-memory payload. +#[derive(Clone, Copy, Debug)] +pub(crate) struct RawBox<'a> { + pub kind: FourCc, + pub payload: &'a [u8], + #[allow(dead_code)] + pub uuid: Option<[u8; 16]>, +} + +/// Iterator over the child boxes of a payload. Yields an error for a malformed box and then stops. +pub(crate) struct Boxes<'a> { + data: &'a [u8], + pos: usize, + done: bool, +} + +pub(crate) fn boxes(data: &[u8]) -> Boxes<'_> { + Boxes { data, pos: 0, done: false } +} + +impl<'a> Iterator for Boxes<'a> { + type Item = Result>; + fn next(&mut self) -> Option { + if self.done { + return None; + } + let rest = &self.data[self.pos..]; + if rest.is_empty() { + self.done = true; + return None; + } + // QuickTime allows a 32-bit zero terminator at the end of some containers. + if rest.len() < 8 { + self.done = true; + if rest.iter().all(|&b| b == 0) { + return None; + } + return Some(Err(Error::Truncated("box header"))); + } + let h = match parse_box_header(rest) { + Ok(h) => h, + Err(e) => { + self.done = true; + return Some(Err(e)); + } + }; + let size = h.size.unwrap_or(rest.len() as u64); + if size > rest.len() as u64 { + self.done = true; + return Some(Err(Error::Invalid(format!("box '{}' size {} exceeds its container ({} bytes left)", h.kind, size, rest.len())))); + } + let size = size as usize; + let payload = &rest[h.header_len as usize..size]; + self.pos += size; + Some(Ok(RawBox { kind: h.kind, payload, uuid: h.uuid })) + } +} + +/// Find the first child of the given type (malformed trailing children are ignored). +pub(crate) fn find<'a>(data: &'a [u8], kind: &[u8; 4]) -> Option<&'a [u8]> { + boxes(data).map_while(|b| b.ok()).find(|b| b.kind == kind).map(|b| b.payload) +} + +/// Growable buffer for building boxes. +#[derive(Default, Clone, Debug)] +pub(crate) struct BoxBuf { + pub buf: Vec, +} + +impl BoxBuf { + pub fn new() -> Self { + Self::default() + } + /// Begin a box; returns a marker to pass to [`end`](Self::end). + pub fn start(&mut self, kind: &[u8; 4]) -> usize { + let at = self.buf.len(); + self.buf.extend_from_slice(&[0, 0, 0, 0]); + self.buf.extend_from_slice(kind); + at + } + pub fn start_full(&mut self, kind: &[u8; 4], version: u8, flags: u32) -> usize { + let at = self.start(kind); + self.u32(((version as u32) << 24) | (flags & 0xFF_FFFF)); + at + } + pub fn end(&mut self, at: usize) { + let size = (self.buf.len() - at) as u32; + self.buf[at..at + 4].copy_from_slice(&size.to_be_bytes()); + } + /// Append a complete box with the given payload. + pub fn leaf(&mut self, kind: &[u8; 4], payload: &[u8]) { + let at = self.start(kind); + self.bytes(payload); + self.end(at); + } + pub fn u8(&mut self, v: u8) { + self.buf.push(v); + } + pub fn u16(&mut self, v: u16) { + self.buf.extend_from_slice(&v.to_be_bytes()); + } + pub fn i16(&mut self, v: i16) { + self.buf.extend_from_slice(&v.to_be_bytes()); + } + pub fn u32(&mut self, v: u32) { + self.buf.extend_from_slice(&v.to_be_bytes()); + } + pub fn i32(&mut self, v: i32) { + self.buf.extend_from_slice(&v.to_be_bytes()); + } + pub fn u64(&mut self, v: u64) { + self.buf.extend_from_slice(&v.to_be_bytes()); + } + pub fn i64(&mut self, v: i64) { + self.buf.extend_from_slice(&v.to_be_bytes()); + } + pub fn f64(&mut self, v: f64) { + self.buf.extend_from_slice(&v.to_be_bytes()); + } + pub fn bytes(&mut self, b: &[u8]) { + self.buf.extend_from_slice(b); + } + pub fn zeros(&mut self, n: usize) { + self.buf.resize(self.buf.len() + n, 0); + } + pub fn len(&self) -> usize { + self.buf.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_forms() { + let h = parse_box_header(&[0, 0, 0, 16, b'f', b'r', b'e', b'e']).unwrap(); + assert_eq!((h.kind, h.header_len, h.size), (FourCc(*b"free"), 8, Some(16))); + let h = parse_box_header(&[0, 0, 0, 0, b'm', b'd', b'a', b't']).unwrap(); + assert_eq!(h.size, None); + let mut v = vec![0, 0, 0, 1, b'm', b'd', b'a', b't']; + v.extend_from_slice(&100u64.to_be_bytes()); + let h = parse_box_header(&v).unwrap(); + assert_eq!((h.header_len, h.size), (16, Some(100))); + let mut v = vec![0, 0, 0, 30, b'u', b'u', b'i', b'd']; + v.extend_from_slice(&[7u8; 16]); + let h = parse_box_header(&v).unwrap(); + assert_eq!((h.header_len, h.uuid), (24, Some([7u8; 16]))); + assert!(parse_box_header(&[0, 0, 0, 4, b'f', b'r', b'e', b'e']).is_err()); + assert!(parse_box_header(&[0, 0, 0]).is_err()); + } + + #[test] + fn iterate_children() { + let mut b = BoxBuf::new(); + b.leaf(b"aaaa", &[1, 2]); + let m = b.start(b"bbbb"); + b.leaf(b"cccc", &[]); + b.end(m); + b.u32(0); // QT terminator + let kids: Vec<_> = boxes(&b.buf).collect::>>().unwrap(); + assert_eq!(kids.len(), 2); + assert_eq!(kids[0].payload, &[1, 2]); + assert_eq!(find(&b.buf, b"bbbb").map(|p| p.len()), Some(8)); + // Oversized child → error, no panic. + let bad = [0u8, 0, 0, 99, b'x', b'x', b'x', b'x', 0]; + assert!(boxes(&bad).next().unwrap().is_err()); + } + + #[test] + fn cursor_bounds() { + let mut c = Cur::new(&[0, 0, 0, 5, 1], "t"); + assert!(c.count(1).is_err()); + let mut c = Cur::new(&[0xFF, 0xFF, 0xFF, 0xFF], "t"); + assert!(c.count(usize::MAX).is_err()); + } +} diff --git a/crates/video/src/isobmff/codec.rs b/crates/video/src/isobmff/codec.rs new file mode 100644 index 00000000..0edb8d5c --- /dev/null +++ b/crates/video/src/isobmff/codec.rs @@ -0,0 +1,1581 @@ +//! Sample entries (`stsd` children) and codec configuration records: parsing and serialization. + +use crate::bitstream::BitReader; +use crate::isobmff::bytes::{BoxBuf, Cur, FourCc, boxes}; +use crate::isobmff::error::{Error, Result}; + +/// One sample description (an entry of `stsd`). +#[derive(Clone, Debug, PartialEq)] +pub struct SampleEntry { + /// Sample entry format (`avc1`, `mp4a`, `apch`, …). + pub format: FourCc, + pub data_reference_index: u16, + pub codec: CodecConfig, + /// Present for visual sample entries. + pub video: Option, + /// Present for audio sample entries. + pub audio: Option, + /// `btrt` box. + pub bitrate: Option, +} + +/// Fields common to all visual sample entries plus the optional descriptive boxes. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct VideoParams { + pub width: u16, + pub height: u16, + /// Horizontal/vertical resolution in pixels per inch (16.16 fixed point; 72 dpi = 0x0048_0000). + pub horiz_resolution: u32, + pub vert_resolution: u32, + pub frame_count: u16, + pub compressor_name: String, + pub depth: u16, + /// QuickTime `vendor` / version fields (zero in ISO files). + pub vendor: FourCc, + pub temporal_quality: u32, + pub spatial_quality: u32, + pub color: Option, + /// `pasp`: (hSpacing, vSpacing). + pub pixel_aspect: Option<(u32, u32)>, + pub clean_aperture: Option, + pub field_info: Option, + /// QuickTime `gama` (16.16 fixed point). + pub gamma: Option, + /// `mdcv`: mastering display colour volume (SMPTE ST 2086). + pub mastering_display: Option, + /// `clli`: content light level (MaxCLL, MaxFALL in cd/m²; 0 = unknown). + pub content_light: Option<(u16, u16)>, +} + +/// SMPTE ST 2086 mastering display metadata, in the units of the `mdcv` box and the H.264/HEVC +/// SEI: chromaticities in 0.00002 steps (order G, B, R as the SEI), luminance in 0.0001 cd/m². +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MasteringDisplay { + pub primaries: [(u16, u16); 3], + pub white_point: (u16, u16), + pub max_luminance: u32, + pub min_luminance: u32, +} + +impl MasteringDisplay { + /// BT.2020 primaries, D65, with the given peak and black in cd/m². + pub fn bt2020(max_nits: f64, min_nits: f64) -> Self { + let q = |v: f64| (v / 0.00002).round() as u16; + MasteringDisplay { + primaries: [(q(0.170), q(0.797)), (q(0.131), q(0.046)), (q(0.708), q(0.292))], + white_point: (q(0.3127), q(0.3290)), + max_luminance: (max_nits * 10_000.0).round() as u32, + min_luminance: (min_nits * 10_000.0).round() as u32, + } + } + pub fn max_nits(&self) -> f64 { + self.max_luminance as f64 / 10_000.0 + } + /// The 24-byte payload shared by `mdcv` and the SEI message. + pub fn to_bytes(&self) -> Vec { + let mut v = Vec::with_capacity(24); + for (x, y) in self.primaries { + v.extend_from_slice(&x.to_be_bytes()); + v.extend_from_slice(&y.to_be_bytes()); + } + v.extend_from_slice(&self.white_point.0.to_be_bytes()); + v.extend_from_slice(&self.white_point.1.to_be_bytes()); + v.extend_from_slice(&self.max_luminance.to_be_bytes()); + v.extend_from_slice(&self.min_luminance.to_be_bytes()); + v + } + pub fn parse(p: &[u8]) -> Option { + if p.len() < 24 { + return None; + } + let u16_at = |i: usize| u16::from_be_bytes([p[i], p[i + 1]]); + let u32_at = |i: usize| u32::from_be_bytes([p[i], p[i + 1], p[i + 2], p[i + 3]]); + Some(MasteringDisplay { + primaries: [(u16_at(0), u16_at(2)), (u16_at(4), u16_at(6)), (u16_at(8), u16_at(10))], + white_point: (u16_at(12), u16_at(14)), + max_luminance: u32_at(16), + min_luminance: u32_at(20), + }) + } +} + +/// `colr` box contents. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ColorInfo { + /// ISO `nclx` (with full-range flag). + Nclx { primaries: u16, transfer: u16, matrix: u16, full_range: bool }, + /// QuickTime `nclc`. + Nclc { primaries: u16, transfer: u16, matrix: u16 }, + /// Embedded ICC profile (`prof` restricted/unrestricted `rICC`). + Icc { kind: FourCc, profile: Vec }, +} + +/// `clap` clean aperture (all rationals num/den; offsets are signed). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CleanAperture { + pub width: (u32, u32), + pub height: (u32, u32), + pub horiz_offset: (i32, u32), + pub vert_offset: (i32, u32), +} + +/// QuickTime `fiel`: field count (1 progressive, 2 interlaced) and ordering detail +/// (0, 1 = top first, 6 = bottom first, 9 / 14 per the QuickTime spec). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FieldInfo { + pub fields: u8, + pub detail: u8, +} + +/// `btrt` box. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BitRate { + pub buffer_size: u32, + pub max_bitrate: u32, + pub avg_bitrate: u32, +} + +/// Fields of audio sample entries (ISO v0 plus QuickTime sound description v1/v2 extensions). +#[derive(Clone, Debug, PartialEq, Default)] +pub struct AudioParams { + /// QuickTime sound description version (0, 1, 2). ISO files use 0. + pub qt_version: u16, + pub channels: u32, + /// Bits per sample from the entry (16 for most compressed formats). + pub sample_size: u16, + pub sample_rate: f64, + pub compression_id: i16, + /// v1 fields. + pub samples_per_packet: u32, + pub bytes_per_packet: u32, + pub bytes_per_frame: u32, + pub bytes_per_sample: u32, + /// v2 fields. + pub const_bits_per_channel: u32, + pub lpcm_flags: u32, + pub const_bytes_per_packet: u32, + pub const_frames_per_packet: u32, + /// `enda` little-endian flag inside a QuickTime `wave` box. + pub little_endian: Option, +} + +/// Codec configuration decoded from the sample entry. +#[derive(Clone, Debug, PartialEq)] +pub enum CodecConfig { + Avc(AvcConfig), + Hevc(HevcConfig), + Av1(Av1Config), + Vp9(VpcConfig), + /// Apple ProRes; fourcc is `apch`/`apcn`/`apcs`/`apco`/`ap4h`/`ap4x`. + ProRes { + fourcc: FourCc, + }, + /// Motion JPEG (`jpeg`, `mjpa`, `mjpb`). + Jpeg { + fourcc: FourCc, + }, + /// Avid DNxHD/DNxHR (`AVdn`, `AVdh`). + Dnx { + fourcc: FourCc, + }, + Aac(AacConfig), + /// MPEG-1/2 audio layer III (esds object type 0x69/0x6B, or QuickTime `.mp3`). + Mp3, + Pcm(PcmConfig), + /// ALAC magic cookie (ALACSpecificConfig, 24 bytes, optionally followed by a channel layout). + Alac { + cookie: Vec, + }, + Opus(OpusConfig), + Flac(FlacConfig), + /// `dac3` payload. + Ac3 { + dac3: Vec, + }, + /// `dec3` payload. + Eac3 { + dec3: Vec, + }, + Timecode(TimecodeConfig), + /// Anything else; `raw` is the full sample entry payload (after the 8-byte box header). + Unknown { + fourcc: FourCc, + raw: Vec, + }, +} + +impl CodecConfig { + /// A short human-readable codec name. + pub fn name(&self) -> &'static str { + match self { + CodecConfig::Avc(_) => "h264", + CodecConfig::Hevc(_) => "hevc", + CodecConfig::Av1(_) => "av1", + CodecConfig::Vp9(_) => "vp9", + CodecConfig::ProRes { .. } => "prores", + CodecConfig::Jpeg { .. } => "mjpeg", + CodecConfig::Dnx { .. } => "dnxhd", + CodecConfig::Aac(_) => "aac", + CodecConfig::Mp3 => "mp3", + CodecConfig::Pcm(_) => "pcm", + CodecConfig::Alac { .. } => "alac", + CodecConfig::Opus(_) => "opus", + CodecConfig::Flac(_) => "flac", + CodecConfig::Ac3 { .. } => "ac3", + CodecConfig::Eac3 { .. } => "eac3", + CodecConfig::Timecode(_) => "timecode", + CodecConfig::Unknown { .. } => "unknown", + } + } +} + +/// `avcC` — AVCDecoderConfigurationRecord (ISO/IEC 14496-15 §5.3.3). +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct AvcConfig { + pub profile: u8, + pub compatibility: u8, + pub level: u8, + /// NAL length field size in bytes (1, 2 or 4). + pub length_size: u8, + pub sps: Vec>, + pub pps: Vec>, + /// Bytes after the PPS list (high-profile extension: chroma format, bit depths, SPS-ext). + pub ext: Vec, +} + +impl AvcConfig { + /// Build from parameter sets (NAL units without start codes); profile/level taken from the first SPS. + pub fn new(sps: Vec>, pps: Vec>, length_size: u8) -> Self { + let (profile, compatibility, level) = sps.first().filter(|s| s.len() >= 4).map(|s| (s[1], s[2], s[3])).unwrap_or((0, 0, 0)); + AvcConfig { profile, compatibility, level, length_size, sps, pps, ext: Vec::new() } + } + + pub fn parse(data: &[u8]) -> Result { + let mut c = Cur::new(data, "avcC"); + let _version = c.u8()?; + let profile = c.u8()?; + let compatibility = c.u8()?; + let level = c.u8()?; + let length_size = (c.u8()? & 3) + 1; + let nsps = (c.u8()? & 0x1F) as usize; + let mut sps = Vec::with_capacity(nsps); + for _ in 0..nsps { + let n = c.u16()? as usize; + sps.push(c.bytes(n)?.to_vec()); + } + let npps = c.u8()? as usize; + let mut pps = Vec::with_capacity(npps); + for _ in 0..npps { + let n = c.u16()? as usize; + pps.push(c.bytes(n)?.to_vec()); + } + Ok(AvcConfig { profile, compatibility, level, length_size, sps, pps, ext: c.rest().to_vec() }) + } + + pub fn to_bytes(&self) -> Vec { + let mut b = BoxBuf::new(); + b.u8(1); + b.u8(self.profile); + b.u8(self.compatibility); + b.u8(self.level); + b.u8(0xFC | (self.length_size.clamp(1, 4) - 1)); + b.u8(0xE0 | (self.sps.len().min(31) as u8)); + for s in self.sps.iter().take(31) { + b.u16(s.len() as u16); + b.bytes(s); + } + b.u8(self.pps.len().min(255) as u8); + for p in self.pps.iter().take(255) { + b.u16(p.len() as u16); + b.bytes(p); + } + b.bytes(&self.ext); + b.buf + } +} + +/// One NAL array of an `hvcC` record. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct HevcNalArray { + pub completeness: bool, + pub nal_type: u8, + pub nalus: Vec>, +} + +/// `hvcC` — HEVCDecoderConfigurationRecord (ISO/IEC 14496-15 §8.3.3). +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct HevcConfig { + pub general_profile_space: u8, + pub general_tier_flag: bool, + pub general_profile_idc: u8, + pub general_profile_compatibility_flags: u32, + /// 48 bits of general constraint indicator flags. + pub general_constraint_indicator_flags: u64, + pub general_level_idc: u8, + pub min_spatial_segmentation_idc: u16, + pub parallelism_type: u8, + pub chroma_format_idc: u8, + pub bit_depth_luma: u8, + pub bit_depth_chroma: u8, + pub avg_frame_rate: u16, + pub constant_frame_rate: u8, + pub num_temporal_layers: u8, + pub temporal_id_nested: bool, + pub length_size: u8, + pub arrays: Vec, +} + +impl HevcConfig { + pub fn parse(data: &[u8]) -> Result { + let mut c = Cur::new(data, "hvcC"); + let _version = c.u8()?; + let b = c.u8()?; + let compat = c.u32()?; + let hi = c.u16()? as u64; + let lo = c.u32()? as u64; + let level = c.u8()?; + let min_seg = c.u16()? & 0x0FFF; + let par = c.u8()? & 3; + let chroma = c.u8()? & 3; + let bdl = (c.u8()? & 7) + 8; + let bdc = (c.u8()? & 7) + 8; + let afr = c.u16()?; + let x = c.u8()?; + let narrays = c.u8()? as usize; + let mut arrays = Vec::new(); + for _ in 0..narrays { + let t = c.u8()?; + let n = c.u16()? as usize; + c.check_count(n, 2)?; + let mut nalus = Vec::with_capacity(n); + for _ in 0..n { + let len = c.u16()? as usize; + nalus.push(c.bytes(len)?.to_vec()); + } + arrays.push(HevcNalArray { completeness: t & 0x80 != 0, nal_type: t & 0x3F, nalus }); + } + Ok(HevcConfig { + general_profile_space: b >> 6, + general_tier_flag: b & 0x20 != 0, + general_profile_idc: b & 0x1F, + general_profile_compatibility_flags: compat, + general_constraint_indicator_flags: (hi << 32) | lo, + general_level_idc: level, + min_spatial_segmentation_idc: min_seg, + parallelism_type: par, + chroma_format_idc: chroma, + bit_depth_luma: bdl, + bit_depth_chroma: bdc, + avg_frame_rate: afr, + constant_frame_rate: x >> 6, + num_temporal_layers: (x >> 3) & 7, + temporal_id_nested: x & 4 != 0, + length_size: (x & 3) + 1, + arrays, + }) + } + + pub fn to_bytes(&self) -> Vec { + let mut b = BoxBuf::new(); + b.u8(1); + b.u8((self.general_profile_space << 6) | ((self.general_tier_flag as u8) << 5) | (self.general_profile_idc & 0x1F)); + b.u32(self.general_profile_compatibility_flags); + b.u16((self.general_constraint_indicator_flags >> 32) as u16); + b.u32(self.general_constraint_indicator_flags as u32); + b.u8(self.general_level_idc); + b.u16(0xF000 | (self.min_spatial_segmentation_idc & 0x0FFF)); + b.u8(0xFC | (self.parallelism_type & 3)); + b.u8(0xFC | (self.chroma_format_idc & 3)); + b.u8(0xF8 | (self.bit_depth_luma.saturating_sub(8) & 7)); + b.u8(0xF8 | (self.bit_depth_chroma.saturating_sub(8) & 7)); + b.u16(self.avg_frame_rate); + b.u8((self.constant_frame_rate << 6) + | ((self.num_temporal_layers & 7) << 3) + | ((self.temporal_id_nested as u8) << 2) + | (self.length_size.clamp(1, 4) - 1)); + b.u8(self.arrays.len() as u8); + for a in &self.arrays { + b.u8(((a.completeness as u8) << 7) | (a.nal_type & 0x3F)); + b.u16(a.nalus.len() as u16); + for n in &a.nalus { + b.u16(n.len() as u16); + b.bytes(n); + } + } + b.buf + } + + fn nalus_of(&self, t: u8) -> Vec<&[u8]> { + self.arrays.iter().filter(|a| a.nal_type == t).flat_map(|a| a.nalus.iter().map(|n| n.as_slice())).collect() + } + pub fn vps(&self) -> Vec<&[u8]> { + self.nalus_of(32) + } + pub fn sps(&self) -> Vec<&[u8]> { + self.nalus_of(33) + } + pub fn pps(&self) -> Vec<&[u8]> { + self.nalus_of(34) + } +} + +/// `av1C` — AV1CodecConfigurationRecord. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct Av1Config { + pub seq_profile: u8, + pub seq_level_idx_0: u8, + pub seq_tier_0: bool, + pub high_bitdepth: bool, + pub twelve_bit: bool, + pub monochrome: bool, + pub chroma_subsampling_x: bool, + pub chroma_subsampling_y: bool, + pub chroma_sample_position: u8, + pub initial_presentation_delay_minus_one: Option, + pub config_obus: Vec, +} + +impl Av1Config { + pub fn parse(data: &[u8]) -> Result { + let mut c = Cur::new(data, "av1C"); + let m = c.u8()?; + if m & 0x80 == 0 { + return Err(Error::Invalid("av1C marker bit not set".into())); + } + let a = c.u8()?; + let b = c.u8()?; + let d = c.u8()?; + Ok(Av1Config { + seq_profile: a >> 5, + seq_level_idx_0: a & 0x1F, + seq_tier_0: b & 0x80 != 0, + high_bitdepth: b & 0x40 != 0, + twelve_bit: b & 0x20 != 0, + monochrome: b & 0x10 != 0, + chroma_subsampling_x: b & 0x08 != 0, + chroma_subsampling_y: b & 0x04 != 0, + chroma_sample_position: b & 3, + initial_presentation_delay_minus_one: if d & 0x10 != 0 { Some(d & 0x0F) } else { None }, + config_obus: c.rest().to_vec(), + }) + } + pub fn to_bytes(&self) -> Vec { + let mut b = BoxBuf::new(); + b.u8(0x81); + b.u8((self.seq_profile << 5) | (self.seq_level_idx_0 & 0x1F)); + b.u8(((self.seq_tier_0 as u8) << 7) + | ((self.high_bitdepth as u8) << 6) + | ((self.twelve_bit as u8) << 5) + | ((self.monochrome as u8) << 4) + | ((self.chroma_subsampling_x as u8) << 3) + | ((self.chroma_subsampling_y as u8) << 2) + | (self.chroma_sample_position & 3)); + b.u8(match self.initial_presentation_delay_minus_one { + Some(d) => 0x10 | (d & 0x0F), + None => 0, + }); + b.bytes(&self.config_obus); + b.buf + } +} + +/// `vpcC` — VP codec configuration (version 1). +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct VpcConfig { + pub profile: u8, + pub level: u8, + pub bit_depth: u8, + pub chroma_subsampling: u8, + pub full_range: bool, + pub colour_primaries: u8, + pub transfer_characteristics: u8, + pub matrix_coefficients: u8, + pub codec_init: Vec, +} + +impl VpcConfig { + pub fn parse(data: &[u8]) -> Result { + let mut c = Cur::new(data, "vpcC"); + let (version, _) = c.full_header()?; + if version == 0 { + // Version 0 draft layout: profile, level, bitDepth(4)|colorSpace(4), chroma(4)|transfer(3)|range(1) + let profile = c.u8()?; + let level = c.u8()?; + let a = c.u8()?; + let b = c.u8()?; + return Ok(VpcConfig { profile, level, bit_depth: a >> 4, chroma_subsampling: b >> 4, full_range: b & 1 != 0, ..Default::default() }); + } + let profile = c.u8()?; + let level = c.u8()?; + let a = c.u8()?; + let colour_primaries = c.u8()?; + let transfer_characteristics = c.u8()?; + let matrix_coefficients = c.u8()?; + let n = c.u16()? as usize; + let codec_init = c.bytes(n)?.to_vec(); + Ok(VpcConfig { + profile, + level, + bit_depth: a >> 4, + chroma_subsampling: (a >> 1) & 7, + full_range: a & 1 != 0, + colour_primaries, + transfer_characteristics, + matrix_coefficients, + codec_init, + }) + } + pub fn to_bytes(&self) -> Vec { + let mut b = BoxBuf::new(); + b.u32(1 << 24); + b.u8(self.profile); + b.u8(self.level); + b.u8((self.bit_depth << 4) | ((self.chroma_subsampling & 7) << 1) | self.full_range as u8); + b.u8(self.colour_primaries); + b.u8(self.transfer_characteristics); + b.u8(self.matrix_coefficients); + b.u16(self.codec_init.len() as u16); + b.bytes(&self.codec_init); + b.buf + } +} + +/// AAC (MPEG-4 audio) configuration from `esds`. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct AacConfig { + /// AudioSpecificConfig bytes (DecoderSpecificInfo). + pub asc: Vec, + /// Audio object type (2 = AAC-LC, 5 = SBR/HE-AAC, 29 = PS …) as signalled first in the ASC. + pub object_type: u8, + /// Sampling rate signalled by the ASC (core rate; 0 if the ASC is missing). + pub sample_rate: u32, + /// Channel configuration from the ASC (0 = defined elsewhere). + pub channel_config: u8, + /// DecoderConfigDescriptor fields. + pub max_bitrate: u32, + pub avg_bitrate: u32, + pub buffer_size: u32, +} + +const AAC_RATES: [u32; 13] = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350]; + +impl AacConfig { + /// Build from an AudioSpecificConfig, decoding its header fields. + pub fn from_asc(asc: Vec) -> Self { + let (object_type, sample_rate, channel_config) = parse_asc(&asc).unwrap_or((0, 0, 0)); + AacConfig { asc, object_type, sample_rate, channel_config, max_bitrate: 0, avg_bitrate: 0, buffer_size: 0 } + } + /// Channel count implied by the channel configuration (0 if unspecified). + pub fn channels(&self) -> u32 { + match self.channel_config { + 1..=6 => self.channel_config as u32, + 7 => 8, + _ => 0, + } + } +} + +/// Decode (audio object type, sampling frequency, channel configuration) from an AudioSpecificConfig. +pub fn parse_asc(asc: &[u8]) -> Option<(u8, u32, u8)> { + let mut r = BitReader::new(asc); + let aot = |r: &mut BitReader| -> Option { + let t = r.read_bits(5).ok()? as u8; + if t == 31 { Some(32 + r.read_bits(6).ok()? as u8) } else { Some(t) } + }; + let rate = |r: &mut BitReader| -> Option { + let i = r.read_bits(4).ok()? as usize; + if i == 15 { r.read_bits(24).ok() } else { AAC_RATES.get(i).copied() } + }; + let t = aot(&mut r)?; + let sr = rate(&mut r)?; + let ch = r.read_bits(4).ok()? as u8; + Some((t, sr, ch)) +} + +/// Uncompressed PCM description. +#[derive(Clone, Copy, Debug, PartialEq, Default)] +pub struct PcmConfig { + pub bits: u16, + pub float: bool, + pub big_endian: bool, + /// Signed integer samples (8-bit `raw ` is unsigned). + pub signed: bool, + pub channels: u32, + pub sample_rate: f64, +} + +impl PcmConfig { + /// Bytes per interleaved audio frame (all channels). + pub fn bytes_per_frame(&self) -> u32 { + (self.bits as u32).div_ceil(8) * self.channels.max(1) + } +} + +/// `dOps` — Opus specific box. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct OpusConfig { + pub output_channels: u8, + pub pre_skip: u16, + pub input_sample_rate: u32, + pub output_gain: i16, + pub channel_mapping_family: u8, + /// StreamCount, CoupledCount and ChannelMapping bytes when family != 0. + pub channel_mapping: Vec, +} + +impl OpusConfig { + pub fn parse(data: &[u8]) -> Result { + let mut c = Cur::new(data, "dOps"); + let _version = c.u8()?; + Ok(OpusConfig { + output_channels: c.u8()?, + pre_skip: c.u16()?, + input_sample_rate: c.u32()?, + output_gain: c.i16()?, + channel_mapping_family: c.u8()?, + channel_mapping: c.rest().to_vec(), + }) + } + pub fn to_bytes(&self) -> Vec { + let mut b = BoxBuf::new(); + b.u8(0); + b.u8(self.output_channels); + b.u16(self.pre_skip); + b.u32(self.input_sample_rate); + b.i16(self.output_gain); + b.u8(self.channel_mapping_family); + b.bytes(&self.channel_mapping); + b.buf + } +} + +/// `dfLa` — FLAC metadata blocks, with STREAMINFO decoded. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct FlacConfig { + /// Raw metadata blocks (each with its 4-byte header) as stored in `dfLa`. + pub metadata_blocks: Vec, + pub sample_rate: u32, + pub channels: u8, + pub bits_per_sample: u8, + pub total_samples: u64, +} + +impl FlacConfig { + pub fn parse(data: &[u8]) -> Result { + let mut c = Cur::new(data, "dfLa"); + c.full_header()?; + let blocks = c.rest().to_vec(); + let mut cfg = FlacConfig { metadata_blocks: blocks.clone(), ..Default::default() }; + let mut b = Cur::new(&blocks, "dfLa"); + let hdr = b.u32()?; + if (hdr >> 24) & 0x7F == 0 { + let si = b.bytes(((hdr & 0xFF_FFFF) as usize).min(34))?; + if si.len() >= 18 { + let mut r = BitReader::new(&si[10..18]); + cfg.sample_rate = r.read_bits(20).unwrap_or(0); + cfg.channels = r.read_bits(3).unwrap_or(0) as u8 + 1; + cfg.bits_per_sample = r.read_bits(5).unwrap_or(0) as u8 + 1; + cfg.total_samples = r.read_bits_u64(36).unwrap_or(0); + } + } + Ok(cfg) + } +} + +/// QuickTime timecode sample description (`tmcd`) plus the start frame read from its first sample. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct TimecodeConfig { + /// Bit 0 drop frame, bit 1 24-hour max, bit 2 negative times OK, bit 3 counter. + pub flags: u32, + pub timescale: u32, + pub frame_duration: u32, + /// Nominal frames per second (`numberOfFrames`), e.g. 30 for 29.97. + pub frames_per_second: u8, + /// Frame number of the first timecode sample (filled in by the demuxer). + pub start_frame: Option, + /// Source reel name (`name` box) if present. + pub name: Option, +} + +impl TimecodeConfig { + pub fn drop_frame(&self) -> bool { + self.flags & 1 != 0 + } + /// Format a frame number as `HH:MM:SS:FF` (`;` before the frames when drop-frame). + pub fn format_frame(&self, frame: u32) -> String { + let fps = self.frames_per_second.max(1) as u64; + let mut f = frame as u64; + if self.drop_frame() && fps >= 30 { + let drop = (fps / 30) * 2; // 2 frames per minute at 30, 4 at 60 + let per_min = fps * 60 - drop; + let per_10 = per_min * 10 + drop; + let d = f / per_10; + let m = f % per_10; + f += drop * 9 * d + if m > drop { drop * ((m - drop) / per_min) } else { 0 }; + } + let ff = f % fps; + let s = (f / fps) % 60; + let mi = (f / fps / 60) % 60; + let h = (f / fps / 3600) % 24; + let sep = if self.drop_frame() { ';' } else { ':' }; + format!("{h:02}:{mi:02}:{s:02}{sep}{ff:02}") + } +} + +// --------------------------------------------------------------------------------------------- +// Parsing + +fn is_video_fourcc(f: &[u8; 4]) -> bool { + matches!( + f, + b"avc1" + | b"avc3" + | b"hvc1" + | b"hev1" + | b"av01" + | b"vp09" + | b"vp08" + | b"apch" + | b"apcn" + | b"apcs" + | b"apco" + | b"ap4h" + | b"ap4x" + | b"jpeg" + | b"mjpa" + | b"mjpb" + | b"AVdn" + | b"AVdh" + | b"mp4v" + | b"encv" + | b"raw " + | b"2vuy" + | b"v210" + ) +} + +fn is_audio_fourcc(f: &[u8; 4]) -> bool { + matches!( + f, + b"mp4a" + | b"lpcm" + | b"sowt" + | b"twos" + | b"in24" + | b"in32" + | b"fl32" + | b"fl64" + | b"ipcm" + | b"fpcm" + | b"alac" + | b"Opus" + | b"fLaC" + | b"ac-3" + | b"ec-3" + | b".mp3" + | b"enca" + | b"ulaw" + | b"alaw" + | b"NONE" + ) +} + +/// Parse one `stsd` entry. `handler` is the track handler type; `is_qt` selects QuickTime +/// interpretation of the sound description version field. +pub(crate) fn parse_sample_entry(format: FourCc, payload: &[u8], handler: FourCc, is_qt: bool) -> Result { + let mut c = Cur::new(payload, "sample entry"); + c.skip(6)?; + let data_reference_index = c.u16()?; + let f = &format.0; + let unknown = || CodecConfig::Unknown { fourcc: format, raw: payload.to_vec() }; + if f == b"tmcd" || (handler == b"tmcd" && !is_video_fourcc(f)) { + let mut tc = TimecodeConfig::default(); + c.skip(4)?; + tc.flags = c.u32()?; + tc.timescale = c.u32()?; + tc.frame_duration = c.u32()?; + tc.frames_per_second = c.u8()?; + c.skip(1)?; + for b in boxes(c.rest()).map_while(|b| b.ok()) { + if b.kind == b"name" && b.payload.len() >= 4 { + let n = u16::from_be_bytes([b.payload[0], b.payload[1]]) as usize; + let s = b.payload.get(4..4 + n).unwrap_or(&b.payload[4..]); + tc.name = Some(String::from_utf8_lossy(s).into_owned()); + } + } + return Ok(SampleEntry { format, data_reference_index, codec: CodecConfig::Timecode(tc), video: None, audio: None, bitrate: None }); + } + let video_like = is_video_fourcc(f) || (handler == b"vide" && !is_audio_fourcc(f)); + let audio_like = is_audio_fourcc(f) || (handler == b"soun" && !video_like); + if video_like { + let mut v = VideoParams::default(); + let _version = c.u16()?; + let _revision = c.u16()?; + v.vendor = c.fourcc()?; + v.temporal_quality = c.u32()?; + v.spatial_quality = c.u32()?; + v.width = c.u16()?; + v.height = c.u16()?; + v.horiz_resolution = c.u32()?; + v.vert_resolution = c.u32()?; + c.skip(4)?; + v.frame_count = c.u16()?; + let name = c.bytes(32)?; + let n = (name[0] as usize).min(31); + v.compressor_name = String::from_utf8_lossy(&name[1..1 + n]).into_owned(); + v.depth = c.u16()?; + let _ctab = c.i16()?; + let mut codec = None; + let mut bitrate = None; + for b in boxes(c.rest()).map_while(|b| b.ok()) { + let p = b.payload; + match &b.kind.0 { + b"avcC" if matches!(f, b"avc1" | b"avc3") => codec = Some(CodecConfig::Avc(AvcConfig::parse(p)?)), + b"hvcC" if matches!(f, b"hvc1" | b"hev1") => codec = Some(CodecConfig::Hevc(HevcConfig::parse(p)?)), + b"av1C" if f == b"av01" => codec = Some(CodecConfig::Av1(Av1Config::parse(p)?)), + b"vpcC" if f == b"vp09" => codec = Some(CodecConfig::Vp9(VpcConfig::parse(p)?)), + b"colr" => v.color = parse_colr(p).ok(), + b"mdcv" => v.mastering_display = MasteringDisplay::parse(p), + b"clli" if p.len() >= 4 => v.content_light = Some((u16::from_be_bytes([p[0], p[1]]), u16::from_be_bytes([p[2], p[3]]))), + b"pasp" => { + let mut c = Cur::new(p, "pasp"); + v.pixel_aspect = Some((c.u32()?, c.u32()?)); + } + b"clap" => { + let mut c = Cur::new(p, "clap"); + v.clean_aperture = Some(CleanAperture { + width: (c.u32()?, c.u32()?), + height: (c.u32()?, c.u32()?), + horiz_offset: (c.i32()?, c.u32()?), + vert_offset: (c.i32()?, c.u32()?), + }); + } + b"fiel" => { + let mut c = Cur::new(p, "fiel"); + v.field_info = Some(FieldInfo { fields: c.u8()?, detail: c.u8()? }); + } + b"gama" => v.gamma = Some(Cur::new(p, "gama").u32()?), + b"btrt" => bitrate = Some(parse_btrt(p)?), + _ => {} + } + } + let codec = match codec { + Some(c) => c, + None => match f { + b"apch" | b"apcn" | b"apcs" | b"apco" | b"ap4h" | b"ap4x" => CodecConfig::ProRes { fourcc: format }, + b"jpeg" | b"mjpa" | b"mjpb" => CodecConfig::Jpeg { fourcc: format }, + b"AVdn" | b"AVdh" => CodecConfig::Dnx { fourcc: format }, + _ => unknown(), + }, + }; + return Ok(SampleEntry { format, data_reference_index, codec, video: Some(v), audio: None, bitrate }); + } + if audio_like { + let mut a = AudioParams { qt_version: c.u16()?, ..Default::default() }; + let _revision = c.u16()?; + let _vendor = c.u32()?; + a.channels = c.u16()? as u32; + a.sample_size = c.u16()?; + a.compression_id = c.i16()?; + let _packet_size = c.u16()?; + a.sample_rate = c.u32()? as f64 / 65536.0; + if is_qt && a.qt_version == 1 { + a.samples_per_packet = c.u32()?; + a.bytes_per_packet = c.u32()?; + a.bytes_per_frame = c.u32()?; + a.bytes_per_sample = c.u32()?; + } else if is_qt && a.qt_version == 2 { + let _size_of_struct = c.u32()?; + a.sample_rate = c.f64()?; + a.channels = c.u32()?; + let _always = c.u32()?; + a.const_bits_per_channel = c.u32()?; + a.lpcm_flags = c.u32()?; + a.const_bytes_per_packet = c.u32()?; + a.const_frames_per_packet = c.u32()?; + } + // Flatten QuickTime `wave` children into the list of extension boxes. + let mut ext: Vec<(FourCc, &[u8])> = Vec::new(); + for b in boxes(c.rest()).map_while(|b| b.ok()) { + if b.kind == b"wave" { + for w in boxes(b.payload).map_while(|b| b.ok()) { + ext.push((w.kind, w.payload)); + } + } else { + ext.push((b.kind, b.payload)); + } + } + let get = |k: &[u8; 4]| ext.iter().find(|(kk, _)| kk == k).map(|(_, p)| *p); + if let Some(p) = get(b"enda") { + let v = if p.len() >= 2 { u16::from_be_bytes([p[0], p[1]]) } else { p.first().copied().unwrap_or(0) as u16 }; + a.little_endian = Some(v != 0); + } + let bitrate = get(b"btrt").and_then(|p| parse_btrt(p).ok()); + let codec = match f { + b"mp4a" => match get(b"esds") { + Some(p) => parse_esds(p)?, + None => unknown(), + }, + b".mp3" => CodecConfig::Mp3, + b"alac" => match get(b"alac") { + Some(p) if p.len() >= 4 => CodecConfig::Alac { cookie: p[4..].to_vec() }, + _ => unknown(), + }, + b"Opus" => match get(b"dOps") { + Some(p) => CodecConfig::Opus(OpusConfig::parse(p)?), + None => unknown(), + }, + b"fLaC" => match get(b"dfLa") { + Some(p) => CodecConfig::Flac(FlacConfig::parse(p)?), + None => unknown(), + }, + b"ac-3" => CodecConfig::Ac3 { dac3: get(b"dac3").unwrap_or(&[]).to_vec() }, + b"ec-3" => CodecConfig::Eac3 { dec3: get(b"dec3").unwrap_or(&[]).to_vec() }, + b"lpcm" | b"sowt" | b"twos" | b"in24" | b"in32" | b"fl32" | b"fl64" | b"raw " | b"NONE" => CodecConfig::Pcm(qt_pcm(f, &a)), + b"ipcm" | b"fpcm" => { + let mut pcm = PcmConfig { + bits: a.sample_size, + float: f == b"fpcm", + big_endian: true, + signed: true, + channels: a.channels, + sample_rate: a.sample_rate, + }; + if let Some(p) = get(b"pcmC") { + let mut c = Cur::new(p, "pcmC"); + c.full_header()?; + let flags = c.u8()?; + pcm.big_endian = flags & 1 == 0; + pcm.bits = c.u8()? as u16; + } + CodecConfig::Pcm(pcm) + } + _ => unknown(), + }; + return Ok(SampleEntry { format, data_reference_index, codec, video: None, audio: Some(a), bitrate }); + } + Ok(SampleEntry { format, data_reference_index, codec: unknown(), video: None, audio: None, bitrate: None }) +} + +fn qt_pcm(f: &[u8; 4], a: &AudioParams) -> PcmConfig { + let mut p = PcmConfig { channels: a.channels, sample_rate: a.sample_rate, signed: true, ..Default::default() }; + match f { + b"lpcm" => { + p.bits = a.const_bits_per_channel as u16; + p.float = a.lpcm_flags & 1 != 0; + p.big_endian = a.lpcm_flags & 2 != 0; + p.signed = a.lpcm_flags & 4 != 0 || p.float; + if p.bits == 0 { + p.bits = a.sample_size; + } + } + b"sowt" => { + p.bits = if a.sample_size == 0 { 16 } else { a.sample_size }; + } + b"twos" => { + p.bits = if a.sample_size == 0 { 16 } else { a.sample_size }; + p.big_endian = true; + } + b"raw " | b"NONE" => { + p.bits = if a.sample_size == 0 { 8 } else { a.sample_size }; + p.signed = p.bits != 8; + p.big_endian = true; + } + b"in24" | b"in32" | b"fl32" | b"fl64" => { + p.bits = match f { + b"in24" => 24, + b"in32" | b"fl32" => 32, + _ => 64, + }; + p.float = f[0] == b'f'; + p.big_endian = !a.little_endian.unwrap_or(false); + } + _ => {} + } + p +} + +fn parse_btrt(p: &[u8]) -> Result { + let mut c = Cur::new(p, "btrt"); + Ok(BitRate { buffer_size: c.u32()?, max_bitrate: c.u32()?, avg_bitrate: c.u32()? }) +} + +fn parse_colr(p: &[u8]) -> Result { + let mut c = Cur::new(p, "colr"); + let kind = c.fourcc()?; + Ok(match &kind.0 { + b"nclx" => { + ColorInfo::Nclx { primaries: c.u16()?, transfer: c.u16()?, matrix: c.u16()?, full_range: c.u8().map(|v| v & 0x80 != 0).unwrap_or(false) } + } + b"nclc" => ColorInfo::Nclc { primaries: c.u16()?, transfer: c.u16()?, matrix: c.u16()? }, + _ => ColorInfo::Icc { kind, profile: c.rest().to_vec() }, + }) +} + +fn descriptor(c: &mut Cur) -> Result<(u8, usize)> { + let tag = c.u8()?; + let mut len = 0usize; + for _ in 0..4 { + let b = c.u8()?; + len = (len << 7) | (b & 0x7F) as usize; + if b & 0x80 == 0 { + break; + } + } + Ok((tag, len)) +} + +fn parse_esds(p: &[u8]) -> Result { + let mut c = Cur::new(p, "esds"); + c.full_header()?; + let (tag, len) = descriptor(&mut c)?; + if tag != 3 { + return Err(Error::Invalid(format!("esds: expected ES_Descriptor, got tag {tag}"))); + } + let es = c.bytes(len.min(c.remaining()))?; + let mut c = Cur::new(es, "ES_Descriptor"); + c.u16()?; + let fl = c.u8()?; + if fl & 0x80 != 0 { + c.u16()?; + } + if fl & 0x40 != 0 { + let n = c.u8()? as usize; + c.skip(n)?; + } + if fl & 0x20 != 0 { + c.u16()?; + } + while c.remaining() > 0 { + let (tag, len) = descriptor(&mut c)?; + let body = c.bytes(len.min(c.remaining()))?; + if tag != 4 { + continue; + } + let mut d = Cur::new(body, "DecoderConfigDescriptor"); + let oti = d.u8()?; + let _stream_type = d.u8()?; + let buffer_size = d.u24()?; + let max_bitrate = d.u32()?; + let avg_bitrate = d.u32()?; + let mut asc = Vec::new(); + while d.remaining() > 0 { + let (t, l) = descriptor(&mut d)?; + let b = d.bytes(l.min(d.remaining()))?; + if t == 5 { + asc = b.to_vec(); + break; + } + } + return Ok(match oti { + 0x69 | 0x6B => CodecConfig::Mp3, + _ => { + let mut a = AacConfig::from_asc(asc); + a.max_bitrate = max_bitrate; + a.avg_bitrate = avg_bitrate; + a.buffer_size = buffer_size; + CodecConfig::Aac(a) + } + }); + } + Err(Error::Invalid("esds without DecoderConfigDescriptor".into())) +} + +// --------------------------------------------------------------------------------------------- +// Serialization (muxer) + +fn put_descriptor(b: &mut BoxBuf, tag: u8, body: &[u8]) { + b.u8(tag); + let n = body.len() as u32; + // Always use the 4-byte length form for simplicity (as many encoders do). + b.u8(0x80 | ((n >> 21) & 0x7F) as u8); + b.u8(0x80 | ((n >> 14) & 0x7F) as u8); + b.u8(0x80 | ((n >> 7) & 0x7F) as u8); + b.u8((n & 0x7F) as u8); + b.bytes(body); +} + +pub(crate) fn esds_payload(oti: u8, asc: &[u8], max_bitrate: u32, avg_bitrate: u32, buffer_size: u32) -> Vec { + let mut dcd = BoxBuf::new(); + dcd.u8(oti); + dcd.u8(0x15); // streamType audio (5) << 2 | upstream 0 | reserved 1 + dcd.u8((buffer_size >> 16) as u8); + dcd.u16(buffer_size as u16); + dcd.u32(max_bitrate); + dcd.u32(avg_bitrate); + if !asc.is_empty() { + put_descriptor(&mut dcd, 5, asc); + } + let mut es = BoxBuf::new(); + es.u16(1); // ES_ID + es.u8(0); + put_descriptor(&mut es, 4, &dcd.buf); + put_descriptor(&mut es, 6, &[2]); // SLConfigDescriptor, predefined = 2 (MP4) + let mut b = BoxBuf::new(); + b.u32(0); + put_descriptor(&mut b, 3, &es.buf); + b.buf +} + +impl SampleEntry { + /// Convenience constructor for a video entry. + pub fn video(format: FourCc, codec: CodecConfig, width: u16, height: u16) -> Self { + let compressor_name = match &codec { + CodecConfig::ProRes { fourcc } => match &fourcc.0 { + b"apch" => "Apple ProRes 422 HQ", + b"apcn" => "Apple ProRes 422", + b"apcs" => "Apple ProRes 422 LT", + b"apco" => "Apple ProRes 422 Proxy", + b"ap4h" => "Apple ProRes 4444", + _ => "Apple ProRes 4444 XQ", + }, + CodecConfig::Jpeg { .. } => "Photo - JPEG", + _ => "", + }; + SampleEntry { + format, + data_reference_index: 1, + codec, + video: Some(VideoParams { + width, + height, + horiz_resolution: 0x0048_0000, + vert_resolution: 0x0048_0000, + frame_count: 1, + compressor_name: compressor_name.into(), + depth: 24, + ..Default::default() + }), + audio: None, + bitrate: None, + } + } + + /// Convenience constructor for an audio entry (ISO v0 layout; the muxer upgrades PCM in MOV to v2). + pub fn audio(format: FourCc, codec: CodecConfig, channels: u32, sample_rate: f64, sample_size: u16) -> Self { + SampleEntry { + format, + data_reference_index: 1, + codec, + video: None, + audio: Some(AudioParams { channels, sample_rate, sample_size, ..Default::default() }), + bitrate: None, + } + } + + /// H.264 entry (`avc1`). + pub fn avc(cfg: AvcConfig, width: u16, height: u16) -> Self { + Self::video(FourCc(*b"avc1"), CodecConfig::Avc(cfg), width, height) + } + /// HEVC entry (`hvc1`, parameter sets in the config only). + pub fn hevc(cfg: HevcConfig, width: u16, height: u16) -> Self { + Self::video(FourCc(*b"hvc1"), CodecConfig::Hevc(cfg), width, height) + } + /// ProRes entry; `fourcc` one of `apch`/`apcn`/`apcs`/`apco`/`ap4h`/`ap4x`. + pub fn prores(fourcc: FourCc, width: u16, height: u16) -> Self { + let mut e = Self::video(fourcc, CodecConfig::ProRes { fourcc }, width, height); + if let Some(v) = e.video.as_mut() + && matches!(&fourcc.0, b"ap4h" | b"ap4x") + { + v.depth = 32; + } + e + } + /// Avid DNxHD / DNxHR entry (`AVdn`, or `AVdh` for DNxHR). + pub fn dnx(fourcc: FourCc, width: u16, height: u16) -> Self { + let mut e = Self::video(fourcc, CodecConfig::Dnx { fourcc }, width, height); + if let Some(v) = e.video.as_mut() { + v.depth = 24; + } + e + } + /// Motion-JPEG entry (`jpeg`). + pub fn jpeg(width: u16, height: u16) -> Self { + let f = FourCc(*b"jpeg"); + Self::video(f, CodecConfig::Jpeg { fourcc: f }, width, height) + } + /// AAC entry (`mp4a` + `esds`) from an AudioSpecificConfig. + pub fn aac(asc: Vec, channels: u32, sample_rate: u32) -> Self { + Self::audio(FourCc(*b"mp4a"), CodecConfig::Aac(AacConfig::from_asc(asc)), channels, sample_rate as f64, 16) + } + /// PCM entry. The muxer picks the on-disk format (`sowt`/`twos`/`lpcm` for MOV, `ipcm`/`fpcm` for MP4). + pub fn pcm(cfg: PcmConfig) -> Self { + Self::audio(FourCc(*b"lpcm"), CodecConfig::Pcm(cfg), cfg.channels, cfg.sample_rate, cfg.bits) + } + + /// Serialize as a complete sample entry box. For PCM the format is chosen from the brand. + pub(crate) fn write(&self, b: &mut BoxBuf, qt: bool) -> Result<()> { + if let CodecConfig::Unknown { fourcc, raw } = &self.codec { + let m = b.start(&fourcc.0); + b.bytes(raw); + b.end(m); + return Ok(()); + } + if let CodecConfig::Timecode(tc) = &self.codec { + let m = b.start(b"tmcd"); + b.zeros(6); + b.u16(self.data_reference_index); + b.u32(0); + b.u32(tc.flags); + b.u32(tc.timescale); + b.u32(tc.frame_duration); + b.u8(tc.frames_per_second); + b.u8(0); + if let Some(name) = &tc.name { + let n = b.start(b"name"); + b.u16(name.len() as u16); + b.u16(0); + b.bytes(name.as_bytes()); + b.end(n); + } + b.end(m); + return Ok(()); + } + if let Some(v) = &self.video { + let m = b.start(&self.format.0); + b.zeros(6); + b.u16(self.data_reference_index); + b.u16(0); + b.u16(0); + b.u32(v.vendor.as_u32()); + b.u32(v.temporal_quality); + b.u32(v.spatial_quality); + b.u16(v.width); + b.u16(v.height); + b.u32(v.horiz_resolution); + b.u32(v.vert_resolution); + b.u32(0); + b.u16(v.frame_count); + let name = v.compressor_name.as_bytes(); + let n = name.len().min(31); + let mut nb = [0u8; 32]; + nb[0] = n as u8; + nb[1..1 + n].copy_from_slice(&name[..n]); + b.bytes(&nb); + b.u16(v.depth); + b.i16(-1); + match &self.codec { + CodecConfig::Avc(c) => b.leaf(b"avcC", &c.to_bytes()), + CodecConfig::Hevc(c) => b.leaf(b"hvcC", &c.to_bytes()), + CodecConfig::Av1(c) => b.leaf(b"av1C", &c.to_bytes()), + CodecConfig::Vp9(c) => b.leaf(b"vpcC", &c.to_bytes()), + _ => {} + } + if let Some(g) = v.gamma { + b.leaf(b"gama", &g.to_be_bytes()); + } + if let Some(fi) = v.field_info { + b.leaf(b"fiel", &[fi.fields, fi.detail]); + } + if let Some(ci) = &v.color { + let m = b.start(b"colr"); + match ci { + ColorInfo::Nclx { primaries, transfer, matrix, full_range } => { + b.bytes(b"nclx"); + b.u16(*primaries); + b.u16(*transfer); + b.u16(*matrix); + b.u8(if *full_range { 0x80 } else { 0 }); + } + ColorInfo::Nclc { primaries, transfer, matrix } => { + b.bytes(b"nclc"); + b.u16(*primaries); + b.u16(*transfer); + b.u16(*matrix); + } + ColorInfo::Icc { kind, profile } => { + b.bytes(&kind.0); + b.bytes(profile); + } + } + b.end(m); + } + if let Some(md) = &v.mastering_display { + b.leaf(b"mdcv", &md.to_bytes()); + } + if let Some((cll, fall)) = v.content_light { + let mut d = cll.to_be_bytes().to_vec(); + d.extend_from_slice(&fall.to_be_bytes()); + b.leaf(b"clli", &d); + } + if let Some((h, vv)) = v.pixel_aspect { + let m = b.start(b"pasp"); + b.u32(h); + b.u32(vv); + b.end(m); + } + if let Some(c) = v.clean_aperture { + let m = b.start(b"clap"); + b.u32(c.width.0); + b.u32(c.width.1); + b.u32(c.height.0); + b.u32(c.height.1); + b.i32(c.horiz_offset.0); + b.u32(c.horiz_offset.1); + b.i32(c.vert_offset.0); + b.u32(c.vert_offset.1); + b.end(m); + } + if let Some(br) = self.bitrate { + write_btrt(b, br); + } + b.end(m); + return Ok(()); + } + let a = self.audio.clone().ok_or_else(|| Error::Mux("sample entry has neither video nor audio params".into()))?; + // Choose the on-disk format and sound description version. + let (format, version, pcm) = match &self.codec { + CodecConfig::Pcm(p) => { + if qt { + if p.bits == 16 && !p.float { + (if p.big_endian { *b"twos" } else { *b"sowt" }, 0u16, Some(*p)) + } else if p.bits == 8 && !p.float { + (*b"raw ", 0, Some(*p)) + } else { + (*b"lpcm", 2, Some(*p)) + } + } else { + (if p.float { *b"fpcm" } else { *b"ipcm" }, 0, Some(*p)) + } + } + _ => (self.format.0, if qt { a.qt_version.min(2) } else { 0 }, None), + }; + let m = b.start(&format); + b.zeros(6); + b.u16(self.data_reference_index); + if version == 2 { + let p = pcm.unwrap_or_default(); + b.u16(2); + b.u16(0); + b.u32(0); + b.u16(3); + b.u16(16); + b.i16(-2); + b.u16(0); + b.u32(65536); + b.u32(72); + b.f64(p.sample_rate); + b.u32(p.channels); + b.u32(0x7F00_0000); + b.u32(p.bits as u32); + let flags = (p.float as u32) | ((p.big_endian as u32) << 1) | (((p.signed && !p.float) as u32) << 2) | 8; + b.u32(flags); + b.u32(p.bytes_per_frame()); + b.u32(1); + } else { + b.u16(version); + b.u16(0); + b.u32(0); + b.u16(a.channels.min(u16::MAX as u32) as u16); + b.u16(match pcm { + Some(p) => p.bits, + None => { + if a.sample_size == 0 { + 16 + } else { + a.sample_size + } + } + }); + b.i16(if version == 1 { a.compression_id } else { 0 }); + b.u16(0); + let sr = if a.sample_rate < 65536.0 { (a.sample_rate * 65536.0).round() as u32 } else { 0 }; + b.u32(sr); + if version == 1 { + b.u32(a.samples_per_packet); + b.u32(a.bytes_per_packet); + b.u32(a.bytes_per_frame); + b.u32(a.bytes_per_sample); + } + } + match &self.codec { + CodecConfig::Aac(c) => { + b.leaf(b"esds", &esds_payload(0x40, &c.asc, c.max_bitrate, c.avg_bitrate, c.buffer_size)); + } + CodecConfig::Mp3 => { + if format == *b"mp4a" { + b.leaf(b"esds", &esds_payload(0x6B, &[], 0, 0, 0)); + } + } + CodecConfig::Alac { cookie } => { + let m = b.start_full(b"alac", 0, 0); + b.bytes(cookie); + b.end(m); + } + CodecConfig::Opus(o) => b.leaf(b"dOps", &o.to_bytes()), + CodecConfig::Flac(fl) => { + let m = b.start_full(b"dfLa", 0, 0); + b.bytes(&fl.metadata_blocks); + b.end(m); + } + CodecConfig::Ac3 { dac3 } => b.leaf(b"dac3", dac3), + CodecConfig::Eac3 { dec3 } => b.leaf(b"dec3", dec3), + CodecConfig::Pcm(p) if !qt => { + let m = b.start_full(b"pcmC", 0, 0); + b.u8(if p.big_endian { 0 } else { 1 }); + b.u8(p.bits as u8); + b.end(m); + } + CodecConfig::Pcm(p) if p.channels == 6 => { + // QuickTime channel layout: kAudioChannelLayoutTag_MPEG_5_1_A (L R C LFE Ls Rs) + let m = b.start_full(b"chan", 0, 0); + b.u32((121 << 16) | 6); + b.u32(0); + b.u32(0); + b.end(m); + } + _ => {} + } + if let Some(br) = self.bitrate { + write_btrt(b, br); + } + b.end(m); + Ok(()) + } +} + +fn write_btrt(b: &mut BoxBuf, br: BitRate) { + let m = b.start(b"btrt"); + b.u32(br.buffer_size); + b.u32(br.max_bitrate); + b.u32(br.avg_bitrate); + b.end(m); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn roundtrip(e: &SampleEntry, qt: bool, handler: &[u8; 4]) -> SampleEntry { + let mut b = BoxBuf::new(); + e.write(&mut b, qt).unwrap(); + let bx = crate::isobmff::bytes::boxes(&b.buf).next().unwrap().unwrap(); + parse_sample_entry(bx.kind, bx.payload, FourCc(*handler), qt).unwrap() + } + + #[test] + fn avcc_roundtrip() { + let cfg = AvcConfig::new(vec![vec![0x67, 0x64, 0x00, 0x1F, 0xAC]], vec![vec![0x68, 0xEB]], 4); + assert_eq!(cfg.profile, 0x64); + assert_eq!(AvcConfig::parse(&cfg.to_bytes()).unwrap(), cfg); + let mut e = SampleEntry::avc(cfg, 1920, 1080); + e.video.as_mut().unwrap().color = Some(ColorInfo::Nclx { primaries: 1, transfer: 1, matrix: 1, full_range: false }); + e.video.as_mut().unwrap().pixel_aspect = Some((4, 3)); + e.video.as_mut().unwrap().field_info = Some(FieldInfo { fields: 2, detail: 9 }); + e.video.as_mut().unwrap().clean_aperture = + Some(CleanAperture { width: (1920, 1), height: (1080, 1), horiz_offset: (-2, 1), vert_offset: (0, 1) }); + e.video.as_mut().unwrap().mastering_display = Some(MasteringDisplay::bt2020(1000.0, 0.0001)); + e.video.as_mut().unwrap().content_light = Some((1000, 400)); + e.bitrate = Some(BitRate { buffer_size: 1, max_bitrate: 2, avg_bitrate: 3 }); + assert_eq!(roundtrip(&e, false, b"vide"), e); + assert_eq!(MasteringDisplay::bt2020(1000.0, 0.0001).max_nits(), 1000.0); + } + + #[test] + fn hvcc_roundtrip() { + let cfg = HevcConfig { + general_profile_idc: 1, + general_profile_compatibility_flags: 0x6000_0000, + general_constraint_indicator_flags: 0x9000_0000_0000, + general_level_idc: 93, + chroma_format_idc: 1, + bit_depth_luma: 8, + bit_depth_chroma: 8, + length_size: 4, + num_temporal_layers: 1, + temporal_id_nested: true, + arrays: vec![ + HevcNalArray { completeness: true, nal_type: 32, nalus: vec![vec![0x40, 1]] }, + HevcNalArray { completeness: true, nal_type: 33, nalus: vec![vec![0x42, 1, 2]] }, + HevcNalArray { completeness: true, nal_type: 34, nalus: vec![vec![0x44, 1]] }, + ], + ..Default::default() + }; + let back = HevcConfig::parse(&cfg.to_bytes()).unwrap(); + assert_eq!(back, cfg); + assert_eq!(back.sps(), vec![&[0x42u8, 1, 2][..]]); + } + + #[test] + fn av1_vp9_opus_roundtrip() { + let a = Av1Config { seq_profile: 1, seq_level_idx_0: 8, high_bitdepth: true, config_obus: vec![1, 2], ..Default::default() }; + assert_eq!(Av1Config::parse(&a.to_bytes()).unwrap(), a); + let v = VpcConfig { + profile: 2, + level: 31, + bit_depth: 10, + chroma_subsampling: 1, + full_range: true, + colour_primaries: 9, + transfer_characteristics: 16, + matrix_coefficients: 9, + codec_init: vec![], + }; + assert_eq!(VpcConfig::parse(&v.to_bytes()).unwrap(), v); + let o = OpusConfig { + output_channels: 2, + pre_skip: 312, + input_sample_rate: 48000, + output_gain: 0, + channel_mapping_family: 0, + channel_mapping: vec![], + }; + assert_eq!(OpusConfig::parse(&o.to_bytes()).unwrap(), o); + } + + #[test] + fn aac_esds_roundtrip() { + let e = SampleEntry::aac(vec![0x12, 0x10], 2, 44100); + match &e.codec { + CodecConfig::Aac(a) => { + assert_eq!((a.object_type, a.sample_rate, a.channel_config), (2, 44100, 2)); + } + _ => panic!(), + } + assert_eq!(roundtrip(&e, false, b"soun"), e); + assert_eq!(roundtrip(&e, true, b"soun"), e); + } + + #[test] + fn pcm_entries() { + for (bits, float, be, qt) in [ + (16, false, false, true), + (16, false, true, true), + (24, false, false, true), + (32, true, false, true), + (24, false, true, false), + (32, true, false, false), + ] { + let p = PcmConfig { bits, float, big_endian: be, signed: true, channels: 2, sample_rate: 48000.0 }; + let e = SampleEntry::pcm(p); + let back = roundtrip(&e, qt, b"soun"); + assert_eq!(back.codec, CodecConfig::Pcm(p), "{bits} {float} {be} {qt}"); + } + } + + #[test] + fn timecode_and_prores() { + let tc = + TimecodeConfig { flags: 1, timescale: 30000, frame_duration: 1001, frames_per_second: 30, start_frame: None, name: Some("reel".into()) }; + let e = SampleEntry { + format: FourCc(*b"tmcd"), + data_reference_index: 1, + codec: CodecConfig::Timecode(tc.clone()), + video: None, + audio: None, + bitrate: None, + }; + assert_eq!(roundtrip(&e, true, b"tmcd"), e); + assert_eq!(tc.format_frame(107892), "01:00:00;00"); + let nd = TimecodeConfig { flags: 0, frames_per_second: 24, ..Default::default() }; + assert_eq!(nd.format_frame(86400), "01:00:00:00"); + let p = SampleEntry::prores(FourCc(*b"apch"), 1920, 1080); + assert_eq!(roundtrip(&p, true, b"vide"), p); + } + + #[test] + fn asc_parse() { + assert_eq!(parse_asc(&[0x11, 0x90]), Some((2, 48000, 2))); + assert_eq!(parse_asc(&[]), None); + } +} diff --git a/crates/video/src/isobmff/demux.rs b/crates/video/src/isobmff/demux.rs new file mode 100644 index 00000000..0b82e5c0 --- /dev/null +++ b/crates/video/src/isobmff/demux.rs @@ -0,0 +1,1173 @@ +//! Demuxer: parses box structure lazily (headers + `moov`/`moof` only) and builds per-track sample tables. + +use crate::isobmff::bytes::{Cur, FourCc, boxes, find, parse_box_header}; +use crate::isobmff::codec::{CodecConfig, SampleEntry, parse_sample_entry}; +use crate::isobmff::error::{Error, Result, invalid}; +use crate::isobmff::source::ByteSource; +use std::collections::HashMap; + +/// Hard cap on per-track sample count (after PCM chunk merging). +const MAX_SAMPLES: usize = 1 << 26; +/// Hard cap on the size of an in-memory `moov`/`moof` box. +const MAX_HEADER_BOX: u64 = 1 << 30; + +/// One sample (access unit) of a track, in the track's media timescale. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub struct Sample { + /// Absolute byte offset in the file. + pub offset: u64, + pub size: u32, + /// Decode timestamp (raw media time, edit lists not applied). + pub dts: i64, + /// Presentation timestamp = dts + composition offset (raw media time). + pub pts: i64, + pub duration: u32, + pub is_sync: bool, + /// 1-based index into [`Track::entries`]. + pub description_index: u32, +} + +/// One edit list entry (`elst`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Edit { + /// Duration of this edit in the movie timescale. + pub segment_duration: u64, + /// Start time within the media (track timescale); `-1` marks an empty edit. + pub media_time: i64, + /// Media rate as 16.16 fixed point (0x0001_0000 = normal speed, 0 = dwell). + pub media_rate: i32, +} + +/// Broad classification of a track by its handler. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TrackKind { + Video, + Audio, + Timecode, + Subtitle, + Other, +} + +/// A sample-to-group mapping (`sbgp`) with the matching descriptions (`sgpd`) kept raw. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct SampleGroup { + pub grouping_type: FourCc, + /// (sample_count, group_description_index) runs. + pub runs: Vec<(u32, u32)>, + /// Raw description entries from `sgpd`. + pub descriptions: Vec>, +} + +/// A demuxed track. +#[derive(Clone, Debug, PartialEq)] +pub struct Track { + pub id: u32, + pub kind: TrackKind, + pub handler: FourCc, + pub handler_name: String, + /// Media timescale (`mdhd`). + pub timescale: u32, + /// Media duration in the media timescale (`mdhd`). + pub duration: u64, + /// Track duration in the movie timescale (`tkhd`). + pub tkhd_duration: u64, + /// ISO 639-2/T language code (e.g. `und`, `eng`). + pub language: String, + /// `tkhd` presentation size (16.16 fixed point converted to integer pixels, rounded down). + pub width: u32, + pub height: u32, + pub enabled: bool, + pub alternate_group: u16, + /// `tkhd` volume (8.8 fixed point). + pub volume: i16, + /// `tkhd` transformation matrix. + pub matrix: [i32; 9], + pub entries: Vec, + pub samples: Vec, + pub edits: Vec, + /// Offset to add to raw pts to get presentation time (track timescale), derived from the + /// leading empty edits and the first non-empty edit's `media_time`. + pub edit_offset: i64, + /// Track references (`tref`): (type, track ids). + pub references: Vec<(FourCc, Vec)>, + /// Raw `sdtp` bytes (one per sample) if present. + pub sdtp: Vec, + pub sample_groups: Vec, + /// True if uncompressed audio samples were merged into one [`Sample`] per chunk. + pub pcm_chunked: bool, + sync_index: Vec, + pts_order: Vec, +} + +impl Track { + /// The first sample entry's codec configuration. + pub fn codec(&self) -> Option<&CodecConfig> { + self.entries.first().map(|e| &e.codec) + } + pub fn video(&self) -> Option<&crate::isobmff::codec::VideoParams> { + self.entries.first().and_then(|e| e.video.as_ref()) + } + /// Clockwise quarter turns (0–3) the `tkhd` matrix applies for display, or None when it is not + /// a pure 0/90/180/270° rotation (mirrored, scaled or sheared — left to the caller). + /// + /// ISO/IEC 14496-12 §8.3.2: a sample point (p, q) is displayed at p′ = a·p + c·q + x, + /// q′ = b·p + d·q + y for the matrix {a, b, u, c, d, v, x, y, w}, with y pointing down. So + /// {0, 1, −1, 0} sends the top row to the right-hand column (90° clockwise — iPhone portrait + /// video), {−1, 0, 0, −1} is 180° and {0, −1, 1, 0} is 270°. Only the signs matter: a·d − b·c + /// carries the scale, which the presentation size already reflects. + pub fn display_rotation(&self) -> Option { + let [a, b, _, c, d, _, _, _, _] = self.matrix; + match (a.signum(), b.signum(), c.signum(), d.signum()) { + (1, 0, 0, 1) => Some(0), + (0, 1, -1, 0) => Some(1), + (-1, 0, 0, -1) => Some(2), + (0, -1, 1, 0) => Some(3), + _ => None, + } + } + pub fn audio(&self) -> Option<&crate::isobmff::codec::AudioParams> { + self.entries.first().and_then(|e| e.audio.as_ref()) + } + /// Presentation (edit-adjusted) pts of sample `i`. + pub fn presentation_pts(&self, i: usize) -> Option { + self.samples.get(i).map(|s| s.pts.saturating_add(self.edit_offset)) + } + /// Index of the sample being presented at raw media time `pts` (the last sample, in + /// presentation order, whose pts is ≤ `pts`; the first sample if `pts` precedes all). + pub fn sample_at_pts(&self, pts: i64) -> Option { + if self.pts_order.is_empty() { + return None; + } + let k = self.pts_order.partition_point(|&i| self.samples[i as usize].pts <= pts); + Some(self.pts_order[k.saturating_sub(1)] as usize) + } + /// Like [`sample_at_pts`](Self::sample_at_pts) but for edit-adjusted presentation time. + pub fn sample_at_presentation_time(&self, t: i64) -> Option { + self.sample_at_pts(t.saturating_sub(self.edit_offset)) + } + /// The closest sync sample at or before `index` (decode order); 0 if there is none. + pub fn sync_sample_before(&self, index: usize) -> usize { + let k = self.sync_index.partition_point(|&i| i as usize <= index); + if k == 0 { 0 } else { self.sync_index[k - 1] as usize } + } + /// Indices of all sync samples. + pub fn sync_samples(&self) -> &[u32] { + &self.sync_index + } + + fn finish_indexes(&mut self) { + self.sync_index = self.samples.iter().enumerate().filter(|(_, s)| s.is_sync).map(|(i, _)| i as u32).collect(); + let mut order: Vec = (0..self.samples.len() as u32).collect(); + order.sort_by_key(|&i| (self.samples[i as usize].pts, i)); + self.pts_order = order; + } +} + +/// Text metadata from `udta` (QuickTime `©xxx` atoms and iTunes-style `meta/ilst`, incl. `keys`). +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct Metadata { + pub entries: Vec<(String, String)>, +} + +impl Metadata { + pub fn get(&self, key: &str) -> Option<&str> { + self.entries.iter().find(|(k, _)| k == key).map(|(_, v)| v.as_str()) + } + /// Title from `©nam` or the `com.apple.quicktime.title` key. + pub fn title(&self) -> Option<&str> { + self.get("©nam").or_else(|| self.get("com.apple.quicktime.title")).or_else(|| self.get("title")) + } +} + +/// A parsed MP4/MOV file (headers and sample tables; media data stays in the source). +#[derive(Clone, Debug, PartialEq)] +pub struct Mp4File { + pub major_brand: FourCc, + pub minor_version: u32, + pub compatible_brands: Vec, + /// QuickTime file (major brand `qt ` or no `ftyp`). + pub is_quicktime: bool, + /// Movie timescale (`mvhd`). + pub timescale: u32, + /// Movie duration in the movie timescale (`mvhd`; for fragmented files, the longest track). + pub duration: u64, + pub tracks: Vec, + pub metadata: Metadata, + /// File uses movie fragments (`mvex` present). + pub fragmented: bool, + /// Byte offset of the `moov` box. + pub moov_offset: u64, + /// Number of `moof` boxes found. + pub fragment_count: usize, +} + +impl Mp4File { + /// Read the bytes of sample `index` of track `track` (index into [`tracks`](Self::tracks)). + pub fn read_sample(&self, src: &S, track: usize, index: usize) -> Result> { + let t = self.tracks.get(track).ok_or(Error::NoSuchTrack(track))?; + let s = t.samples.get(index).ok_or(Error::NoSuchSample(index))?; + read_range(src, s.offset, s.size as u64) + } + pub fn sample_at_pts(&self, track: usize, pts: i64) -> Option { + self.tracks.get(track)?.sample_at_pts(pts) + } + pub fn sync_sample_before(&self, track: usize, index: usize) -> Option { + Some(self.tracks.get(track)?.sync_sample_before(index)) + } + /// First track of the given kind. + pub fn track_of_kind(&self, kind: TrackKind) -> Option { + self.tracks.iter().position(|t| t.kind == kind) + } +} + +fn read_range(src: &S, offset: u64, size: u64) -> Result> { + let end = offset.checked_add(size).ok_or(Error::Truncated("sample"))?; + if end > src.len() { + return Err(Error::Truncated("sample data")); + } + let mut v = vec![0u8; size as usize]; + src.read_at(offset, &mut v)?; + Ok(v) +} + +/// Parse an MP4/MOV file. Only box headers, `ftyp`, `moov` and `moof` boxes are read. +pub fn open(src: S) -> Result { + let len = src.len(); + let mut offset = 0u64; + let mut ftyp: Option> = None; + let mut moov: Option<(u64, Vec)> = None; + let mut moofs: Vec<(u64, Vec)> = Vec::new(); + while offset + 8 <= len { + let n = (len - offset).min(32) as usize; + let mut hb = [0u8; 32]; + src.read_at(offset, &mut hb[..n])?; + let h = parse_box_header(&hb[..n])?; + let size = h.size.unwrap_or(len - offset); + let end = offset.saturating_add(size).min(len); + let body_len = end.saturating_sub(offset + h.header_len); + let read_body = |max: u64| -> Result> { + if body_len > max { + return Err(Error::TooLarge("header box")); + } + read_range(&src, offset + h.header_len, body_len) + }; + match &h.kind.0 { + b"ftyp" if ftyp.is_none() => ftyp = Some(read_body(1 << 16)?), + b"moov" if moov.is_none() => moov = Some((offset, read_body(MAX_HEADER_BOX)?)), + b"moof" => moofs.push((offset, read_body(MAX_HEADER_BOX)?)), + _ => {} + } + if h.size.is_none() { + break; + } + offset = offset.saturating_add(size); + } + let (moov_offset, moov) = moov.ok_or(Error::NoMoov)?; + let mut file = Mp4File { + major_brand: FourCc::default(), + minor_version: 0, + compatible_brands: Vec::new(), + is_quicktime: true, + timescale: 0, + duration: 0, + tracks: Vec::new(), + metadata: Metadata::default(), + fragmented: false, + moov_offset, + fragment_count: moofs.len(), + }; + if let Some(f) = &ftyp { + let mut c = Cur::new(f, "ftyp"); + file.major_brand = c.fourcc()?; + file.minor_version = c.u32().unwrap_or(0); + while c.remaining() >= 4 { + file.compatible_brands.push(c.fourcc()?); + } + file.is_quicktime = file.major_brand == b"qt "; + } + let mut trex: HashMap = HashMap::new(); + let mut trak_payloads = Vec::new(); + for b in boxes(&moov) { + let b = b?; + match &b.kind.0 { + b"cmov" => return Err(Error::Unsupported("compressed movie header (cmov)".into())), + b"mvhd" => { + let mut c = Cur::new(b.payload, "mvhd"); + let (v, _) = c.full_header()?; + if v == 1 { + c.skip(16)?; + file.timescale = c.u32()?; + file.duration = c.u64()?; + } else { + c.skip(8)?; + file.timescale = c.u32()?; + file.duration = c.u32()? as u64; + } + } + b"trak" => trak_payloads.push(b.payload), + b"mvex" => { + file.fragmented = true; + for m in boxes(b.payload).map_while(|b| b.ok()) { + if m.kind == b"trex" { + let mut c = Cur::new(m.payload, "trex"); + c.full_header()?; + let id = c.u32()?; + trex.insert(id, Trex { description_index: c.u32()?, duration: c.u32()?, size: c.u32()?, flags: c.u32()? }); + } + } + } + b"udta" => parse_udta(b.payload, &mut file.metadata), + b"meta" => parse_meta(b.payload, &mut file.metadata), + _ => {} + } + } + for p in trak_payloads { + let t = parse_trak(p, &file, len)?; + file.tracks.push(t); + } + // Movie fragments. + let mut next_dts: Vec = file.tracks.iter().map(|t| t.samples.last().map(|s| s.dts + s.duration as i64).unwrap_or(0)).collect(); + for (moof_off, moof) in &moofs { + parse_moof(*moof_off, moof, &mut file.tracks, &trex, &mut next_dts, len)?; + } + for t in &mut file.tracks { + if file.fragmented && !moofs.is_empty() { + let d: u64 = t.samples.iter().map(|s| s.duration as u64).sum(); + t.duration = t.duration.max(d); + if t.timescale > 0 && file.timescale > 0 { + let md = (t.duration as u128 * file.timescale as u128 / t.timescale as u128) as u64; + file.duration = file.duration.max(md); + } + } + t.finish_indexes(); + // Timecode start frame from the first sample. + if let Some(CodecConfig::Timecode(tc)) = t.entries.first_mut().map(|e| &mut e.codec) + && let Some(s) = t.samples.first() + && s.size >= 4 + { + let mut b = [0u8; 4]; + if src.read_at(s.offset, &mut b).is_ok() { + tc.start_frame = Some(u32::from_be_bytes(b)); + } + } + } + Ok(file) +} + +#[derive(Clone, Copy, Debug, Default)] +struct Trex { + description_index: u32, + duration: u32, + size: u32, + flags: u32, +} + +fn parse_language(code: u16) -> String { + if code < 0x400 { + // QuickTime Macintosh language code. + return if code == 0 { "eng".into() } else { "und".into() }; + } + let c = |s: u16| (((code >> s) & 0x1F) as u8 + 0x60) as char; + let s: String = [c(10), c(5), c(0)].into_iter().collect(); + if s.chars().all(|ch| ch.is_ascii_lowercase()) { s } else { "und".into() } +} + +fn rescale(v: i128, from: u32, to: u32) -> i64 { + if from == 0 { + return 0; + } + (v * to as i128 / from as i128).clamp(i64::MIN as i128, i64::MAX as i128) as i64 +} + +fn parse_trak(p: &[u8], file: &Mp4File, file_len: u64) -> Result { + let mut t = Track { + id: 0, + kind: TrackKind::Other, + handler: FourCc::default(), + handler_name: String::new(), + timescale: 0, + duration: 0, + tkhd_duration: 0, + language: "und".into(), + width: 0, + height: 0, + enabled: true, + alternate_group: 0, + volume: 0, + matrix: [0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x4000_0000], + entries: Vec::new(), + samples: Vec::new(), + edits: Vec::new(), + edit_offset: 0, + references: Vec::new(), + sdtp: Vec::new(), + sample_groups: Vec::new(), + pcm_chunked: false, + sync_index: Vec::new(), + pts_order: Vec::new(), + }; + let mut mdia = None; + for b in boxes(p) { + let b = b?; + match &b.kind.0 { + b"tkhd" => { + let mut c = Cur::new(b.payload, "tkhd"); + let (v, flags) = c.full_header()?; + t.enabled = flags & 1 != 0; + if v == 1 { + c.skip(16)?; + t.id = c.u32()?; + c.skip(4)?; + t.tkhd_duration = c.u64()?; + } else { + c.skip(8)?; + t.id = c.u32()?; + c.skip(4)?; + t.tkhd_duration = c.u32()? as u64; + } + c.skip(8)?; + let _layer = c.i16()?; + t.alternate_group = c.u16()?; + t.volume = c.i16()?; + c.skip(2)?; + for m in t.matrix.iter_mut() { + *m = c.i32()?; + } + t.width = c.u32()? >> 16; + t.height = c.u32()? >> 16; + } + b"edts" => { + if let Some(e) = find(b.payload, b"elst") { + let mut c = Cur::new(e, "elst"); + let (v, _) = c.full_header()?; + let n = c.count(if v == 1 { 20 } else { 12 })?; + for _ in 0..n { + let (segment_duration, media_time) = if v == 1 { (c.u64()?, c.i64()?) } else { (c.u32()? as u64, c.i32()? as i64) }; + t.edits.push(Edit { segment_duration, media_time, media_rate: c.i32()? }); + } + } + } + b"tref" => { + for r in boxes(b.payload).map_while(|b| b.ok()) { + let ids = r.payload.as_chunks::<4>().0.iter().map(|c| u32::from_be_bytes([c[0], c[1], c[2], c[3]])).collect(); + t.references.push((r.kind, ids)); + } + } + b"mdia" => mdia = Some(b.payload), + _ => {} + } + } + let mdia = mdia.ok_or_else(|| Error::Invalid("trak without mdia".into()))?; + let mut minf = None; + for b in boxes(mdia) { + let b = b?; + match &b.kind.0 { + b"mdhd" => { + let mut c = Cur::new(b.payload, "mdhd"); + let (v, _) = c.full_header()?; + if v == 1 { + c.skip(16)?; + t.timescale = c.u32()?; + t.duration = c.u64()?; + } else { + c.skip(8)?; + t.timescale = c.u32()?; + t.duration = c.u32()? as u64; + } + t.language = parse_language(c.u16()?); + } + b"hdlr" => { + let mut c = Cur::new(b.payload, "hdlr"); + c.full_header()?; + let _pre = c.u32()?; + t.handler = c.fourcc()?; + c.skip(12)?; + let name = c.rest(); + // ISO: null-terminated UTF-8; QuickTime: Pascal string. + let s = if !name.is_empty() && name[0] as usize == name.len() - 1 && name.len() > 1 { + &name[1..] + } else { + name.split(|&b| b == 0).next().unwrap_or(&[]) + }; + t.handler_name = String::from_utf8_lossy(s).into_owned(); + } + b"minf" => minf = Some(b.payload), + _ => {} + } + } + t.kind = match &t.handler.0 { + b"vide" => TrackKind::Video, + b"soun" => TrackKind::Audio, + b"tmcd" => TrackKind::Timecode, + b"text" | b"sbtl" | b"subt" | b"clcp" => TrackKind::Subtitle, + _ => TrackKind::Other, + }; + // Edit offset in media timescale. + let mut empty = 0i128; + for e in &t.edits { + if e.media_time == -1 { + empty += e.segment_duration as i128; + } else { + t.edit_offset = rescale(empty, file.timescale, t.timescale).saturating_sub(e.media_time); + break; + } + } + if let Some(stbl) = minf.and_then(|m| find(m, b"stbl")) { + parse_stbl(stbl, &mut t, file.is_quicktime, file_len)?; + } + Ok(t) +} + +struct SttsCursor<'a> { + runs: &'a [(u32, u32)], + run: usize, + used: u32, +} + +impl SttsCursor<'_> { + /// Advance by `n` samples, returning the summed duration. + fn take(&mut self, mut n: u64) -> u64 { + let mut total = 0u64; + while n > 0 && self.run < self.runs.len() { + let (count, delta) = self.runs[self.run]; + let avail = (count - self.used) as u64; + let k = avail.min(n); + total = total.saturating_add(k.saturating_mul(delta as u64)); + n -= k; + self.used += k as u32; + if self.used >= count { + self.run += 1; + self.used = 0; + } + } + total + } + fn next_value(&mut self) -> u32 { + while self.run < self.runs.len() && self.runs[self.run].0 == 0 { + self.run += 1; + } + let Some(&(_, d)) = self.runs.get(self.run) else { return 0 }; + self.take(1); + d + } +} + +fn parse_stbl(stbl: &[u8], t: &mut Track, is_qt: bool, file_len: u64) -> Result<()> { + let mut stts: Vec<(u32, u32)> = Vec::new(); + let mut ctts: Vec<(u32, u32)> = Vec::new(); + let mut stss: Option> = None; + let mut stsc: Vec<(u32, u32, u32)> = Vec::new(); + let mut const_size = 0u32; + let mut sample_count = 0usize; + let mut sizes: Vec = Vec::new(); + let mut chunks: Vec = Vec::new(); + let mut sbgp: Vec<(FourCc, Vec<(u32, u32)>)> = Vec::new(); + let mut sgpd: HashMap>> = HashMap::new(); + for b in boxes(stbl) { + let b = b?; + let mut c = Cur::new(b.payload, "stbl child"); + match &b.kind.0 { + b"stsd" => { + c.full_header()?; + let n = c.count(8)?; + for e in boxes(c.rest()).take(n) { + let e = e?; + t.entries.push(parse_sample_entry(e.kind, e.payload, t.handler, is_qt)?); + } + } + b"stts" => { + c.full_header()?; + let n = c.count(8)?; + stts = (0..n).map(|_| Ok((c.u32()?, c.u32()?))).collect::>()?; + } + b"ctts" => { + c.full_header()?; + let n = c.count(8)?; + ctts = (0..n).map(|_| Ok((c.u32()?, c.u32()?))).collect::>()?; + } + b"stss" => { + c.full_header()?; + let n = c.count(4)?; + let mut v: Vec = (0..n).map(|_| c.u32()).collect::>()?; + v.sort_unstable(); + stss = Some(v); + } + b"stsc" => { + c.full_header()?; + let n = c.count(12)?; + stsc = (0..n).map(|_| Ok((c.u32()?, c.u32()?, c.u32()?))).collect::>()?; + } + b"stsz" => { + c.full_header()?; + const_size = c.u32()?; + let n = c.u32()? as usize; + if const_size == 0 { + c.check_count(n, 4)?; + sizes = (0..n).map(|_| c.u32()).collect::>()?; + sample_count = n; + } else { + // Samples of constant size must fit in the file: clamp absurd counts. + sample_count = (n as u64).min(file_len / const_size as u64) as usize; + } + } + b"stz2" => { + c.full_header()?; + let field = c.u32()? & 0xFF; + let n = c.u32()? as usize; + let bits = match field { + 4 | 8 | 16 => field as usize, + _ => return invalid(format!("stz2 field size {field}")), + }; + if n.checked_mul(bits).is_none_or(|b| b.div_ceil(8) > c.remaining()) { + return Err(Error::Truncated("stz2")); + } + let d = c.rest(); + sizes = (0..n) + .map(|i| match bits { + 4 => ((d[i / 2] >> if i % 2 == 0 { 4 } else { 0 }) & 0xF) as u32, + 8 => d[i] as u32, + _ => u16::from_be_bytes([d[2 * i], d[2 * i + 1]]) as u32, + }) + .collect(); + sample_count = n; + } + b"stco" => { + c.full_header()?; + let n = c.count(4)?; + chunks = (0..n).map(|_| c.u32().map(|v| v as u64)).collect::>()?; + } + b"co64" => { + c.full_header()?; + let n = c.count(8)?; + chunks = (0..n).map(|_| c.u64()).collect::>()?; + } + b"sdtp" => { + c.full_header()?; + t.sdtp = c.rest().to_vec(); + } + b"sbgp" => { + let (v, _) = c.full_header()?; + let gt = c.fourcc()?; + if v == 1 { + c.u32()?; + } + let n = c.count(8)?; + let runs = (0..n).map(|_| Ok((c.u32()?, c.u32()?))).collect::>()?; + sbgp.push((gt, runs)); + } + b"sgpd" => { + let (v, _) = c.full_header()?; + let gt = c.fourcc()?; + let default_len = if v >= 1 { c.u32()? } else { 0 }; + if v >= 2 { + c.u32()?; + } + let n = c.count(0)?; + let mut descs = Vec::new(); + for _ in 0..n { + let l = if v >= 1 && default_len == 0 { c.u32()? } else { default_len }; + if l == 0 { + // Version 0 without known length: keep the remainder as a single entry. + descs.push(c.rest().to_vec()); + break; + } + descs.push(c.bytes(l as usize)?.to_vec()); + } + sgpd.insert(gt, descs); + } + _ => {} + } + } + t.sample_groups = + sbgp.into_iter().map(|(gt, runs)| SampleGroup { grouping_type: gt, runs, descriptions: sgpd.remove(>).unwrap_or_default() }).collect(); + if sample_count == 0 || chunks.is_empty() || stsc.is_empty() { + return Ok(()); + } + if stsc[0].0 == 0 || stsc.windows(2).any(|w| w[1].0 <= w[0].0) { + return invalid("stsc first_chunk values are not increasing from 1"); + } + let pcm = match t.entries.first().map(|e| &e.codec) { + Some(CodecConfig::Pcm(p)) if const_size != 0 => Some(*p), + _ => None, + }; + let mut stts_c = SttsCursor { runs: &stts, run: 0, used: 0 }; + let mut ctts_c = SttsCursor { runs: &ctts, run: 0, used: 0 }; + let mut samples = Vec::new(); + let mut sidx = 0usize; + let mut dts = 0i64; + let mut stsc_i = 0usize; + if let Some(p) = pcm { + t.pcm_chunked = true; + let a = t.audio().cloned().unwrap_or_default(); + let frame_bytes = |frames: u64| -> u64 { + if a.qt_version == 1 && a.bytes_per_frame > 0 && a.samples_per_packet > 0 { + frames / a.samples_per_packet as u64 * a.bytes_per_frame as u64 + } else if const_size > 1 { + frames * const_size as u64 + } else { + frames * p.bytes_per_frame() as u64 + } + }; + if chunks.len() > MAX_SAMPLES { + return Err(Error::TooLarge("chunk count")); + } + samples.reserve(chunks.len()); + for (ci, &off) in chunks.iter().enumerate() { + while stsc_i + 1 < stsc.len() && stsc[stsc_i + 1].0 as usize <= ci + 1 { + stsc_i += 1; + } + let spc = (stsc[stsc_i].1 as usize).min(sample_count - sidx); + if spc == 0 { + continue; + } + let dur = stts_c.take(spc as u64); + let size = frame_bytes(spc as u64).min(u32::MAX as u64) as u32; + samples.push(Sample { + offset: off, + size, + dts, + pts: dts, + duration: dur.min(u32::MAX as u64) as u32, + is_sync: true, + description_index: stsc[stsc_i].2, + }); + dts = dts.saturating_add(dur.min(i64::MAX as u64) as i64); + sidx += spc; + if sidx >= sample_count { + break; + } + } + } else { + if sample_count > MAX_SAMPLES { + return Err(Error::TooLarge("sample count")); + } + samples.reserve(sample_count); + let mut stss_i = 0usize; + 'chunks: for (ci, &off) in chunks.iter().enumerate() { + while stsc_i + 1 < stsc.len() && stsc[stsc_i + 1].0 as usize <= ci + 1 { + stsc_i += 1; + } + let (_, spc, sdi) = stsc[stsc_i]; + let mut pos = off; + for _ in 0..spc { + if sidx >= sample_count { + break 'chunks; + } + let size = if const_size != 0 { const_size } else { sizes[sidx] }; + let duration = stts_c.next_value(); + let cts = ctts_c.next_value() as i32 as i64; + let is_sync = match &stss { + None => true, + Some(v) => { + while stss_i < v.len() && (v[stss_i] as usize) < sidx + 1 { + stss_i += 1; + } + stss_i < v.len() && v[stss_i] as usize == sidx + 1 + } + }; + samples.push(Sample { offset: pos, size, dts, pts: dts.saturating_add(cts), duration, is_sync, description_index: sdi }); + pos = pos.saturating_add(size as u64); + dts = dts.saturating_add(duration as i64); + sidx += 1; + } + } + } + t.samples = samples; + Ok(()) +} + +fn parse_moof(moof_off: u64, moof: &[u8], tracks: &mut [Track], trex: &HashMap, next_dts: &mut [i64], file_len: u64) -> Result<()> { + // End of data of the previous traf, for the base-offset default of subsequent trafs. + let mut prev_end: Option = None; + for b in boxes(moof) { + let b = b?; + if b.kind != b"traf" { + continue; + } + let mut tfhd = None; + let mut tfdt = None; + let mut truns = Vec::new(); + for c in boxes(b.payload) { + let c = c?; + match &c.kind.0 { + b"tfhd" => tfhd = Some(c.payload), + b"tfdt" => tfdt = Some(c.payload), + b"trun" => truns.push(c.payload), + _ => {} + } + } + let tfhd = tfhd.ok_or_else(|| Error::Invalid("traf without tfhd".into()))?; + let mut c = Cur::new(tfhd, "tfhd"); + let (_, fl) = c.full_header()?; + let id = c.u32()?; + let Some(ti) = tracks.iter().position(|t| t.id == id) else { continue }; + let d = trex.get(&id).copied().unwrap_or_default(); + let base = if fl & 1 != 0 { + c.u64()? + } else if fl & 0x20000 != 0 { + moof_off + } else { + prev_end.unwrap_or(moof_off) + }; + let sdi = if fl & 2 != 0 { c.u32()? } else { d.description_index }; + let def_dur = if fl & 8 != 0 { c.u32()? } else { d.duration }; + let def_size = if fl & 0x10 != 0 { c.u32()? } else { d.size }; + let def_flags = if fl & 0x20 != 0 { c.u32()? } else { d.flags }; + if let Some(p) = tfdt { + let mut c = Cur::new(p, "tfdt"); + let (v, _) = c.full_header()?; + let t = if v == 1 { c.u64()? } else { c.u32()? as u64 }; + next_dts[ti] = t.min(i64::MAX as u64) as i64; + } + let mut data_pos = base; + for tr in truns { + let mut c = Cur::new(tr, "trun"); + let (v, tf) = c.full_header()?; + let per = [0x100u32, 0x200, 0x400, 0x800].iter().filter(|&&f| tf & f != 0).count() * 4; + let n = c.u32()? as usize; + if per > 0 { + c.check_count(n, per)?; + } else if (n as u64).saturating_mul(def_size.max(1) as u64) > file_len { + return Err(Error::TooLarge("trun sample count")); + } + if tracks[ti].samples.len() + n > MAX_SAMPLES { + return Err(Error::TooLarge("sample count")); + } + if tf & 1 != 0 { + data_pos = base.wrapping_add_signed(c.i32()? as i64); + } + let first_flags = if tf & 4 != 0 { Some(c.u32()?) } else { None }; + let t = &mut tracks[ti]; + t.samples.reserve(n); + for i in 0..n { + let dur = if tf & 0x100 != 0 { c.u32()? } else { def_dur }; + let size = if tf & 0x200 != 0 { c.u32()? } else { def_size }; + let flags = if tf & 0x400 != 0 { c.u32()? } else { def_flags }; + // first_sample_flags overrides the flags of the first sample only. + let flags = if i == 0 { first_flags.unwrap_or(flags) } else { flags }; + let cto = if tf & 0x800 != 0 { + let raw = c.u32()?; + if v == 0 { raw as i64 } else { raw as i32 as i64 } + } else { + 0 + }; + let dts = next_dts[ti]; + t.samples.push(Sample { + offset: data_pos, + size, + dts, + pts: dts.saturating_add(cto), + duration: dur, + is_sync: flags & 0x1_0000 == 0, + description_index: sdi, + }); + data_pos = data_pos.saturating_add(size as u64); + next_dts[ti] = dts.saturating_add(dur as i64); + } + } + prev_end = Some(data_pos); + } + Ok(()) +} + +fn parse_udta(p: &[u8], md: &mut Metadata) { + for b in boxes(p).map_while(|b| b.ok()) { + if b.kind == b"meta" { + parse_meta(b.payload, md); + } else if b.kind.0[0] == 0xA9 && b.payload.len() >= 4 { + let n = u16::from_be_bytes([b.payload[0], b.payload[1]]) as usize; + if let Some(s) = b.payload.get(4..4 + n) { + md.entries.push((b.kind.to_string_lossy(), String::from_utf8_lossy(s).into_owned())); + } + } + } +} + +fn parse_meta(p: &[u8], md: &mut Metadata) { + // QuickTime `meta` is a plain box; ISO `meta` is a full box. + let body = if p.len() >= 8 && &p[4..8] == b"hdlr" { p } else { p.get(4..).unwrap_or(&[]) }; + let mut keys: Vec = Vec::new(); + if let Some(k) = find(body, b"keys") { + let mut c = Cur::new(k, "keys"); + if c.full_header().is_ok() + && let Ok(n) = c.count(8) + { + for _ in 0..n { + let Ok(sz) = c.u32() else { break }; + let Ok(_ns) = c.u32() else { break }; + let Ok(name) = c.bytes((sz as usize).saturating_sub(8)) else { break }; + keys.push(String::from_utf8_lossy(name).into_owned()); + } + } + } + let Some(ilst) = find(body, b"ilst") else { return }; + for item in boxes(ilst).map_while(|b| b.ok()) { + let idx = item.kind.as_u32() as usize; + let key = if !keys.is_empty() && idx >= 1 && idx <= keys.len() { keys[idx - 1].clone() } else { item.kind.to_string_lossy() }; + if let Some(d) = find(item.payload, b"data") + && d.len() >= 8 + { + let ty = u32::from_be_bytes([d[0], d[1], d[2], d[3]]) & 0xFF_FFFF; + if ty == 1 || ty == 0 { + md.entries.push((key, String::from_utf8_lossy(&d[8..]).into_owned())); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn language_codes() { + // "und" = 0x55C4 + assert_eq!(parse_language(0x55C4), "und"); + assert_eq!(parse_language(0), "eng"); + assert_eq!(parse_language(0x15C7), "eng"); + } + + #[test] + fn stts_cursor() { + let runs = [(2, 10), (0, 99), (3, 5)]; + let mut c = SttsCursor { runs: &runs, run: 0, used: 0 }; + assert_eq!(c.next_value(), 10); + assert_eq!(c.take(3), 10 + 5 + 5); + assert_eq!(c.next_value(), 5); + assert_eq!(c.next_value(), 0); + } + + #[test] + fn rejects_no_moov_and_cmov() { + let mut b = crate::isobmff::bytes::BoxBuf::new(); + b.leaf(b"free", &[0; 4]); + assert!(matches!(open(b.buf.clone()), Err(Error::NoMoov))); + let m = b.start(b"moov"); + b.leaf(b"cmov", &[]); + b.end(m); + assert!(matches!(open(b.buf), Err(Error::Unsupported(_)))); + } + + #[test] + fn metadata_qt_and_ilst() { + let mut b = crate::isobmff::bytes::BoxBuf::new(); + let m = b.start(&[0xA9, b'n', b'a', b'm']); + b.u16(2); + b.u16(0); + b.bytes(b"hi"); + b.end(m); + let meta = b.start_full(b"meta", 0, 0); + b.leaf(b"hdlr", &[0; 24]); + let il = b.start(b"ilst"); + let it = b.start(&[0xA9, b'A', b'R', b'T']); + let d = b.start(b"data"); + b.u32(1); + b.u32(0); + b.bytes(b"me"); + b.end(d); + b.end(it); + b.end(il); + b.end(meta); + let mut md = Metadata::default(); + parse_udta(&b.buf, &mut md); + assert_eq!(md.title(), Some("hi")); + assert_eq!(md.get("©ART"), Some("me")); + } + + use crate::isobmff::bytes::BoxBuf; + + fn full(b: &mut BoxBuf, kind: &[u8; 4], v: u8, f: impl FnOnce(&mut BoxBuf)) { + let m = b.start_full(kind, v, 0); + f(b); + b.end(m); + } + + /// Hand-built file: uuid/wide/skip boxes, stz2 sizes, co64, ctts v1 with negative offsets, + /// stss, sdtp, sbgp/sgpd, and an edit list with a leading empty edit. + fn handmade(stsz_override: Option<(u32, u32)>) -> Vec { + let mut b = BoxBuf::new(); + let m = b.start(b"ftyp"); + b.bytes(b"isom"); + b.u32(0); + b.bytes(b"isom"); + b.end(m); + let m = b.start(b"uuid"); + b.bytes(&[0xAB; 16]); + b.bytes(b"payload"); + b.end(m); + b.leaf(b"wide", &[]); + b.leaf(b"skip", &[1, 2, 3]); + // mdat with 64-bit size: 4 samples of 3,1,2,4 bytes. + let data_start = b.len() as u64 + 16; + b.u32(1); + b.bytes(b"mdat"); + b.u64(16 + 10); + b.bytes(&[1, 1, 1, 2, 3, 3, 4, 4, 4, 4]); + let moov = b.start(b"moov"); + full(&mut b, b"mvhd", 0, |b| { + b.u32(0); + b.u32(0); + b.u32(1000); + b.u32(500); + b.zeros(80); + }); + let trak = b.start(b"trak"); + full(&mut b, b"tkhd", 0, |b| { + b.u32(0); + b.u32(0); + b.u32(7); + b.u32(0); + b.u32(500); + b.zeros(52); + b.u32(320 << 16); + b.u32(240 << 16); + }); + let edts = b.start(b"edts"); + full(&mut b, b"elst", 1, |b| { + b.u32(2); + b.u64(100); + b.i64(-1); + b.i32(0x10000); + b.u64(400); + b.i64(20); + b.i32(0x10000); + }); + b.end(edts); + let mdia = b.start(b"mdia"); + full(&mut b, b"mdhd", 0, |b| { + b.u32(0); + b.u32(0); + b.u32(100); + b.u32(40); + b.u16(0x15C7); + b.u16(0); + }); + full(&mut b, b"hdlr", 0, |b| { + b.u32(0); + b.bytes(b"vide"); + b.zeros(12); + b.bytes(b"Video\0"); + }); + let minf = b.start(b"minf"); + let stbl = b.start(b"stbl"); + full(&mut b, b"stsd", 0, |b| { + b.u32(1); + crate::isobmff::codec::SampleEntry::jpeg(320, 240).write(b, false).unwrap(); + }); + full(&mut b, b"stts", 0, |b| { + b.u32(1); + b.u32(4); + b.u32(10); + }); + full(&mut b, b"ctts", 1, |b| { + b.u32(2); + b.u32(2); + b.i32(-5); + b.u32(2); + b.i32(5); + }); + full(&mut b, b"stss", 0, |b| { + b.u32(2); + b.u32(3); + b.u32(1); + }); + full(&mut b, b"stsc", 0, |b| { + b.u32(2); + b.u32(1); + b.u32(3); + b.u32(1); + b.u32(2); + b.u32(1); + b.u32(1); + }); + match stsz_override { + Some((size, count)) => full(&mut b, b"stsz", 0, |b| { + b.u32(size); + b.u32(count); + }), + None => full(&mut b, b"stz2", 0, |b| { + b.u32(4); + b.u32(4); + b.u8(0x31); + b.u8(0x24); + }), + } + full(&mut b, b"co64", 0, |b| { + b.u32(2); + b.u64(data_start); + b.u64(data_start + 6); + }); + full(&mut b, b"sdtp", 0, |b| b.bytes(&[0x20, 0x10, 0x10, 0x10])); + full(&mut b, b"sbgp", 0, |b| { + b.bytes(b"roll"); + b.u32(1); + b.u32(4); + b.u32(1); + }); + full(&mut b, b"sgpd", 1, |b| { + b.bytes(b"roll"); + b.u32(2); + b.u32(1); + b.i16(-1); + }); + b.end(stbl); + b.end(minf); + b.end(mdia); + b.end(trak); + b.end(moov); + // Trailing box extending to EOF (size 0). + b.u32(0); + b.bytes(b"free"); + b.bytes(&[9; 5]); + b.buf + } + + #[test] + fn handmade_sample_table() { + let d = handmade(None); + let f = open(d.as_slice()).unwrap(); + assert_eq!(f.major_brand, *b"isom"); + let t = &f.tracks[0]; + assert_eq!((t.id, t.kind, t.timescale, t.language.as_str()), (7, TrackKind::Video, 100, "eng")); + assert_eq!((t.width, t.height), (320, 240)); + assert_eq!(t.handler_name, "Video"); + let sizes: Vec = t.samples.iter().map(|s| s.size).collect(); + assert_eq!(sizes, [3, 1, 2, 4]); + let offs: Vec = t.samples.iter().map(|s| s.offset - t.samples[0].offset).collect(); + assert_eq!(offs, [0, 3, 4, 6]); + let pts: Vec = t.samples.iter().map(|s| s.pts).collect(); + assert_eq!(pts, [-5, 5, 25, 35]); + let sync: Vec = t.samples.iter().map(|s| s.is_sync).collect(); + assert_eq!(sync, [true, false, true, false]); + for (i, want) in [vec![1u8, 1, 1], vec![2], vec![3, 3], vec![4; 4]].iter().enumerate() { + assert_eq!(&f.read_sample(d.as_slice(), 0, i).unwrap(), want); + } + // Empty edit of 100 movie ticks = 10 media ticks, then media starts at 20. + assert_eq!(t.edits.len(), 2); + assert_eq!(t.edit_offset, 10 - 20); + assert_eq!(t.presentation_pts(2), Some(15)); + assert_eq!(t.sdtp.len(), 4); + assert_eq!(t.sample_groups[0].grouping_type, *b"roll"); + assert_eq!(t.sample_groups[0].descriptions, vec![vec![0xFF, 0xFF]]); + assert!(f.read_sample(d.as_slice(), 0, 4).is_err()); + assert!(f.read_sample(d.as_slice(), 1, 0).is_err()); + } + + #[test] + fn absurd_counts_are_bounded() { + // Constant-size stsz claiming 4 billion samples is clamped to what fits in the file. + let d = handmade(Some((1, u32::MAX))); + let f = open(d.as_slice()).unwrap(); + assert!(f.tracks[0].samples.len() <= d.len()); + // A table claiming more entries than its box holds fails cleanly. + let mut d = handmade(None); + let p = d.windows(4).position(|w| w == b"stts").unwrap(); + d[p + 8..p + 12].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(matches!(open(d.as_slice()), Err(Error::Truncated(_)))); + // Box larger than its parent. + let mut d = handmade(None); + let p = d.windows(4).position(|w| w == b"stsc").unwrap(); + d[p - 4..p].copy_from_slice(&0x7FFF_FFFFu32.to_be_bytes()); + assert!(open(d.as_slice()).is_err()); + } +} diff --git a/crates/video/src/isobmff/error.rs b/crates/video/src/isobmff/error.rs new file mode 100644 index 00000000..b5dfd752 --- /dev/null +++ b/crates/video/src/isobmff/error.rs @@ -0,0 +1,38 @@ +use std::io; + +/// Errors produced by the demuxer and muxer. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("I/O error: {0}")] + Io(#[from] io::Error), + /// A box or structure ended before all required fields were read. + #[error("truncated data while reading {0}")] + Truncated(&'static str), + /// Structurally invalid data. + #[error("invalid data: {0}")] + Invalid(String), + /// Valid but unsupported feature (e.g. compressed `cmov` movie header). + #[error("unsupported: {0}")] + Unsupported(String), + /// A count or size exceeds sanity limits relative to the file size. + #[error("size limit exceeded: {0}")] + TooLarge(&'static str), + /// No `moov` box was found. + #[error("no moov box found")] + NoMoov, + /// A track index was out of range. + #[error("track index {0} out of range")] + NoSuchTrack(usize), + /// A sample index was out of range. + #[error("sample index {0} out of range")] + NoSuchSample(usize), + /// Muxer usage error (e.g. invalid track config, bad sample). + #[error("muxer: {0}")] + Mux(String), +} + +pub type Result = std::result::Result; + +pub(crate) fn invalid(msg: impl Into) -> Result { + Err(Error::Invalid(msg.into())) +} diff --git a/crates/video/src/isobmff/mod.rs b/crates/video/src/isobmff/mod.rs new file mode 100644 index 00000000..bfb8a51b --- /dev/null +++ b/crates/video/src/isobmff/mod.rs @@ -0,0 +1,26 @@ +//! Clean-room ISO Base Media File Format (MP4, ISO/IEC 14496-12/14/15) and QuickTime (MOV) support. +//! +//! - **Demux:** [`open`] parses box headers plus `moov` (and `moof` for fragmented files) from a +//! [`ByteSource`] and builds per-track [`Sample`] tables; media data is read on demand with +//! [`Mp4File::read_sample`]. +//! - **Mux:** [`Mp4Writer`] writes progressive MP4/MOV (optionally faststart); +//! [`FragmentedWriter`] writes fragmented MP4. +//! +//! See the crate README for supported boxes and limitations. + +mod bytes; +mod codec; +mod demux; +mod error; +mod mux; +mod source; + +pub use bytes::FourCc; +pub use codec::{ + AacConfig, AudioParams, Av1Config, AvcConfig, BitRate, CleanAperture, CodecConfig, ColorInfo, FieldInfo, FlacConfig, HevcConfig, HevcNalArray, + MasteringDisplay, OpusConfig, PcmConfig, SampleEntry, TimecodeConfig, VideoParams, VpcConfig, parse_asc, +}; +pub use demux::{Edit, Metadata, Mp4File, Sample, SampleGroup, Track, TrackKind, open}; +pub use error::{Error, Result}; +pub use mux::{Brand, FragmentedWriter, Mp4Writer, TrackConfig, WriteSample, WriterOptions}; +pub use source::ByteSource; diff --git a/crates/video/src/isobmff/mux.rs b/crates/video/src/isobmff/mux.rs new file mode 100644 index 00000000..6a3d4322 --- /dev/null +++ b/crates/video/src/isobmff/mux.rs @@ -0,0 +1,966 @@ +//! Muxers: progressive MP4/MOV ([`Mp4Writer`], optional faststart) and fragmented MP4 ([`FragmentedWriter`]). + +use crate::isobmff::bytes::BoxBuf; +use crate::isobmff::codec::{CodecConfig, SampleEntry, TimecodeConfig}; +use crate::isobmff::demux::Edit; +use crate::isobmff::error::{Error, Result}; +use std::io::{Read, Seek, SeekFrom, Write}; + +/// Output flavour: determines brands and some sample-entry encodings. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Brand { + /// ISO MP4: `isom` major brand, compatible `isom iso2 avc1 mp41 mp42`. + Mp4, + /// QuickTime MOV: `qt ` major brand. + Mov, +} + +/// Movie-level writer options. +#[derive(Clone, Debug)] +pub struct WriterOptions { + pub brand: Brand, + /// Movie timescale (`mvhd`); default 1000. + pub movie_timescale: u32, + /// Text metadata written to `moov/udta` (keys like `©nam`, `©ART`, `©cmt`, `©too`). + pub metadata: Vec<(String, String)>, +} + +impl WriterOptions { + pub fn new(brand: Brand) -> Self { + WriterOptions { brand, movie_timescale: 1000, metadata: Vec::new() } + } +} + +/// Per-track configuration. +#[derive(Clone, Debug)] +pub struct TrackConfig { + pub entry: SampleEntry, + /// Media timescale. For PCM it must equal the sample rate (one tick per audio frame). + pub timescale: u32, + /// ISO 639-2/T language (`und` by default). + pub language: String, + /// Explicit edit list (segment durations in the movie timescale). Takes precedence over `media_start`. + pub edits: Vec, + /// Convenience: create one edit starting at this media time (e.g. AAC encoder delay / priming, + /// or the first pts of a B-frame stream) spanning the rest of the media. + pub media_start: Option, + /// `hdlr` name; a sensible default is used when `None`. + pub handler_name: Option, +} + +impl TrackConfig { + pub fn new(entry: SampleEntry, timescale: u32) -> Self { + TrackConfig { entry, timescale, language: "und".into(), edits: Vec::new(), media_start: None, handler_name: None } + } +} + +/// A sample handed to the muxers. +#[derive(Clone, Copy, Debug)] +pub struct WriteSample<'a> { + pub data: &'a [u8], + /// Duration in the media timescale. Ignored for PCM (each frame lasts one tick). + pub duration: u32, + /// pts − dts in the media timescale. + pub composition_offset: i32, + pub is_sync: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Handler { + Video, + Audio, + Timecode, + Other, +} + +fn handler_of(e: &SampleEntry) -> Handler { + if matches!(e.codec, CodecConfig::Timecode(_)) { + Handler::Timecode + } else if e.video.is_some() { + Handler::Video + } else if e.audio.is_some() { + Handler::Audio + } else { + Handler::Other + } +} + +/// Accumulated sample tables for one track. +#[derive(Clone, Debug, Default)] +struct Tables { + stts: Vec<(u32, u32)>, + ctts: Vec<(u32, i32)>, + sizes: Vec, + /// Constant sample size (PCM) — then `sizes` is unused. + const_size: Option, + sample_count: u64, + /// 1-based sync sample numbers. + sync: Vec, + all_sync: bool, + /// (offset relative to file start before shifting, sample count) + chunks: Vec<(u64, u32)>, + media_duration: u64, + /// End of the last sample on the composition timeline (decode time + composition offset + + /// duration, maximum over samples). + comp_end: u64, +} + +impl Tables { + fn new() -> Self { + Tables { all_sync: true, ..Default::default() } + } + fn push_run(runs: &mut Vec<(u32, T)>, count: u32, v: T) { + if let Some(last) = runs.last_mut() + && last.1 == v + && last.0.checked_add(count).is_some() + { + last.0 += count; + return; + } + runs.push((count, v)); + } +} + +struct WTrack { + cfg: TrackConfig, + id: u32, + handler: Handler, + pcm_frame_bytes: Option, + t: Tables, + /// Track id of a linked timecode track (`tref/tmcd`). + tmcd_ref: Option, + /// For timecode tracks: (video track index, start frame). + tmcd_source: Option<(usize, u32)>, +} + +fn pack_language(lang: &str) -> u16 { + let b = lang.as_bytes(); + if b.len() == 3 && b.iter().all(|c| c.is_ascii_lowercase()) { + (((b[0] - 0x60) as u16) << 10) | (((b[1] - 0x60) as u16) << 5) | ((b[2] - 0x60) as u16) + } else { + 0x55C4 // "und" + } +} + +fn rescale(v: u64, from: u32, to: u32) -> u64 { + if from == 0 { + return 0; + } + (v as u128 * to as u128 / from as u128).min(u64::MAX as u128) as u64 +} + +const MATRIX: [i32; 9] = [0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x4000_0000]; + +fn write_ftyp(b: &mut BoxBuf, brand: Brand) { + let m = b.start(b"ftyp"); + match brand { + Brand::Mp4 => { + b.bytes(b"isom"); + b.u32(0x200); + for c in [b"isom", b"iso2", b"avc1", b"mp41", b"mp42"] { + b.bytes(c); + } + } + Brand::Mov => { + b.bytes(b"qt "); + b.u32(0x200); + b.bytes(b"qt "); + } + } + b.end(m); +} + +fn movie_duration(tracks: &[WTrack], movie_ts: u32) -> u64 { + tracks.iter().map(|t| presentation_duration(t, movie_ts)).max().unwrap_or(0) +} + +fn effective_edits(t: &WTrack, movie_ts: u32) -> Vec { + if !t.cfg.edits.is_empty() { + return t.cfg.edits.clone(); + } + if let Some(start) = t.cfg.media_start { + // presentation runs from `start` to the end of the last sample in composition order (with + // B-frame reordering that is past the end of the decode timeline) + let rest = t.t.comp_end.max(t.t.media_duration).saturating_sub(start.max(0) as u64); + return vec![Edit { segment_duration: rescale(rest, t.cfg.timescale, movie_ts), media_time: start, media_rate: 0x10000 }]; + } + Vec::new() +} + +fn presentation_duration(t: &WTrack, movie_ts: u32) -> u64 { + let e = effective_edits(t, movie_ts); + if e.is_empty() { rescale(t.t.media_duration, t.cfg.timescale, movie_ts) } else { e.iter().map(|e| e.segment_duration).sum() } +} + +fn write_mvhd(b: &mut BoxBuf, timescale: u32, duration: u64, next_id: u32) { + let v1 = duration > u32::MAX as u64; + let m = b.start_full(b"mvhd", v1 as u8, 0); + if v1 { + b.u64(0); + b.u64(0); + b.u32(timescale); + b.u64(duration); + } else { + b.u32(0); + b.u32(0); + b.u32(timescale); + b.u32(duration as u32); + } + b.u32(0x10000); + b.u16(0x100); + b.zeros(10); + for v in MATRIX { + b.i32(v); + } + b.zeros(24); + b.u32(next_id); + b.end(m); +} + +fn write_trak(b: &mut BoxBuf, t: &WTrack, movie_ts: u32, qt: bool, shift: u64, force_co64: bool) -> Result<()> { + let trak = b.start(b"trak"); + let dur = presentation_duration(t, movie_ts); + let v1 = dur > u32::MAX as u64; + let m = b.start_full(b"tkhd", v1 as u8, 3); + if v1 { + b.u64(0); + b.u64(0); + b.u32(t.id); + b.u32(0); + b.u64(dur); + } else { + b.u32(0); + b.u32(0); + b.u32(t.id); + b.u32(0); + b.u32(dur as u32); + } + b.zeros(8); + b.i16(0); + b.u16(0); + b.i16(if t.handler == Handler::Audio { 0x100 } else { 0 }); + b.u16(0); + for v in MATRIX { + b.i32(v); + } + let (w, h) = match &t.cfg.entry.video { + Some(v) => { + let w = match v.pixel_aspect { + Some((hs, vs)) if hs > 0 && vs > 0 => (v.width as u64 * hs as u64 / vs as u64) as u32, + _ => v.width as u32, + }; + (w, v.height as u32) + } + None => (0, 0), + }; + b.u32(w << 16); + b.u32(h << 16); + b.end(m); + let edits = effective_edits(t, movie_ts); + if !edits.is_empty() { + let e = b.start(b"edts"); + let v1 = edits.iter().any(|e| e.segment_duration > u32::MAX as u64 || e.media_time > i32::MAX as i64 || e.media_time < i32::MIN as i64); + let l = b.start_full(b"elst", v1 as u8, 0); + b.u32(edits.len() as u32); + for ed in &edits { + if v1 { + b.u64(ed.segment_duration); + b.i64(ed.media_time); + } else { + b.u32(ed.segment_duration as u32); + b.i32(ed.media_time as i32); + } + b.i32(ed.media_rate); + } + b.end(l); + b.end(e); + } + if let Some(id) = t.tmcd_ref { + let r = b.start(b"tref"); + let m = b.start(b"tmcd"); + b.u32(id); + b.end(m); + b.end(r); + } + let mdia = b.start(b"mdia"); + let mv1 = t.t.media_duration > u32::MAX as u64; + let m = b.start_full(b"mdhd", mv1 as u8, 0); + if mv1 { + b.u64(0); + b.u64(0); + b.u32(t.cfg.timescale); + b.u64(t.t.media_duration); + } else { + b.u32(0); + b.u32(0); + b.u32(t.cfg.timescale); + b.u32(t.t.media_duration as u32); + } + b.u16(pack_language(&t.cfg.language)); + b.u16(0); + b.end(m); + let (htype, default_name) = match t.handler { + Handler::Video => (b"vide", "VideoHandler"), + Handler::Audio => (b"soun", "SoundHandler"), + Handler::Timecode => (b"tmcd", "TimeCodeHandler"), + Handler::Other => (b"meta", "MetaHandler"), + }; + let name = t.cfg.handler_name.clone().unwrap_or_else(|| default_name.into()); + let m = b.start_full(b"hdlr", 0, 0); + b.bytes(if qt { b"mhlr" } else { &[0, 0, 0, 0] }); + b.bytes(htype); + b.zeros(12); + if qt { + let n = name.len().min(255); + b.u8(n as u8); + b.bytes(&name.as_bytes()[..n]); + } else { + b.bytes(name.as_bytes()); + b.u8(0); + } + b.end(m); + let minf = b.start(b"minf"); + match t.handler { + Handler::Video => { + let m = b.start_full(b"vmhd", 0, 1); + b.zeros(8); + b.end(m); + } + Handler::Audio => { + let m = b.start_full(b"smhd", 0, 0); + b.zeros(4); + b.end(m); + } + Handler::Timecode => { + let g = b.start(b"gmhd"); + let m = b.start_full(b"gmin", 0, 0); + b.u16(0x40); + b.u16(0x8000); + b.u16(0x8000); + b.u16(0x8000); + b.i16(0); + b.u16(0); + b.end(m); + let tm = b.start(b"tmcd"); + let m = b.start_full(b"tcmi", 0, 0); + b.u16(0); + b.u16(0); + b.u16(12); + b.u16(0); + b.bytes(&[0xFF; 6]); + b.zeros(6); + b.u8(0); + b.end(m); + b.end(tm); + b.end(g); + } + Handler::Other => { + let m = b.start_full(b"nmhd", 0, 0); + b.end(m); + } + } + if qt { + // QuickTime data handler reference. + let m = b.start_full(b"hdlr", 0, 0); + b.bytes(b"dhlr"); + b.bytes(b"url "); + b.zeros(12); + b.u8(0); + b.end(m); + } + let d = b.start(b"dinf"); + let r = b.start_full(b"dref", 0, 0); + b.u32(1); + let u = b.start_full(b"url ", 0, 1); + b.end(u); + b.end(r); + b.end(d); + let stbl = b.start(b"stbl"); + let m = b.start_full(b"stsd", 0, 0); + b.u32(1); + t.cfg.entry.write(b, qt)?; + b.end(m); + let tb = &t.t; + let m = b.start_full(b"stts", 0, 0); + b.u32(tb.stts.len() as u32); + for &(c, d) in &tb.stts { + b.u32(c); + b.u32(d); + } + b.end(m); + if tb.ctts.iter().any(|&(_, o)| o != 0) { + let neg = tb.ctts.iter().any(|&(_, o)| o < 0); + let m = b.start_full(b"ctts", neg as u8, 0); + b.u32(tb.ctts.len() as u32); + for &(c, o) in &tb.ctts { + b.u32(c); + b.i32(o); + } + b.end(m); + } + if !tb.all_sync { + let m = b.start_full(b"stss", 0, 0); + b.u32(tb.sync.len() as u32); + for &s in &tb.sync { + b.u32(s); + } + b.end(m); + } + let m = b.start_full(b"stsc", 0, 0); + let mut runs: Vec<(u32, u32)> = Vec::new(); + for (i, &(_, n)) in tb.chunks.iter().enumerate() { + if runs.last().is_none_or(|r| r.1 != n) { + runs.push((i as u32 + 1, n)); + } + } + b.u32(runs.len() as u32); + for (first, n) in runs { + b.u32(first); + b.u32(n); + b.u32(1); + } + b.end(m); + let m = b.start_full(b"stsz", 0, 0); + match tb.const_size { + Some(s) => { + b.u32(s); + b.u32(tb.sample_count.min(u32::MAX as u64) as u32); + } + None => { + let first = tb.sizes.first().copied().unwrap_or(0); + // A uniform size of 0 would mean "table follows" (ISO/IEC 14496-12 §8.7.3), so + // all-empty samples must still be written as a table. + if first != 0 && !tb.sizes.is_empty() && tb.sizes.iter().all(|&s| s == first) { + b.u32(first); + b.u32(tb.sizes.len() as u32); + } else { + b.u32(0); + b.u32(tb.sizes.len() as u32); + for &s in &tb.sizes { + b.u32(s); + } + } + } + } + b.end(m); + let co64 = force_co64 || tb.chunks.iter().any(|&(o, _)| o + shift > u32::MAX as u64); + let m = b.start_full(if co64 { b"co64" } else { b"stco" }, 0, 0); + b.u32(tb.chunks.len() as u32); + for &(o, _) in &tb.chunks { + if co64 { + b.u64(o + shift); + } else { + b.u32((o + shift) as u32); + } + } + b.end(m); + b.end(stbl); + b.end(minf); + b.end(mdia); + b.end(trak); + Ok(()) +} + +fn write_udta(b: &mut BoxBuf, md: &[(String, String)]) { + let items: Vec<([u8; 4], &str)> = md + .iter() + .filter_map(|(k, v)| { + let mut chars = k.chars(); + if chars.next()? != '©' { + return None; + } + let rest: Vec = chars.map(|c| c as u32 as u8).collect(); + (rest.len() == 3).then(|| ([0xA9, rest[0], rest[1], rest[2]], v.as_str())) + }) + .collect(); + if items.is_empty() { + return; + } + let u = b.start(b"udta"); + for (k, v) in items { + let m = b.start(&k); + b.u16(v.len() as u16); + b.u16(0x55C4); + b.bytes(v.as_bytes()); + b.end(m); + } + b.end(u); +} + +fn validate_track(cfg: &TrackConfig) -> Result> { + if cfg.timescale == 0 { + return Err(Error::Mux("timescale must be non-zero".into())); + } + if let CodecConfig::Pcm(p) = &cfg.entry.codec { + if p.bits == 0 || p.channels == 0 { + return Err(Error::Mux("PCM needs bits and channels".into())); + } + if (p.sample_rate - cfg.timescale as f64).abs() > 0.5 { + return Err(Error::Mux("PCM track timescale must equal the sample rate".into())); + } + return Ok(Some(p.bytes_per_frame())); + } + Ok(None) +} + +/// Progressive MP4/MOV writer: `ftyp`, then one `mdat` streamed to `W`, then `moov` at [`finish`](Self::finish) +/// (or moved in front of `mdat` by [`finish_faststart`](Self::finish_faststart)). +pub struct Mp4Writer { + w: W, + opts: WriterOptions, + tracks: Vec, + /// Offset of the placeholder box (8 bytes) preceding the mdat header. + pad_pos: u64, + mdat_pos: u64, + pos: u64, + last_track: Option, +} + +impl Mp4Writer { + /// Start a file: writes `ftyp`, a placeholder box and the `mdat` header. + pub fn new(mut w: W, opts: WriterOptions) -> Result { + let start = w.stream_position()?; + let mut b = BoxBuf::new(); + write_ftyp(&mut b, opts.brand); + let pad_pos = start + b.len() as u64; + b.leaf(if opts.brand == Brand::Mov { b"wide" } else { b"free" }, &[]); + let mdat_pos = start + b.len() as u64; + b.leaf(b"mdat", &[]); + w.write_all(&b.buf)?; + let pos = start + b.len() as u64; + Ok(Mp4Writer { w, opts, tracks: Vec::new(), pad_pos, mdat_pos, pos, last_track: None }) + } + + /// Add a track; returns its index (track ids are index + 1). + pub fn add_track(&mut self, cfg: TrackConfig) -> Result { + let pcm = validate_track(&cfg)?; + let handler = handler_of(&cfg.entry); + let mut t = Tables::new(); + t.const_size = pcm; + self.tracks.push(WTrack { id: self.tracks.len() as u32 + 1, handler, pcm_frame_bytes: pcm, t, cfg, tmcd_ref: None, tmcd_source: None }); + Ok(self.tracks.len() - 1) + } + + /// Add a QuickTime timecode track referenced by `video_track`. Its single sample (the start + /// frame number) is written at finish, spanning the video track's duration. + pub fn add_timecode_track(&mut self, video_track: usize, tc: TimecodeConfig, start_frame: u32) -> Result { + if video_track >= self.tracks.len() { + return Err(Error::NoSuchTrack(video_track)); + } + if tc.timescale == 0 || tc.frame_duration == 0 || tc.frames_per_second == 0 { + return Err(Error::Mux("timecode needs timescale, frame duration and fps".into())); + } + let entry = SampleEntry { + format: crate::isobmff::bytes::FourCc(*b"tmcd"), + data_reference_index: 1, + codec: CodecConfig::Timecode(tc.clone()), + video: None, + audio: None, + bitrate: None, + }; + let idx = self.add_track(TrackConfig::new(entry, tc.timescale))?; + self.tracks[idx].tmcd_source = Some((video_track, start_frame)); + let id = self.tracks[idx].id; + self.tracks[video_track].tmcd_ref = Some(id); + Ok(idx) + } + + /// Append one sample to track `track` (decode order). For PCM tracks `data` may hold any whole + /// number of audio frames. + pub fn write_sample(&mut self, track: usize, s: WriteSample) -> Result<()> { + let tr = self.tracks.get(track).ok_or(Error::NoSuchTrack(track))?; + if tr.tmcd_source.is_some() { + return Err(Error::Mux("timecode track samples are written automatically".into())); + } + let (count, dur) = match tr.pcm_frame_bytes { + Some(fb) => { + if !s.data.len().is_multiple_of(fb as usize) { + return Err(Error::Mux("PCM data is not a whole number of frames".into())); + } + ((s.data.len() / fb as usize) as u32, 1u32) + } + None => { + if s.data.len() > u32::MAX as usize { + return Err(Error::Mux("sample larger than 4 GiB".into())); + } + (1, s.duration) + } + }; + if count == 0 { + return Ok(()); + } + self.w.write_all(s.data)?; + let offset = self.pos; + self.pos += s.data.len() as u64; + let contiguous = self.last_track == Some(track); + self.last_track = Some(track); + let tr = &mut self.tracks[track]; + let t = &mut tr.t; + Tables::push_run(&mut t.stts, count, dur); + Tables::push_run(&mut t.ctts, count, if tr.pcm_frame_bytes.is_some() { 0 } else { s.composition_offset }); + if tr.pcm_frame_bytes.is_none() { + t.sizes.push(s.data.len() as u32); + if s.is_sync { + t.sync.push(t.sample_count as u32 + 1); + } else { + t.all_sync = false; + } + } + t.sample_count += count as u64; + let ct = if tr.pcm_frame_bytes.is_some() { 0 } else { s.composition_offset as i64 }; + t.comp_end = t.comp_end.max((t.media_duration as i64 + ct + count as i64 * dur as i64).max(0) as u64); + t.media_duration += count as u64 * dur as u64; + match t.chunks.last_mut() { + Some(c) if contiguous => c.1 += count, + _ => t.chunks.push((offset, count)), + } + Ok(()) + } + + fn finish_mdat(&mut self) -> Result<()> { + // Timecode samples. + for i in 0..self.tracks.len() { + let Some((vi, start)) = self.tracks[i].tmcd_source else { continue }; + let v = &self.tracks[vi]; + let vd = v.t.media_duration; + let vts = v.cfg.timescale; + let tts = self.tracks[i].cfg.timescale; + let tc_dur = match &self.tracks[i].cfg.entry.codec { + CodecConfig::Timecode(tc) => tc.frame_duration, + _ => 1, + }; + let d = rescale(vd, vts, tts).max(tc_dur as u64).min(u32::MAX as u64) as u32; + self.w.write_all(&start.to_be_bytes())?; + let off = self.pos; + self.pos += 4; + self.last_track = None; + let t = &mut self.tracks[i].t; + t.stts.push((1, d)); + t.sizes.push(4); + t.sync.push(1); + t.sample_count = 1; + t.media_duration = d as u64; + t.chunks.push((off, 1)); + } + let size = self.pos - self.mdat_pos; + if size <= u32::MAX as u64 { + self.w.seek(SeekFrom::Start(self.mdat_pos))?; + self.w.write_all(&(size as u32).to_be_bytes())?; + } else { + // Merge the placeholder into a 16-byte large-size mdat header. + let mut h = [0u8; 16]; + h[..4].copy_from_slice(&1u32.to_be_bytes()); + h[4..8].copy_from_slice(b"mdat"); + h[8..].copy_from_slice(&(self.pos - self.pad_pos).to_be_bytes()); + self.w.seek(SeekFrom::Start(self.pad_pos))?; + self.w.write_all(&h)?; + } + self.w.seek(SeekFrom::Start(self.pos))?; + Ok(()) + } + + fn build_moov(&self, shift: u64, force_co64: bool) -> Result> { + let qt = self.opts.brand == Brand::Mov; + let ts = self.opts.movie_timescale.max(1); + let mut b = BoxBuf::new(); + let m = b.start(b"moov"); + write_mvhd(&mut b, ts, movie_duration(&self.tracks, ts), self.tracks.len() as u32 + 1); + for t in &self.tracks { + write_trak(&mut b, t, ts, qt, shift, force_co64)?; + } + write_udta(&mut b, &self.opts.metadata); + b.end(m); + Ok(b.buf) + } + + /// Finish the file with `moov` after `mdat`. Returns the underlying writer. + pub fn finish(mut self) -> Result { + self.finish_mdat()?; + let moov = self.build_moov(0, false)?; + self.w.write_all(&moov)?; + self.w.flush()?; + Ok(self.w) + } +} + +impl Mp4Writer { + /// Finish with `moov` in front of `mdat` ("fast start"): media data is shifted forward in place. + pub fn finish_faststart(mut self) -> Result { + self.finish_mdat()?; + let mut moov = self.build_moov(0, false)?; + let mut force = false; + loop { + let shift = moov.len() as u64; + let next = self.build_moov(shift, force)?; + if next.len() == moov.len() { + moov = next; + break; + } + // Offsets crossed the 32-bit boundary: switch to co64 (size only grows, so this converges). + force = true; + moov = next; + } + let shift = moov.len() as u64; + let start = self.pad_pos; + let end = self.pos; + let mut buf = vec![0u8; 1 << 20]; + let mut cur = end; + while cur > start { + let n = (cur - start).min(buf.len() as u64) as usize; + cur -= n as u64; + self.w.seek(SeekFrom::Start(cur))?; + self.w.read_exact(&mut buf[..n])?; + self.w.seek(SeekFrom::Start(cur + shift))?; + self.w.write_all(&buf[..n])?; + } + self.w.seek(SeekFrom::Start(start))?; + self.w.write_all(&moov)?; + self.w.seek(SeekFrom::Start(end + shift))?; + self.w.flush()?; + Ok(self.w) + } +} + +/// Buffered fragment sample: (data, duration, composition offset, sync). +type PendingSample = (Vec, u32, i32, bool); + +/// Fragmented MP4 writer: an init segment (`ftyp` + `moov` with `mvex`) followed by +/// `moof`+`mdat` fragments. Samples are buffered until [`flush_fragment`](Self::flush_fragment). +pub struct FragmentedWriter { + w: W, + opts: WriterOptions, + tracks: Vec, + pending: Vec>, + next_dts: Vec, + seq: u32, + init_written: bool, +} + +impl FragmentedWriter { + pub fn new(w: W, opts: WriterOptions) -> Self { + FragmentedWriter { w, opts, tracks: Vec::new(), pending: Vec::new(), next_dts: Vec::new(), seq: 0, init_written: false } + } + + /// Add a track (before the first fragment). PCM is not supported in fragments. + pub fn add_track(&mut self, cfg: TrackConfig) -> Result { + if self.init_written { + return Err(Error::Mux("tracks must be added before the first fragment".into())); + } + if validate_track(&cfg)?.is_some() { + return Err(Error::Mux("PCM is not supported by the fragmented writer".into())); + } + let handler = handler_of(&cfg.entry); + self.tracks.push(WTrack { + id: self.tracks.len() as u32 + 1, + handler, + pcm_frame_bytes: None, + t: Tables::new(), + cfg, + tmcd_ref: None, + tmcd_source: None, + }); + self.pending.push(Vec::new()); + self.next_dts.push(0); + Ok(self.tracks.len() - 1) + } + + fn write_init(&mut self) -> Result<()> { + let qt = self.opts.brand == Brand::Mov; + let ts = self.opts.movie_timescale.max(1); + let mut b = BoxBuf::new(); + write_ftyp(&mut b, self.opts.brand); + let m = b.start(b"moov"); + write_mvhd(&mut b, ts, 0, self.tracks.len() as u32 + 1); + for t in &self.tracks { + write_trak(&mut b, t, ts, qt, 0, false)?; + } + let mv = b.start(b"mvex"); + for t in &self.tracks { + let x = b.start_full(b"trex", 0, 0); + b.u32(t.id); + b.u32(1); + b.u32(0); + b.u32(0); + b.u32(0); + b.end(x); + } + b.end(mv); + write_udta(&mut b, &self.opts.metadata); + b.end(m); + self.w.write_all(&b.buf)?; + self.init_written = true; + Ok(()) + } + + /// Buffer a sample for the next fragment. + pub fn write_sample(&mut self, track: usize, s: WriteSample) -> Result<()> { + let p = self.pending.get_mut(track).ok_or(Error::NoSuchTrack(track))?; + if s.data.len() > u32::MAX as usize { + return Err(Error::Mux("sample larger than 4 GiB".into())); + } + p.push((s.data.to_vec(), s.duration, s.composition_offset, s.is_sync)); + Ok(()) + } + + fn build_moof(&self, data_offsets: &[i32]) -> Vec { + let mut b = BoxBuf::new(); + let m = b.start(b"moof"); + let h = b.start_full(b"mfhd", 0, 0); + b.u32(self.seq); + b.end(h); + for (i, t) in self.tracks.iter().enumerate() { + let p = &self.pending[i]; + if p.is_empty() { + continue; + } + let tf = b.start(b"traf"); + let h = b.start_full(b"tfhd", 0, 0x20000); + b.u32(t.id); + b.end(h); + let h = b.start_full(b"tfdt", 1, 0); + b.u64(self.next_dts[i]); + b.end(h); + let neg = p.iter().any(|s| s.2 < 0); + let h = b.start_full(b"trun", neg as u8, 0x1 | 0x100 | 0x200 | 0x400 | 0x800); + b.u32(p.len() as u32); + b.i32(data_offsets[i]); + for (d, dur, cto, sync) in p { + b.u32(*dur); + b.u32(d.len() as u32); + b.u32(if *sync { 0x0200_0000 } else { 0x0101_0000 }); + b.i32(*cto); + } + b.end(h); + b.end(tf); + } + b.end(m); + b.buf + } + + /// Write all buffered samples as one `moof` + `mdat` fragment (no-op if nothing is buffered). + pub fn flush_fragment(&mut self) -> Result<()> { + if !self.init_written { + self.write_init()?; + } + if self.pending.iter().all(|p| p.is_empty()) { + return Ok(()); + } + self.seq += 1; + let probe = self.build_moof(&vec![0; self.tracks.len()]); + let data_len: u64 = self.pending.iter().flatten().map(|s| s.0.len() as u64).sum(); + let large = data_len + 8 > u32::MAX as u64; + let hdr = if large { 16 } else { 8 }; + let mut offs = Vec::with_capacity(self.tracks.len()); + let mut acc = probe.len() as u64 + hdr; + for p in &self.pending { + offs.push(i32::try_from(acc).map_err(|_| Error::Mux("fragment too large".into()))?); + acc += p.iter().map(|s| s.0.len() as u64).sum::(); + } + let moof = self.build_moof(&offs); + self.w.write_all(&moof)?; + if large { + self.w.write_all(&1u32.to_be_bytes())?; + self.w.write_all(b"mdat")?; + self.w.write_all(&(data_len + 16).to_be_bytes())?; + } else { + self.w.write_all(&((data_len + 8) as u32).to_be_bytes())?; + self.w.write_all(b"mdat")?; + } + for (i, p) in self.pending.iter_mut().enumerate() { + for (d, dur, _, _) in p.drain(..) { + self.w.write_all(&d)?; + self.next_dts[i] += dur as u64; + } + } + Ok(()) + } + + /// Flush remaining samples and return the writer. + pub fn finish(mut self) -> Result { + self.flush_fragment()?; + self.w.flush()?; + Ok(self.w) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::isobmff::codec::{AvcConfig, PcmConfig}; + use std::io::Cursor; + + fn avc_entry() -> SampleEntry { + SampleEntry::avc(AvcConfig::new(vec![vec![0x67, 0x42, 0, 0x1E]], vec![vec![0x68, 0xCE]], 4), 64, 48) + } + + #[test] + fn language_pack() { + assert_eq!(pack_language("und"), 0x55C4); + assert_eq!(pack_language("eng"), 0x15C7); + assert_eq!(pack_language("EN"), 0x55C4); + } + + #[test] + fn simple_roundtrip_and_faststart() { + for fast in [false, true] { + let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let v = w.add_track(TrackConfig::new(avc_entry(), 12800)).unwrap(); + let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 2, sample_rate: 48000.0 }; + let a = w.add_track(TrackConfig::new(SampleEntry::pcm(pcm), 48000)).unwrap(); + for i in 0..10u8 { + let data = vec![i; 100 + i as usize]; + w.write_sample( + v, + WriteSample { data: &data, duration: 512, composition_offset: if i % 2 == 0 { 1024 } else { 0 }, is_sync: i % 5 == 0 }, + ) + .unwrap(); + w.write_sample(a, WriteSample { data: &[i; 400], duration: 0, composition_offset: 0, is_sync: true }).unwrap(); + } + let out = if fast { w.finish_faststart() } else { w.finish() }.unwrap().into_inner(); + let f = crate::isobmff::open(out.as_slice()).unwrap(); + assert_eq!(f.tracks.len(), 2); + let t = &f.tracks[0]; + assert_eq!(t.samples.len(), 10); + for (i, s) in t.samples.iter().enumerate() { + assert_eq!(s.size as usize, 100 + i); + assert_eq!(s.dts, 512 * i as i64); + assert_eq!(s.is_sync, i % 5 == 0); + assert_eq!(f.read_sample(out.as_slice(), 0, i).unwrap(), vec![i as u8; 100 + i]); + } + let at = &f.tracks[1]; + assert!(at.pcm_chunked); + assert_eq!(at.samples.len(), 10); + assert_eq!(at.samples[3].size, 400); + assert_eq!(at.samples[3].duration, 100); + assert_eq!(f.read_sample(out.as_slice(), 1, 3).unwrap(), vec![3u8; 400]); + if fast { + assert!(f.moov_offset < t.samples[0].offset); + } + } + } + + #[test] + fn fragmented_roundtrip() { + let mut w = FragmentedWriter::new(Vec::new(), WriterOptions::new(Brand::Mp4)); + let v = w.add_track(TrackConfig::new(avc_entry(), 1000)).unwrap(); + for i in 0..6u8 { + w.write_sample(v, WriteSample { data: &[i; 10], duration: 40, composition_offset: -(i as i32 % 2), is_sync: i % 3 == 0 }).unwrap(); + if i % 3 == 2 { + w.flush_fragment().unwrap(); + } + } + let out = w.finish().unwrap(); + let f = crate::isobmff::open(out.as_slice()).unwrap(); + assert!(f.fragmented); + assert_eq!(f.fragment_count, 2); + let t = &f.tracks[0]; + assert_eq!(t.samples.len(), 6); + for (i, s) in t.samples.iter().enumerate() { + assert_eq!(s.dts, 40 * i as i64); + assert_eq!(s.pts, s.dts - (i as i64 % 2)); + assert_eq!(s.is_sync, i % 3 == 0); + assert_eq!(f.read_sample(out.as_slice(), 0, i).unwrap(), vec![i as u8; 10]); + } + assert_eq!(t.sync_sample_before(5), 3); + } +} diff --git a/crates/video/src/isobmff/source.rs b/crates/video/src/isobmff/source.rs new file mode 100644 index 00000000..0dad82b2 --- /dev/null +++ b/crates/video/src/isobmff/source.rs @@ -0,0 +1,123 @@ +//! Random-access byte sources for the demuxer. + +use std::io; +use std::sync::Arc; + +/// Random-access, read-only byte source (file, in-memory buffer, web Blob…). +/// +/// Implementations must be able to serve reads at arbitrary offsets through `&self`. +pub trait ByteSource { + /// Total length in bytes. + fn len(&self) -> u64; + /// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end. + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>; + /// True if the source holds no bytes. + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +fn eof() -> io::Error { + io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source") +} + +impl ByteSource for [u8] { + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + let start = usize::try_from(offset).map_err(|_| eof())?; + let end = start.checked_add(buf.len()).ok_or_else(eof)?; + let src = self.get(start..end).ok_or_else(eof)?; + buf.copy_from_slice(src); + Ok(()) + } +} + +impl ByteSource for Vec { + fn len(&self) -> u64 { + self.as_slice().len() as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.as_slice().read_at(offset, buf) + } +} + +impl ByteSource for &T { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} + +impl ByteSource for Arc { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} + +impl ByteSource for Box { + fn len(&self) -> u64 { + (**self).len() + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + (**self).read_at(offset, buf) + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl ByteSource for std::fs::File { + fn len(&self) -> u64 { + self.metadata().map(|m| m.len()).unwrap_or(0) + } + #[cfg(unix)] + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + std::os::unix::fs::FileExt::read_exact_at(self, buf, offset) + } + #[cfg(windows)] + fn read_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> { + use std::os::windows::fs::FileExt; + while !buf.is_empty() { + let n = self.seek_read(buf, offset)?; + if n == 0 { + return Err(eof()); + } + offset += n as u64; + buf = &mut buf[n..]; + } + Ok(()) + } + #[cfg(not(any(unix, windows)))] + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + use std::io::{Read, Seek, SeekFrom}; + let mut f = self; + f.seek(SeekFrom::Start(offset))?; + f.read_exact(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slice_reads() { + let v = vec![1u8, 2, 3, 4]; + let mut b = [0u8; 2]; + v.read_at(1, &mut b).unwrap(); + assert_eq!(b, [2, 3]); + assert!(v.read_at(3, &mut b).is_err()); + assert!(v.read_at(u64::MAX, &mut b).is_err()); + let a: Arc<[u8]> = Arc::from(v.clone()); + a.read_at(2, &mut b).unwrap(); + assert_eq!(b, [3, 4]); + assert_eq!(ByteSource::len(&a), 4); + let s: &[u8] = &v; + assert_eq!(ByteSource::len(&s), 4); + } +} diff --git a/crates/video/src/lib.rs b/crates/video/src/lib.rs new file mode 100644 index 00000000..17a732a1 --- /dev/null +++ b/crates/video/src/lib.rs @@ -0,0 +1,62 @@ +//! SoundCraft video: open a movie (MP4 / QuickTime MOV) from bytes, report its picture format, and +//! decode the frame at a time to RGBA for the Video window and the Video track's thumbnails. +//! +//! - [`Movie::open`] (bytes) / [`Movie::open_source`] (any random-access [`ByteSource`]). +//! - [`Movie::info`]: size, duration, frame rate, codec, whether it can be decoded. +//! - [`Movie::frame_at`]: the frame shown at a time, decoded forward from the previous keyframe, +//! with a small LRU of decoded frames (sequential playback keeps decoding forward). +//! - [`Movie::thumbnail_at`] / [`Movie::thumbnails`]: low-resolution timeline pictures (keyframes). +//! - Unsupported codecs open fine (so the timeline can show the clip) but frame requests return +//! [`VideoError::Unsupported`]. Malformed files return errors; nothing here panics. +//! +//! Picture codecs: H.264 / AVC (Baseline, Main, High; progressive 8-bit 4:2:0; CAVLC and CABAC), +//! Apple ProRes (422 Proxy/LT/Standard/HQ, 4444/XQ) and Motion JPEG (`jpeg`/`mjpa`). +//! +//! ## Origin +//! +//! The container, H.264 and ProRes code is a copy of the clean-room crates of our sibling app +//! FilmCraft (`storytold/filmcraft`, same organisation, MIT OR Apache-2.0): `filmcraft-bitstream` +//! → [`bitstream`], `filmcraft-isobmff` → [`isobmff`], `filmcraft-h264` → [`h264`] (without its +//! test-only synthetic stream module), `filmcraft-prores` → [`prores`]. The crafting apps share no +//! code as dependencies; each keeps its own copy. Only the module paths were changed; fixes found +//! here should be offered back upstream. [`Movie`], the pixel conversion and the frame caches are +//! SoundCraft's own. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +// Codec code indexes fixed-size blocks; index loops read more clearly than iterator chains there. +#![allow(clippy::needless_range_loop, clippy::too_many_arguments)] + +pub mod bitstream; +pub mod convert; +pub mod h264; +pub mod isobmff; +mod movie; +pub mod prores; +pub mod synth; + +pub use isobmff::ByteSource; +pub use movie::{Frame, Movie, MovieInfo, SharedSource, fit_size}; + +/// Errors from opening or decoding a movie. +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum VideoError { + #[error("not a readable movie: {0}")] + Container(String), + #[error("the movie has no video track")] + NoVideoTrack, + #[error("unsupported: {0}")] + Unsupported(String), + #[error("could not decode the picture: {0}")] + Decode(String), + #[error("no picture at that time")] + OutOfRange, +} + +pub type Result = std::result::Result; + +/// Open a movie and return its description (convenience for importers). +pub fn probe(bytes: &[u8]) -> Result { + Movie::open(bytes.to_vec()).map(|m| m.info().clone()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/video/src/movie.rs b/crates/video/src/movie.rs new file mode 100644 index 00000000..00541407 --- /dev/null +++ b/crates/video/src/movie.rs @@ -0,0 +1,506 @@ +//! [`Movie`]: open an MP4/MOV, report its picture format, and decode frames at times to RGBA. + +use crate::convert::{Matrix, planar_to_rgba, scale_rgba}; +use crate::isobmff::{self, ByteSource, CodecConfig, Mp4File, TrackKind}; +use crate::{Result, VideoError}; +use std::collections::VecDeque; +use std::sync::Arc; + +/// A random-access byte source a [`Movie`] can read from (in-memory bytes, or a file on native). +pub type SharedSource = Arc; + +/// Largest picture we decode (bounds every allocation derived from the file). +const MAX_PIXELS: u64 = 8192 * 8192; + +/// A decoded picture, packed RGBA (8 bits per channel, `width * height * 4` bytes). +#[derive(Debug, Clone, PartialEq)] +pub struct Frame { + pub width: u32, + pub height: u32, + pub rgba: Vec, + /// Presentation index (0-based, in display order). + pub index: usize, + /// Presentation time in seconds from the start of the movie. + pub time: f64, +} + +impl Frame { + /// Downscaled copy that fits inside `max_w` × `max_h` (aspect kept; never upscales). + pub fn fit(&self, max_w: u32, max_h: u32) -> Frame { + let (w, h) = fit_size(self.width, self.height, max_w, max_h); + if (w, h) == (self.width, self.height) { + return self.clone(); + } + Frame { + width: w, + height: h, + rgba: scale_rgba(&self.rgba, self.width as usize, self.height as usize, w as usize, h as usize), + index: self.index, + time: self.time, + } + } +} + +/// The size that fits `w` × `h` inside `max_w` × `max_h`, keeping the aspect ratio (≥ 1 px). +pub fn fit_size(w: u32, h: u32, max_w: u32, max_h: u32) -> (u32, u32) { + if w == 0 || h == 0 { + return (1, 1); + } + let s = (f64::from(max_w.max(1)) / f64::from(w)).min(f64::from(max_h.max(1)) / f64::from(h)).min(1.0); + (((f64::from(w) * s).round() as u32).max(1), ((f64::from(h) * s).round() as u32).max(1)) +} + +/// What the movie's picture is, and whether we can decode it. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct MovieInfo { + /// Coded picture size (after cropping), in pixels. + pub width: u32, + pub height: u32, + /// Pixel aspect ratio (1.0 = square pixels). + pub pixel_aspect: f64, + /// Length of the picture track in seconds. + pub duration: f64, + /// Nominal frame rate (frames per second). + pub frame_rate: f64, + pub frame_count: usize, + /// Short codec id: `h264`, `prores`, `mjpeg`, `hevc`, … + pub codec: String, + /// Human-readable codec description (e.g. "H.264 High", "Apple ProRes 422 HQ"). + pub codec_detail: String, + /// True when SoundCraft can decode this picture (otherwise frame requests return `Err`). + pub decodable: bool, + /// The file also has an audio track. + pub has_audio: bool, + /// QuickTime (`.mov`) rather than ISO MP4 brand. + pub quicktime: bool, +} + +impl MovieInfo { + /// Display aspect ratio (width / height, including pixel aspect). + pub fn display_aspect(&self) -> f64 { + if self.height == 0 { 16.0 / 9.0 } else { f64::from(self.width) * self.pixel_aspect / f64::from(self.height) } + } +} + +enum Codec { + H264 { avcc: Vec }, + ProRes, + Jpeg, + Unsupported(String), +} + +struct H264State { + dec: crate::h264::Decoder, + /// Next sample (decode order) to feed. + next: usize, + /// Last presentation index output, if any. + last_out: Option, + /// Output pictures not requested yet (presentation index → picture). + ready: std::collections::BTreeMap, +} + +/// A tiny LRU of decoded frames. +struct Lru { + cap: usize, + items: VecDeque<(K, Arc)>, +} + +impl Lru { + fn new(cap: usize) -> Self { + Lru { cap: cap.max(1), items: VecDeque::new() } + } + fn get(&mut self, k: K) -> Option> { + let i = self.items.iter().position(|(x, _)| *x == k)?; + let item = self.items.remove(i)?; + let f = Arc::clone(&item.1); + self.items.push_back(item); + Some(f) + } + fn put(&mut self, k: K, f: Arc) { + self.items.retain(|(x, _)| *x != k); + self.items.push_back((k, f)); + while self.items.len() > self.cap { + self.items.pop_front(); + } + } + fn contains(&self, k: K) -> bool { + self.items.iter().any(|(x, _)| *x == k) + } +} + +/// An opened movie: container index plus decoder state and frame caches. +pub struct Movie { + src: SharedSource, + mp4: Mp4File, + track: usize, + info: MovieInfo, + codec: Codec, + /// Presentation index → sample index (decode order). + order: Vec, + /// Sample index → presentation index. + rank: Vec, + /// Presentation index → time in seconds. + times: Vec, + h264: Option, + cache: Lru, + thumbs: Lru<(usize, u32)>, + /// Decoded frames since open (statistics for tests and the UI). + pub decoded: u64, +} + +impl std::fmt::Debug for Movie { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Movie").field("info", &self.info).finish_non_exhaustive() + } +} + +impl Movie { + /// Open a movie held in memory. + pub fn open(bytes: impl Into>) -> Result { + let b: Arc<[u8]> = bytes.into(); + Movie::open_source(Arc::new(b)) + } + + /// Open a movie from any random-access source (e.g. a `std::fs::File` on native targets, so + /// large movies are not read into memory). + pub fn open_source(src: SharedSource) -> Result { + let mp4 = isobmff::open(&*src).map_err(|e| VideoError::Container(e.to_string()))?; + let track = mp4 + .tracks + .iter() + .position(|t| t.kind == TrackKind::Video && t.enabled && !t.samples.is_empty()) + .or_else(|| mp4.tracks.iter().position(|t| t.kind == TrackKind::Video && !t.samples.is_empty())) + .ok_or(VideoError::NoVideoTrack)?; + let has_audio = mp4.tracks.iter().any(|t| t.kind == TrackKind::Audio && !t.samples.is_empty()); + let t = mp4.tracks.get(track).ok_or(VideoError::NoVideoTrack)?; + let ts = f64::from(t.timescale.max(1)); + let n = t.samples.len(); + let mut order: Vec = (0..n).collect(); + order.sort_by_key(|&i| (t.samples.get(i).map_or(0, |s| s.pts), i)); + let mut rank = vec![0usize; n]; + for (p, &i) in order.iter().enumerate() { + if let Some(r) = rank.get_mut(i) { + *r = p; + } + } + let times: Vec = order.iter().map(|&i| t.presentation_pts(i).unwrap_or(0) as f64 / ts).collect(); + let first = times.first().copied().unwrap_or(0.0); + let end = order.iter().filter_map(|&i| Some(t.presentation_pts(i)?.saturating_add(i64::from(t.samples.get(i)?.duration)))).max().unwrap_or(0) + as f64 + / ts; + let duration = (end - first.min(0.0)).max(0.0); + // Nominal rate: the most common sample duration. + let mut durs: Vec = t.samples.iter().map(|s| s.duration).filter(|&d| d > 0).collect(); + durs.sort_unstable(); + let mut best = (0u32, 0usize); + let mut k = 0; + while k < durs.len() { + let d = durs.get(k).copied().unwrap_or(0); + let run = durs.iter().skip(k).take_while(|&&x| x == d).count().max(1); + if run > best.1 { + best = (d, run); + } + k += run; + } + let frame_rate = if best.0 > 0 { + ts / f64::from(best.0) + } else if duration > 0.0 { + n as f64 / duration + } else { + 0.0 + }; + let vp = t.video(); + let (mut width, mut height) = vp.map_or((t.width >> 16, t.height >> 16), |v| (u32::from(v.width), u32::from(v.height))); + if width == 0 || height == 0 { + (width, height) = (t.width >> 16, t.height >> 16); + } + let pixel_aspect = vp.and_then(|v| v.pixel_aspect).filter(|&(h, v)| h > 0 && v > 0).map_or(1.0, |(h, v)| f64::from(h) / f64::from(v)); + let (codec, name, detail) = match t.codec() { + Some(CodecConfig::Avc(c)) => { + let (ok, prof) = match c.profile { + 66 => (true, "Baseline"), + 77 => (true, "Main"), + 88 => (true, "Extended"), + 100 => (true, "High"), + 110 => (false, "High 10"), + 122 => (false, "High 4:2:2"), + 244 => (false, "High 4:4:4"), + _ => (true, "?"), + }; + let detail = format!("H.264 {prof}"); + if ok { + (Codec::H264 { avcc: c.to_bytes() }, "h264", detail) + } else { + (Codec::Unsupported(format!("{detail} (only 8-bit 4:2:0 H.264 is supported)")), "h264", detail) + } + } + Some(CodecConfig::ProRes { fourcc }) => { + let f = fourcc.0; + let flavour = match &f { + b"apco" => "422 Proxy", + b"apcs" => "422 LT", + b"apcn" => "422", + b"apch" => "422 HQ", + b"ap4h" => "4444", + b"ap4x" => "4444 XQ", + _ => "", + }; + (Codec::ProRes, "prores", format!("Apple ProRes {flavour}")) + } + Some(CodecConfig::Jpeg { fourcc }) if &fourcc.0 != b"mjpb" => (Codec::Jpeg, "mjpeg", "Motion JPEG".to_string()), + Some(other) => { + let nm = other.name(); + (Codec::Unsupported(format!("{nm} pictures are not supported yet (H.264, ProRes and Motion JPEG are)")), nm, nm.to_ascii_uppercase()) + } + None => (Codec::Unsupported("the video track has no sample description".into()), "unknown", "unknown".into()), + }; + if u64::from(width) * u64::from(height) > MAX_PIXELS { + return Err(VideoError::Unsupported(format!("{width}x{height} is larger than SoundCraft decodes"))); + } + let info = MovieInfo { + width, + height, + pixel_aspect, + duration, + frame_rate, + frame_count: n, + codec: name.to_string(), + codec_detail: detail, + decodable: !matches!(codec, Codec::Unsupported(_)), + has_audio, + quicktime: mp4.is_quicktime, + }; + Ok(Movie { src, mp4, track, info, codec, order, rank, times, h264: None, cache: Lru::new(12), thumbs: Lru::new(256), decoded: 0 }) + } + + pub fn info(&self) -> &MovieInfo { + &self.info + } + + /// Frames kept decoded (full size). Thumbnails have their own cache. + pub fn set_cache_frames(&mut self, n: usize) { + self.cache.cap = n.clamp(1, 512); + } + + /// The frame shown at `secs` (presentation index), or `None` before the first / after the + /// last frame. + pub fn frame_index_at(&self, secs: f64) -> Option { + if !secs.is_finite() || secs < 0.0 || secs >= self.info.duration { + return None; + } + // Allow for rounding of `secs` computed from sample positions. + let k = self.times.partition_point(|&t| t <= secs + 1e-7); + Some(k.saturating_sub(1)) + } + + /// Presentation time (seconds) of frame `index`. + pub fn frame_time(&self, index: usize) -> Option { + self.times.get(index).copied() + } + + /// Decode the frame shown at `secs` (full size). + pub fn frame_at(&mut self, secs: f64) -> Result> { + let i = self.frame_index_at(secs).ok_or(VideoError::OutOfRange)?; + self.frame(i) + } + + /// True if frame `index` is in the cache (no decoding needed). + pub fn is_cached(&self, index: usize) -> bool { + self.cache.contains(index) + } + + /// Decode frame `index` (presentation order), using the cache and decoding forward from the + /// previous keyframe when needed. + pub fn frame(&mut self, index: usize) -> Result> { + if index >= self.order.len() { + return Err(VideoError::OutOfRange); + } + if let Some(f) = self.cache.get(index) { + return Ok(f); + } + let f = match &self.codec { + Codec::Unsupported(why) => return Err(VideoError::Unsupported(why.clone())), + Codec::ProRes | Codec::Jpeg => { + let s = self.order.get(index).copied().ok_or(VideoError::OutOfRange)?; + Arc::new(self.decode_intra(s, index)?) + } + Codec::H264 { .. } => self.decode_h264(index)?, + }; + self.cache.put(index, Arc::clone(&f)); + Ok(f) + } + + /// A small picture for the timeline at `secs`, at most `max_h` pixels high. For long-GOP + /// codecs this is the keyframe at or before `secs` (cheap: one picture is decoded). + pub fn thumbnail_at(&mut self, secs: f64, max_h: u32) -> Result> { + let i = self.frame_index_at(secs).ok_or(VideoError::OutOfRange)?; + let max_h = max_h.clamp(4, 1080); + let key = match self.codec { + Codec::H264 { .. } => { + let s = self.order.get(i).copied().ok_or(VideoError::OutOfRange)?; + let k = self.mp4.tracks.get(self.track).map_or(0, |t| t.sync_sample_before(s)); + self.rank.get(k).copied().unwrap_or(i) + } + _ => i, + }; + if let Some(f) = self.thumbs.get((key, max_h)) { + return Ok(f); + } + let full = if let Some(f) = self.cache.get(key) { + f + } else { + match &self.codec { + Codec::Unsupported(why) => return Err(VideoError::Unsupported(why.clone())), + Codec::H264 { avcc } => { + let avcc = avcc.clone(); + let s = self.order.get(key).copied().ok_or(VideoError::OutOfRange)?; + Arc::new(self.decode_h264_single(&avcc, s, key)?) + } + _ => { + let s = self.order.get(key).copied().ok_or(VideoError::OutOfRange)?; + Arc::new(self.decode_intra(s, key)?) + } + } + }; + let w = (f64::from(max_h) * self.info.display_aspect()).round().clamp(1.0, 4096.0) as u32; + let t = Arc::new(full.fit(w, max_h)); + self.thumbs.put((key, max_h), Arc::clone(&t)); + Ok(t) + } + + /// `count` evenly spaced thumbnails across the movie (time, picture). + pub fn thumbnails(&mut self, count: usize, max_h: u32) -> Vec<(f64, Result>)> { + let count = count.clamp(1, 1000); + let d = self.info.duration; + (0..count) + .map(|k| { + let t = d * k as f64 / count as f64; + (t, self.thumbnail_at(t, max_h)) + }) + .collect() + } + + fn read(&self, sample: usize) -> Result> { + let t = self.mp4.tracks.get(self.track).ok_or(VideoError::NoVideoTrack)?; + let size = t.samples.get(sample).map_or(0, |s| s.size); + if size > 256 << 20 { + return Err(VideoError::Decode("sample larger than 256 MiB".into())); + } + self.mp4.read_sample(&*self.src, self.track, sample).map_err(|e| VideoError::Container(e.to_string())) + } + + fn decode_intra(&mut self, sample: usize, index: usize) -> Result { + let data = self.read(sample)?; + let time = self.times.get(index).copied().unwrap_or(0.0); + self.decoded += 1; + match self.codec { + Codec::ProRes => { + let opts = crate::prores::DecodeOptions { bit_depth: Some(8), threads: false }; + let f = crate::prores::decode_frame_with(&data, &opts).map_err(|e| VideoError::Decode(e.to_string()))?; + if u64::from(f.width) * u64::from(f.height) > MAX_PIXELS { + return Err(VideoError::Decode("picture too large".into())); + } + let (w, h) = (f.width as usize, f.height as usize); + let cw = f.chroma_width() as usize; + let narrow = |p: &[u16]| p.iter().map(|&v| v.min(255) as u8).collect::>(); + let (y, u, v) = (narrow(&f.y), narrow(&f.cb), narrow(&f.cr)); + let cx = if cw < w { 2 } else { 1 }; + let m = Matrix::for_code(f.color.matrix, f.height, false); + let rgba = planar_to_rgba(w, h, &y, w, &u, &v, cw, cx, 1, &m); + Ok(Frame { width: f.width, height: f.height, rgba, index, time }) + } + Codec::Jpeg => { + let img = image::load_from_memory_with_format(&data, image::ImageFormat::Jpeg).map_err(|e| VideoError::Decode(e.to_string()))?; + let rgba = img.to_rgba8(); + let (width, height) = rgba.dimensions(); + if u64::from(width) * u64::from(height) > MAX_PIXELS { + return Err(VideoError::Decode("picture too large".into())); + } + Ok(Frame { width, height, rgba: rgba.into_raw(), index, time }) + } + _ => Err(VideoError::Unsupported("not an intra-frame codec".into())), + } + } + + fn h264_to_frame(&self, p: &crate::h264::Picture, index: usize) -> Frame { + let m = Matrix::for_code(p.color.matrix, p.height, p.color.full_range); + let rgba = planar_to_rgba(p.width as usize, p.height as usize, &p.y, p.y_stride, &p.u, &p.v, p.uv_stride, 2, 2, &m); + Frame { width: p.width, height: p.height, rgba, index, time: self.times.get(index).copied().unwrap_or(0.0) } + } + + /// Decode one sample with a fresh decoder (keyframe thumbnails). + fn decode_h264_single(&mut self, avcc: &[u8], sample: usize, index: usize) -> Result { + let mut dec = crate::h264::Decoder::from_avcc(avcc).map_err(|e| VideoError::Decode(e.to_string()))?; + let data = self.read(sample)?; + let mut pics = dec.decode(&data, index as i64).map_err(|e| VideoError::Decode(e.to_string()))?; + pics.extend(dec.flush()); + self.decoded += 1; + let p = pics.iter().find(|p| p.pts == index as i64).or(pics.first()).ok_or_else(|| VideoError::Decode("no picture decoded".into()))?; + Ok(self.h264_to_frame(p, index)) + } + + fn decode_h264(&mut self, index: usize) -> Result> { + let Codec::H264 { avcc } = &self.codec else { return Err(VideoError::Unsupported("not H.264".into())) }; + let target = self.order.get(index).copied().ok_or(VideoError::OutOfRange)?; + let key = self.mp4.tracks.get(self.track).map_or(0, |t| t.sync_sample_before(target)); + let resume = self + .h264 + .as_ref() + .is_some_and(|st| st.ready.contains_key(&index) || (key < st.next && st.next != usize::MAX && st.last_out.is_none_or(|lo| lo < index))); + if !resume { + let dec = crate::h264::Decoder::from_avcc(avcc).map_err(|e| VideoError::Decode(e.to_string()))?; + self.h264 = Some(H264State { dec, next: key, last_out: None, ready: std::collections::BTreeMap::new() }); + } + let total = self.order.len(); + let mut last_err: Option = None; + // Bounded: each iteration feeds one sample, or flushes once at the end. + for _ in 0..=total.saturating_add(1) { + let Some(st) = self.h264.as_mut() else { break }; + st.ready.retain(|q, _| *q >= index); + if let Some(p) = st.ready.remove(&index) { + return Ok(Arc::new(self.h264_to_frame(&p, index))); + } + if st.next == usize::MAX { + break; + } + let pics = if st.next < total { + let s = st.next; + st.next += 1; + let pts = self.rank.get(s).copied().unwrap_or(0) as i64; + let data = match self.mp4.read_sample(&*self.src, self.track, s) { + Ok(d) => d, + Err(e) => { + last_err = Some(e.to_string()); + continue; + } + }; + let Some(st) = self.h264.as_mut() else { break }; + match st.dec.decode(&data, pts) { + Ok(p) => p, + Err(e) => { + // Corrupt access unit: skip it (the decoder conceals what it can). + last_err = Some(e.to_string()); + Vec::new() + } + } + } else { + st.next = usize::MAX; + st.dec.flush() + }; + let Some(st) = self.h264.as_mut() else { break }; + for p in pics { + let q = usize::try_from(p.pts).unwrap_or(usize::MAX); + st.last_out = Some(q); + self.decoded += 1; + if q >= index { + st.ready.insert(q, p); + } + } + // Pictures waiting for a later request (bounded by the reorder depth in practice). + while st.ready.len() > 32 { + st.ready.pop_last(); + } + } + Err(VideoError::Decode(last_err.unwrap_or_else(|| format!("frame {index} was not produced")))) + } +} diff --git a/crates/video/src/prores/bits.rs b/crates/video/src/prores/bits.rs new file mode 100644 index 00000000..0bb62cc7 --- /dev/null +++ b/crates/video/src/prores/bits.rs @@ -0,0 +1,280 @@ +//! Fast MSB-first bit reader/writer for slice payloads (the hot path of the codec). +//! +//! Frame and picture headers use `crate::bitstream`; the entropy-coded slice data uses these +//! specialised helpers, which read 64-bit windows and decode a whole codeword per call. + +use crate::prores::tables::Codebook; + +/// Bit reader over one slice component with a 64-bit MSB-aligned cache. Reads past the end +/// yield zero bits; callers check [`Reader::overrun`] once a component has been consumed. +pub(crate) struct Reader<'a> { + data: &'a [u8], + /// Next byte of `data` to load into the cache (may run past the end: zeros are loaded). + next: usize, + /// Cached bits, MSB first; bits below `avail` are zero. + cache: u64, + avail: u32, +} + +impl<'a> Reader<'a> { + #[inline] + pub fn new(data: &'a [u8]) -> Self { + let mut r = Self { data, next: 0, cache: 0, avail: 0 }; + r.refill(); + r + } + + /// Ensure at least 56 valid bits are cached. + #[inline(always)] + fn refill(&mut self) { + if self.avail >= 56 { + return; + } + if let Some(s) = self.data.get(self.next..self.next + 8) { + let w = u64::from_be_bytes([s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]]); + self.cache |= w >> self.avail; + self.next += ((63 - self.avail) >> 3) as usize; + self.avail |= 56; + } else { + while self.avail <= 56 { + let b = self.data.get(self.next).copied().unwrap_or(0) as u64; + self.cache |= b << (56 - self.avail); + self.avail += 8; + self.next += 1; + } + } + } + + #[inline(always)] + fn consume(&mut self, n: u32) { + // n <= 56 <= avail + self.cache = if n >= 64 { 0 } else { self.cache << n }; + self.avail -= n; + } + + #[inline] + fn consumed(&self) -> usize { + self.next * 8 - self.avail as usize + } + + #[inline] + pub fn bits_left(&self) -> isize { + (self.data.len() * 8) as isize - self.consumed() as isize + } + + #[inline] + pub fn overrun(&self) -> bool { + self.consumed() > self.data.len() * 8 + } + + /// True when every remaining bit is zero (the component's padding), or nothing is left. + #[inline(always)] + pub fn only_padding_left(&mut self) -> bool { + let left = self.bits_left(); + if left > 56 { + return false; + } + if left <= 0 { + return true; + } + self.refill(); + self.cache >> (64 - left as u32) == 0 + } + + #[inline] + pub fn read(&mut self, n: u32) -> u32 { + debug_assert!(n <= 32); + if n == 0 { + return 0; + } + self.refill(); + let v = (self.cache >> (64 - n)) as u32; + self.consume(n); + v + } + + #[inline(always)] + pub fn read_bit(&mut self) -> bool { + if self.avail == 0 { + self.refill(); + } + let b = self.cache >> 63 == 1; + self.consume(1); + b + } + + /// Decode one codeword; `None` for codewords longer than 56 bits (never produced by valid + /// streams). + #[inline(always)] + pub fn read_cw(&mut self, cb: Codebook) -> Option { + self.refill(); + let w = self.cache; + let q = w.leading_zeros(); + let rq = cb.rice_q_max as u32; + if q <= rq { + let rice = cb.rice as u32; + let len = q + 1 + rice; + let low = if rice == 0 { 0 } else { ((w << (q + 1)) >> (64 - rice)) as u32 }; + self.consume(len); + Some((q << rice) | low) + } else { + let exp = cb.exp as u32; + let len = 2 * q - rq + exp; + if len > 56 { + return None; + } + let code = w >> (64 - len); + self.consume(len); + let v = code - (1u64 << exp) + (((rq + 1) as u64) << cb.rice); + u32::try_from(v).ok() + } + } +} + +/// Number of bits of the codeword for `val` in codebook `cb`. +#[inline] +pub(crate) fn cw_len(cb: Codebook, val: u32) -> u32 { + let rq = cb.rice_q_max as u32; + let rice = cb.rice as u32; + let rice_span = (rq + 1) << rice; + if val < rice_span { + (val >> rice) + 1 + rice + } else { + let exp = cb.exp as u32; + let v = (val - rice_span) as u64 + (1u64 << exp); + let nbits = 64 - v.leading_zeros(); // n + exp + 1 + let n = nbits - exp - 1; + (rq + 1 + n) + nbits + } +} + +/// A bit sink: either a real writer or a bit counter (for rate control trials). +pub(crate) trait Sink { + fn put(&mut self, val: u32, bits: u32); + fn put_cw(&mut self, cb: Codebook, val: u32); +} + +/// Counts bits without storing them. +#[derive(Default)] +pub(crate) struct Counter(pub u64); + +impl Sink for Counter { + #[inline] + fn put(&mut self, _val: u32, bits: u32) { + self.0 += bits as u64; + } + #[inline] + fn put_cw(&mut self, cb: Codebook, val: u32) { + self.0 += cw_len(cb, val) as u64; + } +} + +/// MSB-first bit writer. +#[derive(Default)] +pub(crate) struct Writer { + pub buf: Vec, + acc: u64, + n: u32, +} + +impl Writer { + pub fn new() -> Self { + Self::default() + } + /// Pad with zero bits to a byte boundary and return the bytes. + pub fn finish(mut self) -> Vec { + if self.n > 0 { + let pad = 8 - self.n; + self.put(0, pad); + } + self.buf + } +} + +impl Sink for Writer { + #[inline] + fn put(&mut self, val: u32, bits: u32) { + debug_assert!(bits <= 32); + if bits == 0 { + return; + } + let v = (val as u64) & ((1u64 << bits) - 1); + self.acc = (self.acc << bits) | v; + self.n += bits; + while self.n >= 8 { + self.n -= 8; + self.buf.push((self.acc >> self.n) as u8); + } + self.acc &= (1u64 << self.n) - 1; + } + + fn put_cw(&mut self, cb: Codebook, val: u32) { + let rq = cb.rice_q_max as u32; + let rice = cb.rice as u32; + let rice_span = (rq + 1) << rice; + if val < rice_span { + let q = val >> rice; + // q zeros, a one, then `rice` low bits + self.put(0, q); + self.put(1, 1); + self.put(val & ((1 << rice) - 1), rice); + } else { + let exp = cb.exp as u32; + let v = (val - rice_span) as u64 + (1u64 << exp); + let nbits = 64 - v.leading_zeros(); + let n = nbits - exp - 1; + let zeros = rq + 1 + n; + self.put(0, zeros.min(32)); + if zeros > 32 { + self.put(0, zeros - 32); + } + if nbits > 32 { + self.put((v >> 32) as u32, nbits - 32); + self.put(v as u32, 32); + } else { + self.put(v as u32, nbits); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prores::tables::*; + + #[test] + fn codeword_roundtrip_all_codebooks() { + let mut books = vec![FIRST_DC_CB]; + books.extend_from_slice(&DC_CB); + books.extend_from_slice(&RUN_CB); + books.extend_from_slice(&LEVEL_CB); + let vals: Vec = (0..300).chain([1000, 4095, 65535, 1 << 20]).collect(); + for cb in books { + let mut w = Writer::new(); + let mut total = 0; + for &v in &vals { + w.put_cw(cb, v); + total += cw_len(cb, v); + } + let bits = w.buf.len() as u32 * 8 + w.n; + assert_eq!(bits, total); + let data = w.finish(); + let mut r = Reader::new(&data); + for &v in &vals { + assert_eq!(r.read_cw(cb), Some(v), "{cb:?}"); + } + assert!(!r.overrun()); + } + } + + #[test] + fn exp_golomb5_equivalence() { + // FIRST_DC_CB is Exp-Golomb order 5: value 32 is `01` + 6 bits (000000) + assert_eq!(cw_len(FIRST_DC_CB, 0), 6); + assert_eq!(cw_len(FIRST_DC_CB, 31), 6); + assert_eq!(cw_len(FIRST_DC_CB, 32), 8); + assert_eq!(cw_len(FIRST_DC_CB, 95), 8); + assert_eq!(cw_len(FIRST_DC_CB, 96), 10); + } +} diff --git a/crates/video/src/prores/dct.rs b/crates/video/src/prores/dct.rs new file mode 100644 index 00000000..4c4774e7 --- /dev/null +++ b/crates/video/src/prores/dct.rs @@ -0,0 +1,160 @@ +//! 8×8 orthonormal DCT-II / DCT-III (IDCT) in single precision. +//! +//! The reconstruction IDCT is defined as the exact real-valued 2-D inverse DCT +//! `f(x,y) = Σ_u Σ_v C(u)C(v)/4 · F(v,u) · cos((2x+1)uπ/16) · cos((2y+1)vπ/16)` +//! evaluated separably (rows then columns) in `f32`. For ProRes coefficient ranges the `f32` +//! evaluation error is below 2⁻⁸ in the 12-bit sample domain, so the rounded output equals the +//! rounded exact IDCT except when the exact value lies within that distance of a rounding tie. + +use crate::prores::tables::BASIS; + +/// In-place inverse DCT. `block` holds dequantised coefficients in raster order +/// (`block[v*8 + u]`, `v` = vertical frequency) and receives samples (`block[y*8 + x]`). +/// +/// Each 1-D pass uses the even/odd symmetry of the basis (`B[u][7-x] = ±B[u][x]` for even/odd +/// `u`) and skips all-zero rows; DC-only blocks are filled directly. +#[inline] +pub(crate) fn idct8x8(block: &mut [f32; 64]) { + let mut t = [[0f32; 8]; 8]; + let mut nz = [false; 8]; + let mut dc_only = true; + // Horizontal pass: t[r][x] = Σ_u F[r][u] · B[u][x] + for (r, c) in block.as_chunks::<8>().0.iter().enumerate() { + if c[1..].iter().all(|&v| v == 0.0) { + if c[0] != 0.0 { + t[r] = [c[0] * BASIS[0][0]; 8]; + nz[r] = true; + dc_only &= r == 0; + } + continue; + } + dc_only = false; + nz[r] = true; + let mut out = [0f32; 8]; + for x in 0..4 { + let e = c[0] * BASIS[0][x] + c[2] * BASIS[2][x] + c[4] * BASIS[4][x] + c[6] * BASIS[6][x]; + let o = c[1] * BASIS[1][x] + c[3] * BASIS[3][x] + c[5] * BASIS[5][x] + c[7] * BASIS[7][x]; + out[x] = e + o; + out[7 - x] = e - o; + } + t[r] = out; + } + if dc_only { + *block = [t[0][0] * BASIS[0][0]; 64]; + return; + } + // Vertical pass: f[y][x] = Σ_v t[v][x] · B[v][y] + for y in 0..4 { + let mut e = [0f32; 8]; + let mut o = [0f32; 8]; + for v in [0, 2, 4, 6] { + if nz[v] { + let b = BASIS[v][y]; + for x in 0..8 { + e[x] += t[v][x] * b; + } + } + } + for v in [1, 3, 5, 7] { + if nz[v] { + let b = BASIS[v][y]; + for x in 0..8 { + o[x] += t[v][x] * b; + } + } + } + for x in 0..8 { + block[y * 8 + x] = e[x] + o[x]; + block[(7 - y) * 8 + x] = e[x] - o[x]; + } + } +} + +/// In-place forward DCT (samples in, coefficients out; same layout as [`idct8x8`]). +pub(crate) fn fdct8x8(block: &mut [f32; 64]) { + let mut t = [[0f32; 8]; 8]; + // Horizontal: t[y][u] = Σ_x f[y][x] · B[u][x] + for y in 0..8 { + for u in 0..8 { + let mut s = 0f32; + for x in 0..8 { + s += block[y * 8 + x] * BASIS[u][x]; + } + t[y][u] = s; + } + } + // Vertical: F[v][u] = Σ_y t[y][u] · B[v][y] + for v in 0..8 { + let mut acc = [0f32; 8]; + for y in 0..8 { + let b = BASIS[v][y]; + for u in 0..8 { + acc[u] += t[y][u] * b; + } + } + block[v * 8..v * 8 + 8].copy_from_slice(&acc); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reference_idct(c: &[f32; 64]) -> [f64; 64] { + let mut out = [0f64; 64]; + for y in 0..8 { + for x in 0..8 { + let mut s = 0f64; + for v in 0..8 { + for u in 0..8 { + let cu = if u == 0 { std::f64::consts::FRAC_1_SQRT_2 } else { 1.0 }; + let cv = if v == 0 { std::f64::consts::FRAC_1_SQRT_2 } else { 1.0 }; + s += cu * cv / 4.0 + * c[v * 8 + u] as f64 + * (((2 * x + 1) * u) as f64 * std::f64::consts::PI / 16.0).cos() + * (((2 * y + 1) * v) as f64 * std::f64::consts::PI / 16.0).cos(); + } + } + out[y * 8 + x] = s; + } + } + out + } + + #[test] + fn idct_matches_double_reference() { + let mut seed = 12345u32; + let mut rnd = || { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + seed >> 8 + }; + for _ in 0..200 { + let mut c = [0f32; 64]; + for v in c.iter_mut() { + if rnd() % 3 == 0 { + *v = ((rnd() % 8192) as i32 - 4096) as f32; + } + } + let r = reference_idct(&c); + let mut b = c; + idct8x8(&mut b); + for i in 0..64 { + assert!((b[i] as f64 - r[i]).abs() < 4e-3, "{} vs {}", b[i], r[i]); + } + } + } + + #[test] + fn fdct_inverts_idct() { + let mut b = [0f32; 64]; + for (i, v) in b.iter_mut().enumerate() { + *v = ((i * 37) % 200) as f32 - 100.0; + } + let orig = b; + fdct8x8(&mut b); + idct8x8(&mut b); + for i in 0..64 { + assert!((b[i] - orig[i]).abs() < 1e-3); + } + } +} diff --git a/crates/video/src/prores/decode.rs b/crates/video/src/prores/decode.rs new file mode 100644 index 00000000..11f8fc1d --- /dev/null +++ b/crates/video/src/prores/decode.rs @@ -0,0 +1,509 @@ +//! ProRes frame decoding. + +use crate::prores::bits::Reader; +use crate::prores::dct::idct8x8; +use crate::prores::header::{FrameHeader, PictureHeader, row_slice_widths}; +use crate::prores::tables::*; +use crate::prores::{AlphaType, ChromaFormat, Error, Frame, Interlace, Result}; + +/// Decoder options. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecodeOptions { + /// Output bit depth (8..=16). `None`: 10 for 4:2:2, 12 for 4:4:4 (the precision ProRes + /// codes each format at). + pub bit_depth: Option, + /// Decode slices in parallel (requires the `threads` feature). + pub threads: bool, +} + +impl Default for DecodeOptions { + fn default() -> Self { + DecodeOptions { bit_depth: None, threads: true } + } +} + +/// Parse only the frame header (cheap; for probing geometry and format). +pub fn probe(data: &[u8]) -> Result { + FrameHeader::parse(data).map(|(h, _, _)| h) +} + +/// Decode one ProRes frame (the contents of one QuickTime sample) with default options. +pub fn decode_frame(data: &[u8]) -> Result { + decode_frame_with(data, &DecodeOptions::default()) +} + +/// Maps IDCT output (12-bit-domain signed samples) to the output bit depth. +#[derive(Clone, Copy)] +struct OutConv { + scale: f32, + bias: f32, + min: i32, + max: i32, +} + +impl OutConv { + fn new(depth: u8) -> OutConv { + // The IDCT produces `f`, where the 12-bit sample is `f + 2048`. + let scale = (2f32).powi(depth as i32 - 12); + // Samples are clipped away from the extremes: the four lowest and highest codes are + // excluded at 10 bits and above (0..=3 and 1020..=1023 are the SDI timing-reference codes + // at 10 bits; the same margin is kept at 12 bits, matching common decoders), and the + // single lowest/highest code at 8 and 9 bits. + let reserved = if depth >= 10 { 4 } else { 1 }; + OutConv { scale, bias: (1u32 << (depth - 1)) as f32 + 0.5, min: reserved, max: (1i32 << depth) - 1 - reserved } + } + #[inline(always)] + fn conv(&self, v: f32) -> u16 { + // Truncation equals floor here: negative values clamp to `min` (> 0) either way. + ((v * self.scale + self.bias) as i32).clamp(self.min, self.max) as u16 + } +} + +struct Ctx { + width: usize, + cwidth: usize, + chroma444: bool, + qmat_luma: [u8; 64], + qmat_chroma: [u8; 64], + scan: &'static [u8; 64], + alpha: AlphaType, + conv: OutConv, + depth: u8, +} + +/// Output rows belonging to one macroblock row of every coded picture. +struct Band<'a> { + y: &'a mut [u16], + cb: &'a mut [u16], + cr: &'a mut [u16], + a: Option<&'a mut [u16]>, + rows: usize, +} + +struct Picture { + /// Line parity within the frame (0 = even lines) for field pictures. + parity: usize, + /// Byte ranges of each slice, row-major. + slices: Vec<(usize, usize)>, +} + +struct Scratch { + coeffs: Vec<[f32; 64]>, + alpha: Vec, +} + +impl Scratch { + fn new() -> Scratch { + Scratch { coeffs: vec![[0f32; 64]; 32], alpha: vec![0; 16 * 128] } + } +} + +/// Decode one ProRes frame. +pub fn decode_frame_with(data: &[u8], opts: &DecodeOptions) -> Result { + let mut f = Frame::new(0, 0, ChromaFormat::Yuv422, 10, false); + decode_frame_into(data, opts, &mut f)?; + Ok(f) +} + +/// Decode one ProRes frame into `out`, reusing its plane allocations (the fastest way to decode +/// a sequence). On error the contents of `out` are unspecified. +pub fn decode_frame_into(data: &[u8], opts: &DecodeOptions, out: &mut Frame) -> Result<()> { + let (hdr, mut off, frame_size) = FrameHeader::parse(data)?; + let data = &data[..frame_size]; + let depth = opts.bit_depth.unwrap_or(match hdr.chroma { + ChromaFormat::Yuv422 => 10, + ChromaFormat::Yuv444 => 12, + }); + if !(8..=16).contains(&depth) { + return Err(Error::Unsupported("output bit depth")); + } + let width = hdr.width as usize; + let height = hdr.height as usize; + let cwidth = hdr.chroma.chroma_width(hdr.width as u32) as usize; + let interlaced = hdr.interlace != Interlace::Progressive; + let mb_width = width.div_ceil(16); + let mb_height = if interlaced { height.div_ceil(32) } else { height.div_ceil(16) }; + + let npics = if interlaced { 2 } else { 1 }; + let mut pics = Vec::with_capacity(npics); + for p in 0..npics { + let ph = PictureHeader::parse(data.get(off..).ok_or(Error::Truncated)?)?; + let pic_end = off.checked_add(ph.picture_size).ok_or(Error::Truncated)?; + if pic_end > data.len() || ph.picture_size < ph.header_size { + return Err(Error::Truncated); + } + let widths = row_slice_widths(mb_width, ph.log2_slice_mbs); + let nslices = widths.len() * mb_height; + if ph.num_slices != 0 && ph.num_slices != nslices { + return Err(Error::Invalid("slice count does not match frame geometry")); + } + let table = off + ph.header_size; + let mut pos = table + 2 * nslices; + if pos > pic_end { + return Err(Error::Truncated); + } + let mut slices = Vec::with_capacity(nslices); + for i in 0..nslices { + let sz = u16::from_be_bytes([data[table + 2 * i], data[table + 2 * i + 1]]) as usize; + // Every slice carries at least its 6-byte header. + if sz < 6 || pos + sz > pic_end { + return Err(Error::Truncated); + } + slices.push((pos, sz)); + pos += sz; + } + let parity = match (hdr.interlace, p) { + (Interlace::BottomFieldFirst, 0) | (Interlace::TopFieldFirst, 1) => 1, + _ => 0, + }; + pics.push((Picture { parity, slices }, widths)); + off = pic_end; + } + + let ctx = Ctx { + width, + cwidth, + chroma444: hdr.chroma == ChromaFormat::Yuv444, + qmat_luma: hdr.qmat_luma, + qmat_chroma: hdr.qmat_chroma, + scan: if interlaced { &SCAN_INTERLACED } else { &SCAN_PROGRESSIVE }, + alpha: hdr.alpha, + conv: OutConv::new(depth), + depth, + }; + + let take = |v: &mut Vec, n: usize| { + let mut v = std::mem::take(v); + v.resize(n, 0); + v + }; + let mut y = take(&mut out.y, width * height); + let mut cb = take(&mut out.cb, cwidth * height); + let mut cr = take(&mut out.cr, cwidth * height); + let mut alpha = match (hdr.alpha, out.alpha.take()) { + (AlphaType::None, _) => None, + (_, a) => Some(take(&mut a.unwrap_or_default(), width * height)), + }; + + let band_h = if interlaced { 32 } else { 16 }; + let mut bands: Vec = Vec::with_capacity(mb_height); + { + let mut yc = y.chunks_mut(band_h * width); + let mut cbc = cb.chunks_mut(band_h * cwidth); + let mut crc = cr.chunks_mut(band_h * cwidth); + let mut ac = alpha.as_mut().map(|a| a.chunks_mut(band_h * width)); + while let (Some(y), Some(cb), Some(cr)) = (yc.next(), cbc.next(), crc.next()) { + let rows = y.len() / width; + let a = ac.as_mut().and_then(|it| it.next()); + bands.push(Band { y, cb, cr, a, rows }); + } + } + + let step = if interlaced { 2 } else { 1 }; + let run_band = |(row, band): (usize, &mut Band), scratch: &mut Scratch| -> Result<()> { + for (pi, (pic, widths)) in pics.iter().enumerate() { + let mut mb_x = 0; + for (k, &w) in widths.iter().enumerate() { + let si = row * widths.len() + k; + let (start, len) = pic.slices[si]; + decode_slice(&ctx, &data[start..start + len], mb_x, w, band, step, pic.parity, scratch) + .map_err(|_| Error::Slice { picture: pi, slice: si })?; + mb_x += w; + } + } + Ok(()) + }; + + #[cfg(feature = "threads")] + if opts.threads { + use rayon::prelude::*; + bands.par_iter_mut().enumerate().try_for_each_init(Scratch::new, |s, b| run_band(b, s))?; + } else { + let mut s = Scratch::new(); + bands.iter_mut().enumerate().try_for_each(|b| run_band(b, &mut s))?; + } + #[cfg(not(feature = "threads"))] + { + let mut s = Scratch::new(); + bands.iter_mut().enumerate().try_for_each(|b| run_band(b, &mut s))?; + } + drop(bands); + + *out = Frame { + width: hdr.width as u32, + height: hdr.height as u32, + chroma: hdr.chroma, + bit_depth: depth, + y, + cb, + cr, + alpha, + interlace: hdr.interlace, + color: hdr.color, + aspect_ratio: hdr.aspect_ratio, + frame_rate_code: hdr.frame_rate_code, + }; + Ok(()) +} + +#[inline] +fn be16(d: &[u8], i: usize) -> usize { + u16::from_be_bytes([d[i], d[i + 1]]) as usize +} + +/// Dequantisation factors (matrix × scale) for a slice. +fn qfactors(qmat: &[u8; 64], qs: u32) -> [f32; 64] { + let mut q = [0f32; 64]; + for i in 0..64 { + q[i] = (qmat[i] as u32 * qs) as f32; + } + q +} + +#[allow(clippy::too_many_arguments)] +fn decode_slice( + ctx: &Ctx, + d: &[u8], + mb_x: usize, + mbs: usize, + band: &mut Band, + step: usize, + parity: usize, + s: &mut Scratch, +) -> std::result::Result<(), ()> { + if d.len() < 6 { + return Err(()); + } + let hsize = (d[0] >> 3) as usize; + let has_alpha = ctx.alpha != AlphaType::None; + if hsize < if has_alpha { 8 } else { 6 } || hsize > d.len() { + return Err(()); + } + let qs = qscale(d[1]); + let ysize = be16(d, 2); + let cbsize = be16(d, 4); + let body = d.len() - hsize; + let crsize = if has_alpha { be16(d, 6) } else { body.checked_sub(ysize + cbsize).ok_or(())? }; + if ysize + cbsize + crsize > body { + return Err(()); + } + let mut p = hsize; + let ydata = &d[p..p + ysize]; + p += ysize; + let cbdata = &d[p..p + cbsize]; + p += cbsize; + let crdata = &d[p..p + crsize]; + p += crsize; + let adata = &d[p..]; + + let log2_mbs = mbs.trailing_zeros(); + let x0 = mb_x * 16; + + // Luma + let ql = qfactors(&ctx.qmat_luma, qs); + let nb = 4 * mbs; + decode_component(ydata, log2_mbs + 2, ctx.scan, &ql, &mut s.coeffs[..nb])?; + for (b, blk) in s.coeffs[..nb].iter_mut().enumerate() { + idct8x8(blk); + let (mb, i) = (b >> 2, b & 3); + put_block(blk, band.y, ctx.width, x0 + mb * 16 + (i & 1) * 8, (i >> 1) * 8, band.rows, step, parity, ctx.conv); + } + + // Chroma + let qc = qfactors(&ctx.qmat_chroma, qs); + let (cnb, clog2) = if ctx.chroma444 { (4 * mbs, log2_mbs + 2) } else { (2 * mbs, log2_mbs + 1) }; + for (data, plane) in [(cbdata, &mut *band.cb), (crdata, &mut *band.cr)] { + decode_component(data, clog2, ctx.scan, &qc, &mut s.coeffs[..cnb])?; + for (b, blk) in s.coeffs[..cnb].iter_mut().enumerate() { + idct8x8(blk); + let (bx, by) = if ctx.chroma444 { + // 4:4:4 chroma blocks run down each 8-wide column pair: TL, BL, TR, BR. + let (mb, i) = (b >> 2, b & 3); + (x0 + mb * 16 + (i >> 1) * 8, (i & 1) * 8) + } else { + let (mb, i) = (b >> 1, b & 1); + (mb_x * 8 + mb * 8, i * 8) + }; + put_block(blk, plane, ctx.cwidth, bx, by, band.rows, step, parity, ctx.conv); + } + } + + // Alpha + if let Some(a) = band.a.as_deref_mut() { + let w = mbs * 16; + let n = w * 16; + let buf = &mut s.alpha[..n]; + decode_alpha(adata, ctx.alpha, buf)?; + let depth = ctx.depth as u32; + let conv = |v: u16| -> u16 { + let v = v as u32; + (match ctx.alpha { + AlphaType::Bits8 => { + if depth >= 8 { + let r = (v << (depth - 8)) | (v >> (16 - depth).min(8)); + r & ((1 << depth) - 1) + } else { + v + } + } + _ => v >> (16 - depth), + }) as u16 + }; + for py in 0..16 { + let line = py * step + parity; + if line >= band.rows { + break; + } + let row = &mut a[line * ctx.width..(line + 1) * ctx.width]; + let xe = (x0 + w).min(ctx.width); + if x0 >= xe { + continue; + } + for (o, &v) in row[x0..xe].iter_mut().zip(&buf[py * w..py * w + (xe - x0)]) { + *o = conv(v); + } + } + } + Ok(()) +} + +#[inline] +fn to_signed(code: u32) -> i64 { + let c = code as i64; + (c >> 1) ^ -(c & 1) +} + +/// Entropy-decode and dequantise one slice component into `blocks` (raster-order coefficients). +fn decode_component(data: &[u8], log2_blocks: u32, scan: &[u8; 64], q: &[f32; 64], blocks: &mut [[f32; 64]]) -> std::result::Result<(), ()> { + for b in blocks.iter_mut() { + *b = [0f32; 64]; + } + let n = blocks.len(); + let mut r = Reader::new(data); + + // DC coefficients + let code = r.read_cw(FIRST_DC_CB).ok_or(())?; + let mut dc = to_signed(code); + blocks[0][0] = dc as f32 * q[0]; + let mut code = DC_CTX_INIT; + let mut sign = 0i64; + for blk in blocks.iter_mut().skip(1) { + code = r.read_cw(DC_CB[code.min(6) as usize]).ok_or(())?; + if code == 0 { + sign = 0; + } else { + sign ^= -((code & 1) as i64); + } + let mag = (code as i64 + 1) >> 1; + dc = (dc + ((mag ^ sign) - sign)).clamp(-(1 << 24), 1 << 24); + blk[0] = dc as f32 * q[0]; + } + + // AC coefficients, interleaved across blocks: position = scan_index * n + block + let mask = n - 1; + let total = 64 * n; + let mut pos = mask; + let mut run_ctx = RUN_CTX_INIT; + let mut lev_ctx = LEVEL_CTX_INIT; + while !r.only_padding_left() { + let run = r.read_cw(RUN_CB[run_ctx.min(15) as usize]).ok_or(())?; + run_ctx = run; + pos += run as usize + 1; + if pos >= total { + return Err(()); + } + let lev = r.read_cw(LEVEL_CB[lev_ctx.min(9) as usize]).ok_or(())?.saturating_add(1); + lev_ctx = lev; + let neg = r.read_bit(); + let i = scan[pos >> log2_blocks] as usize; + let v = lev as f32 * q[i]; + blocks[pos & mask][i] = if neg { -v } else { v }; + } + if r.overrun() { + return Err(()); + } + Ok(()) +} + +/// Decode a slice's alpha plane (raster order over the slice's 16 lines) into `out` +/// (native 8- or 16-bit values). +fn decode_alpha(data: &[u8], ty: AlphaType, out: &mut [u16]) -> std::result::Result<(), ()> { + let bits = if ty == AlphaType::Bits16 { 16 } else { 8 }; + let short_bits = if bits == 16 { 7 } else { 4 }; + let mask: u32 = (1 << bits) - 1; + let mut r = Reader::new(data); + let mut val: u32 = mask; + let n = out.len(); + let mut idx = 0; + 'outer: loop { + loop { + let d = if r.read_bit() { + r.read(bits) + } else { + let c = r.read(short_bits); + let m = (c + 2) >> 1; + if c & 1 == 1 { m.wrapping_neg() } else { m } + }; + val = val.wrapping_add(d) & mask; + out[idx] = val as u16; + idx += 1; + if idx >= n { + break 'outer; + } + if r.bits_left() <= 0 || !r.read_bit() { + break; + } + } + let mut run = r.read(4) as usize; + if run == 0 { + run = r.read(11) as usize; + } + let run = run.min(n - idx); + out[idx..idx + run].fill(val as u16); + idx += run; + if idx >= n { + break; + } + if r.bits_left() <= 0 { + // Streams may end early; the remainder repeats the last value. + out[idx..].fill(val as u16); + break; + } + } + if r.bits_left() < -64 { + return Err(()); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +#[inline] +fn put_block(f: &[f32; 64], plane: &mut [u16], stride: usize, x0: usize, y0: usize, rows: usize, step: usize, parity: usize, conv: OutConv) { + if x0 >= stride { + return; + } + let xn = (stride - x0).min(8); + if xn == 8 && (y0 + 7) * step + parity < rows { + for yy in 0..8 { + let o = ((y0 + yy) * step + parity) * stride + x0; + let (Some(row), Some(src)) = (plane.get_mut(o..).and_then(|p| p.first_chunk_mut::<8>()), f.as_chunks::<8>().0.get(yy)) else { + return; + }; + for x in 0..8 { + row[x] = conv.conv(src[x]); + } + } + return; + } + for yy in 0..8 { + let line = (y0 + yy) * step + parity; + if line >= rows { + break; + } + let row = &mut plane[line * stride + x0..line * stride + x0 + xn]; + let src = &f[yy * 8..yy * 8 + xn]; + for (o, &v) in row.iter_mut().zip(src) { + *o = conv.conv(v); + } + } +} diff --git a/crates/video/src/prores/encode.rs b/crates/video/src/prores/encode.rs new file mode 100644 index 00000000..3e5a1f4e --- /dev/null +++ b/crates/video/src/prores/encode.rs @@ -0,0 +1,495 @@ +//! ProRes frame encoding (progressive) with per-slice quantiser rate control. + +use crate::prores::bits::{Counter, Sink, Writer}; +use crate::prores::dct::fdct8x8; +use crate::prores::header::{FRAME_ID, FrameHeader, PictureHeader, row_slice_widths}; +use crate::prores::tables::*; +use crate::prores::{AlphaType, ChromaFormat, ColorInfo, Error, Frame, Interlace, Profile, Result}; + +/// Encoder configuration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncoderConfig { + pub profile: Profile, + pub width: u32, + pub height: u32, + /// Coded-bits budget per macroblock; `None` uses [`Profile::nominal_bits_per_mb`]. + pub bits_per_mb: Option, + /// log2 of the macroblocks per slice (0..=3; 3 = 8 macroblocks, the common choice). + pub log2_slice_mbs: u8, + pub color: ColorInfo, + pub aspect_ratio: u8, + pub frame_rate_code: u8, + /// Encode the frame's alpha plane (4444 profiles only; 16-bit alpha coding). + pub encode_alpha: bool, + /// Encode slices in parallel (requires the `threads` feature). + pub threads: bool, + /// Four-character encoder identifier written into the frame header. + pub encoder_id: [u8; 4], +} + +impl EncoderConfig { + pub fn new(profile: Profile, width: u32, height: u32) -> EncoderConfig { + EncoderConfig { + profile, + width, + height, + bits_per_mb: None, + log2_slice_mbs: 3, + color: ColorInfo::default(), + aspect_ratio: 0, + frame_rate_code: 0, + encode_alpha: true, + threads: true, + encoder_id: *b"fmcr", + } + } +} + +/// ProRes encoder for one stream of same-sized progressive frames. +#[derive(Debug, Clone)] +pub struct Encoder { + cfg: EncoderConfig, +} + +/// Quantiser search range (RDD 36 allows 1..=224). +const Q_MIN: u8 = 1; +const Q_MAX: u8 = 224; + +impl Encoder { + pub fn new(profile: Profile, width: u32, height: u32) -> Encoder { + Encoder { cfg: EncoderConfig::new(profile, width, height) } + } + + pub fn with_config(cfg: EncoderConfig) -> Encoder { + Encoder { cfg } + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + pub fn bits_per_mb(&self) -> u32 { + self.cfg.bits_per_mb.unwrap_or_else(|| self.cfg.profile.nominal_bits_per_mb()) + } + + /// The rate-control target for one frame in bytes (excluding alpha). + pub fn target_frame_bytes(&self) -> usize { + let mbs = (self.cfg.width as usize).div_ceil(16) * (self.cfg.height as usize).div_ceil(16); + mbs * self.bits_per_mb() as usize / 8 + } + + /// Encode one frame. Input planes may be at any bit depth 8..=16 (10 is typical); the frame's + /// chroma format must match the profile. + pub fn encode(&mut self, frame: &Frame) -> Result> { + let cfg = &self.cfg; + if frame.width != cfg.width || frame.height != cfg.height { + return Err(Error::Input(format!("frame is {}x{}, encoder is {}x{}", frame.width, frame.height, cfg.width, cfg.height))); + } + if cfg.width == 0 || cfg.height == 0 || cfg.width > 65535 || cfg.height > 65535 { + return Err(Error::Input("unsupported frame size".into())); + } + let chroma = cfg.profile.chroma(); + if frame.chroma != chroma { + return Err(Error::Input(format!("{:?} input for a {:?} profile", frame.chroma, chroma))); + } + if !(8..=16).contains(&frame.bit_depth) { + return Err(Error::Input("bit depth must be 8..=16".into())); + } + let (w, h) = (cfg.width as usize, cfg.height as usize); + let cw = chroma.chroma_width(cfg.width) as usize; + if frame.y.len() < w * h || frame.cb.len() < cw * h || frame.cr.len() < cw * h { + return Err(Error::Input("plane too small".into())); + } + let with_alpha = cfg.encode_alpha && chroma == ChromaFormat::Yuv444 && frame.alpha.is_some(); + if let Some(a) = &frame.alpha + && with_alpha + && a.len() < w * h + { + return Err(Error::Input("alpha plane too small".into())); + } + if cfg.log2_slice_mbs > 3 { + return Err(Error::Input("log2_slice_mbs must be 0..=3".into())); + } + + let mb_width = w.div_ceil(16); + let mb_height = h.div_ceil(16); + let widths = row_slice_widths(mb_width, cfg.log2_slice_mbs); + let mut jobs = Vec::with_capacity(widths.len() * mb_height); + for row in 0..mb_height { + let mut x = 0; + for &sw in &widths { + jobs.push((x, row, sw)); + x += sw; + } + } + let src = Source { + frame, + w, + h, + cw, + chroma444: chroma == ChromaFormat::Yuv444, + scale: (2f32).powi(12 - frame.bit_depth as i32), + alpha: with_alpha, + bits_per_mb: self.bits_per_mb(), + }; + let threads = cfg.threads; + let prepared: Vec = par_map(threads, &jobs, |&(x, row, sw)| prepare_slice(&src, x, row, sw)); + let qs = choose_quantisers(&prepared, &src, mb_width * mb_height, threads); + let items: Vec<(&SliceData, u8)> = prepared.iter().zip(qs).collect(); + let slices: Vec> = par_map(threads, &items, |&(sd, q)| write_slice(sd, q)); + + let hdr = FrameHeader { + header_size: 0, + bitstream_version: if chroma == ChromaFormat::Yuv444 { 1 } else { 0 }, + encoder_id: cfg.encoder_id, + width: cfg.width as u16, + height: cfg.height as u16, + chroma, + interlace: Interlace::Progressive, + aspect_ratio: cfg.aspect_ratio & 15, + frame_rate_code: cfg.frame_rate_code & 15, + color: cfg.color, + alpha: if with_alpha { AlphaType::Bits16 } else { AlphaType::None }, + qmat_luma: [4; 64], + qmat_chroma: [4; 64], + luma_matrix_present: false, + chroma_matrix_present: false, + }; + let fh = hdr.write(); + let slice_bytes: usize = slices.iter().map(Vec::len).sum(); + let pic_size = 8 + 2 * slices.len() + slice_bytes; + let total = 8 + fh.len() + pic_size; + let mut out = Vec::with_capacity(total); + out.extend_from_slice(&(total as u32).to_be_bytes()); + out.extend_from_slice(&FRAME_ID); + out.extend_from_slice(&fh); + PictureHeader { header_size: 8, picture_size: pic_size, num_slices: slices.len(), log2_slice_mbs: cfg.log2_slice_mbs }.write(&mut out); + for s in &slices { + out.extend_from_slice(&(s.len() as u16).to_be_bytes()); + } + for s in &slices { + out.extend_from_slice(s); + } + debug_assert_eq!(out.len(), total); + Ok(out) + } +} + +struct Source<'a> { + frame: &'a Frame, + w: usize, + h: usize, + cw: usize, + chroma444: bool, + /// Multiplier from input sample values to the 12-bit domain. + scale: f32, + alpha: bool, + bits_per_mb: u32, +} + +/// Load an 8×8 block (edge-replicated) into the 12-bit signed domain and transform it. +fn load_block(plane: &[u16], stride: usize, height: usize, x0: usize, y0: usize, scale: f32) -> [f32; 64] { + let mut b = [0f32; 64]; + for yy in 0..8 { + let y = (y0 + yy).min(height - 1); + let row = &plane[y * stride..(y + 1) * stride]; + for xx in 0..8 { + let x = (x0 + xx).min(stride - 1); + b[yy * 8 + xx] = row[x] as f32 * scale - 2048.0; + } + } + fdct8x8(&mut b); + b +} + +/// Rounding offsets for quantisation: nearest for DC, a slight dead zone for AC. +const AC_ROUND: f32 = 0.45; + +fn quantise(coeffs: &[[f32; 64]], qmat_inv: &[f32; 64], out: &mut Vec<[i32; 64]>) { + out.clear(); + for c in coeffs { + let mut l = [0i32; 64]; + let dc = c[0] * qmat_inv[0]; + l[0] = dc.round() as i32; + for i in 1..64 { + let v = c[i] * qmat_inv[i]; + let a = (v.abs() + AC_ROUND) as i32; + l[i] = if v < 0.0 { -a } else { a }; + } + out.push(l); + } +} + +fn encode_component(s: &mut S, levels: &[[i32; 64]], scan: &[u8; 64]) { + let n = levels.len(); + // DC + let first = levels[0][0]; + let zz = if first >= 0 { (first as u32) << 1 } else { ((-(first as i64)) as u32) * 2 - 1 }; + s.put_cw(FIRST_DC_CB, zz); + let mut prev = first; + let mut ctx = DC_CTX_INIT; + let mut sign = false; + for blk in &levels[1..] { + let dc = blk[0]; + let delta = dc - prev; + prev = dc; + let code = if delta == 0 { + sign = false; + 0 + } else { + let neg = delta < 0; + let mag = delta.unsigned_abs(); + let c = if neg != sign { 2 * mag - 1 } else { 2 * mag }; + sign = neg; + c + }; + s.put_cw(DC_CB[ctx.min(6) as usize], code); + ctx = code; + } + // AC, interleaved across blocks + let mut run = 0u32; + let mut run_ctx = RUN_CTX_INIT; + let mut lev_ctx = LEVEL_CTX_INIT; + for &nat in &scan[1..] { + let nat = nat as usize; + for blk in levels.iter().take(n) { + let l = blk[nat]; + if l == 0 { + run += 1; + } else { + s.put_cw(RUN_CB[run_ctx.min(15) as usize], run); + run_ctx = run; + let a = l.unsigned_abs(); + s.put_cw(LEVEL_CB[lev_ctx.min(9) as usize], a - 1); + lev_ctx = a; + s.put((l < 0) as u32, 1); + run = 0; + } + } + } +} + +fn component_bits(levels: &[[i32; 64]]) -> u64 { + let mut c = Counter::default(); + encode_component(&mut c, levels, &SCAN_PROGRESSIVE); + c.0.div_ceil(8) * 8 +} + +/// A slice's transformed blocks (Y, Cb, Cr) and its coded alpha. +struct SliceData { + comps: [Vec<[f32; 64]>; 3], + alpha: Option>, +} + +impl SliceData { + fn header_size(&self) -> usize { + if self.alpha.is_some() { 8 } else { 6 } + } + /// Largest DCT payload (bits) that keeps every 16-bit size field valid. + fn limit_bits(&self) -> u64 { + (65535 - self.header_size() - self.alpha.as_ref().map_or(0, Vec::len)) as u64 * 8 + } +} + +fn par_map(threads: bool, items: &[T], f: impl Fn(&T) -> R + Sync + Send) -> Vec { + #[cfg(feature = "threads")] + if threads { + use rayon::prelude::*; + return items.par_iter().map(f).collect(); + } + let _ = threads; + items.iter().map(f).collect() +} + +fn prepare_slice(src: &Source, mb_x: usize, mb_y: usize, mbs: usize) -> SliceData { + let f = src.frame; + let x0 = mb_x * 16; + let y0 = mb_y * 16; + let mut luma = Vec::with_capacity(4 * mbs); + for mb in 0..mbs { + for i in 0..4 { + luma.push(load_block(&f.y, src.w, src.h, x0 + mb * 16 + (i & 1) * 8, y0 + (i >> 1) * 8, src.scale)); + } + } + let mut chroma = [Vec::with_capacity(4 * mbs), Vec::with_capacity(4 * mbs)]; + for (k, plane) in [&f.cb, &f.cr].into_iter().enumerate() { + for mb in 0..mbs { + if src.chroma444 { + // 4:4:4 chroma block order within a macroblock: TL, BL, TR, BR. + for i in 0..4 { + chroma[k].push(load_block(plane, src.cw, src.h, x0 + mb * 16 + (i >> 1) * 8, y0 + (i & 1) * 8, src.scale)); + } + } else { + for i in 0..2 { + chroma[k].push(load_block(plane, src.cw, src.h, mb_x * 8 + mb * 8, y0 + i * 8, src.scale)); + } + } + } + } + let [cb, cr] = chroma; + let alpha = src.alpha.then(|| encode_alpha_slice(src, x0, y0, mbs)); + SliceData { comps: [luma, cb, cr], alpha } +} + +fn quantise_all(sd: &SliceData, q: u8, lv: &mut [Vec<[i32; 64]>; 3]) { + let inv = [1.0 / (4.0 * qscale(q) as f32); 64]; + for (c, l) in sd.comps.iter().zip(lv.iter_mut()) { + quantise(c, &inv, l); + } +} + +/// Coded DCT payload bits of a slice at quantiser `q` (components padded to bytes). +fn slice_bits(sd: &SliceData, q: u8) -> u64 { + let mut lv = [Vec::new(), Vec::new(), Vec::new()]; + quantise_all(sd, q, &mut lv); + lv.iter().map(|l| component_bits(l)).sum() +} + +/// Frame-level rate control: the smallest uniform quantiser whose frame fits the budget, then +/// one step finer on as many slices as the remaining budget allows. Uniform quantisation +/// approximately minimises frame MSE for a given size. +fn choose_quantisers(slices: &[SliceData], src: &Source, total_mbs: usize, threads: bool) -> Vec { + let n = slices.len(); + let hsize = slices.first().map_or(6, SliceData::header_size); + let fixed = 8 * (8 + 20 + 8 + n * (2 + hsize)) as u64; + let budget = (src.bits_per_mb as u64 * total_mbs as u64).saturating_sub(fixed); + let bits_at = |q: u8| -> Vec { par_map(threads, slices, |sd| slice_bits(sd, q)) }; + let sum = |v: &[u64]| v.iter().sum::(); + + let mut best = bits_at(Q_MIN); + let mut qg = Q_MIN; + if sum(&best) > budget { + let (mut lo, mut hi) = (Q_MIN + 1, Q_MAX); + let mut hi_bits = None; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let v = bits_at(mid); + if sum(&v) <= budget { + hi = mid; + hi_bits = Some(v); + } else { + lo = mid + 1; + } + } + qg = lo; + best = hi_bits.unwrap_or_else(|| bits_at(qg)); + } + let mut qs = vec![qg; n]; + let mut cur = best; + let total = sum(&cur); + if qg > Q_MIN && total < budget { + let finer = bits_at(qg - 1); + let mut left = budget - total; + for i in 0..n { + let d = finer[i].saturating_sub(cur[i]); + if d <= left && finer[i] <= slices[i].limit_bits() { + left -= d; + qs[i] = qg - 1; + cur[i] = finer[i]; + } + } + } + // Keep every slice within its 16-bit size fields. + for i in 0..n { + while cur[i] > slices[i].limit_bits() && qs[i] < Q_MAX { + qs[i] += 1; + cur[i] = slice_bits(&slices[i], qs[i]); + } + } + qs +} + +fn write_slice(sd: &SliceData, q: u8) -> Vec { + let mut lv = [Vec::new(), Vec::new(), Vec::new()]; + quantise_all(sd, q, &mut lv); + let comps: Vec> = lv + .iter() + .map(|l| { + let mut w = Writer::new(); + encode_component(&mut w, l, &SCAN_PROGRESSIVE); + w.finish() + }) + .collect(); + let hsize = sd.header_size(); + let alpha = sd.alpha.as_deref().unwrap_or(&[]); + let mut out = Vec::with_capacity(hsize + comps.iter().map(Vec::len).sum::() + alpha.len()); + out.push((hsize as u8) << 3); + out.push(q); + out.extend_from_slice(&(comps[0].len() as u16).to_be_bytes()); + out.extend_from_slice(&(comps[1].len() as u16).to_be_bytes()); + if sd.alpha.is_some() { + out.extend_from_slice(&(comps[2].len() as u16).to_be_bytes()); + } + for c in &comps { + out.extend_from_slice(c); + } + out.extend_from_slice(alpha); + out +} + +/// Encode the slice's alpha samples (16-bit coding) in raster order over its 16 lines. +fn encode_alpha_slice(src: &Source, x0: usize, y0: usize, mbs: usize) -> Vec { + let Some(a) = src.frame.alpha.as_ref() else { return Vec::new() }; + let d = src.frame.bit_depth as u32; + let to16 = |v: u16| -> u32 { + let v = v as u32 & ((1 << d) - 1); + if d >= 16 { v } else { ((v << (16 - d)) | (v >> (2 * d).saturating_sub(16).min(d))) & 0xffff } + }; + let sw = mbs * 16; + let mut vals = Vec::with_capacity(sw * 16); + for yy in 0..16 { + let y = (y0 + yy).min(src.h - 1); + for xx in 0..sw { + let x = (x0 + xx).min(src.w - 1); + vals.push(to16(a[y * src.w + x])); + } + } + let mut w = Writer::new(); + encode_alpha(&mut w, &vals, 16); + w.finish() +} + +/// Alpha plane coding: a value (difference from the previous value, short or full width), then +/// either a one bit (another value follows) or a zero bit and a run of repeats of the value. +pub(crate) fn encode_alpha(w: &mut Writer, vals: &[u32], bits: u32) { + let mask = (1u32 << bits) - 1; + let (short_bits, short_max) = if bits == 16 { (7, 64i32) } else { (4, 8i32) }; + let n = vals.len(); + let mut prev = mask; + let mut idx = 0; + while idx < n { + let v = vals[idx]; + let diff = v.wrapping_sub(prev) & mask; + // signed difference in (-2^(bits-1), 2^(bits-1)] + let sd = if diff > mask >> 1 { diff as i32 - (mask as i32 + 1) } else { diff as i32 }; + if sd != 0 && sd.abs() <= short_max { + w.put(0, 1); + let c = (2 * sd.unsigned_abs() - 2) | (sd < 0) as u32; + w.put(c, short_bits); + } else { + w.put(1, 1); + w.put(diff, bits); + } + prev = v; + idx += 1; + if idx >= n { + break; + } + let mut run = 0; + while idx + run < n && vals[idx + run] == prev && run < 2047 { + run += 1; + } + if run == 0 { + w.put(1, 1); + continue; + } + w.put(0, 1); + if run < 16 { + w.put(run as u32, 4); + } else { + w.put(0, 4); + w.put(run as u32, 11); + } + idx += run; + } +} diff --git a/crates/video/src/prores/header.rs b/crates/video/src/prores/header.rs new file mode 100644 index 00000000..c630c256 --- /dev/null +++ b/crates/video/src/prores/header.rs @@ -0,0 +1,234 @@ +//! Frame and picture headers (RDD 36 §5). + +use crate::bitstream::BitReader; +use crate::prores::{AlphaType, ChromaFormat, ColorInfo, Error, Interlace, Result}; + +/// `'icpf'`: the frame identifier following the 32-bit frame size. +pub const FRAME_ID: [u8; 4] = *b"icpf"; + +/// A parsed ProRes frame header. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameHeader { + /// Size of the frame header in bytes (from its own first field). + pub header_size: u16, + pub bitstream_version: u8, + /// Four-character encoder identifier (e.g. `apl0`, `Lavc`, `fmcr`). + pub encoder_id: [u8; 4], + pub width: u16, + pub height: u16, + pub chroma: ChromaFormat, + pub interlace: Interlace, + /// `aspect_ratio_information` (0 = unknown/square, 1 = 1:1, 2 = 4:3, 3 = 16:9). + pub aspect_ratio: u8, + /// `frame_rate_code` (0 = unknown). + pub frame_rate_code: u8, + pub color: ColorInfo, + pub alpha: AlphaType, + /// Luma quantisation matrix in raster order (all 4 when not transmitted). + pub qmat_luma: [u8; 64], + /// Chroma quantisation matrix in raster order (the luma matrix when not transmitted). + pub qmat_chroma: [u8; 64], + pub luma_matrix_present: bool, + pub chroma_matrix_present: bool, +} + +impl FrameHeader { + /// Parse the 8-byte frame prefix and frame header. Returns the header, the byte offset of the + /// first picture, and the frame size (bytes of `data` that belong to this frame). + pub fn parse(data: &[u8]) -> Result<(FrameHeader, usize, usize)> { + if data.len() < 8 + 20 { + return Err(Error::Truncated); + } + let frame_size = u32::from_be_bytes([data[0], data[1], data[2], data[3]]) as usize; + if data[4..8] != FRAME_ID { + return Err(Error::Invalid("missing 'icpf' frame identifier")); + } + if frame_size < 28 || frame_size > data.len() { + return Err(Error::Truncated); + } + let data = &data[..frame_size]; + let h = &data[8..]; + let mut r = BitReader::new(h); + let rd = |r: &mut BitReader, n: u32| r.read_bits(n).map_err(|_| Error::Truncated); + let header_size = rd(&mut r, 16)? as u16; + if (header_size as usize) < 20 || 8 + header_size as usize > data.len() { + return Err(Error::Invalid("bad frame header size")); + } + let _reserved = rd(&mut r, 8)?; + let bitstream_version = rd(&mut r, 8)? as u8; + let encoder_id = [h[4], h[5], h[6], h[7]]; + r.skip(32).map_err(|_| Error::Truncated)?; + let width = rd(&mut r, 16)? as u16; + let height = rd(&mut r, 16)? as u16; + let chroma = match rd(&mut r, 2)? { + 2 => ChromaFormat::Yuv422, + 3 => ChromaFormat::Yuv444, + _ => return Err(Error::Unsupported("chroma format")), + }; + let _ = rd(&mut r, 2)?; + let interlace = match rd(&mut r, 2)? { + 0 => Interlace::Progressive, + 1 => Interlace::TopFieldFirst, + 2 => Interlace::BottomFieldFirst, + _ => return Err(Error::Invalid("interlace mode")), + }; + let _ = rd(&mut r, 2)?; + let aspect_ratio = rd(&mut r, 4)? as u8; + let frame_rate_code = rd(&mut r, 4)? as u8; + let color = ColorInfo { primaries: rd(&mut r, 8)? as u8, transfer: rd(&mut r, 8)? as u8, matrix: rd(&mut r, 8)? as u8 }; + let _ = rd(&mut r, 4)?; + let alpha = match rd(&mut r, 4)? { + 0 => AlphaType::None, + 1 => AlphaType::Bits8, + 2 => AlphaType::Bits16, + _ => return Err(Error::Unsupported("alpha channel type")), + }; + let _ = rd(&mut r, 14)?; + let luma_matrix_present = rd(&mut r, 1)? == 1; + let chroma_matrix_present = rd(&mut r, 1)? == 1; + let mut pos = 20usize; + let mut qmat_luma = [4u8; 64]; + if luma_matrix_present { + let m = h.get(pos..pos + 64).ok_or(Error::Truncated)?; + qmat_luma.copy_from_slice(m); + pos += 64; + } + let mut qmat_chroma = qmat_luma; + if chroma_matrix_present { + let m = h.get(pos..pos + 64).ok_or(Error::Truncated)?; + qmat_chroma.copy_from_slice(m); + pos += 64; + } + if pos > header_size as usize { + return Err(Error::Invalid("quantisation matrices exceed frame header")); + } + if qmat_luma.contains(&0) || qmat_chroma.contains(&0) { + return Err(Error::Invalid("zero quantisation matrix entry")); + } + if width == 0 || height == 0 { + return Err(Error::Invalid("zero frame dimension")); + } + let hdr = FrameHeader { + header_size, + bitstream_version, + encoder_id, + width, + height, + chroma, + interlace, + aspect_ratio, + frame_rate_code, + color, + alpha, + qmat_luma, + qmat_chroma, + luma_matrix_present, + chroma_matrix_present, + }; + Ok((hdr, 8 + header_size as usize, frame_size)) + } + + /// Serialise the frame header (without the 8-byte size/identifier prefix). + pub fn write(&self) -> Vec { + let mut v = Vec::with_capacity(148); + let size = 20 + 64 * (self.luma_matrix_present as usize + self.chroma_matrix_present as usize); + v.extend_from_slice(&(size as u16).to_be_bytes()); + v.push(0); + v.push(self.bitstream_version); + v.extend_from_slice(&self.encoder_id); + v.extend_from_slice(&self.width.to_be_bytes()); + v.extend_from_slice(&self.height.to_be_bytes()); + let cf = match self.chroma { + ChromaFormat::Yuv422 => 2u8, + ChromaFormat::Yuv444 => 3, + }; + let il = match self.interlace { + Interlace::Progressive => 0u8, + Interlace::TopFieldFirst => 1, + Interlace::BottomFieldFirst => 2, + }; + v.push((cf << 6) | (il << 2)); + v.push((self.aspect_ratio << 4) | (self.frame_rate_code & 15)); + v.push(self.color.primaries); + v.push(self.color.transfer); + v.push(self.color.matrix); + v.push(match self.alpha { + AlphaType::None => 0, + AlphaType::Bits8 => 1, + AlphaType::Bits16 => 2, + }); + v.push(0); + v.push(((self.luma_matrix_present as u8) << 1) | self.chroma_matrix_present as u8); + if self.luma_matrix_present { + v.extend_from_slice(&self.qmat_luma); + } + if self.chroma_matrix_present { + v.extend_from_slice(&self.qmat_chroma); + } + v + } +} + +/// A parsed picture header (one per progressive frame, two per interlaced frame). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PictureHeader { + pub header_size: usize, + /// Total picture size in bytes including this header, the slice table and the slices. + pub picture_size: usize, + pub num_slices: usize, + pub log2_slice_mbs: u8, +} + +impl PictureHeader { + pub fn parse(data: &[u8]) -> Result { + if data.len() < 8 { + return Err(Error::Truncated); + } + let header_size = (data[0] >> 3) as usize; + if header_size < 8 || header_size > data.len() { + return Err(Error::Invalid("bad picture header size")); + } + let picture_size = u32::from_be_bytes([data[1], data[2], data[3], data[4]]) as usize; + let num_slices = u16::from_be_bytes([data[5], data[6]]) as usize; + let log2_slice_mbs = (data[7] >> 4) & 3; + Ok(PictureHeader { header_size, picture_size, num_slices, log2_slice_mbs }) + } + + pub fn write(&self, out: &mut Vec) { + out.push(8 << 3); + out.extend_from_slice(&(self.picture_size as u32).to_be_bytes()); + out.extend_from_slice(&(self.num_slices as u16).to_be_bytes()); + out.push((self.log2_slice_mbs & 3) << 4); + } +} + +/// Slice widths (in macroblocks) of one macroblock row: as many `1 << log2` slices as fit, then +/// the remainder split into decreasing powers of two. +pub fn row_slice_widths(mb_width: usize, log2_slice_mbs: u8) -> Vec { + let size = 1usize << log2_slice_mbs; + let mut v = vec![size; mb_width / size]; + let mut rem = mb_width % size; + let mut s = size >> 1; + while rem > 0 { + if rem >= s { + v.push(s); + rem -= s; + } + s >>= 1; + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slice_widths() { + assert_eq!(row_slice_widths(120, 3), vec![8; 15]); + assert_eq!(row_slice_widths(45, 3), vec![8, 8, 8, 8, 8, 4, 1]); + assert_eq!(row_slice_widths(7, 3), vec![4, 2, 1]); + assert_eq!(row_slice_widths(2, 3), vec![2]); + assert_eq!(row_slice_widths(5, 0), vec![1; 5]); + } +} diff --git a/crates/video/src/prores/mod.rs b/crates/video/src/prores/mod.rs new file mode 100644 index 00000000..1e52fd97 --- /dev/null +++ b/crates/video/src/prores/mod.rs @@ -0,0 +1,211 @@ +//! Clean-room Apple ProRes codec (implemented from SMPTE RDD 36). +//! +//! - **Decoder** ([`decode_frame`], [`decode_frame_with`]): ProRes 422 Proxy/LT/Standard/HQ and +//! 4444/4444 XQ including the alpha channel; progressive and interlaced frames; slice-parallel +//! with the `threads` feature (rayon). Output is planar `u16` at 10 bits for 4:2:2 and 12 bits +//! for 4:4:4 by default (any depth 8..=16 on request). +//! - **Encoder** ([`Encoder`]): ProRes 422 (all four flavours) and 4444/4444 XQ (optionally with +//! alpha), progressive, with per-slice quantiser rate control targeting Apple's nominal rates. +//! +//! See the crate README for bitstream details, accuracy and limitations. + +mod bits; +mod dct; +mod decode; +mod encode; +pub mod header; +mod tables; + +pub use decode::{DecodeOptions, decode_frame, decode_frame_into, decode_frame_with, probe}; +pub use encode::{Encoder, EncoderConfig}; +pub use header::{FrameHeader, PictureHeader}; + +/// Codec errors. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("truncated ProRes data")] + Truncated, + #[error("invalid ProRes data: {0}")] + Invalid(&'static str), + #[error("unsupported ProRes feature: {0}")] + Unsupported(&'static str), + #[error("corrupt slice {slice} in picture {picture}")] + Slice { picture: usize, slice: usize }, + #[error("invalid encoder input: {0}")] + Input(String), +} + +pub type Result = std::result::Result; + +/// Chroma subsampling of a frame. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ChromaFormat { + /// Chroma planes are half width, full height. + Yuv422, + /// Chroma planes are full size. + Yuv444, +} + +impl ChromaFormat { + /// Width of a chroma plane for a luma width. + pub fn chroma_width(self, width: u32) -> u32 { + match self { + ChromaFormat::Yuv422 => width.div_ceil(2), + ChromaFormat::Yuv444 => width, + } + } +} + +/// Frame scan structure. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum Interlace { + #[default] + Progressive, + /// Two fields; the first coded picture is the top field (even lines). + TopFieldFirst, + /// Two fields; the first coded picture is the bottom field (odd lines). + BottomFieldFirst, +} + +/// Alpha channel coding in the bitstream. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum AlphaType { + #[default] + None, + Bits8, + Bits16, +} + +/// Colour description codes from the frame header (ITU-T H.273 / ISO 23091-2 code points; +/// 0 = unspecified in ProRes, 2 = unspecified in H.273). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ColorInfo { + pub primaries: u8, + pub transfer: u8, + pub matrix: u8, +} + +impl Default for ColorInfo { + /// BT.709 primaries, transfer and matrix. + fn default() -> Self { + ColorInfo { primaries: 1, transfer: 1, matrix: 1 } + } +} + +/// ProRes profiles (flavours). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Profile { + /// `apco` + Proxy, + /// `apcs` + Lt, + /// `apcn` + Standard, + /// `apch` + Hq, + /// `ap4h` + P4444, + /// `ap4x` + P4444Xq, +} + +impl Profile { + pub const ALL: [Profile; 6] = [Profile::Proxy, Profile::Lt, Profile::Standard, Profile::Hq, Profile::P4444, Profile::P4444Xq]; + + /// The QuickTime sample-entry four-character code. + pub fn fourcc(self) -> [u8; 4] { + *match self { + Profile::Proxy => b"apco", + Profile::Lt => b"apcs", + Profile::Standard => b"apcn", + Profile::Hq => b"apch", + Profile::P4444 => b"ap4h", + Profile::P4444Xq => b"ap4x", + } + } + + pub fn from_fourcc(f: &[u8; 4]) -> Option { + Profile::ALL.into_iter().find(|p| &p.fourcc() == f) + } + + pub fn chroma(self) -> ChromaFormat { + match self { + Profile::P4444 | Profile::P4444Xq => ChromaFormat::Yuv444, + _ => ChromaFormat::Yuv422, + } + } + + /// Apple's published nominal data rate at 1920×1080, 29.97 fps, in Mbit/s. + pub fn nominal_mbps_1080p30(self) -> f64 { + match self { + Profile::Proxy => 45.0, + Profile::Lt => 102.0, + Profile::Standard => 147.0, + Profile::Hq => 220.0, + Profile::P4444 => 330.0, + Profile::P4444Xq => 500.0, + } + } + + /// Nominal coded bits per 16×16 macroblock (the nominal rate spread over the 8160 + /// macroblocks of a 1920×1080 frame at 30000/1001 fps). Frame targets scale with area. + pub fn nominal_bits_per_mb(self) -> u32 { + (self.nominal_mbps_1080p30() * 1e6 * 1001.0 / 30000.0 / 8160.0).round() as u32 + } +} + +/// A decoded (or to-be-encoded) picture: planar `u16` samples, one sample per element, +/// rows packed without padding. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + pub width: u32, + pub height: u32, + pub chroma: ChromaFormat, + /// Significant bits per sample of every plane (including alpha). + pub bit_depth: u8, + /// `width × height` + pub y: Vec, + /// `chroma_width × height` + pub cb: Vec, + /// `chroma_width × height` + pub cr: Vec, + /// `width × height` (full range: 0 = transparent, max = opaque) + pub alpha: Option>, + pub interlace: Interlace, + pub color: ColorInfo, + /// `aspect_ratio_information` from the frame header. + pub aspect_ratio: u8, + /// `frame_rate_code` from the frame header. + pub frame_rate_code: u8, +} + +impl Frame { + /// A black (video-range) frame with the given geometry. + pub fn new(width: u32, height: u32, chroma: ChromaFormat, bit_depth: u8, with_alpha: bool) -> Frame { + let n = width as usize * height as usize; + let cn = chroma.chroma_width(width) as usize * height as usize; + let s = bit_depth.saturating_sub(8) as u32; + Frame { + width, + height, + chroma, + bit_depth, + y: vec![(16u32 << s) as u16; n], + cb: vec![(128u32 << s) as u16; cn], + cr: vec![(128u32 << s) as u16; cn], + alpha: with_alpha.then(|| vec![((1u32 << bit_depth) - 1) as u16; n]), + interlace: Interlace::Progressive, + color: ColorInfo::default(), + aspect_ratio: 0, + frame_rate_code: 0, + } + } + + pub fn chroma_width(&self) -> u32 { + self.chroma.chroma_width(self.width) + } + + pub fn is_interlaced(&self) -> bool { + self.interlace != Interlace::Progressive + } +} diff --git a/crates/video/src/prores/tables.rs b/crates/video/src/prores/tables.rs new file mode 100644 index 00000000..a395e906 --- /dev/null +++ b/crates/video/src/prores/tables.rs @@ -0,0 +1,133 @@ +//! Constant tables defined by the ProRes bitstream format (SMPTE RDD 36). + +/// A ProRes variable-length codebook: a Golomb-Rice code for small values that switches to an +/// Exp-Golomb code for large ones. +/// +/// A codeword starts with `q` zero bits and a one bit. +/// - If `q <= rice_q_max` it is a Rice codeword: `value = (q << rice) | next(rice bits)`. +/// - Otherwise it is an Exp-Golomb codeword of order `exp` whose prefix is `q - rice_q_max - 1` +/// zeros, offset by the number of values the Rice part covers: `(rice_q_max + 1) << rice`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Codebook { + pub rice: u8, + pub exp: u8, + pub rice_q_max: u8, +} + +const fn cb(rice: u8, exp: u8, rice_q_max: u8) -> Codebook { + Codebook { rice, exp, rice_q_max } +} + +/// Codebook of the first DC coefficient in a slice component (equivalent to Exp-Golomb order 5). +pub(crate) const FIRST_DC_CB: Codebook = cb(5, 6, 0); + +/// DC difference codebooks, selected by the previous DC codeword value (clamped to 6). +pub(crate) const DC_CB: [Codebook; 7] = [cb(0, 1, 0), cb(1, 2, 0), cb(1, 2, 0), cb(2, 3, 1), cb(2, 3, 1), cb(3, 4, 0), cb(3, 4, 0)]; + +/// AC run codebooks, selected by the previous run (clamped to 15). +pub(crate) const RUN_CB: [Codebook; 16] = [ + cb(0, 1, 2), + cb(0, 1, 2), + cb(0, 1, 1), + cb(0, 1, 1), + cb(0, 1, 0), + cb(1, 2, 1), + cb(1, 2, 1), + cb(1, 2, 1), + cb(1, 2, 1), + cb(1, 2, 0), + cb(1, 2, 0), + cb(1, 2, 0), + cb(1, 2, 0), + cb(1, 2, 0), + cb(1, 2, 0), + cb(2, 3, 0), +]; + +/// AC level codebooks (level magnitude minus one), selected by the previous level magnitude (clamped to 9). +pub(crate) const LEVEL_CB: [Codebook; 10] = + [cb(0, 1, 0), cb(0, 2, 2), cb(0, 1, 1), cb(0, 1, 2), cb(0, 1, 0), cb(1, 2, 0), cb(1, 2, 0), cb(1, 2, 0), cb(1, 2, 0), cb(2, 3, 0)]; + +/// Initial DC codebook context for the second DC of a component. +pub(crate) const DC_CTX_INIT: u32 = 5; +/// Initial AC run / level codebook contexts. +pub(crate) const RUN_CTX_INIT: u32 = 4; +pub(crate) const LEVEL_CTX_INIT: u32 = 2; + +/// Progressive-frame coefficient scan: scan position → raster index within the 8×8 block. +pub(crate) const SCAN_PROGRESSIVE: [u8; 64] = [ + 0, 1, 8, 9, 2, 3, 10, 11, // + 16, 17, 24, 25, 18, 19, 26, 27, // + 4, 5, 12, 20, 13, 6, 7, 14, // + 21, 28, 29, 22, 15, 23, 30, 31, // + 32, 33, 40, 48, 41, 34, 35, 42, // + 49, 56, 57, 50, 43, 36, 37, 44, // + 51, 58, 59, 52, 45, 38, 39, 46, // + 53, 60, 61, 54, 47, 55, 62, 63, +]; + +/// Interlaced-field coefficient scan: scan position → raster index within the 8×8 block. +pub(crate) const SCAN_INTERLACED: [u8; 64] = [ + 0, 8, 1, 9, 16, 24, 17, 25, // + 2, 10, 3, 11, 18, 26, 19, 27, // + 32, 40, 33, 34, 41, 48, 56, 49, // + 42, 35, 43, 50, 57, 58, 51, 59, // + 4, 12, 5, 6, 13, 20, 28, 21, // + 14, 7, 15, 22, 29, 36, 44, 37, // + 30, 23, 31, 38, 45, 52, 60, 53, // + 46, 39, 47, 54, 61, 62, 55, 63, +]; + +/// Quantisation scale for a slice `quantization_index` (1..=224). +#[inline] +pub(crate) fn qscale(qidx: u8) -> u32 { + let q = qidx.max(1) as u32; + if q <= 128 { q } else { 128 + ((q - 128) << 2) } +} + +/// Orthonormal 1-D DCT-II basis: `BASIS[u][x] = C(u)/2 · cos((2x+1)uπ/16)`, `C(0) = 1/√2`. +pub(crate) const BASIS: [[f32; 8]; 8] = [ + [0.353_553_39, 0.353_553_39, 0.353_553_39, 0.353_553_39, 0.353_553_39, 0.353_553_39, 0.353_553_39, 0.353_553_39], + [0.490_392_64, 0.415_734_8, 0.277_785_12, 0.097_545_16, -0.097_545_16, -0.277_785_12, -0.415_734_8, -0.490_392_64], + [0.461_939_77, 0.191_341_72, -0.191_341_72, -0.461_939_77, -0.461_939_77, -0.191_341_72, 0.191_341_72, 0.461_939_77], + [0.415_734_8, -0.097_545_16, -0.490_392_64, -0.277_785_12, 0.277_785_12, 0.490_392_64, 0.097_545_16, -0.415_734_8], + [0.353_553_39, -0.353_553_39, -0.353_553_39, 0.353_553_39, 0.353_553_39, -0.353_553_39, -0.353_553_39, 0.353_553_39], + [0.277_785_12, -0.490_392_64, 0.097_545_16, 0.415_734_8, -0.415_734_8, -0.097_545_16, 0.490_392_64, -0.277_785_12], + [0.191_341_72, -0.461_939_77, 0.461_939_77, -0.191_341_72, -0.191_341_72, 0.461_939_77, -0.461_939_77, 0.191_341_72], + [0.097_545_16, -0.277_785_12, 0.415_734_8, -0.490_392_64, 0.490_392_64, -0.415_734_8, 0.277_785_12, -0.097_545_16], +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scans_are_permutations() { + for scan in [&SCAN_PROGRESSIVE, &SCAN_INTERLACED] { + let mut seen = [false; 64]; + for &i in scan.iter() { + assert!(!seen[i as usize]); + seen[i as usize] = true; + } + } + } + + #[test] + fn basis_matches_cosines() { + for u in 0..8 { + for x in 0..8 { + let c = if u == 0 { (0.125f64).sqrt() } else { 0.5 }; + let v = c * (((2 * x + 1) * u) as f64 * std::f64::consts::PI / 16.0).cos(); + assert!((v - BASIS[u][x] as f64).abs() < 1e-7); + } + } + } + + #[test] + fn qscale_mapping() { + assert_eq!(qscale(1), 1); + assert_eq!(qscale(128), 128); + assert_eq!(qscale(129), 132); + assert_eq!(qscale(224), 512); + } +} diff --git a/crates/video/src/synth.rs b/crates/video/src/synth.rs new file mode 100644 index 00000000..3642d2e8 --- /dev/null +++ b/crates/video/src/synth.rs @@ -0,0 +1,184 @@ +//! Tiny synthetic movies, generated in code (for tests, demos and screenshots; nothing binary is +//! committed). H.264 pictures are made of I_PCM macroblocks, so their decoded samples are known +//! exactly; ProRes comes from the bundled encoder and Motion JPEG from the `image` encoder. All +//! are muxed with the bundled MP4/MOV writer. + +use crate::bitstream::{BitWriter, escape_rbsp}; +use crate::isobmff::{AvcConfig, Brand, FourCc, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; +use crate::{Result, VideoError}; +use std::io::Cursor; + +/// Macroblock grid of [`h264_mp4`] movies (64x48 pixels). +pub const MB_W: usize = 4; +pub const MB_H: usize = 3; + +fn err(e: impl std::fmt::Display) -> VideoError { + VideoError::Container(e.to_string()) +} + +fn nal_body(ref_idc: u8, t: u8, rbsp: &[u8]) -> Vec { + let mut v = vec![(ref_idc << 5) | t]; + v.extend(escape_rbsp(rbsp)); + v +} + +fn sps(mb_w: usize, mb_h: usize) -> Vec { + let mut w = BitWriter::new(); + w.write_bits(66, 8); // Baseline + w.write_bits(0, 8); + w.write_bits(30, 8); + w.write_ue(0); // sps id + w.write_ue(0); // log2_max_frame_num - 4 + w.write_ue(2); // POC type 2: output order = decoding order + w.write_ue(1); // max_num_ref_frames + w.write_bit(false); + w.write_ue(mb_w.saturating_sub(1) as u32); + w.write_ue(mb_h.saturating_sub(1) as u32); + w.write_bit(true); // frame_mbs_only + w.write_bit(true); // direct_8x8_inference + w.write_bit(false); // cropping + w.write_bit(false); // vui + w.rbsp_trailing(); + nal_body(3, 7, &w.finish()) +} + +fn pps() -> Vec { + let mut w = BitWriter::new(); + w.write_ue(0); + w.write_ue(0); + w.write_bit(false); // CAVLC + w.write_bit(false); + w.write_ue(0); + w.write_ue(0); + w.write_ue(0); + w.write_bit(false); + w.write_bits(0, 2); + w.write_se(0); + w.write_se(0); + w.write_se(0); + w.write_bit(true); + w.write_bit(false); + w.write_bit(false); + w.rbsp_trailing(); + nal_body(3, 8, &w.finish()) +} + +/// One I picture (IDR or non-IDR reference) of I_PCM macroblocks, each a solid Y'CbCr colour. +fn pcm_picture(idr: bool, frame_num: u32, mb_w: usize, mb_h: usize, color: &dyn Fn(usize, usize) -> [u8; 3]) -> Vec { + let mut w = BitWriter::new(); + w.write_ue(0); // first_mb_in_slice + w.write_ue(7); // I (all slices) + w.write_ue(0); // pps id + w.write_bits(frame_num % 16, 4); + if idr { + w.write_ue(0); // idr_pic_id + w.write_bit(false); + w.write_bit(false); + } else { + w.write_bit(false); // adaptive_ref_pic_marking_mode_flag + } + w.write_se(0); // slice_qp_delta + w.write_ue(0); // deblocking on (no effect on I_PCM at qP 0) + w.write_se(0); + w.write_se(0); + for my in 0..mb_h { + for mx in 0..mb_w { + let [y, cb, cr] = color(mx, my); + let mut mb = vec![y; 256]; + mb.extend([cb; 64]); + mb.extend([cr; 64]); + w.write_ue(25); // I_PCM + w.align_zero(); + w.write_bytes(&mb); + } + } + w.rbsp_trailing(); + nal_body(3, if idr { 5 } else { 1 }, &w.finish()) +} + +fn length_prefixed(nal: &[u8]) -> Vec { + let mut v = Vec::with_capacity(nal.len() + 4); + v.extend(u32::try_from(nal.len()).unwrap_or(u32::MAX).to_be_bytes()); + v.extend(nal); + v +} + +/// Y'CbCr of frame `i` of [`h264_mp4`] (limited range; luma steps so every frame differs). +pub fn frame_color(i: usize) -> [u8; 3] { + [(20 + (i * 7) % 200) as u8, (60 + (i * 13) % 130) as u8, (200 - (i * 11) % 130) as u8] +} + +/// H.264 MP4 of `mb_w` × `mb_h` macroblocks, `n` frames at `fps` (integer), an IDR picture every +/// `gop` frames; `color(frame, mb_x, mb_y)` gives each macroblock's Y'CbCr. +pub fn h264_mp4_with(n: usize, gop: usize, mb_w: usize, mb_h: usize, fps: u32, color: &dyn Fn(usize, usize, usize) -> [u8; 3]) -> Result> { + let (mb_w, mb_h) = (mb_w.clamp(1, 120), mb_h.clamp(1, 68)); + let fps = fps.clamp(1, 240); + let avcc = AvcConfig::new(vec![sps(mb_w, mb_h)], vec![pps()], 4); + let mut mw = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).map_err(err)?; + let entry = SampleEntry::avc(avcc, (mb_w * 16) as u16, (mb_h * 16) as u16); + let t = mw.add_track(TrackConfig::new(entry, fps * 100)).map_err(err)?; + let mut frame_num = 0; + for i in 0..n.min(100_000) { + let idr = i % gop.max(1) == 0; + if idr { + frame_num = 0; + } + let au = length_prefixed(&pcm_picture(idr, frame_num, mb_w, mb_h, &|x, y| color(i, x, y))); + frame_num += 1; + mw.write_sample(t, WriteSample { data: &au, duration: 100, composition_offset: 0, is_sync: idr }).map_err(err)?; + } + Ok(mw.finish().map_err(err)?.into_inner()) +} + +/// 64x48 H.264 MP4, `n` solid frames ([`frame_color`]) at 24 fps, an IDR every `gop` frames. +pub fn h264_mp4(n: usize, gop: usize) -> Result> { + h264_mp4_with(n, gop, MB_W, MB_H, 24, &|i, _, _| frame_color(i)) +} + +/// A colour-bar test movie for demos and screenshots: `secs` seconds at 24 fps, 320x192, bars with +/// a white block that moves one macroblock per frame (so frames are visibly different). +pub fn test_pattern_mp4(secs: u32) -> Result> { + // Limited-range BT.601 Y'CbCr of 75 % bars: white, yellow, cyan, green, magenta, red, blue. + const BARS: [[u8; 3]; 7] = [[180, 128, 128], [162, 44, 142], [131, 156, 44], [112, 72, 58], [84, 184, 198], [65, 100, 212], [35, 212, 114]]; + let (w, h) = (20usize, 12usize); + h264_mp4_with(secs.clamp(1, 600) as usize * 24, 24, w, h, 24, &|i, x, y| { + if y == 9 && x == i % w { + [235, 128, 128] + } else if y >= 8 { + [16 + (x * 219 / w) as u8, 128, 128] + } else { + BARS.get(x * BARS.len() / w).copied().unwrap_or([16, 128, 128]) + } + }) +} + +/// A MOV with `n` ProRes 422 HQ frames (64x48) at 25 fps; frame i has luma 16 + 8 i (8-bit scale). +pub fn prores_mov(n: usize) -> Result> { + use crate::prores::{ChromaFormat, Encoder, Frame, Profile}; + let (w, h) = (64u32, 48u32); + let mut enc = Encoder::new(Profile::Hq, w, h); + let mut mw = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).map_err(err)?; + let t = mw.add_track(TrackConfig::new(SampleEntry::prores(FourCc(*b"apch"), w as u16, h as u16), 25)).map_err(err)?; + for i in 0..n.min(10_000) { + let mut f = Frame::new(w, h, ChromaFormat::Yuv422, 10, false); + let y = ((16 + 8 * (i as u32 % 27)) << 2) as u16; + f.y.iter_mut().for_each(|s| *s = y); + let data = enc.encode(&f).map_err(err)?; + mw.write_sample(t, WriteSample { data: &data, duration: 1, composition_offset: 0, is_sync: true }).map_err(err)?; + } + Ok(mw.finish().map_err(err)?.into_inner()) +} + +/// A MOV with `n` Motion JPEG frames (32x24) at 10 fps; frame i is grey level 30 i. +pub fn mjpeg_mov(n: usize) -> Result> { + let (w, h) = (32u32, 24u32); + let mut mw = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).map_err(err)?; + let t = mw.add_track(TrackConfig::new(SampleEntry::jpeg(w as u16, h as u16), 10)).map_err(err)?; + for i in 0..n.min(10_000) { + let img = image::RgbImage::from_pixel(w, h, image::Rgb([(30 * (i % 9)) as u8; 3])); + let mut jpg = Vec::new(); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpg, 95).encode_image(&img).map_err(err)?; + mw.write_sample(t, WriteSample { data: &jpg, duration: 1, composition_offset: 0, is_sync: true }).map_err(err)?; + } + Ok(mw.finish().map_err(err)?.into_inner()) +} diff --git a/crates/video/src/tests.rs b/crates/video/src/tests.rs new file mode 100644 index 00000000..58db4b3c --- /dev/null +++ b/crates/video/src/tests.rs @@ -0,0 +1,208 @@ +//! Movie tests on fixtures generated in process (no binary fixtures): H.264 made of I_PCM +//! macroblocks (exact, known colours), ProRes from the bundled encoder and Motion JPEG from the +//! `image` encoder, muxed with the bundled MP4/MOV writer. When `ffmpeg` with libx264 is on PATH, +//! real High-profile streams with B-frames are checked too (skipped otherwise). + +use crate::isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; +use crate::synth::{frame_color, h264_mp4, mjpeg_mov, prores_mov}; +use crate::{Movie, VideoError}; +use std::io::Cursor; + +fn expected_rgb(i: usize) -> [u8; 3] { + let [y, cb, cr] = frame_color(i); + crate::convert::Matrix::new(false, false).rgb(y, cb, cr) +} + +fn assert_solid(f: &crate::Frame, rgb: [u8; 3], tol: i32) { + assert_eq!(f.rgba.len(), (f.width * f.height * 4) as usize); + for px in f.rgba.as_chunks::<4>().0.iter().step_by(37) { + for c in 0..3 { + assert!((i32::from(px[c]) - i32::from(rgb[c])).abs() <= tol, "frame {} pixel {:?} expected {:?}", f.index, px, rgb); + } + assert_eq!(px[3], 255); + } +} + +#[test] +fn h264_info_and_exact_frames() { + let bytes = h264_mp4(30, 10).unwrap(); + let mut m = Movie::open(bytes).unwrap(); + let info = m.info().clone(); + assert_eq!((info.width, info.height), (64, 48)); + assert_eq!(info.codec, "h264"); + assert_eq!(info.codec_detail, "H.264 Baseline"); + assert!(info.decodable && !info.has_audio && !info.quicktime); + assert_eq!(info.frame_count, 30); + assert!((info.frame_rate - 24.0).abs() < 1e-9); + assert!((info.duration - 1.25).abs() < 1e-9); + // Sequential playback: every frame exact, decoding forward (one decode per frame). + for i in 0..30 { + let f = m.frame_at(i as f64 / 24.0 + 0.001).unwrap(); + assert_eq!(f.index, i); + assert_solid(&f, expected_rgb(i), 1); + } + assert!(m.decoded <= 31, "sequential decode re-decoded frames: {}", m.decoded); + assert!(m.frame_at(1.25).is_err() && m.frame_at(-0.1).is_err() && m.frame_at(f64::NAN).is_err()); +} + +#[test] +fn h264_random_seeks_decode_from_the_keyframe() { + let mut m = Movie::open(h264_mp4(40, 20).unwrap()).unwrap(); + m.set_cache_frames(1); + for &i in &[37usize, 3, 19, 20, 5, 39, 0, 21, 21, 12] { + let f = m.frame(i).unwrap(); + assert_eq!(f.index, i); + assert_solid(&f, expected_rgb(i), 1); + } + // Backwards seek to 12 decoded at most frames 0..=12 again (+ the read-ahead). + let before = m.decoded; + m.frame(13).unwrap(); + assert!(m.decoded - before <= 1, "forward step should not restart"); +} + +#[test] +fn h264_thumbnails_use_keyframes() { + let mut m = Movie::open(h264_mp4(30, 10).unwrap()).unwrap(); + let th = m.thumbnail_at(0.6, 12).unwrap(); + assert_eq!(th.height, 12); + assert_eq!(th.width, 16); + assert_eq!(th.index, 10, "thumbnail snaps to the keyframe before 0.6 s (frame 14)"); + assert_solid(&th, expected_rgb(10), 2); + let all = m.thumbnails(6, 24); + assert_eq!(all.len(), 6); + assert!(all.iter().all(|(_, r)| r.as_ref().is_ok_and(|f| f.height == 24))); +} + +#[test] +fn prores_and_mjpeg_decode() { + let mut m = Movie::open(prores_mov(5).unwrap()).unwrap(); + assert_eq!(m.info().codec, "prores"); + assert_eq!(m.info().codec_detail, "Apple ProRes 422 HQ"); + assert!(m.info().quicktime); + assert!((m.info().frame_rate - 25.0).abs() < 1e-9); + for i in [4usize, 0, 2] { + let f = m.frame(i).unwrap(); + assert_eq!((f.width, f.height), (64, 48)); + let y = 16 + 8 * i as u8; + let grey = crate::convert::Matrix::new(false, false).rgb(y, 128, 128); + assert_solid(&f, grey, 3); + } + let mut j = Movie::open(mjpeg_mov(4).unwrap()).unwrap(); + assert_eq!(j.info().codec, "mjpeg"); + for i in 0..4 { + let f = j.frame_at(i as f64 / 10.0).unwrap(); + assert_eq!((f.width, f.height), (32, 24)); + assert_solid(&f, [(30 * i) as u8; 3], 4); + } +} + +#[test] +fn unsupported_codec_opens_but_does_not_decode() { + let mut mw = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let t = mw.add_track(TrackConfig::new(SampleEntry::hevc(Default::default(), 64, 48), 24)).unwrap(); + mw.write_sample(t, WriteSample { data: &[0, 0, 0, 1, 0x40], duration: 1, composition_offset: 0, is_sync: true }).unwrap(); + let bytes = mw.finish().unwrap().into_inner(); + let mut m = Movie::open(bytes).unwrap(); + assert_eq!(m.info().codec, "hevc"); + assert!(!m.info().decodable); + assert!(matches!(m.frame(0), Err(VideoError::Unsupported(_)))); + assert!(matches!(m.thumbnail_at(0.0, 20), Err(VideoError::Unsupported(_)))); +} + +#[test] +fn garbage_and_audio_only_are_errors() { + assert!(matches!(Movie::open(Vec::new()), Err(VideoError::Container(_)))); + assert!(Movie::open(vec![0u8; 64]).is_err()); + assert!(Movie::open(b"RIFF\0\0\0\0WAVEfmt ".to_vec()).is_err()); +} + +/// Deterministic xorshift. +fn rng(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed +} + +#[test] +fn mutated_and_truncated_movies_never_panic() { + let sources = [h264_mp4(12, 4).unwrap(), prores_mov(3).unwrap(), mjpeg_mov(3).unwrap()]; + let mut seed = 0x5eed_u64; + for src in &sources { + for round in 0..150 { + let mut b = src.clone(); + if round % 3 == 0 { + let cut = (rng(&mut seed) as usize) % b.len().max(1); + b.truncate(cut); + } else { + for _ in 0..1 + rng(&mut seed) % 8 { + let at = (rng(&mut seed) as usize) % b.len().max(1); + if let Some(x) = b.get_mut(at) { + *x = rng(&mut seed) as u8; + } + } + } + if let Ok(mut m) = Movie::open(b) { + let d = m.info().duration; + for k in 0..4 { + let _ = m.frame_at(d * f64::from(k) / 4.0); + } + let _ = m.thumbnails(3, 16); + } + } + } +} + +/// Real encoder output (B-frames, CABAC, 8x8 transform), when ffmpeg + libx264 are available. +#[test] +fn ffmpeg_h264_high_with_bframes() { + let Some(dir) = ffmpeg_dir() else { return }; + let path = dir.join("high_bframes.mp4"); + let ok = std::process::Command::new("ffmpeg") + .args(["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=160x96:rate=24:duration=2", "-c:v", "libx264", "-profile:v", "high"]) + .args(["-pix_fmt", "yuv420p", "-bf", "3", "-g", "12", "-movflags", "+faststart"]) + .arg(&path) + .status() + .is_ok_and(|s| s.success()); + if !ok { + eprintln!("skipping: ffmpeg could not encode H.264 (libx264 missing?)"); + return; + } + let bytes = std::fs::read(&path).unwrap(); + let mut m = Movie::open(bytes.clone()).unwrap(); + assert_eq!(m.info().codec_detail, "H.264 High"); + assert_eq!((m.info().width, m.info().height), (160, 96)); + assert_eq!(m.info().frame_count, 48); + // Sequential frames must equal frames decoded after random seeks (decoder state independence). + let seq: Vec> = (0..48).map(|i| m.frame(i).unwrap().rgba.clone()).collect(); + let mut r = Movie::open(bytes).unwrap(); + r.set_cache_frames(1); + for &i in &[47usize, 13, 0, 25, 24, 11, 36, 2] { + assert!(r.frame(i).unwrap().rgba == seq[i], "frame {i} differs after a seek"); + } + // testsrc2 frames differ from each other. + assert!(seq[0] != seq[1]); + let _ = std::fs::remove_dir_all(dir); +} + +fn ffmpeg_dir() -> Option { + let found = std::process::Command::new("ffmpeg").arg("-version").output().is_ok_and(|o| o.status.success()); + if !found { + eprintln!("skipping: ffmpeg not on PATH"); + return None; + } + let dir = std::env::temp_dir().join(format!("soundcraft-video-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).ok()?; + Some(dir) +} + +#[test] +fn test_pattern_moves() { + let mut m = Movie::open(crate::synth::test_pattern_mp4(1).unwrap()).unwrap(); + assert_eq!((m.info().width, m.info().height, m.info().frame_count), (320, 192, 24)); + let white_x = |f: &crate::Frame| (0..20).find(|&mx| f.rgba[((9 * 16 + 8) * 320 + mx * 16 + 8) * 4] > 250); + let f3 = m.frame(3).unwrap(); + assert_eq!(white_x(&f3), Some(3)); + let f17 = m.frame_at(17.5 / 24.0).unwrap(); + assert_eq!(white_x(&f17), Some(17)); +} diff --git a/crates/vst3-host/Cargo.toml b/crates/vst3-host/Cargo.toml index 406e472d..19d7ef29 100644 --- a/crates/vst3-host/Cargo.toml +++ b/crates/vst3-host/Cargo.toml @@ -17,6 +17,16 @@ log.workspace = true vst3 = "0.3" libloading = "0.8" +# The macOS editor host window (an NSWindow whose content view the plugin's IPlugView attaches to). +[target.'cfg(target_os = "macos")'.dependencies] +objc2 = "0.6" +objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSGeometry", "NSString"] } +objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSApplication", "NSResponder", "NSView", "NSWindow", "NSGraphics"] } + +[[test]] +name = "editor_main_thread" +harness = false + # An isolated unsafe helper crate (craftrules never-crash), like soundcraft-clap-host: this crate # opts out of the workspace `unsafe_code = "forbid"` for itself only. It denies unsafe everywhere # and allows it solely on the FFI module (`src/ffi.rs`). The rest mirrors `[workspace.lints]`. diff --git a/crates/vst3-host/src/ffi.rs b/crates/vst3-host/src/ffi.rs index d4e5c36a..b8ee3e30 100644 --- a/crates/vst3-host/src/ffi.rs +++ b/crates/vst3-host/src/ffi.rs @@ -26,7 +26,7 @@ use vst3::Steinberg::Vst::{ IHostApplicationTrait, IParamValueQueueTrait, IParameterChangesTrait, }; use vst3::Steinberg::{self as sb}; -use vst3::Steinberg::{FUnknown, IBStreamTrait, IPluginBaseTrait, IPluginFactory2Trait, IPluginFactoryTrait}; +use vst3::Steinberg::{FUnknown, IBStreamTrait, IPlugFrameTrait, IPlugViewTrait, IPluginBaseTrait, IPluginFactory2Trait, IPluginFactoryTrait}; use vst3::{Class, ComPtr, ComWrapper, Interface}; /// Most classes one factory may report (hostile-input cap). @@ -41,8 +41,13 @@ const MAX_BUS_CHANNELS: u32 = 64; pub(crate) const EVENT_CAP: usize = 1024; /// Most parameter queues per block (preallocated). pub(crate) const PARAM_QUEUE_CAP: usize = 1024; -/// Largest plugin state we buffer when syncing the controller to the component. +/// Largest plugin state we buffer (component or controller state). const MAX_STATE_BYTES: usize = 64 << 20; +/// Most editor parameter edits buffered between two UI frames. +const EDIT_CAP: usize = 256; +/// Magic of SoundCraft's state container: `SCV3` + u32 LE component length + component state + +/// u32 LE controller length + controller state. +const STATE_MAGIC: &[u8; 4] = b"SCV3"; // ---- strings ------------------------------------------------------------------------------- @@ -322,6 +327,7 @@ impl Bundle { // SAFETY: `ProcessContext` is plain data (numbers and small structs); all-zero is valid. context: unsafe { std::mem::zeroed() }, event_in: false, + editor: Arc::new(EditorLink::default()), component, host, _bundle: self.clone(), @@ -364,6 +370,7 @@ impl Bundle { // SAFETY: valid controller; the handler (host context) outlives it. unsafe { c.setComponentHandler(h.as_ptr()) }; } + inst.editor = Arc::new(EditorLink { inner: Mutex::new(EditorInner { controller: Some(c.clone()), ..EditorInner::default() }) }); } Ok(inst) } @@ -386,12 +393,33 @@ pub(crate) fn hex(cid: &[u8; 16]) -> String { // ---- host-side COM objects ----------------------------------------------------------------- /// The host context handed to `initialize` (`IHostApplication`) and to the controller -/// (`IComponentHandler`: plugin-side parameter edits are accepted and ignored, SoundCraft's -/// session is the source of truth). -#[derive(Default)] +/// (`IComponentHandler`: parameter edits made in the plugin's editor are queued for the UI, which +/// writes them into the session, from where they reach the processor). pub(crate) struct HostContext { pub latency_changed: AtomicBool, pub restart_requested: AtomicBool, + /// `performEdit` values (normalized), bounded by `EDIT_CAP`. + pub edits: Mutex>, +} + +impl Default for HostContext { + fn default() -> Self { + HostContext { + latency_changed: AtomicBool::new(false), + restart_requested: AtomicBool::new(false), + edits: Mutex::new(Vec::with_capacity(EDIT_CAP)), + } + } +} + +impl HostContext { + /// Takes the queued editor edits. + pub fn take_edits(&self) -> Vec<(u32, f64)> { + let mut e = self.edits.lock().unwrap_or_else(PoisonError::into_inner); + let out = e.clone(); + e.clear(); + out + } } impl Class for HostContext { @@ -424,7 +452,16 @@ impl IComponentHandlerTrait for HostContext { unsafe fn beginEdit(&self, _id: sv::ParamID) -> sb::tresult { sb::kResultOk } - unsafe fn performEdit(&self, _id: sv::ParamID, _value: sv::ParamValue) -> sb::tresult { + unsafe fn performEdit(&self, id: sv::ParamID, value: sv::ParamValue) -> sb::tresult { + if value.is_finite() { + let mut e = self.edits.lock().unwrap_or_else(PoisonError::into_inner); + let v = value.clamp(0.0, 1.0); + if let Some(x) = e.iter_mut().find(|(i, _)| *i == id) { + x.1 = v; + } else if e.len() < EDIT_CAP { + e.push((id, v)); + } + } sb::kResultOk } unsafe fn endEdit(&self, _id: sv::ParamID) -> sb::tresult { @@ -625,6 +662,14 @@ impl Class for MemoryStream { } impl MemoryStream { + fn new(data: Vec) -> ComWrapper { + ComWrapper::new(MemoryStream { inner: Mutex::new((data, 0)) }) + } + + fn take(&self) -> Vec { + std::mem::take(&mut self.lock().0) + } + fn lock(&self) -> std::sync::MutexGuard<'_, (Vec, usize)> { self.inner.lock().unwrap_or_else(PoisonError::into_inner) } @@ -844,6 +889,7 @@ pub(crate) struct Instance { out_events: ComWrapper, context: sv::ProcessContext, event_in: bool, + editor: Arc, // Field order matters for drop: `Drop::drop` shuts the plugin down first; then the component // and host objects are released, and the bundle (which keeps the code mapped) goes last. component: ComPtr, @@ -866,6 +912,68 @@ impl Instance { &self.host } + /// A counted reference to the host context (editor handles read its edit queue). + pub fn host_ref(&self) -> ComWrapper { + self.host.clone() + } + + /// The editor link shared with editor handles (`None` without an edit controller). + pub fn editor(&self) -> Option> { + self.editor.has_controller().then(|| Arc::clone(&self.editor)) + } + + /// Writes one state into a fresh memory stream. + fn read_state(f: impl FnOnce(*mut sb::IBStream) -> sb::tresult) -> Option> { + let stream = MemoryStream::new(Vec::new()); + let s = stream.as_com_ref::()?; + ok(f(s.as_ptr())).then(|| stream.take()) + } + + /// The component state plus the controller state, in SoundCraft's container (see + /// `STATE_MAGIC`). `None` when the component refuses `getState`. + pub fn save_state(&self) -> Option> { + // SAFETY: valid component; the stream lives in `read_state` for the whole call. + let comp = Instance::read_state(|s| unsafe { self.component.getState(s) })?; + let ctrl = match &self.controller { + // SAFETY: valid controller; as above. + Some(c) => Instance::read_state(|s| unsafe { c.getState(s) }).unwrap_or_default(), + None => Vec::new(), + }; + let mut out = Vec::with_capacity(12 + comp.len() + ctrl.len()); + out.extend_from_slice(STATE_MAGIC); + out.extend_from_slice(&u32::try_from(comp.len()).ok()?.to_le_bytes()); + out.extend_from_slice(&comp); + out.extend_from_slice(&u32::try_from(ctrl.len()).ok()?.to_le_bytes()); + out.extend_from_slice(&ctrl); + Some(out) + } + + /// Restores a [`Instance::save_state`] blob (or a bare component state): component + /// `setState`, then the controller's `setComponentState` and `setState`. `false` when the + /// blob is malformed or the component refuses it. + pub fn load_state(&self, data: &[u8]) -> bool { + let Some((comp, ctrl)) = split_state(data) else { return false }; + let stream = MemoryStream::new(comp.to_vec()); + let Some(s) = stream.as_com_ref::() else { return false }; + // SAFETY: valid component; the stream lives on this frame for the call. + if !ok(unsafe { self.component.setState(s.as_ptr()) }) { + return false; + } + if let Some(c) = &self.controller { + stream.lock().1 = 0; + // SAFETY: valid controller; the same stream, rewound. + unsafe { c.setComponentState(s.as_ptr()) }; + if !ctrl.is_empty() { + let cs = MemoryStream::new(ctrl.to_vec()); + if let Some(r) = cs.as_com_ref::() { + // SAFETY: valid controller; the stream lives on this frame for the call. + unsafe { c.setState(r.as_ptr()) }; + } + } + } + true + } + /// Copies the component's state into the controller (`getState` → `setComponentState`). fn sync_controller_state(&self) { let Some(c) = &self.controller else { return }; @@ -1148,6 +1256,8 @@ impl Instance { impl Drop for Instance { fn drop(&mut self) { + // Close any editor and cut its handles off before the controller goes away. + self.editor.kill(); self.shutdown(); if let Some((a, b)) = self.connections.take() { // SAFETY: the pair was connected in `instantiate`; disconnect before terminating. @@ -1173,10 +1283,409 @@ impl Drop for Instance { } } +/// Splits a state blob into (component, controller) parts. A blob without SoundCraft's container +/// header is taken as a bare component state. +fn split_state(data: &[u8]) -> Option<(&[u8], &[u8])> { + if data.len() > MAX_STATE_BYTES.saturating_mul(2) { + return None; + } + let Some(rest) = data.strip_prefix(STATE_MAGIC) else { return (!data.is_empty()).then_some((data, &[][..])) }; + let take = |b: &[u8]| -> Option<(usize, usize)> { + let n = u32::from_le_bytes(b.get(..4)?.try_into().ok()?) as usize; + (n <= MAX_STATE_BYTES && b.len() >= 4 + n).then_some((4, 4 + n)) + }; + let (a, b) = take(rest)?; + let comp = rest.get(a..b)?; + let rest = rest.get(b..)?; + let (c, d) = take(rest)?; + let ctrl = rest.get(c..d)?; + (d == rest.len()).then_some((comp, ctrl)) +} + +// ---- editor (IPlugView) -------------------------------------------------------------------- + +/// The `IPlugFrame` handed to a plugin view: `resizeView` resizes our host window. +pub(crate) struct PlugFrame { + /// The host window (an `NSWindow*` on macOS), not owned; null when there is none. + window: std::sync::atomic::AtomicPtr, +} + +impl Class for PlugFrame { + type Interfaces = (sb::IPlugFrame,); +} + +impl IPlugFrameTrait for PlugFrame { + unsafe fn resizeView(&self, view: *mut sb::IPlugView, new_size: *mut sb::ViewRect) -> sb::tresult { + // SAFETY: in-pointer from the plugin (or null, handled). + let Some(r) = (unsafe { new_size.as_mut() }) else { return sb::kInvalidArgument }; + let (w, h) = rect_size(r); + window::resize(self.window.load(Ordering::Relaxed), w, h); + // SAFETY: the plugin passes its own live view (or null, handled). + if let Some(v) = unsafe { vst3::ComRef::from_raw(view) } { + // SAFETY: valid view; `r` is the plugin's own writable rect. + unsafe { v.onSize(r) }; + } + sb::kResultOk + } +} + +/// Width and height of a view rect, clamped to a sane window size. +fn rect_size(r: &sb::ViewRect) -> (f64, f64) { + let w = r.right.saturating_sub(r.left).clamp(1, 16_384); + let h = r.bottom.saturating_sub(r.top).clamp(1, 16_384); + (f64::from(w), f64::from(h)) +} + +#[derive(Default)] +struct EditorInner { + /// `None` once the instance is gone. + controller: Option>, + view: Option>, + frame: Option>, + window: Option, +} + +/// The link between an instance and its editor handles. The instance may process on the audio +/// thread while the editor (the edit controller's `IPlugView`) is driven from the main thread, +/// which is VST3's own threading model (controller and views on the UI thread). All view calls +/// happen under this lock, and the instance closes the view and drops the controller reference +/// under the same lock before it terminates, so a handle never calls into a dead plugin. +#[derive(Default)] +pub(crate) struct EditorLink { + inner: Mutex, +} + +impl EditorLink { + fn lock(&self) -> std::sync::MutexGuard<'_, EditorInner> { + self.inner.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn has_controller(&self) -> bool { + self.lock().controller.is_some() + } + + fn create_view(g: &EditorInner) -> Result, String> { + let c = g.controller.as_ref().ok_or_else(|| "the plugin is gone".to_string())?; + // SAFETY: valid controller; the name is a static NUL-terminated string. + let raw = unsafe { c.createView(c"editor".as_ptr()) }; + // SAFETY: a non-null view comes with one reference owned by the caller. + unsafe { ComPtr::from_raw(raw) }.ok_or_else(|| "this plugin has no editor".to_string()) + } + + /// Creates the view, checks it supports this platform's window type and reads its size, + /// without attaching it anywhere. For diagnostics and tests. + pub fn probe(&self) -> Result<(u32, u32), String> { + let g = self.lock(); + let view = EditorLink::create_view(&g)?; + let platform = window::PLATFORM_TYPE.ok_or_else(|| "plugin editors are not supported on this platform".to_string())?; + // SAFETY: valid view; `platform` is a static NUL-terminated string. + if unsafe { view.isPlatformTypeSupported(platform) } != sb::kResultTrue { + return Err("the editor does not support this platform's windows".into()); + } + let mut r = sb::ViewRect { left: 0, top: 0, right: 0, bottom: 0 }; + // SAFETY: valid view; `r` is writable. + if !ok(unsafe { view.getSize(&mut r) }) { + return Err("the editor reports no size".into()); + } + let (w, h) = rect_size(&r); + Ok((w as u32, h as u32)) + } + + /// Opens (or raises) the editor in a host window. Main thread only. + pub fn open(&self, title: &str) -> Result<(), String> { + if !window::on_main_thread() { + return Err("plugin editors must be opened on the main thread".into()); + } + let platform = window::PLATFORM_TYPE.ok_or_else(|| "VST3 editors are not hosted on this platform yet".to_string())?; + let mut g = self.lock(); + if let Some(w) = &g.window { + if w.is_visible() { + w.raise(); + return Ok(()); + } + // The user closed the window: start over. + EditorLink::teardown(&mut g); + } + let view = EditorLink::create_view(&g)?; + // SAFETY: valid view; `platform` is a static NUL-terminated string. + if unsafe { view.isPlatformTypeSupported(platform) } != sb::kResultTrue { + return Err("the editor does not support this platform's windows".into()); + } + let mut r = sb::ViewRect { left: 0, top: 0, right: 400, bottom: 300 }; + // SAFETY: valid view; `r` is writable. On failure the default size stays. + unsafe { view.getSize(&mut r) }; + let (w, h) = rect_size(&r); + let win = window::HostWindow::new(title, w, h)?; + let frame = ComWrapper::new(PlugFrame { window: std::sync::atomic::AtomicPtr::new(win.raw()) }); + if let Some(f) = frame.as_com_ref::() { + // SAFETY: valid view; the frame outlives the view's attachment (kept in `g.frame` + // and detached in `teardown`). + unsafe { view.setFrame(f.as_ptr()) }; + } + // SAFETY: valid view; the parent is the live content view of `win`, of the type named. + if !ok(unsafe { view.attached(win.parent(), platform) }) { + // SAFETY: valid view; detach our frame again. + unsafe { view.setFrame(std::ptr::null_mut()) }; + win.close(); + return Err("the editor could not attach to its window".into()); + } + win.raise(); + g.view = Some(view); + g.frame = Some(frame); + g.window = Some(win); + Ok(()) + } + + fn teardown(g: &mut EditorInner) { + if let Some(view) = g.view.take() { + // SAFETY: valid, attached view; main thread (callers check). `removed` before the + // window goes, and the frame is detached before it is released. + unsafe { + view.removed(); + view.setFrame(std::ptr::null_mut()); + } + } + g.frame = None; + if let Some(w) = g.window.take() { + w.close(); + } + } + + /// Closes the editor. Main thread only (no-op elsewhere). + pub fn close(&self) { + if window::on_main_thread() { + EditorLink::teardown(&mut self.lock()); + } + } + + pub fn is_open(&self) -> bool { + let g = self.lock(); + match &g.window { + Some(w) if window::on_main_thread() => w.is_visible(), + Some(_) => true, + None => false, + } + } + + /// Main-thread housekeeping: tears the view down when the user closed its window. + pub fn idle(&self) { + if !window::on_main_thread() { + return; + } + let mut g = self.lock(); + if g.window.as_ref().is_some_and(|w| !w.is_visible()) { + EditorLink::teardown(&mut g); + } + } + + /// Normalized → plain through the controller (main thread). + pub fn to_plain(&self, id: u32, n: f64) -> Option { + let g = self.lock(); + let c = g.controller.as_ref()?; + // SAFETY: valid controller. + let v = unsafe { c.normalizedParamToPlain(id, n.clamp(0.0, 1.0)) }; + v.is_finite().then_some(v) + } + + /// Plain → normalized through the controller (main thread). + pub fn to_normalized(&self, id: u32, plain: f64) -> Option { + let g = self.lock(); + let c = g.controller.as_ref()?; + if !plain.is_finite() { + return None; + } + // SAFETY: valid controller. + let v = unsafe { c.plainParamToNormalized(id, plain) }; + v.is_finite().then(|| v.clamp(0.0, 1.0)) + } + + /// Shows a value change in the editor (`setParamNormalized`, main thread). + pub fn set_normalized(&self, id: u32, n: f64) { + if !window::on_main_thread() || !n.is_finite() { + return; + } + let g = self.lock(); + if let Some(c) = &g.controller { + // SAFETY: valid controller, main thread. + unsafe { c.setParamNormalized(id, n.clamp(0.0, 1.0)) }; + } + } + + /// Called by the instance before it terminates: closes the editor and cuts the handles off. + fn kill(&self) { + let mut g = self.lock(); + if g.view.is_some() || g.window.is_some() { + if window::on_main_thread() { + EditorLink::teardown(&mut g); + } else { + // Window calls are main-thread only; leak the view and window rather than crash. + log::warn!("vst3: editor still open while its plugin is destroyed off the main thread"); + if let Some(v) = g.view.take() { + std::mem::forget(v); + } + // `HostWindow` has no destructor: dropping it leaks its retain on purpose. + g.window = None; + if let Some(f) = g.frame.take() { + std::mem::forget(f); + } + } + } + g.controller = None; + } +} + +/// The platform host window for plugin views. +#[cfg(target_os = "macos")] +mod window { + use objc2::rc::Retained; + use objc2::{MainThreadMarker, MainThreadOnly}; + use objc2_app_kit::{NSApplication, NSBackingStoreType, NSFloatingWindowLevel, NSWindow, NSWindowStyleMask}; + use objc2_foundation::{NSPoint, NSRect, NSSize, NSString}; + use std::ffi::{c_char, c_void}; + + pub const PLATFORM_TYPE: Option<*const c_char> = Some(vst3::Steinberg::kPlatformTypeNSView); + + pub fn on_main_thread() -> bool { + MainThreadMarker::new().is_some() + } + + /// An `NSWindow` (titled, closable, floating above the app's windows) whose content view is + /// the plugin view's parent. Holds one retain of the window; main thread only. + pub struct HostWindow { + window: *mut NSWindow, + } + + // SAFETY: the pointer is only dereferenced on the main thread (every method checks + // `MainThreadMarker`); the struct merely carries the retain between threads' locks. + unsafe impl Send for HostWindow {} + + impl HostWindow { + fn get(&self) -> Option<&NSWindow> { + MainThreadMarker::new()?; + // SAFETY: we hold a retain of this window (from `new`) until `close`. + unsafe { self.window.as_ref() } + } + + pub fn new(title: &str, w: f64, h: f64) -> Result { + let mtm = MainThreadMarker::new().ok_or_else(|| "not on the main thread".to_string())?; + let _app = NSApplication::sharedApplication(mtm); + let rect = NSRect::new(NSPoint::new(200.0, 200.0), NSSize::new(w, h)); + let style = NSWindowStyleMask::Titled | NSWindowStyleMask::Closable | NSWindowStyleMask::Miniaturizable; + // SAFETY: a freshly allocated window on the main thread; valid rect and style. + let window = unsafe { + NSWindow::initWithContentRect_styleMask_backing_defer(NSWindow::alloc(mtm), rect, style, NSBackingStoreType::Buffered, false) + }; + // SAFETY: we keep our own retain and release it in `close`, so AppKit must not + // release the window when the user closes it. + unsafe { window.setReleasedWhenClosed(false) }; + window.setTitle(&NSString::from_str(title)); + window.setLevel(NSFloatingWindowLevel); + window.setHidesOnDeactivate(true); + window.center(); + Ok(HostWindow { window: Retained::into_raw(window) }) + } + + /// The raw window pointer (for the plug frame's resizes). + pub fn raw(&self) -> *mut c_void { + self.window.cast() + } + + /// The content view (`NSView*`) the plugin view attaches to; null when unavailable. + pub fn parent(&self) -> *mut c_void { + self.get().and_then(|w| w.contentView()).map_or(std::ptr::null_mut(), |v| Retained::as_ptr(&v).cast_mut().cast()) + } + + pub fn raise(&self) { + if let Some(w) = self.get() { + w.makeKeyAndOrderFront(None); + } + } + + pub fn is_visible(&self) -> bool { + self.get().is_some_and(|w| w.isVisible()) + } + + pub fn close(self) { + if let Some(w) = self.get() { + w.orderOut(None); + w.close(); + } + if MainThreadMarker::new().is_some() { + // SAFETY: balances the retain taken in `new`; nothing uses the pointer after this. + drop(unsafe { Retained::from_raw(self.window) }); + } + } + } + + /// Resizes a host window's content (`window` from [`HostWindow::raw`]); main thread only. + pub fn resize(window: *mut c_void, w: f64, h: f64) { + if MainThreadMarker::new().is_none() { + return; + } + // SAFETY: null or a window the editor link keeps retained while the frame is attached. + if let Some(win) = unsafe { window.cast::().as_ref() } { + win.setContentSize(NSSize::new(w, h)); + } + } +} + +/// Other platforms: no editor windows yet (opening reports "unsupported"). +#[cfg(not(target_os = "macos"))] +mod window { + use std::ffi::{c_char, c_void}; + + pub const PLATFORM_TYPE: Option<*const c_char> = None; + + pub fn on_main_thread() -> bool { + true + } + + pub struct HostWindow; + + impl HostWindow { + pub fn new(_title: &str, _w: f64, _h: f64) -> Result { + Err("VST3 editors are not hosted on this platform yet".into()) + } + pub fn raw(&self) -> *mut c_void { + std::ptr::null_mut() + } + pub fn parent(&self) -> *mut c_void { + std::ptr::null_mut() + } + pub fn raise(&self) {} + pub fn is_visible(&self) -> bool { + false + } + pub fn close(self) {} + } + + pub fn resize(_window: *mut c_void, _w: f64, _h: f64) {} +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn state_container_parses_strictly() { + let mut blob = b"SCV3".to_vec(); + blob.extend_from_slice(&3u32.to_le_bytes()); + blob.extend_from_slice(b"abc"); + blob.extend_from_slice(&2u32.to_le_bytes()); + blob.extend_from_slice(b"xy"); + assert_eq!(split_state(&blob), Some((&b"abc"[..], &b"xy"[..]))); + assert_eq!(split_state(b"raw component"), Some((&b"raw component"[..], &[][..])), "bare component state"); + assert_eq!(split_state(b""), None); + let mut trailing = blob.clone(); + trailing.push(0); + assert_eq!(split_state(&trailing), None); + assert_eq!(split_state(&blob[..blob.len() - 1]), None); + let mut huge = b"SCV3".to_vec(); + huge.extend_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(split_state(&huge), None); + assert_eq!(split_state(b"SCV3"), None); + } + #[test] fn strings_are_bounded() { let mut b = [0u16; 4]; diff --git a/crates/vst3-host/src/lib.rs b/crates/vst3-host/src/lib.rs index 97afb0db..8c2972b0 100644 --- a/crates/vst3-host/src/lib.rs +++ b/crates/vst3-host/src/lib.rs @@ -20,9 +20,22 @@ //! //! On `wasm32` the crate compiles to stubs: scans are empty and nothing can be created. //! -//! Not hosted yet: plugin editors (`IPlugView`), `IMessage`/`IAttributeList` creation through the -//! host (plugins that need it for processor ↔ controller messaging may lose those messages), -//! plugin state save/restore, sidechain (aux buses are kept inactive) and 64-bit processing. +//! Plugin state: [`Plugin::save_state`] stores the component state and the controller state in +//! one blob (`SCV3` + length-prefixed parts); [`Plugin::load_state`] restores both (a bare +//! component state is accepted too) and re-reads the parameter values. +//! +//! Plugin editors (`IPlugView`): on macOS the view is attached to the content `NSView` of a host +//! `NSWindow` (titled and closable, floating above the app, resized on `IPlugFrame::resizeView`). +//! [`Plugin::open_editor`] opens it directly; [`Plugin::editor`] returns a handle usable on the +//! main thread while the instance processes on the audio thread (VST3's own threading model: +//! the controller and its views live on the UI thread). Parameter edits made in the editor +//! (`IComponentHandler::performEdit`) are reported by the handle's `idle` so the host can write +//! them into the session. Windows (HWND) and Linux (X11) editor windows are not hosted yet: +//! opening reports "unsupported". +//! +//! Not hosted yet: `IMessage`/`IAttributeList` creation through the host (plugins that need it +//! for processor ↔ controller messaging may lose those messages), sidechain (aux buses are kept +//! inactive) and 64-bit processing. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -185,6 +198,13 @@ pub fn instantiate(id: &str) -> Result, Vst3Error> { api::instantiate(id) } +/// Like [`instantiate`], but returns the concrete [`Vst3Plugin`] (for its extra methods, such +/// as [`Vst3Plugin::editor_size`]). +#[cfg(not(target_arch = "wasm32"))] +pub fn instantiate_plugin(id: &str) -> Result { + registry::instantiate_plugin(id) +} + /// Creates a plugin by `vst3:`; `None` if unknown or it fails to load. pub fn create(id: &str) -> Option> { parse_id(id)?; diff --git a/crates/vst3-host/src/plugin.rs b/crates/vst3-host/src/plugin.rs index 661420b6..254671f5 100644 --- a/crates/vst3-host/src/plugin.rs +++ b/crates/vst3-host/src/plugin.rs @@ -1,8 +1,10 @@ //! `Vst3Plugin`: a loaded VST3 instance behind the safe `soundcraft_dsp::Plugin` trait. -use crate::ffi::{BusBuffers, EVENT_CAP, Instance, NoteEvent, PARAM_QUEUE_CAP}; -use soundcraft_dsp::{Plugin, PluginInfo, Unit}; +use crate::ffi::{BusBuffers, EVENT_CAP, EditorLink, HostContext, Instance, NoteEvent, PARAM_QUEUE_CAP}; +use soundcraft_dsp::{Plugin, PluginEditor, PluginInfo, Unit}; +use std::sync::Arc; use std::sync::atomic::Ordering; +use vst3::ComWrapper; /// Largest block we set a plugin up for. const MAX_BLOCK: usize = 1 << 16; @@ -125,6 +127,26 @@ impl Vst3Plugin { n.is_finite().then(|| n.clamp(0.0, 1.0)) } + /// The editor's size in pixels without opening it (creates and releases a view). For + /// diagnostics and tests; real plugins expect this on the main thread. + pub fn editor_size(&self) -> Result<(u32, u32), String> { + self.inst.editor().ok_or_else(|| "this plugin has no edit controller".to_string())?.probe() + } + + /// Re-reads every parameter value from the controller (after a state load) and drops + /// changes still queued from before it. + fn refresh_values(&mut self) { + for i in 0..self.ids.len() { + let (Some(pi), Some(&id)) = (self.info.params.get(i), self.ids.get(i)) else { continue }; + if let Some(v) = self.inst.normalized_value(id).and_then(|n| self.plain_from_normalized(i, n)) + && let Some(slot) = self.values.get_mut(i) + { + *slot = pi.clamp(v); + } + } + self.pending.clear(); + } + /// Inserts keeping the queue time-sorted; drops the event when the queue is full (never /// allocates on the audio thread). fn push_note(&mut self, e: NoteEvent) { @@ -295,4 +317,102 @@ impl Plugin for Vst3Plugin { } } } + + fn save_state(&mut self) -> Option> { + self.inst.save_state() + } + + fn load_state(&mut self, data: &[u8]) -> bool { + let ok = self.inst.load_state(data); + if ok { + self.refresh_values(); + } + ok + } + + fn open_editor(&mut self) -> Result<(), String> { + self.inst.editor().ok_or_else(|| "this plugin has no editor".to_string())?.open(self.info.name) + } + + fn close_editor(&mut self) { + if let Some(e) = self.inst.editor() { + e.close(); + } + } + + fn editor(&mut self) -> Option> { + let link = self.inst.editor()?; + Some(Box::new(Vst3Editor { link, host: self.inst.host_ref(), info: self.info, ids: self.ids.clone(), maps: self.maps.clone() })) + } +} + +/// A VST3 plugin's editor (`IPlugView` in a host window), driven from the main thread. +struct Vst3Editor { + link: Arc, + host: ComWrapper, + info: &'static PluginInfo, + ids: Vec, + maps: Vec, +} + +impl Vst3Editor { + fn plain(&self, i: usize, n: f64) -> Option { + let pi = self.info.params.get(i)?; + let id = *self.ids.get(i)?; + let v = match self.maps.get(i)? { + Mapping::Stepped(steps) => (n * steps).round(), + Mapping::Plain => self.link.to_plain(id, n)?, + Mapping::Linear => f64::from(pi.min) + n * f64::from(pi.max - pi.min), + }; + v.is_finite().then(|| pi.clamp(v as f32)) + } + + fn normalized(&self, i: usize, v: f32) -> Option { + let pi = self.info.params.get(i)?; + let id = *self.ids.get(i)?; + let span = f64::from(pi.max - pi.min); + let linear = if span > 0.0 { (f64::from(v) - f64::from(pi.min)) / span } else { 0.0 }; + let n = match self.maps.get(i)? { + Mapping::Stepped(steps) if *steps > 0.0 => f64::from(v) / steps, + Mapping::Stepped(_) => 0.0, + Mapping::Plain => self.link.to_normalized(id, f64::from(v)).unwrap_or(linear), + Mapping::Linear => linear, + }; + n.is_finite().then(|| n.clamp(0.0, 1.0)) + } +} + +impl PluginEditor for Vst3Editor { + fn open(&mut self) -> Result<(), String> { + // Edits reported before the editor opened (e.g. during setup) are stale. + let _ = self.host.take_edits(); + self.link.open(self.info.name) + } + + fn close(&mut self) { + self.link.close(); + } + + fn is_open(&self) -> bool { + self.link.is_open() + } + + fn idle(&mut self) -> Vec<(String, f32)> { + self.link.idle(); + let mut out = Vec::new(); + for (vid, n) in self.host.take_edits() { + let Some(i) = self.ids.iter().position(|&x| x == vid) else { continue }; + if let (Some(p), Some(v)) = (self.info.params.get(i), self.plain(i, n)) { + out.push((p.id.to_string(), v)); + } + } + out + } + + fn set_param(&mut self, id: &str, value: f32) { + let Some(i) = self.info.params.iter().position(|p| p.id == id) else { return }; + if let (Some(&vid), Some(n)) = (self.ids.get(i), self.normalized(i, value)) { + self.link.set_normalized(vid, n); + } + } } diff --git a/crates/vst3-host/src/registry.rs b/crates/vst3-host/src/registry.rs index c63db7bc..453fbe3b 100644 --- a/crates/vst3-host/src/registry.rs +++ b/crates/vst3-host/src/registry.rs @@ -225,10 +225,14 @@ fn instantiate_raw(id: &str) -> Result<(Instance, &'static PluginInfo), Vst3Erro } pub fn instantiate(id: &str) -> Result, Vst3Error> { + Ok(Box::new(instantiate_plugin(id)?)) +} + +pub fn instantiate_plugin(id: &str) -> Result { let (inst, info) = instantiate_raw(id)?; let mut p = Vst3Plugin::new(inst, info); p.prepare(48_000.0, 1024, 2); - Ok(Box::new(p)) + Ok(p) } pub fn plugin_info(id: &str) -> Option<&'static PluginInfo> { diff --git a/crates/vst3-host/tests/common/mod.rs b/crates/vst3-host/tests/common/mod.rs new file mode 100644 index 00000000..e49b5a0e --- /dev/null +++ b/crates/vst3-host/tests/common/mod.rs @@ -0,0 +1,41 @@ +//! Shared by the integration tests: builds the fixture plugin (a cdylib outside the workspace) +//! and installs it as a `.vst3` bundle in a temp folder. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; + +/// Builds the fixture once and returns a folder containing `Fixture.vst3`. +pub fn fixture_dir() -> &'static Path { + static DIR: OnceLock = OnceLock::new(); + DIR.get_or_init(|| { + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture/Cargo.toml"); + let target = Path::new(env!("CARGO_TARGET_TMPDIR")).join("vst3-fixture-target"); + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); + let st = Command::new(cargo) + .args(["build", "--quiet", "--manifest-path"]) + .arg(&manifest) + .arg("--target-dir") + .arg(&target) + .env_remove("RUSTFLAGS") + .status() + .expect("run cargo"); + assert!(st.success(), "building the VST3 fixture failed"); + let (lib, folder, exe) = if cfg!(target_os = "macos") { + ("libvst3_fixture.dylib", "MacOS", "Fixture") + } else if cfg!(windows) { + ("vst3_fixture.dll", "x86_64-win", "Fixture.vst3") + } else { + ("libvst3_fixture.so", if cfg!(target_arch = "aarch64") { "aarch64-linux" } else { "x86_64-linux" }, "Fixture.so") + }; + let built = target.join("debug").join(lib); + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("vst3-fixture-install-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + // Install as a real bundle to exercise bundle resolution and the module entry. + let bin = dir.join("vendor/Fixture.vst3/Contents").join(folder); + std::fs::create_dir_all(&bin).unwrap(); + std::fs::copy(&built, bin.join(exe)).unwrap(); + assert_eq!(soundcraft_vst3_host::add_search_dir(&dir), 2); + dir + }) +} diff --git a/crates/vst3-host/tests/editor_main_thread.rs b/crates/vst3-host/tests/editor_main_thread.rs new file mode 100644 index 00000000..ab5d28c3 --- /dev/null +++ b/crates/vst3-host/tests/editor_main_thread.rs @@ -0,0 +1,25 @@ +//! VST3 editors on the process main thread (`harness = false`: the standard harness never runs +//! tests there). Opening a real window would flash on screen, so this only checks what the +//! main thread changes without one: the view probe and the parameter round trip through the +//! controller. The window itself is exercised manually (see `installed_plugins_smoke`). + +mod common; + +fn main() { + let dir = common::fixture_dir(); + let gain = soundcraft_vst3_host::scan_paths(&[dir.to_path_buf()]).into_iter().find(|d| d.name == "Fixture Gain").expect("fixture").id; + let mut p = soundcraft_vst3_host::instantiate_plugin(&gain).expect("instantiate"); + assert_eq!(p.editor_size(), Ok((320, 240))); + let mut ed = soundcraft_dsp::Plugin::editor(&mut p).expect("editor handle"); + assert!(!ed.is_open()); + assert_eq!(ed.idle().len(), 1, "the fixture's setup edit"); + // Session → editor: the controller follows (seen through the component handler round trip + // below and through the editor's own value). + ed.set_param("7", 1.5); + ed.set_param("nope", 1.0); + assert!(ed.idle().is_empty(), "set_param is not reported back as an edit"); + drop(p); + assert!(!ed.is_open()); + assert!(ed.open().is_err(), "the plugin is gone"); + println!("editor_main_thread: ok"); +} diff --git a/crates/vst3-host/tests/fixture/src/lib.rs b/crates/vst3-host/tests/fixture/src/lib.rs index 69c584d3..dd7c382b 100644 --- a/crates/vst3-host/tests/fixture/src/lib.rs +++ b/crates/vst3-host/tests/fixture/src/lib.rs @@ -4,7 +4,9 @@ //! (connected through IConnectionPoint, synced through the component state). Params: 7 "Gain" //! (plain 0..2, linear, units dB), 3 "Mode" (3 steps: A/B/C), 12 "Freq" (20..20000 Hz, log), //! 9 hidden, 11 bypass. Starts at gain 0.5 (its state) although the declared default is 1.0. -//! Reports 3 samples of latency. +//! Reports 3 samples of latency. State: the component stores the gain (one LE f64), the +//! controller the Freq value (one LE f64). Its controller has an editor view (`IPlugView`, +//! 320×240, no real drawing) supporting NSView/HWND/X11. //! - "Fixture Synth" (`Instrument|Synth`): a single-component instrument (the component is also //! its edit controller). No audio input, one stereo output, one event input; every output //! sample is the velocity of the held note (sample-accurate), 0 when none. @@ -117,7 +119,18 @@ impl IComponentTrait for GainProcessor { unsafe fn setActive(&self, _state: TBool) -> tresult { kResultOk } - unsafe fn setState(&self, _state: *mut IBStream) -> tresult { + unsafe fn setState(&self, state: *mut IBStream) -> tresult { + let Some(s) = (unsafe { ComRef::from_raw(state) }) else { return kInvalidArgument }; + let mut bytes = [0u8; 8]; + let mut n = 0; + if unsafe { s.read(bytes.as_mut_ptr().cast(), 8, &mut n) } != kResultOk || n != 8 { + return kResultFalse; + } + let g = f64::from_le_bytes(bytes); + if !(0.0..=1.0).contains(&g) { + return kResultFalse; + } + self.gain.store(g.to_bits(), Ordering::Relaxed); kResultOk } unsafe fn getState(&self, state: *mut IBStream) -> tresult { @@ -195,6 +208,62 @@ impl IAudioProcessorTrait for GainProcessor { struct GainController { gain: AtomicU64, + freq: AtomicU64, +} + +struct View { + attached: std::sync::atomic::AtomicBool, +} + +impl Class for View { + type Interfaces = (IPlugView,); +} + +impl IPlugViewTrait for View { + unsafe fn isPlatformTypeSupported(&self, t: FIDString) -> tresult { + let t = unsafe { std::ffi::CStr::from_ptr(t) }.to_bytes(); + if t == b"NSView" || t == b"HWND" || t == b"X11EmbedWindowID" { kResultTrue } else { kResultFalse } + } + unsafe fn attached(&self, parent: *mut c_void, _t: FIDString) -> tresult { + if parent.is_null() { + return kInvalidArgument; + } + self.attached.store(true, Ordering::Relaxed); + kResultOk + } + unsafe fn removed(&self) -> tresult { + assert!(self.attached.swap(false, Ordering::Relaxed), "removed without attached"); + kResultOk + } + unsafe fn onWheel(&self, _d: f32) -> tresult { + kResultFalse + } + unsafe fn onKeyDown(&self, _k: char16, _c: i16, _m: i16) -> tresult { + kResultFalse + } + unsafe fn onKeyUp(&self, _k: char16, _c: i16, _m: i16) -> tresult { + kResultFalse + } + unsafe fn getSize(&self, size: *mut ViewRect) -> tresult { + let r = unsafe { &mut *size }; + *r = ViewRect { left: 0, top: 0, right: 320, bottom: 240 }; + kResultOk + } + unsafe fn onSize(&self, _s: *mut ViewRect) -> tresult { + kResultOk + } + unsafe fn onFocus(&self, _s: TBool) -> tresult { + kResultOk + } + unsafe fn setFrame(&self, _f: *mut IPlugFrame) -> tresult { + kResultOk + } + unsafe fn canResize(&self) -> tresult { + kResultFalse + } + unsafe fn checkSizeConstraint(&self, _r: *mut ViewRect) -> tresult { + kResultOk + } } impl Class for GainController { @@ -240,11 +309,20 @@ impl IEditControllerTrait for GainController { } kResultOk } - unsafe fn setState(&self, _state: *mut IBStream) -> tresult { + unsafe fn setState(&self, state: *mut IBStream) -> tresult { + let Some(s) = (unsafe { ComRef::from_raw(state) }) else { return kInvalidArgument }; + let mut bytes = [0u8; 8]; + let mut n = 0; + if unsafe { s.read(bytes.as_mut_ptr().cast(), 8, &mut n) } == kResultOk && n == 8 { + self.freq.store(f64::from_le_bytes(bytes).clamp(0.0, 1.0).to_bits(), Ordering::Relaxed); + } kResultOk } - unsafe fn getState(&self, _state: *mut IBStream) -> tresult { - kResultOk + unsafe fn getState(&self, state: *mut IBStream) -> tresult { + let Some(s) = (unsafe { ComRef::from_raw(state) }) else { return kInvalidArgument }; + let mut bytes = f64::from_bits(self.freq.load(Ordering::Relaxed)).to_le_bytes(); + let mut n = 0; + unsafe { s.write(bytes.as_mut_ptr().cast(), 8, &mut n) } } unsafe fn getParameterCount(&self) -> i32 { GAIN_PARAMS.len() as i32 @@ -293,13 +371,15 @@ impl IEditControllerTrait for GainController { unsafe fn getParamNormalized(&self, id: u32) -> f64 { match id { 7 => f64::from_bits(self.gain.load(Ordering::Relaxed)), - 12 => 0.5, + 12 => f64::from_bits(self.freq.load(Ordering::Relaxed)), _ => 0.0, } } unsafe fn setParamNormalized(&self, id: u32, v: f64) -> tresult { if id == 7 { self.gain.store(v.to_bits(), Ordering::Relaxed); + } else if id == 12 { + self.freq.store(v.to_bits(), Ordering::Relaxed); } kResultOk } @@ -312,8 +392,14 @@ impl IEditControllerTrait for GainController { } kResultOk } - unsafe fn createView(&self, _name: FIDString) -> *mut IPlugView { - std::ptr::null_mut() + unsafe fn createView(&self, name: FIDString) -> *mut IPlugView { + if name.is_null() || unsafe { std::ffi::CStr::from_ptr(name) }.to_bytes() != b"editor" { + return std::ptr::null_mut(); + } + match ComWrapper::new(View { attached: std::sync::atomic::AtomicBool::new(false) }).to_com_ptr::() { + Some(p) => p.into_raw(), + None => std::ptr::null_mut(), + } } } @@ -534,7 +620,7 @@ impl IPluginFactoryTrait for Factory { let unknown = if cid == GAIN_CID { ComWrapper::new(GainProcessor { gain: AtomicU64::new(0.25f64.to_bits()), arrangement: AtomicU64::new(SpeakerArr::kStereo) }).to_com_ptr::() } else if cid == GAIN_CTRL_CID { - ComWrapper::new(GainController { gain: AtomicU64::new(0.5f64.to_bits()) }).to_com_ptr::() + ComWrapper::new(GainController { gain: AtomicU64::new(0.5f64.to_bits()), freq: AtomicU64::new(0.5f64.to_bits()) }).to_com_ptr::() } else if cid == SYNTH_CID { ComWrapper::new(Synth { level: AtomicU32::new(0) }).to_com_ptr::() } else { diff --git a/crates/vst3-host/tests/fixture_e2e.rs b/crates/vst3-host/tests/fixture_e2e.rs index 59bd21c5..23970d2c 100644 --- a/crates/vst3-host/tests/fixture_e2e.rs +++ b/crates/vst3-host/tests/fixture_e2e.rs @@ -3,44 +3,9 @@ //! notes. use soundcraft_dsp::{Category, Taper, Unit}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::OnceLock; -/// Builds the fixture once and returns a folder containing `Fixture.vst3`. -fn fixture_dir() -> &'static Path { - static DIR: OnceLock = OnceLock::new(); - DIR.get_or_init(|| { - let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture/Cargo.toml"); - let target = Path::new(env!("CARGO_TARGET_TMPDIR")).join("vst3-fixture-target"); - let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); - let st = Command::new(cargo) - .args(["build", "--quiet", "--manifest-path"]) - .arg(&manifest) - .arg("--target-dir") - .arg(&target) - .env_remove("RUSTFLAGS") - .status() - .expect("run cargo"); - assert!(st.success(), "building the VST3 fixture failed"); - let (lib, folder, exe) = if cfg!(target_os = "macos") { - ("libvst3_fixture.dylib", "MacOS", "Fixture") - } else if cfg!(windows) { - ("vst3_fixture.dll", "x86_64-win", "Fixture.vst3") - } else { - ("libvst3_fixture.so", if cfg!(target_arch = "aarch64") { "aarch64-linux" } else { "x86_64-linux" }, "Fixture.so") - }; - let built = target.join("debug").join(lib); - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("vst3-fixture-install-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - // Install as a real bundle to exercise bundle resolution and the module entry. - let bin = dir.join("vendor/Fixture.vst3/Contents").join(folder); - std::fs::create_dir_all(&bin).unwrap(); - std::fs::copy(&built, bin.join(exe)).unwrap(); - assert_eq!(soundcraft_vst3_host::add_search_dir(&dir), 2); - dir - }) -} +mod common; +use common::fixture_dir; fn id_of(name: &str) -> String { let found = soundcraft_vst3_host::scan_paths(&[fixture_dir().to_path_buf()]); @@ -206,3 +171,59 @@ fn installed_plugins_smoke() { } } } + +#[test] +fn state_saves_component_and_controller_and_restores_them() { + let id = id_of("Fixture Gain"); + let mut p = soundcraft_vst3_host::create(&id).unwrap(); + p.prepare(48_000.0, 64, 2); + assert!(p.set_param("7", 1.5)); + let mut io = vec![vec![0.5f32; 64]; 2]; + p.process(&mut io, 64); + let blob = p.save_state().expect("component state"); + assert!(blob.starts_with(b"SCV3")); + let mut q = soundcraft_vst3_host::create(&id).unwrap(); + q.prepare(48_000.0, 64, 2); + assert_eq!(q.param("7"), Some(0.5)); + assert!(q.load_state(&blob)); + assert_eq!(q.param("7"), Some(1.5), "controller synced from the restored component state"); + let mut io = vec![vec![0.5f32; 64]; 2]; + q.process(&mut io, 64); + assert!(io[0].iter().all(|&x| (x - 0.75).abs() < 1e-6), "{:?}", &io[0][..4]); + // The controller part (Freq) round-trips too: write one by hand. + let mut custom = b"SCV3".to_vec(); + custom.extend_from_slice(&8u32.to_le_bytes()); + custom.extend_from_slice(&0.25f64.to_le_bytes()); + custom.extend_from_slice(&8u32.to_le_bytes()); + custom.extend_from_slice(&1.0f64.to_le_bytes()); + assert!(q.load_state(&custom)); + assert_eq!(q.param("7"), Some(0.5)); + assert!((q.param("12").unwrap() - 20_000.0).abs() < 1.0, "{:?}", q.param("12")); + // A bare component state is accepted; hostile blobs are refused. + assert!(q.load_state(&0.5f64.to_le_bytes())); + assert_eq!(q.param("7"), Some(1.0)); + for bad in [&b""[..], b"SCV3", b"SCV3\xff\xff\xff\xff", &[0xffu8; 8][..], b"abc"] { + assert!(!q.load_state(bad), "{bad:?}"); + } + assert_eq!(q.param("7"), Some(1.0)); +} + +#[test] +fn editor_view_is_probed_and_edits_come_back_as_plain_values() { + let id = id_of("Fixture Gain"); + let mut p = soundcraft_vst3_host::instantiate_plugin(&id).unwrap(); + assert_eq!(p.editor_size(), Ok((320, 240))); + let mut ed = soundcraft_dsp::Plugin::editor(&mut p).expect("editor handle"); + // The fixture "edits" Gain to 0.9 (normalized) when it gets the component handler. + assert_eq!(ed.idle(), vec![("7".to_string(), 1.8)]); + assert!(ed.idle().is_empty()); + if cfg!(target_os = "macos") { + assert!(ed.open().unwrap_err().contains("main thread"), "test threads are not the main thread"); + } else { + assert!(ed.open().is_err(), "unsupported elsewhere"); + } + assert!(!ed.is_open()); + // The synth's controller has no view. + let synth = soundcraft_vst3_host::instantiate_plugin(&id_of("Fixture Synth")).unwrap(); + assert!(synth.editor_size().is_err()); +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 54267335..9ccb8fe8 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -21,9 +21,11 @@ const LAYERS: &[(&str, u8)] = &[ ("time", 0), ("audio-io", 0), ("midi", 0), + ("video", 0), ("dsp", 1), ("clap-host", 1), ("vst3-host", 1), + ("au-host", 1), ("model", 2), ("mix", 3), ("engine", 4), @@ -32,7 +34,7 @@ const LAYERS: &[(&str, u8)] = &[ ("ui-egui", 6), ]; /// Standalone format crates: no workspace dependencies at all (publishable on their own). -const STANDALONE: &[&str] = &["time", "audio-io", "midi", "dsp"]; +const STANDALONE: &[&str] = &["time", "audio-io", "midi", "video", "dsp"]; const UI_CRATES: &[&str] = &["egui", "eframe", "winit", "egui_kittest", "rfd", "egui-wgpu", "egui_glow"]; const EXEMPT: &[&str] = &["soundcraft", "soundcraft-cli", "soundcraft-web", "xtask"]; @@ -119,6 +121,7 @@ const WASM: &[&str] = &[ "soundcraft-time", "soundcraft-audio-io", "soundcraft-midi", + "soundcraft-video", "soundcraft-dsp", "soundcraft-model", "soundcraft-mix",