From 280f7d509c29c2c8328719450ce06eea9ea349af Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 05:01:43 -0400 Subject: [PATCH] M13.4: HEVC / AV1 output modules (MP4 hvc1/av01, WebM AV1), codec + Opus options in engine, UI, CLI Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + apps/effectcraft-cli/src/main.rs | 34 ++- crates/engine/src/commands/render_queue.rs | 79 ++++++- crates/engine/src/lib.rs | 2 + crates/engine/src/render_queue.rs | 2 +- crates/engine/src/tests_codec_options.rs | 55 +++++ crates/export/Cargo.toml | 4 +- crates/export/src/encode.rs | 17 +- crates/export/src/hevc_av1.rs | 203 +++++++++++++++++ crates/export/src/lib.rs | 6 +- crates/export/src/webm.rs | 50 ++-- crates/export/src/webm_av1.rs | 172 ++++++++++++++ crates/project/src/render_queue.rs | 251 ++++++++++++++++++++- crates/ui-egui/src/panels/render_queue.rs | 72 +++++- docs/architecture.md | 6 +- 15 files changed, 906 insertions(+), 49 deletions(-) create mode 100644 crates/engine/src/tests_codec_options.rs create mode 100644 crates/export/src/hevc_av1.rs create mode 100644 crates/export/src/webm_av1.rs diff --git a/Cargo.lock b/Cargo.lock index 8f8a55a2..63cd9b3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1242,7 +1242,9 @@ dependencies = [ name = "effectcraft-export" version = "0.1.0" dependencies = [ + "effectcraft-av1enc", "effectcraft-color", + "effectcraft-hevcenc", "effectcraft-media", "effectcraft-opusenc", "effectcraft-project", diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 8aa1381b..09a9f233 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -9,9 +9,11 @@ //! effectcraft-cli get [--time S] read a property //! effectcraft-cli set [--time S] [--expression E] //! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png] -//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] +//! effectcraft-cli render [--comp C] --out FILE [--format h264|hevc|av1|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] +//! [--profile main|main10] [--level auto|4.1] [--rate-control bitrate|quality] [--video-quality 1-100] +//! [--keyint FRAMES] [--webm-codec vp9|av1] [--audio-bitrate KBPS] [--opus-app audio|voice] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue //! effectcraft-cli bench [--comp C] [--time S] [--scale K] [--n N] [--play N] [--gpu] render timings //! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half) @@ -44,7 +46,11 @@ const USAGE: &str = "usage: effectcraft-cli Result<(), Failure> { if let Some(v) = args.num("--bitrate")? { p["bitrate"] = json!(v); } + for (flag, key) in [ + ("--profile", "profile"), + ("--level", "level"), + ("--rate-control", "rateControl"), + ("--webm-codec", "webmCodec"), + ("--opus-app", "opusApplication"), + ] { + if let Some(v) = args.opt(flag) { + p[key] = json!(v); + } + } + for (flag, key) in [("--video-quality", "quality"), ("--keyint", "keyframeInterval"), ("--audio-bitrate", "audioBitrate")] { + if let Some(v) = args.num(flag)? { + p[key] = json!(v); + } + } // Render exactly this item: unqueue whatever the project's queue already holds. for k in 0..s.project.render_queue.len() { let _ = s.execute("renderQueue.setRender", json!({"index": k + 1, "render": false})); diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index bb2b8ce5..c6f59a36 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -135,7 +135,7 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { let mut any = false; if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|hevc|av1|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); any = true; } @@ -145,11 +145,12 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { "rgba" | "rgbalpha" => Channels::Rgba, _ => return Err(bad(cmd, "channels: rgb|rgba")), }; - if om.channels == Channels::Rgba && !om.format.supports_alpha() { + if om.channels == Channels::Rgba && !om.supports_alpha() { return Err(bad(cmd, format!("{} has no alpha channel", om.format.label()))); } any = true; } + any |= apply_codec_options(om, p, cmd)?; if let Some(q) = f_p(p, "quality") { om.quality = q.clamp(1.0, 100.0) as u8; any = true; @@ -199,6 +200,66 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { Ok(any) } +/// HEVC / AV1 codec options, the WebM video codec and the Opus audio options. +fn apply_codec_options(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { + use effectcraft_project::render_queue::{CodecProfile, OpusApplication, RateControlMode, VideoCodecOptions, WebmVideoCodec}; + let mut any = false; + if let Some(c) = str_p(p, "webmCodec") { + om.webm_codec = match c.to_ascii_lowercase().as_str() { + "vp9" => WebmVideoCodec::Vp9, + "av1" => WebmVideoCodec::Av1, + _ => return Err(bad(cmd, "webmCodec: vp9|av1")), + }; + if !om.supports_alpha() { + om.channels = Channels::Rgb; + } + any = true; + } + if let Some(v) = str_p(p, "profile") { + om.codec.profile = CodecProfile::from_name(v).ok_or_else(|| bad(cmd, "profile: main|main10"))?; + any = true; + } + match p.get("level") { + Some(Value::Null) => { + om.codec.level = None; + any = true; + } + Some(v) => { + let s = match v { + Value::String(s) => s.clone(), + other => other.to_string(), + }; + om.codec.level = VideoCodecOptions::parse_level(&s).ok_or_else(|| bad(cmd, "level: auto|null or a level like 4.1"))?; + any = true; + } + None => {} + } + if let Some(v) = str_p(p, "rateControl") { + om.codec.rate_control = RateControlMode::from_name(v).ok_or_else(|| bad(cmd, "rateControl: bitrate|quality"))?; + any = true; + } + if let Some(q) = f_p(p, "quality") { + om.codec.quality = q.clamp(1.0, 100.0) as u8; + } + if let Some(k) = f_p(p, "keyframeInterval") { + om.codec.keyframe_interval = k.clamp(0.0, 100_000.0) as u32; + any = true; + } + if let Some(b) = f_p(p, "audioBitrate") { + om.opus_bitrate_kbps = b.clamp(6.0, 510.0) as u32; + any = true; + } + if let Some(a) = str_p(p, "opusApplication") { + om.opus_application = match a.to_ascii_lowercase().as_str() { + "audio" | "music" => OpusApplication::Audio, + "voip" | "voice" | "speech" => OpusApplication::Voip, + _ => return Err(bad(cmd, "opusApplication: audio|voice")), + }; + any = true; + } + Ok(any) +} + /// `module` (1-based; 1 = the first output module). fn module_p(s: &Session, i: usize, p: &Value, cmd: &str) -> Result { let n = p.get("module").and_then(Value::as_u64).unwrap_or(1) as usize; @@ -419,7 +480,17 @@ fn formats(s: &mut Session, _: &Value) -> Result { }) }) .collect(); - Ok(json!({"formats": v, "proresProfiles": ProResProfile::ALL.iter().map(|p| p.label()).collect::>()})) + use effectcraft_project::render_queue::{CodecProfile, VideoCodecOptions}; + let level = |l: &u8| format!("{}.{}", l / 10, l % 10); + Ok(json!({ + "formats": v, + "proresProfiles": ProResProfile::ALL.iter().map(|p| p.label()).collect::>(), + "codecProfiles": CodecProfile::ALL.iter().map(|p| p.label()).collect::>(), + "hevcLevels": VideoCodecOptions::HEVC_LEVELS.iter().map(level).collect::>(), + "av1Levels": VideoCodecOptions::AV1_LEVELS.iter().map(level).collect::>(), + "webmCodecs": ["vp9", "av1"], + "opusApplications": ["audio", "voice"], + })) } /// `item?|index?`, else the last item (the queue's newest entry). @@ -540,7 +611,7 @@ pub fn specs() -> Vec { "Output Module Settings...", [], None, - "{item?|index?, module?: n (1 = first), format?, channels?: rgb|rgba, quality?: 1-100, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool, output?}", + "{item?|index?, module?: n (1 = first), format?: h264|hevc|av1|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, channels?: rgb|rgba, quality?: 1-100, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, profile?: main|main10 (HEVC/AV1), level?: auto|4.1, rateControl?: bitrate|quality, keyframeInterval?: frames (0 = auto), webmCodec?: vp9|av1, audioBitrate?: Opus kbps, opusApplication?: audio|voice, audio?: auto|on|off, sampleRate?, loop?: bool, output?}", has_items, set_output_module ), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f7fda487..421ddc8c 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -766,6 +766,8 @@ mod tests_anim_tools; #[cfg(test)] mod tests_camera_track; #[cfg(test)] +mod tests_codec_options; +#[cfg(test)] mod tests_color_view; #[cfg(test)] mod tests_disk_cache; diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index c01d8513..e5609aa8 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -162,7 +162,7 @@ pub fn bytes_per_frame(om: &OutputModule, w: u32, h: u32, fps: f64) -> f64 { let px = (w as f64 * h as f64) / (1920.0 * 1080.0); let mbps = |m: f64| m * 1_000_000.0 / 8.0; match om.format { - OutputFormat::H264 => om.bitrate_kbps as f64 * 1000.0 / 8.0 / fps, + OutputFormat::H264 | OutputFormat::Hevc | OutputFormat::Av1 => om.bitrate_kbps as f64 * 1000.0 / 8.0 / fps, OutputFormat::ProRes => { let rate = match om.prores_profile { Proxy => 45.0, diff --git a/crates/engine/src/tests_codec_options.rs b/crates/engine/src/tests_codec_options.rs new file mode 100644 index 00000000..7a42bf13 --- /dev/null +++ b/crates/engine/src/tests_codec_options.rs @@ -0,0 +1,55 @@ +//! M13.4: HEVC / AV1 output modules and the WebM codec / Opus options over the command API. + +use effectcraft_project::render_queue::{Channels, CodecProfile, OpusApplication, OutputFormat, RateControlMode, VideoCodecOptions, WebmVideoCodec}; +use serde_json::json; + +use crate::Session; + +fn session() -> Session { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s +} + +#[test] +fn hevc_and_av1_output_module_options() { + let mut s = session(); + let a = s + .execute( + "renderQueue.add", + json!({"format": "hevc", "profile": "main10", "level": "4.1", "rateControl": "quality", "quality": 85, "keyframeInterval": 30}), + ) + .unwrap(); + assert!(a["outputPath"].as_str().unwrap().ends_with(".mp4"), "{a}"); + let om = &s.project.render_queue[0].output; + assert_eq!(om.format, OutputFormat::Hevc); + assert_eq!( + om.codec, + VideoCodecOptions { profile: CodecProfile::Main10, level: Some(41), rate_control: RateControlMode::Quality, quality: 85, keyframe_interval: 30 } + ); + assert!(a["outputModuleSummary"].as_str().unwrap_or_default().contains("Main 10"), "{a}"); + let id = a["item"].clone(); + s.execute("renderQueue.setOutputModule", json!({"item": id, "format": "av1", "level": null, "rateControl": "bitrate", "bitrate": 3000})).unwrap(); + let om = &s.project.render_queue[0].output; + assert_eq!((om.format, om.codec.level, om.codec.rate_control, om.bitrate_kbps), (OutputFormat::Av1, None, RateControlMode::Bitrate, 3000)); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": id, "channels": "rgba"})).is_err(), "AV1 MP4 has no alpha"); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": id, "profile": "high"})).is_err()); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": id, "level": "0.5"})).is_err()); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": id, "rateControl": "cbr2"})).is_err()); + + // WebM: VP9 with alpha, then AV1 (alpha dropped) with voice-tuned Opus. + s.execute("renderQueue.setOutputModule", json!({"item": id, "format": "webm", "channels": "rgba"})).unwrap(); + assert_eq!(s.project.render_queue[0].output.channels, Channels::Rgba); + s.execute("renderQueue.setOutputModule", json!({"item": id, "webmCodec": "av1", "audioBitrate": 24, "opusApplication": "voice"})).unwrap(); + let om = &s.project.render_queue[0].output; + assert_eq!((om.webm_codec, om.channels, om.opus_bitrate_kbps, om.opus_application), (WebmVideoCodec::Av1, Channels::Rgb, 24, OpusApplication::Voip)); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": id, "webmCodec": "theora"})).is_err()); + + // The formats query lists the new formats and option values. + let f = s.execute("renderQueue.formats", json!({})).unwrap(); + let ids: Vec<&str> = f["formats"].as_array().unwrap().iter().filter_map(|v| v["id"].as_str()).collect(); + assert!(ids.contains(&"Hevc") && ids.contains(&"Av1"), "{ids:?}"); + assert_eq!(f["codecProfiles"], json!(["Main", "Main 10"])); + assert!(f["hevcLevels"].as_array().unwrap().contains(&json!("4.1"))); + assert_eq!(f["webmCodecs"], json!(["vp9", "av1"])); +} diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 15f9b17d..e53bac35 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-export" -description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, WebM (VP9 + Opus), PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF" +description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft, HEVC / AV1 MP4 (+AAC) and WebM (VP9 or AV1 + Opus) via EffectCraft encoders, PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF" version.workspace = true edition.workspace = true license.workspace = true @@ -20,6 +20,8 @@ filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } effectcraft-opusenc = { workspace = true } effectcraft-vp9enc = { workspace = true } +effectcraft-hevcenc = { workspace = true } +effectcraft-av1enc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } diff --git a/crates/export/src/encode.rs b/crates/export/src/encode.rs index d6cf3150..860baacb 100644 --- a/crates/export/src/encode.rs +++ b/crates/export/src/encode.rs @@ -1,4 +1,5 @@ -//! Movie export: FilmCraft H.264 → MP4 (+ AAC) and ProRes → MOV (+ PCM), muxed by +//! Movie export: FilmCraft H.264, HEVC (`effectcraft-hevcenc`) and AV1 (`effectcraft-av1enc`) +//! → MP4 (+ AAC) and ProRes → MOV (+ PCM), muxed by //! FilmCraft's ISO BMFF / QuickTime writer. use std::io::Write; @@ -18,14 +19,14 @@ fn mux_err(e: impl std::fmt::Display) -> ExportError { ExportError::Io(e.to_string()) } -struct Packet { - data: Vec, - key: bool, +pub(crate) struct Packet { + pub(crate) data: Vec, + pub(crate) key: bool, /// pts − dts in the track timescale. - cto: i32, + pub(crate) cto: i32, } -trait VideoEncoder { +pub(crate) trait VideoEncoder { fn sample_entry(&self) -> SampleEntry; fn encode(&mut self, rgba: &[u8], index: u64) -> Result>; fn flush(&mut self) -> Result>; @@ -226,9 +227,11 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R let channels = if fmt == OutputFormat::ProRes { job.output.channels } else { Channels::Rgb }; let mut venc: Box = match fmt { OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps)?), + OutputFormat::Hevc => Box::new(crate::hevc_av1::Hevc::new(w, h, rate, job.output)?), + OutputFormat::Av1 => Box::new(crate::hevc_av1::Av1::new(w, h, rate, job.output)?), _ => Box::new(ProRes::new(w, h, job.output.prores_profile, channels)), }; - let brand = if fmt == OutputFormat::H264 { Brand::Mp4 } else { Brand::Mov }; + let brand = if fmt == OutputFormat::ProRes { Brand::Mov } else { Brand::Mp4 }; let batch = batch_size(); let render = |k: u64| -> Vec { rgba8(&render_frame(job, comp, k), comp, channels, w, h) }; diff --git a/crates/export/src/hevc_av1.rs b/crates/export/src/hevc_av1.rs new file mode 100644 index 00000000..ba447dba --- /dev/null +++ b/crates/export/src/hevc_av1.rs @@ -0,0 +1,203 @@ +//! HEVC (`effectcraft-hevcenc`) and AV1 (`effectcraft-av1enc`) video encoders for the MP4 +//! movie writer (`hvc1` / `av01` sample entries) and AV1 in WebM, configured from the Output +//! Module's [`VideoCodecOptions`]. + +use effectcraft_project::render_queue::{CodecProfile, OutputModule, RateControlMode, VideoCodecOptions}; +use effectcraft_time::FrameRate; +use filmcraft_isobmff::{Av1Config, CodecConfig, FourCc, HevcConfig, SampleEntry}; +use rayon::prelude::*; + +use crate::encode::{Packet, VideoEncoder}; +use crate::{ExportError, Result}; + +fn enc_err(e: impl std::fmt::Display) -> ExportError { + ExportError::Encode(e.to_string()) +} + +/// BT.709 limited-range 4:2:0 planes at `bits` (8 or 10) from RGBA8 (2×2 chroma average; +/// alpha ignored). Chroma planes are `ceil(w/2)` × `ceil(h/2)`. +pub(crate) fn rgba_to_yuv420_bits(rgba: &[u8], w: usize, h: usize, bits: u8, y: &mut Vec, u: &mut Vec, v: &mut Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let scale = (1u32 << (bits - 8)) as f32; + let (lo, hi) = (scale, 254.0 * scale + (scale - 1.0)); + y.resize(w * h, 0); + u.resize(cw * ch, 0); + v.resize(cw * ch, 0); + y.par_chunks_mut(w * 2).zip(u.par_chunks_mut(cw).zip(v.par_chunks_mut(cw))).enumerate().for_each(|(cy, (yr, (ur, vr)))| { + let rows = yr.len() / w; + let mut acc = vec![(0f32, 0f32, 0f32); cw]; + for dy in 0..rows { + let src = &rgba[(cy * 2 + dy) * w * 4..][..w * 4]; + for x in 0..w { + let (r, g, b) = (src[x * 4] as f32 / 255.0, src[x * 4 + 1] as f32 / 255.0, src[x * 4 + 2] as f32 / 255.0); + let l = 0.2126 * r + 0.7152 * g + 0.0722 * b; + yr[dy * w + x] = ((16.0 + 219.0 * l) * scale).round().clamp(lo, hi) as u16; + let a = &mut acc[x / 2]; + a.0 += (b - l) / 1.8556; + a.1 += (r - l) / 1.5748; + a.2 += 1.0; + } + } + for (cx, a) in acc.iter().enumerate() { + ur[cx] = ((128.0 + 224.0 * a.0 / a.2) * scale).round().clamp(lo, hi) as u16; + vr[cx] = ((128.0 + 224.0 * a.1 / a.2) * scale).round().clamp(lo, hi) as u16; + } + }); +} + +#[derive(Default)] +struct Planes { + y: Vec, + u: Vec, + v: Vec, +} + +// ---------------------------------------------------------------- HEVC + +pub(crate) struct Hevc { + enc: effectcraft_hevcenc::Encoder, + w: u32, + h: u32, + bits: u8, + planes: Planes, +} + +/// The HEVC encoder settings for an output module. +pub(crate) fn hevc_config(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> effectcraft_hevcenc::EncoderConfig { + use effectcraft_hevcenc as he; + let o: &VideoCodecOptions = &om.codec; + let mut cfg = he::EncoderConfig::new(w, h, rate.num as u32, rate.den as u32); + cfg.profile = match o.profile { + CodecProfile::Main => he::Profile::Main, + CodecProfile::Main10 => he::Profile::Main10, + }; + cfg.level_idc = o.hevc_level_idc(); + cfg.rate = match o.rate_control { + RateControlMode::Bitrate => he::RateControl::Bitrate { kbps: om.bitrate_kbps.max(50) }, + RateControlMode::Quality => he::RateControl::ConstantQp(o.hevc_qp()), + }; + cfg.keyint = o.keyint(rate.as_f64()); + cfg +} + +impl Hevc { + pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> Result { + let cfg = hevc_config(w, h, rate, om); + let bits = cfg.profile.bit_depth(); + let enc = effectcraft_hevcenc::Encoder::new(cfg).map_err(enc_err)?; + Ok(Hevc { enc, w, h, bits, planes: Planes::default() }) + } +} + +impl VideoEncoder for Hevc { + fn sample_entry(&self) -> SampleEntry { + let cfg = HevcConfig::parse(&self.enc.hvcc()).unwrap_or_default(); + SampleEntry::hevc(cfg, self.w as u16, self.h as u16) + } + fn encode(&mut self, rgba: &[u8], _index: u64) -> Result> { + let (w, h) = (self.w as usize, self.h as usize); + let p = &mut self.planes; + rgba_to_yuv420_bits(rgba, w, h, self.bits, &mut p.y, &mut p.u, &mut p.v); + let frame = effectcraft_hevcenc::Frame { y: &p.y, u: &p.u, v: &p.v, y_stride: w, uv_stride: w.div_ceil(2) }; + let pk = self.enc.encode(&frame); + Ok(vec![Packet { data: pk.data, key: pk.keyframe, cto: 0 }]) + } + fn flush(&mut self) -> Result> { + Ok(vec![]) + } +} + +// ---------------------------------------------------------------- AV1 + +pub(crate) struct Av1 { + enc: effectcraft_av1enc::Encoder, + w: u32, + h: u32, + bits: u8, + planes: Planes, +} + +/// The AV1 encoder settings for an output module. +pub(crate) fn av1_config(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> effectcraft_av1enc::EncoderConfig { + use effectcraft_av1enc as ae; + let o: &VideoCodecOptions = &om.codec; + let mut cfg = ae::EncoderConfig::new(w, h, rate.num as u32, rate.den as u32); + cfg.bit_depth = o.profile.bit_depth(); + cfg.level_idx = o.av1_level_idx(); + cfg.rate = match o.rate_control { + RateControlMode::Bitrate => ae::RateControl::Bitrate { kbps: om.bitrate_kbps.max(50) }, + RateControlMode::Quality => ae::RateControl::ConstantQ(o.av1_qindex()), + }; + cfg.keyint = o.keyint(rate.as_f64()); + cfg +} + +impl Av1 { + pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> Result { + let cfg = av1_config(w, h, rate, om); + let bits = cfg.bit_depth; + let enc = effectcraft_av1enc::Encoder::new(cfg).map_err(enc_err)?; + Ok(Av1 { enc, w, h, bits, planes: Planes::default() }) + } + /// The `av1C` record (also the WebM CodecPrivate). + pub(crate) fn av1c(&self) -> Vec { + self.enc.av1c() + } +} + +impl VideoEncoder for Av1 { + fn sample_entry(&self) -> SampleEntry { + let cfg = Av1Config::parse(&self.enc.av1c()).unwrap_or_default(); + SampleEntry::video(FourCc(*b"av01"), CodecConfig::Av1(cfg), self.w as u16, self.h as u16) + } + fn encode(&mut self, rgba: &[u8], _index: u64) -> Result> { + let (w, h) = (self.w as usize, self.h as usize); + let p = &mut self.planes; + rgba_to_yuv420_bits(rgba, w, h, self.bits, &mut p.y, &mut p.u, &mut p.v); + let frame = effectcraft_av1enc::Frame { y: &p.y, u: &p.u, v: &p.v, y_stride: w, uv_stride: w.div_ceil(2) }; + let pk = self.enc.encode(&frame); + Ok(vec![Packet { data: pk.data, key: pk.keyframe, cto: 0 }]) + } + fn flush(&mut self) -> Result> { + Ok(vec![]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ten_bit_planes_scale_eight_bit_ones() { + let rgba = [255u8, 255, 255, 255, 0, 0, 0, 255, 255, 0, 0, 255, 0, 255, 0, 255]; + let (mut y, mut u, mut v) = (vec![], vec![], vec![]); + rgba_to_yuv420_bits(&rgba, 2, 2, 8, &mut y, &mut u, &mut v); + assert_eq!((y[0], y[1]), (235, 16)); + rgba_to_yuv420_bits(&rgba, 2, 2, 10, &mut y, &mut u, &mut v); + assert_eq!((y[0], y[1]), (940, 64)); + assert_eq!(u.len(), 1); + } + + #[test] + fn codec_options_map_to_encoder_settings() { + let mut om = OutputModule::for_format(effectcraft_project::render_queue::OutputFormat::Hevc); + om.codec.profile = CodecProfile::Main10; + om.codec.level = Some(41); + om.codec.rate_control = RateControlMode::Quality; + om.codec.quality = 100; + om.codec.keyframe_interval = 12; + let rate = FrameRate::new(30, 1); + let h = hevc_config(64, 32, rate, &om); + assert_eq!(h.profile, effectcraft_hevcenc::Profile::Main10); + assert_eq!(h.level_idc, Some(123)); + assert_eq!(h.rate, effectcraft_hevcenc::RateControl::ConstantQp(4)); + assert_eq!(h.keyint, 12); + let a = av1_config(64, 32, rate, &om); + assert_eq!((a.bit_depth, a.level_idx, a.keyint), (10, Some(9), 12)); + om.codec.rate_control = RateControlMode::Bitrate; + om.codec.keyframe_interval = 0; + om.bitrate_kbps = 3000; + let a = av1_config(64, 32, rate, &om); + assert_eq!((a.rate, a.keyint), (effectcraft_av1enc::RateControl::Bitrate { kbps: 3000 }, 60)); + } +} diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 1f0c4407..1b194996 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -8,6 +8,8 @@ //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | //! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | HEVC | MP4 (`hvc1`) | `effectcraft-hevcenc` Main / Main 10 4:2:0, I + P slices, bitrate or constant quality, level, key-frame interval | no | AAC-LC (FilmCraft), stereo | +//! | AV1 | MP4 (`av01`), or WebM with the WebM format's AV1 codec | `effectcraft-av1enc` Main 8/10-bit 4:2:0, key + inter frames | no | AAC-LC in MP4, Opus in WebM | //! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to @@ -19,8 +21,10 @@ //! output is a single pixel wide/high). mod encode; +mod hevc_av1; mod out; mod webm; +mod webm_av1; use web_time::Instant; @@ -134,7 +138,7 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result< st.advance(0)?; let (w, h) = output_size(comp, job.settings, job.output); let mut report = match job.output.format { - OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, + OutputFormat::H264 | OutputFormat::Hevc | OutputFormat::Av1 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index eb99cd22..4aa170fd 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -19,25 +19,24 @@ use rayon::prelude::*; use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; -const OPUS_RATE: u32 = 48_000; -const OPUS_BITRATE: u32 = 192_000; +pub(crate) const OPUS_RATE: u32 = 48_000; // Element ids (with their length markers). -const EBML: u32 = 0x1A45_DFA3; -const SEGMENT: u32 = 0x1853_8067; -const INFO: u32 = 0x1549_A966; -const TRACKS: u32 = 0x1654_AE6B; -const TRACK_ENTRY: u32 = 0xAE; -const CLUSTER: u32 = 0x1F43_B675; +pub(crate) const EBML: u32 = 0x1A45_DFA3; +pub(crate) const SEGMENT: u32 = 0x1853_8067; +pub(crate) const INFO: u32 = 0x1549_A966; +pub(crate) const TRACKS: u32 = 0x1654_AE6B; +pub(crate) const TRACK_ENTRY: u32 = 0xAE; +pub(crate) const CLUSTER: u32 = 0x1F43_B675; -fn id_bytes(id: u32) -> Vec { +pub(crate) fn id_bytes(id: u32) -> Vec { let b = id.to_be_bytes(); let skip = b.iter().position(|x| *x != 0).unwrap_or(3); b[skip..].to_vec() } /// EBML variable-size integer for an element data size (shortest form). -fn vint(n: u64) -> Vec { +pub(crate) fn vint(n: u64) -> Vec { for len in 1..=8u32 { let max = (1u64 << (7 * len)) - 2; if n <= max { @@ -50,23 +49,23 @@ fn vint(n: u64) -> Vec { v } -fn el(out: &mut Vec, id: u32, data: &[u8]) { +pub(crate) fn el(out: &mut Vec, id: u32, data: &[u8]) { out.extend(id_bytes(id)); out.extend(vint(data.len() as u64)); out.extend_from_slice(data); } -fn el_uint(out: &mut Vec, id: u32, v: u64) { +pub(crate) fn el_uint(out: &mut Vec, id: u32, v: u64) { let b = v.to_be_bytes(); let skip = b.iter().position(|x| *x != 0).unwrap_or(7); el(out, id, &b[skip..]); } -fn el_float(out: &mut Vec, id: u32, v: f64) { +pub(crate) fn el_float(out: &mut Vec, id: u32, v: f64) { el(out, id, &v.to_be_bytes()); } -fn el_str(out: &mut Vec, id: u32, s: &str) { +pub(crate) fn el_str(out: &mut Vec, id: u32, s: &str) { el(out, id, s.as_bytes()); } @@ -79,14 +78,14 @@ fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { b } -struct Frame { - ms: i64, - track: u64, - data: Vec, - alpha: Option>, +pub(crate) struct Frame { + pub(crate) ms: i64, + pub(crate) track: u64, + pub(crate) data: Vec, + pub(crate) alpha: Option>, } -fn cluster(frames: &mut [Frame]) -> Vec { +pub(crate) fn cluster(frames: &mut [Frame]) -> Vec { frames.sort_by_key(|f| (f.ms, f.track)); let base = frames.first().map_or(0, |f| f.ms); let mut body = vec![]; @@ -113,13 +112,22 @@ fn cluster(frames: &mut [Frame]) -> Vec { out } +/// The Opus encoder of a WebM output (stereo, the module's bitrate and application). +pub(crate) fn opus_encoder(job: &Job) -> effectcraft_opusenc::OpusEncoder { + // TEMP(M13.4): application mapping lands with the SILK encoder. + effectcraft_opusenc::OpusEncoder::new(2, job.output.opus_bitrate_kbps.clamp(6, 510) * 1000) +} + pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + if job.output.webm_codec == effectcraft_project::render_queue::WebmVideoCodec::Av1 { + return crate::webm_av1::webm_av1(job, comp, w, h, st); + } let rate = job.settings.rate(comp); let alpha = job.output.channels == Channels::Rgba; let quality = job.output.quality.clamp(1, 100); let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; let with_audio = wants_audio(job); - let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let mut opus = with_audio.then(|| opus_encoder(job)); let total = st.total; let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; let duration_ms = frame_ms(total) as f64; diff --git a/crates/export/src/webm_av1.rs b/crates/export/src/webm_av1.rs new file mode 100644 index 00000000..5c965325 --- /dev/null +++ b/crates/export/src/webm_av1.rs @@ -0,0 +1,172 @@ +//! WebM with AV1 video (`V_AV1`, CodecPrivate = the `av1C` record, per the AV1-in-Matroska +//! codec mapping) and Opus audio. Frames are rendered in parallel batches and encoded in order +//! (AV1 inter frames reference the previous frame); each batch becomes one Cluster whose +//! SimpleBlocks carry the key-frame flag only on AV1 key frames (the Cues-less layout of the +//! VP9 writer: players seek by scanning clusters). + +use std::io::{Seek, SeekFrom, Write}; + +use effectcraft_project::Comp; +use effectcraft_project::render_queue::Channels; +use effectcraft_time::{TICKS_PER_SECOND, Tick}; +use rayon::prelude::*; + +use crate::encode::VideoEncoder; +use crate::webm::{CLUSTER, EBML, INFO, OPUS_RATE, SEGMENT, TRACK_ENTRY, TRACKS, el, el_float, el_str, el_uint, id_bytes, opus_encoder, vint}; +use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; + +struct Block { + ms: i64, + track: u64, + key: bool, + data: Vec, +} + +fn cluster(blocks: &mut [Block]) -> Vec { + blocks.sort_by_key(|b| (b.ms, b.track)); + let base = blocks.first().map_or(0, |b| b.ms); + let mut body = vec![]; + el_uint(&mut body, 0xE7, base.max(0) as u64); + for b in blocks.iter() { + let rel = (b.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; + let mut p = vint(b.track); + p.extend_from_slice(&rel.to_be_bytes()); + p.push(if b.key { 0x80 } else { 0x00 }); + p.extend_from_slice(&b.data); + el(&mut body, 0xA3, &p); + } + let mut out = vec![]; + el(&mut out, CLUSTER, &body); + out +} + +pub(crate) fn webm_av1(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let rate = job.settings.rate(comp); + let mut venc = crate::hevc_av1::Av1::new(w, h, rate, job.output)?; + let with_audio = wants_audio(job); + let mut opus = with_audio.then(|| opus_encoder(job)); + let total = st.total; + let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; + let duration_ms = frame_ms(total) as f64; + + let mut file = crate::out::create(job.sink, job.path)?; + let mut head = vec![]; + let mut eb = vec![]; + el_uint(&mut eb, 0x4286, 1); + el_uint(&mut eb, 0x42F7, 1); + el_uint(&mut eb, 0x42F2, 4); + el_uint(&mut eb, 0x42F3, 8); + el_str(&mut eb, 0x4282, "webm"); + el_uint(&mut eb, 0x4287, 4); + el_uint(&mut eb, 0x4285, 2); + el(&mut head, EBML, &eb); + head.extend(id_bytes(SEGMENT)); + let size_pos = head.len() as u64; + head.extend_from_slice(&[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + let seg_start = head.len() as u64; + let mut info = vec![]; + el_uint(&mut info, 0x2AD7B1, 1_000_000); + el_float(&mut info, 0x4489, duration_ms); + el_str(&mut info, 0x4D80, "EffectCraft"); + el_str(&mut info, 0x5741, "EffectCraft"); + el(&mut head, INFO, &info); + let mut tracks = vec![]; + let mut v = vec![]; + el_uint(&mut v, 0xD7, 1); + el_uint(&mut v, 0x73C5, 1); + el_uint(&mut v, 0x83, 1); + el_uint(&mut v, 0x9C, 0); + el_str(&mut v, 0x86, "V_AV1"); + el(&mut v, 0x63A2, &venc.av1c()); + el_uint(&mut v, 0x23E383, (1_000_000_000u128 * rate.den as u128 / rate.num.max(1) as u128) as u64); + let mut video = vec![]; + el_uint(&mut video, 0xB0, w as u64); + el_uint(&mut video, 0xBA, h as u64); + el(&mut v, 0xE0, &video); + el(&mut tracks, TRACK_ENTRY, &v); + if let Some(o) = &opus { + let mut a = vec![]; + el_uint(&mut a, 0xD7, 2); + el_uint(&mut a, 0x73C5, 2); + el_uint(&mut a, 0x83, 2); + el_uint(&mut a, 0x9C, 0); + el_str(&mut a, 0x86, "A_OPUS"); + el(&mut a, 0x63A2, &o.opus_head()); + el_uint(&mut a, 0x56AA, o.pre_skip() as u64 * 1_000_000_000 / OPUS_RATE as u64); + el_uint(&mut a, 0x56BB, 80_000_000); + let mut au = vec![]; + el_float(&mut au, 0xB5, OPUS_RATE as f64); + el_uint(&mut au, 0x9F, 2); + el(&mut a, 0xE1, &au); + el(&mut tracks, TRACK_ENTRY, &a); + } + el(&mut head, TRACKS, &tracks); + file.write_all(&head).map_err(io)?; + let mut seg_len = head.len() as u64 - seg_start; + + let (span_start, span_end) = job.settings.span(comp); + let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); + let mut pcm_left: Vec = vec![]; + let mut packets = 0u64; + let mut audio_until = |until: Tick, blocks: &mut Vec, finish: bool| { + let Some(enc) = opus.as_mut() else { return }; + let end = until.to_units_floor(OPUS_RATE as i64); + if end > cursor { + let n = (end - cursor) as usize; + let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); + pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + cursor = end; + } + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + if finish && !pcm_left.is_empty() { + let pad = (fs - pcm_left.len() % fs) % fs + fs; + pcm_left.extend(std::iter::repeat_n(0.0, pad)); + } + while pcm_left.len() >= fs { + let pkt = enc.encode_float(&pcm_left[..fs]); + pcm_left.drain(..fs); + blocks.push(Block { ms: packets as i64 * 20, track: 2, key: true, data: pkt }); + packets += 1; + } + }; + + let batch = batch_size(); + let mut i = 0; + let flush_blocks = |blocks: &mut Vec, file: &mut crate::out::Out, seg_len: &mut u64| -> Result<()> { + if !blocks.is_empty() { + let c = cluster(blocks); + file.write_all(&c).map_err(io)?; + *seg_len += c.len() as u64; + blocks.clear(); + } + Ok(()) + }; + let mut blocks = vec![]; + while i < total { + let end = (i + batch).min(total); + let frames: Vec> = (i..end).into_par_iter().map(|k| rgba8(&render_frame(job, comp, k), comp, Channels::Rgb, w, h)).collect(); + for (j, px) in frames.iter().enumerate() { + let k = i + j as u64; + for p in venc.encode(px, k)? { + blocks.push(Block { ms: frame_ms(k), track: 1, key: p.key, data: p.data }); + } + } + let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; + audio_until(t_end, &mut blocks, end >= total); + flush_blocks(&mut blocks, &mut file, &mut seg_len)?; + st.advance(end - i)?; + i = end; + } + if total == 0 { + audio_until(span_end, &mut blocks, true); + flush_blocks(&mut blocks, &mut file, &mut seg_len)?; + } + let mut size = [0u8; 8]; + size[0] = 0x01; + size[1..].copy_from_slice(&seg_len.to_be_bytes()[1..]); + file.seek(SeekFrom::Start(size_pos)).map_err(io)?; + file.write_all(&size).map_err(io)?; + file.seek(SeekFrom::End(0)).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) +} diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index d0f0c91f..5e9f2949 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -190,11 +190,18 @@ pub enum OutputFormat { Wav, /// Audio only: AIFF (16-bit PCM). Aiff, + /// HEVC (H.265) video (+ AAC audio) in MP4 (`hvc1`). + Hevc, + /// AV1 video (+ AAC audio) in MP4 (`av01`). WebM with AV1: [`OutputFormat::WebM`] with + /// [`WebmVideoCodec::Av1`]. + Av1, } impl OutputFormat { - pub const ALL: [OutputFormat; 10] = [ + pub const ALL: [OutputFormat; 12] = [ OutputFormat::H264, + OutputFormat::Hevc, + OutputFormat::Av1, OutputFormat::ProRes, OutputFormat::WebM, OutputFormat::PngSequence, @@ -218,6 +225,8 @@ impl OutputFormat { OutputFormat::WebM => "WebM (VP9 + Opus)", OutputFormat::Wav => "WAV", OutputFormat::Aiff => "AIFF", + OutputFormat::Hevc => "HEVC (H.265)", + OutputFormat::Av1 => "AV1", } } pub fn extension(self) -> &'static str { @@ -232,29 +241,34 @@ impl OutputFormat { OutputFormat::WebM => "webm", OutputFormat::Wav => "wav", OutputFormat::Aiff => "aif", + OutputFormat::Hevc | OutputFormat::Av1 => "mp4", } } pub fn is_sequence(self) -> bool { matches!(self, OutputFormat::PngSequence | OutputFormat::JpegSequence | OutputFormat::TiffSequence | OutputFormat::ExrSequence) } pub fn is_movie(self) -> bool { - matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM) + matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM | OutputFormat::Hevc | OutputFormat::Av1) + } + /// Formats whose video codec options live in [`OutputModule::codec`] (HEVC, AV1). + pub fn has_codec_options(self) -> bool { + matches!(self, OutputFormat::Hevc | OutputFormat::Av1) } /// Audio-only outputs (no video is rendered). pub fn is_audio_only(self) -> bool { matches!(self, OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_alpha(self) -> bool { - !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff) + !matches!(self, OutputFormat::H264 | OutputFormat::Hevc | OutputFormat::Av1 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_audio(self) -> bool { self.is_movie() || self.is_audio_only() } - /// The written frame size for a rendered size: H.264 needs even dimensions (rounded down, + /// The written frame size for a rendered size: H.264 and HEVC need even dimensions (rounded down, /// at least 2). pub fn coded_size(self, w: u32, h: u32) -> (u32, u32) { let even = |v: u32| if v < 2 { 2 } else { v & !1 }; - if self == OutputFormat::H264 { (even(w), even(h)) } else { (w, h) } + if matches!(self, OutputFormat::H264 | OutputFormat::Hevc) { (even(w), even(h)) } else { (w, h) } } /// Parse a format name: `h264`, `mp4`, `prores`, `mov`, `png`, `jpeg`/`jpg`, `tiff`/`tif`, /// `exr`, `gif`, or a label. @@ -262,6 +276,8 @@ impl OutputFormat { let n = s.trim().to_ascii_lowercase().replace([' ', '.', '-', '_'], ""); Some(match n.as_str() { "h264" | "mp4" | "avc" => OutputFormat::H264, + "hevc" | "h265" | "hevch265" | "hvc1" | "x265" => OutputFormat::Hevc, + "av1" | "av01" | "mp4av1" => OutputFormat::Av1, "prores" | "mov" | "quicktime" | "quicktimeprores" => OutputFormat::ProRes, "png" | "pngsequence" => OutputFormat::PngSequence, "jpg" | "jpeg" | "jpegsequence" => OutputFormat::JpegSequence, @@ -338,6 +354,161 @@ impl ProResProfile { } } +/// HEVC / AV1 profile: 8-bit (HEVC Main, AV1 Main 8-bit) or 10-bit (HEVC Main 10, AV1 Main 10-bit). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum CodecProfile { + #[default] + Main, + Main10, +} + +impl CodecProfile { + pub const ALL: [CodecProfile; 2] = [CodecProfile::Main, CodecProfile::Main10]; + pub fn label(self) -> &'static str { + match self { + CodecProfile::Main => "Main", + CodecProfile::Main10 => "Main 10", + } + } + pub fn bit_depth(self) -> u8 { + match self { + CodecProfile::Main => 8, + CodecProfile::Main10 => 10, + } + } + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "main" | "main8" | "8" | "8bit" => Some(CodecProfile::Main), + "main10" | "10" | "10bit" => Some(CodecProfile::Main10), + _ => None, + } + } +} + +/// HEVC / AV1 rate control: a target bitrate ([`OutputModule::bitrate_kbps`]) or constant +/// quality ([`VideoCodecOptions::quality`], CRF-like). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum RateControlMode { + #[default] + Bitrate, + Quality, +} + +impl RateControlMode { + pub fn label(self) -> &'static str { + match self { + RateControlMode::Bitrate => "Target Bitrate", + RateControlMode::Quality => "Constant Quality", + } + } + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "bitrate" | "vbr" | "abr" | "targetbitrate" => Some(RateControlMode::Bitrate), + "quality" | "crf" | "cq" | "constantquality" | "qp" => Some(RateControlMode::Quality), + _ => None, + } + } +} + +/// Video codec options of the HEVC and AV1 formats (and AV1 in WebM). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct VideoCodecOptions { + pub profile: CodecProfile, + /// Level × 10 (`41` = level 4.1); `None` picks the lowest level that fits the frame size and rate. + pub level: Option, + pub rate_control: RateControlMode, + /// Constant-quality setting 1–100 (100 ≈ visually lossless), used with [`RateControlMode::Quality`]. + pub quality: u8, + /// Frames between key frames; 0 = automatic (about 2 seconds). + pub keyframe_interval: u32, +} + +impl Default for VideoCodecOptions { + fn default() -> Self { + VideoCodecOptions { profile: CodecProfile::Main, level: None, rate_control: RateControlMode::Bitrate, quality: 70, keyframe_interval: 0 } + } +} + +impl VideoCodecOptions { + /// HEVC levels (× 10) offered in the UI (Main tier). + pub const HEVC_LEVELS: [u8; 13] = [10, 20, 21, 30, 31, 40, 41, 50, 51, 52, 60, 61, 62]; + /// AV1 levels (× 10) offered in the UI. + pub const AV1_LEVELS: [u8; 14] = [20, 21, 30, 31, 40, 41, 50, 51, 52, 53, 60, 61, 62, 63]; + /// The key-frame interval in frames at `fps` (automatic: 2 s). + pub fn keyint(&self, fps: f64) -> u32 { + if self.keyframe_interval > 0 { self.keyframe_interval } else { (fps * 2.0).round().max(1.0) as u32 } + } + /// HEVC QP (0–51) for the constant-quality setting. + pub fn hevc_qp(&self) -> u8 { + (4.0 + (100 - self.quality.clamp(1, 100)) as f64 * 0.45).round() as u8 + } + /// AV1 `base_q_idx` (1–255) for the constant-quality setting. + pub fn av1_qindex(&self) -> u8 { + (4.0 + (100 - self.quality.clamp(1, 100)) as f64 * 2.4).round() as u8 + } + /// `general_level_idc` (30 × level). + pub fn hevc_level_idc(&self) -> Option { + self.level.map(|l| l.saturating_mul(3)) + } + /// AV1 `seq_level_idx`: (major − 2) × 4 + minor. + pub fn av1_level_idx(&self) -> Option { + self.level.map(|l| ((l / 10).clamp(2, 7) - 2) * 4 + (l % 10).min(3)) + } + pub fn level_label(&self) -> String { + match self.level { + None => "Auto".into(), + Some(l) => format!("{}.{}", l / 10, l % 10), + } + } + /// Parse `"auto"`, `"4.1"`, `"41"` (level × 10). `None` when unparsable; `Some(None)` = auto. + pub fn parse_level(s: &str) -> Option> { + let s = s.trim().to_ascii_lowercase(); + if s == "auto" || s.is_empty() { + return Some(None); + } + let v: f64 = s.trim_start_matches('l').parse().ok()?; + let l = if v >= 10.0 { v.round() } else { (v * 10.0).round() }; + (10.0..=73.0).contains(&l).then_some(Some(l as u8)) + } +} + +/// The video codec of a WebM output. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum WebmVideoCodec { + #[default] + Vp9, + /// AV1 (`V_AV1`) with the [`OutputModule::codec`] options; no alpha. + Av1, +} + +impl WebmVideoCodec { + pub fn label(self) -> &'static str { + match self { + WebmVideoCodec::Vp9 => "VP9", + WebmVideoCodec::Av1 => "AV1", + } + } +} + +/// Opus coding application: general audio (music and mixed content) or voice (speech-tuned +/// SILK / hybrid coding at low bitrates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum OpusApplication { + #[default] + Audio, + Voip, +} + +impl OpusApplication { + pub fn label(self) -> &'static str { + match self { + OpusApplication::Audio => "Audio", + OpusApplication::Voip => "Voice", + } + } +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct OutputModule { @@ -347,13 +518,20 @@ pub struct OutputModule { pub output: String, /// JPEG quality 1–100. pub quality: u8, - /// H.264 target bitrate. + /// H.264 target bitrate (also HEVC / AV1 with [`RateControlMode::Bitrate`]). pub bitrate_kbps: u32, pub prores_profile: ProResProfile, pub audio: AudioOutput, pub audio_sample_rate: u32, /// GIF: loop forever. pub gif_loop: bool, + /// HEVC / AV1 options. + pub codec: VideoCodecOptions, + /// WebM video codec. + pub webm_codec: WebmVideoCodec, + /// WebM Opus audio bitrate (stereo total), kbit/s. + pub opus_bitrate_kbps: u32, + pub opus_application: OpusApplication, } impl Default for OutputModule { @@ -368,6 +546,10 @@ impl Default for OutputModule { audio: AudioOutput::Auto, audio_sample_rate: 48_000, gif_loop: true, + codec: VideoCodecOptions::default(), + webm_codec: WebmVideoCodec::Vp9, + opus_bitrate_kbps: 192, + opus_application: OpusApplication::Audio, } } } @@ -384,7 +566,7 @@ impl OutputModule { pub fn set_format(&mut self, format: OutputFormat) { let old = self.format; self.format = format; - if !format.supports_alpha() { + if !self.supports_alpha() { self.channels = Channels::Rgb; } if self.output.contains("[fileExtension]") { @@ -404,6 +586,20 @@ impl OutputModule { } self.output = format!("{stem}.{}", format.extension()); } + /// Whether this module can write alpha (the format can, and WebM uses VP9). + pub fn supports_alpha(&self) -> bool { + self.format.supports_alpha() && !(self.format == OutputFormat::WebM && self.webm_codec == WebmVideoCodec::Av1) + } + /// Codec options summary: `Main · 10000 kbps · Level Auto · Key every auto`. + fn codec_summary(&self) -> String { + let c = &self.codec; + let rate = match c.rate_control { + RateControlMode::Bitrate => format!("{} kbps", self.bitrate_kbps), + RateControlMode::Quality => format!("Quality {}", c.quality), + }; + let gop = if c.keyframe_interval == 0 { "auto".to_string() } else { c.keyframe_interval.to_string() }; + format!("{} · {rate} · Level {} · Key every {gop}", c.profile.label(), c.level_label()) + } /// One-line summary shown next to "Output Module:". pub fn summary(&self) -> String { let ch = match self.channels { @@ -416,6 +612,10 @@ impl OutputModule { "{} · {ch}", if self.channels == Channels::Rgba && !self.prores_profile.is_4444() { "Apple ProRes 4444" } else { self.prores_profile.label() } ), + OutputFormat::Hevc | OutputFormat::Av1 => format!("{} · {}", self.format.label(), self.codec_summary()), + OutputFormat::WebM if self.webm_codec == WebmVideoCodec::Av1 => { + format!("WebM (AV1 + Opus {} kbps) · {}", self.opus_bitrate_kbps, self.codec_summary()) + } f => format!("{} · {ch}", f.label()), } } @@ -632,4 +832,41 @@ mod tests { assert_eq!(s.first_frame(&c), 30); assert!(s.frame_time(&c, 0) >= Tick::from_seconds_f64(1.0)); } + + #[test] + fn hevc_av1_codec_options() { + assert_eq!(OutputFormat::from_name("H.265"), Some(OutputFormat::Hevc)); + assert_eq!(OutputFormat::from_name("hevc"), Some(OutputFormat::Hevc)); + assert_eq!(OutputFormat::from_name("AV1"), Some(OutputFormat::Av1)); + assert_eq!(OutputFormat::Hevc.coded_size(33, 17), (32, 16)); + assert_eq!(OutputFormat::Av1.coded_size(33, 17), (33, 17)); + assert!(!OutputFormat::Hevc.supports_alpha() && OutputFormat::Av1.is_movie() && OutputFormat::Av1.supports_audio()); + assert_eq!(VideoCodecOptions::parse_level("4.1"), Some(Some(41))); + assert_eq!(VideoCodecOptions::parse_level("51"), Some(Some(51))); + assert_eq!(VideoCodecOptions::parse_level("auto"), Some(None)); + assert_eq!(VideoCodecOptions::parse_level("9"), None); + let mut o = VideoCodecOptions { level: Some(41), ..Default::default() }; + assert_eq!((o.hevc_level_idc(), o.av1_level_idx()), (Some(123), Some(9))); + o.level = Some(20); + assert_eq!(o.av1_level_idx(), Some(0)); + o.quality = 100; + assert_eq!((o.hevc_qp(), o.av1_qindex()), (4, 4)); + o.quality = 1; + assert!(o.hevc_qp() <= 51 && o.av1_qindex() >= 240); + assert_eq!(o.keyint(29.97), 60); + o.keyframe_interval = 1; + assert_eq!(o.keyint(29.97), 1); + // WebM with AV1 has no alpha; switching drops RGB + Alpha. + let mut m = OutputModule::for_format(OutputFormat::WebM); + m.channels = Channels::Rgba; + assert!(m.supports_alpha()); + m.webm_codec = WebmVideoCodec::Av1; + assert!(!m.supports_alpha()); + assert!(m.summary().starts_with("WebM (AV1 + Opus 192 kbps)"), "{}", m.summary()); + let h = OutputModule::for_format(OutputFormat::Hevc); + assert_eq!(h.summary(), "HEVC (H.265) · Main · 10000 kbps · Level Auto · Key every auto"); + // Older projects (no codec fields) load with the defaults. + let old: OutputModule = serde_json::from_str(r#"{"format":"H264","bitrate_kbps":5000}"#).unwrap(); + assert_eq!((old.codec, old.webm_codec, old.opus_bitrate_kbps), (VideoCodecOptions::default(), WebmVideoCodec::Vp9, 192)); + } } diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index c02090bc..013b8ff8 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -7,7 +7,9 @@ //! registers an automation id: `renderQueue.render`, `renderQueue.stop`, //! `renderQueue.item..{row,twirl,render,renderSettings,outputModule,outputTo,outputPath}`. -use effectcraft_engine::project::render_queue::{AudioOutput, Channels, OutputFormat, ProResProfile, RenderQuality, RenderQueueItem, RenderStatus, TimeSpan}; +use effectcraft_engine::project::render_queue::{ + AudioOutput, Channels, OpusApplication, OutputFormat, ProResProfile, RenderQuality, RenderQueueItem, RenderStatus, TimeSpan, WebmVideoCodec, +}; use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use serde_json::{Value, json}; @@ -96,7 +98,7 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, } v.push(("-".into(), Value::Null)); v.push((mark(o.channels == Channels::Rgb, "Channels: RGB"), json!({"channels": "rgb"}))); - if o.format.supports_alpha() { + if o.supports_alpha() { v.push((mark(o.channels == Channels::Rgba, "Channels: RGB + Alpha"), json!({"channels": "rgba"}))); } match o.format { @@ -112,7 +114,29 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } - OutputFormat::JpegSequence | OutputFormat::WebM => { + OutputFormat::Hevc | OutputFormat::Av1 => codec_menu(o, &mut v), + OutputFormat::WebM => { + v.push(("-".into(), Value::Null)); + for c in [WebmVideoCodec::Vp9, WebmVideoCodec::Av1] { + v.push((mark(o.webm_codec == c, &format!("Video Codec: {}", c.label())), json!({"webmCodec": c.label().to_ascii_lowercase()}))); + } + if o.webm_codec == WebmVideoCodec::Av1 { + codec_menu(o, &mut v); + } else { + v.push(("-".into(), Value::Null)); + for q in [60u8, 80, 90, 100] { + v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); + } + } + v.push(("-".into(), Value::Null)); + for kbps in [24u32, 32, 64, 96, 128, 192, 256] { + v.push((mark(o.opus_bitrate_kbps == kbps, &format!("Opus Bitrate: {kbps} kbps")), json!({"audioBitrate": kbps}))); + } + for (a, k) in [(OpusApplication::Audio, "audio"), (OpusApplication::Voip, "voice")] { + v.push((mark(o.opus_application == a, &format!("Opus Tuning: {}", a.label())), json!({"opusApplication": k}))); + } + } + OutputFormat::JpegSequence => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); @@ -138,6 +162,48 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v } +/// HEVC / AV1 options: profile, level, rate control (bitrate or constant quality), key frames. +fn codec_menu(o: &effectcraft_engine::project::render_queue::OutputModule, v: &mut Vec<(String, Value)>) { + use effectcraft_engine::project::render_queue::{CodecProfile, RateControlMode, VideoCodecOptions}; + let mark = |on: bool, l: &str| if on { format!("✓ {l}") } else { format!(" {l}") }; + let c = &o.codec; + v.push(("-".into(), Value::Null)); + for p in CodecProfile::ALL { + v.push((mark(c.profile == p, &format!("Profile: {}", p.label())), json!({"profile": format!("{p:?}").to_ascii_lowercase()}))); + } + v.push(("-".into(), Value::Null)); + v.push((mark(c.level.is_none(), "Level: Auto"), json!({"level": "auto"}))); + // The common HEVC / AV1 levels (every level is reachable over the control channel). + for l in [30u8, 31, 40, 41, 50, 51, 52] { + debug_assert!(VideoCodecOptions::HEVC_LEVELS.contains(&l) && VideoCodecOptions::AV1_LEVELS.contains(&l)); + let name = format!("{}.{}", l / 10, l % 10); + v.push((mark(c.level == Some(l), &format!("Level: {name}")), json!({"level": name}))); + } + v.push(("-".into(), Value::Null)); + for r in [RateControlMode::Bitrate, RateControlMode::Quality] { + v.push((mark(c.rate_control == r, &format!("Rate Control: {}", r.label())), json!({"rateControl": format!("{r:?}").to_ascii_lowercase()}))); + } + if c.rate_control == RateControlMode::Bitrate { + for kbps in [1_000u32, 2_000, 5_000, 10_000, 20_000, 40_000] { + let l = if kbps < 2_000 { format!("Bitrate: {kbps} kbps") } else { format!("Bitrate: {} Mbps", kbps / 1000) }; + v.push((mark(o.bitrate_kbps == kbps, &l), json!({"bitrate": kbps}))); + } + } else { + for q in [40u8, 60, 70, 80, 90, 100] { + v.push((mark(c.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); + } + } + v.push(("-".into(), Value::Null)); + for k in [0u32, 1, 12, 30, 60, 120, 250] { + let l = match k { + 0 => "Key Frames: Auto (2 s)".to_string(), + 1 => "Key Frames: Every Frame (all intra)".to_string(), + k => format!("Key Frames: Every {k} Frames"), + }; + v.push((mark(c.keyframe_interval == k, &l), json!({"keyframeInterval": k}))); + } +} + const OUTPUT_TEMPLATES: [&str; 4] = [ "[compName].[fileExtension]", "[compName]_[width]x[height].[fileExtension]", diff --git a/docs/architecture.md b/docs/architecture.md index e5a6a836..a48eb5e0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,9 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | | L0 | `vp9enc` | VP9 intra-frame encoder (profile 0, 8-bit 4:2:0; lossless at quality 100) for WebM export, from the VP9 bitstream specification | -| L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | +| L0 | `opusenc` | Opus encoder: RFC 6716 SILK (NB/MB/WB), hybrid (SWB/FB) and CELT (FB) modes chosen by bitrate and application (audio / voice), 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | +| L0 | `hevcenc` | HEVC (H.265) encoder from ITU-T H.265: Main / Main 10 4:2:0, IDR + P slices, CABAC, bitrate or constant-QP rate control, for MP4 (`hvc1`) export | +| L0 | `av1enc` | AV1 encoder from the AV1 bitstream specification: Main profile 8/10-bit 4:2:0, key + inter frames, for MP4 (`av01`) and WebM export | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | | L1 | `psd` | Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit; RGB/CMYK/Gray/Lab) and a minimal writer, from Adobe's published format specification | @@ -31,7 +33,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | | L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, Photoshop and SVG stills, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | -| L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | +| L3 | `export` | Render queue encoding: H.264, HEVC and AV1 MP4, ProRes, WebM (VP9 + alpha or AV1, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus |