From b936ed0055e364ed652dc7def086124e90035455 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:41:29 -0400 Subject: [PATCH 1/6] OUT-2: Opus (CELT) encoder (effectcraft-opusenc) New L0 crate effectcraft-opusenc: clean-room RFC 6716 CELT-only encoder (fullband 48 kHz, 20 ms packets, mono/stereo as dual L/R stereo, constant packet size from the bitrate) with the RFC 7845 OpusHead and pre-skip. Forward low-overlap MDCT, inter/intra coarse energy, fine/final energy, the decoder's exact allocation, recursive band splitting, PVQ search and codeword indexing, silence frames. crates/export depends on it; filmcraft-opus (dev) is the round-trip oracle in tests/opus_roundtrip.rs, with ffmpeg reading an in-test Ogg Opus file as a second oracle when installed. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 + Cargo.toml | 6 + crates/export/Cargo.toml | 2 + crates/export/tests/opus_roundtrip.rs | 364 +++++++++++++++++ crates/opusenc/Cargo.toml | 12 + crates/opusenc/README.md | 33 ++ crates/opusenc/src/bands.rs | 497 +++++++++++++++++++++++ crates/opusenc/src/energy.rs | 230 +++++++++++ crates/opusenc/src/lib.rs | 311 ++++++++++++++ crates/opusenc/src/mdct.rs | 293 ++++++++++++++ crates/opusenc/src/range.rs | 472 ++++++++++++++++++++++ crates/opusenc/src/rate.rs | 561 ++++++++++++++++++++++++++ crates/opusenc/src/tables.rs | 122 ++++++ docs/architecture.md | 1 + xtask/src/main.rs | 1 + 15 files changed, 2911 insertions(+) create mode 100644 crates/export/tests/opus_roundtrip.rs create mode 100644 crates/opusenc/Cargo.toml create mode 100644 crates/opusenc/README.md create mode 100644 crates/opusenc/src/bands.rs create mode 100644 crates/opusenc/src/energy.rs create mode 100644 crates/opusenc/src/lib.rs create mode 100644 crates/opusenc/src/mdct.rs create mode 100644 crates/opusenc/src/range.rs create mode 100644 crates/opusenc/src/rate.rs create mode 100644 crates/opusenc/src/tables.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..1a1f0f60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1218,6 +1218,7 @@ version = "0.1.0" dependencies = [ "effectcraft-color", "effectcraft-media", + "effectcraft-opusenc", "effectcraft-project", "effectcraft-raster", "effectcraft-render", @@ -1225,6 +1226,7 @@ dependencies = [ "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", + "filmcraft-opus", "filmcraft-prores", "gif", "image", @@ -1338,6 +1340,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-opusenc" +version = "0.1.0" + [[package]] name = "effectcraft-path" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1941cf4..da4f7047 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } +effectcraft-opusenc = { path = "crates/opusenc" } # foundation thiserror = "2" @@ -66,6 +67,8 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "89c filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -129,6 +132,9 @@ opt-level = 3 [profile.dev.package.effectcraft-media] opt-level = 3 +[profile.dev.package.effectcraft-opusenc] +opt-level = 3 + [profile.dev.package.effectcraft-export] opt-level = 3 diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 34c83aab..5bdd0fee 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -18,6 +18,7 @@ filmcraft-h264enc = { workspace = true } filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } +effectcraft-opusenc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -28,3 +29,4 @@ log = { workspace = true } [dev-dependencies] effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } +filmcraft-opus = { workspace = true } diff --git a/crates/export/tests/opus_roundtrip.rs b/crates/export/tests/opus_roundtrip.rs new file mode 100644 index 00000000..5ea099f0 --- /dev/null +++ b/crates/export/tests/opus_roundtrip.rs @@ -0,0 +1,364 @@ +//! Round trip of `effectcraft-opusenc` through FilmCraft's spec-derived Opus decoder, plus an +//! ffmpeg oracle (skipped when ffmpeg is not installed) reading an Ogg Opus file written here. + +use std::process::Command; + +use effectcraft_opusenc::OpusEncoder; + +const SR: f32 = 48_000.0; +const FRAME: usize = OpusEncoder::FRAME_SIZE; + +/// Encodes interleaved `pcm`, padding the tail so the encoder's look-ahead is flushed. +fn encode(pcm: &[f32], channels: u8, bitrate: u32) -> (OpusEncoder, Vec>) { + let c = channels as usize; + let mut enc = OpusEncoder::new(channels, bitrate); + let total = pcm.len() / c + enc.pre_skip() as usize; + let frames = total.div_ceil(FRAME); + let mut padded = pcm.to_vec(); + padded.resize(frames * FRAME * c, 0.0); + let packets = padded.chunks(FRAME * c).map(|f| enc.encode_float(f)).collect(); + (enc, packets) +} + +/// Decodes with filmcraft-opus, drops the pre-skip and returns planar channels of `len` samples. +fn decode(head: &[u8], packets: &[Vec], len: usize) -> Vec> { + let mut dec = filmcraft_opus::Decoder::new(head).expect("OpusHead"); + dec.set_trim_pre_skip(true); + let mut out: Vec> = vec![Vec::new(); dec.channels()]; + for p in packets { + let planes = dec.decode(Some(p)).expect("decode"); + for (o, p) in out.iter_mut().zip(planes) { + o.extend_from_slice(&p); + } + } + for o in &mut out { + assert!(o.len() >= len, "decoded {} < {len}", o.len()); + o.truncate(len); + } + out +} + +fn roundtrip(planar: &[Vec], bitrate: u32) -> (Vec>, Vec>) { + let c = planar.len(); + let len = planar[0].len(); + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, c as u8, bitrate); + let decoded = decode(&enc.opus_head(), &packets, len); + (packets, decoded) +} + +/// SNR in dB over the steady part (the first and last 20 ms are excluded). +fn snr(orig: &[f32], dec: &[f32]) -> f64 { + let (a, b) = (FRAME, orig.len() - FRAME); + let mut s = 0f64; + let mut e = 0f64; + for i in a..b { + s += (orig[i] as f64).powi(2); + e += (orig[i] as f64 - dec[i] as f64).powi(2); + } + 10.0 * (s / e.max(1e-30)).log10() +} + +fn sine(freq: f32, amp: f32, len: usize) -> Vec { + (0..len).map(|i| amp * (2.0 * std::f32::consts::PI * freq * i as f32 / SR).sin()).collect() +} + +fn chirp(f0: f32, f1: f32, amp: f32, len: usize) -> Vec { + let dur = len as f32 / SR; + let k = (f1 - f0) / dur; + (0..len) + .map(|i| { + let t = i as f32 / SR; + amp * (2.0 * std::f32::consts::PI * (f0 * t + 0.5 * k * t * t)).sin() + }) + .collect() +} + +fn noise(amp: f32, len: usize, seed: u32) -> Vec { + let mut s = seed; + (0..len) + .map(|_| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + amp * ((s >> 8) as f32 / (1u32 << 24) as f32 * 2.0 - 1.0) + }) + .collect() +} + +/// Energy (dB) of `x` in each of a few octave bands up to 16 kHz, from a Hann-windowed FFT +/// averaged over 2048-sample blocks. +fn band_energies(x: &[f32]) -> Vec { + const N: usize = 2048; + let edges = [100.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0]; + let mut acc = vec![0f64; edges.len() - 1]; + let mut start = N; + while start + N <= x.len() - N { + let mut re: Vec = (0..N).map(|i| x[start + i] as f64 * (0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / N as f64).cos())).collect(); + let mut im = vec![0f64; N]; + fft(&mut re, &mut im); + for k in 1..N / 2 { + let f = k as f64 * SR as f64 / N as f64; + if let Some(b) = edges.windows(2).position(|w| f >= w[0] && f < w[1]) { + acc[b] += re[k] * re[k] + im[k] * im[k]; + } + } + start += N; + } + acc.iter().map(|e| 10.0 * (e + 1e-20).log10()).collect() +} + +fn fft(re: &mut [f64], im: &mut [f64]) { + let n = re.len(); + let mut j = 0; + for i in 1..n { + let mut bit = n >> 1; + while j & bit != 0 { + j ^= bit; + bit >>= 1; + } + j |= bit; + if i < j { + re.swap(i, j); + im.swap(i, j); + } + } + let mut len = 2; + while len <= n { + let ang = -2.0 * std::f64::consts::PI / len as f64; + for s in (0..n).step_by(len) { + for k in 0..len / 2 { + let (wr, wi) = ((ang * k as f64).cos(), (ang * k as f64).sin()); + let (a, b) = (s + k, s + k + len / 2); + let tr = re[b] * wr - im[b] * wi; + let ti = re[b] * wi + im[b] * wr; + re[b] = re[a] - tr; + im[b] = im[a] - ti; + re[a] += tr; + im[a] += ti; + } + } + len <<= 1; + } +} + +#[test] +fn sine_440_mono() { + let x = sine(440.0, 0.5, 48_000); + for rate in [64_000, 128_000] { + let (packets, d) = roundtrip(std::slice::from_ref(&x), rate); + assert!(packets.iter().all(|p| p.len() == rate as usize / 400)); + let s = snr(&x, &d[0]); + eprintln!("sine 440 Hz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 20.0, "sine SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn chirp_mono() { + let x = chirp(100.0, 10_000.0, 0.4, 96_000); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("chirp 100 Hz-10 kHz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "chirp SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn white_noise_mono() { + let x = noise(0.3, 48_000, 12345); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + let (eo, ed) = (band_energies(&x), band_energies(&d[0])); + let err: Vec = x.iter().zip(&d[0]).map(|(a, b)| a - b).collect(); + let ee = band_energies(&err); + let octave_snr: Vec = eo.iter().zip(&ee).map(|(a, b)| a - b).collect(); + let worst = eo.iter().zip(&ed).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max); + let min_octave = octave_snr.iter().copied().fold(f64::MAX, f64::min); + eprintln!( + "white noise mono @ {} kb/s: full-band SNR {s:.1} dB, octave SNRs 100 Hz-16 kHz {:.1?} dB, worst octave energy error {worst:.2} dB", + rate / 1000, + octave_snr + ); + // Full-band noise loses everything above 20 kHz (CELT's last band ends there) and spends + // few bits per coefficient on 16-20 kHz, so the full-band waveform SNR is low; the band + // energies must match closely and the octaves below 16 kHz must be waveform-coded. + assert!(worst < 1.5, "band energy error {worst:.2} dB at {rate}: {eo:?} vs {ed:?}"); + assert!(s > 0.5, "noise SNR {s:.1} dB at {rate}"); + assert!(min_octave > if rate >= 128_000 { 10.0 } else { 3.0 }, "octave SNRs {octave_snr:?} at {rate}"); + } +} + +#[test] +fn lowpass_noise_mono() { + // White noise through a 4-pole low-pass at about 2 kHz: a dense spectrum the encoder can + // afford to code as a waveform. + let w = noise(0.8, 48_000, 999); + let a = (-2.0 * std::f32::consts::PI * 2_000.0 / SR).exp(); + let mut st = [0f32; 4]; + let x: Vec = w + .iter() + .map(|&v| { + let mut y = v; + for s in st.iter_mut() { + *s = (1.0 - a) * y + a * *s; + y = *s; + } + y + }) + .collect(); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("low-pass noise mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "low-pass noise SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn silence_stays_silent() { + let x = vec![0f32; 48_000]; + let (packets, d) = roundtrip(&[x.clone(), x], 128_000); + assert!(packets.iter().all(|p| p.len() == 3), "silence frames are 3-byte packets"); + for ch in &d { + let peak = ch.iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(peak < 1e-6, "peak {peak}"); + } +} + +#[test] +fn stereo_distinct_channels() { + let l = sine(440.0, 0.5, 48_000); + let r = chirp(200.0, 6_000.0, 0.3, 48_000); + let (packets, d) = roundtrip(&[l.clone(), r.clone()], 256_000); + assert!(packets.iter().all(|p| p[0] == 0xFC && p.len() == 640)); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 256 kb/s: L (sine) SNR {sl:.1} dB, R (chirp) SNR {sr:.1} dB"); + assert!(sl > 20.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); + // No leakage of the chirp into the left channel (2-4 kHz octave: chirp only). + let (el, er) = (band_energies(&d[0]), band_energies(&r)); + assert!(el[4] < er[4] - 30.0, "chirp energy leaked into L: {} vs {}", el[4], er[4]); +} + +#[test] +fn stereo_128k_total() { + let l = sine(1000.0, 0.4, 48_000); + let r = sine(330.0, 0.4, 48_000); + let (_, d) = roundtrip(&[l.clone(), r.clone()], 128_000); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 128 kb/s: L SNR {sl:.1} dB, R SNR {sr:.1} dB"); + assert!(sl > 15.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); +} + +#[test] +fn signal_after_silence_and_loud_transient() { + // Silence, then a burst at full scale, then silence again: energy prediction must recover. + let mut x = vec![0f32; 24_000]; + x.extend(sine(2_000.0, 0.99, 24_000)); + x.extend(vec![0f32; 24_000]); + let (_, d) = roundtrip(std::slice::from_ref(&x), 96_000); + let s = snr(&x[24_000 + 2 * FRAME..48_000], &d[0][24_000 + 2 * FRAME..48_000]); + eprintln!("burst after silence @ 96 kb/s: SNR {s:.1} dB"); + assert!(s > 15.0, "burst SNR {s:.1}"); + let tail_peak = d[0][48_000 + 2 * FRAME..].iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(tail_peak < 1e-3, "tail peak {tail_peak}"); +} + +// ---- Ogg Opus (RFC 3533 + RFC 7845) for the ffmpeg oracle ---- + +fn ogg_crc(data: &[u8]) -> u32 { + let mut crc = 0u32; + for &b in data { + crc ^= (b as u32) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +} + +fn ogg_page(out: &mut Vec, packet: &[u8], header_type: u8, granule: u64, seq: u32) { + let mut lacing = vec![255u8; packet.len() / 255]; + lacing.push((packet.len() % 255) as u8); + let start = out.len(); + out.extend_from_slice(b"OggS"); + out.push(0); + out.push(header_type); + out.extend_from_slice(&granule.to_le_bytes()); + out.extend_from_slice(&0x4543_4F50u32.to_le_bytes()); // serial + out.extend_from_slice(&seq.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); // CRC placeholder + out.push(lacing.len() as u8); + out.extend_from_slice(&lacing); + out.extend_from_slice(packet); + let crc = ogg_crc(&out[start..]); + out[start + 22..start + 26].copy_from_slice(&crc.to_le_bytes()); +} + +fn ogg_opus(enc: &OpusEncoder, packets: &[Vec], len: usize) -> Vec { + let mut out = Vec::new(); + ogg_page(&mut out, &enc.opus_head(), 0x02, 0, 0); + let vendor = b"effectcraft-opusenc"; + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&(vendor.len() as u32).to_le_bytes()); + tags.extend_from_slice(vendor); + tags.extend_from_slice(&0u32.to_le_bytes()); + ogg_page(&mut out, &tags, 0, 0, 1); + let pre_skip = enc.pre_skip() as u64; + for (i, p) in packets.iter().enumerate() { + let last = i + 1 == packets.len(); + let granule = if last { pre_skip + len as u64 } else { ((i + 1) * FRAME) as u64 }; + ogg_page(&mut out, p, if last { 0x04 } else { 0 }, granule, i as u32 + 2); + } + out +} + +#[test] +fn ogg_crc_known_value() { + // CRC-32 (poly 0x04C11DB7, no reflection, init 0) of "123456789". + assert_eq!(ogg_crc(b"123456789"), 0x89A1_897F); +} + +#[test] +fn ffmpeg_oracle_decodes_ogg_opus() { + if Command::new("ffmpeg").arg("-version").output().is_err() { + eprintln!("ffmpeg not installed; skipping the oracle test"); + return; + } + for (channels, rate) in [(1u8, 96_000u32), (2, 192_000)] { + let len = 48_000; + let planar: Vec> = + if channels == 1 { vec![chirp(150.0, 9_000.0, 0.4, len)] } else { vec![sine(440.0, 0.5, len), chirp(150.0, 9_000.0, 0.3, len)] }; + let c = channels as usize; + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, channels, rate); + let path = std::env::temp_dir().join(format!("effectcraft-opusenc-{}-{channels}.opus", std::process::id())); + std::fs::write(&path, ogg_opus(&enc, &packets, len)).unwrap(); + let out = Command::new("ffmpeg") + .args(["-v", "error", "-i"]) + .arg(&path) + .args(["-f", "f32le", "-ac", &channels.to_string(), "-ar", "48000", "-"]) + .output() + .expect("run ffmpeg"); + let _ = std::fs::remove_file(&path); + assert!(out.status.success(), "ffmpeg failed: {}", String::from_utf8_lossy(&out.stderr)); + let dec: Vec = out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + assert_eq!(dec.len(), len * c, "ffmpeg must honour pre-skip and the end granule"); + for ch in 0..c { + let d: Vec = (0..len).map(|i| dec[i * c + ch]).collect(); + let s = snr(&planar[ch], &d); + eprintln!("ffmpeg {channels} ch @ {} kb/s, channel {ch}: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "ffmpeg-decoded SNR {s:.1} dB (channel {ch})"); + } + } +} diff --git a/crates/opusenc/Cargo.toml b/crates/opusenc/Cargo.toml new file mode 100644 index 00000000..90b0fd02 --- /dev/null +++ b/crates/opusenc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-opusenc" +description = "Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only, fullband 48 kHz, 20 ms; RFC 7845 OpusHead) for WebM export audio" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/opusenc/README.md b/crates/opusenc/README.md new file mode 100644 index 00000000..0c76d49f --- /dev/null +++ b/crates/opusenc/README.md @@ -0,0 +1,33 @@ +# effectcraft-opusenc + +Clean-room, pure-Rust Opus encoder for EffectCraft's WebM export audio (layer L0, `std` only, +no `unsafe`). + +- **CELT-only** packets (TOC configuration 31: fullband 20 kHz, 20 ms, one frame per packet), + 48 kHz input, mono or stereo (stereo coded as dual L/R stereo), constant packet size from the + requested bitrate; all-zero frames become 3-byte silence packets. +- Forward MDCT with the low-overlap window, band energies with inter/intra coarse prediction + (cheaper of the two per frame), fine and final energy bits, the decoder's exact bit allocation, + spreading and recursive band splitting, greedy PVQ search and codeword indexing. +- Not implemented: SILK/hybrid modes, transient (short-block) frames, TF changes, pitch + post-filter, intensity stereo, dynamic allocation boosts, VBR rate control. +- `OpusEncoder::pre_skip()` is 120 samples (the MDCT overlap); `opus_head()` returns the RFC 7845 + identification header. + +```rust +let mut enc = effectcraft_opusenc::OpusEncoder::new(2, 128_000); +let packet = enc.encode_float(&vec![0.0; effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2]); +``` + +## Specifications + +- RFC 6716, *Definition of the Opus Audio Codec* (September 2012), with the RFC 8251 updates + (October 2017) as implemented by the decoder we round-trip against. +- RFC 7845, *Ogg Encapsulation for the Opus Audio Codec* (April 2016), §5.1 `OpusHead` and + §4.2 pre-skip. + +Implemented from the RFC text. Constant tables (band edges, `eMeans`, the coarse-energy Laplace +model, the allocation table, ICDFs) and the decoder-side allocation arithmetic are mirrored from +FilmCraft's first-party, spec-derived `filmcraft-opus` decoder (MIT OR Apache-2.0), which is also +the round-trip oracle in `crates/export/tests/opus_roundtrip.rs` (ffmpeg is a second, external +oracle when installed). No third-party encoder or decoder source was consulted. diff --git a/crates/opusenc/src/bands.rs b/crates/opusenc/src/bands.rs new file mode 100644 index 00000000..744de3fe --- /dev/null +++ b/crates/opusenc/src/bands.rs @@ -0,0 +1,497 @@ +//! Band shape coding: PVQ search and codeword indexing, spreading, recursive band splitting +//! (RFC 6716 §4.3.4, encoder direction). +//! +//! The encoder codes 20 ms long-block frames only (no transients, no TF changes), and stereo +//! as dual (independent L/R) stereo, so a band is always coded by `quant_partition` with a single +//! block. Every bit-budget computation mirrors the decoder so both sides take the same +//! split/pulse decisions. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::rate::{Mode, QTHETA_OFFSET, get_pulses}; +use crate::tables::*; + +pub const SPREAD_NORMAL: usize = 2; + +#[inline] +fn frac_mul16(a: i32, b: i32) -> i32 { + (16384 + (a as i16 as i32) * (b as i16 as i32)) >> 15 +} + +fn bitexact_cos(x: i32) -> i32 { + let tmp = (4096 + x * x) >> 13; + let x2 = tmp; + let x2 = (32767 - x2) + frac_mul16(x2, -7651 + frac_mul16(x2, 8277 + frac_mul16(-626, x2))); + 1 + x2 +} + +fn bitexact_log2tan(isin: i32, icos: i32) -> i32 { + let lc = ilog(icos as u32); + let ls = ilog(isin as u32); + let icos = icos << (15 - lc); + let isin = isin << (15 - ls); + (ls - lc) * (1 << 11) + frac_mul16(isin, frac_mul16(isin, -2597) + 7932) - frac_mul16(icos, frac_mul16(icos, -2597) + 7932) +} + +fn compute_qn(n: i32, b: i32, offset: i32, pulse_cap: i32, stereo: bool) -> i32 { + const EXP2_TABLE8: [i32; 8] = [16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048]; + let mut n2 = 2 * n - 1; + if stereo && n == 2 { + n2 -= 1; + } + let mut qb = (b + n2 * offset) / n2; + qb = qb.min(b - pulse_cap - (4 << BITRES)); + qb = qb.min(8 << BITRES); + if qb < (1 << BITRES >> 1) { + 1 + } else { + let qn = EXP2_TABLE8[(qb & 0x7) as usize] >> (14 - (qb >> BITRES)); + (qn + 1) >> 1 << 1 + } +} + +/// One elementary pass of the spreading rotation, inverted (the decoder applies the forward +/// pass; the encoder undoes it before the pulse search). +fn exp_rotation1_inv(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + if len <= stride { + return; + } + if len >= 2 * stride + 1 { + for i in 0..=len - 2 * stride - 1 { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } + } + for i in (0..len - stride).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } +} + +/// Inverse of the decoder's spreading rotation (§4.3.4.3) for one block of `len` samples. +pub fn exp_rotation_inv(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + // The decoder applies (stride2: s, c) then (1: c, s); undo in reverse order. + exp_rotation1_inv(x, len, 1, c, s); + if stride2 != 0 { + exp_rotation1_inv(x, len, stride2, s, c); + } +} + +/// Greedy PVQ search: the integer vector `y` with `sum |y| = k` closest in direction to `x`. +pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) { + let n = x.len(); + y[..n].fill(0); + let abs: Vec = x.iter().map(|v| v.abs()).collect(); + let sum: f32 = abs.iter().sum(); + if !sum.is_finite() || sum <= 1e-15 { + y[0] = k as i32; + return; + } + let mut placed = 0usize; + let mut xy = 0f32; + let mut yy = 0f32; + if k > n / 2 { + // Pre-project onto the pyramid (rounding down) and place the rest greedily. + let r = (k as f32 - 1.0) / sum; + for j in 0..n { + let p = (abs[j] * r).floor() as i32; + y[j] = p; + placed += p as usize; + xy += abs[j] * p as f32; + yy += (p * p) as f32; + } + } + while placed < k { + let mut best = 0usize; + let mut best_num = -1f32; + let mut best_den = 1f32; + for j in 0..n { + let num = xy + abs[j]; + let num = num * num; + let den = yy + 2.0 * y[j] as f32 + 1.0; + if num * best_den > best_num * den { + best_num = num; + best_den = den; + best = j; + } + } + xy += abs[best]; + yy += 2.0 * y[best] as f32 + 1.0; + y[best] += 1; + placed += 1; + } + for j in 0..n { + if x[j] < 0.0 { + y[j] = -y[j]; + } + } +} + +/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps +/// back to `y`. +pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec) { + let n = y.len(); + let w = k + 1; + v.clear(); + v.resize((n + 1) * w, 0); + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + let a = v[(nn - 1) * w + kk]; + let b = v[nn * w + kk - 1]; + let c = v[(nn - 1) * w + kk - 1]; + v[nn * w + kk] = a.saturating_add(b).saturating_add(c); + } + } + let total = v[n * w + k]; + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut index = 0u64; + let mut kk = k; + for j in 0..n { + if kk == 0 { + break; + } + let rem = n - j; + let t = y[j].unsigned_abs() as usize; + // Codewords with |y_j| = t start at sum_{m < kk - t} V(rem-1, m) within their sign half. + let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum(); + if y[j] < 0 { + base += (vv(rem - 1, kk) + vv(rem, kk)) / 2; + } + index += base; + kk -= t; + } + debug_assert!(total <= u32::MAX as u64); + if total >= 2 { + enc.uint(index as u32, total as u32); + } +} + +pub struct BandEnc<'m, 'e> { + pub m: &'m Mode, + pub enc: &'e mut RangeEncoder, + pub i: usize, + pub spread: usize, + pub remaining_bits: i32, + pvq_v: Vec, + iy: Vec, + tmp: Vec, +} + +impl BandEnc<'_, '_> { + /// `compute_theta` for a mono split (single block): measures the energy split between the + /// two halves and codes the quantised angle with the triangular pdf. + fn compute_theta_split(&mut self, xa: &[f32], xb: &[f32], b: &mut i32, lm: i32) -> (i32, i32, i32) { + let n = xa.len() as i32; + let i = self.i; + let pulse_cap = self.m.log_n[i] + lm * (1 << BITRES); + let offset = (pulse_cap >> 1) - QTHETA_OFFSET; + let qn = compute_qn(n, *b, offset, pulse_cap, false); + let tell = self.enc.tell_frac(); + let mut itheta = 0i32; + if qn != 1 { + let mid = xa.iter().map(|v| v * v).sum::().sqrt(); + let side = xb.iter().map(|v| v * v).sum::().sqrt(); + let ang = (side as f64).atan2(mid as f64) * std::f64::consts::FRAC_2_PI; + let it = ((ang * qn as f64 + 0.5).floor() as i32).clamp(0, qn) as u32; + let q = qn as u32; + let half = q >> 1; + let ft = (half + 1) * (half + 1); + let (fl, fs) = if it <= half { (it * (it + 1) >> 1, it + 1) } else { (ft - ((q + 1 - it) * (q + 2 - it) >> 1), q + 1 - it) }; + self.enc.encode(fl, fl + fs, ft); + itheta = ((it * 16384) / q) as i32; + } + let qalloc = self.enc.tell_frac() - tell; + *b -= qalloc; + let delta = if itheta == 0 { + -16384 + } else if itheta == 16384 { + 16384 + } else { + let imid = bitexact_cos(itheta); + let iside = bitexact_cos(16384 - itheta); + frac_mul16((n - 1) << 7, bitexact_log2tan(iside, imid)) + }; + (itheta, delta, qalloc) + } + + /// `quant_partition` (single block, encoder side). + pub fn quant_partition(&mut self, x: &[f32], mut b: i32, mut lm: i32) { + let n = x.len(); + let i = self.i; + let cache = self.m.cache(lm, i); + let cache_max = cache[cache[0] as usize] as i32; + if lm != -1 && b > cache_max + 12 && n > 2 { + let half = n >> 1; + let (xa, xb) = x.split_at(half); + lm -= 1; + let (itheta, delta, qalloc) = self.compute_theta_split(xa, xb, &mut b, lm); + let mut mbits = 0.max(b.min((b - delta) / 2)); + let mut sbits = b - mbits; + self.remaining_bits -= qalloc; + let mut rebalance = self.remaining_bits; + if mbits >= sbits { + self.quant_partition(xa, mbits, lm); + rebalance = mbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 0 { + sbits += rebalance - (3 << BITRES); + } + self.quant_partition(xb, sbits, lm); + } else { + self.quant_partition(xb, sbits, lm); + rebalance = sbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 16384 { + mbits += rebalance - (3 << BITRES); + } + self.quant_partition(xa, mbits, lm); + } + return; + } + let m = self.m; + let mut q = m.bits2pulses(i, lm, b); + let mut curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + while self.remaining_bits < 0 && q > 0 { + self.remaining_bits += curr_bits; + q -= 1; + curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + } + if q != 0 { + let k = get_pulses(q) as usize; + let mut tmp = std::mem::take(&mut self.tmp); + tmp.clear(); + tmp.extend_from_slice(x); + exp_rotation_inv(&mut tmp, n, k, self.spread); + let mut iy = std::mem::take(&mut self.iy); + iy.clear(); + iy.resize(n, 0); + pvq_search(&tmp, k, &mut iy); + encode_pulses(&iy, k, self.enc, &mut self.pvq_v); + self.iy = iy; + self.tmp = tmp; + } + } +} + +/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds `c` channels +/// of `n` normalised MDCT coefficients each. +#[allow(clippy::too_many_arguments)] +pub fn quant_all_bands( + m: &Mode, + start: usize, + end: usize, + x: &[f32], + n_total: usize, + c: usize, + pulses: &[i32; NB_EBANDS], + spread: usize, + dual_stereo: bool, + intensity: usize, + total_bits: i32, + mut balance: i32, + enc: &mut RangeEncoder, + lm: usize, + coded_bands: usize, +) { + let mm = 1usize << lm; + debug_assert!(c == 1 || dual_stereo || intensity <= start, "stereo frames are coded as dual stereo"); + let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, pvq_v: Vec::new(), iy: Vec::new(), tmp: Vec::new() }; + for i in start..end { + ctx.i = i; + let xo = mm * EBANDS[i] as usize; + let n = mm * EBANDS[i + 1] as usize - xo; + let tell = ctx.enc.tell_frac(); + if i != start { + balance -= tell; + } + let remaining_bits = total_bits - tell - 1; + ctx.remaining_bits = remaining_bits; + let b = if i < coded_bands { + let curr_balance = balance / 3.min(coded_bands as i32 - i as i32); + 0.max(16383.min((remaining_bits + 1).min(pulses[i] + curr_balance))) + } else { + 0 + }; + // Bands at and above `intensity` (= coded_bands) carry no bits, so they emit no symbols. + if i < coded_bands || c == 1 { + if c == 2 && dual_stereo && i < intensity { + ctx.quant_partition(&x[xo..xo + n], b / 2, lm as i32); + ctx.quant_partition(&x[n_total + xo..n_total + xo + n], b / 2, lm as i32); + } else { + ctx.quant_partition(&x[xo..xo + n], b, lm as i32); + } + } + balance += pulses[i] + tell; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + fn exp_rotation1(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + let ms = -s; + if len <= stride { + return; + } + for i in 0..len - stride { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + if len >= 2 * stride + 1 { + for i in (0..=len - 2 * stride - 1).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + } + } + + /// The decoder's spreading rotation (single block). + fn exp_rotation(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + if stride2 != 0 { + exp_rotation1(x, len, stride2, s, c); + } + exp_rotation1(x, len, 1, c, s); + } + + #[test] + fn spreading_inverse_is_exact() { + for &len in &[8usize, 16, 24, 48, 96, 176] { + for k in [1usize, 2, 3, 5] { + let orig: Vec = (0..len).map(|i| ((i * 31 % 17) as f32 - 8.0) / 8.0).collect(); + let mut x = orig.clone(); + exp_rotation_inv(&mut x, len, k, SPREAD_NORMAL); + exp_rotation(&mut x, len, k, SPREAD_NORMAL); + for (a, b) in x.iter().zip(&orig) { + assert!((a - b).abs() < 1e-4, "len={len} k={k}"); + } + } + } + } + + /// PVQ decoder of RFC 6716 §4.3.4.2 (as in FilmCraft's decoder). + fn decode_pulses(n: usize, k: usize, dec: &mut RangeDecoder) -> Vec { + let w = k + 1; + let mut v = vec![0u64; (n + 1) * w]; + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + v[nn * w + kk] = v[(nn - 1) * w + kk].saturating_add(v[nn * w + kk - 1]).saturating_add(v[(nn - 1) * w + kk - 1]); + } + } + let total = v[n * w + k]; + let mut i = if total >= 2 { dec.uint(total as u32) as u64 } else { 0 }; + let mut y = vec![0i32; n]; + let mut kk = k; + for j in 0..n { + let rem = n - j; + if kk == 0 { + break; + } + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut p = (vv(rem - 1, kk) + vv(rem, kk)) / 2; + let neg = i >= p; + if neg { + i -= p; + } + let k0 = kk; + p -= vv(rem - 1, kk); + while p > i && kk > 0 { + kk -= 1; + p -= vv(rem - 1, kk); + } + let mag = (k0 - kk) as i32; + y[j] = if neg { -mag } else { mag }; + i -= p.min(i); + } + y + } + + fn all_codewords(n: usize, k: usize) -> Vec> { + if n == 0 { + return if k == 0 { vec![vec![]] } else { vec![] }; + } + let mut out = Vec::new(); + for t in -(k as i32)..=(k as i32) { + for mut rest in all_codewords(n - 1, k - t.unsigned_abs() as usize) { + rest.insert(0, t); + out.push(rest); + } + } + out + } + + #[test] + fn pvq_index_roundtrip_exhaustive() { + let mut v = Vec::new(); + for n in 1..6usize { + for k in 1..6usize { + for y in all_codewords(n, k) { + let mut e = RangeEncoder::new(); + encode_pulses(&y, k, &mut e, &mut v); + let buf = e.done(16); + let mut d = RangeDecoder::new(&buf); + assert_eq!(decode_pulses(n, k, &mut d), y, "n={n} k={k}"); + } + } + } + } + + #[test] + fn pvq_search_finds_good_direction() { + let x: Vec = (0..32).map(|i| ((i as f32) * 0.7).sin()).collect(); + let mut y = vec![0i32; 32]; + for k in [1usize, 4, 16, 64] { + pvq_search(&x, k, &mut y); + assert_eq!(y.iter().map(|v| v.unsigned_abs() as usize).sum::(), k); + let xy: f32 = x.iter().zip(&y).map(|(a, &b)| a * b as f32).sum(); + let yy: f32 = y.iter().map(|&b| (b * b) as f32).sum(); + let xx: f32 = x.iter().map(|a| a * a).sum(); + let corr = xy / (xx * yy).sqrt(); + assert!(corr > if k >= 16 { 0.9 } else { 0.2 }, "k={k} corr={corr}"); + } + } +} diff --git a/crates/opusenc/src/energy.rs b/crates/opusenc/src/energy.rs new file mode 100644 index 00000000..0057ae33 --- /dev/null +++ b/crates/opusenc/src/energy.rs @@ -0,0 +1,230 @@ +//! Band energy quantisation: Laplace-coded coarse energy with inter/intra prediction, fine +//! energy and the final refinement bits (RFC 6716 §4.3.2, encoder direction). +//! +//! The quantised energies are updated with exactly the decoder's arithmetic so the prediction +//! state never drifts between encoder and decoder. + +use crate::range::RangeEncoder; +use crate::rate::MAX_FINE_BITS; +use crate::tables::*; + +const LAPLACE_MINP: u32 = 1; +const LAPLACE_NMIN: u32 = 16; + +fn laplace_get_freq1(fs0: u32, decay: u32) -> u32 { + let ft = 32768 - LAPLACE_MINP * (2 * LAPLACE_NMIN) - fs0; + (ft * (16384 - decay)) >> 15 +} + +/// Encodes a Laplace-distributed integer (`fs` = probability of 0 in Q15, `decay` in Q14). +/// Returns the value actually coded (very large magnitudes are clamped to the representable +/// range). +pub fn laplace_encode(enc: &mut RangeEncoder, value: i32, mut fs: u32, decay: u32) -> i32 { + let mut fl = 0u32; + let mut coded = value; + if value != 0 { + let s: i32 = if value < 0 { -1 } else { 0 }; + let val = value.abs(); + fl = fs; + fs = laplace_get_freq1(fs, decay); + let mut i = 1; + while fs > 0 && i < val { + fs *= 2; + fl += fs + 2 * LAPLACE_MINP; + fs = (fs * decay) >> 15; + i += 1; + } + if fs == 0 { + let ndi_max = (32768 - fl + LAPLACE_MINP - 1) as i32; + let ndi_max = (ndi_max - s) >> 1; + let di = (val - i).min(ndi_max - 1); + fl = (fl as i32 + (2 * di + 1 + s) * LAPLACE_MINP as i32) as u32; + fs = LAPLACE_MINP.min(32768 - fl); + let mag = i + di; + coded = if s < 0 { -mag } else { mag }; + } else { + fs += LAPLACE_MINP; + if s == 0 { + fl += fs; + } + } + } + enc.encode_bin(fl, fl + fs, 15); + coded +} + +/// `quant_coarse_energy` for one prediction mode. `e` holds the target log2 energies (relative +/// to `E_MEANS`), `old_e` the quantised energies, updated in place. `budget` is the frame size in +/// bits. +#[allow(clippy::too_many_arguments)] +pub fn quant_coarse_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + intra: bool, + enc: &mut RangeEncoder, + c: usize, + lm: usize, + budget: i32, +) { + let prob = &E_PROB_MODEL[lm][intra as usize]; + let (coef, beta) = if intra { (0.0, BETA_INTRA) } else { (PRED_COEF[lm], BETA_COEF[lm]) }; + let mut prev = [0f32; 2]; + for i in start..end { + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let oldc = old_e[idx].max(-9.0); + let x = e[idx]; + let f = x - coef * oldc - prev[ch]; + let mut qi = (f + 0.5).floor().clamp(-64.0, 64.0) as i32; + let tell = enc.tell(); + // Keep a few bits per remaining band in reserve when the budget runs out. + let bits_left = budget - tell - 3 * c as i32 * (end - i) as i32; + if i != start && bits_left < 30 { + if bits_left < 24 { + qi = qi.min(1); + } + if bits_left < 16 { + qi = qi.max(-1); + } + } + if budget - tell >= 15 { + let pi = 2 * i.min(20); + qi = laplace_encode(enc, qi, (prob[pi] as u32) << 7, (prob[pi + 1] as u32) << 6); + } else if budget - tell >= 2 { + qi = qi.clamp(-1, 1); + enc.icdf(if qi < 0 { (-2 * qi - 1) as usize } else { (2 * qi) as usize }, &SMALL_ENERGY_ICDF, 2); + } else if budget - tell >= 1 { + qi = qi.clamp(-1, 0); + enc.bit_logp(qi == -1, 1); + } else { + qi = -1; + } + let q = qi as f32; + old_e[idx] = oldc; + let tmp = coef * old_e[idx] + prev[ch] + q; + old_e[idx] = tmp; + prev[ch] = prev[ch] + q - beta * q; + } + } +} + +/// `quant_fine_energy`: raw bits refining each band's quantised energy. +pub fn quant_fine_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + enc: &mut RangeEncoder, + c: usize, +) { + for i in start..end { + let fq = fine_quant[i]; + if fq <= 0 { + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let err = e[idx] - old_e[idx]; + let levels = 1i32 << fq; + let q2 = (((err + 0.5) * levels as f32).floor() as i32).clamp(0, levels - 1); + enc.bits(q2 as u32, fq as u32); + let offset = (q2 as f32 + 0.5) * (1 << (14 - fq)) as f32 / 16384.0 - 0.5; + old_e[idx] += offset; + } + } +} + +/// `quant_energy_finalise`: spends the bits left at the end of the frame on one more bit of +/// energy resolution per band, in priority order. +#[allow(clippy::too_many_arguments)] +pub fn quant_energy_finalise( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + fine_priority: &[i32; NB_EBANDS], + mut bits_left: i32, + enc: &mut RangeEncoder, + c: usize, +) { + for prio in 0..2 { + let mut i = start; + while i < end && bits_left >= c as i32 { + if fine_quant[i] >= MAX_FINE_BITS || fine_priority[i] != prio { + i += 1; + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let q2 = (e[idx] - old_e[idx] >= 0.0) as u32; + enc.bits(q2, 1); + let offset = (q2 as f32 - 0.5) * (1 << (14 - fine_quant[i] - 1)) as f32 / 16384.0; + old_e[idx] += offset; + bits_left -= 1; + } + i += 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + /// Laplace decoder of RFC 6716 §4.3.2.1 (as in FilmCraft's decoder). + fn laplace_decode(dec: &mut RangeDecoder, mut fs: u32, decay: u32) -> i32 { + let mut val = 0i32; + let fm = dec.decode(32768); + let mut fl = 0u32; + if fm >= fs { + val += 1; + fl = fs; + fs = laplace_get_freq1(fs, decay) + LAPLACE_MINP; + while fs > LAPLACE_MINP && fm >= fl + 2 * fs { + fs *= 2; + fl += fs; + fs = (((fs - 2 * LAPLACE_MINP) * decay) >> 15) + LAPLACE_MINP; + val += 1; + } + if fs <= LAPLACE_MINP { + let di = (fm - fl) >> 1; + val += di as i32; + fl += 2 * di * LAPLACE_MINP; + } + if fm < fl + fs { + val = -val; + } else { + fl += fs; + } + } + dec.update(fl, (fl + fs).min(32768), 32768); + val + } + + #[test] + fn laplace_roundtrip_all_models() { + for lm in 0..4 { + for intra in 0..2 { + for band in 0..21 { + let prob = &E_PROB_MODEL[lm][intra]; + let (fs, decay) = ((prob[2 * band] as u32) << 7, (prob[2 * band + 1] as u32) << 6); + let vals: Vec = (-64..=64).collect(); + let mut e = RangeEncoder::new(); + let coded: Vec = vals.iter().map(|&v| laplace_encode(&mut e, v, fs, decay)).collect(); + let size = (e.tell() as usize).div_ceil(8); + let buf = e.done(size); + let mut d = RangeDecoder::new(&buf); + for (&v, &c) in vals.iter().zip(&coded) { + assert!(c == v || (c.signum() == v.signum() && c.abs() < v.abs()), "clamp {v} -> {c}"); + assert_eq!(laplace_decode(&mut d, fs, decay), c, "lm={lm} intra={intra} band={band} v={v}"); + } + } + } + } + } +} diff --git a/crates/opusenc/src/lib.rs b/crates/opusenc/src/lib.rs new file mode 100644 index 00000000..1e7a2cec --- /dev/null +++ b/crates/opusenc/src/lib.rs @@ -0,0 +1,311 @@ +//! Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only mode; RFC 7845 `OpusHead`). +//! +//! Produces fullband (20 kHz) 48 kHz Opus packets of 20 ms, mono or stereo, at a constant +//! packet size derived from the requested bitrate. Each packet is a single CELT frame (TOC +//! configuration 31, frame-count code 0). Stereo is coded as dual (independent L/R) stereo. +//! +//! Layer L0: no dependencies beyond `std`; no `unsafe`; builds for `wasm32-unknown-unknown`. +//! +//! ``` +//! use effectcraft_opusenc::OpusEncoder; +//! let mut enc = OpusEncoder::new(2, 128_000); +//! let pcm = vec![0.0f32; OpusEncoder::FRAME_SIZE * 2]; +//! let packet = enc.encode_float(&pcm); +//! assert_eq!(packet[0] >> 3, 31); // CELT-only, fullband, 20 ms +//! assert_eq!(enc.opus_head().len(), 19); +//! ``` + +// The bit-allocation code mirrors the normative integer arithmetic; keep C-like shift expressions. +#![allow(clippy::precedence, clippy::int_plus_one)] + +mod bands; +mod energy; +mod mdct; +mod range; +mod rate; +mod tables; + +use std::sync::OnceLock; + +use bands::{SPREAD_NORMAL, quant_all_bands}; +use energy::{quant_coarse_energy, quant_energy_finalise, quant_fine_energy}; +use mdct::Mdct; +use range::{BITRES, RangeEncoder}; +use rate::{Mode, compute_allocation}; +use tables::*; + +/// Samples per channel in one 20 ms frame at 48 kHz. +const N: usize = 960; +/// LM of a 20 ms frame (8 short blocks). +const LM: usize = 3; +/// Smallest CELT payload the encoder produces (bytes). +const MIN_FRAME_BYTES: usize = 16; +/// Largest CELT payload allowed in one frame (RFC 6716 §3.4, R2). +const MAX_FRAME_BYTES: usize = 1275; + +fn mode() -> &'static Mode { + static MODE: OnceLock = OnceLock::new(); + MODE.get_or_init(Mode::new) +} + +/// Opus encoder for one mono or stereo stream at 48 kHz. +pub struct OpusEncoder { + channels: usize, + /// CELT payload bytes per packet (the packet is one byte longer: the TOC). + frame_bytes: usize, + mdct: Mdct, + /// Last input sample of each channel (pre-emphasis filter memory, ×32768). + preemph_mem: [f32; 2], + /// Last `OVERLAP` pre-emphasised samples of each channel. + hist: Vec, + /// Quantised band energies, exactly as the decoder holds them. + old_band_e: [f32; 2 * NB_EBANDS], + block: Vec, + freq: Vec, +} + +impl OpusEncoder { + /// Samples per channel per packet (20 ms at 48 kHz). + pub const FRAME_SIZE: usize = N; + + /// Creates an encoder for `channels` (1 or 2) channels of 48 kHz audio at `bitrate_bps` + /// (total for all channels; packets are constant-size, clamped to 6.8–510 kb/s). + pub fn new(channels: u8, bitrate_bps: u32) -> Self { + assert!(channels == 1 || channels == 2, "Opus family-0 streams have 1 or 2 channels"); + let c = channels as usize; + let packet_bytes = (bitrate_bps as usize).div_ceil(400); + let frame_bytes = packet_bytes.saturating_sub(1).clamp(MIN_FRAME_BYTES, MAX_FRAME_BYTES); + OpusEncoder { + channels: c, + frame_bytes, + mdct: Mdct::new(N), + preemph_mem: [0.0; 2], + hist: vec![0.0; c * OVERLAP], + old_band_e: [-28.0; 2 * NB_EBANDS], + block: vec![0.0; N + OVERLAP], + freq: vec![0.0; c * N], + } + } + + /// Number of channels. + pub fn channels(&self) -> u8 { + self.channels as u8 + } + + /// Actual bitrate in bits per second (packet size × 50 packets per second). + pub fn bitrate(&self) -> u32 { + ((self.frame_bytes + 1) * 8 * 50) as u32 + } + + /// Samples at 48 kHz the decoder must discard from the start of the stream (the MDCT + /// overlap); this is the `OpusHead` pre-skip. + pub fn pre_skip(&self) -> u16 { + OVERLAP as u16 + } + + /// The 19-byte `OpusHead` identification header (RFC 7845 §5.1): version 1, channel count, + /// pre-skip, input sample rate 48000, output gain 0, channel mapping family 0. + pub fn opus_head(&self) -> Vec { + let mut h = Vec::with_capacity(19); + h.extend_from_slice(b"OpusHead"); + h.push(1); + h.push(self.channels as u8); + h.extend_from_slice(&self.pre_skip().to_le_bytes()); + h.extend_from_slice(&48_000u32.to_le_bytes()); + h.extend_from_slice(&0i16.to_le_bytes()); + h.push(0); + h + } + + /// Encodes one 20 ms frame of interleaved f32 samples in [-1, 1] (exactly + /// `FRAME_SIZE * channels` values) and returns one Opus packet (TOC byte + CELT frame). + pub fn encode_float(&mut self, pcm: &[f32]) -> Vec { + let c = self.channels; + assert_eq!(pcm.len(), N * c, "encode_float takes exactly FRAME_SIZE samples per channel"); + + // Pre-emphasis, the MDCT and the band energies. + let mut band_e = [0f32; 2 * NB_EBANDS]; + let mut log_e = [0f32; 2 * NB_EBANDS]; + let mut peak = 0f32; + for ch in 0..c { + self.block[..OVERLAP].copy_from_slice(&self.hist[ch * OVERLAP..(ch + 1) * OVERLAP]); + let mut mem = self.preemph_mem[ch]; + for j in 0..N { + let s = pcm[j * c + ch]; + let s = if s.is_finite() { s.clamp(-2.0, 2.0) * 32768.0 } else { 0.0 }; + let v = s - PREEMPH * mem; + mem = s; + self.block[OVERLAP + j] = v; + } + self.preemph_mem[ch] = mem; + self.hist[ch * OVERLAP..(ch + 1) * OVERLAP].copy_from_slice(&self.block[N..]); + peak = self.block.iter().fold(peak, |p, v| p.max(v.abs())); + let freq = &mut self.freq[ch * N..(ch + 1) * N]; + self.mdct.forward(&self.block, freq); + for i in 0..NB_EBANDS { + let lo = (EBANDS[i] as usize) << LM; + let hi = (EBANDS[i + 1] as usize) << LM; + let e = (1e-27f32 + freq[lo..hi].iter().map(|v| v * v).sum::()).sqrt(); + band_e[ch * NB_EBANDS + i] = e; + log_e[ch * NB_EBANDS + i] = e.log2() - E_MEANS[i]; + let g = 1.0 / e; + for v in freq[lo..hi].iter_mut() { + *v *= g; + } + } + } + let silence = peak < 1e-4; + let toc = (31u8 << 3) | if c == 2 { 4 } else { 0 }; + let mut packet = vec![toc]; + packet.extend_from_slice(&self.encode_celt(&log_e, silence)); + packet + } + + /// Codes one CELT frame (RFC 6716 §4.3, in bitstream order). + fn encode_celt(&mut self, log_e: &[f32; 2 * NB_EBANDS], silence: bool) -> Vec { + let m = mode(); + let c = self.channels; + let (start, end) = (0usize, NB_EBANDS); + let mut enc = RangeEncoder::new(); + if c == 1 { + for i in 0..NB_EBANDS { + self.old_band_e[i] = self.old_band_e[i].max(self.old_band_e[NB_EBANDS + i]); + } + } + if silence { + // The silence flag; the decoder treats every remaining bit as consumed. + enc.bit_logp(true, 15); + self.old_band_e = [-28.0; 2 * NB_EBANDS]; + return enc.done(2); + } + let len = self.frame_bytes; + let total_bits_raw = len as i32 * 8; + enc.bit_logp(false, 15); + // No pitch post-filter. + if enc.tell() + 16 <= total_bits_raw { + enc.bit_logp(false, 1); + } + // Long blocks only. + if enc.tell() + 3 <= total_bits_raw { + enc.bit_logp(false, 3); + } + // Coarse energy, choosing intra or inter prediction by trial (whichever is cheaper). + if enc.tell() + 3 <= total_bits_raw { + let mut enc_intra = enc.clone(); + let mut old_intra = self.old_band_e; + enc_intra.bit_logp(true, 3); + quant_coarse_energy(start, end, log_e, &mut old_intra, true, &mut enc_intra, c, LM, total_bits_raw); + enc.bit_logp(false, 3); + let mut old_inter = self.old_band_e; + quant_coarse_energy(start, end, log_e, &mut old_inter, false, &mut enc, c, LM, total_bits_raw); + if enc_intra.tell_frac() < enc.tell_frac() { + enc = enc_intra; + self.old_band_e = old_intra; + } else { + self.old_band_e = old_inter; + } + } else { + quant_coarse_energy(start, end, log_e, &mut self.old_band_e, false, &mut enc, c, LM, total_bits_raw); + } + // TF resolution: no changes. + { + let mut budget = total_bits_raw; + let mut tell = enc.tell(); + let mut logp = 4; + let tf_select_rsv = LM > 0 && tell + logp + 1 <= budget; + budget -= tf_select_rsv as i32; + for _ in start..end { + if tell + logp <= budget { + enc.bit_logp(false, logp as u32); + tell = enc.tell(); + } + logp = 5; + } + if tf_select_rsv && TF_SELECT_TABLE[LM][0] != TF_SELECT_TABLE[LM][2] { + enc.bit_logp(false, 1); + } + } + if enc.tell() + 4 <= total_bits_raw { + enc.icdf(SPREAD_NORMAL, &SPREAD_ICDF, 5); + } + // Dynamic allocation: no boosts. + let cap = m.init_caps(LM, c); + let offsets = [0i32; NB_EBANDS]; + let total_bits = total_bits_raw << BITRES; + let mut tellf = enc.tell_frac(); + for &cp in cap.iter().take(end).skip(start) { + if tellf + (6 << BITRES) < total_bits && 0 < cp { + enc.bit_logp(false, 6); + tellf = enc.tell_frac(); + } + } + let alloc_trim = 5; + if tellf + (6 << BITRES) <= total_bits { + enc.icdf(alloc_trim as usize, &TRIM_ICDF, 7); + } + let bits = (total_bits_raw << BITRES) - enc.tell_frac() - 1; + let alloc = compute_allocation(m, start, end, &offsets, &cap, alloc_trim, bits, c, LM, &mut enc); + quant_fine_energy(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc, c); + quant_all_bands( + m, + start, + end, + &self.freq, + N, + c, + &alloc.pulses, + SPREAD_NORMAL, + alloc.dual_stereo, + alloc.intensity, + total_bits_raw << BITRES, + alloc.balance, + &mut enc, + LM, + alloc.coded_bands, + ); + let bits_left = total_bits_raw - enc.tell(); + quant_energy_finalise(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &alloc.fine_priority, bits_left, &mut enc, c); + debug_assert!(enc.tell() <= total_bits_raw, "frame overflow: {} > {}", enc.tell(), total_bits_raw); + if c == 1 { + let (a, b) = self.old_band_e.split_at_mut(NB_EBANDS); + b.copy_from_slice(a); + } + enc.done(len) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn opus_head_layout() { + let e = OpusEncoder::new(2, 96_000); + let h = e.opus_head(); + assert_eq!(&h[..8], b"OpusHead"); + assert_eq!(h[8], 1); + assert_eq!(h[9], 2); + assert_eq!(u16::from_le_bytes([h[10], h[11]]), e.pre_skip()); + assert_eq!(u32::from_le_bytes([h[12], h[13], h[14], h[15]]), 48_000); + assert_eq!(&h[16..], &[0, 0, 0]); + } + + #[test] + fn packet_sizes_follow_bitrate() { + for (ch, rate) in [(1u8, 64_000u32), (2, 128_000), (2, 256_000)] { + let mut e = OpusEncoder::new(ch, rate); + let pcm: Vec = (0..N * ch as usize).map(|i| (i as f32 * 0.05).sin() * 0.5).collect(); + let p = e.encode_float(&pcm); + assert_eq!(p.len(), rate as usize / 400); + assert_eq!(p[0], 0xF8 | if ch == 2 { 4 } else { 0 }); + assert_eq!(e.bitrate(), rate); + } + } + + #[test] + fn silence_is_a_tiny_packet() { + let mut e = OpusEncoder::new(1, 64_000); + let p = e.encode_float(&vec![0.0; N]); + assert_eq!(p.len(), 3); + } +} diff --git a/crates/opusenc/src/mdct.rs b/crates/opusenc/src/mdct.rs new file mode 100644 index 00000000..814de037 --- /dev/null +++ b/crates/opusenc/src/mdct.rs @@ -0,0 +1,293 @@ +//! Forward MDCT with the CELT low-overlap window (RFC 6716 §4.3.7, run in the analysis +//! direction). +//! +//! A 20 ms frame is a 1920-point MDCT (960 coefficients) whose window is zero for 420 samples, +//! rises over the 120-sample overlap, is flat for 840 samples, falls over the next 120 samples +//! and is zero again for the last 420. The input block is therefore the 120 overlap samples kept +//! from the previous frame followed by the 960 new samples. The MDCT is folded into a DCT-IV, +//! which is computed with an N/2-point complex FFT. + +use crate::tables::OVERLAP; + +#[derive(Clone, Copy, Default, Debug)] +pub struct Cpx { + pub re: f32, + pub im: f32, +} + +impl Cpx { + #[inline] + fn mul(self, o: Cpx) -> Cpx { + Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re } + } + #[inline] + fn add(self, o: Cpx) -> Cpx { + Cpx { re: self.re + o.re, im: self.im + o.im } + } +} + +/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive). +pub struct Fft { + n: usize, + factors: Vec, + twiddle: Vec, +} + +impl Fft { + pub fn new(n: usize) -> Fft { + let mut factors = Vec::new(); + let mut m = n; + for p in [4usize, 2, 3, 5] { + while m.is_multiple_of(p) { + factors.push(p); + m /= p; + } + } + assert_eq!(m, 1, "unsupported FFT size {n}"); + let twiddle = (0..n) + .map(|k| { + let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + Fft { n, factors, twiddle } + } + + pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec) { + scratch.clear(); + scratch.extend_from_slice(&data[..self.n]); + self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1); + } + + fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) { + let p = self.factors[fi]; + let m = n / p; + if m == 1 { + for (i, o) in out.iter_mut().enumerate().take(p) { + *o = inp[i * in_stride]; + } + } else { + for q in 0..p { + self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p); + } + } + let big = self.n; + let mut t = [Cpx::default(); 5]; + for k in 0..m { + for (q, tq) in t.iter_mut().enumerate().take(p) { + *tq = out[q * m + k].mul(self.twiddle[(q * k * tw_stride) % big]); + } + for q2 in 0..p { + let mut acc = Cpx::default(); + for (q, tq) in t.iter().enumerate().take(p) { + acc = acc.add(tq.mul(self.twiddle[((q * q2) % p) * (big / p)])); + } + out[q2 * m + k] = acc; + } + } + } +} + +/// Forward MDCT producing `n` coefficients from a block of `n + OVERLAP` samples. +pub struct Mdct { + /// Number of coefficients (half the MDCT length). + n: usize, + fft: Fft, + pre: Vec, + post: Vec, + /// Rising half of the low-overlap window (RFC 6716 §4.3.7). + pub window: [f32; OVERLAP], + scale: f32, + z: Vec, + scratch: Vec, + v: Vec, +} + +impl Mdct { + pub fn new(n: usize) -> Mdct { + let n2 = n / 2; + let pi = std::f64::consts::PI; + let pre = (0..n2) + .map(|m| { + let ph = -pi * m as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let post = (0..n2) + .map(|p| { + let ph = -pi * (p as f64 + 0.25) / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let mut window = [0f32; OVERLAP]; + for (i, w) in window.iter_mut().enumerate() { + let x = std::f64::consts::FRAC_PI_2 * (i as f64 + 0.5) / OVERLAP as f64; + let s = x.sin(); + *w = (std::f64::consts::FRAC_PI_2 * s * s).sin() as f32; + } + Mdct { n, fft: Fft::new(n2), pre, post, window, scale: 2.0 / n as f32, z: Vec::new(), scratch: Vec::new(), v: vec![0.0; n] } + } + + /// Transforms `block` (`n + OVERLAP` samples: the previous frame's last `OVERLAP` samples, then + /// `n` new ones) into `out[..n]`, scaled so that the RFC 6716 decoder's inverse MDCT and + /// overlap-add reproduce the input. + pub fn forward(&mut self, block: &[f32], out: &mut [f32]) { + let n = self.n; + let n2 = n / 2; + debug_assert_eq!(block.len(), n + OVERLAP); + // Place the windowed block into the (virtual) 2n-sample MDCT input u at offset `pad`. + let pad = (n - OVERLAP) / 2; + let w = &self.window; + let u = |t: usize| -> f32 { + if t < pad || t >= pad + n + OVERLAP { + return 0.0; + } + let j = t - pad; + let g = if j < OVERLAP { + w[j] + } else if j >= n { + w[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + block[j] * g + }; + // Fold to a DCT-IV input: v = (-c_r - d, a - b_r) over the four quarters of u. + for m in 0..n2 { + self.v[m] = -u(n + n2 - 1 - m) - u(n + n2 + m); + self.v[n2 + m] = u(m) - u(n2 + n2 - 1 - m); + } + // DCT-IV via an n/2-point complex FFT. + self.z.clear(); + for m in 0..n2 { + let wv = Cpx { re: self.v[2 * m], im: self.v[n - 1 - 2 * m] }; + self.z.push(wv.mul(self.pre[m])); + } + self.fft.process(&mut self.z, &mut self.scratch); + for p in 0..n2 { + let y = self.z[p].mul(self.post[p]); + out[2 * p] = y.re * self.scale; + out[n - 1 - 2 * p] = -y.im * self.scale; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Direct MDCT definition X[k] = sum u[t] cos(pi/N (t + 1/2 + N/2)(k + 1/2)). + fn direct(block: &[f32], n: usize, window: &[f32; OVERLAP]) -> Vec { + let pad = (n - OVERLAP) / 2; + let mut u = vec![0f64; 2 * n]; + for (j, &b) in block.iter().enumerate() { + let g = if j < OVERLAP { + window[j] + } else if j >= n { + window[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + u[pad + j] = (b * g) as f64; + } + (0..n) + .map(|k| { + (0..2 * n).map(|t| u[t] * (std::f64::consts::PI / n as f64 * (t as f64 + 0.5 + n as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum::() * 2.0 + / n as f64 + }) + .collect() + } + + #[test] + fn matches_direct_formula() { + for &n in &[240usize, 960] { + let block: Vec = (0..n + OVERLAP).map(|i| ((i * 37 % 101) as f32 - 50.0) / 7.0).collect(); + let mut m = Mdct::new(n); + let mut out = vec![0f32; n]; + m.forward(&block, &mut out); + let d = direct(&block, n, &m.window); + for k in 0..n { + assert!((out[k] as f64 - d[k]).abs() < 1e-2, "n={n} k={k}: {} vs {}", out[k], d[k]); + } + } + } + + // ---- Replica of the RFC 6716 decoder inverse MDCT (as in FilmCraft's decoder) ---- + + fn imdct_backward(n_full: usize, input: &[f32], out: &mut [f32], window: &[f32]) { + let n2 = n_full >> 1; + let n4 = n_full >> 2; + let t: Vec = (0..n2).map(|i| (2.0 * std::f64::consts::PI * (i as f64 + 0.125) / n_full as f64).cos() as f32).collect(); + let fft = Fft::new(n4); + let mut z = Vec::new(); + for i in 0..n4 { + let x1 = input[2 * i]; + let x2 = input[n2 - 1 - 2 * i]; + let yr = x2 * t[i] + x1 * t[n4 + i]; + let yi = x1 * t[i] - x2 * t[n4 + i]; + z.push(Cpx { re: yi, im: yr }); + } + fft.process(&mut z, &mut Vec::new()); + let overlap = OVERLAP; + let base = overlap >> 1; + let y = &mut out[base..base + n2]; + for i in 0..n4.div_ceil(2) { + let front = z[i]; + let back = z[n4 - 1 - i]; + let (re, im) = (front.im, front.re); + let (t0, t1) = (t[i], t[n4 + i]); + let yr0 = re * t0 + im * t1; + let yi0 = re * t1 - im * t0; + let (re, im) = (back.im, back.re); + let (t0, t1) = (t[n4 - i - 1], t[n2 - i - 1]); + let yr1 = re * t0 + im * t1; + let yi1 = re * t1 - im * t0; + y[2 * i] = yr0; + y[n2 - 1 - 2 * i] = yi0; + y[n2 - 2 - 2 * i] = yr1; + y[2 * i + 1] = yi1; + } + for i in 0..overlap / 2 { + let x1 = out[overlap - 1 - i]; + let x2 = out[i]; + let w1 = window[i]; + let w2 = window[overlap - 1 - i]; + out[i] = x2 * w2 - x1 * w1; + out[overlap - 1 - i] = x2 * w1 + x1 * w2; + } + } + + /// MDCT analysis followed by the decoder's IMDCT + TDAC overlap-add reproduces the input + /// delayed by exactly `OVERLAP` samples. + #[test] + fn perfect_reconstruction_through_decoder_imdct() { + let n = 960; + let frames = 5; + let sig: Vec = (0..n * frames).map(|i| (i as f32 * 0.013).sin() * 1000.0 + ((i * 7919) % 211) as f32 - 105.0).collect(); + let mut m = Mdct::new(n); + let mut hist = vec![0f32; OVERLAP]; + // Decoder memory: [frame output n][folded tail OVERLAP/2] laid out like decode_mem. + let mut mem = vec![0f32; n + OVERLAP]; + let mut decoded = Vec::new(); + for f in 0..frames { + let mut block = hist.clone(); + block.extend_from_slice(&sig[f * n..(f + 1) * n]); + hist.copy_from_slice(&block[n..]); + let mut coefs = vec![0f32; n]; + m.forward(&block, &mut coefs); + // Shift: the previous folded tail (at n..n+OVERLAP/2) moves to 0..OVERLAP/2. + mem.copy_within(n..n + OVERLAP / 2, 0); + imdct_backward(2 * n, &coefs, &mut mem, &m.window); + decoded.extend_from_slice(&mem[..n]); + } + let mut err = 0f64; + let mut sum = 0f64; + for t in OVERLAP..n * frames { + let d = decoded[t] as f64 - sig[t - OVERLAP] as f64; + err += d * d; + sum += (sig[t - OVERLAP] as f64).powi(2); + } + let snr = 10.0 * (sum / err).log10(); + assert!(snr > 80.0, "reconstruction SNR {snr:.1} dB"); + } +} diff --git a/crates/opusenc/src/range.rs b/crates/opusenc/src/range.rs new file mode 100644 index 00000000..daedd58a --- /dev/null +++ b/crates/opusenc/src/range.rs @@ -0,0 +1,472 @@ +//! Range encoder (RFC 6716 §5.1), the exact inverse of the §4.1 range decoder. +//! +//! Range-coded symbols are written from the front of the buffer and raw bits from the back; the +//! bit accounting ([`RangeEncoder::tell`], [`RangeEncoder::tell_frac`]) matches the decoder's, so +//! every budget decision the decoder takes can be mirrored exactly. + +const SYM_BITS: u32 = 8; +const CODE_BITS: u32 = 32; +const SYM_MAX: u32 = (1 << SYM_BITS) - 1; +const CODE_SHIFT: u32 = CODE_BITS - SYM_BITS - 1; +const CODE_TOP: u32 = 1 << (CODE_BITS - 1); +const CODE_BOT: u32 = CODE_TOP >> SYM_BITS; +/// Bit resolution of [`RangeEncoder::tell_frac`] (1/8 bit). +pub const BITRES: u32 = 3; + +/// Number of bits needed to represent `x` (0 for 0). +#[inline] +pub fn ilog(x: u32) -> i32 { + (32 - x.leading_zeros()) as i32 +} + +#[derive(Clone, Debug)] +pub struct RangeEncoder { + out: Vec, + end: Vec, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + low: u32, + rem: i32, + ext: u32, +} + +impl Default for RangeEncoder { + fn default() -> Self { + Self::new() + } +} + +impl RangeEncoder { + pub fn new() -> Self { + RangeEncoder { + out: Vec::new(), + end: Vec::new(), + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1) as i32, + rng: CODE_TOP, + low: 0, + rem: -1, + ext: 0, + } + } + + /// Current range (equals the decoder's range after the same symbols). + #[cfg(test)] + pub fn rng(&self) -> u32 { + self.rng + } + + fn carry_out(&mut self, c: u32) { + if c != SYM_MAX { + let carry = c >> SYM_BITS; + if self.rem >= 0 { + self.out.push((self.rem as u32 + carry) as u8); + } + if self.ext > 0 { + let sym = (SYM_MAX + carry) & SYM_MAX; + for _ in 0..self.ext { + self.out.push(sym as u8); + } + self.ext = 0; + } + self.rem = (c & SYM_MAX) as i32; + } else { + self.ext += 1; + } + } + + #[inline] + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.carry_out(self.low >> CODE_SHIFT); + self.low = (self.low << SYM_BITS) & (CODE_TOP - 1); + self.rng <<= SYM_BITS; + self.nbits_total += SYM_BITS as i32; + } + } + + /// Encodes the symbol with cumulative range `[fl, fh)` of total `ft`. + pub fn encode(&mut self, fl: u32, fh: u32, ft: u32) { + let r = self.rng / ft; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Like [`Self::encode`] with `ft = 1 << bits`. + pub fn encode_bin(&mut self, fl: u32, fh: u32, bits: u32) { + let r = self.rng >> bits; + let ft = 1u32 << bits; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Encodes a binary symbol whose probability of being 1 is `1/2^logp`. + pub fn bit_logp(&mut self, val: bool, logp: u32) { + let r = self.rng; + let s = r >> logp; + let r2 = r - s; + if val { + self.low += r2; + self.rng = s; + } else { + self.rng = r2; + } + self.normalize(); + } + + /// Encodes symbol `s` with an "inverse CDF" table (`ft = 1 << ftb`). + pub fn icdf(&mut self, s: usize, icdf: &[u8], ftb: u32) { + let r = self.rng >> ftb; + if s > 0 { + self.low += self.rng - r * icdf[s - 1] as u32; + self.rng = r * (icdf[s - 1] as u32 - icdf[s] as u32); + } else { + self.rng -= r * icdf[s] as u32; + } + self.normalize(); + } + + /// Encodes a uniformly distributed integer `fl` in `[0, ft)` (`ft >= 2`). + pub fn uint(&mut self, fl: u32, ft: u32) { + debug_assert!(ft > 1 && fl < ft); + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let v = fl >> ftb; + self.encode(v, v + 1, top); + self.bits(fl & ((1 << ftb) - 1), ftb); + } else { + self.encode(fl, fl + 1, ft); + } + } + + /// Writes `bits` raw bits (0..=25) to the end of the buffer. + pub fn bits(&mut self, val: u32, bits: u32) { + if bits == 0 { + return; + } + let mut window = self.end_window; + let mut used = self.nend_bits; + if used as u32 + bits > CODE_BITS { + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + } + window |= val << used; + used += bits as i32; + self.end_window = window; + self.nend_bits = used; + self.nbits_total += bits as i32; + } + + /// Number of bits written so far (rounded up), identical to the decoder's `tell`. + #[inline] + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + + /// Number of 1/8 bits written so far (rounded up), identical to the decoder's `tell_frac`. + pub fn tell_frac(&self) -> i32 { + let nbits = (self.nbits_total as u32) << BITRES; + let mut l = ilog(self.rng) as u32; + let mut r = self.rng >> (l - 16); + for _ in 0..BITRES { + r = (r * r) >> 15; + let b = r >> 16; + l = (l << 1) | b; + r >>= b; + } + nbits.wrapping_sub(l) as i32 + } + + /// Finalises the stream into a buffer of exactly `size` bytes (range data at the front, raw + /// bits at the back, zero padding in between; a partial raw-bit byte is merged into the byte + /// just before the full raw-bit bytes). + pub fn done(mut self, size: usize) -> Vec { + let mut l = CODE_BITS as i32 - ilog(self.rng); + let mut msk = (CODE_TOP - 1) >> l; + let mut end = (self.low.wrapping_add(msk)) & !msk; + if (end | msk) >= self.low.wrapping_add(self.rng) { + l += 1; + msk >>= 1; + end = (self.low.wrapping_add(msk)) & !msk; + } + while l > 0 { + self.carry_out(end >> CODE_SHIFT); + end = (end << SYM_BITS) & (CODE_TOP - 1); + l -= SYM_BITS as i32; + } + if self.rem >= 0 || self.ext > 0 { + self.carry_out(0); + } + let mut window = self.end_window; + let mut used = self.nend_bits; + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + let mut buf = vec![0u8; size]; + let full = self.end.len().min(size); + let room = size - full; + let n = self.out.len().min(room); + buf[..n].copy_from_slice(&self.out[..n]); + for (k, &b) in self.end.iter().take(full).enumerate() { + buf[size - 1 - k] = b; + } + if used > 0 && room > 0 { + buf[room - 1] |= window as u8; + } + buf + } +} + +#[cfg(test)] +pub(crate) mod dec { + //! Minimal range decoder (RFC 6716 §4.1) used to verify the encoder. + use super::*; + const CODE_EXTRA: u32 = (CODE_BITS - 2) % SYM_BITS + 1; + + pub struct RangeDecoder<'a> { + buf: &'a [u8], + offs: usize, + end_offs: usize, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + val: u32, + rem: u32, + } + + impl<'a> RangeDecoder<'a> { + pub fn new(buf: &'a [u8]) -> Self { + let mut d = RangeDecoder { + buf, + offs: 0, + end_offs: 0, + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1 - ((CODE_BITS - CODE_EXTRA) / SYM_BITS) * SYM_BITS) as i32, + rng: 1 << CODE_EXTRA, + val: 0, + rem: 0, + }; + d.rem = d.read_byte(); + d.val = d.rng - 1 - (d.rem >> (SYM_BITS - CODE_EXTRA)); + d.normalize(); + d + } + fn read_byte(&mut self) -> u32 { + if self.offs < self.buf.len() { + self.offs += 1; + self.buf[self.offs - 1] as u32 + } else { + 0 + } + } + fn read_byte_from_end(&mut self) -> u32 { + if self.end_offs < self.buf.len() { + self.end_offs += 1; + self.buf[self.buf.len() - self.end_offs] as u32 + } else { + 0 + } + } + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.nbits_total += SYM_BITS as i32; + self.rng <<= SYM_BITS; + let sym = self.rem; + self.rem = self.read_byte(); + let sym = ((sym << SYM_BITS) | self.rem) >> (SYM_BITS - CODE_EXTRA); + self.val = ((self.val << SYM_BITS).wrapping_add(SYM_MAX & !sym)) & (CODE_TOP - 1); + } + } + pub fn rng(&self) -> u32 { + self.rng + } + pub fn decode(&mut self, ft: u32) -> u32 { + let ext = self.rng / ft; + let s = self.val / ext; + ft - (s + 1).min(ft) + } + pub fn update(&mut self, fl: u32, fh: u32, ft: u32) { + let ext = self.rng / ft; + let s = ext * (ft - fh); + self.val = self.val.wrapping_sub(s); + self.rng = if fl > 0 { ext * (fh - fl) } else { self.rng - s }; + self.normalize(); + } + pub fn bit_logp(&mut self, logp: u32) -> bool { + let r = self.rng; + let d = self.val; + let s = r >> logp; + let ret = d < s; + if !ret { + self.val = d - s; + } + self.rng = if ret { s } else { r - s }; + self.normalize(); + ret + } + pub fn icdf(&mut self, icdf: &[u8], ftb: u32) -> usize { + let mut s = self.rng; + let d = self.val; + let r = s >> ftb; + let mut ret = 0usize; + let mut t; + loop { + t = s; + s = r * icdf[ret] as u32; + if d >= s { + break; + } + ret += 1; + } + self.val = d - s; + self.rng = t - s; + self.normalize(); + ret + } + pub fn uint(&mut self, ft: u32) -> u32 { + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let s = self.decode(top); + self.update(s, s + 1, top); + (s << ftb) | self.bits(ftb) + } else { + let s = self.decode(ft); + self.update(s, s + 1, ft); + s + } + } + pub fn bits(&mut self, bits: u32) -> u32 { + let mut window = self.end_window; + let mut available = self.nend_bits; + if (available as u32) < bits { + loop { + window |= self.read_byte_from_end() << available; + available += SYM_BITS as i32; + if available > (CODE_BITS - SYM_BITS) as i32 { + break; + } + } + } + let ret = window & ((1u32 << bits) - 1); + self.end_window = window >> bits; + self.nend_bits = available - bits as i32; + self.nbits_total += bits as i32; + ret + } + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + } +} + +#[cfg(test)] +mod tests { + use super::dec::RangeDecoder; + use super::*; + + struct Lcg(u32); + impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(1664525).wrapping_add(1013904223); + self.0 >> 8 + } + } + + #[test] + fn starts_at_one_bit() { + let e = RangeEncoder::new(); + assert_eq!(e.tell(), 1); + assert_eq!(e.tell_frac(), 8); + } + + #[test] + fn roundtrip_mixed_symbols_and_tell_matches() { + let mut rng = Lcg(7); + let icdf: [u8; 4] = [200, 120, 30, 0]; + for trial in 0..300 { + let n = 1 + (rng.next() % 300) as usize; + let mut ops = Vec::new(); + for _ in 0..n { + match rng.next() % 5 { + 0 => { + let ft = 2 + rng.next() % 1000; + ops.push((0u8, rng.next() % ft, ft)); + } + 1 => { + let logp = 1 + rng.next() % 15; + ops.push((1, rng.next().is_multiple_of(5) as u32, logp)); + } + 2 => { + let bits = 1 + rng.next() % 24; + ops.push((2, rng.next() & ((1 << bits) - 1), bits)); + } + 3 => { + let ft = 2 + rng.next() % 100_000_000; + ops.push((3, rng.next() % ft, ft)); + } + _ => ops.push((4, rng.next() % 4, 0)), + } + } + let mut e = RangeEncoder::new(); + let mut tells = Vec::new(); + for &(k, v, p) in &ops { + match k { + 0 => e.encode(v, v + 1, p), + 1 => e.bit_logp(v != 0, p), + 2 => e.bits(v, p), + 3 => e.uint(v, p), + _ => e.icdf(v as usize, &icdf, 8), + } + tells.push(e.tell()); + } + // Exactly the number of bytes `tell` says were used must suffice. + let size = (e.tell() as usize).div_ceil(8); + let final_rng = e.rng(); + let buf = e.done(size); + assert_eq!(buf.len(), size); + let mut d = RangeDecoder::new(&buf); + for (i, &(k, v, p)) in ops.iter().enumerate() { + let got = match k { + 0 => { + let s = d.decode(p); + d.update(s, s + 1, p); + s + } + 1 => d.bit_logp(p) as u32, + 2 => d.bits(p), + 3 => d.uint(p), + _ => d.icdf(&icdf, 8) as u32, + }; + assert_eq!(got, v, "trial {trial} op {i} kind {k}"); + assert_eq!(d.tell(), tells[i], "tell mismatch trial {trial} op {i}"); + } + assert_eq!(d.rng(), final_rng); + } + } +} diff --git a/crates/opusenc/src/rate.rs b/crates/opusenc/src/rate.rs new file mode 100644 index 00000000..d8665bf1 --- /dev/null +++ b/crates/opusenc/src/rate.rs @@ -0,0 +1,561 @@ +//! Pulse cache and bit allocation (RFC 6716 §4.3.3, §4.3.4.1). +//! +//! The allocation must be bit-identical to the decoder's, so this mirrors FilmCraft's +//! spec-derived `filmcraft-opus` decoder (first-party, MIT OR Apache-2.0) step by step; the only +//! difference is that the encoder *writes* the band-skip, intensity and dual-stereo symbols. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::tables::*; + +const MAX_PSEUDO: i32 = 40; +const LOG_MAX_PSEUDO: i32 = 6; +pub const MAX_FINE_BITS: i32 = 8; +const FINE_OFFSET: i32 = 21; +pub const QTHETA_OFFSET: i32 = 4; +pub const QTHETA_OFFSET_TWOPHASE: i32 = 16; +const ALLOC_STEPS: i32 = 6; + +/// Number of pulses represented by pseudo-pulse index `i`. +pub fn get_pulses(i: i32) -> i32 { + if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) } +} + +/// Conservative (rounded up) log2 of `val` with `frac` fractional bits. +pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 { + let mut l = ilog(val); + if val & val.wrapping_sub(1) != 0 { + if l > 16 { + val = ((val - 1) >> (l - 16)) + 1; + } else { + val <<= 16 - l; + } + let mut l = (l - 1) << frac; + loop { + let b = (val >> 16) as i32; + l += b << frac; + val = (val + b as u32) >> b; + val = (val * val + 0x7FFF) >> 15; + if frac == 0 { + break; + } + frac -= 1; + } + l + (val > 0x8000) as i32 + } else { + l -= 1; + l << frac + } +} + +/// PVQ codebook size V(N, K) (saturating far above 2^32). +pub fn pvq_v(n: usize, k: usize) -> u128 { + if k == 0 { + return 1; + } + if n == 0 { + return 0; + } + let mut row = vec![0u128; k + 1]; + row[0] = 1; + for _ in 0..n { + let mut prev_diag = row[0]; + for kk in 1..=k { + let up = row[kk]; + let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag); + prev_diag = up; + row[kk] = v.min(u128::MAX >> 8); + } + row[0] = 1; + } + row[k] +} + +fn fits_in32(n: usize, k: i32) -> bool { + pvq_v(n, k as usize) <= u32::MAX as u128 +} + +/// Precomputed per-mode data. +pub struct Mode { + /// log2 of band width in 1/8 bits. + pub log_n: [i32; NB_EBANDS], + cache_index: Vec, + cache_bits: Vec, + caps: Vec, +} + +impl Mode { + pub fn new() -> Mode { + let mut log_n = [0i32; NB_EBANDS]; + for (i, l) in log_n.iter_mut().enumerate() { + *l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES); + } + let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32); + let mut m = Mode { log_n, cache_index, cache_bits, caps: Vec::new() }; + m.caps = compute_caps(&m); + m + } + + #[inline] + pub fn cache(&self, lm: i32, band: usize) -> &[u8] { + let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band]; + &self.cache_bits[idx.max(0) as usize..] + } + + pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 { + let cache = self.cache(lm, band); + let mut lo = 0i32; + let mut hi = cache[0] as i32; + let bits = bits - 1; + for _ in 0..LOG_MAX_PSEUDO { + let mid = (lo + hi + 1) >> 1; + if cache[mid as usize] as i32 >= bits { + hi = mid; + } else { + lo = mid; + } + } + let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 }; + if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi } + } + + pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 { + if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 } + } + + /// Per-band allocation caps for this frame (`init_caps`). + pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] { + let mut cap = [0i32; NB_EBANDS]; + for (i, cp) in cap.iter_mut().enumerate() { + let n = (EBANDS[i + 1] - EBANDS[i]) << lm; + *cp = (self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n >> 2; + } + cap + } +} + +fn compute_pulse_cache(lm_max: i32) -> (Vec, Vec) { + let nb = NB_EBANDS; + let rows = (lm_max + 2) as usize; + let mut index = vec![-1i16; nb * rows]; + let mut entries: Vec<(usize, i32, usize)> = Vec::new(); + let mut curr = 0usize; + for i in 0..rows { + for j in 0..nb { + let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize; + index[i * nb + j] = -1; + 'search: for k in 0..=i { + for nn in 0..nb { + if k == i && nn >= j { + break; + } + if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize { + index[i * nb + j] = index[k * nb + nn]; + break 'search; + } + } + } + if index[i * nb + j] == -1 && n != 0 { + let mut k = 0; + while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) { + k += 1; + } + index[i * nb + j] = curr as i16; + entries.push((n, k, curr)); + curr += (k + 1) as usize; + } + } + } + let mut bits = vec![0u8; curr]; + for &(n, k, off) in &entries { + let maxk = get_pulses(k) as usize; + let tmp = required_bits(n, maxk, BITRES); + for j in 1..=k { + bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8; + } + bits[off] = k as u8; + } + (index, bits) +} + +fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec { + let mut out = vec![0i32; maxk + 1]; + if n == 1 { + for o in out.iter_mut().skip(1) { + *o = 1 << frac; + } + } else { + for (k, o) in out.iter_mut().enumerate().skip(1) { + *o = log2_frac(pvq_v(n, k) as u32, frac); + } + } + out +} + +fn compute_caps(m: &Mode) -> Vec { + let nb = NB_EBANDS; + let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb]; + for i in 0..=MAX_LM as i32 { + for c in 1..=2i32 { + for j in 0..nb { + let mut n0 = EBANDS[j + 1] - EBANDS[j]; + let max_bits; + if n0 << i == 1 { + max_bits = c * (1 + MAX_FINE_BITS) << BITRES; + } else { + let mut lm0 = 0; + if n0 > 2 { + n0 >>= 1; + lm0 -= 1; + } else if n0 <= 1 { + lm0 = i.min(1); + n0 <<= lm0; + } + let pcache = m.cache(lm0, j); + let mut mb = pcache[pcache[0] as usize] as i32 + 1; + let mut n = n0; + for k in 0..(i - lm0) { + mb <<= 1; + let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET; + let num = 459 * ((2 * n - 1) * offset + mb); + let den = ((2 * n - 1) << 9) - 459; + let qb = ((num + (den >> 1)) / den).min(57); + mb += qb; + n <<= 1; + } + if c == 2 { + mb <<= 1; + let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET }; + let ndof = 2 * n - 1 - (n == 2) as i32; + let p = if n == 2 { 512 } else { 487 }; + let num = p * (mb + ndof * offset); + let den = (ndof << 9) - p; + let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 }); + mb += qb; + } + let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 }; + let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET; + if n == 2 { + offset += (1 << BITRES) >> 2; + } + let num = mb + ndof * offset; + let den = (ndof - 1) << BITRES; + let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS); + mb += c * qb << BITRES; + max_bits = mb; + } + let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i); + let v = (4 * max_bits / w) - 64; + caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8; + } + } + } + caps +} + +/// Result of the allocation process. +pub struct Allocation { + pub coded_bands: usize, + pub intensity: usize, + pub dual_stereo: bool, + pub balance: i32, + pub pulses: [i32; NB_EBANDS], + pub fine_quant: [i32; NB_EBANDS], + pub fine_priority: [i32; NB_EBANDS], +} + +/// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo +/// with no intensity bands, and no band is skipped voluntarily. +#[allow(clippy::too_many_arguments)] +pub fn compute_allocation( + m: &Mode, + start: usize, + end: usize, + offsets: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + alloc_trim: i32, + total: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let lmi = lm as i32; + let mut total = total.max(0); + let mut skip_start = start; + let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= skip_rsv; + let mut intensity_rsv = 0; + let mut dual_stereo_rsv = 0; + if c == 2 { + intensity_rsv = LOG2_FRAC_TABLE[end - start]; + if intensity_rsv > total { + intensity_rsv = 0; + } else { + total -= intensity_rsv; + dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= dual_stereo_rsv; + } + } + let mut bits1 = [0i32; NB_EBANDS]; + let mut bits2 = [0i32; NB_EBANDS]; + let mut thresh = [0i32; NB_EBANDS]; + let mut trim_offset = [0i32; NB_EBANDS]; + for j in start..end { + let w = EBANDS[j + 1] - EBANDS[j]; + thresh[j] = (ci << BITRES).max((3 * w << lmi << BITRES) >> 4); + trim_offset[j] = ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32)) >> 6; + if w << lmi == 1 { + trim_offset[j] -= ci << BITRES; + } + } + let nb_alloc = ALLOC.len() as i32; + let mut lo = 1; + let mut hi = nb_alloc - 1; + loop { + let mut done = false; + let mut psum = 0; + let mid = (lo + hi) >> 1; + for j in (start..end).rev() { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bitsj = ci * n * (ALLOC[mid as usize][j] as i32) << lmi >> 2; + if bitsj > 0 { + bitsj = (bitsj + trim_offset[j]).max(0); + } + bitsj += offsets[j]; + if bitsj >= thresh[j] || done { + done = true; + psum += bitsj.min(cap[j]); + } else if bitsj >= ci << BITRES { + psum += ci << BITRES; + } + } + if psum > total { + hi = mid - 1; + } else { + lo = mid + 1; + } + if lo > hi { + break; + } + } + hi = lo; + lo -= 1; + for j in start..end { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bits1j = ci * n * (ALLOC[lo as usize][j] as i32) << lmi >> 2; + let mut bits2j = if hi >= nb_alloc { cap[j] } else { ci * n * (ALLOC[hi as usize][j] as i32) << lmi >> 2 }; + if bits1j > 0 { + bits1j = (bits1j + trim_offset[j]).max(0); + } + if bits2j > 0 { + bits2j = (bits2j + trim_offset[j]).max(0); + } + if lo > 0 { + bits1j += offsets[j]; + } + bits2j += offsets[j]; + if offsets[j] > 0 { + skip_start = j; + } + bits2j = (bits2j - bits1j).max(0); + bits1[j] = bits1j; + bits2[j] = bits2j; + } + interp_bits2pulses(m, start, end, skip_start, &bits1, &bits2, &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv, c, lm, enc) +} + +#[allow(clippy::too_many_arguments)] +fn interp_bits2pulses( + m: &Mode, + start: usize, + end: usize, + skip_start: usize, + bits1: &[i32; NB_EBANDS], + bits2: &[i32; NB_EBANDS], + thresh: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + mut total: i32, + skip_rsv: i32, + mut intensity_rsv: i32, + mut dual_stereo_rsv: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let alloc_floor = ci << BITRES; + let stereo = (c > 1) as i32; + let log_m = (lm as i32) << BITRES; + let mut lo = 0; + let mut hi = 1 << ALLOC_STEPS; + for _ in 0..ALLOC_STEPS { + let mid = (lo + hi) >> 1; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let tmp = bits1[j] + (mid * bits2[j] >> ALLOC_STEPS); + if tmp >= thresh[j] || done { + done = true; + psum += tmp.min(cap[j]); + } else if tmp >= alloc_floor { + psum += alloc_floor; + } + } + if psum > total { + hi = mid; + } else { + lo = mid; + } + } + let mut bits = [0i32; NB_EBANDS]; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let mut tmp = bits1[j] + (lo * bits2[j] >> ALLOC_STEPS); + if tmp < thresh[j] && !done { + tmp = if tmp >= alloc_floor { alloc_floor } else { 0 }; + } else { + done = true; + } + tmp = tmp.min(cap[j]); + bits[j] = tmp; + psum += tmp; + } + let s = start; + let mut coded_bands = end; + loop { + let j = coded_bands - 1; + if j <= skip_start { + total += skip_rsv; + break; + } + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + let rem = (left - (EBANDS[j] - EBANDS[s])).max(0); + let band_width = EBANDS[coded_bands] - EBANDS[j]; + let band_bits = bits[j] + percoeff * band_width + rem; + if band_bits >= thresh[j].max(alloc_floor + (1 << BITRES)) { + // Keep this band (and every band below it). + enc.bit_logp(true, 1); + break; + } + psum -= bits[j] + intensity_rsv; + if intensity_rsv > 0 { + intensity_rsv = LOG2_FRAC_TABLE[j - s]; + } + psum += intensity_rsv; + if band_bits >= alloc_floor { + psum += alloc_floor; + bits[j] = alloc_floor; + } else { + bits[j] = 0; + } + coded_bands -= 1; + } + let intensity = if intensity_rsv > 0 { + // No intensity stereo: every coded band is coded in full. + enc.uint((coded_bands - s) as u32, (coded_bands + 1 - s) as u32); + coded_bands + } else { + 0 + }; + if intensity <= s { + total += dual_stereo_rsv; + dual_stereo_rsv = 0; + } + let dual_stereo = if dual_stereo_rsv > 0 { + enc.bit_logp(true, 1); + true + } else { + false + }; + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + for j in s..coded_bands { + bits[j] += percoeff * (EBANDS[j + 1] - EBANDS[j]); + } + for j in s..coded_bands { + let tmp = left.min(EBANDS[j + 1] - EBANDS[j]); + bits[j] += tmp; + left -= tmp; + } + let mut ebits = [0i32; NB_EBANDS]; + let mut fine_priority = [0i32; NB_EBANDS]; + let mut balance = 0; + for j in s..coded_bands { + let n0 = EBANDS[j + 1] - EBANDS[j]; + let n = n0 << lm; + let bit = bits[j] + balance; + let mut excess; + if n > 1 { + excess = (bit - cap[j]).max(0); + bits[j] = bit - excess; + let den = ci * n + if c == 2 && n > 2 && !dual_stereo && j < intensity { 1 } else { 0 }; + let nclogn = den * (m.log_n[j] + log_m); + let mut offset = (nclogn >> 1) - den * FINE_OFFSET; + if n == 2 { + offset += (den << BITRES) >> 2; + } + if bits[j] + offset < den * 2 << BITRES { + offset += nclogn >> 2; + } else if bits[j] + offset < den * 3 << BITRES { + offset += nclogn >> 3; + } + let mut eb = (bits[j] + offset + (den << (BITRES - 1))).max(0); + eb = (eb / den) >> BITRES; + if ci * eb > (bits[j] >> BITRES) { + eb = bits[j] >> stereo >> BITRES; + } + eb = eb.min(MAX_FINE_BITS); + ebits[j] = eb; + fine_priority[j] = (eb * (den << BITRES) >= bits[j] + offset) as i32; + bits[j] -= ci * eb << BITRES; + } else { + excess = (bit - (ci << BITRES)).max(0); + bits[j] = bit - excess; + ebits[j] = 0; + fine_priority[j] = 1; + } + if excess > 0 { + let extra_fine = (excess >> (stereo + BITRES as i32)).min(MAX_FINE_BITS - ebits[j]); + ebits[j] += extra_fine; + let extra_bits = extra_fine * ci << BITRES; + fine_priority[j] = (extra_bits >= excess - balance) as i32; + excess -= extra_bits; + } + balance = excess; + } + for j in coded_bands..end { + ebits[j] = bits[j] >> stereo >> BITRES; + bits[j] = 0; + fine_priority[j] = (ebits[j] < 1) as i32; + } + Allocation { coded_bands, intensity, dual_stereo, balance, pulses: bits, fine_quant: ebits, fine_priority } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caps_match_decoder_reference() { + let m = Mode::new(); + let first_rows: [u8; 42] = [ + 224, 224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 185, 185, 185, 178, 178, 168, 134, 61, 37, 224, 224, 224, 224, 224, 224, 224, 224, 240, + 240, 240, 240, 207, 207, 207, 198, 198, 183, 144, 66, 40, + ]; + assert_eq!(&m.caps[..42], &first_rows[..]); + } + + #[test] + fn pvq_counts() { + assert_eq!(pvq_v(5, 1), 10); + assert_eq!(pvq_v(1, 7), 2); + assert_eq!(pvq_v(2, 9), 36); + assert_eq!(pvq_v(3, 2), 18); + } +} diff --git a/crates/opusenc/src/tables.rs b/crates/opusenc/src/tables.rs new file mode 100644 index 00000000..00d623af --- /dev/null +++ b/crates/opusenc/src/tables.rs @@ -0,0 +1,122 @@ +//! Constant tables of the 48 kHz CELT mode (RFC 6716 §4.3, Tables 55–58 and §4.3.2). +//! +//! Transcribed from RFC 6716; copied from FilmCraft's spec-derived `filmcraft-opus` decoder +//! (first-party, MIT OR Apache-2.0) so both sides share the exact same values. + +/// Number of energy bands. +pub const NB_EBANDS: usize = 21; +/// Band edges in units of 2.5 ms MDCT bins (Table 55). +pub const EBANDS: [i32; NB_EBANDS + 1] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100]; +/// Overlap of the low-overlap window (samples at 48 kHz). +pub const OVERLAP: usize = 120; +/// Largest LM (20 ms frames). +pub const MAX_LM: usize = 3; +/// Pre-emphasis coefficient. +pub const PREEMPH: f32 = 0.850_006_1; + +/// Static allocation table (Table 57), indexed `[quality][band]`, in 1/32 bit per MDCT bin. +pub const ALLOC: [[u8; NB_EBANDS]; 11] = transpose([ + [0, 90, 110, 118, 126, 134, 144, 152, 162, 172, 200], + [0, 80, 100, 110, 119, 127, 137, 145, 155, 165, 200], + [0, 75, 90, 103, 112, 120, 130, 138, 148, 158, 200], + [0, 69, 84, 93, 104, 114, 124, 132, 142, 152, 200], + [0, 63, 78, 86, 95, 103, 113, 123, 133, 143, 200], + [0, 56, 71, 80, 89, 97, 107, 117, 127, 137, 200], + [0, 49, 65, 75, 83, 91, 101, 111, 121, 131, 200], + [0, 40, 58, 70, 78, 85, 95, 105, 115, 125, 200], + [0, 34, 51, 65, 72, 78, 88, 98, 108, 118, 198], + [0, 29, 45, 59, 66, 72, 82, 92, 102, 112, 193], + [0, 20, 39, 53, 60, 66, 76, 86, 96, 106, 188], + [0, 18, 32, 47, 54, 60, 70, 80, 90, 100, 183], + [0, 10, 26, 40, 47, 54, 64, 74, 84, 94, 178], + [0, 0, 20, 31, 39, 47, 57, 67, 77, 87, 173], + [0, 0, 12, 23, 32, 41, 51, 61, 71, 81, 168], + [0, 0, 0, 15, 25, 35, 45, 55, 65, 75, 163], + [0, 0, 0, 4, 17, 29, 39, 49, 59, 69, 158], + [0, 0, 0, 0, 12, 23, 33, 43, 53, 63, 153], + [0, 0, 0, 0, 1, 16, 26, 36, 46, 56, 148], + [0, 0, 0, 0, 0, 10, 15, 20, 30, 45, 129], + [0, 0, 0, 0, 0, 1, 1, 1, 1, 20, 104], +]); + +const fn transpose(t: [[u8; 11]; NB_EBANDS]) -> [[u8; NB_EBANDS]; 11] { + let mut out = [[0u8; NB_EBANDS]; 11]; + let mut b = 0; + while b < NB_EBANDS { + let mut q = 0; + while q < 11 { + out[q][b] = t[b][q]; + q += 1; + } + b += 1; + } + out +} + +/// Mean band energy (log2 domain) removed before coarse quantisation. +pub const E_MEANS: [f32; 25] = [ + 6.4375, 6.25, 5.75, 5.3125, 5.0625, 4.8125, 4.5, 4.375, 4.875, 4.6875, 4.5625, 4.4375, 4.875, 4.625, 4.3125, 4.5, 4.375, 4.625, 4.75, 4.4375, 3.75, 3.75, + 3.75, 3.75, 3.75, +]; + +/// Inter-frame prediction coefficient (alpha) per LM, Q15. +pub const PRED_COEF: [f32; 4] = [29440.0 / 32768.0, 26112.0 / 32768.0, 21248.0 / 32768.0, 16384.0 / 32768.0]; +/// Inter-band prediction coefficient (beta) per LM for inter frames, Q15. +pub const BETA_COEF: [f32; 4] = [30147.0 / 32768.0, 22282.0 / 32768.0, 12124.0 / 32768.0, 6554.0 / 32768.0]; +/// Beta for intra frames. +pub const BETA_INTRA: f32 = 4915.0 / 32768.0; + +/// Laplace model parameters for coarse energy: `[LM][intra][2*band + {0: prob, 1: decay}]`. +pub const E_PROB_MODEL: [[[u8; 42]; 2]; 4] = [ + [ + [ + 72, 127, 65, 129, 66, 128, 65, 128, 64, 128, 62, 128, 64, 128, 64, 128, 92, 78, 92, 79, 92, 78, 90, 79, 116, 41, 115, 40, 114, 40, 132, 26, 132, + 26, 145, 17, 161, 12, 176, 10, 177, 11, + ], + [ + 24, 179, 48, 138, 54, 135, 54, 132, 53, 134, 56, 133, 55, 132, 55, 132, 61, 114, 70, 96, 74, 88, 75, 88, 87, 74, 89, 66, 91, 67, 100, 59, 108, 50, + 120, 40, 122, 37, 97, 43, 78, 50, + ], + ], + [ + [ + 83, 78, 84, 81, 88, 75, 86, 74, 87, 71, 90, 73, 93, 74, 93, 74, 109, 40, 114, 36, 117, 34, 117, 34, 143, 17, 145, 18, 146, 19, 162, 12, 165, 10, + 178, 7, 189, 6, 190, 8, 177, 9, + ], + [ + 23, 178, 54, 115, 63, 102, 66, 98, 69, 99, 74, 89, 71, 91, 73, 91, 78, 89, 86, 80, 92, 66, 93, 64, 102, 59, 103, 60, 104, 60, 117, 52, 123, 44, + 138, 35, 133, 31, 97, 38, 77, 45, + ], + ], + [ + [ + 61, 90, 93, 60, 105, 42, 107, 41, 110, 45, 116, 38, 113, 38, 112, 38, 124, 26, 132, 27, 136, 19, 140, 20, 155, 14, 159, 16, 158, 18, 170, 13, 177, + 10, 187, 8, 192, 6, 175, 9, 159, 10, + ], + [ + 21, 178, 59, 110, 71, 86, 75, 85, 84, 83, 91, 66, 88, 73, 87, 72, 92, 75, 98, 72, 105, 58, 107, 54, 115, 52, 114, 55, 112, 56, 129, 51, 132, 40, + 150, 33, 140, 29, 98, 35, 77, 42, + ], + ], + [ + [ + 42, 121, 96, 66, 108, 43, 111, 40, 117, 44, 123, 32, 120, 36, 119, 33, 127, 33, 134, 34, 139, 21, 147, 23, 152, 20, 158, 25, 154, 26, 166, 21, 173, + 16, 184, 13, 184, 10, 150, 13, 139, 15, + ], + [ + 22, 178, 63, 114, 74, 82, 84, 83, 92, 82, 103, 62, 96, 72, 96, 67, 101, 73, 107, 72, 113, 55, 118, 52, 125, 52, 118, 52, 117, 55, 135, 49, 137, 39, + 157, 32, 145, 29, 97, 33, 77, 40, + ], + ], +]; + +/// Inverse CDF for coarse energy when only a few bits remain. +pub const SMALL_ENERGY_ICDF: [u8; 3] = [2, 1, 0]; +/// Conservative log2 in 1/8 bits of the number of coded bands (intensity reservation). +pub const LOG2_FRAC_TABLE: [i32; 24] = [0, 8, 13, 16, 19, 21, 23, 24, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 34, 35, 36, 36, 37, 37]; +/// TF resolution changes: `[LM][4*transient + 2*tf_select + tf_flag]`. +pub const TF_SELECT_TABLE: [[i8; 8]; 4] = [[0, -1, 0, -1, 0, -1, 0, -1], [0, -1, 0, -2, 1, 0, 1, -1], [0, -2, 0, -3, 2, 0, 1, -1], [0, -2, 0, -3, 3, 0, 1, -1]]; +/// Spreading decision `{7, 2, 21, 2}/32`. +pub const SPREAD_ICDF: [u8; 4] = [25, 23, 2, 0]; +/// Allocation trim (Table 58). +pub const TRIM_ICDF: [u8; 11] = [126, 124, 119, 109, 87, 41, 19, 9, 4, 2, 0]; diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb..c3d869c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,6 +16,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 76ff6fa3..0cef3307 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -24,6 +24,7 @@ const LAYERS: &[(&str, u8)] = &[ ("geom", 0), ("color", 0), ("testkit", 0), + ("opusenc", 0), ("raster", 1), ("keyframe", 1), ("path", 1), From bab4f88c22e525ac69474a8293b155fe70fccab4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:44:51 -0400 Subject: [PATCH 2/6] M12.6: camera tracker solve Structure from motion for the 3D Camera Tracker in effectcraft-track (camtrack): long KLT feature tracks with re-detection, parallax keyframes, two-view initialisation (normalised 8-point essential matrix in RANSAC or planar homography decomposition), incremental resection + triangulation, sparse Levenberg-Marquardt bundle adjustment with the Schur complement and Huber loss, focal length search (Fixed / Variable Zoom / Specify Angle of View), tripod pan solve, Auto Detect, ground plane / target / world mapping. Synthetic-scene tests (path, focal, error, tripod, flat, rendered frames). Co-Authored-By: Claude Opus 5.5 --- crates/track/Cargo.toml | 2 +- crates/track/src/camtrack/bundle.rs | 416 ++++++++++++ crates/track/src/camtrack/features.rs | 115 ++++ crates/track/src/camtrack/geometry.rs | 432 +++++++++++++ crates/track/src/camtrack/linalg.rs | 476 ++++++++++++++ crates/track/src/camtrack/mod.rs | 376 +++++++++++ crates/track/src/camtrack/solver.rs | 894 ++++++++++++++++++++++++++ crates/track/src/camtrack/tests.rs | 360 +++++++++++ crates/track/src/lib.rs | 1 + 9 files changed, 3071 insertions(+), 1 deletion(-) create mode 100644 crates/track/src/camtrack/bundle.rs create mode 100644 crates/track/src/camtrack/features.rs create mode 100644 crates/track/src/camtrack/geometry.rs create mode 100644 crates/track/src/camtrack/linalg.rs create mode 100644 crates/track/src/camtrack/mod.rs create mode 100644 crates/track/src/camtrack/solver.rs create mode 100644 crates/track/src/camtrack/tests.rs diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92..9bd46e26 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 00000000..84e8a5ad --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,416 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + match ps.w.iter_mut().find(|e| e.0 == *ia) { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points. + let mut vinv: Vec> = vec![None; npts]; + if !o.fix_points { + for (j, ps) in systems.iter().enumerate() { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + let Some(vi) = inv3(&v) else { continue }; + vinv[j] = Some(vi); + let vb = mv(&vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w { + s[ia * n + ib] -= dot(wv[a], *wb); + } + } + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-10 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 00000000..311ec711 --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 00000000..6956ac2b --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 00000000..a7067f26 --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,476 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j are independent. + use rayon::prelude::*; + let work = |row: &mut [f64]| { + let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); + row[j] = s / d; + }; + if n - j > 192 { + tail.par_chunks_mut(n).for_each(work); + } else { + tail.chunks_mut(n).for_each(work); + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 00000000..056d7ad0 --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,376 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one. +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 00000000..da3a9290 --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,894 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels). + noise: f64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise + } + fn huber(&self) -> f64 { + 1.0 * self.noise + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared +/// features drop below 24. The first and last frames with features are keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + if try_tripod && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + if cancelled() { + return err("cancelled"); + } + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 00000000..258161f9 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,360 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5..bd4e21dc 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,6 +18,7 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; From 94e1c86d89a866771f80ad1610afda9aa12e85d8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:10:48 -0400 Subject: [PATCH 3/6] IO-2/SHP-5/PRV-2: PSD import, SVG import + vector/text conversions, persistent disk cache - effectcraft-psd (L1): PSD/PSB reader from Adobe's published file format spec (layers, groups, masks, blend modes, text engine data, lfx2 effects, fill/adjustment layers, vector masks; 1/8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer for fixtures. - File > Import: importAs footage|composition|compositionLayerSizes, layer (Choose Layer); groups -> precomps (pass-through collapses), text -> editable text, styles -> Layer Styles, adjustments -> adjustment layers with effects; UI asks the Import Kind for Photoshop files. - effectcraft-svg (L2): SVG parser (shapes, paths, transforms, use, CSS, gradients) and rasteriser; .svg imports as footage, continuously rasterised when the switch is on. - layer.create: shapesFromVector, shapesFromText, masksFromText, editableText (stub removed). - Disk cache (render::disk_cache): LZ4 content-keyed frames and layer buffers, LRU size limit from prefs, atomic writes, restart persistence; edit.purge clears it, cache.diskStats query; viewer frames read/write it and the timeline shows disk-only frames in blue. Co-Authored-By: Claude Opus 5.5 --- ATTRIBUTION.md | 2 + Cargo.lock | 44 + Cargo.toml | 2 + crates/engine/Cargo.toml | 3 + crates/engine/src/commands/create.rs | 139 +++ crates/engine/src/commands/edit.rs | 32 +- crates/engine/src/commands/file.rs | 98 +- crates/engine/src/commands/file_more.rs | 1 + crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 12 + crates/engine/src/psd_import.rs | 634 ++++++++++++ crates/engine/src/session_settings.rs | 37 + crates/engine/src/tests_disk_cache.rs | 121 +++ crates/engine/src/tests_stubs.rs | 1 + crates/engine/src/tests_track.rs | 2 + crates/engine/src/tests_vector_import.rs | 237 +++++ crates/engine/src/vector.rs | 250 +++++ crates/gpu/src/tests.rs | 1 + crates/host/Cargo.toml | 4 + crates/host/tests/vector_import.rs | 163 +++ crates/lottie/src/import.rs | 1 + crates/lottie/src/tests.rs | 1 + crates/media/Cargo.toml | 2 + crates/media/src/convert.rs | 17 + crates/media/src/layered.rs | 62 ++ crates/media/src/lib.rs | 6 +- crates/media/src/pool.rs | 16 + crates/media/src/probe.rs | 7 +- crates/project/src/lib.rs | 15 + crates/psd/Cargo.toml | 13 + crates/psd/README.md | 17 + crates/psd/src/descriptor.rs | 430 ++++++++ crates/psd/src/engine_data.rs | 288 ++++++ crates/psd/src/lib.rs | 965 ++++++++++++++++++ crates/psd/src/read.rs | 86 ++ crates/psd/src/tests.rs | 189 ++++ crates/psd/src/write.rs | 491 +++++++++ crates/render/Cargo.toml | 1 + crates/render/src/audio.rs | 1 + crates/render/src/cache.rs | 62 +- crates/render/src/disk_cache.rs | 829 +++++++++++++++ crates/render/src/lib.rs | 29 +- crates/render/src/tests_audio_fx.rs | 1 + crates/render/src/tests_color.rs | 1 + crates/render/src/tests_frame_blend.rs | 1 + crates/svg/Cargo.toml | 18 + crates/svg/README.md | 26 + crates/svg/src/color.rs | 254 +++++ crates/svg/src/lib.rs | 949 +++++++++++++++++ crates/svg/src/pathdata.rs | 293 ++++++ crates/svg/src/raster.rs | 220 ++++ crates/svg/src/tests.rs | 108 ++ crates/svg/tests/fixtures/basic-shapes.svg | 15 + .../fixtures/basic-shapes.svg.attribution | 7 + crates/svg/tests/fixtures/paths-gradients.svg | 26 + .../fixtures/paths-gradients.svg.attribution | 7 + crates/ui-egui/src/frames.rs | 108 +- crates/ui-egui/src/lib.rs | 18 +- crates/ui-egui/src/menus.rs | 6 +- crates/ui-egui/src/panels/forms.rs | 20 + crates/ui-egui/src/panels/timeline.rs | 25 + crates/ui-egui/src/theme.rs | 3 + xtask/src/main.rs | 2 + 64 files changed, 7388 insertions(+), 34 deletions(-) create mode 100644 crates/engine/src/commands/create.rs create mode 100644 crates/engine/src/psd_import.rs create mode 100644 crates/engine/src/tests_disk_cache.rs create mode 100644 crates/engine/src/tests_vector_import.rs create mode 100644 crates/engine/src/vector.rs create mode 100644 crates/host/tests/vector_import.rs create mode 100644 crates/media/src/layered.rs create mode 100644 crates/psd/Cargo.toml create mode 100644 crates/psd/README.md create mode 100644 crates/psd/src/descriptor.rs create mode 100644 crates/psd/src/engine_data.rs create mode 100644 crates/psd/src/lib.rs create mode 100644 crates/psd/src/read.rs create mode 100644 crates/psd/src/tests.rs create mode 100644 crates/psd/src/write.rs create mode 100644 crates/render/src/disk_cache.rs create mode 100644 crates/svg/Cargo.toml create mode 100644 crates/svg/README.md create mode 100644 crates/svg/src/color.rs create mode 100644 crates/svg/src/lib.rs create mode 100644 crates/svg/src/pathdata.rs create mode 100644 crates/svg/src/raster.rs create mode 100644 crates/svg/src/tests.rs create mode 100644 crates/svg/tests/fixtures/basic-shapes.svg create mode 100644 crates/svg/tests/fixtures/basic-shapes.svg.attribution create mode 100644 crates/svg/tests/fixtures/paths-gradients.svg create mode 100644 crates/svg/tests/fixtures/paths-gradients.svg.attribution diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 9caf1c27..d0b936b8 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -36,6 +36,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/basic-shapes.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (basic shapes, groups, transforms) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/paths-gradients.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (path data, gradients, CSS, use) | MIT OR Apache-2.0 | ## Screenshots diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..1d14bab9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1198,12 +1198,15 @@ dependencies = [ "effectcraft-lottie", "effectcraft-path", "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-text", "effectcraft-time", "effectcraft-track", "image", + "kurbo", "log", "rayon", "serde", @@ -1287,6 +1290,10 @@ dependencies = [ "effectcraft-expr", "effectcraft-media", "effectcraft-project", + "effectcraft-psd", + "effectcraft-raster", + "effectcraft-svg", + "effectcraft-time", "serde_json", ] @@ -1324,8 +1331,10 @@ name = "effectcraft-media" version = "0.1.0" dependencies = [ "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-time", "filmcraft-codecs", "filmcraft-color", @@ -1363,6 +1372,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-psd" +version = "0.1.0" +dependencies = [ + "miniz_oxide 0.8.9", +] + [[package]] name = "effectcraft-raster" version = "0.1.0" @@ -1387,11 +1403,24 @@ dependencies = [ "effectcraft-time", "kurbo", "log", + "lz4_flex", "rayon", "serde", "web-time", ] +[[package]] +name = "effectcraft-svg" +version = "0.1.0" +dependencies = [ + "effectcraft-geom", + "effectcraft-path", + "effectcraft-raster", + "kurbo", + "rayon", + "roxmltree", +] + [[package]] name = "effectcraft-text" version = "0.1.0" @@ -2749,6 +2778,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + [[package]] name = "mach2" version = "0.4.3" @@ -3970,6 +4005,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1941cf4..7aabc4af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,8 @@ effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } +effectcraft-psd = { path = "crates/psd" } +effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 132bdcac..0830c78d 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -22,6 +22,9 @@ effectcraft-effects = { workspace = true } effectcraft-render = { workspace = true } effectcraft-track = { workspace = true } effectcraft-lottie = { workspace = true } +effectcraft-psd = { workspace = true } +kurbo = { workspace = true } +effectcraft-svg = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/engine/src/commands/create.rs b/crates/engine/src/commands/create.rs new file mode 100644 index 00000000..72cb2420 --- /dev/null +++ b/crates/engine/src/commands/create.rs @@ -0,0 +1,139 @@ +//! Layer ▸ Create: Convert to Editable Text (Photoshop type layers), Create Shapes from Text, +//! Create Masks from Text and Create Shapes from Vector Layer (SVG footage). + +use effectcraft_color::Label; +use effectcraft_project::{ItemKind, LayerSource, Solid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, has_layers, layers_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +const OPS: &str = "editableText|shapesFromText|masksFromText|shapesFromVector"; + +fn create(s: &mut Session, p: &Value) -> Result { + let op = str_p(p, "op").ok_or_else(|| bad("layer.create", format!("missing `op` ({OPS})")))?; + let (cid, lids) = layers_p(s, p)?; + if lids.is_empty() { + return Err(bad("layer.create", "select a layer first")); + } + let time = s.time(); + let mut created = vec![]; + let mut skipped = vec![]; + for lid in lids { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let Some(src) = comp.layer(lid).cloned() else { continue }; + let footage = src.source.item().and_then(|i| match &s.project.item(i)?.kind { + ItemKind::Footage(f) => Some(f.clone()), + _ => None, + }); + match op { + "shapesFromVector" => { + let Some(f) = footage.filter(|f| f.codec == "SVG" || f.path.to_ascii_lowercase().ends_with(".svg")) else { + skipped.push(format!("{}: not a vector (SVG) footage layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let doc = effectcraft_svg::parse(&bytes).map_err(|e| EngineError::Other(format!("{}: {e}", f.path)))?; + let id = s.edit("Create Shapes from Vector Layer", None, |proj, st| { + let l = crate::vector::shapes_from_vector(proj, &comp, &src, &doc); + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "shapesFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let snapshot = s.project.clone(); + let id = s.edit("Create Shapes from Text", None, |proj, st| { + let l = crate::vector::shapes_from_text(proj, &comp, &snapshot, cid, &src, time) + .ok_or_else(|| bad("layer.create", "the text layer has no glyphs"))?; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "masksFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let outlines = crate::vector::text_outlines_in_comp(&s.project, cid, &comp, &src, time); + if outlines.is_empty() { + skipped.push(format!("{}: no glyphs", src.name)); + continue; + } + let id = s.edit("Create Masks from Text", None, |proj, st| { + let name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + &name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = crate::vector::masks_layer(proj, &comp, &name, outlines, sid); + l.in_point = src.in_point; + l.out_point = src.out_point; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "editableText" => { + let Some((f, sl)) = footage.and_then(|f| f.layer.clone().map(|l| (f, l))) else { + skipped.push(format!("{}: not a Photoshop layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let id = s.edit("Convert to Editable Text", None, |proj, st| { + let l = crate::psd_import::editable_text(proj, &comp, &src, &psd, sl.index as usize) + .ok_or_else(|| bad("layer.create", format!("{}: the Photoshop layer has no text", src.name)))?; + let id = l.id; + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == lid).ok_or(EngineError::NoComp)?; + c.layers[at] = l; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + other => return Err(bad("layer.create", format!("`{other}` is not available yet (supported: {OPS})"))), + } + } + if created.is_empty() && !skipped.is_empty() { + return Err(bad("layer.create", skipped.join("; "))); + } + Ok(json!({"layers": created, "skipped": skipped})) +} + +/// Insert `l` directly above layer `below`; optionally switch the source's video off (as After +/// Effects does after converting). +fn insert_above( + proj: &mut effectcraft_project::Project, + cid: effectcraft_project::ItemId, + below: effectcraft_project::LayerId, + l: effectcraft_project::Layer, + hide_source: bool, +) -> Result<()> { + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == below).unwrap_or(0); + if hide_source && let Some(src) = c.layers.get_mut(at) { + src.switches.video = false; + } + c.layers.insert(at, l); + Ok(()) +} + +pub fn specs() -> Vec { + vec![cmd!("layer.create", "Create", [], None, "{op: editableText|shapesFromText|masksFromText|shapesFromVector, layers?}", has_layers, create)] +} diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index e7ca4bd7..9a134b05 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -525,9 +525,38 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result { s.snapshot = None; s.state.viewer.show_snapshot = false; } + let memory = matches!(what.as_str(), "all" | "memoryAndDisk" | "memory" | "image"); + let disk = matches!(what.as_str(), "all" | "memoryAndDisk" | "disk"); + if memory { + s.layer_cache.clear(); + } + let mut disk_entries = 0; + if disk && let Some(dc) = &s.disk_cache { + disk_entries = dc.stats().entries; + dc.clear(); + } s.events.push(crate::Event::PurgeCaches); s.toast(format!("Purged {what} cache")); - Ok(json!({"purged": what})) + Ok(json!({"purged": what, "diskEntries": disk_entries})) +} + +/// `cache.diskStats`: the disk cache's folder, limit and contents. +pub(crate) fn disk_stats(s: &mut Session, _: &Value) -> Result { + let Some(dc) = &s.disk_cache else { return Ok(json!({"enabled": false})) }; + let st = dc.stats(); + Ok(json!({ + "enabled": true, + "folder": dc.folder().to_string_lossy(), + "entries": st.entries, + "frames": dc.keys(effectcraft_render::disk_cache::Kind::Frame).len(), + "layers": dc.keys(effectcraft_render::disk_cache::Kind::Layer).len(), + "bytes": st.bytes, + "maxBytes": st.max_bytes, + "hits": st.hits, + "misses": st.misses, + "writes": st.writes, + "evictions": st.evictions, + })) } /// Path of the footage behind the selected layer or project item. @@ -596,6 +625,7 @@ pub fn specs() -> Vec { cmd!("edit.selectLabelGroup", "Select Label Group", ["Edit", "Label"], None, "{}", has_props_or_layers_or_items, select_label_group), cmd!("edit.purgeUndo", "Undo", ["Edit", "Purge"], None, "{}", always_ok, purge), cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), + crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats), cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original), ] } diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index a85a391f..a176c400 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -84,10 +84,48 @@ fn import(s: &mut Session, p: &Value) -> Result { Some(Value::String(x)) => vec![x.clone()], _ => return Err(bad("file.import", "missing `paths`")), }; - let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; + // Photoshop documents as compositions (Import As: Composition / – Retain Layer Sizes). + let import_as = str_p(p, "importAs").unwrap_or("footage"); + let retain = match import_as { + "footage" => None, + "composition" | "comp" => Some(false), + "compositionLayerSizes" | "compositionRetainLayerSizes" | "layerSizes" => Some(true), + other => return Err(bad("file.import", format!("importAs: footage|composition|compositionLayerSizes, not `{other}`"))), + }; + let mut paths = paths; + let mut out_comps = vec![]; let mut ids = vec![]; let mut errors = vec![]; + if let Some(retain) = retain { + let mut rest = vec![]; + for path in paths { + let bytes = match s.services.read_file(&path) { + Ok(b) if effectcraft_psd::is_psd(&b) => b, + _ => { + rest.push(path); + continue; + } + }; + match import_psd_comp(s, &path, bytes, retain) { + Ok((comp, items, warnings)) => { + out_comps.push(comp); + ids.extend(items); + errors.extend(warnings); + } + Err(e) => errors.push(format!("{path}: {e}")), + } + } + paths = rest; + if paths.is_empty() { + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + return Ok(json!({"items": ids, "comps": out_comps, "errors": errors})); + } + } + let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; let mut probed = vec![]; + let psd_layer = p.get("layer").cloned(); let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps); for path in &paths { match importer.probe(path) { @@ -99,6 +137,18 @@ fn import(s: &mut Session, p: &Value) -> Result { f.frame_rate = r; f.duration = r.tick_of(frames.max(1)); } + // Choose Layer: one layer of a Photoshop document (document-sized). + if let Some(sel) = &psd_layer + && f.codec == "PSD" + { + match s.services.read_file(path).ok().and_then(|b| effectcraft_psd::Psd::parse(b).ok()).and_then(|d| find_psd_layer(&d, sel)) { + Some((index, name)) => f.layer = Some(effectcraft_project::SourceLayer { index: index as u32, name, layer_size: false }), + None => { + errors.push(format!("{path}: no layer {sel}")); + continue; + } + } + } probed.push((path.clone(), f)) } Err(e) => errors.push(format!("{path}: {e}")), @@ -107,6 +157,10 @@ fn import(s: &mut Session, p: &Value) -> Result { s.edit("Import", None, |proj, st| { for (path, f) in probed { let name = std::path::Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(path.clone()); + let name = match &f.layer { + Some(l) => format!("{}/{name}", l.name), + None => name, + }; let label = match f.kind { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, @@ -118,7 +172,37 @@ fn import(s: &mut Session, p: &Value) -> Result { } Ok(()) })?; - Ok(json!({"items": ids, "errors": errors})) + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + Ok(json!({"items": ids, "comps": out_comps, "errors": errors})) +} + +/// A Photoshop layer by index or name (pixel layers only). +fn find_psd_layer(d: &effectcraft_psd::Psd, sel: &Value) -> Option<(usize, String)> { + let l = match sel { + Value::Number(n) => d.layers.get(n.as_u64()? as usize)?, + Value::String(name) => d.layers.iter().find(|l| &l.name == name)?, + _ => return None, + }; + Some((l.index, l.name.clone())) +} + +/// Import a Photoshop document as a composition (one undo step). Returns (comp, items, warnings). +fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec, retain: bool) -> Result<(u64, Vec, Vec)> { + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Photoshop".into()); + let rate = effectcraft_time::FrameRate::FPS_29_97; + let secs = if s.prefs.import.still_footage == "seconds" { s.prefs.import.still_seconds } else { 10.0 }; + let duration = rate.snap_nearest(effectcraft_time::Tick::from_seconds_f64(secs)); + let r = s.edit("Import", None, |proj, st| { + let r = crate::psd_import::import(proj, &psd, path, &name, retain, rate, duration); + st.project_selection = vec![r.comp]; + Ok(r) + })?; + let mut items = vec![r.folder.0]; + items.extend(r.items.iter().map(|i| i.0)); + Ok((r.comp.0, items, r.warnings)) } fn project_settings(s: &mut Session, p: &Value) -> Result { @@ -224,7 +308,15 @@ pub fn specs() -> Vec { cmd!("file.saveAs", "Save As...", ["File", "Save As"], Some("Cmd+Shift+S"), "{path}", always, save_as), cmd!("file.incrementAndSave", "Increment and Save", ["File"], Some("Cmd+Alt+Shift+S"), "{}", has_path, increment_save), cmd!("file.revert", "Revert", ["File"], None, "{}", has_path, revert), - cmd!("file.import", "File...", ["File", "Import"], Some("Cmd+I"), "{paths: [string]}", always, import), + cmd!( + "file.import", + "File...", + ["File", "Import"], + Some("Cmd+I"), + "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer)}", + always, + import + ), cmd!( "file.projectSettings", "Project Settings...", diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 87c1e2df..b62664af 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -107,6 +107,7 @@ fn placeholder_footage(p: &Value) -> Footage { missing: true, sequence: vec![], color_profile: None, + layer: None, } } diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f..0791d1a3 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -6,6 +6,7 @@ pub mod anim_tools; mod animation; mod comp; mod comp_more; +mod create; mod edit; mod effect; mod file; @@ -118,6 +119,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(help::specs()); v.extend(query::specs()); v.extend(layer_menu::specs()); + v.extend(create::specs()); v.extend(animation::specs()); v.extend(view::specs()); v.extend(viewer_cmds::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd7..f8bca154 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -34,7 +34,6 @@ pub fn specs() -> Vec { // Time (timeline milestone). stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), stub!("layer.environment", "Environment Layer", ["Layer"], "{}"), - stub!("layer.create", "Create", [], "{op}"), stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), // Cameras / lights / materials / 3D views (3D milestone). diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488a..9d6afcc0 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -16,10 +16,12 @@ pub mod links; pub mod mask_track; pub mod menus; pub mod prefs; +pub mod psd_import; pub mod render_queue; mod session_settings; pub mod shortcuts; pub mod tracking; +pub mod vector; pub mod viewer; pub mod warp; @@ -268,6 +270,8 @@ pub struct Session { pub journal: Vec<(String, Value)>, /// Processed-layer pixels reused across frames and edits (content-keyed, never stale). pub layer_cache: Arc, + /// The persistent disk cache (Settings ▸ Media & Disk Cache), when enabled. + pub disk_cache: Option>, /// Settings (Preferences). pub prefs: prefs::Prefs, /// Bumped whenever settings change (frontends re-apply theme, labels…). @@ -311,6 +315,7 @@ impl Default for Session { events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), + disk_cache: None, prefs: prefs::Prefs::default(), prefs_revision: 0, config: None, @@ -393,6 +398,9 @@ impl Session { pub fn bump(&mut self) { self.revision += 1; + if self.disk_cache.is_some() { + self.layer_cache.set_disk_salt(effectcraft_render::disk_cache::footage_salt(&self.project)); + } self.events.push(Event::ProjectChanged { revision: self.revision }); } @@ -574,6 +582,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_disk_cache; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; @@ -602,6 +612,8 @@ mod tests_timeline; #[cfg(test)] mod tests_track; #[cfg(test)] +mod tests_vector_import; +#[cfg(test)] mod tests_viewer; /// Font families available to text layers (bundled + scanned system fonts). diff --git a/crates/engine/src/psd_import.rs b/crates/engine/src/psd_import.rs new file mode 100644 index 00000000..c14888ea --- /dev/null +++ b/crates/engine/src/psd_import.rs @@ -0,0 +1,634 @@ +//! Photoshop import as a composition (File ▸ Import ▸ File… with Import As: Composition or +//! Composition – Retain Layer Sizes), and Layer ▸ Create ▸ Convert to Editable Text. +//! +//! Every pixel layer becomes a footage item that names its layer (`Footage::layer`), placed in +//! a composition the size of the document. Groups become precomps (pass-through groups collapse +//! so their blend modes act on the layers below, like Photoshop); type layers become editable +//! text layers; layer masks are baked into the layer's alpha and vector masks become masks; +//! layer effects map to Layer Styles; adjustment layers become adjustment layers with the +//! matching effect (best effort); clipped layers preserve the transparency underneath. + +use std::sync::Arc; + +use effectcraft_color::BlendMode; +use effectcraft_color::Label; +use effectcraft_keyframe::{Gradient, Justify, ShapePath, TextDoc, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::styles::{self, STYLE_BLEND_MODES}; +use effectcraft_project::{AlphaMode, Comp, Footage, FootageKind, ItemId, ItemKind, Layer, LayerSource, MaskMode, Project, PropGroup, Solid, SourceLayer}; +use effectcraft_psd::{Adjustment, DValue, Descriptor, Node, Psd}; +use effectcraft_time::{FrameRate, Tick}; + +/// Photoshop blend mode key → compositor mode (`pass` = pass through, handled by collapsing). +pub fn blend_mode(key: &str) -> BlendMode { + match key { + "diss" => BlendMode::Dissolve, + "dark" => BlendMode::Darken, + "mul " => BlendMode::Multiply, + "idiv" => BlendMode::ColorBurn, + "lbrn" => BlendMode::LinearBurn, + "dkCl" => BlendMode::DarkerColor, + "lite" => BlendMode::Lighten, + "scrn" => BlendMode::Screen, + "div " => BlendMode::ColorDodge, + "lddg" => BlendMode::LinearDodge, + "lgCl" => BlendMode::LighterColor, + "over" => BlendMode::Overlay, + "sLit" => BlendMode::SoftLight, + "hLit" => BlendMode::HardLight, + "vLit" => BlendMode::VividLight, + "lLit" => BlendMode::LinearLight, + "pLit" => BlendMode::PinLight, + "hMix" => BlendMode::HardMix, + "diff" => BlendMode::Difference, + "smud" => BlendMode::Exclusion, + "fsub" => BlendMode::Subtract, + "fdiv" => BlendMode::Divide, + "hue " => BlendMode::Hue, + "sat " => BlendMode::Saturation, + "colr" => BlendMode::Color, + "lum " => BlendMode::Luminosity, + _ => BlendMode::Normal, + } +} + +/// Descriptor blend mode enum (`BlnM`) → compositor mode. +fn desc_blend(v: Option<&str>) -> Option { + Some(match v? { + "Nrml" => BlendMode::Normal, + "Dslv" => BlendMode::Dissolve, + "Drkn" => BlendMode::Darken, + "Mltp" => BlendMode::Multiply, + "CBrn" => BlendMode::ColorBurn, + "linearBurn" => BlendMode::LinearBurn, + "darkerColor" => BlendMode::DarkerColor, + "Lghn" => BlendMode::Lighten, + "Scrn" => BlendMode::Screen, + "CDdg" => BlendMode::ColorDodge, + "linearDodge" => BlendMode::LinearDodge, + "lighterColor" => BlendMode::LighterColor, + "Ovrl" => BlendMode::Overlay, + "SftL" => BlendMode::SoftLight, + "HrdL" => BlendMode::HardLight, + "vividLight" => BlendMode::VividLight, + "linearLight" => BlendMode::LinearLight, + "pinLight" => BlendMode::PinLight, + "hardMix" => BlendMode::HardMix, + "Dfrn" => BlendMode::Difference, + "Xclu" => BlendMode::Exclusion, + "blendSubtraction" => BlendMode::Subtract, + "blendDivide" => BlendMode::Divide, + "H " => BlendMode::Hue, + "Strt" => BlendMode::Saturation, + "Clr " => BlendMode::Color, + "Lmns" => BlendMode::Luminosity, + _ => return None, + }) +} + +/// What the import created. +pub struct PsdImport { + pub comp: ItemId, + pub folder: ItemId, + pub items: Vec, + pub warnings: Vec, +} + +struct Ctx<'a> { + psd: &'a Psd, + path: &'a str, + retain: bool, + folder: ItemId, + rate: FrameRate, + duration: Tick, + items: Vec, + warnings: Vec, +} + +/// Import `psd` (read from `path`) as a composition named `name`. +pub fn import(proj: &mut Project, psd: &Psd, path: &str, name: &str, retain: bool, rate: FrameRate, duration: Tick) -> PsdImport { + let folder = proj.add_item(&format!("{name} Layers"), Label::Yellow, None, ItemKind::Folder); + let mut cx = Ctx { psd, path, retain, folder, rate, duration, items: vec![], warnings: vec![] }; + let comp = build_comp(proj, &mut cx, name, &psd.tree(), None); + PsdImport { comp, folder, items: cx.items, warnings: cx.warnings } +} + +fn new_comp(cx: &Ctx) -> Comp { + Comp::new(cx.psd.width, cx.psd.height, cx.rate, cx.duration) +} + +fn build_comp(proj: &mut Project, cx: &mut Ctx, name: &str, nodes: &[Node], parent: Option) -> ItemId { + let cid = proj.add_item(name, Label::Sandstone, parent, ItemKind::Comp(Arc::new(new_comp(cx)))); + cx.items.push(cid); + let mut comp = new_comp(cx); + for n in nodes { + if let Some(l) = build_layer(proj, cx, &comp, n) { + comp.layers.push(l); + } + } + styles::sync_comp(&mut comp); + if let Some(item) = proj.item_mut(cid) { + item.kind = ItemKind::Comp(Arc::new(comp)); + } + cid +} + +fn footage_item(proj: &mut Project, cx: &mut Ctx, index: usize, name: &str) -> (ItemId, (u32, u32)) { + let l = &cx.psd.layers[index]; + let retain = cx.retain; + let (w, h) = if retain && !l.rect.is_empty() { (l.rect.width(), l.rect.height()) } else { (cx.psd.width, cx.psd.height) }; + let file = std::path::Path::new(cx.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let f = Footage { + path: cx.path.to_string(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: cx.rate, + native_rate: None, + duration: Tick::ZERO, + has_video: true, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PSD".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: retain && !l.rect.is_empty() }), + }; + let id = proj.add_item(&format!("{name}/{file}"), Label::Lavender, Some(cx.folder), ItemKind::Footage(f)); + cx.items.push(id); + (id, (w, h)) +} + +fn build_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, n: &Node) -> Option { + let (index, children) = match n { + Node::Layer(i) => (*i, None), + Node::Group { layer, children } => (*layer, Some(children)), + }; + let pl = cx.psd.layers[index].clone(); + let name = if pl.name.is_empty() { format!("Layer {}", index + 1) } else { pl.name.clone() }; + let mut layer = if let Some(children) = children { + let sub = build_comp(proj, cx, &name, children, Some(cx.folder)); + let mut l = build::layer(proj, comp, &name, LayerSource::Comp { item: sub }, (comp.width, comp.height), None); + // Pass-through groups composite their layers straight into the parent. + l.switches.collapse = pl.blend_key() == "pass"; + l + } else if let Some(t) = &pl.text { + text_layer(proj, comp, &name, t) + } else if let Some(adj) = &pl.adjustment { + let mut l = adjustment_layer(proj, comp, &name); + add_adjustment_effect(proj, &mut l, adj, comp, &mut cx.warnings, &name); + l + } else if pl.has_pixels() || pl.fill.is_some() { + let (item, size) = footage_item(proj, cx, index, &name); + let mut l = build::layer(proj, comp, &name, LayerSource::Footage { item }, size, None); + if cx.retain && !pl.rect.is_empty() { + set_vec2(&mut l, "transform/position", [pl.rect.left as f64 + size.0 as f64 / 2.0, pl.rect.top as f64 + size.1 as f64 / 2.0]); + } + l + } else { + // Empty layers have nothing to show. + return None; + }; + layer.name = comp.unique_layer_name(&name); + layer.blend_mode = blend_mode(pl.blend_key()); + layer.switches.video = !pl.hidden; + layer.preserve_transparency = pl.clipping; + set_scalar(&mut layer, "transform/opacity", pl.opacity as f64 / 255.0 * 100.0); + // Vector mask → a mask in layer space. + let origin = layer_origin(&pl, cx.retain, &layer); + if let Some(vm) = pl.vector_mask.as_ref().filter(|v| !v.disabled) + && let Some(masks) = layer.props.sub_mut("masks") + { + let mut ids = Ids(&mut proj.next_id); + for (k, sp) in vm.subpaths.iter().enumerate() { + let n = sp.knots.len(); + let mut path = ShapePath { vertices: vec![], in_tangents: vec![], out_tangents: vec![], closed: sp.closed }; + for kn in &sp.knots { + let v = [kn[1][0] - origin[0], kn[1][1] - origin[1]]; + path.vertices.push(v); + path.in_tangents.push([kn[0][0] - kn[1][0], kn[0][1] - kn[1][1]]); + path.out_tangents.push([kn[2][0] - kn[1][0], kn[2][1] - kn[1][1]]); + } + if n == 0 { + continue; + } + let mode = if vm.inverted { MaskMode::Subtract } else { MaskMode::Add }; + masks.children.push(build::mask(&mut ids, &format!("Mask {}", k + 1), path, mode, [255, 255, 0]).into()); + } + } + // Layer effects → Layer Styles; Fill Opacity → Advanced Blending. + let gl = comp.global_light.clone(); + if let Some(fx) = &pl.effects + && fx.bool("masterFXSwitch") != Some(false) + && layer.can_have_styles() + { + apply_effects(proj, &mut layer, fx, &gl); + } + if pl.fill_opacity < 255 && layer.can_have_styles() { + let mut ids = Ids(&mut proj.next_id); + if layer.props.sub(styles::GROUP).is_none() { + let g = styles::layer_styles(&mut ids, &gl); + let at = layer.props.children.iter().position(|c| c.match_id() == "transform").map(|i| i + 1).unwrap_or(layer.props.children.len()); + layer.props.children.insert(at, g.into()); + } + if let Some(p) = layer.props.prop_mut("layerStyles/blendingOptions/advancedBlending/fillOpacity") { + p.value = Value::Scalar(pl.fill_opacity as f64 / 255.0 * 100.0); + } + } + Some(layer) +} + +/// Where layer space's origin sits in document pixels. +fn layer_origin(pl: &effectcraft_psd::Layer, retain: bool, layer: &Layer) -> [f64; 2] { + if let Some(t) = &pl.text { + return [t.transform[4], t.transform[5]]; + } + if matches!(layer.source, LayerSource::Footage { .. }) && retain && !pl.rect.is_empty() { + return [pl.rect.left as f64, pl.rect.top as f64]; + } + [0.0, 0.0] +} + +fn set_scalar(l: &mut Layer, path: &str, v: f64) { + if let Some(p) = l.props.prop_mut(path) { + p.value = Value::Scalar(v); + } +} + +fn set_vec2(l: &mut Layer, path: &str, v: [f64; 2]) { + if let Some(p) = l.props.prop_mut(path) { + p.value = match p.value { + Value::Vec3(o) => Value::Vec3([v[0], v[1], o[2]]), + _ => Value::Vec2(v), + }; + } +} + +/// Split a PostScript font name into family and style (`Inter-Bold` → `Inter`, `Bold`). +fn font_names(ps: &str) -> (String, String) { + match ps.split_once('-') { + Some((fam, style)) => (fam.to_string(), style.to_string()), + None => (ps.to_string(), "Regular".to_string()), + } +} + +fn text_doc(t: &effectcraft_psd::TextInfo) -> TextDoc { + let mut d = TextDoc::default(); + d.set_text(&t.text); + let [xx, xy, yx, yy, _, _] = t.transform; + let scale = (xx * yy - xy * yx).abs().sqrt().max(1e-6); + if let Some(s) = t.style.size { + d.size = s * scale; + } + if let Some(f) = &t.style.font { + let (fam, style) = font_names(f); + d.font = fam; + d.style = style; + } + if let Some(c) = t.style.color { + d.fill = [c[0] as f32, c[1] as f32, c[2] as f32, 1.0]; + } + d.justify = match t.style.justification { + Some(1) => Justify::Right, + Some(2) => Justify::Center, + Some(3) => Justify::JustifyLastLeft, + Some(4) => Justify::JustifyLastRight, + Some(5) => Justify::JustifyLastCenter, + Some(6) => Justify::JustifyAll, + _ => Justify::Left, + }; + if let Some(tr) = t.style.tracking { + d.tracking = tr; + } + if let Some(l) = t.style.leading { + d.leading = Some(l * scale); + } + d.faux_bold = t.style.faux_bold; + d.faux_italic = t.style.faux_italic; + if t.style.box_text + && let Some([l, top, r, b]) = t.bounds + { + d.box_size = Some([(r - l) * scale, (b - top) * scale]); + d.box_pos = [l * scale, top * scale]; + } + d +} + +fn text_layer(proj: &mut Project, comp: &Comp, name: &str, t: &effectcraft_psd::TextInfo) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Text, (0, 0), None); + if let Some(p) = l.props.prop_mut("text/sourceText") { + p.value = Value::Text(Box::new(text_doc(t))); + } + set_vec2(&mut l, "transform/position", [t.transform[4], t.transform[5]]); + let [xx, xy, _, _, _, _] = t.transform; + let rot = xy.atan2(xx).to_degrees(); + if rot.abs() > 1e-6 { + set_scalar(&mut l, "transform/rotation", rot); + } + l +} + +fn adjustment_layer(proj: &mut Project, comp: &Comp, name: &str) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (comp.width, comp.height), None); + l.switches.adjustment = true; + l.label = Label::Purple; + l +} + +fn add_effect(proj: &mut Project, l: &mut Layer, effect: &str, comp: &Comp, params: &[(&str, Value)]) { + let Some(spec) = effectcraft_effects::lookup(effect) else { return }; + let mut ids = Ids(&mut proj.next_id); + let mut g = effectcraft_effects::instantiate(spec, &mut ids, spec.name, [comp.width as f64, comp.height as f64]); + for (k, v) in params { + if let Some(p) = g.get_mut(k) { + p.value = v.clone(); + } + } + if let Some(fx) = l.props.sub_mut("effects") { + fx.children.push(g.into()); + } +} + +fn add_adjustment_effect(proj: &mut Project, l: &mut Layer, adj: &Adjustment, comp: &Comp, warnings: &mut Vec, name: &str) { + match adj { + Adjustment::Invert => add_effect(proj, l, "Invert", comp, &[]), + Adjustment::BrightnessContrast { brightness, contrast, .. } => { + add_effect(proj, l, "Brightness & Contrast", comp, &[("brightness", Value::Scalar(*brightness)), ("contrast", Value::Scalar(*contrast))]) + } + Adjustment::HueSaturation { hue, saturation, lightness, colorize } => { + let mut p = vec![("hue", Value::Scalar(*hue)), ("saturation", Value::Scalar(*saturation)), ("lightness", Value::Scalar(*lightness))]; + if *colorize { + p = vec![ + ("colorize", Value::Bool(true)), + ("colorizeHue", Value::Scalar(*hue)), + ("colorizeSaturation", Value::Scalar(*saturation)), + ("colorizeLightness", Value::Scalar(*lightness)), + ]; + } + add_effect(proj, l, "Hue/Saturation", comp, &p) + } + Adjustment::Levels { in_black, in_white, gamma, out_black, out_white } => add_effect( + proj, + l, + "Levels", + comp, + &[ + ("inBlack", Value::Scalar(in_black / 255.0)), + ("inWhite", Value::Scalar(in_white / 255.0)), + ("gamma", Value::Scalar(*gamma)), + ("outBlack", Value::Scalar(out_black / 255.0)), + ("outWhite", Value::Scalar(out_white / 255.0)), + ], + ), + Adjustment::Exposure { exposure, offset, gamma } => add_effect( + proj, + l, + "Exposure", + comp, + &[("exposure", Value::Scalar(*exposure)), ("offset", Value::Scalar(*offset)), ("gamma", Value::Scalar(*gamma))], + ), + Adjustment::Other(k) => warnings.push(format!("{name}: adjustment `{k}` is not converted (layer kept as an empty adjustment layer)")), + } +} + +// ---------------------------------------------------------------- layer effects → styles + +fn style_mode(m: BlendMode) -> Value { + Value::Enum(STYLE_BLEND_MODES.iter().position(|x| x.1 == m).unwrap_or(0) as u32) +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn color4(c: [f64; 3]) -> Value { + Value::Color([c[0], c[1], c[2], 1.0]) +} + +/// Copy the common style settings (`Md `, `Clr `, `Opct`, angle, distance, size…). +fn common(g: &mut PropGroup, d: &Descriptor) { + if let Some(m) = desc_blend(d.enum_value("Md ")) { + set(g, "blendMode", style_mode(m)); + } + if let Some(c) = d.color("Clr ") { + set(g, "color", color4(c)); + } + for (k, prop) in [("Opct", "opacity"), ("Nose", "noise"), ("Dstn", "distance"), ("blur", "size"), ("lagl", "angle"), ("ShdN", "jitter"), ("Inpr", "range")] + { + if let Some(v) = d.num(k) { + set(g, prop, Value::Scalar(v)); + } + } + if let Some(b) = d.bool("uglg") { + set(g, "useGlobalLight", Value::Bool(b)); + } +} + +fn gradient_of(d: &Descriptor) -> Option { + let g = d.obj("Grad")?; + let list = |k: &str| match g.get(k) { + Some(DValue::List(v)) => v.clone(), + _ => vec![], + }; + let colors: Vec<(f64, [f32; 4])> = list("Clrs") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => { + let col = s.color("Clr ")?; + Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, [col[0] as f32, col[1] as f32, col[2] as f32, 1.0])) + } + _ => None, + }) + .collect(); + let opacities: Vec<(f64, f32)> = list("Trns") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, (s.num("Opct").unwrap_or(100.0) / 100.0) as f32)), + _ => None, + }) + .collect(); + (!colors.is_empty()).then(|| Gradient { colors, opacities: if opacities.is_empty() { vec![(0.0, 1.0), (1.0, 1.0)] } else { opacities } }) +} + +/// The effect descriptor for `key`, or the first enabled one of its `…Multi` list. +fn effect<'a>(fx: &'a Descriptor, key: &str, multi: &str) -> Option<&'a Descriptor> { + fx.obj(key).or_else(|| match fx.get(multi) { + Some(DValue::List(v)) => v.iter().find_map(|x| match x { + DValue::Descriptor(d) if d.bool("enab") != Some(false) => Some(d), + _ => None, + }), + _ => None, + }) +} + +fn apply_effects(proj: &mut Project, layer: &mut Layer, fx: &Descriptor, gl: &styles::GlobalLight) { + const MAP: &[(&str, &str, &str)] = &[ + ("DrSh", "dropShadowMulti", "dropShadow"), + ("IrSh", "innerShadowMulti", "innerShadow"), + ("OrGl", "outerGlowMulti", "outerGlow"), + ("IrGl", "innerGlowMulti", "innerGlow"), + ("ebbl", "bevelEmbossMulti", "bevelEmboss"), + ("ChFX", "satinMulti", "satin"), + ("SoFi", "solidFillMulti", "colorOverlay"), + ("GrFl", "gradientFillMulti", "gradientOverlay"), + ("FrFX", "frameFXMulti", "stroke"), + ]; + for (key, multi, id) in MAP { + let Some(d) = effect(fx, key, multi) else { continue }; + let enabled = d.bool("enab").unwrap_or(true); + let mut ids = Ids(&mut proj.next_id); + let Some(uid) = styles::add_style(layer, &mut ids, id, gl, enabled) else { continue }; + let Some(g) = layer.props.find_group_mut(uid) else { continue }; + common(g, d); + match *id { + "dropShadow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + if let Some(b) = d.bool("layerConceals") { + set(g, "knocksOut", Value::Bool(b)); + } + } + "innerShadow" | "innerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "choke", Value::Scalar(v)); + } + if d.enum_value("glwS") == Some("SrcC") { + set(g, "source", Value::Enum(0)); + } + } + "outerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + } + "bevelEmboss" => { + let style = match d.enum_value("bvlS") { + Some("OtrB") => 0, + Some("Embs") => 2, + Some("PlEb") => 3, + Some("strokeEmboss") => 4, + _ => 1, + }; + set(g, "style", Value::Enum(style)); + let tech = match d.enum_value("bvlT") { + Some("PrBL") => 1, + Some("Slmt") => 2, + _ => 0, + }; + set(g, "technique", Value::Enum(tech)); + if let Some(v) = d.num("srgR") { + set(g, "depth", Value::Scalar(v)); + } + set(g, "direction", Value::Enum(u32::from(d.enum_value("bvlD") == Some("Out ")))); + if let Some(v) = d.num("Sftn") { + set(g, "soften", Value::Scalar(v)); + } + if let Some(v) = d.num("Lald") { + set(g, "altitude", Value::Scalar(v)); + } + for (k, p) in [("hglM", "highlightMode"), ("sdwM", "shadowMode")] { + if let Some(m) = desc_blend(d.enum_value(k)) { + set(g, p, style_mode(m)); + } + } + for (k, p) in [("hglC", "highlightColor"), ("sdwC", "shadowColor")] { + if let Some(c) = d.color(k) { + set(g, p, color4(c)); + } + } + for (k, p) in [("hglO", "highlightOpacity"), ("sdwO", "shadowOpacity")] { + if let Some(v) = d.num(k) { + set(g, p, Value::Scalar(v)); + } + } + } + "satin" => { + if let Some(b) = d.bool("Invr") { + set(g, "invert", Value::Bool(b)); + } + } + "gradientOverlay" => { + if let Some(gr) = gradient_of(d) { + set(g, "colors", Value::Gradient(gr)); + } + if let Some(a) = d.num("Angl") { + set(g, "angle", Value::Scalar(a)); + } + let st = match d.enum_value("Type") { + Some("Rdl ") => 1, + Some("Angl") => 2, + Some("Rflc") => 3, + Some("Dmnd") => 4, + _ => 0, + }; + set(g, "style", Value::Enum(st)); + if let Some(b) = d.bool("Rvrs") { + set(g, "reverse", Value::Bool(b)); + } + if let Some(b) = d.bool("Algn") { + set(g, "alignWithLayer", Value::Bool(b)); + } + if let Some(v) = d.num("Scl ") { + set(g, "scale", Value::Scalar(v)); + } + } + "stroke" => { + if let Some(v) = d.num("Sz ") { + set(g, "size", Value::Scalar(v)); + } + let pos = match d.enum_value("Styl") { + Some("InsF") => 1, + Some("CtrF") => 2, + _ => 0, + }; + set(g, "position", Value::Enum(pos)); + } + _ => {} + } + } +} + +/// Layer ▸ Create ▸ Convert to Editable Text: a footage layer showing a Photoshop type layer +/// becomes a text layer (`None` when the layer's source has no text data). +pub fn editable_text(proj: &mut Project, comp: &Comp, layer: &Layer, psd: &Psd, index: usize) -> Option { + let t = psd.layers.get(index)?.text.as_ref()?; + let mut l = text_layer(proj, comp, &layer.name, t); + l.name = layer.name.clone(); + l.blend_mode = layer.blend_mode; + l.switches.video = layer.switches.video; + l.in_point = layer.in_point; + l.out_point = layer.out_point; + l.start_time = layer.start_time; + if let (Some(o), Some(p)) = (layer.props.prop("transform/opacity"), l.props.prop_mut("transform/opacity")) { + *p = effectcraft_project::Property { uid: p.uid, ..o.clone() }; + } + Some(l) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blend_keys() { + assert_eq!(blend_mode("mul "), BlendMode::Multiply); + assert_eq!(blend_mode("smud"), BlendMode::Exclusion); + assert_eq!(blend_mode("pass"), BlendMode::Normal); + assert_eq!(desc_blend(Some("linearDodge")), Some(BlendMode::LinearDodge)); + assert_eq!(font_names("Inter-Bold"), ("Inter".into(), "Bold".into())); + } +} diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index b0512a6a..e54eb93b 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -27,6 +27,7 @@ impl Session { self.prefs.normalize(); self.layer_cache.set_budget(self.prefs.layer_cache_bytes()); self.footage.set_cache_budget(self.prefs.media_cache_bytes()); + self.configure_disk_cache(); // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; if self.history.undo.len() > levels { @@ -36,6 +37,42 @@ impl Session { self.prefs_revision += 1; } + /// Apply Settings ▸ Media & Disk Cache ▸ Disk Cache: open (or close) the persistent cache + /// and attach it to the layer cache. Without a folder setting the platform cache folder is + /// used, but only by frontends with a settings store (headless sessions stay off disk). + pub fn configure_disk_cache(&mut self) { + use effectcraft_render::disk_cache::{DiskCache, default_folder, footage_salt}; + let d = &self.prefs.disk; + let folder = if d.disk_cache_folder.trim().is_empty() { + self.config.is_some().then(default_folder) + } else { + Some(std::path::PathBuf::from(d.disk_cache_folder.trim())) + }; + let max = d.disk_cache_max_gb.max(1) as u64 * (1 << 30); + let Some(folder) = folder.filter(|_| d.disk_cache_enabled) else { + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + }; + let dc = match &self.disk_cache { + Some(c) if c.folder() == folder.as_path() => { + c.set_max_bytes(max); + c.clone() + } + _ => match DiskCache::open(&folder, max) { + Ok(c) => c, + Err(e) => { + log::warn!("disk cache at {}: {e}", folder.display()); + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + } + }, + }; + self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project)))); + self.disk_cache = Some(dc); + } + /// Write settings to the config store. pub fn save_prefs(&self) { if let Some(c) = &self.config diff --git a/crates/engine/src/tests_disk_cache.rs b/crates/engine/src/tests_disk_cache.rs new file mode 100644 index 00000000..65180ba8 --- /dev/null +++ b/crates/engine/src/tests_disk_cache.rs @@ -0,0 +1,121 @@ +//! The persistent disk cache (PRV-2) through the session: settings, layer cache persistence +//! across restarts, invalidation on edits, purging. + +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn folder(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-session-disk-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d.to_string_lossy().to_string() +} + +fn session(dir: &str) -> Session { + let mut s = Session::default(); + s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir, "disk.diskCacheEnabled": true}})).unwrap(); + // Write every layer buffer, however quick to render. + s.disk_cache.as_ref().unwrap().set_min_layer_ms(0); + s +} + +fn project(s: &mut Session) -> effectcraft_project::ItemId { + s.execute("comp.new", json!({"name": "D", "width": 160, "height": 90, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#3080ff", "width": 80, "height": 40})).unwrap(); + s.execute("effect.apply", json!({"layer": "#1", "effect": "Gaussian Blur"})).unwrap(); + s.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 12})).unwrap(); + s.active_comp_id().unwrap() +} + +#[test] +fn disk_cache_settings_persistence_and_invalidation() { + let dir = folder("persist"); + let mut s = session(&dir); + let st = s.execute_checked("cache.diskStats", json!({})).unwrap(); + assert_eq!(st["enabled"], json!(true)); + assert_eq!(st["maxBytes"], json!(100u64 << 30)); + let cid = project(&mut s); + let t = s.time(); + let first = s.render(cid, t, RenderOpts::default()); + let dc = s.disk_cache.clone().unwrap(); + dc.flush(); + let written = dc.stats().entries; + assert!(written > 0, "layer buffers written"); + // "Restart": a new session on the same folder and project renders from disk. + let saved = s.project.clone(); + drop(s); + let mut s2 = session(&dir); + s2.project = saved; + s2.bump(); + s2.open_comp(cid); + let dc2 = s2.disk_cache.clone().unwrap(); + assert_eq!(dc2.stats().entries, written); + let again = s2.render(cid, t, RenderOpts::default()); + assert!(dc2.stats().hits > 0, "{:?}", dc2.stats()); + assert_eq!(again.data, first.data, "disk-cached pixels are exact"); + // An edit changes the keys: a miss, and new entries. + let hits = dc2.stats().hits; + s2.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 20})).unwrap(); + let edited = s2.render(cid, t, RenderOpts::default()); + assert_ne!(edited.data, first.data); + assert_eq!(dc2.stats().hits, hits); + dc2.flush(); + assert!(dc2.stats().entries > written); + // Purge ▸ All Disk Cache empties it; memory purge leaves the disk alone. + let r = s2.execute_checked("edit.purge", json!({"what": "memory"})).unwrap(); + assert_eq!(r["diskEntries"], json!(0)); + assert!(dc2.stats().entries > 0); + let r = s2.execute_checked("edit.purge", json!({"what": "disk"})).unwrap(); + assert!(r["diskEntries"].as_u64().unwrap() > 0); + assert_eq!(dc2.stats().entries, 0); + // Disabling detaches it. + s2.execute_checked("prefs.set", json!({"key": "disk.diskCacheEnabled", "value": false})).unwrap(); + assert!(s2.disk_cache.is_none()); + assert!(s2.layer_cache.disk().is_none()); +} + +#[test] +fn footage_changes_change_the_salt() { + let dir = folder("salt"); + let mut s = session(&dir); + let foot = std::env::temp_dir().join(format!("effectcraft-salt-{}.png", std::process::id())); + std::fs::write(&foot, b"one").unwrap(); + s.edit("add", None, |p, _| { + let f = effectcraft_project::Footage { + path: foot.to_string_lossy().to_string(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + pixel_aspect: 1.0, + frame_rate: effectcraft_time::FrameRate::FPS_30, + native_rate: None, + duration: effectcraft_time::Tick::ZERO, + has_video: true, + has_audio: false, + alpha: effectcraft_project::AlphaMode::Ignore, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PNG".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: None, + }; + p.add_item("f.png", effectcraft_color::Label::Lavender, None, effectcraft_project::ItemKind::Footage(f)); + Ok(()) + }) + .unwrap(); + let a = effectcraft_render::disk_cache::footage_salt(&s.project); + std::fs::write(&foot, b"two, longer").unwrap(); + let b = effectcraft_render::disk_cache::footage_salt(&s.project); + assert_ne!(a, b); + let cid = project(&mut s); + let k1 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap(); + let k2 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + assert_ne!(k1, k2); + // Unrelated comps do not change it. + s.execute("comp.new", json!({"name": "Other", "width": 10, "height": 10, "frameRate": 30, "duration": 1})).unwrap(); + assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k2); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858ca..17ffda7f 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -257,6 +257,7 @@ impl Importer for MockImporter { missing: false, sequence: vec![], color_profile: None, + layer: None, }) } } diff --git a/crates/engine/src/tests_track.rs b/crates/engine/src/tests_track.rs index adcf7df0..8cf020f9 100644 --- a/crates/engine/src/tests_track.rs +++ b/crates/engine/src/tests_track.rs @@ -101,6 +101,7 @@ pub(crate) fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Ses missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); @@ -410,6 +411,7 @@ fn analyze_1080p_single_point_speed() { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); diff --git a/crates/engine/src/tests_vector_import.rs b/crates/engine/src/tests_vector_import.rs new file mode 100644 index 00000000..d205630c --- /dev/null +++ b/crates/engine/src/tests_vector_import.rs @@ -0,0 +1,237 @@ +//! Photoshop import as a composition (IO-2) and Layer ▸ Create text conversions (SHP-5). + +use effectcraft_color::BlendMode; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{ItemKind, LayerSource, MaskMode}; +use effectcraft_psd::write::*; +use effectcraft_psd::{DValue, Descriptor, Rect}; +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-psd-tests-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +/// A layered document exercising groups, masks, blend modes, text, styles and adjustments. +pub(crate) fn sample_psd() -> Vec { + let mut d = WDoc::new(120, 80); + let shadow = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 0.5]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("uglg", DValue::Bool(false)) + .with("lagl", DValue::UnitFloat("#Ang".into(), 45.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 9.0)) + .with("blur", DValue::UnitFloat("#Pxl".into(), 4.0)), + ), + ); + let stroke = Descriptor::new("null").with( + "FrFX", + DValue::Descriptor( + Descriptor::new("FrFX") + .with("enab", DValue::Bool(true)) + .with("Styl", DValue::Enum("FStl".into(), "InsF".into())) + .with("Md ", DValue::Enum("BlnM".into(), "Nrml".into())) + .with("Opct", DValue::UnitFloat("#Prc".into(), 100.0)) + .with("Sz ", DValue::UnitFloat("#Pxl".into(), 2.0)) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 1.0, 0.0]))), + ), + ); + let mut masked = WLayer::solid("Masked Square", Rect::new(10, 10, 40, 40), [0.0, 0.8, 0.2, 1.0]).with_blend(b"mul "); + masked.mask = Some(WMask { + rect: Rect::new(10, 10, 40, 40), + data: (0..1600).map(|i| if i % 40 < 20 { 1.0 } else { 0.0 }).collect(), + default_color: 0, + disabled: false, + }); + let mut hidden = WLayer::solid("Hidden", Rect::new(0, 0, 10, 10), [1.0, 0.0, 1.0, 1.0]); + hidden.hidden = true; + let mut clipped = WLayer::solid("Clipped", Rect::new(60, 30, 20, 20), [1.0, 1.0, 1.0, 1.0]).with_blend(b"scrn").with_opacity(128); + clipped.clipping = true; + let tri = vec![[[70.0, 10.0]; 3], [[110.0, 10.0]; 3], [[90.0, 40.0]; 3]]; + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 120, 80), [0.2, 0.3, 0.4, 1.0]), + WLayer::group_end(), + masked, + hidden, + WLayer::group("Group A", true, *b"pass"), + WLayer::solid("Base", Rect::new(55, 25, 30, 30), [0.9, 0.1, 0.1, 1.0]).with_block(effects_block(&shadow)), + clipped, + WLayer::solid("Vector Masked", Rect::new(60, 0, 60, 50), [0.1, 0.1, 0.9, 1.0]) + .with_block(vector_mask_block(&[tri], 120, 80, false)) + .with_block(effects_block(&stroke)), + WLayer::empty("Hue Shift", vec![hue_sat_block(40, 10, -5)]), + WLayer::empty("Title", vec![text_block("Hello PSD", [8.0, 70.0], "Inter-Bold", 14.0, [1.0, 1.0, 1.0], 0)]), + ]; + write(&d) +} + +#[test] +fn psd_imports_as_composition() { + let path = tmp("sample.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + assert_eq!(s.state.active_comp, Some(cid)); + assert_eq!(s.project.item(cid).unwrap().name, "sample"); + let c = s.project.comp(cid).unwrap().clone(); + assert_eq!((c.width, c.height), (120, 80)); + let names: Vec<&str> = c.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Title", "Hue Shift", "Vector Masked", "Clipped", "Base", "Group A", "Background"]); + // Text layer: editable text, positioned at the type origin. + let title = &c.layers[0]; + assert!(matches!(title.source, LayerSource::Text)); + let Some(KV::Text(doc)) = title.props.prop("text/sourceText").map(|p| p.value.clone()) else { panic!("text") }; + assert_eq!(doc.text, "Hello PSD"); + assert_eq!(doc.size, 14.0); + assert_eq!(doc.font, "Inter"); + assert_eq!(doc.style, "Bold"); + assert_eq!(title.props.prop("transform/position").unwrap().value.as_vec2(), [8.0, 70.0]); + // Adjustment layer with Hue/Saturation. + let hue = &c.layers[1]; + assert!(hue.switches.adjustment); + let fx = hue.effects().unwrap().groups().next().unwrap(); + assert_eq!(fx.get("hue").unwrap().value.as_f64(), 40.0); + assert_eq!(fx.get("saturation").unwrap().value.as_f64(), 10.0); + // Vector mask → mask; Stroke style. + let vm = &c.layers[2]; + let m = vm.masks().unwrap().groups().next().unwrap(); + assert!(matches!(m.kind, effectcraft_project::GroupKind::Mask { mode: MaskMode::Add, .. })); + let Some(KV::Path(p)) = m.get("path").map(|p| p.value.clone()) else { panic!("mask path") }; + assert_eq!(p.vertices.len(), 3); + assert!((p.vertices[1][0] - 110.0).abs() < 1e-3 && (p.vertices[2][1] - 40.0).abs() < 1e-3); + let st = vm.props.group("layerStyles/stroke").unwrap(); + assert_eq!(st.get("size").unwrap().value.as_f64(), 2.0); + assert_eq!(st.get("position").unwrap().value, KV::Enum(1)); + // Clipped → preserve transparency; opacity and blend mode. + let clipped = &c.layers[3]; + assert!(clipped.preserve_transparency); + assert_eq!(clipped.blend_mode, BlendMode::Screen); + assert!((clipped.props.prop("transform/opacity").unwrap().value.as_f64() - 128.0 / 255.0 * 100.0).abs() < 1e-9); + // Drop shadow. + let ds = c.layers[4].props.group("layerStyles/dropShadow").unwrap(); + assert_eq!(ds.get("distance").unwrap().value.as_f64(), 9.0); + assert_eq!(ds.get("angle").unwrap().value.as_f64(), 45.0); + assert_eq!(ds.get("useGlobalLight").unwrap().value, KV::Bool(false)); + assert_eq!(ds.get("opacity").unwrap().value.as_f64(), 60.0); + // Group → collapsed precomp (pass through) with its layers. + let g = &c.layers[5]; + assert!(g.switches.collapse); + let LayerSource::Comp { item } = g.source else { panic!("precomp") }; + let inner = s.project.comp(item).unwrap(); + let inner_names: Vec<&str> = inner.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(inner_names, ["Hidden", "Masked Square"]); + assert!(!inner.layers[0].switches.video); + assert_eq!(inner.layers[1].blend_mode, BlendMode::Multiply); + // Footage items name their Photoshop layer; document-sized footage. + let LayerSource::Footage { item } = inner.layers[1].source else { panic!("footage") }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + let sl = f.layer.as_ref().unwrap(); + assert_eq!((sl.name.as_str(), sl.layer_size), ("Masked Square", false)); + assert_eq!((f.width, f.height), (120, 80)); + assert_eq!(s.project.item(item).unwrap().name, "Masked Square/sample.psd"); + // One undo step removes everything. + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.comp(cid).is_none()); +} + +#[test] +fn psd_retain_layer_sizes() { + let path = tmp("retain.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "compositionLayerSizes"})).unwrap(); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + let c = s.project.comp(cid).unwrap(); + let base = c.layer_by_name("Base").unwrap(); + let LayerSource::Footage { item } = base.source else { panic!() }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + assert_eq!((f.width, f.height), (30, 30)); + assert!(f.layer.as_ref().unwrap().layer_size); + assert_eq!(base.props.prop("transform/position").unwrap().value.as_vec2(), [70.0, 40.0]); + assert_eq!(base.props.prop("transform/anchor").unwrap().value.as_vec2(), [15.0, 15.0]); + // Vector mask vertices are in the layer's own space. + let vm = c.layer_by_name("Vector Masked").unwrap(); + let Some(KV::Path(p)) = vm.masks().unwrap().groups().next().unwrap().get("path").map(|p| p.value.clone()) else { panic!() }; + assert!((p.vertices[0][0] - 10.0).abs() < 1e-3, "{:?}", p.vertices); +} + +#[test] +fn import_rejects_unknown_kind() { + let mut s = Session::default(); + assert!(s.execute_checked("file.import", json!({"paths": ["x.psd"], "importAs": "banana"})).is_err()); +} + +fn text_session() -> (Session, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "T", "width": 320, "height": 120, "frameRate": 30, "duration": 2})).unwrap(); + let r = s.execute_checked("layer.newText", json!({"text": "Ab8", "size": 64, "position": [160, 80], "fill": "#ffcc00"})).unwrap(); + (s, r["layer"].as_u64().unwrap()) +} + +fn alpha_diff(a: &effectcraft_raster::Image, b: &effectcraft_raster::Image) -> (f32, f32) { + let mut sum = 0.0; + let mut mass = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + sum += (p[3] - q[3]).abs(); + mass += p[3]; + } + (sum / a.data.len() as f32, mass) +} + +#[test] +fn create_shapes_from_text_matches_text_render() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let opts = RenderOpts::default(); + let text_img = s.render(cid, s.time(), opts); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromText"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(matches!(c.layers[0].source, LayerSource::Shape)); + assert_eq!(c.layers[0].name, "Ab8 Outlines"); + assert!(!c.layer(effectcraft_project::LayerId(lid)).unwrap().switches.video, "source hidden"); + let groups: Vec = c.layers[0].props.sub("contents").unwrap().groups().map(|g| g.name.clone()).collect(); + assert_eq!(groups, ["A", "b", "8"]); + let shape_img = s.render(cid, s.time(), opts); + let (d, mass) = alpha_diff(&text_img, &shape_img); + assert!(mass > 500.0, "text drew something"); + assert!(d < 0.01, "mean alpha difference {d}"); + // Fill colour carried over. + let i = shape_img.data.iter().position(|p| p[3] > 0.99).unwrap(); + let p = shape_img.data[i]; + assert!((p[0] - 1.0).abs() < 0.02 && (p[1] - 0.8).abs() < 0.02 && p[2] < 0.02, "{p:?}"); +} + +#[test] +fn create_masks_from_text() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let text_img = s.render(cid, s.time(), RenderOpts::default()); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "masksFromText"})).unwrap(); + let mid = effectcraft_project::LayerId(r["layers"][0].as_u64().unwrap()); + let c = s.active_comp().unwrap(); + let l = c.layer(mid).unwrap(); + assert!(matches!(l.source, LayerSource::Solid { .. })); + // "A", "b" and "8" have 2, 2 and 3 contours. + assert_eq!(l.masks().unwrap().groups().count(), 7); + let img = s.render(cid, s.time(), RenderOpts::default()); + let (d, _) = alpha_diff(&text_img, &img); + assert!(d < 0.01, "mean alpha difference {d}"); + // Other ops need the right layer type. + assert!(s.execute_checked("layer.create", json!({"op": "shapesFromVector", "layers": [lid]})).is_err()); + assert!(s.execute_checked("layer.create", json!({"op": "nullTracing", "layers": [lid]})).is_err()); +} diff --git a/crates/engine/src/vector.rs b/crates/engine/src/vector.rs new file mode 100644 index 00000000..a3f3e040 --- /dev/null +++ b/crates/engine/src/vector.rs @@ -0,0 +1,250 @@ +//! Vector conversions behind Layer ▸ Create: Create Shapes from Vector Layer (SVG footage → +//! shape layer), Create Shapes from Text and Create Masks from Text (glyph outlines). + +use effectcraft_keyframe::{Gradient, ShapePath, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, Layer, LayerSource, MaskMode, Project, PropGroup}; +use effectcraft_svg::{Affine, Doc, Geom, GradientKind, Node, Paint}; +use kurbo::{BezPath, Point}; + +fn shape_paths(ids: &mut Ids, path: &BezPath) -> Vec { + effectcraft_path::from_kurbo(path) + .into_iter() + .enumerate() + .map(|(i, sp)| { + let mut g = build::shape_path(ids, sp); + g.name = format!("Path {}", i + 1); + g + }) + .collect() +} + +fn det_scale(m: Affine) -> f64 { + let c = m.as_coeffs(); + (c[0] * c[3] - c[1] * c[2]).abs().sqrt() +} + +/// Axis-aligned scale + translation only (basic shapes stay parametric). +fn axis_aligned(m: Affine) -> Option<(f64, f64, f64, f64)> { + let c = m.as_coeffs(); + (c[1].abs() < 1e-9 && c[2].abs() < 1e-9 && c[0] > 0.0 && c[3] > 0.0).then_some((c[0], c[3], c[4], c[5])) +} + +fn gradient_value(g: &effectcraft_svg::Gradient) -> Gradient { + Gradient { + colors: g.stops.iter().map(|(o, c)| (*o, [c[0] as f32, c[1] as f32, c[2] as f32, 1.0])).collect(), + opacities: g.stops.iter().map(|(o, c)| (*o, c[3] as f32)).collect(), + } +} + +/// Gradient start/end points in shape-layer space (`m`: shape user space → layer space). +fn gradient_points(g: &effectcraft_svg::Gradient, m: Affine, bbox: kurbo::Rect) -> (bool, [f64; 2], [f64; 2]) { + let unit = if g.bbox_units { Affine::new([bbox.width(), 0.0, 0.0, bbox.height(), bbox.x0, bbox.y0]) } else { Affine::IDENTITY }; + let full = m * unit * g.transform; + let p = |x: f64, y: f64| { + let q = full * Point::new(x, y); + [q.x, q.y] + }; + match g.kind { + GradientKind::Linear { x1, y1, x2, y2 } => (false, p(x1, y1), p(x2, y2)), + GradientKind::Radial { cx, cy, r, .. } => (true, p(cx, cy), p(cx + r, cy)), + } +} + +fn shape_items(ids: &mut Ids, s: &effectcraft_svg::Shape, m: Affine) -> Vec { + use kurbo::Shape as _; + let mut items = vec![]; + // Geometry: parametric when the transform keeps rectangles/ellipses axis-aligned. + match (&s.geom, axis_aligned(m)) { + (Geom::Rect { x, y, w, h, rx, ry }, Some((sx, sy, tx, ty))) if (rx - ry).abs() < 1e-9 && (*rx == 0.0 || (sx - sy).abs() < 1e-9) => { + let (w2, h2) = (w * sx, h * sy); + let pos = [tx + (x + w / 2.0) * sx, ty + (y + h / 2.0) * sy]; + let mut g = build::shape_rect(ids, [w2, h2], pos, rx * sx); + g.name = "Rectangle Path 1".into(); + items.push(g); + } + (Geom::Ellipse { cx, cy, rx, ry }, Some((sx, sy, tx, ty))) => { + items.push(build::shape_ellipse(ids, [2.0 * rx * sx, 2.0 * ry * sy], [tx + cx * sx, ty + cy * sy])); + } + (g, _) => items.extend(shape_paths(ids, &(m * g.to_path()))), + } + let bbox = s.geom.to_path().bounding_box(); + if let Some(st) = &s.stroke { + let width = st.width * det_scale(m); + let mut g = match &st.paint { + Paint::Color(c) => build::shape_stroke(ids, [c[0], c[1], c[2], 1.0], width), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_stroke(ids, radial, a, b, gradient_value(gr), width) + } + }; + set(&mut g, "opacity", Value::Scalar(st.opacity * 100.0)); + set(&mut g, "cap", Value::Enum(st.cap as u32)); + set(&mut g, "join", Value::Enum(st.join as u32)); + set(&mut g, "miter", Value::Scalar(st.miter)); + if let Some((d, off)) = &st.dash + && let Some(dg) = g.sub_mut("dashes") + { + let k = det_scale(m); + set(dg, "dash", Value::Scalar(d.first().copied().unwrap_or(0.0) * k)); + set(dg, "gap", Value::Scalar(d.get(1).or(d.first()).copied().unwrap_or(0.0) * k)); + set(dg, "offset", Value::Scalar(off * k)); + } + items.push(g); + } + if let Some(f) = &s.fill { + let mut g = match f { + Paint::Color(c) => build::shape_fill(ids, [c[0], c[1], c[2], 1.0]), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_fill(ids, radial, a, b, gradient_value(gr)) + } + }; + set(&mut g, "opacity", Value::Scalar(s.fill_opacity * 100.0)); + set(&mut g, "rule", Value::Enum(u32::from(s.fill_rule == effectcraft_svg::FillRule::EvenOdd))); + items.push(g); + } + items +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn group(ids: &mut Ids, name: &str, items: Vec, opacity: f64) -> PropGroup { + let mut g = build::shape_group(ids, name, items); + if let Some(p) = g.prop_mut("transform/opacity") { + p.value = Value::Scalar(opacity * 100.0); + } + g +} + +/// Shape layer contents for an SVG document, in document pixels (transforms are baked into the +/// geometry; group and element opacity go to the group transforms). Top of the paint order first. +pub fn svg_contents(ids: &mut Ids, doc: &Doc) -> Vec { + fn walk(ids: &mut Ids, nodes: &[Node], m: Affine) -> Vec { + let mut out = vec![]; + for n in nodes.iter().rev() { + match n { + Node::Group(g) => { + let items = walk(ids, &g.children, m * g.transform); + if !items.is_empty() { + out.push(group(ids, &g.name, items, g.opacity)); + } + } + Node::Shape(s) => { + let items = shape_items(ids, s, m * s.transform); + if !items.is_empty() { + out.push(group(ids, &s.name, items, s.opacity)); + } + } + } + } + out + } + let root = &doc.root; + let items = walk(ids, &root.children, root.transform); + if (root.opacity - 1.0).abs() > 1e-9 { vec![group(ids, "svg", items, root.opacity)] } else { items } +} + +/// A copy of `src`'s transform group with fresh uids. +fn copy_transform(proj: &mut Project, src: &Layer, dst: &mut Layer) { + let Some(tr) = src.props.sub("transform") else { return }; + let mut tr = tr.clone(); + let mut n = proj.next_id; + tr.reassign_uids(&mut n); + proj.next_id = n + 1; + if let Some(slot) = dst.props.children.iter_mut().find(|c| c.match_id() == "transform") { + *slot = tr.into(); + } +} + +fn like_source(dst: &mut Layer, src: &Layer) { + dst.start_time = src.start_time; + dst.in_point = src.in_point; + dst.out_point = src.out_point; + dst.stretch = src.stretch; + dst.parent = src.parent; + dst.switches.three_d = src.switches.three_d; +} + +/// Layer ▸ Create ▸ Create Shapes from Vector Layer: `doc` is the SVG shown by `src`. +pub fn shapes_from_vector(proj: &mut Project, comp: &Comp, src: &Layer, doc: &Doc) -> Layer { + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let items = svg_contents(&mut Ids(&mut proj.next_id), doc); + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(items.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + l +} + +/// Layer ▸ Create ▸ Create Shapes from Text: one group per character with its outline, fill and +/// stroke (as laid out at the comp time `ctx` was made for). +pub fn shapes_from_text(proj: &mut Project, comp: &Comp, ctx_project: &Project, cid: ItemId, src: &Layer, time: effectcraft_time::Tick) -> Option { + let ctx = effectcraft_render::EvalCtx::new(ctx_project, cid, comp, time); + let geom = effectcraft_render::text::text_geom(&ctx, src)?; + let chars: Vec = geom.doc.text.chars().filter(|c| !c.is_whitespace()).collect(); + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let mut groups = vec![]; + { + let mut ids = Ids(&mut proj.next_id); + for (i, g) in geom.glyphs.iter().enumerate() { + let path = g.path(); + if path.elements().is_empty() { + continue; + } + let mut items = shape_paths(&mut ids, &path); + if g.stroke_width > 0.0 && g.stroke[3] > 0.0 { + let c = g.stroke; + items.push(build::shape_stroke(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0], g.stroke_width)); + } + if g.apply_fill { + let c = g.fill; + let mut f = build::shape_fill(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0]); + set(&mut f, "opacity", Value::Scalar(c[3] as f64 * 100.0)); + items.push(f); + } + let name = chars.get(i).map(|c| c.to_string()).unwrap_or_else(|| format!("Glyph {}", i + 1)); + groups.push(group(&mut ids, &name, items, 1.0)); + } + } + // First character on top, as in After Effects. + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(groups.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + Some(l) +} + +/// Glyph outlines of a text layer in comp space (for Create Masks from Text). +pub fn text_outlines_in_comp(project: &Project, cid: ItemId, comp: &Comp, src: &Layer, time: effectcraft_time::Tick) -> Vec { + let ctx = effectcraft_render::EvalCtx::new(project, cid, comp, time); + let Some(geom) = effectcraft_render::text::text_geom(&ctx, src) else { return vec![] }; + let (m, _) = ctx.layer_to_comp(src); + let mut out = vec![]; + for g in &geom.glyphs { + let p = g.path(); + let p = effectcraft_path::transform(std::slice::from_ref(&p), &m).remove(0); + out.extend(effectcraft_path::from_kurbo(&p)); + } + out +} + +/// A comp-sized white solid with one mask per glyph contour (Difference, so counters stay open). +pub fn masks_layer(proj: &mut Project, comp: &Comp, name: &str, outlines: Vec, solid: ItemId) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: solid }, (comp.width, comp.height), None); + let mut ids = Ids(&mut proj.next_id); + if let Some(masks) = l.props.sub_mut("masks") { + for (i, sp) in outlines.into_iter().enumerate() { + masks.children.push(build::mask(&mut ids, &format!("Mask {}", i + 1), sp, MaskMode::Difference, [255, 255, 0]).into()); + } + } + l +} diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index ca2fb1b0..cf63e593 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -100,6 +100,7 @@ impl Scene { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = self.p.add_item("Pattern", Label::Lavender, None, ItemKind::Footage(f)); build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None) diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index b43271c7..7143510a 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -18,3 +18,7 @@ effectcraft-export = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-time = { workspace = true } diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs new file mode 100644 index 00000000..eb5be076 --- /dev/null +++ b/crates/host/tests/vector_import.rs @@ -0,0 +1,163 @@ +//! Photoshop and SVG import through the fully wired session (media decoding included): pixels +//! of imported compositions, SVG footage at any scale, and Create Shapes from Vector Layer. + +use effectcraft_engine::render::RenderOpts; +use effectcraft_project::ItemId; +use effectcraft_psd::Rect; +use effectcraft_psd::write::*; +use effectcraft_raster::Image; +use serde_json::json; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-host-vector-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +fn mean_diff(a: &Image, b: &Image) -> f32 { + assert_eq!((a.width, a.height), (b.width, b.height)); + let mut s = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + for c in 0..4 { + s += (p[c] - q[c]).abs(); + } + } + s / (a.data.len() * 4) as f32 +} + +fn psd_doc() -> WDoc { + let mut d = WDoc::new(64, 48); + let mut masked = WLayer::solid("Masked", Rect::new(30, 8, 24, 24), [0.1, 0.9, 0.3, 1.0]); + masked.mask = Some(WMask { + rect: Rect::new(30, 8, 24, 24), + data: (0..576).map(|i| if (i / 24) < 12 { 1.0 } else { 0.25 }).collect(), + default_color: 0, + disabled: false, + }); + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 64, 48), [0.2, 0.25, 0.3, 1.0]), + WLayer::solid("Half Red", Rect::new(4, 4, 30, 20), [1.0, 0.0, 0.0, 1.0]).with_opacity(128), + masked, + WLayer::group_end(), + WLayer::solid("In Group", Rect::new(10, 30, 20, 12), [0.9, 0.9, 0.1, 1.0]), + WLayer::group("Group", true, *b"norm"), + ]; + d +} + +/// Straight-alpha over of the layers as Photoshop composites them (normal blending), as the +/// merged image to compare the imported composition with. +fn reference(bytes: &[u8]) -> Vec<[f32; 4]> { + let p = effectcraft_psd::Psd::parse(bytes.to_vec()).unwrap(); + let mut acc = vec![[0.0f32; 4]; (p.width * p.height) as usize]; + for l in p.layers.iter().filter(|l| l.has_pixels() && !l.hidden) { + let px = p.layer_pixels(l.index, true).unwrap(); + let o = l.opacity as f32 / 255.0; + for (d, s) in acc.iter_mut().zip(&px.data) { + let a = s[3] * o; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (s[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +#[test] +fn psd_composition_renders_like_the_document() { + let mut doc = psd_doc(); + let bytes = write(&doc); + let merged = reference(&bytes); + doc.composite = Some(merged.clone()); + let bytes = write(&doc); + let path = tmp("layers.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + // Footage (merged image). + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + let merged_item = r["items"][0].as_u64().unwrap(); + // Composition and Composition – Retain Layer Sizes. + let mut renders = vec![]; + for kind in ["composition", "compositionLayerSizes"] { + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": kind})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + renders.push(s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default())); + } + // The merged footage in a comp of the same size. + s.execute("comp.new", json!({"name": "Merged", "width": 64, "height": 48, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.addItem", json!({"item": merged_item})).unwrap(); + let mcid = s.active_comp_id().unwrap(); + let merged_img = s.render(mcid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + assert!(mean_diff(&merged_img, &expected) < 2.0 / 255.0, "merged footage"); + for (i, img) in renders.iter().enumerate() { + let d = mean_diff(img, &expected); + assert!(d < 2.0 / 255.0, "import kind {i}: mean diff {d}"); + } + // A single layer as footage (Choose Layer). + let r = s.execute_checked("file.import", json!({"paths": [path], "layer": "Masked"})).unwrap(); + let it = s.project.item(ItemId(r["items"][0].as_u64().unwrap())).unwrap(); + assert_eq!(it.name, "Masked/layers.psd"); +} + +#[test] +fn psd_16bit_cmyk_composition_imports() { + let mut doc = psd_doc(); + doc.depth = 16; + doc.mode = effectcraft_psd::ColorMode::Cmyk; + doc.rle = false; + let bytes = write(&doc); + let merged = reference(&bytes); + let path = tmp("layers16.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + let d = mean_diff(&img, &expected); + assert!(d < 2.0 / 255.0, "mean diff {d}"); +} + +const SVG: &[u8] = include_bytes!("../../svg/tests/fixtures/basic-shapes.svg"); +const SVG2: &[u8] = include_bytes!("../../svg/tests/fixtures/paths-gradients.svg"); + +#[test] +fn svg_footage_and_shapes_from_vector_layer() { + for (name, bytes) in [("basic-shapes.svg", SVG), ("paths-gradients.svg", SVG2)] { + let path = tmp(name); + std::fs::write(&path, bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let item = r["items"][0].as_u64().unwrap(); + let doc = effectcraft_svg::parse(bytes).unwrap(); + let (w, h) = doc.pixel_size(); + s.execute("comp.new", json!({"name": "V", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let foot = s.render(cid, t, RenderOpts::default()); + let direct = effectcraft_svg::rasterize(&doc, w, h, 1.0); + assert!(mean_diff(&foot, &direct) < 1.0 / 255.0, "{name}: footage = rasterised SVG"); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromVector"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(!c.layers[1].switches.video); + let shapes = s.render(cid, t, RenderOpts::default()); + let d = mean_diff(&shapes, &direct); + assert!(d < 0.02, "{name}: shapes vs SVG mean diff {d}"); + // Scaled up 3×, the shape layer stays sharp and matches the SVG rasterised at 3×. + s.execute("prop.set", json!({"layer": sid, "path": "transform/scale", "value": [300, 300]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/position", "value": [0, 0]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/anchor", "value": [0, 0]})).unwrap(); + let big = s.render(cid, t, RenderOpts::default()); + let direct3 = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let d3 = mean_diff(&big, &direct3); + assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); + } +} diff --git a/crates/lottie/src/import.rs b/crates/lottie/src/import.rs index c87dc187..22bc5023 100644 --- a/crates/lottie/src/import.rs +++ b/crates/lottie/src/import.rs @@ -171,6 +171,7 @@ pub fn import( missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = project.add_item(&nm, Label::Lavender, Some(folder), ItemKind::Footage(foot)); cx.items.push(item); diff --git a/crates/lottie/src/tests.rs b/crates/lottie/src/tests.rs index 5140133c..01c724d7 100644 --- a/crates/lottie/src/tests.rs +++ b/crates/lottie/src/tests.rs @@ -592,6 +592,7 @@ fn images_embed_as_base64_and_come_back() { missing: false, sequence: vec![], color_profile: None, + layer: None, }), ); let c = comp(&p, cid); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 89992569..6e83c949 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -14,6 +14,8 @@ effectcraft-time = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } filmcraft-frame = { workspace = true } diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index fa5df238..dce33d8a 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -284,6 +284,22 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. +pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px[y * wu + x]; + *o = op.apply([s[0], s[1], s[2]], s[3].clamp(0.0, 1.0)); + } + }); + out +} + #[cfg(test)] mod tests { use super::*; @@ -376,3 +392,4 @@ mod tests { } } } + diff --git a/crates/media/src/layered.rs b/crates/media/src/layered.rs new file mode 100644 index 00000000..204312a5 --- /dev/null +++ b/crates/media/src/layered.rs @@ -0,0 +1,62 @@ +//! Layered and vector stills: Photoshop documents (`effectcraft-psd`: the merged image, or one +//! layer for footage imported as a composition) and SVG (`effectcraft-svg`, rasterised). + +use effectcraft_project::{AlphaMode, Footage}; +use effectcraft_raster::Image; + +use crate::convert::{AlphaOp, straight_to_image}; +use crate::{MediaError, Result}; + +fn is_svg(path: &str, bytes: &[u8]) -> bool { + path.to_ascii_lowercase().ends_with(".svg") || effectcraft_svg::looks_like_svg(bytes) +} + +/// Footage for a Photoshop or SVG file (`None` for other formats). +pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let mut f = crate::probe::still_footage(path, psd.width, psd.height, image::ImageFormat::Png, true); + f.codec = if psd.psb { "PSB".into() } else { "PSD".into() }; + f.alpha = if psd.merged_alpha || !psd.layers.is_empty() { AlphaMode::Straight } else { AlphaMode::Ignore }; + return Ok(Some(f)); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let mut f = crate::probe::still_footage(path, w, h, image::ImageFormat::Png, true); + f.codec = "SVG".into(); + f.alpha = AlphaMode::Straight; + return Ok(Some(f)); + } + Ok(None) +} + +/// Decode a Photoshop or SVG still (`None` for other formats). +pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let px = match &footage.layer { + Some(l) => psd.layer_pixels(l.index as usize, !l.layer_size), + None => psd.composite(), + } + .map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + return Ok(Some(straight_to_image(px.width, px.height, &px.data, op))); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let img = effectcraft_svg::rasterize(&doc, w, h, 1.0); + // The rasteriser produces premultiplied pixels already. + let _ = op; + return Ok(Some(img)); + } + Ok(None) +} + +/// An SVG rasterised at `scale` × its pixel size (Continuously Rasterize). +pub(crate) fn rasterize_svg(bytes: &[u8], scale: f64) -> Option { + let doc = effectcraft_svg::parse(bytes).ok()?; + let (w, h) = doc.pixel_size(); + let (sw, sh) = (((w as f64 * scale).ceil() as u32).clamp(1, 16384), ((h as f64 * scale).ceil() as u32).clamp(1, 16384)); + Some(effectcraft_svg::rasterize(&doc, sw, sh, scale)) +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 29546f7e..246054b3 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -10,9 +10,11 @@ //! Video containers (MP4/MOV, Matroska/WebM) and codecs (H.264, HEVC, VP9, AV1, ProRes, DNxHD, //! MJPEG; AAC, Opus, PCM, MP3/FLAC/Vorbis) come from FilmCraft's pure-Rust crates (git dependency, //! pinned; `plan/adr/0001`). FilmCraft types never leave this crate. Stills use the `image` crate -//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). +//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR); Photoshop documents (merged image or one layer) come +//! from `effectcraft-psd` and SVG from `effectcraft-svg` (rasterised at any scale). mod convert; +mod layered; mod pool; mod probe; @@ -44,7 +46,7 @@ impl From for MediaError { } /// File extensions recognised as stills. -pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"]; +pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "psd", "psb", "svg"]; /// File extensions recognised as audio-only files. pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; /// File extensions recognised as movies. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b4cc51ef..b5d94299 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -359,6 +359,11 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } + // A layer of a layered still is keyed by its layer index and size mode. + FootageKind::Still if footage.layer.is_some() => { + let l = footage.layer.as_ref().map_or(0, |l| 1 + l.index as i64 * 2 + l.layer_size as i64); + Loc { key: key(&footage.path, l), media_t: None } + } FootageKind::Still | FootageKind::Sequence => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; @@ -437,6 +442,9 @@ impl Inner { match loc.media_t { None => { let bytes = self.read(path)?; + if let Some(img) = crate::layered::decode(path, &bytes, footage, op)? { + return Ok(img); + } let img = image::load_from_memory(&bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; Ok(dynamic_to_image(&img, op)) } @@ -470,6 +478,14 @@ impl FootageSource for MediaPool { } } + fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option> { + if footage.missing || !effectcraft_render::is_vector_footage(footage) { + return None; + } + let bytes = self.inner.read(&footage.path).ok()?; + crate::layered::rasterize_svg(&bytes, scale).map(Arc::new) + } + fn audio(&self, _item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { if !footage.has_audio || footage.missing { return None; diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 4e1c5611..23432cc6 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -33,6 +33,9 @@ pub fn probe_single(path: impl AsRef) -> Result { /// Probe in-memory file contents; `path` names the footage (and is stored as `Footage::path`). pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { + if let Some(f) = crate::layered::probe(path, &bytes)? { + return Ok(f); + } if let Ok(fmt) = image::guess_format(&bytes) { return probe_still_bytes(path, &bytes, fmt); } @@ -72,6 +75,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - missing: false, sequence: Vec::new(), color_profile: v.and_then(|v| profile_of(&v.color)), + layer: None, } } @@ -88,7 +92,7 @@ fn profile_of(c: &filmcraft_color::ColorInfo) -> Option Footage { +pub(crate) fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: bool) -> Footage { let codec = fmt.extensions_str().first().map_or("image", |s| s).to_ascii_uppercase(); // OpenEXR stores premultiplied colour by convention; other formats straight alpha. let alpha = match (has_alpha, fmt) { @@ -114,6 +118,7 @@ fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: missing: false, sequence: Vec::new(), color_profile: None, + layer: None, } } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 1ed35c09..b4854f54 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -579,6 +579,21 @@ pub struct Footage { /// Colour profile (from the file's metadata, or Interpret Footage). `None` = sRGB. #[serde(default, skip_serializing_if = "Option::is_none")] pub color_profile: Option, + /// One layer of a layered file (Photoshop) instead of its merged image. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layer: Option, +} + +/// A layer of a layered still (a Photoshop document) used as footage +/// (File ▸ Import ▸ Composition / Composition – Retain Layer Sizes, or Choose Layer). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceLayer { + /// Layer record index in the file (bottom of the stack = 0). + pub index: u32, + pub name: String, + /// The footage is the layer's own bounds (Retain Layer Sizes) rather than the document size. + #[serde(default)] + pub layer_size: bool, } fn one() -> u32 { diff --git a/crates/psd/Cargo.toml b/crates/psd/Cargo.toml new file mode 100644 index 00000000..49dda6a3 --- /dev/null +++ b/crates/psd/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "effectcraft-psd" +description = "Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer, from Adobe's published file format specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +miniz_oxide = "0.8" diff --git a/crates/psd/README.md b/crates/psd/README.md new file mode 100644 index 00000000..e238b9b2 --- /dev/null +++ b/crates/psd/README.md @@ -0,0 +1,17 @@ +# effectcraft-psd + +Reads Photoshop documents (`.psd`, and `.psb` large documents) for File ▸ Import (as footage, as a +composition, or as a composition with layer sizes retained), and writes a minimal subset (used to +generate test fixtures). + +Implemented from Adobe's public **Adobe Photoshop File Formats Specification** (November 2019 +edition): file header, colour mode data, image resources, layer and mask information (layer +records, channel image data, layer mask data, additional layer information: `luni`, `lsct`/`lsdk`, +`lyid`, `iOpa`, `TySh`, `lfx2`, `SoCo`, `nvrt`, `brit`, `hue2`, `levl`, `expA`, `vmsk`/`vsms`, +`Lr16`/`Lr32`), the descriptor (action) structure, and the merged image data. Raw, RLE (PackBits) +and ZIP (with and without prediction, inflated with `miniz_oxide`) compression; 1/8/16/32-bit; +Bitmap, Grayscale, Indexed, RGB, CMYK (naive conversion) and Lab colour modes. + +Text engine data (`EngineData`) is read best-effort for the font, size, fill colour and +justification of the first style run. No Adobe code, sample files or assets were used; test +fixtures are generated in-test by the writer in this crate ([`write`](src/write.rs)). diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs new file mode 100644 index 00000000..e121f823 --- /dev/null +++ b/crates/psd/src/descriptor.rs @@ -0,0 +1,430 @@ +//! The descriptor ("action descriptor") structure: typed key/value trees used by text layers, +//! layer effects, fill layers and newer adjustment layers (spec: "Descriptor structure"). + +use crate::read::Reader; +use crate::{Error, Result}; + +/// A descriptor: a class and a list of keyed items (in file order). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Descriptor { + pub class_name: String, + pub class_id: String, + pub items: Vec<(String, DValue)>, +} + +/// One descriptor value. +#[derive(Clone, Debug, PartialEq)] +pub enum DValue { + Descriptor(Descriptor), + List(Vec), + Double(f64), + /// Unit (`#Pxl`, `#Prc`, `#Ang`, `#Pnt`…) and value. + UnitFloat(String, f64), + UnitFloats(String, Vec), + Text(String), + /// Enumerated value: type and value ids. + Enum(String, String), + Integer(i32), + LargeInteger(i64), + Bool(bool), + Class(String, String), + Raw(Vec), + Reference(Vec), +} + +/// An item of a reference (`obj `). +#[derive(Clone, Debug, PartialEq)] +pub enum RefItem { + Property(String, String), + Class(String), + Enum(String, String, String), + Offset(String, i32), + Identifier(i32), + Index(i32), + Name(String, String), +} + +impl Descriptor { + pub fn get(&self, key: &str) -> Option<&DValue> { + self.items.iter().find(|(k, _)| k == key).map(|(_, v)| v) + } + pub fn obj(&self, key: &str) -> Option<&Descriptor> { + match self.get(key)? { + DValue::Descriptor(d) => Some(d), + _ => None, + } + } + /// A number: doubles, unit floats, integers. + pub fn num(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Double(v) | DValue::UnitFloat(_, v) => Some(*v), + DValue::Integer(v) => Some(*v as f64), + DValue::LargeInteger(v) => Some(*v as f64), + _ => None, + } + } + pub fn bool(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Bool(b) => Some(*b), + _ => None, + } + } + pub fn text(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Text(s) => Some(s), + _ => None, + } + } + /// The value id of an enumerated item. + pub fn enum_value(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Enum(_, v) => Some(v), + _ => None, + } + } + pub fn raw(&self, key: &str) -> Option<&[u8]> { + match self.get(key)? { + DValue::Raw(v) => Some(v), + _ => None, + } + } + /// An RGB colour object (`RGBC` with `Rd `/`Grn `/`Bl ` in 0..255) as 0..1 values; also + /// reads grayscale (`Gry `) colours. + pub fn color(&self, key: &str) -> Option<[f64; 3]> { + let c = self.obj(key)?; + if let (Some(r), Some(g), Some(b)) = (c.num("Rd "), c.num("Grn "), c.num("Bl ")) { + return Some([r / 255.0, g / 255.0, b / 255.0]); + } + if let (Some(r), Some(g), Some(b)) = (c.num("redFloat"), c.num("greenFloat"), c.num("blueFloat")) { + return Some([r, g, b]); + } + c.num("Gry ").map(|g| { + let v = 1.0 - g / 100.0; + [v, v, v] + }) + } +} + +/// A key or class id: length-prefixed string, or a 4-character code when the length is 0. +pub(crate) fn read_id(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let n = if n == 0 { 4 } else { n }; + let b = r.bytes(n)?; + Ok(String::from_utf8_lossy(b).into_owned()) +} + +pub(crate) fn read_unicode(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n); + for _ in 0..n { + v.push(r.u16()?); + } + while v.last() == Some(&0) { + v.pop(); + } + Ok(String::from_utf16_lossy(&v)) +} + +/// Read a descriptor (after its version field). +pub fn read_descriptor(r: &mut Reader) -> Result { + let class_name = read_unicode(r)?; + let class_id = read_id(r)?; + let n = r.u32()? as usize; + if n > 100_000 { + return Err(Error::Invalid("descriptor item count".into())); + } + let mut items = Vec::with_capacity(n.min(64)); + for _ in 0..n { + let key = read_id(r)?; + let ty = r.tag()?; + let v = read_value(r, &ty, 0)?; + items.push((key, v)); + } + Ok(Descriptor { class_name, class_id, items }) +} + +fn read_value(r: &mut Reader, ty: &[u8; 4], depth: usize) -> Result { + if depth > 64 { + return Err(Error::Invalid("descriptor nesting".into())); + } + Ok(match ty { + b"Objc" | b"GlbO" => DValue::Descriptor(read_descriptor(r)?), + b"VlLs" => { + let n = r.u32()? as usize; + if n > 1_000_000 { + return Err(Error::Invalid("descriptor list length".into())); + } + let mut v = Vec::with_capacity(n.min(256)); + for _ in 0..n { + let t = r.tag()?; + v.push(read_value(r, &t, depth + 1)?); + } + DValue::List(v) + } + b"doub" => DValue::Double(r.f64()?), + b"UntF" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + DValue::UnitFloat(unit, r.f64()?) + } + b"UnFl" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n.min(1024)); + for _ in 0..n { + v.push(r.f64()?); + } + DValue::UnitFloats(unit, v) + } + b"TEXT" => DValue::Text(read_unicode(r)?), + b"enum" => { + let t = read_id(r)?; + DValue::Enum(t, read_id(r)?) + } + b"long" => DValue::Integer(r.u32()? as i32), + b"comp" => DValue::LargeInteger(r.u64()? as i64), + b"bool" => DValue::Bool(r.u8()? != 0), + b"type" | b"GlbC" => { + let name = read_unicode(r)?; + DValue::Class(name, read_id(r)?) + } + b"alis" | b"tdta" | b"Pth " => { + let n = r.u32()? as usize; + DValue::Raw(r.bytes(n)?.to_vec()) + } + b"obj " => { + let n = r.u32()? as usize; + let mut items = Vec::with_capacity(n.min(16)); + for _ in 0..n { + let t = r.tag()?; + items.push(match &t { + b"prop" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Property(class, read_id(r)?) + } + b"Clss" => { + let _name = read_unicode(r)?; + RefItem::Class(read_id(r)?) + } + b"Enmr" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + let ty = read_id(r)?; + RefItem::Enum(class, ty, read_id(r)?) + } + b"rele" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Offset(class, r.u32()? as i32) + } + b"Idnt" => RefItem::Identifier(r.u32()? as i32), + b"indx" => RefItem::Index(r.u32()? as i32), + b"name" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Name(class, read_unicode(r)?) + } + other => return Err(Error::Invalid(format!("reference item `{}`", String::from_utf8_lossy(other)))), + }); + } + DValue::Reference(items) + } + other => return Err(Error::Invalid(format!("descriptor type `{}`", String::from_utf8_lossy(other)))), + }) +} + +// ---------------------------------------------------------------- writing + +fn put_u32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +fn put_id(out: &mut Vec, id: &str) { + if id.len() == 4 { + put_u32(out, 0); + } else { + put_u32(out, id.len() as u32); + } + out.extend_from_slice(id.as_bytes()); +} + +pub(crate) fn put_unicode(out: &mut Vec, s: &str) { + let v: Vec = s.encode_utf16().chain(std::iter::once(0)).collect(); + put_u32(out, v.len() as u32); + for c in v { + out.extend_from_slice(&c.to_be_bytes()); + } +} + +/// Serialise a descriptor (without the version field). +pub fn write_descriptor(out: &mut Vec, d: &Descriptor) { + put_unicode(out, &d.class_name); + put_id(out, &d.class_id); + put_u32(out, d.items.len() as u32); + for (k, v) in &d.items { + put_id(out, k); + write_value(out, v); + } +} + +fn write_value(out: &mut Vec, v: &DValue) { + match v { + DValue::Descriptor(d) => { + out.extend_from_slice(b"Objc"); + write_descriptor(out, d); + } + DValue::List(l) => { + out.extend_from_slice(b"VlLs"); + put_u32(out, l.len() as u32); + for x in l { + write_value(out, x); + } + } + DValue::Double(x) => { + out.extend_from_slice(b"doub"); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloat(u, x) => { + out.extend_from_slice(b"UntF"); + out.extend_from_slice(&tag4(u)); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloats(u, xs) => { + out.extend_from_slice(b"UnFl"); + out.extend_from_slice(&tag4(u)); + put_u32(out, xs.len() as u32); + for x in xs { + out.extend_from_slice(&x.to_be_bytes()); + } + } + DValue::Text(s) => { + out.extend_from_slice(b"TEXT"); + put_unicode(out, s); + } + DValue::Enum(t, e) => { + out.extend_from_slice(b"enum"); + put_id(out, t); + put_id(out, e); + } + DValue::Integer(i) => { + out.extend_from_slice(b"long"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::LargeInteger(i) => { + out.extend_from_slice(b"comp"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::Bool(b) => { + out.extend_from_slice(b"bool"); + out.push(*b as u8); + } + DValue::Class(n, c) => { + out.extend_from_slice(b"type"); + put_unicode(out, n); + put_id(out, c); + } + DValue::Raw(b) => { + out.extend_from_slice(b"tdta"); + put_u32(out, b.len() as u32); + out.extend_from_slice(b); + } + DValue::Reference(items) => { + out.extend_from_slice(b"obj "); + put_u32(out, items.len() as u32); + for it in items { + match it { + RefItem::Property(c, k) => { + out.extend_from_slice(b"prop"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, k); + } + RefItem::Class(c) => { + out.extend_from_slice(b"Clss"); + put_unicode(out, ""); + put_id(out, c); + } + RefItem::Enum(c, t, e) => { + out.extend_from_slice(b"Enmr"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, t); + put_id(out, e); + } + RefItem::Offset(c, o) => { + out.extend_from_slice(b"rele"); + put_unicode(out, ""); + put_id(out, c); + out.extend_from_slice(&o.to_be_bytes()); + } + RefItem::Identifier(i) => { + out.extend_from_slice(b"Idnt"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Index(i) => { + out.extend_from_slice(b"indx"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Name(c, n) => { + out.extend_from_slice(b"name"); + put_unicode(out, ""); + put_id(out, c); + put_unicode(out, n); + } + } + } + } + } +} + +fn tag4(s: &str) -> [u8; 4] { + let mut t = *b" "; + for (i, b) in s.bytes().take(4).enumerate() { + t[i] = b; + } + t +} + +/// Builder helpers for descriptors (tests and the writer). +impl Descriptor { + pub fn new(class_id: &str) -> Descriptor { + Descriptor { class_name: String::new(), class_id: class_id.into(), items: vec![] } + } + pub fn with(mut self, key: &str, v: DValue) -> Descriptor { + self.items.push((key.into(), v)); + self + } + /// An `RGBC` colour object from 0..1 values. + pub fn rgb(c: [f64; 3]) -> Descriptor { + Descriptor::new("RGBC").with("Rd ", DValue::Double(c[0] * 255.0)).with("Grn ", DValue::Double(c[1] * 255.0)).with("Bl ", DValue::Double(c[2] * 255.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + let d = Descriptor::new("null") + .with("Txt ", DValue::Text("Héllo".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 0.5, 0.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 75.0)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("enab", DValue::Bool(true)) + .with("longKeyName", DValue::List(vec![DValue::Integer(3), DValue::Double(1.5)])) + .with("EngineData", DValue::Raw(b"<< >>".to_vec())) + .with("ref", DValue::Reference(vec![RefItem::Enum("Lyr ".into(), "Ordn".into(), "Trgt".into()), RefItem::Index(2)])); + let mut out = vec![]; + write_descriptor(&mut out, &d); + let back = read_descriptor(&mut Reader::new(&out)).unwrap(); + assert_eq!(back.class_id, "null"); + assert_eq!(back.text("Txt "), Some("Héllo")); + assert_eq!(back.color("Clr "), Some([1.0, 0.5, 0.0])); + assert_eq!(back.num("Opct"), Some(75.0)); + assert_eq!(back.enum_value("Md "), Some("Mltp")); + assert_eq!(back.bool("enab"), Some(true)); + assert_eq!(back.raw("EngineData"), Some(&b"<< >>"[..])); + assert_eq!(back, d); + } +} diff --git a/crates/psd/src/engine_data.rs b/crates/psd/src/engine_data.rs new file mode 100644 index 00000000..c4f23038 --- /dev/null +++ b/crates/psd/src/engine_data.rs @@ -0,0 +1,288 @@ +//! The text engine data of type layers (`EngineData` in the `TySh` descriptor): a PostScript-like +//! tree of dictionaries (`<< /Key value >>`), arrays (`[ ]`), numbers, names, booleans and +//! strings (`(þÿ` + UTF-16BE with backslash escapes `)`). Read best-effort. + +#[derive(Clone, Debug, PartialEq)] +pub enum EValue { + Dict(Vec<(String, EValue)>), + Array(Vec), + Num(f64), + Bool(bool), + Name(String), + Str(String), +} + +impl EValue { + pub fn get(&self, key: &str) -> Option<&EValue> { + match self { + EValue::Dict(v) => v.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + pub fn idx(&self, i: usize) -> Option<&EValue> { + match self { + EValue::Array(v) => v.get(i), + _ => None, + } + } + /// Follow a `/`-separated path; numeric segments index arrays. + pub fn path(&self, p: &str) -> Option<&EValue> { + let mut cur = self; + for seg in p.split('/').filter(|s| !s.is_empty()) { + cur = match seg.parse::() { + Ok(i) => cur.idx(i)?, + Err(_) => cur.get(seg)?, + }; + } + Some(cur) + } + pub fn num(&self) -> Option { + match self { + EValue::Num(n) => Some(*n), + _ => None, + } + } + pub fn str(&self) -> Option<&str> { + match self { + EValue::Str(s) | EValue::Name(s) => Some(s), + _ => None, + } + } +} + +struct P<'a> { + b: &'a [u8], + i: usize, +} + +impl P<'_> { + fn ws(&mut self) { + while self.i < self.b.len() && (self.b[self.i].is_ascii_whitespace() || self.b[self.i] == 0) { + self.i += 1; + } + } + fn peek(&self, s: &[u8]) -> bool { + self.b[self.i..].starts_with(s) + } + fn value(&mut self, depth: usize) -> Option { + if depth > 128 { + return None; + } + self.ws(); + if self.i >= self.b.len() { + return None; + } + if self.peek(b"<<") { + self.i += 2; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Dict(v)); + } + if self.peek(b">>") { + self.i += 2; + return Some(EValue::Dict(v)); + } + if self.b[self.i] != b'/' { + // Unexpected token: skip it. + self.i += 1; + continue; + } + let key = self.name(); + let val = self.value(depth + 1)?; + v.push((key, val)); + } + } + match self.b[self.i] { + b'[' => { + self.i += 1; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Array(v)); + } + if self.b[self.i] == b']' { + self.i += 1; + return Some(EValue::Array(v)); + } + let start = self.i; + match self.value(depth + 1) { + Some(x) => v.push(x), + None => return Some(EValue::Array(v)), + } + if self.i == start { + self.i += 1; + } + } + } + b'/' => Some(EValue::Name(self.name())), + b'(' => Some(EValue::Str(self.string())), + _ => { + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + let tok = std::str::from_utf8(&self.b[start..self.i]).unwrap_or(""); + if self.i == start { + self.i += 1; + return Some(EValue::Name(String::new())); + } + Some(match tok { + "true" => EValue::Bool(true), + "false" => EValue::Bool(false), + t => t.parse::().map(EValue::Num).unwrap_or_else(|_| EValue::Name(t.to_string())), + }) + } + } + } + fn name(&mut self) -> String { + self.i += 1; // '/' + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + String::from_utf8_lossy(&self.b[start..self.i]).into_owned() + } + fn string(&mut self) -> String { + self.i += 1; // '(' + let mut raw = vec![]; + while self.i < self.b.len() { + let c = self.b[self.i]; + self.i += 1; + match c { + b'\\' if self.i < self.b.len() => { + let e = self.b[self.i]; + self.i += 1; + raw.push(match e { + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + other => other, + }); + } + b')' => break, + _ => raw.push(c), + } + } + if raw.starts_with(&[0xFE, 0xFF]) { + let u: Vec = raw[2..].chunks_exact(2).map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + String::from_utf16_lossy(&u).trim_end_matches('\0').to_string() + } else { + String::from_utf8_lossy(&raw).into_owned() + } + } +} + +/// Parse engine data (returns an empty dictionary on garbage). +pub fn parse(b: &[u8]) -> EValue { + let mut p = P { b, i: 0 }; + p.value(0).unwrap_or(EValue::Dict(vec![])) +} + +/// Encode a string the way engine data stores it (UTF-16BE with a BOM, escaped). +pub fn encode_string(s: &str) -> Vec { + let mut out = vec![b'(', 0xFE, 0xFF]; + for u in s.encode_utf16() { + for b in u.to_be_bytes() { + if matches!(b, b'(' | b')' | b'\\') { + out.push(b'\\'); + } + out.push(b); + } + } + out.push(b')'); + out +} + +/// The first style run's character attributes. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextStyle { + pub font: Option, + pub size: Option, + /// RGB 0..1. + pub color: Option<[f64; 3]>, + /// 0 left, 1 right, 2 centre, 3+ justified variants. + pub justification: Option, + pub tracking: Option, + pub leading: Option, + pub faux_bold: bool, + pub faux_italic: bool, + /// Paragraph (box) text rather than point text. + pub box_text: bool, +} + +pub fn text_style(ed: &EValue) -> TextStyle { + let engine = ed.get("EngineDict").unwrap_or(ed); + let resources = ed.get("ResourceDict").or_else(|| ed.get("DocumentResources")); + let run = engine.path("StyleRun/RunArray/0/StyleSheet/StyleSheetData"); + let normal = resources.and_then(|r| r.path("StyleSheetSet/0/StyleSheetData")); + let attr = |k: &str| run.and_then(|r| r.get(k)).or_else(|| normal.and_then(|n| n.get(k))); + let font = + attr("Font").and_then(EValue::num).and_then(|i| resources?.path(&format!("FontSet/{}/Name", i as usize))).and_then(EValue::str).map(str::to_string); + let color = attr("FillColor").and_then(|c| c.get("Values")).and_then(|v| match v { + EValue::Array(a) if a.len() >= 4 => Some([a[1].num()?, a[2].num()?, a[3].num()?]), + _ => None, + }); + let para = engine.path("ParagraphRun/RunArray/0/ParagraphSheet/Properties"); + let para_normal = resources.and_then(|r| r.path("ParagraphSheetSet/0/Properties")); + let justification = + para.and_then(|p| p.get("Justification")).or_else(|| para_normal.and_then(|p| p.get("Justification"))).and_then(EValue::num).map(|n| n as u8); + let auto_leading = attr("AutoLeading").is_none_or(|v| *v != EValue::Bool(false)); + let box_text = engine.path("Rendered/Shapes/Children/0/ShapeType").and_then(EValue::num) == Some(1.0); + TextStyle { + font, + size: attr("FontSize").and_then(EValue::num), + color, + justification, + tracking: attr("Tracking").and_then(EValue::num), + leading: if auto_leading { None } else { attr("Leading").and_then(EValue::num) }, + faux_bold: attr("FauxBold") == Some(&EValue::Bool(true)), + faux_italic: attr("FauxItalic") == Some(&EValue::Bool(true)), + box_text, + } +} + +/// Minimal engine data for a type layer with one style run (fixtures, and what a writer needs). +pub fn minimal(text: &str, font: &str, size: f64, color: [f64; 3], justification: u8) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(b"\n\n<<\n\t/EngineDict\n\t<<\n\t\t/Editor << /Text "); + out.extend_from_slice(&encode_string(&format!("{text}\r"))); + out.extend_from_slice(b" >>\n\t\t/ParagraphRun << /RunArray [ << /ParagraphSheet << /Properties << /Justification "); + out.extend_from_slice(justification.to_string().as_bytes()); + out.extend_from_slice(b" >> >> >> ] >>\n\t\t/StyleRun << /RunArray [ << /StyleSheet << /StyleSheetData << /Font 0 /FontSize "); + out.extend_from_slice(format!("{size:.1}").as_bytes()); + out.extend_from_slice( + format!(" /FillColor << /Type 1 /Values [ 1.0 {:.5} {:.5} {:.5} ] >> /AutoLeading true >> >> >> ] >>\n\t>>\n", color[0], color[1], color[2]).as_bytes(), + ); + out.extend_from_slice(b"\t/ResourceDict << /FontSet [ << /Name "); + out.extend_from_slice(&encode_string(font)); + out.extend_from_slice(b" /Type 0 >> ] >>\n>>"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_minimal() { + let b = minimal("Hi (there)", "Arial-BoldMT", 36.0, [1.0, 0.5, 0.0], 2); + let v = parse(&b); + assert_eq!(v.path("EngineDict/Editor/Text").and_then(EValue::str), Some("Hi (there)\r")); + let st = text_style(&v); + assert_eq!(st.font.as_deref(), Some("Arial-BoldMT")); + assert_eq!(st.size, Some(36.0)); + assert_eq!(st.color, Some([1.0, 0.5, 0.0])); + assert_eq!(st.justification, Some(2)); + assert!(!st.box_text); + } + + #[test] + fn garbage_is_harmless() { + let _ = parse(b"<< /A [ 1 2 << /B (unterminated"); + let _ = parse(b"]]]>>>> /x"); + let _ = parse(&[0xFF; 64]); + } +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs new file mode 100644 index 00000000..885ab3c2 --- /dev/null +++ b/crates/psd/src/lib.rs @@ -0,0 +1,965 @@ +//! Photoshop document (PSD / PSB) reader, implemented from Adobe's public *Photoshop File Formats +//! Specification* (see README.md), and a minimal writer ([`write`]) for fixtures. +//! +//! [`Psd::parse`] reads the structure (header, layer records and their additional information, +//! the layer tree) without decoding pixels; [`Psd::composite`] decodes the merged image and +//! [`Psd::layer_pixels`] one layer (with its layer mask applied to alpha) on demand. Pixels come +//! out as straight (unpremultiplied) RGBA `f32` in 0..1, display-encoded (32-bit documents, +//! which are linear, are encoded to sRGB). + +pub mod descriptor; +pub mod engine_data; +mod read; +pub mod write; + +use std::sync::Arc; + +pub use descriptor::{DValue, Descriptor}; +use read::Reader; + +#[derive(Debug, Clone, PartialEq)] +pub enum Error { + NotPsd, + Truncated, + Unsupported(String), + Invalid(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::NotPsd => write!(f, "not a Photoshop document"), + Error::Truncated => write!(f, "truncated Photoshop document"), + Error::Unsupported(s) => write!(f, "unsupported Photoshop feature: {s}"), + Error::Invalid(s) => write!(f, "invalid Photoshop document: {s}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Whether `bytes` start like a Photoshop document. +pub fn is_psd(bytes: &[u8]) -> bool { + bytes.len() >= 6 && &bytes[..4] == b"8BPS" && matches!(u16::from_be_bytes([bytes[4], bytes[5]]), 1 | 2) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ColorMode { + Bitmap, + Grayscale, + Indexed, + Rgb, + Cmyk, + Multichannel, + Duotone, + Lab, +} + +impl ColorMode { + pub fn from_u16(v: u16) -> Option { + Some(match v { + 0 => ColorMode::Bitmap, + 1 => ColorMode::Grayscale, + 2 => ColorMode::Indexed, + 3 => ColorMode::Rgb, + 4 => ColorMode::Cmyk, + 7 => ColorMode::Multichannel, + 8 => ColorMode::Duotone, + 9 => ColorMode::Lab, + _ => return None, + }) + } + pub fn to_u16(self) -> u16 { + match self { + ColorMode::Bitmap => 0, + ColorMode::Grayscale => 1, + ColorMode::Indexed => 2, + ColorMode::Rgb => 3, + ColorMode::Cmyk => 4, + ColorMode::Multichannel => 7, + ColorMode::Duotone => 8, + ColorMode::Lab => 9, + } + } + /// Number of colour channels (before alpha). + pub fn color_channels(self) -> usize { + match self { + ColorMode::Rgb | ColorMode::Lab => 3, + ColorMode::Cmyk => 4, + _ => 1, + } + } +} + +/// A rectangle in document pixels (`right`/`bottom` exclusive). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Rect { + pub top: i32, + pub left: i32, + pub bottom: i32, + pub right: i32, +} + +impl Rect { + pub fn new(left: i32, top: i32, width: i32, height: i32) -> Rect { + Rect { top, left, bottom: top + height, right: left + width } + } + pub fn width(&self) -> u32 { + (self.right - self.left).max(0) as u32 + } + pub fn height(&self) -> u32 { + (self.bottom - self.top).max(0) as u32 + } + pub fn is_empty(&self) -> bool { + self.width() == 0 || self.height() == 0 + } +} + +/// One channel of a layer: id (0.. colour, -1 transparency, -2 layer mask, -3 real user mask) and +/// where its data (compression + image data) sits in the file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ChannelInfo { + pub id: i16, + pub offset: usize, + pub len: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LayerMask { + pub rect: Rect, + /// Value outside the mask rectangle (0 or 255). + pub default_color: u8, + pub disabled: bool, + pub inverted: bool, +} + +/// Layer kind from the section divider setting (`lsct`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Section { + #[default] + Layer, + OpenFolder, + ClosedFolder, + /// The hidden "" record that closes a group (below its children). + Divider, +} + +/// A type layer (`TySh`). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextInfo { + /// Text with `\n` line breaks. + pub text: String, + /// Affine transform (xx, xy, yx, yy, tx, ty): the text origin sits at (tx, ty). + pub transform: [f64; 6], + pub style: engine_data::TextStyle, + /// Text bounds (left, top, right, bottom) relative to the origin, when stored. + pub bounds: Option<[f64; 4]>, +} + +/// Adjustment layers (best effort). +#[derive(Clone, Debug, PartialEq)] +pub enum Adjustment { + Invert, + BrightnessContrast { + brightness: f64, + contrast: f64, + legacy: bool, + }, + HueSaturation { + hue: f64, + saturation: f64, + lightness: f64, + colorize: bool, + }, + Levels { + in_black: f64, + in_white: f64, + gamma: f64, + out_black: f64, + out_white: f64, + }, + Exposure { + exposure: f64, + offset: f64, + gamma: f64, + }, + /// Another adjustment (its tagged-block key, e.g. `curv`, `blnc`). + Other(String), +} + +/// A vector mask path: sub-paths of (in tangent, vertex, out tangent) points in document pixels. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct VectorMask { + pub inverted: bool, + pub disabled: bool, + pub subpaths: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SubPath { + pub closed: bool, + /// Per knot: [control point preceding, anchor, control point leaving] as (x, y). + pub knots: Vec<[[f64; 2]; 3]>, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Layer { + /// Index in [`Psd::layers`] (file order: bottom of the stack first). + pub index: usize, + pub name: String, + pub id: Option, + pub rect: Rect, + /// Blend mode key (`norm`, `mul `, `pass`…). + pub blend: [u8; 4], + pub opacity: u8, + /// Fill opacity (`iOpa`), 255 when absent. + pub fill_opacity: u8, + /// Clipped to the layer below. + pub clipping: bool, + pub hidden: bool, + pub protect_transparency: bool, + pub section: Section, + pub channels: Vec, + pub mask: Option, + pub text: Option, + pub adjustment: Option, + /// Solid colour fill layer (`SoCo`), RGB 0..1. + pub fill: Option<[f64; 3]>, + /// Layer effects (`lfx2` descriptor). + pub effects: Option, + pub vector_mask: Option, + /// Keys of every additional-information block the layer carries. + pub keys: Vec<[u8; 4]>, +} + +impl Layer { + pub fn is_group(&self) -> bool { + matches!(self.section, Section::OpenFolder | Section::ClosedFolder) + } + pub fn blend_key(&self) -> &str { + std::str::from_utf8(&self.blend).unwrap_or("norm") + } + pub fn has_pixels(&self) -> bool { + !self.rect.is_empty() && self.channels.iter().any(|c| c.id >= 0) + } +} + +/// The layer tree, top of the stack first. +#[derive(Clone, Debug, PartialEq)] +pub enum Node { + Layer(usize), + Group { layer: usize, children: Vec }, +} + +/// Decoded pixels: straight RGBA, row-major. +#[derive(Clone, Debug, PartialEq)] +pub struct Pixels { + pub width: u32, + pub height: u32, + pub data: Vec<[f32; 4]>, +} + +/// A parsed Photoshop document. +#[derive(Clone, Debug)] +pub struct Psd { + /// PSB (large document format). + pub psb: bool, + pub width: u32, + pub height: u32, + pub channels: u16, + pub depth: u16, + pub mode: ColorMode, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// The merged image's first extra channel is transparency. + pub merged_alpha: bool, + /// Indexed colour palette (256 R, then G, then B). + pub palette: Vec, + data: Arc<[u8]>, + merged_offset: usize, +} + +impl Psd { + pub fn parse(data: impl Into>) -> Result { + let data: Arc<[u8]> = data.into(); + let mut r = Reader::new(&data); + if r.bytes(4).ok() != Some(b"8BPS") { + return Err(Error::NotPsd); + } + let version = r.u16()?; + let psb = match version { + 1 => false, + 2 => true, + _ => return Err(Error::NotPsd), + }; + r.skip(6)?; + let channels = r.u16()?; + let height = r.u32()?; + let width = r.u32()?; + let depth = r.u16()?; + let mode_v = r.u16()?; + let mode = ColorMode::from_u16(mode_v).ok_or_else(|| Error::Unsupported(format!("colour mode {mode_v}")))?; + if !matches!(depth, 1 | 8 | 16 | 32) { + return Err(Error::Unsupported(format!("{depth}-bit"))); + } + if width == 0 || height == 0 || width > 300_000 || height > 300_000 { + return Err(Error::Invalid(format!("size {width}x{height}"))); + } + // Colour mode data. + let n = r.u32()? as usize; + let palette = r.bytes(n)?.to_vec(); + // Image resources. + let n = r.u32()? as usize; + r.skip(n)?; + // Layer and mask information. + let mut psd = Psd { psb, width, height, channels, depth, mode, layers: vec![], merged_alpha: false, palette, data: data.clone(), merged_offset: 0 }; + let lm_len = r.length(psb)?; + let lm_end = r.pos.checked_add(lm_len).filter(|&e| e <= data.len()).ok_or(Error::Truncated)?; + if lm_len > 0 { + let li_len = r.length(psb)?; + let li_end = r.pos.checked_add(li_len).filter(|&e| e <= lm_end).ok_or(Error::Truncated)?; + if li_len > 0 { + psd.layer_info(&data, r.pos, li_end)?; + } + r.pos = li_end; + if r.pos + 4 <= lm_end { + let g = r.u32()? as usize; + r.pos = (r.pos + g).min(lm_end); + } + // Global additional layer information (16/32-bit layer info lives here). + while let Some((key, start, end)) = next_block(&data, &mut r.pos, lm_end, psb) { + if matches!(&key, b"Lr16" | b"Lr32" | b"Layr") && psd.layers.is_empty() && end > start { + // The block holds the layer info, normally without its length field; accept + // both layouts. + if psd.layer_info(&data, start, end).is_err() || psd.layers.is_empty() { + psd.layers.clear(); + let skip = if psb { 8 } else { 4 }; + if start + skip < end && psd.layer_info(&data, start + skip, end).is_err() { + psd.layers.clear(); + } + } + } + r.pos = end; + } + } + psd.merged_offset = lm_end; + Ok(psd) + } + + fn layer_info(&mut self, data: &[u8], start: usize, end: usize) -> Result<()> { + let mut r = Reader::at(&data[..end], start); + let count = r.i16()?; + self.merged_alpha = count < 0; + let n = count.unsigned_abs() as usize; + let mut layers = Vec::with_capacity(n); + for i in 0..n { + layers.push(self.layer_record(&mut r, i)?); + } + // Channel image data follows in record order. + for l in &mut layers { + for c in &mut l.channels { + c.offset = r.pos; + r.skip(c.len)?; + } + } + self.layers = layers; + Ok(()) + } + + fn layer_record(&self, r: &mut Reader, index: usize) -> Result { + let psb = self.psb; + let (top, left, bottom, right) = (r.i32()?, r.i32()?, r.i32()?, r.i32()?); + let nch = r.u16()? as usize; + if nch > 64 { + return Err(Error::Invalid("channel count".into())); + } + let mut channels = Vec::with_capacity(nch); + for _ in 0..nch { + let id = r.i16()?; + let len = r.length(psb)?; + channels.push(ChannelInfo { id, offset: 0, len }); + } + if r.bytes(4)? != b"8BIM" { + return Err(Error::Invalid("layer record signature".into())); + } + let blend = r.tag()?; + let opacity = r.u8()?; + let clipping = r.u8()? != 0; + let flags = r.u8()?; + r.skip(1)?; + let extra = r.u32()? as usize; + let extra_end = r.pos.checked_add(extra).filter(|&e| e <= r.data.len()).ok_or(Error::Truncated)?; + let mut l = Layer { + index, + rect: Rect { top, left, bottom, right }, + blend, + opacity, + fill_opacity: 255, + clipping, + hidden: flags & 2 != 0, + protect_transparency: flags & 1 != 0, + channels, + ..Default::default() + }; + // Layer mask data. + let mlen = r.u32()? as usize; + let mstart = r.pos; + if mlen >= 18 { + let rect = Rect { top: r.i32()?, left: r.i32()?, bottom: r.i32()?, right: r.i32()? }; + let default_color = r.u8()?; + let mflags = r.u8()?; + l.mask = Some(LayerMask { rect, default_color, disabled: mflags & 2 != 0, inverted: mflags & 4 != 0 }); + } + r.pos = mstart + mlen; + // Blending ranges. + let blen = r.u32()? as usize; + r.skip(blen)?; + l.name = r.pascal(4)?; + let mut pos = r.pos; + let data = r.data; + while let Some((key, start, end)) = next_block(data, &mut pos, extra_end, psb) { + l.keys.push(key); + let block = &data[start..end]; + // A malformed block only loses that block's information. + let _ = self.layer_block(&mut l, &key, block); + pos = end; + } + r.pos = extra_end; + Ok(l) + } + + fn layer_block(&self, l: &mut Layer, key: &[u8; 4], b: &[u8]) -> Result<()> { + let mut r = Reader::new(b); + match key { + b"luni" => { + let name = descriptor::read_unicode(&mut r)?; + if !name.is_empty() { + l.name = name; + } + } + b"lyid" => l.id = Some(r.u32()?), + b"iOpa" => l.fill_opacity = r.u8()?, + b"lsct" | b"lsdk" => { + l.section = match r.u32()? { + 1 => Section::OpenFolder, + 2 => Section::ClosedFolder, + 3 => Section::Divider, + _ => Section::Layer, + }; + if b.len() >= 12 && &b[4..8] == b"8BIM" { + l.blend.copy_from_slice(&b[8..12]); + } + } + b"TySh" => { + let _ver = r.u16()?; + let mut t = [0.0; 6]; + for v in &mut t { + *v = r.f64()?; + } + let _text_ver = r.u16()?; + let _desc_ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + let text = d.text("Txt ").unwrap_or_default().replace("\r\n", "\n").replace('\r', "\n"); + let text = text.strip_suffix('\n').map(str::to_string).unwrap_or(text); + let style = d.raw("EngineData").map(|e| engine_data::text_style(&engine_data::parse(e))).unwrap_or_default(); + let bounds = + d.obj("bounds").or_else(|| d.obj("boundingBox")).and_then(|o| Some([o.num("Left")?, o.num("Top ")?, o.num("Rght")?, o.num("Btom")?])); + l.text = Some(TextInfo { text, transform: t, style, bounds }); + } + b"lfx2" => { + let _ver = r.u32()?; + let _dver = r.u32()?; + l.effects = Some(descriptor::read_descriptor(&mut r)?); + } + b"SoCo" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.fill = d.color("Clr "); + } + b"nvrt" => l.adjustment = Some(Adjustment::Invert), + b"brit" => { + let brightness = r.i16()? as f64; + let contrast = r.i16()? as f64; + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::BrightnessContrast { brightness, contrast, legacy: true }); + } + } + b"CgEd" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.adjustment = Some(Adjustment::BrightnessContrast { + brightness: d.num("Brgh").unwrap_or(0.0), + contrast: d.num("Cntr").unwrap_or(0.0), + legacy: d.bool("useLegacy").unwrap_or(false), + }); + } + b"hue2" | b"hue " => { + let _ver = r.u16()?; + let colorize = r.u8()? != 0; + r.skip(1)?; + let ch = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let m = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let (hue, saturation, lightness) = if colorize { ch } else { m }; + l.adjustment = Some(Adjustment::HueSaturation { hue, saturation, lightness, colorize }); + } + b"levl" => { + let _ver = r.u16()?; + let (ib, iw, ob, ow, g) = (r.i16()?, r.i16()?, r.i16()?, r.i16()?, r.i16()?); + l.adjustment = + Some(Adjustment::Levels { in_black: ib as f64, in_white: iw as f64, gamma: g as f64 / 100.0, out_black: ob as f64, out_white: ow as f64 }); + } + b"expA" => { + let _ver = r.u16()?; + let exposure = r.f32()? as f64; + let offset = r.f32()? as f64; + let gamma = r.f32()? as f64; + l.adjustment = Some(Adjustment::Exposure { exposure, offset, gamma }); + } + b"curv" | b"blnc" | b"post" | b"thrs" | b"selc" | b"mixr" | b"phfl" | b"vibA" | b"blwh" | b"grdm" | b"clrL" => { + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::Other(String::from_utf8_lossy(key).into_owned())); + } + } + b"vmsk" | b"vsms" => { + let _ver = r.u32()?; + let flags = r.u32()?; + l.vector_mask = Some(self.vector_paths(&mut r, flags)?); + } + _ => {} + } + Ok(()) + } + + fn vector_paths(&self, r: &mut Reader, flags: u32) -> Result { + let (w, h) = (self.width as f64, self.height as f64); + let mut vm = VectorMask { inverted: flags & 1 != 0, disabled: flags & 4 != 0, subpaths: vec![] }; + let fixed = |r: &mut Reader| -> Result { Ok(r.i32()? as f64 / (1 << 24) as f64) }; + while r.remaining() >= 26 { + let sel = r.u16()?; + match sel { + 0 | 3 => { + let _n = r.u16()?; + r.skip(22)?; + vm.subpaths.push(SubPath { closed: sel == 0, knots: vec![] }); + } + 1 | 2 | 4 | 5 => { + let mut pts = [[0.0; 2]; 3]; + for p in &mut pts { + let y = fixed(r)?; + let x = fixed(r)?; + *p = [x * w, y * h]; + } + if let Some(sp) = vm.subpaths.last_mut() { + sp.knots.push(pts); + } + } + _ => r.skip(24)?, + } + } + vm.subpaths.retain(|s| !s.knots.is_empty()); + Ok(vm) + } + + /// The layer tree, top of the stack first. + pub fn tree(&self) -> Vec { + let mut stack: Vec> = vec![vec![]]; + for l in &self.layers { + match l.section { + Section::Divider => stack.push(vec![]), + Section::OpenFolder | Section::ClosedFolder => { + let children = if stack.len() > 1 { stack.pop().unwrap_or_default() } else { vec![] }; + if let Some(top) = stack.last_mut() { + top.push(Node::Group { layer: l.index, children }); + } + } + Section::Layer => { + if let Some(top) = stack.last_mut() { + top.push(Node::Layer(l.index)); + } + } + } + } + while stack.len() > 1 { + let c = stack.pop().unwrap_or_default(); + if let Some(top) = stack.last_mut() { + top.extend(c); + } + } + fn rev(v: &mut Vec) { + v.reverse(); + for n in v { + if let Node::Group { children, .. } = n { + rev(children); + } + } + } + let mut out = stack.pop().unwrap_or_default(); + rev(&mut out); + out + } + + fn bytes_per_sample(&self) -> usize { + match self.depth { + 16 => 2, + 32 => 4, + _ => 1, + } + } + + /// Decode one channel of `w`×`h` samples at `offset` (compression field first). Values 0..1. + fn decode_channel(&self, offset: usize, len: usize, w: usize, h: usize) -> Result> { + let end = offset.checked_add(len).filter(|&e| e <= self.data.len()).ok_or(Error::Truncated)?; + let mut r = Reader::at(&self.data[..end], offset); + let comp = r.u16()?; + let rowb = self.row_bytes(w); + let raw = match comp { + 0 => r.bytes(rowb * h)?.to_vec(), + 1 => { + let mut counts = Vec::with_capacity(h); + for _ in 0..h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + let mut out = Vec::with_capacity(rowb * h); + for c in counts { + unpack_bits(r.bytes(c)?, rowb, &mut out)?; + } + out + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let mut v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + v.resize(rowb * h, 0); + if comp == 3 { + self.unpredict(&mut v, w, h); + } + v + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + }; + Ok(self.samples(&raw, w, h)) + } + + fn row_bytes(&self, w: usize) -> usize { + if self.depth == 1 { w.div_ceil(8) } else { w * self.bytes_per_sample() } + } + + fn unpredict(&self, v: &mut [u8], w: usize, h: usize) { + match self.depth { + 16 => { + for row in v.chunks_exact_mut(w * 2).take(h) { + let mut prev = 0u16; + for c in row.chunks_exact_mut(2) { + let x = u16::from_be_bytes([c[0], c[1]]).wrapping_add(prev); + c.copy_from_slice(&x.to_be_bytes()); + prev = x; + } + } + } + 32 => { + for row in v.chunks_exact_mut(w * 4).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + // Bytes are stored plane by plane (all first bytes, then all second bytes…). + let planar = row.to_vec(); + for x in 0..w { + for b in 0..4 { + row[x * 4 + b] = planar[b * w + x]; + } + } + } + } + _ => { + for row in v.chunks_exact_mut(w).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + } + } + } + } + + fn samples(&self, raw: &[u8], w: usize, h: usize) -> Vec { + match self.depth { + 1 => { + let rowb = w.div_ceil(8); + let mut out = Vec::with_capacity(w * h); + for y in 0..h { + for x in 0..w { + let b = raw.get(y * rowb + x / 8).copied().unwrap_or(0); + // 1 = black. + out.push(if b & (0x80 >> (x % 8)) != 0 { 0.0 } else { 1.0 }); + } + } + out + } + 16 => raw.chunks_exact(2).take(w * h).map(|c| u16::from_be_bytes([c[0], c[1]]) as f32 / 65535.0).collect(), + 32 => raw.chunks_exact(4).take(w * h).map(|c| f32::from_be_bytes([c[0], c[1], c[2], c[3]])).collect(), + _ => raw.iter().take(w * h).map(|&b| b as f32 / 255.0).collect(), + } + } + + /// Convert colour channels (in the document's mode) to RGB. + fn to_rgb(&self, ch: &[Option>], i: usize) -> [f32; 3] { + let g = |k: usize| ch.get(k).and_then(|c| c.as_ref()).map_or(0.0, |c| c[i]); + let rgb = match self.mode { + ColorMode::Rgb => [g(0), g(1), g(2)], + ColorMode::Cmyk => { + // Stored inverted (1 = no ink). + let k = g(3); + [g(0) * k, g(1) * k, g(2) * k] + } + ColorMode::Indexed if self.palette.len() >= 768 => { + let idx = (g(0) * 255.0).round() as usize & 255; + [self.palette[idx] as f32 / 255.0, self.palette[256 + idx] as f32 / 255.0, self.palette[512 + idx] as f32 / 255.0] + } + ColorMode::Lab => lab_to_rgb(g(0) * 100.0, g(1) * 255.0 - 128.0, g(2) * 255.0 - 128.0), + _ => [g(0); 3], + }; + if self.depth == 32 && self.mode != ColorMode::Lab { rgb.map(srgb_encode) } else { rgb } + } + + /// The merged (flattened) image. + pub fn composite(&self) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let mut r = Reader::at(&self.data, self.merged_offset); + let comp = r.u16()?; + let nch = self.channels as usize; + let cc = self.mode.color_channels(); + let want = (cc + usize::from(self.merged_alpha)).min(nch); + let rowb = self.row_bytes(w); + let mut planes: Vec>> = vec![None; nch]; + match comp { + 0 => { + for p in planes.iter_mut().take(want) { + *p = Some(self.samples(r.bytes(rowb * h)?, w, h)); + } + } + 1 => { + let mut counts = Vec::with_capacity(nch * h); + for _ in 0..nch * h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut out = Vec::with_capacity(rowb * h); + for row in 0..h { + unpack_bits(r.bytes(counts[c * h + row])?, rowb, &mut out)?; + } + *p = Some(self.samples(&out, w, h)); + } + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut plane = v.get(c * rowb * h..(c + 1) * rowb * h).ok_or(Error::Truncated)?.to_vec(); + if comp == 3 { + self.unpredict(&mut plane, w, h); + } + *p = Some(self.samples(&plane, w, h)); + } + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + } + let alpha = if self.merged_alpha && nch > cc { planes[cc].take() } else { None }; + let mut data = Vec::with_capacity(w * h); + for i in 0..w * h { + let [r, g, b] = self.to_rgb(&planes, i); + let a = alpha.as_ref().map_or(1.0, |a| a[i]); + data.push([r, g, b, a]); + } + Ok(Pixels { width: w as u32, height: h as u32, data }) + } + + /// One layer's pixels with its layer mask (and vector-less fill colour) applied to alpha. + /// `canvas`: document-sized with the layer in place; otherwise the layer's own bounds. + pub fn layer_pixels(&self, index: usize, canvas: bool) -> Result { + let l = self.layers.get(index).ok_or_else(|| Error::Invalid(format!("no layer {index}")))?; + let doc = Rect { top: 0, left: 0, bottom: self.height as i32, right: self.width as i32 }; + // Fill layers often have no pixel data: they cover the canvas (or the mask). + let fill_rect = + if l.fill.is_some() && !l.has_pixels() { l.mask.filter(|m| !m.rect.is_empty() && m.default_color == 0).map_or(doc, |m| m.rect) } else { l.rect }; + let out_rect = if canvas { + doc + } else if fill_rect.is_empty() { + Rect { top: 0, left: 0, bottom: 1, right: 1 } + } else { + fill_rect + }; + let (ow, oh) = (out_rect.width() as usize, out_rect.height() as usize); + let mut out = vec![[0.0f32; 4]; ow * oh]; + let src = fill_rect; + let (sw, sh) = (src.width() as usize, src.height() as usize); + if sw > 0 && sh > 0 { + let mut planes: Vec>> = vec![None; self.mode.color_channels()]; + let mut alpha = None; + if l.has_pixels() { + for c in &l.channels { + match c.id { + -1 => alpha = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + id if id >= 0 && (id as usize) < planes.len() => planes[id as usize] = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + _ => {} + } + } + } + let fill = l.fill.map(|c| c.map(|v| v as f32)); + for y in 0..sh { + let dy = src.top + y as i32 - out_rect.top; + if dy < 0 || dy >= oh as i32 { + continue; + } + for x in 0..sw { + let dx = src.left + x as i32 - out_rect.left; + if dx < 0 || dx >= ow as i32 { + continue; + } + let i = y * sw + x; + let (rgb, a) = match fill { + Some(f) if !l.has_pixels() => (f, 1.0), + _ => (self.to_rgb(&planes, i), alpha.as_ref().map_or(1.0, |a| a[i])), + }; + out[dy as usize * ow + dx as usize] = [rgb[0], rgb[1], rgb[2], a]; + } + } + } + // Layer mask: multiplies alpha (also outside the mask rect, by its default colour). + if let Some(m) = l.mask.filter(|m| !m.disabled) { + let mc = l.channels.iter().find(|c| c.id == -2); + let (mw, mh) = (m.rect.width() as usize, m.rect.height() as usize); + let mask = match mc { + Some(c) if mw > 0 && mh > 0 => Some(self.decode_channel(c.offset, c.len, mw, mh)?), + _ => None, + }; + let def = m.default_color as f32 / 255.0; + for y in 0..oh { + let py = out_rect.top + y as i32; + for x in 0..ow { + let px = out_rect.left + x as i32; + let inside = py >= m.rect.top && py < m.rect.bottom && px >= m.rect.left && px < m.rect.right; + let mut v = match (&mask, inside) { + (Some(mk), true) => mk[(py - m.rect.top) as usize * mw + (px - m.rect.left) as usize], + _ => def, + }; + if m.inverted { + v = 1.0 - v; + } + out[y * ow + x][3] *= v; + } + } + } + Ok(Pixels { width: ow as u32, height: oh as u32, data: out }) + } +} + +/// The next additional-information block at `*pos` (signature `8BIM`/`8B64`, key, length): +/// returns (key, data start, data end). Skips up to 3 padding bytes before a signature. +fn next_block(data: &[u8], pos: &mut usize, end: usize, psb: bool) -> Option<([u8; 4], usize, usize)> { + let mut p = *pos; + for _ in 0..4 { + if p + 12 > end { + return None; + } + if &data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64" { + break; + } + p += 1; + } + if p + 12 > end || !(&data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64") { + return None; + } + let mut key = [0u8; 4]; + key.copy_from_slice(&data[p + 4..p + 8]); + let wide = + psb && matches!(&key, b"LMsk" | b"Lr16" | b"Lr32" | b"Layr" | b"Mt16" | b"Mt32" | b"Mtrn" | b"Alph" | b"FMsk" | b"lnk2" | b"FEid" | b"FXid" | b"PxSD"); + let mut r = Reader::at(&data[..end], p + 8); + let len = r.length(wide).ok()?; + let start = r.pos; + let stop = start.checked_add(len)?; + if stop > end { + return None; + } + *pos = stop; + Some((key, start, stop)) +} + +/// PackBits decoding of one row (appends exactly `row` bytes, zero-padded). +fn unpack_bits(src: &[u8], row: usize, out: &mut Vec) -> Result<()> { + let start = out.len(); + let mut i = 0; + while i < src.len() && out.len() - start < row { + let n = src[i] as i8; + i += 1; + if n >= 0 { + let k = n as usize + 1; + let s = src.get(i..i + k).ok_or(Error::Truncated)?; + out.extend_from_slice(s); + i += k; + } else if n != -128 { + let k = (1 - n as isize) as usize; + let v = *src.get(i).ok_or(Error::Truncated)?; + out.extend(std::iter::repeat_n(v, k)); + i += 1; + } + } + out.resize(start + row, 0); + Ok(()) +} + +/// PackBits encoding of one row. +pub(crate) fn pack_bits(src: &[u8]) -> Vec { + let mut out = vec![]; + let mut i = 0; + while i < src.len() { + // run of identical bytes + let mut run = 1; + while i + run < src.len() && src[i + run] == src[i] && run < 128 { + run += 1; + } + if run >= 2 { + out.push((1 - run as isize) as i8 as u8); + out.push(src[i]); + i += run; + continue; + } + // literal run + let start = i; + let mut n = 0; + while i < src.len() && n < 128 { + if i + 1 < src.len() && src[i + 1] == src[i] { + break; + } + i += 1; + n += 1; + } + if n == 0 { + i += 1; + n = 1; + } + out.push((n - 1) as u8); + out.extend_from_slice(&src[start..start + n]); + } + out +} + +fn srgb_encode(v: f32) -> f32 { + if v <= 0.003_130_8 { v * 12.92 } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 } +} + +/// CIE L*a*b* (D50) to sRGB (D65 via the Bradford-adapted matrix), clamped. +fn lab_to_rgb(l: f32, a: f32, b: f32) -> [f32; 3] { + let fy = (l + 16.0) / 116.0; + let fx = fy + a / 500.0; + let fz = fy - b / 200.0; + let f = |t: f32| if t > 6.0 / 29.0 { t * t * t } else { 3.0 * (6.0f32 / 29.0).powi(2) * (t - 4.0 / 29.0) }; + let (x, y, z) = (0.964_22 * f(fx), f(fy), 0.825_21 * f(fz)); + // XYZ (D50) → linear sRGB. + let r = 3.133_856 * x - 1.616_867 * y - 0.490_615 * z; + let g = -0.978_768 * x + 1.916_142 * y + 0.033_454 * z; + let bb = 0.071_945 * x - 0.228_991 * y + 1.405_243 * z; + [r, g, bb].map(|v| srgb_encode(v.clamp(0.0, 1.0))) +} + +#[cfg(test)] +mod tests; diff --git a/crates/psd/src/read.rs b/crates/psd/src/read.rs new file mode 100644 index 00000000..97651b2b --- /dev/null +++ b/crates/psd/src/read.rs @@ -0,0 +1,86 @@ +//! Big-endian byte reader. + +use crate::{Error, Result}; + +#[derive(Clone)] +pub struct Reader<'a> { + pub data: &'a [u8], + pub pos: usize, +} + +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Reader { data, pos: 0 } + } + pub fn at(data: &'a [u8], pos: usize) -> Self { + Reader { data, pos } + } + pub fn remaining(&self) -> usize { + self.data.len().saturating_sub(self.pos) + } + pub fn bytes(&mut self, n: usize) -> Result<&'a [u8]> { + if self.remaining() < n { + return Err(Error::Truncated); + } + let s = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(s) + } + pub fn skip(&mut self, n: usize) -> Result<()> { + self.bytes(n).map(|_| ()) + } + pub fn u8(&mut self) -> Result { + Ok(self.bytes(1)?[0]) + } + pub fn u16(&mut self) -> Result { + let b = self.bytes(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + } + pub fn i16(&mut self) -> Result { + Ok(self.u16()? as i16) + } + pub fn u32(&mut self) -> Result { + let b = self.bytes(4)?; + Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + pub fn i32(&mut self) -> Result { + Ok(self.u32()? as i32) + } + pub fn u64(&mut self) -> Result { + let b = self.bytes(8)?; + let mut a = [0u8; 8]; + a.copy_from_slice(b); + Ok(u64::from_be_bytes(a)) + } + pub fn f32(&mut self) -> Result { + Ok(f32::from_bits(self.u32()?)) + } + pub fn f64(&mut self) -> Result { + Ok(f64::from_bits(self.u64()?)) + } + pub fn tag(&mut self) -> Result<[u8; 4]> { + let b = self.bytes(4)?; + Ok([b[0], b[1], b[2], b[3]]) + } + /// A length field: 4 bytes in PSD, 8 bytes in PSB (`wide`). + pub fn length(&mut self, wide: bool) -> Result { + let v = if wide { self.u64()? } else { self.u32()? as u64 }; + usize::try_from(v).map_err(|_| Error::Invalid("length".into())) + } + /// A Pascal string padded so that length byte + text is a multiple of `pad`. + pub fn pascal(&mut self, pad: usize) -> Result { + let n = self.u8()? as usize; + let s = self.bytes(n)?; + let total = 1 + n; + let padded = total.div_ceil(pad) * pad; + self.skip(padded - total)?; + // Pascal strings are in the system (Mac Roman) encoding; ASCII covers the common case. + Ok(s.iter().map(|&b| if b < 0x80 { b as char } else { mac_roman(b) }).collect()) + } +} + +/// Mac OS Roman upper half (0x80..=0xFF), as published by the Unicode Consortium mapping table. +fn mac_roman(b: u8) -> char { + const T: &str = "ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñóòôöõúùûü†°¢£§•¶ß®©™´¨≠ÆØ∞±≤≥¥µ∂∑∏π∫ªºΩæø¿¡¬√ƒ≈∆«»… ÀÃÕŒœ–—“”‘’÷◊ÿŸ⁄€‹›fifl‡·‚„‰ÂÊÁËÈÍÎÏÌÓÔ\u{F8FF}ÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ"; + T.chars().nth((b - 0x80) as usize).unwrap_or('?') +} diff --git a/crates/psd/src/tests.rs b/crates/psd/src/tests.rs new file mode 100644 index 00000000..68680fe7 --- /dev/null +++ b/crates/psd/src/tests.rs @@ -0,0 +1,189 @@ +use super::write::*; +use super::*; + +fn gradient(w: u32, h: u32) -> Vec<[f32; 4]> { + (0..w * h).map(|i| [(i % w) as f32 / (w - 1).max(1) as f32, (i / w) as f32 / (h - 1).max(1) as f32, 0.25, 1.0]).collect() +} + +fn close(a: &[[f32; 4]], b: &[[f32; 4]], tol: f32) -> bool { + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| (0..4).all(|c| (x[c] - y[c]).abs() <= tol)) +} + +fn doc_with_layers(depth: u16, mode: ColorMode, rle: bool) -> WDoc { + let mut d = WDoc::new(40, 30); + d.depth = depth; + d.mode = mode; + d.rle = rle; + let mut bg = WLayer::pixels("Background", Rect::new(0, 0, 40, 30), gradient(40, 30)); + bg.opacity = 255; + let mut red = WLayer::solid("Red box", Rect::new(5, 6, 10, 8), [1.0, 0.0, 0.0, 1.0]).with_blend(b"mul ").with_opacity(128); + red.hidden = true; + let mut masked = WLayer::solid("Masked", Rect::new(20, 10, 12, 12), [0.0, 0.5, 1.0, 1.0]).with_blend(b"scrn"); + // Mask: left half opaque, right half transparent; default (outside) black. + let mrect = Rect::new(20, 10, 12, 12); + masked.mask = Some(WMask { rect: mrect, data: (0..144).map(|i| if i % 12 < 6 { 1.0 } else { 0.0 }).collect(), default_color: 0, disabled: false }); + d.layers = vec![bg, red, masked]; + d +} + +#[test] +fn layers_round_trip_8bit_rle() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + assert!(is_psd(&bytes)); + let p = Psd::parse(bytes).unwrap(); + assert_eq!((p.width, p.height, p.depth, p.mode), (40, 30, 8, ColorMode::Rgb)); + assert_eq!(p.layers.len(), 3); + let names: Vec<&str> = p.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Background", "Red box", "Masked"]); + assert_eq!(p.layers[1].blend_key(), "mul "); + assert_eq!(p.layers[1].opacity, 128); + assert!(p.layers[1].hidden && !p.layers[0].hidden); + assert_eq!(p.layers[2].blend_key(), "scrn"); + assert_eq!(p.layers[1].rect, Rect::new(5, 6, 10, 8)); + // Pixels (8-bit quantisation). + let bg = p.layer_pixels(0, false).unwrap(); + assert!(close(&bg.data, &gradient(40, 30), 1.0 / 255.0)); + // Retain layer size vs canvas. + let red = p.layer_pixels(1, false).unwrap(); + assert_eq!((red.width, red.height), (10, 8)); + let red_c = p.layer_pixels(1, true).unwrap(); + assert_eq!((red_c.width, red_c.height), (40, 30)); + assert_eq!(red_c.data[6 * 40 + 5], [1.0, 0.0, 0.0, 1.0]); + assert_eq!(red_c.data[0][3], 0.0); + // Layer mask becomes alpha. + let m = p.layer_pixels(2, false).unwrap(); + assert_eq!(m.data[0][3], 1.0); + assert_eq!(m.data[11][3], 0.0); + // Merged composite. + let c = p.composite().unwrap(); + assert_eq!((c.width, c.height), (40, 30)); + assert!(p.merged_alpha); +} + +#[test] +fn layers_round_trip_16bit_raw_and_cmyk_gray() { + for (depth, mode, rle) in + [(16, ColorMode::Rgb, false), (16, ColorMode::Rgb, true), (8, ColorMode::Cmyk, true), (16, ColorMode::Cmyk, false), (8, ColorMode::Grayscale, false)] + { + let bytes = write(&doc_with_layers(depth, mode, rle)); + let p = Psd::parse(bytes).unwrap_or_else(|e| panic!("{depth} {mode:?} {rle}: {e}")); + assert_eq!(p.layers.len(), 3, "{depth} {mode:?}"); + assert_eq!(p.depth, depth); + let bg = p.layer_pixels(0, true).unwrap(); + let tol = if depth == 16 { 1.0 / 30000.0 } else { 1.5 / 255.0 }; + let want: Vec<[f32; 4]> = + if mode == ColorMode::Grayscale { gradient(40, 30).iter().map(|p| [p[0], p[0], p[0], 1.0]).collect() } else { gradient(40, 30) }; + assert!(close(&bg.data, &want, tol), "{depth} {mode:?} {rle}: {:?} vs {:?}", bg.data[41], want[41]); + let c = p.composite().unwrap(); + assert_eq!(c.data.len(), 1200); + } +} + +#[test] +fn groups_build_a_tree() { + let mut d = WDoc::new(16, 16); + d.layers = vec![ + WLayer::solid("Bottom", Rect::new(0, 0, 16, 16), [1.0; 4]), + WLayer::group_end(), + WLayer::solid("Inner A", Rect::new(0, 0, 4, 4), [1.0, 0.0, 0.0, 1.0]), + WLayer::group_end(), + WLayer::solid("Deep", Rect::new(2, 2, 4, 4), [0.0, 1.0, 0.0, 1.0]), + WLayer::group("Nested", false, *b"norm"), + WLayer::group("Group 1", true, *b"pass"), + WLayer::solid("Top", Rect::new(8, 8, 4, 4), [0.0, 0.0, 1.0, 1.0]), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[6].section, Section::OpenFolder); + assert_eq!(p.layers[6].blend_key(), "pass"); + assert_eq!(p.layers[5].section, Section::ClosedFolder); + let t = p.tree(); + assert_eq!( + t, + vec![ + Node::Layer(7), + Node::Group { layer: 6, children: vec![Node::Group { layer: 5, children: vec![Node::Layer(4)] }, Node::Layer(2)] }, + Node::Layer(0) + ] + ); +} + +#[test] +fn text_effects_fill_and_adjustments() { + let mut d = WDoc::new(64, 32); + let fx = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 1.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 7.0)), + ), + ); + d.layers = vec![ + WLayer::empty("Fill", vec![solid_color_block([0.0, 1.0, 0.0])]), + WLayer::empty("Title", vec![text_block("Hello\nWorld", [10.0, 20.0], "Inter-Bold", 18.0, [1.0, 0.0, 0.0], 2)]).with_block(effects_block(&fx)), + WLayer::empty("Hue", vec![hue_sat_block(30, -20, 10)]), + WLayer::empty("Lev", vec![levels_block(10, 240, 5, 250, 1.2)]), + WLayer::empty("Inv", vec![invert_block()]), + WLayer::empty("BC", vec![brightness_block(20, -10)]), + WLayer::empty("Exp", vec![exposure_block(1.5, 0.0, 1.0)]), + WLayer::solid("Vec", Rect::new(0, 0, 64, 32), [1.0; 4]).with_block(vector_mask_block( + &[vec![[[8.0, 4.0]; 3], [[40.0, 4.0]; 3], [[40.0, 20.0]; 3]]], + 64, + 32, + false, + )), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[0].fill, Some([0.0, 1.0, 0.0])); + let fill = p.layer_pixels(0, false).unwrap(); + assert_eq!((fill.width, fill.height), (64, 32)); + assert_eq!(fill.data[5], [0.0, 1.0, 0.0, 1.0]); + let t = p.layers[1].text.as_ref().unwrap(); + assert_eq!(t.text, "Hello\nWorld"); + assert_eq!(t.transform[4..], [10.0, 20.0]); + assert_eq!(t.style.font.as_deref(), Some("Inter-Bold")); + assert_eq!(t.style.size, Some(18.0)); + assert_eq!(t.style.color, Some([1.0, 0.0, 0.0])); + assert_eq!(t.style.justification, Some(2)); + let e = p.layers[1].effects.as_ref().unwrap(); + let ds = e.obj("DrSh").unwrap(); + assert_eq!(ds.enum_value("Md "), Some("Mltp")); + assert_eq!(ds.num("Dstn"), Some(7.0)); + assert_eq!(p.layers[2].adjustment, Some(Adjustment::HueSaturation { hue: 30.0, saturation: -20.0, lightness: 10.0, colorize: false })); + assert_eq!(p.layers[3].adjustment, Some(Adjustment::Levels { in_black: 10.0, in_white: 240.0, gamma: 1.2, out_black: 5.0, out_white: 250.0 })); + assert_eq!(p.layers[4].adjustment, Some(Adjustment::Invert)); + assert_eq!(p.layers[5].adjustment, Some(Adjustment::BrightnessContrast { brightness: 20.0, contrast: -10.0, legacy: true })); + assert_eq!(p.layers[6].adjustment, Some(Adjustment::Exposure { exposure: 1.5, offset: 0.0, gamma: 1.0 })); + let vm = p.layers[7].vector_mask.as_ref().unwrap(); + assert_eq!(vm.subpaths.len(), 1); + assert!(vm.subpaths[0].closed); + let k = &vm.subpaths[0].knots; + assert_eq!(k.len(), 3); + assert!((k[1][1][0] - 40.0).abs() < 1e-4 && (k[2][1][1] - 20.0).abs() < 1e-4); +} + +#[test] +fn packbits_round_trip() { + for src in [vec![], vec![1u8], vec![5; 300], (0..=255).collect::>(), vec![1, 1, 2, 3, 3, 3, 4, 5, 5]] { + let enc = pack_bits(&src); + let mut out = vec![]; + unpack_bits(&enc, src.len(), &mut out).unwrap(); + assert_eq!(out, src); + } +} + +#[test] +fn truncated_and_garbage_input_errors() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + for n in [0, 3, 10, 26, 40, 100, bytes.len() / 2] { + let r = Psd::parse(bytes[..n].to_vec()); + if let Ok(p) = r { + let _ = p.composite(); + } + } + assert!(Psd::parse(b"8BPS\0\x09garbage".to_vec()).is_err()); + assert!(Psd::parse(b"PNG".to_vec()).is_err()); +} diff --git a/crates/psd/src/write.rs b/crates/psd/src/write.rs new file mode 100644 index 00000000..fb78cc78 --- /dev/null +++ b/crates/psd/src/write.rs @@ -0,0 +1,491 @@ +//! A minimal Photoshop document writer, per the same specification as the reader: RGB, CMYK or +//! Grayscale, 8 or 16 bits, raw or RLE channels, pixel layers with layer masks, groups and any +//! additional-information blocks (helpers below build text, effects, fill and adjustment +//! blocks). Used to generate test fixtures; not a full exporter. + +use crate::descriptor::{self, DValue, Descriptor}; +use crate::{ColorMode, Rect, Section, engine_data, pack_bits}; + +/// A layer mask for [`WLayer`]. +#[derive(Clone, Debug)] +pub struct WMask { + pub rect: Rect, + /// `rect.width() * rect.height()` values in 0..1. + pub data: Vec, + pub default_color: u8, + pub disabled: bool, +} + +/// A layer to write. +#[derive(Clone, Debug)] +pub struct WLayer { + pub name: String, + pub rect: Rect, + /// Straight RGBA, `rect.width() * rect.height()` pixels. + pub pixels: Vec<[f32; 4]>, + pub blend: [u8; 4], + pub opacity: u8, + pub hidden: bool, + pub clipping: bool, + pub mask: Option, + pub section: Section, + /// Extra additional-information blocks (key, data). + pub blocks: Vec<([u8; 4], Vec)>, +} + +impl WLayer { + /// A pixel layer. + pub fn pixels(name: &str, rect: Rect, pixels: Vec<[f32; 4]>) -> WLayer { + WLayer { + name: name.into(), + rect, + pixels, + blend: *b"norm", + opacity: 255, + hidden: false, + clipping: false, + mask: None, + section: Section::Layer, + blocks: vec![], + } + } + /// A layer filled with one colour. + pub fn solid(name: &str, rect: Rect, rgba: [f32; 4]) -> WLayer { + let n = (rect.width() * rect.height()) as usize; + WLayer::pixels(name, rect, vec![rgba; n]) + } + /// The top record of a group (its children come *before* it in file order, after a + /// [`WLayer::group_end`]). + pub fn group(name: &str, open: bool, blend: [u8; 4]) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.section = if open { Section::OpenFolder } else { Section::ClosedFolder }; + l.blend = blend; + l + } + /// The hidden record that closes a group (written below its children). + pub fn group_end() -> WLayer { + let mut l = WLayer::pixels("", Rect::default(), vec![]); + l.section = Section::Divider; + l + } + /// A layer without pixels carrying `blocks` (adjustment and fill layers, type layers). + pub fn empty(name: &str, blocks: Vec<([u8; 4], Vec)>) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.blocks = blocks; + l + } + pub fn with_blend(mut self, key: &[u8; 4]) -> WLayer { + self.blend = *key; + self + } + pub fn with_opacity(mut self, o: u8) -> WLayer { + self.opacity = o; + self + } + pub fn with_block(mut self, b: ([u8; 4], Vec)) -> WLayer { + self.blocks.push(b); + self + } +} + +/// A document to write. +#[derive(Clone, Debug)] +pub struct WDoc { + pub width: u32, + pub height: u32, + /// 8 or 16. + pub depth: u16, + /// Rgb, Cmyk or Grayscale. + pub mode: ColorMode, + pub rle: bool, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// Merged image (straight RGBA, document-sized); `None` composites visible pixel layers + /// with Normal blending. + pub composite: Option>, +} + +impl WDoc { + pub fn new(width: u32, height: u32) -> WDoc { + WDoc { width, height, depth: 8, mode: ColorMode::Rgb, rle: true, layers: vec![], composite: None } + } +} + +fn u16b(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_be_bytes()); +} +fn u32b(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +/// Colour channels for a straight RGB pixel in the document mode. +fn mode_channels(mode: ColorMode, p: [f32; 4]) -> Vec { + match mode { + ColorMode::Cmyk => { + // Stored values are inverted ink amounts: k = max, c = r / max (reader: r = c * k). + let m = p[0].max(p[1]).max(p[2]); + if m <= 0.0 { vec![0.0, 0.0, 0.0, 0.0] } else { vec![p[0] / m, p[1] / m, p[2] / m, m] } + } + ColorMode::Grayscale => vec![p[0]], + _ => vec![p[0], p[1], p[2]], + } +} + +fn encode_samples(v: &[f32], depth: u16) -> Vec { + let mut out = Vec::with_capacity(v.len() * if depth == 16 { 2 } else { 1 }); + for &x in v { + let x = x.clamp(0.0, 1.0); + if depth == 16 { + u16b(&mut out, (x * 65535.0).round() as u16); + } else { + out.push((x * 255.0).round() as u8); + } + } + out +} + +/// One channel's data (compression field + data) for a `w`×`h` plane. +fn channel_data(v: &[f32], w: usize, h: usize, depth: u16, rle: bool) -> Vec { + let raw = encode_samples(v, depth); + let mut out = vec![]; + let rowb = raw.len() / h.max(1); + if !rle || w == 0 || h == 0 { + u16b(&mut out, 0); + out.extend_from_slice(&raw); + return out; + } + u16b(&mut out, 1); + let rows: Vec> = raw.chunks(rowb.max(1)).map(pack_bits).collect(); + for r in &rows { + u16b(&mut out, r.len() as u16); + } + for r in rows { + out.extend_from_slice(&r); + } + out +} + +fn pascal4(out: &mut Vec, s: &str) { + let b: Vec = s.bytes().map(|c| if c < 0x80 { c } else { b'?' }).take(255).collect(); + out.push(b.len() as u8); + out.extend_from_slice(&b); + let total = 1 + b.len(); + out.resize(out.len() + total.div_ceil(4) * 4 - total, 0); +} + +fn block(out: &mut Vec, key: &[u8; 4], data: &[u8]) { + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(key); + let padded = data.len().div_ceil(2) * 2; + u32b(out, padded as u32); + out.extend_from_slice(data); + out.resize(out.len() + padded - data.len(), 0); +} + +fn layer_info(doc: &WDoc) -> Vec { + let mut out = vec![]; + out.extend_from_slice(&(-(doc.layers.len() as i16)).to_be_bytes()); + let mut chan_data: Vec)>> = vec![]; + for l in &doc.layers { + let (w, h) = (l.rect.width() as usize, l.rect.height() as usize); + let mut chans: Vec<(i16, Vec)> = vec![]; + let n = w * h; + let alpha: Vec = l.pixels.iter().take(n).map(|p| p[3]).collect(); + chans.push((-1, channel_data(&alpha, w, h, doc.depth, doc.rle))); + let cc = doc.mode.color_channels(); + let planes: Vec> = l.pixels.iter().take(n).map(|p| mode_channels(doc.mode, *p)).collect(); + for c in 0..cc { + let plane: Vec = planes.iter().map(|v| v[c]).collect(); + chans.push((c as i16, channel_data(&plane, w, h, doc.depth, doc.rle))); + } + if let Some(m) = &l.mask { + chans.push((-2, channel_data(&m.data, m.rect.width() as usize, m.rect.height() as usize, doc.depth, doc.rle))); + } + // record + for v in [l.rect.top, l.rect.left, l.rect.bottom, l.rect.right] { + out.extend_from_slice(&v.to_be_bytes()); + } + u16b(&mut out, chans.len() as u16); + for (id, d) in &chans { + out.extend_from_slice(&id.to_be_bytes()); + u32b(&mut out, d.len() as u32); + } + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(&l.blend); + out.push(l.opacity); + out.push(l.clipping as u8); + let mut flags = 8u8; + if l.hidden { + flags |= 2; + } + out.push(flags); + out.push(0); + let mut extra = vec![]; + match &l.mask { + Some(m) => { + u32b(&mut extra, 20); + for v in [m.rect.top, m.rect.left, m.rect.bottom, m.rect.right] { + extra.extend_from_slice(&v.to_be_bytes()); + } + extra.push(m.default_color); + extra.push(if m.disabled { 2 } else { 0 }); + extra.extend_from_slice(&[0, 0]); + } + None => u32b(&mut extra, 0), + } + u32b(&mut extra, 0); // blending ranges + pascal4(&mut extra, &l.name); + let luni = unicode_plain(&l.name); + block(&mut extra, b"luni", &luni); + match l.section { + Section::Layer => {} + s => { + let mut d = vec![]; + u32b( + &mut d, + match s { + Section::OpenFolder => 1, + Section::ClosedFolder => 2, + _ => 3, + }, + ); + d.extend_from_slice(b"8BIM"); + d.extend_from_slice(&l.blend); + block(&mut extra, b"lsct", &d); + } + } + for (k, d) in &l.blocks { + block(&mut extra, k, d); + } + u32b(&mut out, extra.len() as u32); + out.extend_from_slice(&extra); + chan_data.push(chans); + } + for chans in chan_data { + for (_, d) in chans { + out.extend_from_slice(&d); + } + } + if out.len() % 2 == 1 { + out.push(0); + } + out +} + +fn unicode_plain(s: &str) -> Vec { + let v: Vec = s.encode_utf16().collect(); + let mut out = vec![]; + u32b(&mut out, v.len() as u32); + for c in v { + u16b(&mut out, c); + } + out +} + +/// Normal "over" of the visible pixel layers (groups ignored), straight alpha. +fn flatten(doc: &WDoc) -> Vec<[f32; 4]> { + let (w, h) = (doc.width as i32, doc.height as i32); + let mut acc = vec![[0.0f32; 4]; (w * h) as usize]; + for l in doc.layers.iter().filter(|l| !l.hidden && l.section == Section::Layer) { + let lw = l.rect.width() as i32; + for (i, p) in l.pixels.iter().enumerate() { + let (x, y) = (l.rect.left + i as i32 % lw.max(1), l.rect.top + i as i32 / lw.max(1)); + if x < 0 || y < 0 || x >= w || y >= h { + continue; + } + let a = p[3] * l.opacity as f32 / 255.0; + let d = &mut acc[(y * w + x) as usize]; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (p[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +/// Serialise a document. +pub fn write(doc: &WDoc) -> Vec { + let cc = doc.mode.color_channels(); + let mut out = vec![]; + out.extend_from_slice(b"8BPS"); + u16b(&mut out, 1); + out.extend_from_slice(&[0; 6]); + u16b(&mut out, (cc + 1) as u16); + u32b(&mut out, doc.height); + u32b(&mut out, doc.width); + u16b(&mut out, doc.depth); + u16b(&mut out, doc.mode.to_u16()); + u32b(&mut out, 0); // colour mode data + u32b(&mut out, 0); // image resources + // Layer and mask information. + let mut lm = vec![]; + if !doc.layers.is_empty() { + let li = layer_info(doc); + if doc.depth == 16 { + u32b(&mut lm, 0); + u32b(&mut lm, 0); // global layer mask info + let mut padded = li; + while !padded.len().is_multiple_of(4) { + padded.push(0); + } + lm.extend_from_slice(b"8BIM"); + lm.extend_from_slice(b"Lr16"); + u32b(&mut lm, padded.len() as u32); + lm.extend_from_slice(&padded); + } else { + u32b(&mut lm, li.len() as u32); + lm.extend_from_slice(&li); + u32b(&mut lm, 0); + } + } + u32b(&mut out, lm.len() as u32); + out.extend_from_slice(&lm); + // Merged image. + let comp = doc.composite.clone().unwrap_or_else(|| flatten(doc)); + let (w, h) = (doc.width as usize, doc.height as usize); + let planes: Vec> = comp.iter().map(|p| mode_channels(doc.mode, *p)).collect(); + let mut chans: Vec> = (0..cc).map(|c| planes.iter().map(|v| v[c]).collect()).collect(); + chans.push(comp.iter().map(|p| p[3]).collect()); + if doc.rle { + u16b(&mut out, 1); + let rows: Vec>> = + chans.iter().map(|c| encode_samples(c, doc.depth).chunks((w * if doc.depth == 16 { 2 } else { 1 }).max(1)).map(pack_bits).collect()).collect(); + for c in &rows { + for r in c { + u16b(&mut out, r.len() as u16); + } + } + for c in rows { + for r in c { + out.extend_from_slice(&r); + } + } + } else { + u16b(&mut out, 0); + for c in &chans { + out.extend_from_slice(&encode_samples(c, doc.depth)); + } + } + let _ = h; + out +} + +// ---------------------------------------------------------------- block helpers + +/// `TySh`: a type layer with one style run; the text origin sits at `origin`. +pub fn text_block(text: &str, origin: [f64; 2], font: &str, size: f64, color: [f64; 3], justification: u8) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [1.0, 0.0, 0.0, 1.0, origin[0], origin[1]] { + d.extend_from_slice(&f64::to_be_bytes(v)); + } + u16b(&mut d, 50); + u32b(&mut d, 16); + let desc = Descriptor::new("TxLr") + .with("Txt ", DValue::Text(text.replace('\n', "\r"))) + .with("textGridding", DValue::Enum("textGridding".into(), "None".into())) + .with("Ornt", DValue::Enum("Ornt".into(), "Hrzn".into())) + .with("EngineData", DValue::Raw(engine_data::minimal(text, font, size, color, justification))); + descriptor::write_descriptor(&mut d, &desc); + // Warp: version, descriptor version, an empty warp descriptor. + u16b(&mut d, 1); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("warp").with("warpStyle", DValue::Enum("warpStyle".into(), "warpNone".into()))); + for _ in 0..4 { + u32b(&mut d, 0); + } + (*b"TySh", d) +} + +/// `lfx2`: layer effects from an effects descriptor (`null` class with `DrSh`, `FrFX`…). +pub fn effects_block(desc: &Descriptor) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 0); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, desc); + (*b"lfx2", d) +} + +/// `SoCo`: solid colour fill. +pub fn solid_color_block(rgb: [f64; 3]) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("null").with("Clr ", DValue::Descriptor(Descriptor::rgb(rgb)))); + (*b"SoCo", d) +} + +/// `hue2`: Hue/Saturation (master). +pub fn hue_sat_block(hue: i16, sat: i16, light: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + d.extend_from_slice(&[0, 0]); + for v in [0i16, 0, 0, hue, sat, light] { + d.extend_from_slice(&v.to_be_bytes()); + } + // Six colour ranges (unused). + d.resize(d.len() + 6 * 24, 0); + (*b"hue2", d) +} + +/// `levl`: Levels (composite record). +pub fn levels_block(in_black: i16, in_white: i16, out_black: i16, out_white: i16, gamma: f64) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + for v in [in_black, in_white, out_black, out_white, (gamma * 100.0).round() as i16] { + d.extend_from_slice(&v.to_be_bytes()); + } + for _ in 0..28 { + for v in [0i16, 255, 0, 255, 100] { + d.extend_from_slice(&v.to_be_bytes()); + } + } + (*b"levl", d) +} + +pub fn invert_block() -> ([u8; 4], Vec) { + (*b"nvrt", vec![]) +} + +pub fn brightness_block(brightness: i16, contrast: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + for v in [brightness, contrast, 127] { + d.extend_from_slice(&v.to_be_bytes()); + } + d.push(0); + (*b"brit", d) +} + +pub fn exposure_block(exposure: f32, offset: f32, gamma: f32) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [exposure, offset, gamma] { + d.extend_from_slice(&v.to_be_bytes()); + } + (*b"expA", d) +} + +/// `vmsk`: a vector mask from closed polygons/curves given as (in, anchor, out) knots in pixels. +pub fn vector_mask_block(subpaths: &[Vec<[[f64; 2]; 3]>], width: u32, height: u32, inverted: bool) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 3); + u32b(&mut d, inverted as u32); + // Path fill rule record. + u16b(&mut d, 6); + d.resize(d.len() + 24, 0); + let fx = |v: f64| ((v * (1 << 24) as f64).round() as i32).to_be_bytes(); + for sp in subpaths { + u16b(&mut d, 0); + u16b(&mut d, sp.len() as u16); + d.resize(d.len() + 22, 0); + for k in sp { + u16b(&mut d, 1); + for p in k { + d.extend_from_slice(&fx(p[1] / height as f64)); + d.extend_from_slice(&fx(p[0] / width as f64)); + } + } + } + (*b"vmsk", d) +} diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 4c8a9787..cdaa85c1 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -19,6 +19,7 @@ effectcraft-raster = { workspace = true } effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } +lz4_flex = { version = "0.11", default-features = false, features = ["std", "safe-encode", "safe-decode", "checked-decode"] } kurbo = { workspace = true } rayon = { workspace = true } web-time = { workspace = true } diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 4884411f..4b872ad2 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -330,6 +330,7 @@ mod tests { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f)); let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 8cd27d59..23702fd6 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -25,9 +25,13 @@ use effectcraft_project::{ItemKind, Layer, LayerSource, Node, PropGroup}; use crate::eval::EvalCtx; -/// Thread-safe, memory-budgeted cache of processed layer buffers. +/// Thread-safe, memory-budgeted cache of processed layer buffers, optionally backed by the +/// persistent [`DiskCache`](crate::disk_cache::DiskCache): a memory miss reads the disk, and +/// buffers that were slow to render are written there (keys salted per project, see +/// [`LayerCache::set_disk`]). pub struct LayerCache { inner: Mutex, + disk: Mutex, u64)>>, } struct Entry { @@ -43,6 +47,8 @@ struct Inner { clock: u64, hits: u64, misses: u64, + /// When each recent miss happened (to time the render that follows it). + missed_at: HashMap, } /// Hit/miss counters and memory use (for perf readouts and tests). @@ -63,7 +69,37 @@ impl Default for LayerCache { impl LayerCache { /// A cache holding at most `budget` bytes of pixels. pub fn new(budget: usize) -> LayerCache { - LayerCache { inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0 }) } + LayerCache { + inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0, missed_at: HashMap::new() }), + disk: Mutex::new(None), + } + } + + /// Attach (or detach) the disk cache. `salt` separates projects and footage versions (see + /// `footage_salt`); keys on disk are `(salt, key)`. + pub fn set_disk(&self, disk: Option<(Arc, u64)>) { + if let Ok(mut d) = self.disk.lock() { + *d = disk; + } + } + + /// Change the disk salt (after footage changed) keeping the attached disk cache. + pub fn set_disk_salt(&self, salt: u64) { + if let Ok(mut d) = self.disk.lock() + && let Some((_, s)) = d.as_mut() + { + *s = salt; + } + } + + pub fn disk(&self) -> Option> { + self.disk.lock().ok()?.as_ref().map(|d| d.0.clone()) + } + + fn disk_key(&self, key: u64) -> Option<(Arc, u128)> { + let d = self.disk.lock().ok()?; + let (dc, salt) = d.as_ref()?; + Some((dc.clone(), ((*salt as u128) << 64) | key as u128)) } pub fn get(&self, key: u64) -> Option> { @@ -79,12 +115,32 @@ impl LayerCache { } None => { g.misses += 1; - None + if g.missed_at.len() > 4096 { + g.missed_at.clear(); + } + g.missed_at.insert(key, web_time::Instant::now()); + drop(g); + // Disk cache: a hit comes back into memory. + let (dc, dk) = self.disk_key(key)?; + let buf = Arc::new(dc.get_layer(dk)?); + self.insert_mem(key, buf.clone()); + Some(buf) } } } pub fn insert(&self, key: u64, buf: Arc) { + let started = self.inner.lock().ok().and_then(|mut g| g.missed_at.remove(&key)); + if let Some((dc, dk)) = self.disk_key(key) { + let slow = started.is_some_and(|t| t.elapsed().as_secs_f64() * 1e3 >= dc.min_layer_ms() as f64); + if slow { + dc.put_layer(dk, &buf); + } + } + self.insert_mem(key, buf); + } + + fn insert_mem(&self, key: u64, buf: Arc) { let bytes = buf.img.data.len() * std::mem::size_of::(); let Ok(mut g) = self.inner.lock() else { return }; if bytes > g.budget / 4 { diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs new file mode 100644 index 00000000..1afc9bbb --- /dev/null +++ b/crates/render/src/disk_cache.rs @@ -0,0 +1,829 @@ +//! The persistent disk cache (Settings ▸ Media & Disk Cache ▸ Disk Cache): RAM-preview frames +//! and layer cache entries kept on disk between sessions, like After Effects' disk cache. +//! +//! * Entries are files keyed by a 128-bit content hash (`/v1//<2 hex>/<32 hex>.ecc`): +//! frames by the hash of the composition's content (the comp and everything it uses, footage +//! files' size and modification time included) plus frame, scale and view; layer buffers by +//! the layer cache's content key salted with the project's footage fingerprint. A changed +//! project therefore never reads stale pixels: its keys simply change. +//! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer +//! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh +//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and +//! are deleted. +//! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the +//! folder when the cache opens (file modification times order it), so the cache survives +//! restarts; reads refresh a file's time. +//! * Safe concurrent access: writes go to a unique temporary file that is renamed into place +//! (atomic on every desktop OS), so readers in this or another process never see a partial +//! entry; a missing or damaged file is just a miss. Writes run on a background thread with a +//! bounded queue (a full queue drops the write rather than stalling rendering). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +use effectcraft_effects::Buf; +use effectcraft_raster::Image; + +/// File format version (part of the folder layout). +const VERSION: &str = "v1"; +const MAGIC: &[u8; 4] = b"ECDC"; +/// Pending writes before new ones are dropped. +const QUEUE_LIMIT: usize = 64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Kind { + Frame, + Layer, +} + +impl Kind { + fn dir(self) -> &'static str { + match self { + Kind::Frame => "frames", + Kind::Layer => "layers", + } + } + fn tag(self) -> u8 { + match self { + Kind::Frame => 1, + Kind::Layer => 2, + } + } +} + +/// A cached viewer frame: 8-bit premultiplied RGBA. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Frame8 { + pub width: u32, + pub height: u32, + pub rgba: Vec, +} + +/// Counters for tests, the Info panel and `cache.diskStats`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DiskStats { + pub entries: usize, + pub bytes: u64, + pub max_bytes: u64, + pub hits: u64, + pub misses: u64, + pub writes: u64, + pub evictions: u64, + pub dropped: u64, +} + +struct Meta { + bytes: u64, + last_use: u64, +} + +#[derive(Default)] +struct Index { + map: HashMap<(Kind, u128), Meta>, + total: u64, + clock: u64, +} + +enum Job { + Write(Kind, u128, Vec), +} + +struct Queue { + jobs: std::collections::VecDeque, + busy: bool, +} + +pub struct DiskCache { + dir: PathBuf, + index: Mutex, + max_bytes: AtomicU64, + hits: AtomicU64, + misses: AtomicU64, + writes: AtomicU64, + evictions: AtomicU64, + dropped: AtomicU64, + tmp_counter: AtomicU64, + queue: Arc<(Mutex, Condvar)>, + /// Layer buffers that took less than this to render are not written (ms). + min_layer_ms: AtomicU64, +} + +/// FNV-1a 128-bit (content keys). +#[derive(Clone, Copy)] +pub struct Hash128(u128); + +impl Default for Hash128 { + fn default() -> Self { + Hash128(0x6c62_272e_07bb_0142_62b8_2175_6295_c58d) + } +} + +impl Hash128 { + pub fn write(&mut self, bytes: &[u8]) { + for &b in bytes { + self.0 ^= b as u128; + self.0 = self.0.wrapping_mul(0x0000_0000_0100_0000_0000_0000_0000_013b); + } + } + pub fn write_u64(&mut self, v: u64) { + self.write(&v.to_le_bytes()); + } + pub fn finish(&self) -> u128 { + self.0 + } +} + +fn checksum(data: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for &b in data { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// The platform's default disk cache folder. +pub fn default_folder() -> PathBuf { + let home = std::env::var_os("HOME").map(PathBuf::from); + if cfg!(target_os = "macos") + && let Some(h) = &home + { + return h.join("Library/Caches/EffectCraft/Disk Cache"); + } + if cfg!(windows) + && let Some(l) = std::env::var_os("LOCALAPPDATA") + { + return PathBuf::from(l).join("EffectCraft").join("Disk Cache"); + } + if let Some(x) = std::env::var_os("XDG_CACHE_HOME") { + return PathBuf::from(x).join("effectcraft/disk-cache"); + } + home.map(|h| h.join(".cache/effectcraft/disk-cache")).unwrap_or_else(|| std::env::temp_dir().join("effectcraft-disk-cache")) +} + +impl DiskCache { + /// Open (or create) the cache in `folder`, holding at most `max_bytes`. + pub fn open(folder: impl AsRef, max_bytes: u64) -> std::io::Result> { + let dir = folder.as_ref().join(VERSION); + std::fs::create_dir_all(&dir)?; + let dc = Arc::new(DiskCache { + dir, + index: Mutex::new(Index::default()), + max_bytes: AtomicU64::new(max_bytes), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + writes: AtomicU64::new(0), + evictions: AtomicU64::new(0), + dropped: AtomicU64::new(0), + tmp_counter: AtomicU64::new(0), + queue: Arc::new((Mutex::new(Queue { jobs: Default::default(), busy: false }), Condvar::new())), + min_layer_ms: AtomicU64::new(30), + }); + dc.rescan(); + dc.evict(); + #[cfg(not(target_arch = "wasm32"))] + { + let weak = Arc::downgrade(&dc); + let q = dc.queue.clone(); + std::thread::Builder::new().name("ec-disk-cache".into()).spawn(move || writer(weak, q))?; + } + Ok(dc) + } + + pub fn folder(&self) -> &Path { + self.dir.parent().unwrap_or(&self.dir) + } + + /// Rebuild the index from the folder (oldest files first). + pub fn rescan(&self) { + let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![]; + for kind in [Kind::Frame, Kind::Layer] { + let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue }; + for shard in shards.flatten() { + let Ok(files) = std::fs::read_dir(shard.path()) else { continue }; + for f in files.flatten() { + let p = f.path(); + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if name.contains(".tmp") { + // A writer that died mid-write; old temporaries are removed. + if f.metadata().and_then(|m| m.modified()).is_ok_and(|t| t.elapsed().is_ok_and(|e| e.as_secs() > 600)) { + let _ = std::fs::remove_file(&p); + } + continue; + } + let Some(hex) = name.strip_suffix(".ecc") else { continue }; + let Ok(key) = u128::from_str_radix(hex, 16) else { continue }; + let Ok(md) = f.metadata() else { continue }; + found.push((md.modified().unwrap_or(std::time::UNIX_EPOCH), kind, key, md.len())); + } + } + } + found.sort_by_key(|e| e.0); + let mut idx = Index::default(); + for (_, kind, key, bytes) in found { + idx.clock += 1; + let clock = idx.clock; + idx.total += bytes; + idx.map.insert((kind, key), Meta { bytes, last_use: clock }); + } + if let Ok(mut g) = self.index.lock() { + *g = idx; + } + } + + fn path(&self, kind: Kind, key: u128) -> PathBuf { + let hex = format!("{key:032x}"); + self.dir.join(kind.dir()).join(&hex[..2]).join(format!("{hex}.ecc")) + } + + pub fn max_bytes(&self) -> u64 { + self.max_bytes.load(Ordering::Relaxed) + } + + /// Change the size limit (evicts the least recently used entries when shrinking). + pub fn set_max_bytes(&self, b: u64) { + self.max_bytes.store(b, Ordering::Relaxed); + self.evict(); + } + + /// Layer buffers faster to render than this are not worth a disk write. + pub fn set_min_layer_ms(&self, ms: u64) { + self.min_layer_ms.store(ms, Ordering::Relaxed); + } + + pub fn min_layer_ms(&self) -> u64 { + self.min_layer_ms.load(Ordering::Relaxed) + } + + pub fn contains(&self, kind: Kind, key: u128) -> bool { + self.index.lock().is_ok_and(|g| g.map.contains_key(&(kind, key))) + } + + /// Keys of one kind (unordered). + pub fn keys(&self, kind: Kind) -> Vec { + self.index.lock().map(|g| g.map.keys().filter(|k| k.0 == kind).map(|k| k.1).collect()).unwrap_or_default() + } + + pub fn stats(&self) -> DiskStats { + let (entries, bytes) = self.index.lock().map(|g| (g.map.len(), g.total)).unwrap_or_default(); + DiskStats { + entries, + bytes, + max_bytes: self.max_bytes(), + hits: self.hits.load(Ordering::Relaxed), + misses: self.misses.load(Ordering::Relaxed), + writes: self.writes.load(Ordering::Relaxed), + evictions: self.evictions.load(Ordering::Relaxed), + dropped: self.dropped.load(Ordering::Relaxed), + } + } + + /// Delete every entry (Edit ▸ Purge ▸ All Disk Cache / Empty Disk Cache). + pub fn clear(&self) { + self.flush(); + if let Ok(mut g) = self.index.lock() { + *g = Index::default(); + } + for kind in [Kind::Frame, Kind::Layer] { + let _ = std::fs::remove_dir_all(self.dir.join(kind.dir())); + } + } + + /// Wait until queued writes are on disk. + pub fn flush(&self) { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + #[cfg(target_arch = "wasm32")] + { + q.jobs.clear(); + } + while !q.jobs.is_empty() || q.busy { + q = match cv.wait(q) { + Ok(q) => q, + Err(_) => return, + }; + } + } + + fn read(&self, kind: Kind, key: u128) -> Option> { + if !self.contains(kind, key) { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + } + let path = self.path(kind, key); + let data = std::fs::read(&path).ok(); + let ok = data.as_deref().and_then(|d| verify(d, kind)); + match ok { + Some(payload) => { + let payload = payload.to_vec(); + self.hits.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + if let Some(m) = g.map.get_mut(&(kind, key)) { + m.last_use = c; + } + } + // Persist the recency for the next session (best effort). + if let Ok(f) = std::fs::File::options().append(true).open(&path) { + let _ = f.set_modified(std::time::SystemTime::now()); + } + Some(payload) + } + None => { + self.misses.fetch_add(1, Ordering::Relaxed); + // Gone (another process evicted it) or damaged: forget it. + if data.is_some() { + let _ = std::fs::remove_file(&path); + } + self.forget(kind, key); + None + } + } + } + + fn forget(&self, kind: Kind, key: u128) { + if let Ok(mut g) = self.index.lock() + && let Some(m) = g.map.remove(&(kind, key)) + { + g.total -= m.bytes; + } + } + + fn enqueue(&self, kind: Kind, key: u128, payload: Vec) { + #[cfg(target_arch = "wasm32")] + { + self.write_now(kind, key, &payload); + } + #[cfg(not(target_arch = "wasm32"))] + { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + if q.jobs.len() >= QUEUE_LIMIT { + self.dropped.fetch_add(1, Ordering::Relaxed); + return; + } + q.jobs.push_back(Job::Write(kind, key, payload)); + cv.notify_all(); + } + } + + /// Write an entry now (atomic rename), then evict down to the limit. + fn write_now(&self, kind: Kind, key: u128, payload: &[u8]) { + if self.contains(kind, key) { + return; + } + let path = self.path(kind, key); + let Some(parent) = path.parent() else { return }; + if std::fs::create_dir_all(parent).is_err() { + return; + } + let mut data = Vec::with_capacity(payload.len() + 16); + data.extend_from_slice(MAGIC); + data.push(kind.tag()); + data.extend_from_slice(&[0, 0, 0]); + data.extend_from_slice(payload); + let sum = checksum(&data); + data.extend_from_slice(&sum.to_le_bytes()); + let n = self.tmp_counter.fetch_add(1, Ordering::Relaxed); + let tmp = parent.join(format!("{key:032x}.tmp.{}.{n}", std::process::id())); + if std::fs::write(&tmp, &data).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + if std::fs::rename(&tmp, &path).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + self.writes.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + let bytes = data.len() as u64; + if let Some(old) = g.map.insert((kind, key), Meta { bytes, last_use: c }) { + g.total -= old.bytes; + } + g.total += bytes; + } + self.evict(); + } + + /// Remove least recently used entries until the cache fits its limit. + fn evict(&self) { + let max = self.max_bytes(); + let victims: Vec<(Kind, u128)> = { + let Ok(mut g) = self.index.lock() else { return }; + if g.total <= max { + return; + } + let mut by_age: Vec<(u64, (Kind, u128), u64)> = g.map.iter().map(|(k, m)| (m.last_use, *k, m.bytes)).collect(); + by_age.sort_unstable_by_key(|e| e.0); + let mut out = vec![]; + for (_, k, bytes) in by_age { + if g.total <= max { + break; + } + g.map.remove(&k); + g.total -= bytes; + out.push(k); + } + out + }; + for (kind, key) in victims { + let _ = std::fs::remove_file(self.path(kind, key)); + self.evictions.fetch_add(1, Ordering::Relaxed); + } + } + + // ------------------------------------------------------------ typed entries + + pub fn get_frame(&self, key: u128) -> Option { + let p = self.read(Kind::Frame, key)?; + decode_frame(&p) + } + + /// Queue a frame for writing (8-bit premultiplied RGBA, `width * height * 4` bytes). + pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + if self.contains(Kind::Frame, key) || rgba.len() != width as usize * height as usize * 4 { + return; + } + self.enqueue(Kind::Frame, key, encode_frame(width, height, rgba)); + } + + pub fn get_layer(&self, key: u128) -> Option { + let p = self.read(Kind::Layer, key)?; + decode_layer(&p) + } + + /// Queue a layer buffer for writing. + pub fn put_layer(&self, key: u128, buf: &Buf) { + if self.contains(Kind::Layer, key) { + return; + } + self.enqueue(Kind::Layer, key, encode_layer(buf)); + } + + /// Write a layer buffer synchronously (tests). + pub fn put_layer_now(&self, key: u128, buf: &Buf) { + self.write_now(Kind::Layer, key, &encode_layer(buf)); + } + + /// Write a frame synchronously (tests). + pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba)); + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { + let (m, cv) = &*q; + loop { + let job = { + let Ok(mut g) = m.lock() else { return }; + loop { + if let Some(j) = g.jobs.pop_front() { + g.busy = true; + break j; + } + if dc.strong_count() == 0 { + return; + } + g = match cv.wait_timeout(g, std::time::Duration::from_millis(500)) { + Ok((g, _)) => g, + Err(_) => return, + }; + } + }; + let Some(cache) = dc.upgrade() else { return }; + match job { + Job::Write(kind, key, payload) => cache.write_now(kind, key, &payload), + } + drop(cache); + if let Ok(mut g) = m.lock() { + g.busy = false; + } + cv.notify_all(); + } +} + +/// Check magic, kind and checksum; returns the payload. +fn verify(d: &[u8], kind: Kind) -> Option<&[u8]> { + if d.len() < 16 || &d[..4] != MAGIC || d[4] != kind.tag() { + return None; + } + let (body, sum) = d.split_at(d.len() - 8); + let mut s = [0u8; 8]; + s.copy_from_slice(sum); + (checksum(body) == u64::from_le_bytes(s)).then_some(&body[8..]) +} + +fn encode_frame(w: u32, h: u32, rgba: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rgba.len() / 2 + 16); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(rgba)); + out +} + +fn decode_frame(p: &[u8]) -> Option { + if p.len() < 8 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let rgba = lz4_flex::decompress_size_prepended(&p[8..]).ok()?; + (rgba.len() == w as usize * h as usize * 4).then_some(Frame8 { width: w, height: h, rgba }) +} + +fn encode_layer(b: &Buf) -> Vec { + let (w, h) = (b.img.width, b.img.height); + let n = b.img.data.len(); + // Byte planes: all first bytes of every float, then all second bytes… (compresses well). + let mut planes = vec![0u8; n * 16]; + for (i, px) in b.img.data.iter().enumerate() { + for c in 0..4 { + let bytes = px[c].to_le_bytes(); + for (k, byte) in bytes.iter().enumerate() { + planes[(k * 4 + c) * n + i] = *byte; + } + } + } + let mut out = Vec::with_capacity(planes.len() / 4 + 40); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&b.offset[0].to_le_bytes()); + out.extend_from_slice(&b.offset[1].to_le_bytes()); + out.extend_from_slice(&b.scale.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(&planes)); + out +} + +fn decode_layer(p: &[u8]) -> Option { + if p.len() < 32 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let f = |i: usize| -> Option { Some(f64::from_le_bytes(p[i..i + 8].try_into().ok()?)) }; + let offset = [f(8)?, f(16)?]; + let scale = f(24)?; + let planes = lz4_flex::decompress_size_prepended(&p[32..]).ok()?; + let n = w as usize * h as usize; + if planes.len() != n * 16 { + return None; + } + let mut img = Image::new(w, h); + for (i, px) in img.data.iter_mut().enumerate() { + for c in 0..4 { + let mut bytes = [0u8; 4]; + for (k, byte) in bytes.iter_mut().enumerate() { + *byte = planes[(k * 4 + c) * n + i]; + } + px[c] = f32::from_le_bytes(bytes); + } + } + Some(Buf { img, offset, scale }) +} + +// ---------------------------------------------------------------- content keys + +/// Bump when rendering changes in a way that makes old disk entries wrong. +const RENDER_VERSION: &str = "effectcraft-render-1"; + +fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) { + h.write(format!("{v:?}").as_bytes()); + h.write(&[0xff]); +} + +/// A file's size and modification time (so edited footage gets new keys). +fn file_stamp(h: &mut Hash128, path: &str) { + if path.is_empty() { + return; + } + if let Ok(md) = std::fs::metadata(path) { + h.write_u64(md.len()); + if let Ok(t) = md.modified() + && let Ok(d) = t.duration_since(std::time::UNIX_EPOCH) + { + h.write_u64(d.as_nanos() as u64); + } + } +} + +fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) { + file_stamp(h, &f.path); + for p in &f.sequence { + file_stamp(h, p); + } +} + +/// Salt for layer cache keys on disk: the renderer version and every footage file's identity +/// (layer keys name footage items, not file contents). +pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + for (id, it) in &project.items { + if let effectcraft_project::ItemKind::Footage(f) = &it.kind { + h.write_u64(id.0); + hash_debug(&mut h, f); + footage_stamp(&mut h, f); + } + } + let v = h.finish(); + (v as u64) ^ ((v >> 64) as u64) +} + +/// Content key of a composition's pixels: project render settings, the comp and everything it +/// uses (nested comps, footage, solids, footage files' size and time). With expressions in play +/// (which can read any comp) the whole project counts. +pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 { + use effectcraft_project::{ItemKind, LayerSource}; + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + hash_debug(&mut h, &project.settings); + let mut seen = std::collections::BTreeSet::new(); + let mut stack = vec![comp]; + let mut expressions = false; + while let Some(id) = stack.pop() { + if !seen.insert(id) { + continue; + } + let Some(it) = project.item(id) else { continue }; + if let ItemKind::Comp(c) = &it.kind { + for l in &c.layers { + if let LayerSource::Comp { item } | LayerSource::Footage { item } | LayerSource::Solid { item } = l.source { + stack.push(item); + } + if !expressions { + l.props.walk("", &mut |_, p| expressions |= p.expr.is_some()); + } + } + } + } + let ids: Vec = if expressions { project.items.keys().copied().collect() } else { seen.into_iter().collect() }; + for id in ids { + let Some(it) = project.item(id) else { continue }; + h.write_u64(id.0); + h.write(it.name.as_bytes()); + hash_debug(&mut h, &it.kind); + if let ItemKind::Footage(f) = &it.kind { + footage_stamp(&mut h, f); + } + } + h.write_u64(comp.0); + h.finish() +} + +/// Disk key of one viewer frame. +pub fn frame_key(content: u128, frame: i64, scale_milli: u32, view: u64) -> u128 { + let mut h = Hash128::default(); + h.write(&content.to_le_bytes()); + h.write_u64(frame as u64); + h.write_u64(scale_milli as u64); + h.write_u64(view); + h.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmpdir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("effectcraft-disk-cache-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d + } + + fn buf(seed: u32, w: u32, h: u32) -> Buf { + let mut img = Image::new(w, h); + for (i, p) in img.data.iter_mut().enumerate() { + let v = ((i as u32).wrapping_mul(2654435761u32).wrapping_add(seed) % 1000) as f32 / 999.0; + *p = [v, v * 0.5, 1.0 - v, if i % 7 == 0 { 0.0 } else { 1.0 }]; + } + Buf { img, offset: [1.5, -2.25], scale: 0.5 } + } + + #[test] + fn hit_miss_and_exact_round_trip() { + let dir = tmpdir("hit"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert!(dc.get_layer(42).is_none()); + let b = buf(1, 33, 17); + dc.put_layer(42, &b); + dc.flush(); + let back = dc.get_layer(42).unwrap(); + assert_eq!(back.img, b.img); + assert_eq!((back.offset, back.scale), (b.offset, b.scale)); + let rgba: Vec = (0..20 * 10 * 4).map(|i| (i % 251) as u8).collect(); + dc.put_frame(7, 20, 10, &rgba); + dc.flush(); + assert_eq!(dc.get_frame(7).unwrap().rgba, rgba); + let st = dc.stats(); + assert_eq!((st.entries, st.hits, st.misses, st.writes), (2, 2, 1, 2)); + // Wrong kind is a miss. + assert!(dc.get_frame(42).is_none()); + } + + #[test] + fn persists_across_restarts_and_clears() { + let dir = tmpdir("persist"); + { + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_layer_now(1, &buf(1, 8, 8)); + dc.put_frame_now(2, 2, 2, &[9; 16]); + } + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(dc.stats().entries, 2); + assert!(dc.contains(Kind::Layer, 1) && dc.contains(Kind::Frame, 2)); + assert_eq!(dc.get_frame(2).unwrap().rgba, vec![9; 16]); + dc.clear(); + assert_eq!(dc.stats().entries, 0); + assert!(dc.get_frame(2).is_none()); + let again = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(again.stats().entries, 0); + } + + #[test] + fn lru_eviction_respects_the_limit() { + let dir = tmpdir("lru"); + let dc = DiskCache::open(&dir, u64::MAX).unwrap(); + for k in 0..6u128 { + dc.put_layer_now(k, &buf(k as u32, 64, 64)); + } + let per = dc.stats().bytes / 6; + // Touch 0 so it is the most recently used. + assert!(dc.get_layer(0).is_some()); + dc.set_max_bytes(per * 3 + per / 2); + let st = dc.stats(); + assert!(st.bytes <= per * 3 + per / 2, "{st:?}"); + assert_eq!(st.entries, 3); + assert!(dc.contains(Kind::Layer, 0), "recently used survives"); + assert!(!dc.contains(Kind::Layer, 1) && !dc.contains(Kind::Layer, 2) && !dc.contains(Kind::Layer, 3)); + assert_eq!(st.evictions, 3); + // Files are really gone. + let files = walk(&dir); + assert_eq!(files, 3); + // New writes keep evicting. + dc.put_layer_now(99, &buf(9, 64, 64)); + assert!(dc.stats().bytes <= per * 3 + per / 2); + } + + fn walk(d: &Path) -> usize { + let mut n = 0; + if let Ok(rd) = std::fs::read_dir(d) { + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + n += walk(&p); + } else if p.extension().is_some_and(|x| x == "ecc") { + n += 1; + } + } + } + n + } + + #[test] + fn damaged_files_are_misses() { + let dir = tmpdir("damaged"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_frame_now(5, 4, 4, &[1; 64]); + let p = dc.path(Kind::Frame, 5); + let mut d = std::fs::read(&p).unwrap(); + let n = d.len(); + d[n / 2] ^= 0xff; + std::fs::write(&p, d).unwrap(); + assert!(dc.get_frame(5).is_none()); + assert!(!dc.contains(Kind::Frame, 5)); + assert!(!p.exists()); + } + + #[test] + fn concurrent_readers_and_writers() { + let dir = tmpdir("concurrent"); + let a = DiskCache::open(&dir, 1 << 30).unwrap(); + // A second instance on the same folder (another process, in effect). + let b = DiskCache::open(&dir, 1 << 30).unwrap(); + std::thread::scope(|s| { + for t in 0..4u32 { + let (a, b) = (a.clone(), b.clone()); + s.spawn(move || { + for i in 0..20u128 { + let k = i % 5; + let w = if t % 2 == 0 { &a } else { &b }; + w.put_layer_now(k, &buf(k as u32, 16, 16)); + if let Some(x) = w.get_layer(k) { + assert_eq!(x.img, buf(k as u32, 16, 16).img); + } + } + }); + } + }); + b.rescan(); + for k in 0..5u128 { + assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img); + } + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 14a81575..a4e718a3 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -9,6 +9,7 @@ pub mod audio; pub mod cache; pub mod color; +pub mod disk_cache; use color::Region; pub mod eval; pub mod masks; @@ -46,6 +47,16 @@ pub trait FootageSource: Send + Sync { fn cache_budget(&self) -> Option { None } + /// Vector footage (SVG) rasterised at `scale` × its pixel size, for Continuously Rasterize. + /// `None` when the footage is not vector or cannot be read. + fn vector_frame(&self, _item: ItemId, _footage: &Footage, _scale: f64) -> Option> { + None + } +} + +/// Footage that can be rasterised at any scale (SVG). +pub fn is_vector_footage(f: &Footage) -> bool { + f.codec == "SVG" } /// Layer parameters and audio for effects (see [`effectcraft_effects::EffectHost`]). @@ -348,7 +359,12 @@ impl<'a> Renderer<'a> { /// stays sharp when scaled up). pub fn raster_scale(&self, ctx: &EvalCtx, layer: &Layer) -> f64 { let s = self.opts.scale; - if !layer.switches.collapse || !matches!(layer.source, LayerSource::Text | LayerSource::Shape) { + let vector = match layer.source { + LayerSource::Text | LayerSource::Shape => true, + LayerSource::Footage { item } => matches!(self.project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if is_vector_footage(f)), + _ => false, + }; + if !layer.switches.collapse || !vector { return s; } let (l2c, _) = ctx.layer_to_comp(layer); @@ -631,6 +647,17 @@ impl<'a> Renderer<'a> { if !f.has_video { return None; } + // Continuously rasterised vector footage: drawn at the on-screen scale. + if layer.switches.collapse && is_vector_footage(f) { + let k = self.raster_scale(ctx, layer); + if let Some(img) = self.footage.vector_frame(*item, f, k) { + let mut buf = Buf { img: (*img).clone(), offset: [0.0; 2], scale: k }; + if let Some(c) = self.pipe.media_in(f.color_profile) { + color::convert(&mut buf.img, &c); + } + return Some(buf); + } + } let lt = ctx.source_time(layer); let img = self.footage_frame(ctx, layer, *item, f, lt)?; let k = if img.width > 0 { f.width.max(1) as f64 / img.width as f64 } else { 1.0 }; diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index dfd49ccc..991fac32 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -51,6 +51,7 @@ fn audio_footage() -> Footage { missing: false, sequence: vec![], color_profile: None, + layer: None, } } diff --git a/crates/render/src/tests_color.rs b/crates/render/src/tests_color.rs index 87f9dd1a..016cdc93 100644 --- a/crates/render/src/tests_color.rs +++ b/crates/render/src/tests_color.rs @@ -172,6 +172,7 @@ fn footage(profile: Option) -> Footage { missing: false, sequence: vec![], color_profile: profile, + layer: None, } } diff --git a/crates/render/src/tests_frame_blend.rs b/crates/render/src/tests_frame_blend.rs index c172b81c..3c301d38 100644 --- a/crates/render/src/tests_frame_blend.rs +++ b/crates/render/src/tests_frame_blend.rs @@ -80,6 +80,7 @@ fn setup(mode: FrameBlend, size: (u32, u32)) -> (Project, ItemId) { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); let mut l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, size, None); diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml new file mode 100644 index 00000000..0e1af771 --- /dev/null +++ b/crates/svg/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "effectcraft-svg" +description = "SVG import: parses SVG 1.1 / SVG 2 static vector content (paths, basic shapes, groups, transforms, use, CSS styles, solid and gradient fills/strokes, opacity) and rasterises it at any scale" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-geom = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-path = { workspace = true } +kurbo = { workspace = true } +roxmltree = "0.21" +rayon = { workspace = true } diff --git a/crates/svg/README.md b/crates/svg/README.md new file mode 100644 index 00000000..862bd01d --- /dev/null +++ b/crates/svg/README.md @@ -0,0 +1,26 @@ +# effectcraft-svg + +SVG import for EffectCraft: `.svg` files import as footage (rasterised at any scale, so +Continuously Rasterize stays sharp) and Layer ▸ Create ▸ Create Shapes from Vector Layer turns +them into editable shape layers. + +Implemented from the W3C **Scalable Vector Graphics (SVG) 1.1 (Second Edition)** recommendation +(16 August 2011) and the **SVG 2** candidate recommendation (4 October 2018) for the parts listed +here, plus **CSS Color Module Level 3** for colour keywords and functions: + +- document: `svg` (width/height, `viewBox`, `preserveAspectRatio`), nested `svg`, `g`, `defs`, + `use` (x/y, `href` / `xlink:href`), `symbol` (through `use`), `title`/`desc`/`metadata` ignored; +- shapes: `path` (all path data commands incl. elliptical arcs), `rect` (rx/ry), `circle`, + `ellipse`, `line`, `polyline`, `polygon`; +- `transform` lists (matrix, translate, scale, rotate with centre, skewX, skewY); +- painting: `fill`, `fill-rule`, `fill-opacity`, `stroke`, `stroke-width`, `stroke-linecap`, + `stroke-linejoin`, `stroke-miterlimit`, `stroke-dasharray`, `stroke-dashoffset`, + `stroke-opacity`, `opacity`, `display`, `visibility`, `color`/`currentColor`; +- `linearGradient` and `radialGradient` (with `fx`/`fy` focal point), `gradientUnits`, + `gradientTransform`, `spreadMethod`, `href` inheritance of stops and attributes; +- styling through presentation attributes, the `style` attribute and ` + + + + + + + + + + + + + + + + + diff --git a/crates/svg/tests/fixtures/paths-gradients.svg.attribution b/crates/svg/tests/fixtures/paths-gradients.svg.attribution new file mode 100644 index 00000000..d4b8758f --- /dev/null +++ b/crates/svg/tests/fixtures/paths-gradients.svg.attribution @@ -0,0 +1,7 @@ +asset: paths-gradients.svg +title: SVG test fixture: path data, gradients, CSS styles and use +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft contributors (Claude agent, SVG import) +notes: Hand-written for the effectcraft-svg parser and Create Shapes from Vector Layer tests. diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 62239f19..d0e5a6d9 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -7,6 +7,11 @@ //! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] renders them //! on the UI thread between egui frames, within a time budget. //! +//! With the disk cache on (Settings ▸ Media & Disk Cache), every CPU-rendered frame is also +//! written to disk under a content key (the comp's content hash, frame, scale, view and render +//! options), and a job first looks there: frames survive restarts and project reopenings. The +//! timeline shows disk-only frames as a blue cache bar. +//! //! With Mercury GPU Acceleration (a [`Gpu`] on egui-wgpu's device and the project's renderer set //! to the GPU) frames stay on the GPU: the compositor writes an RGBA8 texture that the viewer //! registers with egui-wgpu and draws directly; pixels are read back only when something needs @@ -16,6 +21,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; use effectcraft_gpu::{DisplayFrame, Gpu}; @@ -122,6 +128,37 @@ pub struct RenderSource { /// Show Channel / exposure / region-of-interest drawing read the CPU frame texture, so GPU /// frames are read back (the compositing still runs on the GPU). pub gpu_display: bool, + /// The persistent disk cache, when enabled. + pub disk: Option>, +} + +/// Content keys of (project revision, comp), computed once per revision. +type ContentKeys = Arc>>; + +fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 { + if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) { + return k; + } + let k = disk_cache::comp_content_key(project, ItemId(comp)); + if let Ok(mut m) = keys.lock() { + if m.len() > 256 { + m.clear(); + } + m.insert((revision, comp), k); + } + k +} + +fn opts_hash(o: &RenderOpts) -> u64 { + let mut h = disk_cache::Hash128::default(); + h.write(format!("{o:?}").as_bytes()); + h.finish() as u64 +} + +/// Disk key of a viewer frame (render options included: region of interest, draft…). +fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 { + let content = content_key(keys, project, key.revision, key.comp); + disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash) } pub struct Frames { @@ -135,6 +172,7 @@ pub struct Frames { ctx: Option, /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, + content_keys: ContentKeys, } impl Default for Frames { @@ -149,6 +187,7 @@ impl Default for Frames { pool: rayon::ThreadPoolBuilder::new().num_threads(threads).thread_name(|i| format!("ec-frame-{i}")).build().expect("frame pool"), ctx: None, last_ms: Arc::new(Mutex::new(0.0)), + content_keys: Arc::default(), } } } @@ -195,6 +234,21 @@ impl Frames { v } + /// Frames of (revision, comp, scale) in the disk cache but not in RAM — the blue cache bar. + /// `opts` are the viewer's render options (part of the disk key). + pub fn disk_frames(&self, src: &RenderSource, revision: u64, comp: u64, scale: u32, view: u64, frames: i64, opts: &RenderOpts) -> Vec { + let Some(dc) = &src.disk else { return vec![] }; + let ram: HashSet = self.cached_frames(revision, comp, scale).into_iter().collect(); + let oh = opts_hash(opts); + (0..frames.clamp(0, 100_000)) + .filter(|f| !ram.contains(f)) + .filter(|f| { + let key = FrameKey { revision, comp, frame: *f, scale, view }; + dc.contains(disk_cache::Kind::Frame, frame_disk_key(&self.content_keys, &src.project, &key, oh)) + }) + .collect() + } + /// Change the RAM preview cache budget (Settings ▸ Memory & CPU), evicting the oldest /// frames when it shrinks. pub fn set_budget(&self, bytes: usize) { @@ -278,7 +332,14 @@ impl Frames { } fn worker(&self) -> Worker { - Worker { queue: self.queue.clone(), cache: self.cache.clone(), inflight: self.inflight.clone(), ctx: self.ctx.clone(), last: self.last_ms.clone() } + Worker { + queue: self.queue.clone(), + cache: self.cache.clone(), + inflight: self.inflight.clone(), + ctx: self.ctx.clone(), + last: self.last_ms.clone(), + content_keys: self.content_keys.clone(), + } } /// Render queued frames on this thread, most important first, until the queue is empty or @@ -306,12 +367,13 @@ struct Worker { inflight: Arc>>, ctx: Option, last: Arc>, + content_keys: ContentKeys, } impl Worker { /// Render the queued job that matters most right now. `false` when the queue was empty. fn run_next(&self) -> bool { - let Worker { queue, cache, inflight, ctx, last } = self; + let Worker { queue, cache, inflight, ctx, last, content_keys } = self; let job = { let Ok(mut q) = queue.lock() else { return false }; // Urgent first (newest), then prefetch in request order. @@ -325,19 +387,18 @@ impl Worker { job }; let t0 = web_time::Instant::now(); - let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); - r.expr = job.src.expr.as_deref(); - r.cache = Some(&job.src.layer_cache); - r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); - // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no - // adapter, or a frame the GPU cannot finish here) the CPU renders it. - let gpu_frame = match (&job.src.gpu, r.active_accel()) { - (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), - _ => None, - }; - let ci = match gpu_frame { - Some(f) => FrameImage::Gpu(Arc::new(f)), - None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts)))); + let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k)); + let ci = match from_disk { + Some(f) => FrameImage::Cpu(Arc::new(egui::ColorImage::from_rgba_premultiplied([f.width as usize, f.height as usize], &f.rgba))), + None => { + let ci = self.render_job(&job); + if let (Some((dc, k)), FrameImage::Cpu(img)) = (&disk, &ci) { + let rgba: Vec = img.pixels.iter().flat_map(|c| c.to_array()).collect(); + dc.put_frame(*k, img.size[0] as u32, img.size[1] as u32, &rgba); + } + ci + } }; if job.urgent && let Ok(mut l) = last.lock() @@ -361,6 +422,23 @@ impl Worker { } true } + + fn render_job(&self, job: &Job) -> FrameImage { + let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); + r.expr = job.src.expr.as_deref(); + r.cache = Some(&job.src.layer_cache); + r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); + // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no + // adapter, or a frame the GPU cannot finish here) the CPU renders it. + let gpu_frame = match (&job.src.gpu, r.active_accel()) { + (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), + _ => None, + }; + match gpu_frame { + Some(f) => FrameImage::Gpu(Arc::new(f)), + None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + } + } } struct Job { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c1948..ec6a2278 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -308,6 +308,7 @@ impl EffectcraftApp { layer_cache: self.session.layer_cache.clone(), gpu: self.gpu.clone(), gpu_display: false, + disk: self.session.disk_cache.clone(), } } @@ -405,13 +406,11 @@ impl EffectcraftApp { self.request_frame_with(comp, frame, scale, true); } - fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { - let Some(c) = self.session.project.comp(comp) else { return }; - let key = self.frame_key(comp, frame, scale); - let t = c.frame_rate.tick_of(frame); + /// Render options of viewer frames of `comp` at `scale`. + pub fn frame_opts(&self, comp: ItemId, scale: f64) -> RenderOpts { let (draft, _) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting); let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp)); - let opts = RenderOpts { + RenderOpts { scale, motion_blur: true, guides: true, @@ -419,7 +418,14 @@ impl EffectcraftApp { view: self.session.view_camera(comp), roi, backend: effectcraft_engine::render::Backend::Auto, - }; + } + } + + fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { + let Some(c) = self.session.project.comp(comp) else { return }; + let key = self.frame_key(comp, frame, scale); + let t = c.frame_rate.tick_of(frame); + let opts = self.frame_opts(comp, scale); if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); } else { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index da5eaec2..7ebadbf4 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -716,7 +716,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, @@ -739,6 +739,10 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option bool { } /// If `id` is a dialog command invoked without its parameters, open its form and return true. +/// The import includes a Photoshop document. +fn is_layered(p: &Value) -> bool { + let psd = |v: &Value| v.as_str().is_some_and(|s| matches!(s.rsplit('.').next().map(str::to_ascii_lowercase).as_deref(), Some("psd" | "psb"))); + match p.get("paths").or(p.get("path")) { + Some(Value::Array(a)) => a.iter().any(psd), + Some(v) => psd(v), + None => false, + } +} + pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let s = &app.session; let comp = s.active_comp(); @@ -116,6 +126,16 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let lt = layer.map(|l| l.layer_time(t)).unwrap_or(t); let base = p.clone(); let (title, fields): (String, Vec) = match id { + // Photoshop files: Import Kind (Footage / Composition / – Retain Layer Sizes). + "file.import" if !has(p, &["importAs"]) && is_layered(p) => ( + "Import Photoshop File".into(), + vec![Field::choice( + "importAs", + "Import Kind", + &[("Footage", json!("footage")), ("Composition", json!("composition")), ("Composition - Retain Layer Sizes", json!("compositionLayerSizes"))], + 1, + )], + ), "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 5b6c085f..785491d7 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -602,6 +602,31 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if let Some((a, b)) = run { draw_run(a, b); } + // Disk-cached frames not in RAM (blue), like After Effects. + if app.session.disk_cache.is_some() { + let frames = comp.frame_rate.frame_at(comp.duration); + let opts = app.frame_opts(cid, scale_key as f64 / 1000.0); + let disk = app.frames.disk_frames(&app.render_source(), app.session.revision, cid.0, scale_key, app.view_hash(cid), frames, &opts); + let draw_blue = |a: i64, b: i64| { + let x0 = tm.x(a as f64 * fd); + let x1 = tm.x((b + 1) as f64 * fd); + p.rect_filled(Rect::from_min_max(pos2(x0, cy0), pos2(x1, cy0 + 2.5)), 0.0, t.cache_blue); + }; + let mut run: Option<(i64, i64)> = None; + for f in disk { + run = match run { + Some((a, b)) if f == b + 1 => Some((a, f)), + Some((a, b)) => { + draw_blue(a, b); + Some((f, f)) + } + None => Some((f, f)), + }; + } + if let Some((a, b)) = run { + draw_blue(a, b); + } + } // Ticks + labels. // Label spacing in whole frames (AE: `00:15f`-style seconds:frames labels). let fps_i = fr.as_f64().round().max(1.0) as i64; diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index 5c82e916..c32ecb24 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -60,6 +60,8 @@ pub struct Tokens { pub cti: Color32, pub work_area: Color32, pub cache_green: Color32, + /// Timeline cache bar for frames held in the disk cache (not in RAM). + pub cache_blue: Color32, pub keyframe: Color32, pub keyframe_selected: Color32, pub pasteboard: Color32, @@ -108,6 +110,7 @@ impl Tokens { cti: Color32::from_rgb(0x2d, 0x8c, 0xeb), work_area: Color32::from_rgb(0x4a, 0x4a, 0x4a), cache_green: Color32::from_rgb(0x3c, 0xa6, 0x4c), + cache_blue: Color32::from_rgb(0x3a, 0x6f, 0xd8), keyframe: Color32::from_rgb(0xa8, 0xa8, 0xa8), keyframe_selected: Color32::from_rgb(0x3d, 0x8f, 0xf5), pasteboard: Color32::from_rgb(0x1a, 0x1a, 0x1a), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 76ff6fa3..09e15047 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -27,10 +27,12 @@ const LAYERS: &[(&str, u8)] = &[ ("raster", 1), ("keyframe", 1), ("path", 1), + ("psd", 1), ("project", 2), ("text", 2), ("effects", 2), ("track", 2), + ("svg", 2), ("render", 3), ("media", 3), ("expr", 3), From 406706e3d55b47f6addf342d1ddc08db12ff2b09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:28:15 -0400 Subject: [PATCH 4/6] OUT-2: WebM export (VP9 intra encoder + alpha, Opus), WAV/AIFF audio-only outputs - effectcraft-vp9enc (L0): VP9 profile 0 key-frame encoder from the VP9 bitstream spec (8x8 intra blocks, DC/V/H/TM, 4x4 DCT/ADST, lossless WHT at quality 100, tile columns); round-trips exactly through FilmCraft's VP9 decoder and ffmpeg. - export: WebM muxer (EBML/Matroska spec) with VP9, VP9 alpha in BlockAdditional (AlphaMode 1), Opus (CodecDelay = pre-skip); WAV and AIFF outputs. - OutputFormat WebM/Wav/Aiff in the Render Queue, renderQueue.add formats, CLI --format. - Tests: FilmCraft decode-back (frames, size, pixels, audio RMS), ffprobe/ffmpeg oracles. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 + Cargo.toml | 5 + apps/effectcraft-cli/src/main.rs | 2 +- crates/engine/src/commands/render_queue.rs | 8 +- crates/engine/src/rq_tests.rs | 2 +- crates/export/Cargo.toml | 4 +- crates/export/src/lib.rs | 6 + crates/export/src/webm.rs | 368 +++++++++ crates/export/tests/roundtrip.rs | 92 +++ crates/export/tests/vp9_roundtrip.rs | 149 ++++ crates/host/tests/vector_import.rs | 23 + crates/media/src/convert.rs | 1 - crates/project/src/render_queue.rs | 30 +- crates/ui-egui/src/panels/render_queue.rs | 2 +- crates/vp9enc/Cargo.toml | 12 + crates/vp9enc/README.md | 15 + crates/vp9enc/src/bool.rs | 193 +++++ crates/vp9enc/src/lib.rs | 857 +++++++++++++++++++++ crates/vp9enc/src/tables.rs | 130 ++++ docs/architecture.md | 15 +- docs/parity.md | 10 +- xtask/src/main.rs | 1 + 22 files changed, 1911 insertions(+), 20 deletions(-) create mode 100644 crates/export/src/webm.rs create mode 100644 crates/export/tests/vp9_roundtrip.rs create mode 100644 crates/vp9enc/Cargo.toml create mode 100644 crates/vp9enc/README.md create mode 100644 crates/vp9enc/src/bool.rs create mode 100644 crates/vp9enc/src/lib.rs create mode 100644 crates/vp9enc/src/tables.rs diff --git a/Cargo.lock b/Cargo.lock index 866ae15f..9b28b8ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1226,11 +1226,13 @@ dependencies = [ "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "effectcraft-vp9enc", "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", "filmcraft-opus", "filmcraft-prores", + "filmcraft-vp9", "gif", "image", "log", @@ -1485,6 +1487,10 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-vp9enc" +version = "0.1.0" + [[package]] name = "effectcraft-web" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 08a7d19d..dc2ce073 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } effectcraft-psd = { path = "crates/psd" } +effectcraft-vp9enc = { path = "crates/vp9enc" } effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } @@ -71,6 +72,7 @@ filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf2 filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -140,6 +142,9 @@ opt-level = 3 [profile.dev.package.effectcraft-export] opt-level = 3 +[profile.dev.package.effectcraft-vp9enc] +opt-level = 3 + # FilmCraft codec hot loops (same settings as FilmCraft's own workspace). [profile.dev.package.filmcraft-h264] opt-level = 3 diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 90d7ef66..3a04e15c 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -9,7 +9,7 @@ //! effectcraft-cli get [--time S] read a property //! effectcraft-cli set [--time S] [--expression E] //! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png] -//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|png|jpeg|tiff|exr|gif] [--start S] [--end S] +//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index c905ce6c..be6cebfb 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -131,7 +131,7 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { let mut any = false; if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); any = true; } @@ -256,7 +256,7 @@ fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; it.output = OutputModule::for_format(f); } apply_settings(&mut it.settings, p, "renderQueue.add")?; @@ -434,7 +434,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { let n = it.extra_outputs.len() + 2; let mut om = it.output.clone(); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om = OutputModule::for_format(f); } if str_p(p, "output").is_none() && str_p(p, "path").is_none() { @@ -509,7 +509,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", can_add, add ), diff --git a/crates/engine/src/rq_tests.rs b/crates/engine/src/rq_tests.rs index c4aa08b8..f172b5e9 100644 --- a/crates/engine/src/rq_tests.rs +++ b/crates/engine/src/rq_tests.rs @@ -159,6 +159,6 @@ fn without_exporter() { s.execute("renderQueue.add", json!({})).unwrap(); assert!(!s.is_enabled("renderQueue.render")); let f = s.execute("renderQueue.formats", json!({})).unwrap(); - assert_eq!(f["formats"].as_array().unwrap().len(), 7); + assert_eq!(f["formats"].as_array().unwrap().len(), 10); assert_eq!(f["formats"][0]["available"], false); } diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 5bdd0fee..15f9b17d 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-export" -description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, PNG/JPEG/TIFF/EXR sequences, animated GIF" +description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, WebM (VP9 + Opus), PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF" version.workspace = true edition.workspace = true license.workspace = true @@ -19,6 +19,7 @@ filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } effectcraft-opusenc = { workspace = true } +effectcraft-vp9enc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -30,3 +31,4 @@ log = { workspace = true } effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } filmcraft-opus = { workspace = true } +filmcraft-vp9 = { workspace = true } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index dc4c35c8..a2af6a4f 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,6 +7,8 @@ //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | +//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -18,6 +20,7 @@ mod encode; mod out; +mod webm; use web_time::Instant; @@ -131,6 +134,9 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result< let mut report = match job.output.format { OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs new file mode 100644 index 00000000..eb99cd22 --- /dev/null +++ b/crates/export/src/webm.rs @@ -0,0 +1,368 @@ +//! WebM output (VP9 video, optional VP9 alpha, Opus audio) and audio-only WAV / AIFF. +//! +//! The WebM container is written from the Matroska / WebM specifications (IETF RFC 8794 EBML, +//! the Matroska element registry and the WebM container guidelines): EBML header, Segment +//! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are +//! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose +//! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every +//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit +//! extended sample rate) follow their published file layouts. + +use std::io::{Seek, SeekFrom, Write}; + +use effectcraft_project::Comp; +use effectcraft_project::render_queue::Channels; +use effectcraft_time::{TICKS_PER_SECOND, Tick}; +use rayon::prelude::*; + +use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; + +const OPUS_RATE: u32 = 48_000; +const OPUS_BITRATE: u32 = 192_000; + +// Element ids (with their length markers). +const EBML: u32 = 0x1A45_DFA3; +const SEGMENT: u32 = 0x1853_8067; +const INFO: u32 = 0x1549_A966; +const TRACKS: u32 = 0x1654_AE6B; +const TRACK_ENTRY: u32 = 0xAE; +const CLUSTER: u32 = 0x1F43_B675; + +fn id_bytes(id: u32) -> Vec { + let b = id.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(3); + b[skip..].to_vec() +} + +/// EBML variable-size integer for an element data size (shortest form). +fn vint(n: u64) -> Vec { + for len in 1..=8u32 { + let max = (1u64 << (7 * len)) - 2; + if n <= max { + let marked = n | (1u64 << (7 * len)); + return marked.to_be_bytes()[8 - len as usize..].to_vec(); + } + } + let mut v = vec![0x01]; + v.extend_from_slice(&n.to_be_bytes()[1..]); + v +} + +fn el(out: &mut Vec, id: u32, data: &[u8]) { + out.extend(id_bytes(id)); + out.extend(vint(data.len() as u64)); + out.extend_from_slice(data); +} + +fn el_uint(out: &mut Vec, id: u32, v: u64) { + let b = v.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(7); + el(out, id, &b[skip..]); +} + +fn el_float(out: &mut Vec, id: u32, v: f64) { + el(out, id, &v.to_be_bytes()); +} + +fn el_str(out: &mut Vec, id: u32, s: &str) { + el(out, id, s.as_bytes()); +} + +/// A block payload: track number, timestamp relative to the cluster, flags, frame. +fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { + let mut b = vint(track); + b.extend_from_slice(&rel.to_be_bytes()); + b.push(flags); + b.extend_from_slice(data); + b +} + +struct Frame { + ms: i64, + track: u64, + data: Vec, + alpha: Option>, +} + +fn cluster(frames: &mut [Frame]) -> Vec { + frames.sort_by_key(|f| (f.ms, f.track)); + let base = frames.first().map_or(0, |f| f.ms); + let mut body = vec![]; + el_uint(&mut body, 0xE7, base.max(0) as u64); + for f in frames.iter() { + let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; + match &f.alpha { + None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + Some(a) => { + let mut g = vec![]; + el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + let mut more = vec![]; + el_uint(&mut more, 0xEE, 1); + el(&mut more, 0xA5, a); + let mut adds = vec![]; + el(&mut adds, 0xA6, &more); + el(&mut g, 0x75A1, &adds); + el(&mut body, 0xA0, &g); + } + } + } + let mut out = vec![]; + el(&mut out, CLUSTER, &body); + out +} + +pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let rate = job.settings.rate(comp); + let alpha = job.output.channels == Channels::Rgba; + let quality = job.output.quality.clamp(1, 100); + let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let with_audio = wants_audio(job); + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let total = st.total; + let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; + let duration_ms = frame_ms(total) as f64; + + let mut file = crate::out::create(job.sink, job.path)?; + // EBML header. + let mut head = vec![]; + let mut eb = vec![]; + el_uint(&mut eb, 0x4286, 1); + el_uint(&mut eb, 0x42F7, 1); + el_uint(&mut eb, 0x42F2, 4); + el_uint(&mut eb, 0x42F3, 8); + el_str(&mut eb, 0x4282, "webm"); + el_uint(&mut eb, 0x4287, 4); + el_uint(&mut eb, 0x4285, 2); + el(&mut head, EBML, &eb); + // Segment with an 8-byte size patched at the end. + head.extend(id_bytes(SEGMENT)); + let size_pos = head.len() as u64; + head.extend_from_slice(&[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + let seg_start = head.len() as u64; + let mut info = vec![]; + el_uint(&mut info, 0x2AD7B1, 1_000_000); + el_float(&mut info, 0x4489, duration_ms); + el_str(&mut info, 0x4D80, "EffectCraft"); + el_str(&mut info, 0x5741, "EffectCraft"); + el(&mut head, INFO, &info); + let mut tracks = vec![]; + let mut v = vec![]; + el_uint(&mut v, 0xD7, 1); + el_uint(&mut v, 0x73C5, 1); + el_uint(&mut v, 0x83, 1); + el_uint(&mut v, 0x9C, 0); + el_str(&mut v, 0x86, "V_VP9"); + el_uint(&mut v, 0x23E383, (1_000_000_000u128 * rate.den as u128 / rate.num.max(1) as u128) as u64); + if alpha { + el_uint(&mut v, 0x55EE, 1); + } + let mut video = vec![]; + el_uint(&mut video, 0xB0, w as u64); + el_uint(&mut video, 0xBA, h as u64); + if alpha { + el_uint(&mut video, 0x53C0, 1); + } + el(&mut v, 0xE0, &video); + el(&mut tracks, TRACK_ENTRY, &v); + if let Some(o) = &opus { + let mut a = vec![]; + el_uint(&mut a, 0xD7, 2); + el_uint(&mut a, 0x73C5, 2); + el_uint(&mut a, 0x83, 2); + el_uint(&mut a, 0x9C, 0); + el_str(&mut a, 0x86, "A_OPUS"); + el(&mut a, 0x63A2, &o.opus_head()); + el_uint(&mut a, 0x56AA, o.pre_skip() as u64 * 1_000_000_000 / OPUS_RATE as u64); + el_uint(&mut a, 0x56BB, 80_000_000); + let mut au = vec![]; + el_float(&mut au, 0xB5, OPUS_RATE as f64); + el_uint(&mut au, 0x9F, 2); + el(&mut a, 0xE1, &au); + el(&mut tracks, TRACK_ENTRY, &a); + } + el(&mut head, TRACKS, &tracks); + file.write_all(&head).map_err(io)?; + let mut seg_len = head.len() as u64 - seg_start; + + // Audio: mixed in blocks, encoded in 20 ms packets. + let (span_start, span_end) = job.settings.span(comp); + let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); + let mut pcm_left: Vec = vec![]; + let mut packets = 0u64; + let mut audio_until = |until: Tick, frames: &mut Vec, finish: bool| { + let Some(enc) = opus.as_mut() else { return }; + let end = until.to_units_floor(OPUS_RATE as i64); + if end > cursor { + let n = (end - cursor) as usize; + let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); + pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + cursor = end; + } + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + if finish && !pcm_left.is_empty() { + // Pad the last packet (pre-skip + padding cover the encoder delay). + let pad = (fs - pcm_left.len() % fs) % fs + fs; + pcm_left.extend(std::iter::repeat_n(0.0, pad)); + } + while pcm_left.len() >= fs { + let pkt = enc.encode_float(&pcm_left[..fs]); + pcm_left.drain(..fs); + frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + packets += 1; + } + }; + + let batch = batch_size(); + let mut i = 0; + while i < total { + let end = (i + batch).min(total); + let encoded: Vec<(Vec, Option>)> = (i..end) + .into_par_iter() + .map(|k| { + let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); + let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); + let color = enc.encode_yuv420(&y, &u, &vv); + let a = alpha.then(|| { + let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); + ea.encode_yuv420(&y, &u, &vv) + }); + (color, a) + }) + .collect(); + let mut frames: Vec = + encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; + audio_until(t_end, &mut frames, end >= total); + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + st.advance(end - i)?; + i = end; + } + if total == 0 { + let mut frames = vec![]; + audio_until(span_end, &mut frames, true); + if !frames.is_empty() { + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + } + } + // Patch the Segment size. + let mut size = [0u8; 8]; + size[0] = 0x01; + size[1..].copy_from_slice(&seg_len.to_be_bytes()[1..]); + file.seek(SeekFrom::Start(size_pos)).map_err(io)?; + file.write_all(&size).map_err(io)?; + file.seek(SeekFrom::End(0)).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) +} + +// ---------------------------------------------------------------- audio-only outputs + +fn ieee_extended(v: f64) -> [u8; 10] { + // 80-bit IEEE 754 extended precision (AIFF COMM sample rate). + let mut out = [0u8; 10]; + if v <= 0.0 { + return out; + } + let e = v.log2().floor() as i32; + let mant = (v / 2f64.powi(e) * (1u64 << 63) as f64) as u64; + out[..2].copy_from_slice(&((e + 16383) as u16).to_be_bytes()); + out[2..].copy_from_slice(&mant.to_be_bytes()); + out +} + +/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. +pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { + let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let (span_start, span_end) = job.settings.span(comp); + let start = span_start.to_units_floor(sr as i64); + let end = span_end.to_units_floor(sr as i64); + let n = (end - start).max(0) as usize; + let mut pcm = Vec::with_capacity(n * 4); + // Mix in one-second blocks, reporting progress in frames. + let block = sr as usize; + let mut done = 0usize; + let total_frames = st.total.max(1); + let mut reported = 0u64; + while done < n { + let k = block.min(n - done); + let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); + for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { + let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; + pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); + } + done += k; + let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); + if now > reported && st.total > 0 { + st.advance(now - reported)?; + reported = now; + } + } + if reported < st.total { + st.advance(st.total - reported)?; + } + let mut out = vec![]; + let frames = (pcm.len() / 4) as u32; + if aiff { + let mut comm = vec![]; + comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&frames.to_be_bytes()); + comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&ieee_extended(sr as f64)); + let ssnd_len = 8 + pcm.len(); + out.extend_from_slice(b"FORM"); + out.extend_from_slice(&((4 + 8 + comm.len() + 8 + ssnd_len + ssnd_len % 2) as u32).to_be_bytes()); + out.extend_from_slice(b"AIFF"); + out.extend_from_slice(b"COMM"); + out.extend_from_slice(&(comm.len() as u32).to_be_bytes()); + out.extend_from_slice(&comm); + out.extend_from_slice(b"SSND"); + out.extend_from_slice(&(ssnd_len as u32).to_be_bytes()); + out.extend_from_slice(&[0; 8]); + out.extend_from_slice(&pcm); + if ssnd_len % 2 == 1 { + out.push(0); + } + } else { + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&sr.to_le_bytes()); + out.extend_from_slice(&(sr * 4).to_le_bytes()); + out.extend_from_slice(&4u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); + out.extend_from_slice(&pcm); + } + let mut file = crate::out::create(job.sink, job.path)?; + file.write_all(&out).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vints_and_ids() { + assert_eq!(vint(0), vec![0x80]); + assert_eq!(vint(126), vec![0xFE]); + assert_eq!(vint(127), vec![0x40, 0x7F]); + assert_eq!(id_bytes(0xA3), vec![0xA3]); + assert_eq!(id_bytes(EBML), vec![0x1A, 0x45, 0xDF, 0xA3]); + assert_eq!(ieee_extended(44100.0), [0x40, 0x0E, 0xAC, 0x44, 0, 0, 0, 0, 0, 0]); + assert_eq!(ieee_extended(48000.0)[..4], [0x40, 0x0E, 0xBB, 0x80]); + } +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 186aa1a9..1ade594f 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -363,3 +363,95 @@ fn movies_carry_audio() { assert!(!effectcraft_media::probe(&path).expect("probe").has_audio); } } + +#[test] +fn webm_vp9_opus_roundtrip() { + let d = out_dir("webm"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + let om = OutputModule { quality: 90, audio: AudioOutput::Auto, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("comp.webm"); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio && r.bytes > 200, "{r:?}"); + // Decoded back through FilmCraft (Matroska demuxer, VP9 and Opus decoders). + let f = decode_movie(&path, FRAMES, 0.08); + assert!(f.has_audio); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + let l = rms(s.iter().step_by(2).copied()); + assert!((l - 0.3536).abs() < 0.06, "left RMS {l}"); + ffmpeg_check(&path, 0.08); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("vp9|64|48|yuv420p"), "{s}"); + assert!(s.contains("opus"), "{s}"); + } +} + +#[test] +fn webm_with_alpha() { + let d = out_dir("webm-alpha"); + let (p, cid) = project(None); + // Transparent outside the red solid. + let om = OutputModule { channels: Channels::Rgba, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("alpha.webm"); + run(&p, cid, &NoFootage, &om, &path); + let f = effectcraft_media::probe(&path).expect("probe our output"); + assert_eq!((f.width, f.height), (W, H)); + let bytes = std::fs::read(&path).unwrap(); + // AlphaMode element (0x53C0) and BlockAdditions (0x75A1). + assert!(bytes.windows(3).any(|w| w == [0x53, 0xC0, 0x81]), "AlphaMode 1"); + assert!(bytes.windows(2).any(|w| w == [0x75, 0xA1]), "BlockAdditions"); + // ffmpeg's libvpx decoder reads the alpha channel. + let png = path.with_extension("alpha.png"); + let ok = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y", "-c:v", "libvpx-vp9", "-i"]) + .arg(&path) + .args(["-vf", "select=eq(n\\,8)", "-frames:v", "1", "-pix_fmt", "rgba"]) + .arg(&png) + .status() + .map(|s| s.success()) + .unwrap_or(false); + if ok { + let img = image::open(&png).unwrap().to_rgba8(); + assert!(img.get_pixel(2, 2)[3] < 20, "corner transparent"); + assert!(img.get_pixel(W / 2, H / 2)[3] > 235, "centre opaque"); + } else { + eprintln!("ffmpeg with libvpx not available: skipping alpha oracle"); + } +} + +#[test] +fn wav_and_aiff_audio_only() { + let d = out_dir("audio-only"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + for (fmt, name, magic) in [(OutputFormat::Wav, "out.wav", &b"RIFF"[..]), (OutputFormat::Aiff, "out.aif", &b"FORM"[..])] { + let om = OutputModule::for_format(fmt); + let path = d.join(name); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio); + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(&bytes[..4], magic); + // 1.2 s of 48 kHz 16-bit stereo. + assert!((bytes.len() as i64 - 57_600 * 4).abs() < 200, "{name}: {}", bytes.len()); + let l = if fmt == OutputFormat::Wav { + // Decoded by FilmCraft. + let f = effectcraft_media::probe(&path).expect("probe audio"); + assert!(f.has_audio && !f.has_video, "{name}"); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + rms(s.iter().step_by(2).copied()) + } else { + // AIFF: big-endian samples after the SSND chunk's offset/block size. + let at = bytes.windows(4).position(|w| w == b"SSND").expect("SSND") + 16; + let left = bytes[at..].chunks_exact(4).skip(14_400).take(24_000).map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32767.0); + rms(left) + }; + assert!((l - 0.3536).abs() < 0.01, "{name}: left RMS {l}"); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("pcm_s16"), "{s}"); + } + } +} diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs new file mode 100644 index 00000000..8156d850 --- /dev/null +++ b/crates/export/tests/vp9_roundtrip.rs @@ -0,0 +1,149 @@ +//! The VP9 intra encoder (`effectcraft-vp9enc`) round-trips through FilmCraft's VP9 decoder (and +//! ffmpeg as an external oracle when installed): sizes, lossless exactness, PSNR. + +use effectcraft_vp9enc::{EncoderConfig, Vp9Encoder}; +use filmcraft_vp9::{Decoder, Plane}; + +fn plane_u8(p: &Plane) -> Vec { + match p { + Plane::U8(v) => v.clone(), + Plane::U16(v) => v.iter().map(|x| *x as u8).collect(), + } +} + +/// Synthetic planes: gradients, noise and sharp edges. +fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut s = 99u32 + kind; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 24) as u8 + }; + let y: Vec = (0..w * h) + .map(|i| { + let (x, yy) = (i % w, i / w); + match kind { + 0 => ((x * 255) / w.max(1)) as u8 / 2 + ((yy * 255) / h.max(1)) as u8 / 2, + 1 => rnd(), + _ => { + if (x / 7 + yy / 5) % 2 == 0 { + 30 + } else { + 220 + } + } + } + }) + .collect(); + let u: Vec = (0..cw * ch).map(|i| (64 + (i % cw) * 128 / cw.max(1)) as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (200 - (i / cw) * 100 / ch.max(1)) as u8).collect(); + (y, u, v) +} + +fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { + let mut d = Decoder::new(); + let mut pics = d.decode(frame, 0).expect("decode"); + assert_eq!(pics.len(), 1); + pics.remove(0) +} + +fn crop(p: &[u8], stride: usize, w: usize, h: usize) -> Vec { + (0..h).flat_map(|y| p[y * stride..y * stride + w].to_vec()).collect() +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse: f64 = a.iter().zip(b).map(|(x, y)| (*x as f64 - *y as f64).powi(2)).sum::() / a.len() as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } +} + +#[test] +fn lossless_is_exact_for_odd_sizes() { + for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { + for kind in 0..3 { + let (y, u, v) = source(w, h, kind); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert_eq!((p.width, p.height), (w as u32, h as u32)); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + assert_eq!(crop(&plane_u8(&p.y), p.y_stride, w, h), y, "{w}x{h} kind {kind}: luma"); + assert_eq!(crop(&plane_u8(&p.u), p.uv_stride, cw, ch), u, "{w}x{h} kind {kind}: u"); + assert_eq!(crop(&plane_u8(&p.v), p.uv_stride, cw, ch), v, "{w}x{h} kind {kind}: v"); + } + } +} + +#[test] +fn lossy_quality_and_size() { + let (w, h) = (200, 120); + for kind in [0, 2] { + let (y, u, v) = source(w, h, kind); + let mut last = 0usize; + for q in [30u8, 60, 80, 95] { + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + let got = crop(&plane_u8(&p.y), p.y_stride, w, h); + let db = psnr(&got, &y); + let min = match q { + 30 => 20.0, + 60 => 28.0, + 80 => 32.0, + _ => 38.0, + }; + assert!(db > min, "kind {kind} q{q}: PSNR {db:.1} dB, {} bytes", f.len()); + assert!(f.len() >= last, "higher quality, more bytes"); + last = f.len(); + } + } +} + +#[test] +fn wide_frames_use_tile_columns() { + // 4160 px > 64 superblocks: two tile columns are required. + let (w, h) = (4160, 16); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); +} + +#[test] +fn ffmpeg_decodes_our_stream() { + let (w, h) = (96, 64); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let frame = e.encode_yuv420(&y, &u, &v); + // IVF container (one frame). + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + ivf.extend_from_slice(&(frame.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&0u64.to_le_bytes()); + ivf.extend_from_slice(&frame); + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); + assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); +} diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs index eb5be076..590112c9 100644 --- a/crates/host/tests/vector_import.rs +++ b/crates/host/tests/vector_import.rs @@ -161,3 +161,26 @@ fn svg_footage_and_shapes_from_vector_layer() { assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); } } + +#[test] +fn svg_footage_continuously_rasterizes() { + let path = tmp("cr.svg"); + std::fs::write(&path, SVG).unwrap(); + let doc = effectcraft_svg::parse(SVG).unwrap(); + let (w, h) = doc.pixel_size(); + let mut s = effectcraft_host::session(); + let item = s.execute_checked("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "C", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + for (k, v) in [("transform/anchor", json!([0, 0])), ("transform/position", json!([0, 0])), ("transform/scale", json!([300, 300]))] { + s.execute("prop.set", json!({"layer": lid, "path": k, "value": v})).unwrap(); + } + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let want = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let soft = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + s.execute("layer.setSwitch", json!({"layers": [lid], "switch": "collapse", "value": true})).unwrap(); + let sharp = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + assert!(sharp < 0.004, "continuously rasterised diff {sharp}"); + assert!(sharp < soft, "sharper than the upscaled pixels ({sharp} vs {soft})"); +} diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index dce33d8a..1edcf5fd 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -392,4 +392,3 @@ mod tests { } } } - diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index c41462b5..d1e6683c 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -136,17 +136,26 @@ pub enum OutputFormat { ExrSequence, /// Animated GIF. Gif, + /// WebM: VP9 video (alpha with RGB + Alpha) + Opus audio. + WebM, + /// Audio only: WAV (16-bit PCM). + Wav, + /// Audio only: AIFF (16-bit PCM). + Aiff, } impl OutputFormat { - pub const ALL: [OutputFormat; 7] = [ + pub const ALL: [OutputFormat; 10] = [ OutputFormat::H264, OutputFormat::ProRes, + OutputFormat::WebM, OutputFormat::PngSequence, OutputFormat::JpegSequence, OutputFormat::TiffSequence, OutputFormat::ExrSequence, OutputFormat::Gif, + OutputFormat::Wav, + OutputFormat::Aiff, ]; pub fn label(self) -> &'static str { @@ -158,6 +167,9 @@ impl OutputFormat { OutputFormat::TiffSequence => "TIFF Sequence", OutputFormat::ExrSequence => "OpenEXR Sequence", OutputFormat::Gif => "Animated GIF", + OutputFormat::WebM => "WebM (VP9 + Opus)", + OutputFormat::Wav => "WAV", + OutputFormat::Aiff => "AIFF", } } pub fn extension(self) -> &'static str { @@ -169,19 +181,26 @@ impl OutputFormat { OutputFormat::TiffSequence => "tif", OutputFormat::ExrSequence => "exr", OutputFormat::Gif => "gif", + OutputFormat::WebM => "webm", + OutputFormat::Wav => "wav", + OutputFormat::Aiff => "aif", } } pub fn is_sequence(self) -> bool { matches!(self, OutputFormat::PngSequence | OutputFormat::JpegSequence | OutputFormat::TiffSequence | OutputFormat::ExrSequence) } pub fn is_movie(self) -> bool { - matches!(self, OutputFormat::H264 | OutputFormat::ProRes) + matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM) + } + /// Audio-only outputs (no video is rendered). + pub fn is_audio_only(self) -> bool { + matches!(self, OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_alpha(self) -> bool { - !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence) + !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_audio(self) -> bool { - self.is_movie() + self.is_movie() || self.is_audio_only() } /// The written frame size for a rendered size: H.264 needs even dimensions (rounded down, /// at least 2). @@ -201,6 +220,9 @@ impl OutputFormat { "tif" | "tiff" | "tiffsequence" => OutputFormat::TiffSequence, "exr" | "openexr" | "openexrsequence" => OutputFormat::ExrSequence, "gif" | "animatedgif" => OutputFormat::Gif, + "webm" | "vp9" | "webmvp9opus" => OutputFormat::WebM, + "wav" | "wave" => OutputFormat::Wav, + "aif" | "aiff" | "aifc" => OutputFormat::Aiff, _ => return None, }) } diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index 154dd4c2..06403181 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -108,7 +108,7 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } - OutputFormat::JpegSequence => { + OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml new file mode 100644 index 00000000..62144b25 --- /dev/null +++ b/crates/vp9enc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-vp9enc" +description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md new file mode 100644 index 00000000..c8f781b8 --- /dev/null +++ b/crates/vp9enc/README.md @@ -0,0 +1,15 @@ +# effectcraft-vp9enc + +A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, +BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 +DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 +(mathematically lossless). + +Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March +2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode +info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) +and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The +boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec +tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR +Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an +external test oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs new file mode 100644 index 00000000..45db92dd --- /dev/null +++ b/crates/vp9enc/src/bool.rs @@ -0,0 +1,193 @@ +//! Boolean (arithmetic) encoder: the inverse of the VP9 boolean decoder (spec §9.2), written as +//! the classic carry-propagating range coder described in RFC 6386 §7.3 (VP8 and VP9 share the +//! bool coder). + +pub struct BoolEncoder { + out: Vec, + range: u32, + bottom: u32, + bit_count: i32, +} + +impl Default for BoolEncoder { + fn default() -> Self { + BoolEncoder::new() + } +} + +impl BoolEncoder { + pub fn new() -> Self { + BoolEncoder { out: Vec::new(), range: 255, bottom: 0, bit_count: 24 } + } + + fn add_one(&mut self) { + let mut i = self.out.len(); + while i > 0 { + i -= 1; + if self.out[i] == 255 { + self.out[i] = 0; + } else { + self.out[i] += 1; + return; + } + } + } + + /// Write `bit` with probability `prob`/256 of it being 0. + #[inline] + pub fn write(&mut self, bit: bool, prob: u8) { + let split = 1 + (((self.range - 1) * prob as u32) >> 8); + if bit { + self.bottom = self.bottom.wrapping_add(split); + self.range -= split; + } else { + self.range = split; + } + while self.range < 128 { + self.range <<= 1; + if self.bottom & (1 << 31) != 0 { + self.add_one(); + } + self.bottom <<= 1; + self.bit_count -= 1; + if self.bit_count == 0 { + self.out.push((self.bottom >> 24) as u8); + self.bottom &= (1 << 24) - 1; + self.bit_count = 8; + } + } + } + + pub fn literal(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.write((v >> i) & 1 != 0, 128); + } + } + + /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). + pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { + // Find the path to the leaf. + fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.push((node >> 1, b == 1)); + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if path(tree, t as usize, value, out) { + return true; + } + out.pop(); + } + false + } + let mut p = Vec::with_capacity(8); + path(tree, 0, value, &mut p); + for (i, bit) in p { + self.write(bit, probs[i]); + } + } + + /// Flush (RFC 6386 §7.3) and return the bytes. + pub fn finish(mut self) -> Vec { + let mut c = self.bit_count; + let mut v = self.bottom; + if v & (1u32 << (32 - c)) != 0 { + self.add_one(); + } + v <<= c & 7; + c >>= 3; + while { + c -= 1; + c >= 0 + } { + v <<= 8; + } + for _ in 0..4 { + self.out.push((v >> 24) as u8); + v <<= 8; + } + self.out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A straightforward decoder per VP9 spec §9.2. + struct Dec<'a> { + d: &'a [u8], + pos: usize, + value: u64, + bits: i32, + range: u32, + } + impl<'a> Dec<'a> { + fn new(d: &'a [u8]) -> Self { + let mut x = Dec { d, pos: 0, value: 0, bits: 0, range: 255 }; + x.fill(); + x + } + fn fill(&mut self) { + while self.bits <= 48 { + let b = if self.pos < self.d.len() { self.d[self.pos] } else { 0 }; + self.pos += 1; + self.value |= (b as u64) << (56 - self.bits); + self.bits += 8; + } + } + fn read(&mut self, p: u8) -> bool { + let split = 1 + (((self.range - 1) * p as u32) >> 8); + let big = (split as u64) << 56; + let bit = if self.value >= big { + self.range -= split; + self.value -= big; + true + } else { + self.range = split; + false + }; + while self.range < 128 { + self.range <<= 1; + self.value <<= 1; + self.bits -= 1; + } + self.fill(); + bit + } + } + + #[test] + fn round_trip() { + let mut e = BoolEncoder::new(); + let mut seq = vec![]; + let mut s = 12345u32; + for i in 0..20000 { + s = s.wrapping_mul(1103515245).wrapping_add(12345); + let p = ((s >> 8) % 255 + 1) as u8; + let bit = (s >> 20) % 256 >= p as u32 || i % 97 == 0; + seq.push((bit, p)); + e.write(bit, p); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for (i, (b, p)) in seq.into_iter().enumerate() { + assert_eq!(dec.read(p), b, "bit {i}"); + } + } + + #[test] + fn all_ones_carry() { + let mut e = BoolEncoder::new(); + for _ in 0..5000 { + e.write(true, 1); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for _ in 0..5000 { + assert!(dec.read(1)); + } + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs new file mode 100644 index 00000000..02cbb0e2 --- /dev/null +++ b/crates/vp9enc/src/lib.rs @@ -0,0 +1,857 @@ +//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & +//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see +//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width +//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard +//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by +//! prediction error, default probabilities (no probability updates). The encoder reconstructs +//! exactly what a decoder reconstructs (the specification's integer inverse transforms and +//! intra edge rules), so prediction never drifts. + +mod bool; +mod tables; + +use bool::BoolEncoder; +use tables::*; + +const DC_PRED: u8 = 0; +const V_PRED: u8 = 1; +const H_PRED: u8 = 2; +const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; + +const DCT_DCT: u8 = 0; +const ADST_DCT: u8 = 1; +const DCT_ADST: u8 = 2; +const ADST_ADST: u8 = 3; + +/// mode2txfm_map for the modes used here. +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// Encoder settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + /// 1..=100; 100 is lossless. + pub quality: u8, + /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. + pub full_range: bool, +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +// ---------------------------------------------------------------- transforms (8.7) + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms, so `inverse4(forward(res)) ≈ res`. +fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +// ---------------------------------------------------------------- planes and prediction + +struct Plane { + w: usize, + /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). + max_x: usize, + max_y: usize, + src: Vec, + rec: Vec, +} + +impl Plane { + fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { + let mut src = vec![0u8; w * h]; + for y in 0..h { + let sy = y.min(vh.saturating_sub(1)); + for x in 0..w { + let sx = x.min(vw.saturating_sub(1)); + src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); + } + } + Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + } +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- the encoder + +pub struct Vp9Encoder { + cfg: EncoderConfig, + qidx: u8, + lossless: bool, +} + +/// Per-tile coding state. +struct Tile<'a> { + enc: &'a Vp9Encoder, + bc: BoolEncoder, + planes: &'a mut [Plane; 3], + mi_cols: usize, + mi_rows: usize, + col_start: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// y mode of every 8×8 block (for mode contexts). + modes: &'a mut [u8], +} + +impl Vp9Encoder { + pub fn new(cfg: EncoderConfig) -> Self { + let q = cfg.quality.clamp(1, 100); + let lossless = q >= 100; + // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; + Vp9Encoder { cfg, qidx, lossless } + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// ((w+1)/2)×((h+1)/2), tightly packed). + pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let mi_cols = w.div_ceil(8); + let mi_rows = h.div_ceil(8); + let sb_cols = mi_cols.div_ceil(8); + let sb_rows = mi_rows.div_ceil(8); + let (pw, ph) = (sb_cols * 64, sb_rows * 64); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut planes = [ + Plane::new(y, w, h, pw, ph, mi_cols * 8 - 1, mi_rows * 8 - 1), + Plane::new(u, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + Plane::new(v, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + ]; + // Tile columns: as few as the width allows (tile_info, 6.2.13). + let mut min_log2 = 0; + while (64usize << min_log2) < sb_cols { + min_log2 += 1; + } + let mut max_log2 = 1; + while (sb_cols >> max_log2) >= 4 { + max_log2 += 1; + } + max_log2 -= 1; + let tile_cols = 1usize << min_log2; + let mut modes = vec![0u8; mi_cols * mi_rows]; + let mut tiles = vec![]; + for t in 0..tile_cols { + let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); + let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); + let mut tile = Tile { + enc: self, + bc: BoolEncoder::new(), + planes: &mut planes, + mi_cols, + mi_rows, + col_start: start, + above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; mi_cols + 8], + left_part: vec![0; 8], + modes: &mut modes, + }; + tile.bc.write(false, 128); + for r in (0..mi_rows).step_by(8) { + for p in 0..3 { + tile.left_nz[p].fill(0); + } + tile.left_part.fill(0); + for c in (start..end).step_by(8) { + tile.partition(r, c, 12); + } + } + tiles.push(tile.bc.finish()); + } + // Compressed header. + let mut ch_enc = BoolEncoder::new(); + ch_enc.write(false, 128); + if !self.lossless { + ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + } + ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) + for _ in 0..3 { + ch_enc.write(false, 252); // skip probabilities unchanged + } + let compressed = ch_enc.finish(); + // Uncompressed header. + let mut bw = BitWriter::default(); + bw.put(2, 2); // frame_marker + bw.put(0, 1); // profile low bit + bw.put(0, 1); // profile high bit + bw.put(0, 1); // show_existing_frame + bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(1, 1); // show_frame + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + bw.put(0, 2); // frame_context_idx + bw.put(0, 6); // loop_filter_level + bw.put(0, 3); // sharpness + bw.put(0, 1); // mode_ref_delta_enabled + bw.put(self.qidx as u32, 8); + bw.put(0, 1); // delta_q_y_dc + bw.put(0, 1); // delta_q_uv_dc + bw.put(0, 1); // delta_q_uv_ac + bw.put(0, 1); // segmentation_enabled + if min_log2 < max_log2 { + bw.put(0, 1); // no extra tile columns + } + bw.put(0, 1); // tile_rows_log2 = 0 + bw.put(compressed.len() as u32, 16); + let mut out = bw.finish(); + out.extend_from_slice(&compressed); + let n = tiles.len(); + for (i, t) in tiles.into_iter().enumerate() { + if i + 1 < n { + out.extend_from_slice(&(t.len() as u32).to_be_bytes()); + } + out.extend_from_slice(&t); + } + out + } +} + +#[derive(Default)] +struct BitWriter { + out: Vec, + acc: u32, + n: u32, +} + +impl BitWriter { + fn put(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1); + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out + } +} + +const BLOCK_8X8: u8 = 3; +const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; +const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; + +impl Tile<'_> { + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + if r >= self.mi_rows || c >= self.mi_cols { + return; + } + let num8 = NUM_8X8[bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < self.mi_rows; + let has_cols = (c + half) < self.mi_cols; + let bsl = MI_WIDTH_LOG2[bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let above = (above >> boffset) & 1; + let left = (left >> boffset) & 1; + let ctx = bsl * 4 + left as usize * 2 + above as usize; + let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; + if bsize == BLOCK_8X8 { + // PARTITION_NONE. + self.bc.write(false, probs[0]); + self.block(r, c); + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.bc.tree(&PARTITION_TREE, probs, 3); + } else if has_cols { + self.bc.write(true, probs[1]); + } else if has_rows { + self.bc.write(true, probs[2]); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + fn block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. + let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). + self.bc.write(false, DEFAULT_SKIP_PROB[0]); + let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; + let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); + let o = y_mode as usize * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); + self.modes[r * self.mi_cols + c] = y_mode; + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.mi_cols * 8) >> ss; + let max_y = (self.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); + // Quantised levels (raster order). + let levels: [i32; 16] = if self.enc.lossless { + forward_wht(&res) + } else { + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let qs = if k == 0 { dcq } else { acq } as f64; + let a = coef.abs() / qs; + // Rounding with a small dead zone for AC coefficients. + let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; + lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; + } + lv + }; + let sc = scan(tx_type); + let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); + // Tokens. + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, tx_type, &levels, eob, ctx0); + // Reconstruct exactly as the decoder does. + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { dcq } else { acq }; + } + let resid = if eob == 0 { + [0i32; 16] + } else if self.enc.lossless { + inverse_wht(&deq) + } else { + inverse4(&deq, tx_type) + }; + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. + let base = (ptype * 2) * 6 * 6 * 3; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = base + (band * 6 + ctx) * 3; + let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; + if check_eob { + let more = c < eob; + self.bc.write(more, p[0]); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.bc.write(false, p[1]); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.bc.write(true, p[1]); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.bc.write(false, p[2]); + } else { + self.bc.write(true, p[2]); + let pp = pareto(p[2]); + let path: &[(usize, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.bc.write(b, pp[n]); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.bc.write(levels[pos] < 0, 128); + c += 1; + } + } +} + +// ---------------------------------------------------------------- colour conversion + +/// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). +pub fn rgba_to_yuv420(rgba: &[u8], w: u32, h: u32, full_range: bool) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut y = vec![0u8; w * h]; + let mut u = vec![0u8; cw * ch]; + let mut v = vec![0u8; cw * ch]; + let (ys, yo, cs) = if full_range { (255.0, 0.0, 255.0) } else { (219.0, 16.0, 224.0) }; + let px = |x: usize, yy: usize| -> [f32; 3] { + let i = (yy.min(h - 1) * w + x.min(w - 1)) * 4; + [rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0] + }; + let luma = |c: [f32; 3]| 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; + for yy in 0..h { + for x in 0..w { + y[yy * w + x] = (luma(px(x, yy)) * ys + yo).round().clamp(0.0, 255.0) as u8; + } + } + for cy in 0..ch { + for cx in 0..cw { + let mut acc = [0.0f32; 3]; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let c = px(cx * 2 + dx, cy * 2 + dy); + for k in 0..3 { + acc[k] += c[k] / 4.0; + } + } + let l = luma(acc); + let cb = (acc[2] - l) / 1.8556; + let cr = (acc[0] - l) / 1.5748; + u[cy * cw + cx] = (cb * cs + 128.0).round().clamp(0.0, 255.0) as u8; + v[cy * cw + cx] = (cr * cs + 128.0).round().clamp(0.0, 255.0) as u8; + } + } + (y, u, v) +} + +/// The alpha channel as a luma plane (full range) with neutral chroma — the second VP9 stream of +/// WebM alpha. +pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let y: Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); + (y, vec![128; cw * ch], vec![128; cw * ch]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs new file mode 100644 index 00000000..dc8027a6 --- /dev/null +++ b/crates/vp9enc/src/tables.rs @@ -0,0 +1,130 @@ +//! Tables from the VP9 Bitstream & Decoding Process Specification v0.6 (section 10), as +//! transcribed in FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0, the same +//! licence as this crate). + +#![allow(clippy::unreadable_literal)] + +pub const DC_QLOOKUP: [i32; 256] = [ + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, + 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 70, 70, 71, 72, + 73, 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, 111, 113, 114, + 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, 177, 180, + 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, + 284, 288, 292, 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, + 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, + 775, 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, +]; + +pub const AC_QLOOKUP: [i32; 256] = [ + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, + 136, 138, 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, 373, 380, 387, + 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, + 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, + 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, +]; + +pub const DEFAULT_SCAN_4X4: [u16; 16] = [0, 4, 1, 5, 8, 2, 12, 9, 3, 6, 13, 10, 7, 14, 11, 15]; + +pub const COL_SCAN_4X4: [u16; 16] = [0, 4, 8, 1, 12, 5, 9, 2, 13, 6, 10, 3, 7, 14, 11, 15]; + +pub const ROW_SCAN_4X4: [u16; 16] = [0, 1, 4, 2, 5, 3, 6, 8, 9, 7, 12, 10, 13, 11, 14, 15]; + +pub const COEFBAND_4X4: [u8; 16] = [0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5]; + +pub const ENERGY_CLASS: [u8; 12] = [0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5]; + +pub const PARETO_TABLE: [u8; 1024] = [ + 3, 86, 128, 6, 86, 23, 88, 29, 9, 86, 129, 17, 88, 61, 94, 76, 15, 87, 129, 28, 89, 93, 100, 110, 20, 88, 130, 38, 91, 118, 106, 136, 26, 89, 131, 48, 92, + 139, 111, 156, 31, 90, 131, 58, 94, 156, 117, 171, 37, 90, 132, 66, 95, 171, 122, 184, 42, 91, 132, 75, 97, 183, 127, 194, 47, 92, 133, 83, 98, 193, 132, + 202, 52, 93, 133, 90, 100, 201, 137, 208, 57, 94, 134, 98, 101, 208, 142, 214, 62, 94, 135, 105, 103, 214, 146, 218, 66, 95, 135, 111, 104, 219, 151, 222, + 71, 96, 136, 117, 106, 224, 155, 225, 76, 97, 136, 123, 107, 227, 159, 228, 80, 98, 137, 129, 109, 231, 162, 231, 84, 98, 138, 134, 110, 234, 166, 233, 89, + 99, 138, 140, 112, 236, 170, 235, 93, 100, 139, 145, 113, 238, 173, 236, 97, 101, 140, 149, 115, 240, 176, 238, 101, 102, 140, 154, 116, 242, 179, 239, + 105, 103, 141, 158, 118, 243, 182, 240, 109, 104, 141, 162, 119, 244, 185, 241, 113, 104, 142, 166, 120, 245, 187, 242, 116, 105, 143, 170, 122, 246, 190, + 243, 120, 106, 143, 173, 123, 247, 192, 244, 123, 107, 144, 177, 125, 248, 195, 244, 127, 108, 145, 180, 126, 249, 197, 245, 130, 109, 145, 183, 128, 249, + 199, 245, 134, 110, 146, 186, 129, 250, 201, 246, 137, 111, 147, 189, 131, 251, 203, 246, 140, 112, 147, 192, 132, 251, 205, 247, 143, 113, 148, 194, 133, + 251, 207, 247, 146, 114, 149, 197, 135, 252, 208, 248, 149, 115, 149, 199, 136, 252, 210, 248, 152, 115, 150, 201, 138, 252, 211, 248, 155, 116, 151, 204, + 139, 253, 213, 249, 158, 117, 151, 206, 140, 253, 214, 249, 161, 118, 152, 208, 142, 253, 216, 249, 163, 119, 153, 210, 143, 253, 217, 249, 166, 120, 153, + 212, 144, 254, 218, 250, 168, 121, 154, 213, 146, 254, 220, 250, 171, 122, 155, 215, 147, 254, 221, 250, 173, 123, 155, 217, 148, 254, 222, 250, 176, 124, + 156, 218, 150, 254, 223, 250, 178, 125, 157, 220, 151, 254, 224, 251, 180, 126, 157, 221, 152, 254, 225, 251, 183, 127, 158, 222, 153, 254, 226, 251, 185, + 128, 159, 224, 155, 255, 227, 251, 187, 129, 160, 225, 156, 255, 228, 251, 189, 131, 160, 226, 157, 255, 228, 251, 191, 132, 161, 227, 159, 255, 229, 251, + 193, 133, 162, 228, 160, 255, 230, 252, 195, 134, 163, 230, 161, 255, 231, 252, 197, 135, 163, 231, 162, 255, 231, 252, 199, 136, 164, 232, 163, 255, 232, + 252, 201, 137, 165, 233, 165, 255, 233, 252, 202, 138, 166, 233, 166, 255, 233, 252, 204, 139, 166, 234, 167, 255, 234, 252, 206, 140, 167, 235, 168, 255, + 235, 252, 207, 141, 168, 236, 169, 255, 235, 252, 209, 142, 169, 237, 171, 255, 236, 252, 210, 144, 169, 237, 172, 255, 236, 252, 212, 145, 170, 238, 173, + 255, 237, 252, 214, 146, 171, 239, 174, 255, 237, 253, 215, 147, 172, 240, 175, 255, 238, 253, 216, 148, 173, 240, 176, 255, 238, 253, 218, 149, 173, 241, + 177, 255, 239, 253, 219, 150, 174, 241, 179, 255, 239, 253, 220, 152, 175, 242, 180, 255, 240, 253, 222, 153, 176, 242, 181, 255, 240, 253, 223, 154, 177, + 243, 182, 255, 240, 253, 224, 155, 178, 244, 183, 255, 241, 253, 225, 156, 178, 244, 184, 255, 241, 253, 226, 158, 179, 244, 185, 255, 242, 253, 228, 159, + 180, 245, 186, 255, 242, 253, 229, 160, 181, 245, 187, 255, 242, 253, 230, 161, 182, 246, 188, 255, 243, 253, 231, 163, 183, 246, 189, 255, 243, 253, 232, + 164, 184, 247, 190, 255, 243, 253, 233, 165, 185, 247, 191, 255, 244, 253, 234, 166, 185, 247, 192, 255, 244, 253, 235, 168, 186, 248, 193, 255, 244, 253, + 236, 169, 187, 248, 194, 255, 244, 253, 236, 170, 188, 248, 195, 255, 245, 253, 237, 171, 189, 249, 196, 255, 245, 254, 238, 173, 190, 249, 197, 255, 245, + 254, 239, 174, 191, 249, 198, 255, 245, 254, 240, 175, 192, 249, 199, 255, 246, 254, 240, 177, 193, 250, 200, 255, 246, 254, 241, 178, 194, 250, 201, 255, + 246, 254, 242, 179, 195, 250, 202, 255, 246, 254, 242, 181, 196, 250, 203, 255, 247, 254, 243, 182, 197, 251, 204, 255, 247, 254, 244, 184, 198, 251, 205, + 255, 247, 254, 244, 185, 199, 251, 206, 255, 247, 254, 245, 186, 200, 251, 207, 255, 247, 254, 246, 188, 201, 252, 207, 255, 248, 254, 246, 189, 202, 252, + 208, 255, 248, 254, 247, 191, 203, 252, 209, 255, 248, 254, 247, 192, 204, 252, 210, 255, 248, 254, 248, 194, 205, 252, 211, 255, 248, 254, 248, 195, 206, + 252, 212, 255, 249, 254, 249, 197, 207, 253, 213, 255, 249, 254, 249, 198, 208, 253, 214, 255, 249, 254, 250, 200, 210, 253, 215, 255, 249, 254, 250, 201, + 211, 253, 215, 255, 249, 254, 250, 203, 212, 253, 216, 255, 249, 254, 251, 204, 213, 253, 217, 255, 250, 254, 251, 206, 214, 254, 218, 255, 250, 254, 252, + 207, 216, 254, 219, 255, 250, 254, 252, 209, 217, 254, 220, 255, 250, 254, 252, 211, 218, 254, 221, 255, 250, 254, 253, 213, 219, 254, 222, 255, 250, 254, + 253, 214, 221, 254, 223, 255, 250, 254, 253, 216, 222, 254, 224, 255, 251, 254, 253, 218, 224, 254, 225, 255, 251, 254, 254, 220, 225, 254, 225, 255, 251, + 254, 254, 222, 227, 255, 226, 255, 251, 254, 254, 224, 228, 255, 227, 255, 251, 254, 254, 226, 230, 255, 228, 255, 251, 254, 255, 228, 231, 255, 230, 255, + 251, 254, 255, 230, 233, 255, 231, 255, 252, 254, 255, 232, 235, 255, 232, 255, 252, 254, 255, 235, 237, 255, 233, 255, 252, 254, 255, 238, 240, 255, 235, + 255, 252, 255, 255, 241, 243, 255, 236, 255, 252, 254, 255, 246, 247, 255, 239, 255, 253, 255, +]; + +pub const KF_PARTITION_PROBS: [u8; 48] = [ + 158, 97, 94, 93, 24, 99, 85, 119, 44, 62, 59, 67, 149, 53, 53, 94, 20, 48, 83, 53, 24, 52, 18, 18, 150, 40, 39, 78, 12, 26, 67, 33, 11, 24, 7, 5, 174, 35, + 49, 68, 11, 27, 57, 15, 9, 12, 3, 3, +]; + +pub const KF_Y_MODE_PROBS: [u8; 900] = [ + 137, 30, 42, 148, 151, 207, 70, 52, 91, 92, 45, 102, 136, 116, 180, 74, 90, 100, 73, 32, 19, 187, 222, 215, 46, 34, 100, 91, 30, 32, 116, 121, 186, 93, 86, + 94, 72, 35, 36, 149, 68, 206, 68, 63, 105, 73, 31, 28, 138, 57, 124, 55, 122, 151, 67, 23, 21, 140, 126, 197, 40, 37, 171, 86, 27, 28, 128, 154, 212, 45, + 43, 53, 74, 32, 27, 107, 86, 160, 63, 134, 102, 59, 67, 44, 140, 161, 202, 78, 67, 119, 63, 36, 126, 146, 123, 158, 60, 90, 96, 43, 46, 168, 134, 107, 128, + 69, 142, 92, 44, 29, 68, 159, 201, 177, 50, 57, 77, 58, 38, 76, 114, 97, 172, 78, 133, 92, 46, 41, 76, 140, 63, 184, 69, 112, 57, 38, 32, 85, 140, 46, 112, + 54, 151, 133, 39, 27, 61, 131, 110, 175, 44, 75, 136, 52, 30, 74, 113, 130, 175, 51, 64, 58, 47, 35, 80, 100, 74, 143, 64, 163, 74, 36, 61, 116, 114, 128, + 162, 80, 125, 82, 82, 26, 26, 171, 208, 204, 44, 32, 105, 55, 44, 68, 166, 179, 192, 57, 57, 108, 42, 26, 11, 199, 241, 228, 23, 15, 85, 68, 42, 19, 131, + 160, 199, 55, 52, 83, 58, 50, 25, 139, 115, 232, 39, 52, 118, 50, 35, 33, 153, 104, 162, 64, 59, 131, 44, 24, 16, 150, 177, 202, 33, 19, 156, 55, 27, 12, + 153, 203, 218, 26, 27, 49, 53, 49, 21, 110, 116, 168, 59, 80, 76, 38, 72, 19, 168, 203, 212, 50, 50, 107, 103, 26, 36, 129, 132, 201, 83, 80, 93, 59, 38, + 83, 112, 103, 162, 98, 136, 90, 62, 30, 23, 158, 200, 207, 59, 57, 50, 67, 30, 29, 84, 86, 191, 102, 91, 59, 60, 32, 33, 112, 71, 220, 64, 89, 104, 53, 26, + 34, 130, 56, 149, 84, 120, 103, 53, 21, 23, 133, 109, 210, 56, 77, 172, 77, 19, 29, 112, 142, 228, 55, 66, 36, 61, 29, 29, 93, 97, 165, 83, 175, 162, 47, + 47, 43, 114, 137, 181, 100, 99, 95, 69, 23, 29, 128, 83, 199, 46, 44, 101, 53, 40, 55, 139, 69, 183, 61, 80, 110, 40, 29, 19, 161, 180, 207, 43, 24, 91, + 60, 34, 19, 105, 61, 198, 53, 64, 89, 52, 31, 22, 158, 40, 209, 58, 62, 89, 44, 31, 29, 147, 46, 158, 56, 102, 198, 35, 19, 12, 135, 87, 209, 41, 45, 167, + 55, 25, 21, 118, 95, 215, 38, 39, 66, 51, 38, 25, 113, 58, 164, 70, 93, 97, 47, 54, 34, 146, 108, 203, 72, 103, 151, 64, 19, 37, 156, 66, 138, 49, 95, 133, + 46, 27, 80, 150, 55, 124, 55, 121, 135, 36, 23, 27, 165, 149, 166, 54, 64, 118, 53, 21, 36, 131, 63, 163, 60, 109, 81, 40, 26, 35, 154, 40, 185, 51, 97, + 123, 35, 19, 34, 179, 19, 97, 48, 129, 124, 36, 20, 26, 136, 62, 164, 33, 77, 154, 45, 18, 32, 130, 90, 157, 40, 79, 91, 45, 26, 28, 129, 45, 129, 49, 147, + 123, 38, 44, 51, 136, 74, 162, 57, 97, 121, 75, 17, 22, 136, 138, 185, 32, 34, 166, 56, 39, 58, 133, 117, 173, 48, 53, 187, 35, 21, 12, 161, 212, 207, 20, + 23, 145, 56, 29, 19, 117, 109, 181, 55, 68, 112, 47, 29, 17, 153, 64, 220, 59, 51, 114, 46, 16, 24, 136, 76, 147, 41, 64, 172, 34, 17, 11, 108, 152, 187, + 13, 15, 209, 51, 24, 14, 115, 133, 209, 32, 26, 104, 55, 30, 18, 122, 79, 179, 44, 88, 116, 37, 49, 25, 129, 168, 164, 41, 54, 148, 82, 22, 32, 127, 143, + 213, 39, 41, 70, 62, 44, 61, 123, 105, 189, 48, 57, 64, 47, 25, 17, 175, 222, 220, 24, 30, 86, 68, 36, 17, 106, 102, 206, 59, 74, 74, 57, 39, 23, 151, 68, + 216, 55, 63, 58, 49, 30, 35, 141, 70, 168, 82, 40, 115, 51, 25, 15, 136, 129, 202, 38, 35, 139, 68, 26, 16, 111, 141, 215, 29, 28, 28, 59, 39, 19, 114, 75, + 180, 77, 104, 42, 40, 61, 26, 126, 152, 206, 61, 59, 93, 78, 23, 39, 111, 117, 170, 74, 124, 94, 48, 34, 86, 101, 92, 146, 78, 179, 134, 47, 22, 24, 138, + 187, 178, 68, 69, 59, 56, 25, 33, 105, 112, 187, 95, 177, 129, 48, 31, 27, 114, 63, 183, 82, 116, 56, 43, 28, 37, 121, 63, 123, 61, 192, 169, 42, 17, 24, + 109, 97, 177, 56, 76, 122, 58, 18, 28, 105, 139, 182, 70, 92, 63, 46, 23, 32, 74, 86, 150, 67, 183, 88, 36, 38, 48, 92, 122, 165, 88, 137, 91, 65, 70, 60, + 155, 159, 199, 61, 60, 81, 44, 78, 115, 132, 119, 173, 71, 112, 93, 39, 38, 21, 184, 227, 206, 42, 32, 64, 58, 47, 36, 124, 137, 193, 80, 82, 78, 49, 50, + 35, 144, 95, 205, 63, 78, 59, 41, 53, 52, 148, 71, 142, 65, 128, 51, 40, 36, 28, 143, 143, 202, 40, 55, 137, 52, 34, 29, 129, 183, 227, 42, 35, 43, 42, 44, + 44, 104, 105, 164, 64, 130, 80, 43, 81, 53, 140, 169, 204, 68, 84, 72, +]; + +pub const KF_UV_MODE_PROBS: [u8; 90] = [ + 144, 11, 54, 157, 195, 130, 46, 58, 108, 118, 15, 123, 148, 131, 101, 44, 93, 131, 113, 12, 23, 188, 226, 142, 26, 32, 125, 120, 11, 50, 123, 163, 135, 64, + 77, 103, 113, 9, 36, 155, 111, 157, 32, 44, 161, 116, 9, 55, 176, 76, 96, 37, 61, 149, 115, 9, 28, 141, 161, 167, 21, 25, 193, 120, 12, 32, 145, 195, 142, + 32, 38, 86, 116, 12, 64, 120, 140, 125, 49, 115, 121, 102, 19, 66, 162, 182, 122, 35, 59, 128, +]; + +pub const DEFAULT_SKIP_PROB: [u8; 3] = [192, 128, 64]; + +pub const DEFAULT_COEF_PROBS: [u8; 432] = [ + 195, 29, 183, 84, 49, 136, 8, 42, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 107, 169, 35, 99, 159, 17, 82, 140, 8, 66, 114, 2, 44, 76, 1, 19, 32, 40, 132, 201, + 29, 114, 187, 13, 91, 157, 7, 75, 127, 3, 58, 95, 1, 28, 47, 69, 142, 221, 42, 122, 201, 15, 91, 159, 6, 67, 121, 1, 42, 77, 1, 17, 31, 102, 148, 228, 67, + 117, 204, 17, 82, 154, 6, 59, 114, 2, 39, 75, 1, 15, 29, 156, 57, 233, 119, 57, 212, 58, 48, 163, 29, 40, 124, 12, 30, 81, 3, 12, 31, 191, 107, 226, 124, + 117, 204, 25, 99, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 148, 210, 37, 126, 194, 8, 93, 157, 2, 68, 118, 1, 39, 69, 1, 17, 33, 41, 151, 213, 27, 123, 193, 3, + 82, 144, 1, 58, 105, 1, 32, 60, 1, 13, 26, 59, 159, 220, 23, 126, 198, 4, 88, 151, 1, 66, 114, 1, 38, 71, 1, 18, 34, 114, 136, 232, 51, 114, 207, 11, 83, + 155, 3, 56, 105, 1, 33, 65, 1, 17, 34, 149, 65, 234, 121, 57, 215, 61, 49, 166, 28, 36, 114, 12, 25, 76, 3, 16, 42, 214, 49, 220, 132, 63, 188, 42, 65, + 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 137, 221, 104, 131, 216, 49, 111, 192, 21, 87, 155, 2, 49, 87, 1, 16, 28, 89, 163, 230, 90, 137, 220, 29, 100, 183, 10, + 70, 135, 2, 42, 81, 1, 17, 33, 108, 167, 237, 55, 133, 222, 15, 97, 179, 4, 72, 135, 1, 45, 85, 1, 19, 38, 124, 146, 240, 66, 124, 224, 17, 88, 175, 4, 58, + 122, 1, 36, 75, 1, 18, 37, 141, 79, 241, 126, 70, 227, 66, 58, 182, 30, 44, 136, 12, 34, 96, 2, 20, 47, 229, 99, 249, 143, 111, 235, 46, 109, 192, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 82, 158, 236, 94, 146, 224, 25, 117, 191, 9, 87, 149, 3, 56, 99, 1, 33, 57, 83, 167, 237, 68, 145, 222, 10, 103, 177, 2, 72, 131, 1, 41, + 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, + 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, +]; diff --git a/docs/architecture.md b/docs/architecture.md index c3d869c0..096c75bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,18 +16,21 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `vp9enc` | VP9 intra-frame encoder (profile 0, 8-bit 4:2:0; lossless at quality 100) for WebM export, from the VP9 bitstream specification | | L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | +| L1 | `psd` | Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit; RGB/CMYK/Gray/Lab) and a minimal writer, from Adobe's published format specification | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `svg` | SVG import (W3C SVG 1.1/2 static subset: shapes, paths, transforms, `use`, CSS, gradients) and rasterisation at any scale | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | -| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | +| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, Photoshop and SVG stills, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | -| L3 | `export` | Render queue encoding: H.264, ProRes, PNG/JPEG/TIFF/EXR sequences, GIF, audio | +| L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | @@ -76,6 +79,14 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +A persistent **disk cache** (`render::disk_cache`, Settings ▸ Media & Disk Cache) backs both +the layer cache and the viewer's RAM preview: layer buffers that were slow to render and every +preview frame are written (LZ4, atomic rename, checksummed) under 128-bit content keys — the +comp's content hash (the comp, what it uses, footage file size and time) plus frame, scale and +view for frames; the layer key salted with the footage fingerprint for layers — so entries +survive restarts and never go stale. LRU eviction keeps it under the size limit; Edit ▸ Purge +empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. + **Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through `EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* diff --git a/docs/parity.md b/docs/parity.md index d259fec2..5d291ca7 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -75,7 +75,7 @@ library groundwork for mask tracking and Warp Stabilizer. What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of -the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, +the new tracking library, a JavaScript scripting object model, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and wasm threads. @@ -84,9 +84,9 @@ wasm threads. | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 85% | 4.2 | multiple output modules, pre-render, WebM, audio-only | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | -| Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | +| Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | 76% | 3.3 | a JavaScript scripting object model | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | | Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | @@ -98,7 +98,7 @@ wasm threads. | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | -| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | +| Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | @@ -125,7 +125,7 @@ Mocha and Cineware are third-party and not counted. 10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. 11. Auto-save, crash recovery, recent projects (in progress). 12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). -13. Lottie (in progress), WebM, SVG and PSD import. +13. ~~Lottie, WebM, SVG and PSD import~~ (landed; a disk cache too). 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7e7616a5..d822a1b9 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,6 +23,7 @@ const LAYERS: &[(&str, u8)] = &[ ("time", 0), ("geom", 0), ("color", 0), + ("vp9enc", 0), ("testkit", 0), ("opusenc", 0), ("raster", 1), From cba8d7e737db9438c9b0e46fd882875df5e7946a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:02:39 -0400 Subject: [PATCH 5/6] M12.6: 3D Camera Tracker effect + UI Effect > Perspective > 3D Camera Tracker (Shot Type, Horizontal Angle of View, Show/Render Track Points, Track Point Size, Target Size, Advanced: Solve Method, Method Used, Average Error, Detailed Analysis, Auto-delete Points Across Time, Hide Warning Banner); tracks and solve stored in the effect with keys (frame edits re-track, setting / deleted-point edits only re-solve). Background job (Analyzing in background (step 1 of 2) / Solving camera banner, cancel, one undo step). Commands: track.camera (replaces the stub), camera.analyze/cancel/solveStatus/points/selectPoints/ createFromSolve/create/setGroundPlane/deletePoints; the one-node 3D Tracker Camera keyed per frame and text / solids / nulls / shadow catcher + light on the target plane, Create Multiple. Viewer: depth-sized coloured targets, hover target plane, click / shift / marquee selection, AE right-click menu, Delete; Effect Controls Analyze / Cancel / Create Camera; Tracker panel Track Camera. Solver: adaptive noise thresholds, denser keyframes, quick tripod check, faster Cholesky and parallel Schur complement. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + ROADMAP.md | 4 +- crates/effects/src/camera_tracker.rs | 291 ++++++++ crates/effects/src/lib.rs | 4 + crates/engine/src/camera_track.rs | 640 ++++++++++++++++++ crates/engine/src/commands/camera_cmds.rs | 574 ++++++++++++++++ crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 22 + crates/engine/src/tests_camera_track.rs | 307 +++++++++ crates/engine/src/tests_stubs.rs | 2 +- crates/track/Cargo.toml | 1 + crates/track/src/camtrack/bundle.rs | 90 ++- crates/track/src/camtrack/features.rs | 2 +- crates/track/src/camtrack/linalg.rs | 30 +- crates/track/src/camtrack/mod.rs | 8 +- crates/track/src/camtrack/solver.rs | 83 ++- crates/track/src/camtrack/tests.rs | 78 +++ crates/ui-egui/src/lib.rs | 5 + .../ui-egui/src/panels/camera_tracker_ui.rs | 438 ++++++++++++ crates/ui-egui/src/panels/effect_controls.rs | 2 + crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/tracker.rs | 7 +- crates/ui-egui/src/panels/viewer.rs | 3 + crates/ui-egui/tests/ui_camera_tracker.rs | 175 +++++ docs/agents.md | 28 + docs/architecture.md | 16 + docs/parity.md | 10 +- 28 files changed, 2771 insertions(+), 54 deletions(-) create mode 100644 crates/effects/src/camera_tracker.rs create mode 100644 crates/engine/src/camera_track.rs create mode 100644 crates/engine/src/commands/camera_cmds.rs create mode 100644 crates/engine/src/tests_camera_track.rs create mode 100644 crates/ui-egui/src/panels/camera_tracker_ui.rs create mode 100644 crates/ui-egui/tests/ui_camera_tracker.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..ce792ecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1420,6 +1420,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979c..7780c164 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,10 +28,10 @@ once. Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): - **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. + professional daily-use features, 21% of the long tail); 258 of 298 effects, 16 of them on the GPU. - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: Roto Brush, Advanced 3D, a scripting object model (the 3D Camera Tracker landed). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs new file mode 100644 index 00000000..dd97b585 --- /dev/null +++ b/crates/effects/src/camera_tracker.rs @@ -0,0 +1,291 @@ +//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera). +//! +//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks +//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves +//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the +//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made +//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the +//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing +//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. +//! +//! The effect passes its input through; with **Render Track Points** on it draws the solved +//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! commands live in the engine and the UI. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.perspective.cameratracker"; + +/// Hidden parameters (see the module docs). +pub const TRACKS: &str = "tracks"; +pub const TRACKS_KEY: &str = "tracksKey"; +pub const SOLVE: &str = "solve"; +pub const SOLVE_KEY: &str = "solveKey"; +/// Deleted track point ids (JSON array). +pub const DELETED: &str = "deleted"; +/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none). +pub const CAMERA: &str = "camera"; +pub const METHOD_USED: &str = "advanced/methodUsed"; +pub const AVERAGE_ERROR: &str = "advanced/averageError"; + +pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"]; +pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"]; +pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "3D Camera Tracker", + category: "Perspective", + params: vec![ + p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)), + p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)), + p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)), + p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox), + p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)), + p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)), + p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)), + p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), + p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden), + p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden), + p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Deleted point ids of an instance (stored as a JSON array of numbers). +pub fn deleted(params: &Params) -> Vec { + s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect() +} + +/// The solve settings of an instance. +pub fn settings(params: &Params) -> SolveSettings { + let mut del = deleted(params); + del.sort_unstable(); + del.dedup(); + SolveSettings { + shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(), + hfov: f(params, "horizontalAngleOfView"), + method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), + deleted: del, + } +} + +/// What a solve depends on besides the tracks. +pub fn solve_key(st: &SolveSettings) -> String { + let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; + format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +type Cache = Mutex>>; + +fn cached(c: &'static OnceLock>, json: &str, parse: impl Fn(&str) -> Option) -> Option> { + let key = fnv(json.as_bytes()); + let c = c.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(a.clone()); + } + let a = Arc::new(parse(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some(a) +} + +/// The stored feature tracks of an instance (`None` when not analysed). +pub fn tracks(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, TRACKS); + if j.is_empty() { + return None; + } + cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty()) +} + +/// The stored camera solve of an instance (`None` when not solved). +pub fn solve(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, SOLVE); + if j.is_empty() { + return None; + } + cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty()) +} + +/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the +/// median depth of the frame's points. +pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { + (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) +} + +/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + let Some(cam) = solve.frames.get(k) else { return vec![] }; + solve + .visible(k) + .filter_map(|p| { + let c = cam.to_cam(p.pos); + cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + }) + .collect() +} + +/// Median of the depths. +pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { + let mut z: Vec = v.iter().map(|x| x.2).collect(); + if z.is_empty() { + return 1.0; + } + z.sort_by(f64::total_cmp); + z[z.len() / 2] +} + +fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { + if !b(ctx.params, "renderTrackPoints") { + return buf; + } + let Some(sol) = solve(ctx.params) else { return buf }; + if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { + return buf; + } + let Some(k) = sol.frame_at(ctx.time) else { return buf }; + let pts = projected(&sol, k); + let med = median_depth(&pts); + let size = f(ctx.params, "trackPointSize"); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for (id, q, z) in pts { + let c = point_color(id); + let r = target_radius(size, z, med) * sc; + let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]]; + ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]); + disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]); + } + buf +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) { + let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = cov(d).clamp(0.0, 1.0) as f32; + if a > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, a); + } + } + } +} + +fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + cover(img, c, r, |d| r + 0.5 - d, col); +} + +fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) { + cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col); +} + +/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of +/// radius `size / 2` in the plane through `center` with normal `normal`. +pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec { + let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + let u = linalg::normalize(linalg::cross(normal, a)); + let v = linalg::cross(normal, u); + (0..n) + .map(|i| { + let t = i as f64 / n as f64 * std::f64::consts::TAU; + linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin()))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint}; + + fn params_with(solve: &CameraSolve, render: bool) -> Params { + let mut p = Params::default(); + p.values.insert(SOLVE.into(), Value::Str(solve.to_json())); + p.values.insert("renderTrackPoints".into(), Value::Bool(render)); + p.values.insert("trackPointSize".into(), Value::Scalar(100.0)); + p + } + + #[test] + fn render_track_points_draws_targets_and_passes_through_otherwise() { + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }], + ..Default::default() + }; + let img = Image::new(64, 48); + for on in [false, true] { + let params = params_with(&sol, on); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let px = out.img.data[out.img.idx(32, 24)]; + assert_eq!(px[3] > 0.5, on); + } + let st = settings(&Params::default()); + assert_eq!(st.shot, ShotType::FixedAngle); + assert!(solve_key(&st).contains("FixedAngle")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a..9c2aefb5 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,6 +8,7 @@ pub mod audio_fx; mod blur2; +pub mod camera_tracker; mod channel; mod channel2; mod color2; @@ -354,6 +355,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(camera_tracker::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -494,6 +496,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.pixelmotionblur", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Render Track Points draws each frame's solved points. + camera_tracker::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs new file mode 100644 index 00000000..1b984aac --- /dev/null +++ b/crates/engine/src/camera_track.rs @@ -0,0 +1,640 @@ +//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping +//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`). +//! +//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame +//! between the layer's In and Out points on a background thread and tracks features through +//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving +//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method, +//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into +//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear +//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects. +//! +//! Composition space: the solve's canonical frame maps to comp world by +//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's +//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created +//! layers sit on the selected points. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; +use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::warp::{analysis_times, signature}; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CameraProgress { + /// 1 = tracking features, 2 = solving the camera. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, + /// Re-solving stored tracks (no tracking step). + pub solve_only: bool, +} + +impl CameraProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Solving camera".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +struct CamResult { + /// New tracks (None for a re-solve). + tracks: Option, + solve: Result, +} + +#[derive(Default)] +pub struct CameraShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) camera analysis. +pub struct CameraJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the tracks are made from, and the solve settings key. + key: String, + solve_key: String, +} + +impl CameraJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> CameraProgress { + lock(&self.shared.state).clone() + } +} + +struct CamWork { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + size: [f64; 2], + detailed: bool, + times: Vec, + /// Stored tracks to re-solve (skips step 1). + tracks: Option>, + settings: SolveSettings, +} + +pub(crate) fn is_tracker(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.prop(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis. +fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String { + let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool); + format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" }) +} + +/// Parameters of an instance from static values (the tracker's settings are not animated). +pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params { + effectcraft_effects::flatten_params(g, &mut |p| p.value.clone()) +} + +fn clear(g: &mut PropGroup, all: bool) { + let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED]; + if all { + ids.extend([TRACKS, TRACKS_KEY]); + } + for m in ids { + if let Some(p) = g.prop_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + if let Some(p) = g.prop_mut(AVERAGE_ERROR) { + p.value = Value::Scalar(0.0); + } +} + +/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their +/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis: +/// cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clears = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_tracker) { + continue; + } + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_tracker(g)) { + let tracked = !str_of(g, TRACKS).is_empty(); + if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) { + clears.push((*cid, l.id, g.uid, true)); + todo.push((*cid, l.id, g.uid)); + } else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) { + clears.push((*cid, l.id, g.uid, false)); + todo.push((*cid, l.id, g.uid)); + } else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, all) in clears { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + clear(g, all); + } + } + todo +} + +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: CamWork, shared: &CameraShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let cancelled = || { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let (tracks, fresh) = match w.tracks.clone() { + Some(t) => (t, false), + None => { + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + (Arc::new(an.finish(lt0, fd)), true) + } + }; + lock(&shared.state).step = 2; + let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0); + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + *lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve }); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +impl Session { + pub fn is_camera_analyzing(&self) -> bool { + self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn camera_progress(&self) -> Option { + self.camera_job.as_ref().map(|j| j.progress()) + } + + /// The tracker being analysed: (comp, layer, effect uid). + pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_camera(&mut self) -> bool { + match &self.camera_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when + /// they are still valid). `wait`: block until done (wasm32 always does). Returns the number + /// of frames. + pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_camera_analyzing() { + return Err("a 3D Camera Tracker analysis is already running".into()); + } + self.poll_camera_job(); + self.camera_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?; + if l.switches.adjustment { + return Err("3D Camera Tracker needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let params = static_params(g); + let key = tracks_key(&self.project, comp, l, g); + let settings = ct::settings(¶ms); + let solve_key = ct::solve_key(&settings); + let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key); + let n = times.len(); + let work = CamWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()), + times, + tracks: tracks.clone(), + settings, + }; + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(CameraShared::default()); + { + let mut st = lock(&shared.state); + st.solve_only = tracks.is_some(); + st.step = if tracks.is_some() { 2 } else { 1 }; + st.total = n as u64; + } + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_camera_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_camera_job(&mut self) -> bool { + let Some(job) = &self.camera_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.camera_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(res), None) if !st.cancelled => { + let (cid, lid, uid) = (job.comp, job.layer, job.effect); + let (key, skey) = (job.key.clone(), job.solve_key.clone()); + let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json); + let (solve_json, method, avg, msg, failed) = match &res.solve { + Ok(s) => ( + s.to_json(), + s.method_used.label().to_string(), + s.average_error, + format!( + "3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)", + s.frames.len(), + s.points.len(), + s.average_error, + st.elapsed + ), + false, + ), + Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true), + }; + let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?; + let mut set = |m: &str, v: Value| { + if let Some(pr) = g.prop_mut(m) { + pr.value = v; + } + }; + if let Some(t) = &tracks_json { + set(TRACKS, Value::Str(t.clone())); + set(TRACKS_KEY, Value::Str(key.clone())); + } + set(SOLVE, Value::Str(solve_json.clone())); + set(SOLVE_KEY, Value::Str(skey.clone())); + set(METHOD_USED, Value::Str(method.clone())); + set(AVERAGE_ERROR, Value::Scalar(avg)); + Ok(()) + }); + let (msg, failed) = match r { + Ok(()) => (msg, failed), + Err(e) => (format!("3D Camera Tracker: {e}"), true), + }; + self.events.push(Event::Toast { message: msg, error: failed }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("3D Camera Tracker analysis failed: {e}"), + None => "3D Camera Tracker analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// (re-)analyses in the background, like After Effects. + pub fn poll_camera(&mut self, auto: bool) -> bool { + let changed = self.poll_camera_job(); + if auto && self.camera_job.is_none() { + while let Some((c, l, e)) = self.camera_pending.first().copied() { + self.camera_pending.remove(0); + if self.start_camera(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} + +// ---------------------------------------------------------------- solve → composition + +/// A tracker's solve placed in its composition. +pub struct Placed { + pub solve: Arc, + /// Canonical → comp world. + pub world: Similarity, + /// Comp pixels per layer pixel. + pub scale: f64, + /// Comp times of the analysed frames. + pub times: Vec, + pub comp_size: [f64; 2], +} + +impl Placed { + /// Comp-world position of a canonical point. + pub fn to_world(&self, x: V3) -> V3 { + self.world.apply(x) + } + /// Canonical position of a comp-world point. + pub fn from_world(&self, x: V3) -> V3 { + let r = &self.world; + linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s) + } + /// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom). + pub fn camera(&self, k: usize) -> (V3, V3, f64) { + let c = &self.solve.frames[k]; + let m = linalg::mm(&self.world.r, &linalg::mt(&c.r())); + (self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale) + } + /// Analysed frame at comp time `t`. + pub fn frame_at(&self, t: Tick) -> usize { + self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1)) + } +} + +/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point). +fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 { + let Some(comp) = project.comp(cid) else { return 1.0 }; + let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO)); + let m = ctx.world_matrix(layer).0; + let s = m[0][0].hypot(m[1][0]); + if s.is_finite() && s > 1e-9 { s } else { 1.0 } +} + +/// The solve of tracker `uid` on `lid`, placed in the comp. +pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option { + let comp = project.comp(cid)?; + let layer = comp.layer(lid)?; + let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?; + let solve = ct::solve(&static_params(g))?; + let scale = layer_scale(project, cid, layer); + let k0 = solve.first_solved()?; + let comp_size = [comp.width as f64, comp.height as f64]; + let world = solve.world(comp_size, solve.frames[k0].focal * scale); + let mut times = analysis_times(comp, layer); + times.truncate(solve.frames.len()); + Some(Placed { solve, world, scale, times, comp_size }) +} + +/// What to create from a target (`camera.createFromSolve`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CreateKind { + Text, + Solid, + Null, + ShadowCatcher, + /// Only the camera. + Camera, +} + +impl CreateKind { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "text" => Some(CreateKind::Text), + "solid" => Some(CreateKind::Solid), + "null" => Some(CreateKind::Null), + "shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher), + "camera" => Some(CreateKind::Camera), + _ => None, + } + } +} + +/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera), +/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera. +pub fn plane_orientation(normal: V3, right: V3) -> V3 { + let z = linalg::scale(normal, -1.0); + let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z))); + if linalg::norm(x) < 1e-9 { + x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z))); + } + let x = linalg::normalize(x); + let y = linalg::cross(z, x); + let m = linalg::from_cols(x, y, z); + norm_angles(linalg::euler_xyz(&m)) +} + +fn norm_angles(a: V3) -> V3 { + a.map(|v| { + let r = v.rem_euclid(360.0); + if (r - 360.0).abs() < 1e-9 { 0.0 } else { r } + }) +} + +/// Key `path` on `l` at every `(time, value)`. +pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) { + if let Some(pr) = l.props.prop_mut(path) { + pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect(); + if let Some((_, v)) = keys.first() { + pr.value = v.clone(); + } + } +} + +/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame; +/// Zoom keyed for Variable Zoom). +pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) { + l.auto_orient = AutoOrient::Off; + let n = pl.times.len().min(pl.solve.frames.len()); + let mut pos = Vec::with_capacity(n); + let mut ori = Vec::with_capacity(n); + let mut zoom = Vec::with_capacity(n); + let mut prev: Option = None; + for k in 0..n { + let (p, o, z) = pl.camera(k); + // Unwrap the angles so per-frame keys interpolate the short way. + let o = match prev { + None => norm_angles(o), + Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0), + }; + prev = Some(o); + let t = l.layer_time(pl.times[k]); + pos.push((t, Value::Vec3(p))); + ori.push((t, Value::Vec3(o))); + zoom.push((t, Value::Scalar(z))); + } + key_all(l, "transform/position", &pos); + key_all(l, "transform/orientation", &ori); + let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal); + if variable { + key_all(l, "cameraOptions/zoom", &zoom); + } else if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys.clear(); + pr.value = z.clone(); + } + if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance") + { + pr.value = z.clone(); + } + // A one-node camera has no Point of Interest. + if let Some(tr) = l.props.sub_mut("transform") { + tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi")); + } + for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + if let Some(pr) = l.props.prop_mut(rot) { + pr.keys.clear(); + pr.value = Value::Scalar(0.0); + } + } +} + +/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest +/// solved neighbours (in 3D) complete the plane. +pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option { + let mut use_ids: Vec = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect(); + if use_ids.is_empty() { + return None; + } + if use_ids.len() < 3 { + let c = solve.point(use_ids[0])?.pos; + let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect(); + others.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) { + use_ids.push(id); + } + let mut t = solve.target(&use_ids, k)?; + // The target sits on the chosen point(s). + let pts: Vec = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect(); + t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64); + return Some(t); + } + solve.target(&use_ids, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plane_orientation_faces_the_normal() { + let n = linalg::normalize([0.2, -0.9, -0.3]); + let o = plane_orientation(n, [1.0, 0.0, 0.0]); + let m = linalg::from_euler_xyz(o); + // Layer −z (its front) is the normal. + let front = linalg::mv(&m, [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, n)) < 1e-9); + } +} diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs new file mode 100644 index 00000000..5d454aed --- /dev/null +++ b/crates/engine/src/commands/camera_cmds.rs @@ -0,0 +1,574 @@ +//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze / +//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu +//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera, +//! Create Multiple …, Delete Selected Points) and the status / points queries. + +use effectcraft_color::Label; +use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE}; +use effectcraft_keyframe::{Justify, TextDoc, Value as KV}; +use effectcraft_project::build; +use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid}; +use effectcraft_track::camtrack::linalg::{self, V3}; +use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p}; +use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.camera", + "Track Camera", + ["Animation"], + None, + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + can_track, + track_camera + ), + cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze), + cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))), + query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status), + query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points), + cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points), + cmd!( + "camera.createFromSolve", + "Create from Camera Solve", + [], + None, + "{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}", + has_comp, + create_from_solve + ), + cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), + cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), + ] +} + +fn with_kind(p: &Value, kind: &str) -> Value { + let mut q = p.clone(); + if !q.is_object() { + q = json!({}); + } + q["kind"] = json!(kind); + q +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()), + Some(_) => Err("Track Camera needs a footage or composition layer".into()), + None => Err("select a layer to track".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) } +} + +/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based +/// index), else a selected one, else the first in the comp. +fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_tracker(g))), + }; + if let Some(g) = g.filter(|g| is_tracker(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)")) +} + +fn shot_p(p: &Value) -> Result> { + match p.get("shotType") { + None => Ok(None), + Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")), + Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")), + Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")), + } +} + +fn method_p(p: &Value) -> Result> { + match p.get("solveMethod") { + None => Ok(None), + Some(Value::Number(n)) => { + SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3")) + } + Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + } +} + +/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer +/// (reusing one it already has), set the given options and start the analysis. +fn track_camera(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.camera")?; + let shot = shot_p(p)?; + let method = method_p(p)?; + let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView")); + if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) { + return Err(bad("track.camera", "aov: 1..170 degrees")); + } + let detailed = b_p(p, "detailed"); + let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); + let uid = match existing { + Some(u) => u, + None => { + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?; + r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? + } + }; + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + s.edit("3D Camera Tracker Settings", None, |proj, _| { + let g = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + let mut set = |m: &str, v: KV| { + if let Some(pr) = g.prop_mut(m) { + pr.keys.clear(); + pr.value = v; + } + }; + if let Some(sh) = shot { + set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32)); + } + if let Some(a) = aov { + set("horizontalAngleOfView", KV::Scalar(a)); + if shot.is_none() { + set("shotType", KV::Enum(2)); + } + } + if let Some(m) = method { + set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32)); + } + if let Some(d) = detailed { + set("advanced/detailedAnalysis", KV::Bool(d)); + } + Ok(()) + })?; + } + // Select the effect so the viewer shows its track points. + s.state.selected_props = vec![(lid, uid)]; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_camera(false); + let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid))); + let tracks = ct::tracks(¶ms); + out["analyzed"] = json!(tracks.is_some()); + if let Some(t) = &tracks { + out["tracks"] = json!(t.tracks.len()); + out["frames"] = json!(t.frames); + } + let settings = ct::settings(¶ms); + out["shotType"] = json!(settings.shot.label()); + out["solveMethod"] = json!(settings.method.label()); + out["deleted"] = json!(settings.deleted.len()); + let solve = ct::solve(¶ms); + out["solved"] = json!(solve.is_some()); + if let Some(sv) = &solve { + let f = sv.focal(); + out["methodUsed"] = json!(sv.method_used.label()); + out["averageError"] = json!(sv.average_error); + out["points"] = json!(sv.points.len()); + out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count()); + out["focalLength"] = json!(f); + out["horizontalAngleOfView"] = json!(sv.hfov(f)); + out["groundPlane"] = json!(sv.ground.is_some()); + if let Some(pl) = placed(&s.project, cid, lid, uid) { + let k = pl.frame_at(s.time_of(cid)); + let (pos, ori, zoom) = pl.camera(k); + out["frame"] = json!(k); + out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom}); + out["visible"] = json!(sv.visible(k).count()); + } + } + let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0); + out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null }; + out["selected"] = json!(s.state.camera_points); + Ok(out) +} + +/// The visible solved points at the current (or given) time, with their comp positions. +fn points(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.points")?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?; + let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let k = pl.frame_at(t); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); + let m = ctx.world_matrix(layer); + let list: Vec = ct::projected(&pl.solve, k) + .into_iter() + .map(|(id, q, z)| { + let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0)); + let sp = pl.solve.point(id).copied().unwrap_or_default(); + let col = point_color(id); + json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)}) + }) + .collect(); + Ok(json!({"frame": k, "points": list})) +} + +fn ids_p(s: &Session, p: &Value) -> Vec { + match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(), + _ => s.state.camera_points.clone(), + } +} + +fn select_points(s: &mut Session, p: &Value) -> Result { + let ids: Vec = match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + _ => vec![], + }; + if b_p(p, "toggle").unwrap_or(false) { + for id in ids { + match s.state.camera_points.iter().position(|x| *x == id) { + Some(i) => { + s.state.camera_points.remove(i); + } + None => s.state.camera_points.push(id), + } + } + } else if b_p(p, "add").unwrap_or(false) { + for id in ids { + if !s.state.camera_points.contains(&id) { + s.state.camera_points.push(id); + } + } + } else { + s.state.camera_points = ids; + } + s.bump(); + Ok(json!({"selected": s.state.camera_points})) +} + +/// The target of a create / ground command in canonical solve coordinates. +fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result { + if let Some(t) = p.get("target") { + let v3 = |k: &str| -> Option { + let a = t.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0])); + let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0); + let nc = linalg::normalize(linalg::mtv(&pl.world.r, n)); + return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s }); + } + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points` / `target`)")); + } + target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane")) +} + +fn next_name(c: &effectcraft_project::Comp, base: &str) -> String { + let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1); + format!("{base} {n}") +} + +fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) { + l.switches.three_d = true; + l.in_point = src.in_point; + l.out_point = src.out_point; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.value = KV::Vec3(pos); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.value = KV::Vec3(ori); + } +} + +/// Insert `l` above the tracked layer. +fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0); + let id = l.id; + comp.layers.insert(at, l); + Ok(id) +} + +fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 })); + build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None) +} + +/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker +/// Camera when the tracker has none yet — one undo step. +fn create_from_solve(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.createFromSolve"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?; + let k = pl.frame_at(s.time_of(cid)); + let multiple = b_p(p, "multiple").unwrap_or(false); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let size_pct = effectcraft_effects::warp_stab::param(¶ms, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0; + let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera)); + // Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane. + let targets: Vec = if kind == CreateKind::Camera { + vec![] + } else { + let t = target_p(s, p, &pl, k, cmd)?; + if multiple { + let ids = ids_p(s, p); + ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect() + } else { + vec![t] + } + }; + // Camera right vector on this frame (world), to align created layers with the view. + let cam = &pl.solve.frames[k]; + let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0])); + let label = match (kind, multiple) { + (CreateKind::Text, false) => "Create Text and Camera", + (CreateKind::Solid, false) => "Create Solid and Camera", + (CreateKind::Null, false) => "Create Null and Camera", + (CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light", + (CreateKind::Text, true) => "Create Multiple Text Layers and Camera", + (CreateKind::Solid, true) => "Create Multiple Solids and Camera", + (CreateKind::Null, true) => "Create Multiple Nulls and Camera", + (CreateKind::Camera, _) => "Create Camera", + }; + let (camera, created) = s.edit(label, None, |proj, st| { + let mut created = vec![]; + // The camera: reuse the tracker's, else create "3D Tracker Camera" at the top. + let camera = match existing_cam { + Some(id) if kind != CreateKind::Camera => LayerId(id), + Some(id) => { + let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?; + write_camera(l, &pl); + LayerId(id) + } + None => { + let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None); + l.in_point = src.in_point; + l.out_point = src.out_point; + write_camera(&mut l, &pl); + let id = l.id; + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l); + if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) { + pr.value = KV::Scalar(id.0 as f64); + } + id + } + }; + for t in &targets { + let center = pl.to_world(t.center); + let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal)); + let ori = plane_orientation(normal, right); + let size = (t.size * pl.world.s * size_pct).max(1.0); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + match kind { + CreateKind::Text => { + let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None); + let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() }; + if let Some(pr) = l.props.prop_mut("text/sourceText") { + pr.value = KV::Text(Box::new(doc)); + } + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Solid => { + let name = next_name(&cur, "Track Solid"); + let c = point_color((created.len() as u32).wrapping_add(7)); + let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c); + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Null => { + let name = next_name(&cur, "Track Null"); + let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("transform/scale") { + pr.value = KV::Vec3([size, size, size]); + } + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::ShadowCatcher => { + let name = next_name(&cur, "Shadow Catcher"); + let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") { + pr.value = KV::Enum(2); + } + if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") { + pr.value = KV::Enum(0); + } + let sc = created.len(); + created.push(insert_above(proj, cid, lid, l)?); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lname = next_name(&cur, "Light"); + let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None); + light.in_point = src.in_point; + light.out_point = src.out_point; + let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size))); + if let Some(pr) = light.props.prop_mut("transform/position") { + pr.value = KV::Vec3(lp); + } + if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") { + pr.value = KV::Bool(true); + } + let _ = sc; + created.push(insert_above(proj, cid, created[created.len() - 1], light)?); + } + CreateKind::Camera => {} + } + } + st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() }; + st.selected_props.clear(); + Ok((camera, created)) + })?; + let t0 = targets.first().map( + |t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}), + ); + Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::>(), "target": t0})) +} + +fn set_ground(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.setGroundPlane"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?; + let k = pl.frame_at(s.time_of(cid)); + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + let mut sv = (*pl.solve).clone(); + let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + g.origin = t.center; + sv.ground = Some(g); + let json = sv.to_json(); + s.edit("Set Ground Plane and Origin", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(SOLVE)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + Ok(json!({"origin": g.origin, "normal": g.normal})) +} + +/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the +/// same 3D feature), then re-solve. +fn delete_points(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.deletePoints"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let mut del = ct::deleted(¶ms); + let auto = effectcraft_effects::warp_stab::param(¶ms, "advanced/autoDeletePoints").is_some_and(KV::as_bool); + let mut add = ids.clone(); + if auto && let Some(sv) = ct::solve(¶ms) { + // Same feature tracked again at other times: solved within 1 % of its depth. + let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]); + for id in &ids { + let Some(pt) = sv.point(*id) else { continue }; + let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01; + for q in &sv.points { + let disjoint = q.last < pt.first || q.first > pt.last; + if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r { + add.push(q.id); + } + } + } + } + for id in add { + if !del.contains(&id) { + del.push(id); + } + } + del.sort_unstable(); + let json = serde_json::to_string(&del).unwrap_or_default(); + s.edit("Delete Track Points", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(DELETED)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + s.state.camera_points.retain(|x| !del.contains(x)); + // Re-solve now (After Effects re-solves after deleting points). + let wait = b_p(p, "wait").unwrap_or(false); + if !s.is_camera_analyzing() { + s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + } + Ok(json!({"deleted": del, "running": s.is_camera_analyzing()})) +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f..04034281 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,6 +4,7 @@ mod anim; pub mod anim_tools; mod animation; +mod camera_cmds; mod comp; mod comp_more; mod edit; @@ -129,6 +130,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(camera_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd7..74de3aec 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -59,7 +59,6 @@ pub fn specs() -> Vec { stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), - stub!("track.camera", "Track Camera", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488a..fbd2b391 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,7 @@ //! (compositions are `Arc`s, so untouched comps are shared). pub mod autosave; +pub mod camera_track; pub mod commands; pub mod config; pub mod demo; @@ -41,6 +42,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -170,6 +172,9 @@ pub struct EditorState { /// Tracker panel ▸ Method in mask mode (Track Mask). #[serde(default)] pub mask_track_method: mask_track::MaskMethod, + /// 3D Camera Tracker: the selected track points (ids) in the viewer. + #[serde(default)] + pub camera_points: Vec, /// Mask Interpolation panel options. #[serde(default)] pub mask_interp: commands::mask_interp::MaskInterpOptions, @@ -263,6 +268,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) 3D Camera Tracker analysis. + pub camera_job: Option, + /// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid). + pub camera_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -308,6 +317,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + camera_job: None, + camera_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -374,6 +385,12 @@ impl Session { self.warp_pending.push(w); } } + // So are 3D Camera Tracker tracks (and solves made with other settings). + for w in camera_track::invalidate(&before, &mut p) { + if !self.camera_pending.contains(&w) { + self.camera_pending.push(w); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -551,6 +568,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_camera(); + self.camera_job = None; + self.camera_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -574,6 +594,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_camera_track; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; diff --git a/crates/engine/src/tests_camera_track.rs b/crates/engine/src/tests_camera_track.rs new file mode 100644 index 00000000..5bd5539c --- /dev/null +++ b/crates/engine/src/tests_camera_track.rs @@ -0,0 +1,307 @@ +//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test: +//! textured planes (a floor, a back wall and a box face) seen through a known moving camera. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_geom::vec3; +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::camera::active_camera; +use effectcraft_track::camtrack::linalg::{self, M3, V3}; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup, texture}; + +const F: f64 = 260.0; + +/// The true camera on frame `f`: (world → camera rotation, centre). +fn true_camera(f: u32) -> (M3, V3) { + let t = f as f64 / 24.0; + let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t]; + let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c)); + let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd)); + ([right, linalg::cross(fwd, right), fwd], c) +} + +/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite). +const PLANES: [(V3, V3, V3, V3, f64); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0), + ([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1), +]; + +fn scene_frame(f: u32) -> Image { + let (r, c) = true_camera(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]); + let mut best = (f64::INFINITY, 0.2f32); + for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() { + let den = linalg::dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den; + if tt <= 0.1 || tt >= best.0 { + continue; + } + let hit = linalg::add(c, linalg::scale(d, tt)); + let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va)); + if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) { + continue; + } + best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32)); + } + img.set(x, y, [best.1, best.1, best.1, 1.0]); + } + } + img +} + +/// The comp with the clip tracked (blocking) with `params`. +fn tracked(params: Value) -> (Session, LayerId) { + let (mut s, clip, _) = setup(scene_frame); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + assert!(s.is_enabled("track.camera")); + let mut p = params; + p["wait"] = json!(true); + let r = s.execute_checked("track.camera", p).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert!(s.camera_job.is_none()); + (s, clip) +} + +fn status(s: &mut Session) -> Value { + s.execute("camera.solveStatus", json!({})).unwrap() +} + +#[test] +fn track_camera_solves_and_creates_layers_on_the_plane() { + let (mut s, clip) = tracked(json!({})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + let err = st["averageError"].as_f64().unwrap(); + assert!(err < 0.5, "average error {err}"); + let f = st["focalLength"].as_f64().unwrap(); + assert!((f - F).abs() / F < 0.05, "focal {f}"); + assert_ne!(st["methodUsed"], "Tripod Pan"); + // Method Used / Average Error are reported in the effect. + let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty())); + + // Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the + // ones seen below the horizon at the bottom of the frame). + s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap(); + let pts = s.execute("camera.points", json!({})).unwrap(); + let list = pts["points"].as_array().unwrap(); + assert!(list.len() > 30, "{}", list.len()); + let floor: Vec = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect(); + assert!(floor.len() >= 3, "{floor:?}"); + s.execute("camera.selectPoints", json!({"points": floor})).unwrap(); + assert_eq!(s.state.camera_points.len(), floor.len()); + + // Create Solid and Camera: one undo step, camera + solid. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let cam = LayerId(r["camera"].as_u64().unwrap()); + let solid = LayerId(r["layers"][0].as_u64().unwrap()); + let comp = s.active_comp().unwrap().clone(); + assert!(comp.layer(cam).unwrap().is_camera()); + assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera"); + assert_eq!(comp.layers[0].id, cam); + let sl = comp.layer(solid).unwrap(); + assert!(sl.switches.three_d); + assert!(sl.name.starts_with("Track Solid")); + // The camera is keyed on every frame. + assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + + // Every solved point projects through the created camera exactly where the solve puts it, + // and onto its tracked position (the footage fills the comp, so layer pixels are comp + // pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers: + // features on occlusion edges), 0.5 px on average. + let cid = s.active_comp_id().unwrap(); + let uid = g.uid; + let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap(); + let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap(); + let mut errs = vec![]; + for k in [0u32, 6, 12, 18, 24] { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k)); + let camst = active_camera(&ctx); + for sp in pl.solve.visible(k as usize) { + let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue }; + let w = pl.to_world(sp.pos); + let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap(); + let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap(); + assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve"); + errs.push((q.x - obs[0]).hypot(q.y - obs[1])); + } + } + assert!(errs.len() > 100, "{}", errs.len()); + errs.sort_by(f64::total_cmp); + let p95 = errs[errs.len() * 95 / 100]; + let mean = errs.iter().sum::() / errs.len() as f64; + assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px"); + + // The solid lies on the selected points' plane. + let tr = sl.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + let ori = tr.get("orientation").unwrap().value.as_vec3(); + let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]); + let size = r["target"]["size"].as_f64().unwrap(); + for id in &s.state.camera_points { + let w = pl.to_world(pl.solve.point(*id).unwrap().pos); + let d = linalg::dot(linalg::sub(w, pos), normal).abs(); + assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane"); + } + // The solid faces the camera. + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12)); + let eye = active_camera(&ctx).eye; + assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0); + + // Undo removes both layers; redo brings them back. + let before = comp.layers.len(); + assert!(s.undo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before - 2); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before); + + // A second create reuses the camera: Create Multiple Nulls and Camera. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap(); + assert_eq!(r["camera"].as_u64().unwrap(), cam.0); + assert_eq!(r["layers"].as_array().unwrap().len(), floor.len()); + // Text and Shadow Catcher (+ light). + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap(); + assert_eq!(r["layers"].as_array().unwrap().len(), 1); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect(); + let comp = s.active_comp().unwrap(); + assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher")); + assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2); + assert!(comp.layer(ids[1]).unwrap().is_light()); + assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err()); + + // Set Ground Plane and Origin: the target becomes the origin; a null created there sits at + // (0, 0, 0) lying on the X-Z plane (its front facing up, −y). + s.execute_checked("camera.setGroundPlane", json!({})).unwrap(); + assert_eq!(status(&mut s)["groundPlane"], true); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap(); + let null = LayerId(r["layers"][0].as_u64().unwrap()); + let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(pos) < 1e-6, "{pos:?}"); + let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}"); + + // An explicit target (comp world) places the layer there, its front facing the normal. + let r = s + .execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}})) + .unwrap(); + let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone(); + let tr = l.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}"); + let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}"); + + // Serde keeps the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap(); + let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap(); + assert!(sv.ground.is_some()); + assert_eq!(sv.points.len(), pl.solve.points.len()); +} + +#[test] +fn invalidation_resolve_cancel_and_undo() { + let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["shotType"], "Specify Angle of View"); + assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F); + let cid = s.active_comp_id().unwrap(); + let uid = st["effect"].as_u64().unwrap(); + // Undo removes the analysis (one step); redo restores it. + assert!(s.undo()); + assert_eq!(status(&mut s)["analyzed"], false); + assert!(s.redo()); + assert_eq!(status(&mut s)["solved"], true); + + // Changing the Solve Method keeps the tracks and only re-solves. + s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], true); + assert_eq!(st["solved"], false); + assert_eq!(st["pending"], true); + s.execute_checked("camera.analyze", json!({"wait": true})).unwrap(); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["methodUsed"], "Typical"); + + // Deleting points re-solves without them. + let pts = s.execute("camera.points", json!({})).unwrap(); + let del: Vec = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect(); + s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap(); + s.poll_camera(false); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert!(st["deleted"].as_u64().unwrap() >= 3); + let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap(); + assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64)))); + + // Trimming the layer changes its frames: tracks and solve are cleared and queued. + s.edit("trim", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2); + Ok(()) + }) + .unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], false); + assert_eq!(st["pending"], true); + + // A background analysis can be cancelled: nothing is written. + s.execute_checked("camera.analyze", json!({})).unwrap(); + assert!(s.is_camera_analyzing() || s.camera_job.is_some()); + let banner = status(&mut s)["banner"].as_str().map(str::to_string); + if s.is_camera_analyzing() { + assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)")); + } + s.execute("camera.cancel", json!({})).ok(); + let t0 = std::time::Instant::now(); + while s.camera_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_camera(false); + } + // (When the job finished before the cancel landed it wrote a valid analysis.) + let st = status(&mut s); + assert_eq!(st["running"], false); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err()); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err()); +} + +#[test] +fn effect_parameters_match_after_effects() { + let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap(); + assert_eq!(spec.name, "3D Camera Tracker"); + assert_eq!(spec.category, "Perspective"); + let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect(); + for n in [ + "Shot Type", + "Horizontal Angle of View", + "Show Track Points", + "Render Track Points", + "Track Point Size", + "Target Size", + "Solve Method", + "Detailed Analysis", + "Auto-delete Points Across Time", + "Hide Warning Banner", + ] { + assert!(names.contains(&n), "{n}"); + } + // The Animation menu entry is a real command now. + let c = crate::find_command("track.camera").unwrap(); + assert_eq!(c.label, "Track Camera"); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858ca..bde98add 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -331,7 +331,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "file.createProxy"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index 9bd46e26..4800e06e 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 84e8a5ad..72cb2f17 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -173,6 +173,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> } let mut cur = cost(cams, pts, obs, o); let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; let mut lambda = 1e-4; let mut iters = 0; if l.n == 0 && o.fix_points { @@ -231,7 +234,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> for (ia, ja) in &jc { let wv = [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; - match ps.w.iter_mut().find(|e| e.0 == *ia) { + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), } @@ -294,31 +300,67 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> // ---- Try steps with increasing damping. let mut accepted = false; for _try in 0..12 { + tries += 1; let mut s = base_s.clone(); let mut rhs = base_bc.clone(); for i in 0..n { let d = s[i * n + i]; s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); } - // Schur complement over the points. - let mut vinv: Vec> = vec![None; npts]; + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; if !o.fix_points { - for (j, ps) in systems.iter().enumerate() { - let mut v = ps.v; - let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; - for k in 0..3 { - v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; - } - let Some(vi) = inv3(&v) else { continue }; - vinv[j] = Some(vi); - let vb = mv(&vi, ps.bp); - let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); - for (a, (ia, wa)) in ps.w.iter().enumerate() { - rhs[*ia] -= dot(*wa, vb); - for (ib, wb) in &ps.w { - s[ia * n + ib] -= dot(wv[a], *wb); + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } } } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); } } let mut dc = rhs; @@ -363,9 +405,18 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> let gain = cur - c2; cur = c2; lambda = (lambda / 3.0).max(1e-9); - accepted = gain > 1e-10 * cur.max(1e-12); + accepted = gain > 1e-6 * cur.max(1e-12); if !accepted { // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } return BaReport { initial_cost: initial, cost: cur, iterations: iters }; } break; @@ -376,6 +427,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> break; } } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } BaReport { initial_cost: initial, cost: cur, iterations: iters } } diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 311ec711..30b019e5 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -89,7 +89,7 @@ impl TrackAnalyzer { let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); !occ.get(cy * gw + cx).copied().unwrap_or(true) }; - let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; for p in good_features(&pyr, &opts, Some(&free)) { let l = pyr.to_layer(p); let id = self.tracks.len() as u32; diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs index a7067f26..b719cc8e 100644 --- a/crates/track/src/camtrack/linalg.rs +++ b/crates/track/src/camtrack/linalg.rs @@ -298,6 +298,23 @@ pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { (vals, vecs) } +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + /// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by /// Cholesky factorisation. Returns false when `A` is not positive definite. pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { @@ -313,16 +330,9 @@ pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { a[j * n + j] = d; let (head, tail) = a.split_at_mut((j + 1) * n); let rowj = &head[j * n..j * n + j]; - // Rows below j are independent. - use rayon::prelude::*; - let work = |row: &mut [f64]| { - let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); - row[j] = s / d; - }; - if n - j > 192 { - tail.par_chunks_mut(n).for_each(work); - } else { - tail.chunks_mut(n).for_each(work); + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; } } // Forward: L y = b. diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 056d7ad0..76c0637f 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -13,7 +13,10 @@ //! the other keyframes are added incrementally (**resection** by robust pose refinement, //! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse //! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points -//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. //! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it //! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal //! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment @@ -22,7 +25,8 @@ //! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame //! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle //! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the -//! tripod solution when it explains the tracks as well as the general one. +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). //! //! The camera model has square pixels, the principal point at the layer centre and no lens //! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index da3a9290..85edd2bd 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -37,8 +37,9 @@ struct Data { tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, /// Per frame: (track index, position). by_frame: Vec>, - /// Tracking noise scale (pixels). - noise: f64, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, } impl Data { @@ -58,23 +59,31 @@ impl Data { } tracks.push((tr.id, tr.start as usize, pts)); } - Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } } fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { let (_, s, p) = &self.tracks[ti as usize]; k.checked_sub(*s).and_then(|i| p.get(i).copied()) } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } fn inlier_thr(&self) -> f64 { - 4.0 * self.noise + 4.0 * self.noise() } fn huber(&self) -> f64 { - 1.0 * self.noise + self.noise() } } /// Parallax-based keyframes: a new keyframe when the median feature motion since the last -/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared -/// features drop below 24. The first and last frames with features are keyframes. +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. fn keyframes(d: &Data) -> Vec { let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; let mut kf = vec![first]; @@ -97,7 +106,7 @@ fn keyframes(d: &Data) -> Vec { disp.sort_by(f64::total_cmp); let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); let ratio = disp.len() as f64 / base.len().max(1) as f64; - if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { last = j; kf.push(j); } @@ -405,7 +414,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); - let thr = 2.0 * d.noise / f; + let thr = 2.0 * d.noise() / f; let mut cands: Vec = vec![]; if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); @@ -415,7 +424,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } if use_h { let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); - if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { let h = fit.h.0; // Normalised: K⁻¹ H K with K = diag(f, f, 1). let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; @@ -651,6 +660,12 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; let mut results: Vec<(Attempt, f64)> = vec![]; if try_general { let f = match fixed_f { @@ -666,7 +681,20 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo results.push((a, f)); } } - if try_tripod && !cancelled() { + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { let f = match fixed_f { Some(f) => f, None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), @@ -696,7 +724,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if debug() { eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); } - if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } } (Some(g), None) => g, (None, Some(t)) => t, @@ -717,15 +745,44 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (None, true) => FocalMode::PerCamera(F_PRIOR), (None, false) => FocalMode::Shared, }; + lap("tripod"); // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. let kf_frames: Vec = r.cams.keys().copied().collect(); r.adjust(&d, &kf_frames, &cfg, focal, 40); r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } r.triangulate(&d, kind, 1.0, 1.0); r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); if cancelled() { return err("cancelled"); } + lap("keyframe adjustment"); // Every other frame: resect from the points (nearest registered camera first). let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); @@ -746,6 +803,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if cancelled() { return err("cancelled"); } + lap("resection"); // Points for the tracks that only lived between keyframes, then the final adjustment over // every frame (when that stays affordable). r.triangulate(&d, kind, 1.0, 1.0); @@ -786,6 +844,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } (m, _) => m, }; + lap("final adjustment"); Ok(finish(&d, tracks, s, r, method, kind)) } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 258161f9..4b5a8346 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -358,3 +358,81 @@ fn perf_long_clip() { eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); assert!(s.average_error < 0.5); } + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c1948..d581ef35 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -727,6 +727,11 @@ impl EffectcraftApp { self.session.poll_warp(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs new file mode 100644 index 00000000..41fb7204 --- /dev/null +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -0,0 +1,438 @@ +//! 3D Camera Tracker in the Composition viewer and Effect Controls: +//! +//! - while the effect is selected, the solved 3D track points are drawn as coloured targets +//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source); +//! - hovering between three points shows the target plane they span (a disc in perspective); +//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a +//! triangle to select its three points; the selection's target is drawn; +//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … +//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection; +//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner; +//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average +//! Error. +//! +//! Everything is dispatched to the `camera.*` commands. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid}; +use effectcraft_engine::render::EvalCtx; +use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The tracker whose points the viewer shows. +struct Active { + layer: LayerId, + uid: Uid, + solve: Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f..11640d88 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -545,6 +545,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +921,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f737502..ead1fe99 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,6 +1,7 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b2..364e114e 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb8..c9dd70db 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1144,6 +1144,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 00000000..45374fba --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/docs/agents.md b/docs/agents.md index 510bc356..bda72461 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -146,6 +146,34 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb..f9f0586f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,6 +176,22 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index d259fec2..25e5a080 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **257 of 298** After Effects 2026 effects by name (86%); ≈ 70% allowing for simplified implementations | +| **Effects** | **258 of 298** After Effects 2026 effects by name (87%); ≈ 70% allowing for simplified implementations | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,7 +73,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, +What is left is concentrated in large systems: Roto Brush and Refine Edge, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and @@ -93,13 +93,13 @@ wasm threads. | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 85% | 1.5 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no text on 3D bevels; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | +| 3D | 62% | 18.5 | Advanced 3D (models, PBR, image-based light), multi-view layouts (the 3D Camera Tracker landed in M12.6) | | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, rolling-shutter repair, lens distortion in the camera solve (point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing @@ -107,7 +107,7 @@ wasm threads. Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, -Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX +Keylight (our Screen Key stands in) and Color Profile Converter. Boris FX Mocha and Cineware are third-party and not counted. ## Highest-value gaps, in order From 5df98d0982ac963b508631cc0b6a1f56c00a4403 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:41:28 -0400 Subject: [PATCH 6/6] =?UTF-8?q?M0.11:=20parity=20audit=20at=20fa26ad9=20(?= =?UTF-8?q?=E2=89=8893%=20weighted;=20298/298=20effects;=20=E2=89=8811?= =?UTF-8?q?=E2=80=9314=20h=20wall=20clock=20left)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 13 ++++++------- docs/parity.md | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 2bf7d8ba..724c9d70 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,13 +25,12 @@ once. ## How far from full parity -Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): +Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026): -- **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 298 of 298 effects, 16 of them on the GPU. -- **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in - parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** - by the audit's conservative per-feature figures. -- Hardest remaining pieces: Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7, the 3D Camera Tracker in M12.6). +- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of + professional daily-use features, 65% of the long tail); all 298 After Effects effects. +- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude + Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features. +- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API. Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/docs/parity.md b/docs/parity.md index 0f3103ac..32cb65b7 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -1,9 +1,36 @@ # Parity with After Effects How close EffectCraft is to After Effects 2026, feature by feature, and how much work is left. -Audited against the code at commit `5273f1b` (2 October 2026). Every feature in our catalog was -graded from the code itself: a disabled menu entry, or a setting that is stored but never -rendered, does not count as done. + +## Current status (audit at commit `fa26ad9`, 2 October 2026, late) + +| Measure | Value | +|---|---| +| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 93%** | +| Unweighted | ≈ 91% | +| P0 / P1 / P2 | ≈ 96% / 90% / 65% | +| Features done / partial / missing | 41 / 50 / 1 of 92 (missing: a plug-in API) | +| **Effects** | **298 of 298** After Effects 2026 effects exist (some still simplified) | +| Disabled menu entries left | 26 commands (31 menu items) | +| Remaining work | ≈ 50–55 agent-hours at the pace measured so far (≈ 150 by the original conservative audit scale) | +| **Wall-clock estimate** | **≈ 11–14 hours** with five agents in parallel; ≈ 6–8 hours for 100% of P0 + P1 | + +By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, +Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, +Tracking 88%, Effects 85%, Web 65%. + +What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking +renders); stroke taper/wave and multi-segment dashes; variable mask feather points; real +`sampleImage` and `footage()` in expressions; camera iris/bokeh and focus-link commands; GPU +rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; +approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur +and blend modes; a few viewer and colour-management menu items; OpenType features and variable +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; proxies; ScriptUI; +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and +the "better than After Effects" items (a plug-in API, branching history, GPU particles). + +The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for +history. ## Summary